From d208a87ae8f9646ebbde19ae3e21748df786b65b Mon Sep 17 00:00:00 2001 From: David Date: Mon, 9 Nov 2026 00:04:59 +0300 Subject: [PATCH 1/5] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BD=D0=B5=D1=81?= =?UTF-8?q?=D1=82=D0=B8=20=D1=80=D0=B0=D0=B1=D0=BE=D1=87=D0=B8=D0=B9=20?= =?UTF-8?q?=D1=81=D1=82=D0=B5=D0=BD=D0=B4=20lab3=20=D0=B2=20=D1=80=D0=B5?= =?UTF-8?q?=D0=BF=D0=BE=D0=B7=D0=B8=D1=82=D0=BE=D1=80=D0=B8=D0=B9=20lab4.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit API, Postgres, Redis и HAProxy остаются базой для вложений: сессии и история уже общие для нескольких реплик. --- .env.example | 30 ++- Dockerfile | 29 +++ app/__init__.py | 1 + app/cache.py | 151 ++++++++++++++ app/config.py | 42 ++++ app/cursor_codec.py | 112 ++++++++++ app/db.py | 87 ++++++++ app/docs.py | 52 +++++ app/errors.py | 49 +++++ app/main.py | 146 +++++++++++++ app/middleware.py | 48 +++++ app/migrate.py | 29 +++ app/passwords.py | 23 +++ app/rediscli.py | 40 ++++ app/routes.py | 284 ++++++++++++++++++++++++++ app/sessions.py | 56 +++++ app/store.py | 361 +++++++++++++++++++++++++++++++++ app/tools/__init__.py | 1 + app/tools/explain.py | 73 +++++++ app/validation.py | 102 ++++++++++ compose.yaml | 131 ++++++++++++ infra/haproxy.cfg | 35 ++++ migrations/001_init.sql | 49 +++++ migrations/002_idempotency.sql | 14 ++ requirements.txt | 5 + scripts/restart.sh | 10 + scripts/rolling-update.sh | 55 +++++ scripts/stop-one.sh | 32 +++ 28 files changed, 2040 insertions(+), 7 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/cache.py create mode 100644 app/config.py create mode 100644 app/cursor_codec.py create mode 100644 app/db.py create mode 100644 app/docs.py create mode 100644 app/errors.py create mode 100644 app/main.py create mode 100644 app/middleware.py create mode 100644 app/migrate.py create mode 100644 app/passwords.py create mode 100644 app/rediscli.py create mode 100644 app/routes.py create mode 100644 app/sessions.py create mode 100644 app/store.py create mode 100644 app/tools/__init__.py create mode 100644 app/tools/explain.py create mode 100644 app/validation.py create mode 100644 compose.yaml create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 requirements.txt create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh diff --git a/.env.example b/.env.example index d99d26a2..9b4f9ff3 100644 --- a/.env.example +++ b/.env.example @@ -3,18 +3,34 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +LOG_LEVEL=info +SHUTDOWN_GRACE_SECONDS=10 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +CACHE_TTL_SECONDS=60 +# Sessions live in Redis (lab2 grade 4 path carried from lab3). Not Postgres. +SESSION_STORE=redis API_REPLICAS=2 +API_CPUS=0.50 +API_MEM=256m +POSTGRES_CPUS=1.0 +POSTGRES_MEM=512m +DB_POOL_MAX=4 +LB_DRAIN_SECONDS=2 IDEMPOTENCY_TTL_SECONDS=86400 + +# Compose interpolates these. Synthetic local values only; replace after copy. +POSTGRES_PASSWORD=lab4-dev-only-change-me +DATABASE_URL=postgresql://messenger:lab4-dev-only-change-me@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 + S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Root is used only by bucket bootstrap. The API uses the app key below. +MINIO_ROOT_USER=lab4-minio-root +MINIO_ROOT_PASSWORD=lab4-dev-only-change-me +S3_ACCESS_KEY_ID=lab4appkey00000001 +S3_SECRET_ACCESS_KEY=lab4-dev-only-change-me MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +ORPHAN_GRACE_SECONDS=120 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..65e21fb0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12.8-slim-bookworm AS runtime + +WORKDIR /app + +RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app \ + && apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations +COPY contracts/openapi.json ./contracts/openapi.json + +USER app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + PYTHONUNBUFFERED=1 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + +CMD ["python", "-m", "app.main"] diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..eeb0eb82 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger API (lab 2): PostgreSQL source of truth, Redis sessions/cache.""" diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..ed9261df --- /dev/null +++ b/app/cache.py @@ -0,0 +1,151 @@ +import asyncio +import json +from collections.abc import Awaitable, Callable +from typing import TypeVar + +from app import config, rediscli +from app.errors import AppError + +T = TypeVar("T") +CFG_SINGLEFLIGHT = "cfg:singleflight" +CFG_INVALIDATE = "cfg:invalidate" +CFG_DELAY_MS = "cfg:delay_ms" +SQL_HISTORY = "metrics:sql:messages_select" + + +class SingleFlight: + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._inflight: dict[str, asyncio.Task] = {} + + async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T: + async with self._lock: + task = self._inflight.get(key) + if task is None: + task = asyncio.create_task(factory()) + self._inflight[key] = task + owner = True + else: + owner = False + try: + return await asyncio.shield(task) + finally: + if owner: + async with self._lock: + self._inflight.pop(key, None) + + +_sf = SingleFlight() + + +async def _flag(name: str, default: bool = True) -> bool: + if not config.lab_mode(): + return default + try: + raw = await rediscli.require().get(name) + except Exception: + return default + if raw is None: + return default + return raw not in {"0", "false", "off"} + + +async def singleflight_enabled() -> bool: + return await _flag(CFG_SINGLEFLIGHT, True) + + +async def invalidate_enabled() -> bool: + return await _flag(CFG_INVALIDATE, True) + + +async def incr_history_sql() -> None: + try: + await rediscli.require().incr(SQL_HISTORY) + except Exception: + pass + + +async def history_sql_count() -> int: + try: + return int(await rediscli.require().get(SQL_HISTORY) or 0) + except Exception: + return 0 + + +async def reset_history_sql_count() -> None: + try: + await rediscli.require().set(SQL_HISTORY, 0) + except Exception: + pass + + +async def load_delay_seconds() -> float: + if not config.lab_mode(): + return 0.0 + try: + raw = await rediscli.require().get(CFG_DELAY_MS) + except Exception: + return 0.0 + try: + return max(0.0, float(raw or 0) / 1000.0) + except (TypeError, ValueError): + return 0.0 + + +async def set_flags( + *, + singleflight: bool | None = None, + invalidate: bool | None = None, + delay_ms: int | None = None, +) -> dict: + r = rediscli.require() + if singleflight is not None: + await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0") + if invalidate is not None: + await r.set(CFG_INVALIDATE, "1" if invalidate else "0") + if delay_ms is not None: + await r.set(CFG_DELAY_MS, str(int(delay_ms))) + return { + "singleflight": await singleflight_enabled(), + "invalidate": await invalidate_enabled(), + "history_sql": await history_sql_count(), + "delay_ms": int((await load_delay_seconds()) * 1000), + } + + +async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict: + r = rediscli.require() + try: + version = await r.get("histver:" + chat_id) or "0" + key = f"hist:{chat_id}:{version}:{cursor}:{limit}" + cached = await r.get(key) + if cached: + return json.loads(cached) + + async def load_and_store() -> dict: + delay = await load_delay_seconds() + if delay: + await asyncio.sleep(delay) + page = await loader() + try: + await r.set(key, json.dumps(page, ensure_ascii=False), ex=config.CACHE_TTL_SECONDS) + except Exception: + pass + return page + + if await singleflight_enabled(): + return await _sf.do(key, load_and_store) + return await load_and_store() + except AppError: + raise + except Exception: + return await loader() + + +async def invalidate_history(chat_id: str) -> None: + if not await invalidate_enabled(): + return + try: + await rediscli.require().incr("histver:" + chat_id) + except Exception: + pass diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..b6d1f3b1 --- /dev/null +++ b/app/config.py @@ -0,0 +1,42 @@ +import os +import socket +from pathlib import Path + + +def _env(name: str, default: str = "") -> str: + value = os.environ.get(name) + return default if value is None or value == "" else value + + +def _file_or_env(name: str, default: str = "") -> str: + path = os.environ.get(name + "_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return _env(name, default) + + +HOST = _env("APP_HOST", "0.0.0.0") +PORT = int(_env("APP_PORT", "8080")) +ENV = _env("APP_ENV", "lab") +LOG_LEVEL = _env("LOG_LEVEL", "info").lower() +SHUTDOWN_GRACE_SECONDS = float(_env("SHUTDOWN_GRACE_SECONDS", "10")) +SESSION_TTL_SECONDS = int(_env("SESSION_TTL_SECONDS", "3600")) +CACHE_TTL_SECONDS = int(_env("CACHE_TTL_SECONDS", "60")) +DATABASE_URL = _file_or_env("DATABASE_URL") +REDIS_URL = _file_or_env("REDIS_URL", "redis://redis:6379/0") +SESSION_STORE = _env("SESSION_STORE", "redis") +DB_POOL_MIN = int(_env("DB_POOL_MIN", "1")) +DB_POOL_MAX = int(_env("DB_POOL_MAX", "10")) +DB_COMMAND_TIMEOUT = float(_env("DB_COMMAND_TIMEOUT", "5")) +DB_CONNECT_TIMEOUT = float(_env("DB_CONNECT_TIMEOUT", "5")) +REDIS_SOCKET_TIMEOUT = float(_env("REDIS_SOCKET_TIMEOUT", "2")) +IDEMPOTENCY_TTL_SECONDS = int(_env("IDEMPOTENCY_TTL_SECONDS", "86400")) +MIGRATIONS_ON_START = _env("MIGRATIONS_ON_START", "1") == "1" +LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2")) +# Stable for the life of this process. Compose leaves it unset so each +# container uses its own hostname; stop-one.sh maps that value to one replica. +INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname()) + + +def lab_mode() -> bool: + return ENV == "lab" diff --git a/app/cursor_codec.py b/app/cursor_codec.py new file mode 100644 index 00000000..40e6ce20 --- /dev/null +++ b/app/cursor_codec.py @@ -0,0 +1,112 @@ +import base64 + +import json + +from datetime import datetime + +from typing import Any + +from uuid import UUID + + + + + +def encode_cursor(chat_id: str, created_at: str, entity_id: str) -> str: + + payload = json.dumps({"c": chat_id, "t": created_at, "i": entity_id}, separators=(",", ":")) + + return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=") + + + + + +def _validate_cursor_timestamp(value: str) -> None: + + if not value.endswith("Z"): + + raise ValueError("invalid cursor timestamp") + + try: + + datetime.fromisoformat(value[:-1] + "+00:00") + + except ValueError as exc: + + raise ValueError("invalid cursor timestamp") from exc + + + + + +def _require_str_field(data: dict, key: str) -> str: + + if key not in data: + + raise ValueError("invalid cursor") + + value = data[key] + + if not isinstance(value, str): + + raise ValueError("invalid cursor") + + return value + + + + + +def decode_cursor(cursor: str, expected_chat_id: str) -> tuple[str, str]: + + pad = "=" * (-len(cursor) % 4) + + try: + + raw = base64.urlsafe_b64decode((cursor + pad).encode("ascii")) + + data: Any = json.loads(raw.decode("utf-8")) + + except (ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc: + + raise ValueError("invalid cursor") from exc + + if not isinstance(data, dict): + + raise ValueError("invalid cursor") + + try: + + chat_id = _require_str_field(data, "c") + + created_at = _require_str_field(data, "t") + + entity_id = _require_str_field(data, "i") + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + if chat_id != expected_chat_id: + + raise ValueError("cursor chat mismatch") + + try: + + UUID(entity_id) + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + try: + + _validate_cursor_timestamp(created_at) + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + return created_at, entity_id + diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..f98fd341 --- /dev/null +++ b/app/db.py @@ -0,0 +1,87 @@ +from pathlib import Path + +import asyncpg + +from app import config +from app.errors import AppError + +pool: asyncpg.Pool | None = None +INTEGRITY = ( + asyncpg.UniqueViolationError, + asyncpg.ForeignKeyViolationError, + asyncpg.CheckViolationError, +) + + +def _migrations_dir() -> Path: + here = Path(__file__).resolve().parent + for candidate in (here.parent / "migrations", Path("/app/migrations")): + if candidate.is_dir(): + return candidate + raise RuntimeError("migrations directory not found") + + +async def connect() -> asyncpg.Pool: + global pool + if not config.DATABASE_URL: + raise RuntimeError("DATABASE_URL is not set") + pool = await asyncpg.create_pool( + dsn=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + command_timeout=config.DB_COMMAND_TIMEOUT, + timeout=config.DB_CONNECT_TIMEOUT, + ) + return pool + + +async def close() -> None: + global pool + if pool is not None: + await pool.close() + pool = None + + +def require() -> asyncpg.Pool: + if pool is None: + raise AppError(503, "service unavailable") + return pool + + +async def ping() -> None: + try: + async with require().acquire() as conn: + await conn.execute("SELECT 1") + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + + +_MIGRATION_LOCK = 87133103 + + +async def apply_migrations() -> None: + """Apply pending SQL files once. A session advisory lock stops replica races.""" + directory = _migrations_dir() + async with require().acquire() as conn: + await conn.execute("SELECT pg_advisory_lock($1)", _MIGRATION_LOCK) + try: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")} + for path in sorted(p for p in directory.glob("*.sql") if p.is_file()): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute("INSERT INTO schema_migrations(filename) VALUES($1)", path.name) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", _MIGRATION_LOCK) diff --git a/app/docs.py b/app/docs.py new file mode 100644 index 00000000..7f68e1d9 --- /dev/null +++ b/app/docs.py @@ -0,0 +1,52 @@ +import json +from functools import lru_cache +from pathlib import Path + +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse, Response + +_SPEC_CANDIDATES = ( + Path(__file__).resolve().parents[1] / "contracts" / "openapi.json", + Path("/app/contracts/openapi.json"), +) + +_SWAGGER_HTML = """ + + + + MTUSI Messenger — Swagger (lab2) + + + +
+ + + + +""" + + +@lru_cache(maxsize=1) +def _load_spec() -> dict: + for path in _SPEC_CANDIDATES: + if path.is_file(): + return json.loads(path.read_text(encoding="utf-8")) + return {"openapi": "3.1.0", "info": {"title": "MTUSI Messenger", "version": "1.2.0"}, "paths": {}} + + +async def openapi_json(_: Request) -> Response: + return JSONResponse(_load_spec()) + + +async def swagger_ui(_: Request) -> Response: + return HTMLResponse(_SWAGGER_HTML) diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..105e8ca3 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,49 @@ +from typing import Dict, Optional + +from starlette.responses import JSONResponse + + +ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", +} + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class AppError(Exception): + def __init__(self, status: int, message: str, www_authenticate: str | None = None) -> None: + super().__init__(message) + self.status = status + self.message = message + self.www_authenticate = www_authenticate + + +def error_response( + status: int, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, +) -> JSONResponse: + code = ERROR_CODES.get(status, "internal_error") + headers = {"X-Request-Id": request_id, "Content-Type": "application/json"} + if extra_headers: + headers.update(extra_headers) + return JSONResponse( + status_code=status, + headers=headers, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + ) + + +def from_app_error(exc: AppError, request_id: str) -> JSONResponse: + extra = {"WWW-Authenticate": exc.www_authenticate} if exc.www_authenticate else None + return error_response(exc.status, exc.message, request_id, extra) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..c59ca752 --- /dev/null +++ b/app/main.py @@ -0,0 +1,146 @@ +import asyncio +import logging +import signal +import sys +import uuid +from pathlib import Path + +import uvicorn +from starlette.applications import Starlette +from starlette.exceptions import HTTPException +from starlette.requests import Request +from starlette.responses import Response +from starlette.routing import Route + +from app import config, db, rediscli +from app.docs import openapi_json, swagger_ui +from app.errors import AppError, error_response, from_app_error +from app.middleware import RequestContextMiddleware +from app.routes import ( + create_chat, + create_message, + create_session, + current_user, + get_chat, + health_live, + health_ready, + lab_flags, + lab_metrics, + list_chats, + list_messages, + register_user, + revoke_session, +) + + +def _request_id(request: Request) -> str: + existing = getattr(request.state, "request_id", None) + if existing: + return existing + return request.headers.get("X-Request-Id") or str(uuid.uuid4()) + + +async def not_found_handler(request: Request, exc: HTTPException) -> Response: + return error_response(404, "not found", _request_id(request)) + + +async def app_error_handler(request: Request, exc: AppError) -> Response: + return from_app_error(exc, _request_id(request)) + + +def publish_instance_id() -> None: + path = Path(config._env("INSTANCE_ID_FILE", "/tmp/instance-id")) + try: + path.write_text(config.INSTANCE_ID, encoding="utf-8") + except OSError: + logging.getLogger("messenger").warning("instance id file was not written") + + +async def startup() -> None: + publish_instance_id() + logging.getLogger("messenger").info("instance_id=%s", config.INSTANCE_ID) + await db.connect() + if config.MIGRATIONS_ON_START: + await db.apply_migrations() + await rediscli.connect() + + +async def shutdown() -> None: + await rediscli.close() + await db.close() + + +def build_app() -> Starlette: + app = Starlette( + routes=[ + Route("/docs", swagger_ui, methods=["GET"]), + Route("/openapi.json", openapi_json, methods=["GET"]), + Route("/health/live", health_live, methods=["GET"]), + Route("/health/ready", health_ready, methods=["GET"]), + Route("/api/v1/users", register_user, methods=["POST"]), + Route("/api/v1/users/me", current_user, methods=["GET"]), + Route("/api/v1/auth/sessions", create_session, methods=["POST"]), + Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]), + Route("/api/v1/chats", create_chat, methods=["POST"]), + Route("/api/v1/chats", list_chats, methods=["GET"]), + Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), + Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]), + Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]), + ], + exception_handlers={404: not_found_handler, AppError: app_error_handler}, + on_startup=[startup], + on_shutdown=[shutdown], + ) + app.state.shutting_down = False + app.add_middleware(RequestContextMiddleware) + return app + + +def configure_logging() -> None: + logging.basicConfig( + level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + ) + + +def main() -> None: + configure_logging() + app = build_app() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + server = uvicorn.Server( + uvicorn.Config( + app, + host=config.HOST, + port=config.PORT, + log_level=config.LOG_LEVEL, + access_log=False, + timeout_graceful_shutdown=int(config.SHUTDOWN_GRACE_SECONDS), + ) + ) + + def _begin_shutdown() -> None: + if app.state.shutting_down: + return + logging.getLogger("messenger").info( + "shutdown signal received; readiness fails now, exit in %.1fs", + config.LB_DRAIN_SECONDS, + ) + app.state.shutting_down = True + loop.call_later(config.LB_DRAIN_SECONDS, lambda: setattr(server, "should_exit", True)) + + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, _begin_shutdown) + except NotImplementedError: + signal.signal(sig, lambda *_: _begin_shutdown()) + + loop.run_until_complete(server.serve()) + + +if __name__ == "__main__": + main() diff --git a/app/middleware.py b/app/middleware.py new file mode 100644 index 00000000..4b05cca3 --- /dev/null +++ b/app/middleware.py @@ -0,0 +1,48 @@ +import logging +import time +import uuid +from typing import Callable + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from app.config import INSTANCE_ID +from app.errors import AppError, error_response, from_app_error + +logger = logging.getLogger("messenger.access") + + +def _stamp(response: Response, request_id: str) -> Response: + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +class RequestContextMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next: Callable) -> Response: + request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = request_id + + if request.app.state.shutting_down and request.url.path not in ("/health/live", "/health/ready"): + return _stamp(error_response(503, "service is shutting down", request_id), request_id) + + started = time.perf_counter() + try: + response = await call_next(request) + except AppError as exc: + return _stamp(from_app_error(exc, request_id), request_id) + except Exception: + logger.exception("request_id=%s unhandled error path=%s", request_id, request.url.path) + return _stamp(error_response(500, "internal error", request_id), request_id) + elapsed_ms = (time.perf_counter() - started) * 1000 + _stamp(response, request_id) + logger.info( + "request_id=%s method=%s path=%s status=%s duration_ms=%.2f", + request_id, + request.method, + request.url.path, + response.status_code, + elapsed_ms, + ) + return response diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..03bc8db8 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,29 @@ +"""One-shot schema apply. API replicas do not run this on startup.""" +import asyncio +import logging +import sys + +from app import config, db + + +async def _main() -> None: + if not config.DATABASE_URL: + raise SystemExit("DATABASE_URL is not set") + await db.connect() + try: + await db.apply_migrations() + finally: + await db.close() + + +def main() -> None: + logging.basicConfig( + level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + ) + asyncio.run(_main()) + + +if __name__ == "__main__": + main() diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..8a640cd0 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,23 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + +hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1) +_dummy = None + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def dummy_hash() -> str: + global _dummy + if _dummy is None: + _dummy = hash_password("timing-dummy-password") + return _dummy + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError, TypeError, ValueError): + return False diff --git a/app/rediscli.py b/app/rediscli.py new file mode 100644 index 00000000..de173117 --- /dev/null +++ b/app/rediscli.py @@ -0,0 +1,40 @@ +import redis.asyncio as redis + +from app import config +from app.errors import AppError + +client: redis.Redis | None = None + + +async def connect() -> redis.Redis: + global client + client = redis.from_url( + config.REDIS_URL, + socket_connect_timeout=config.REDIS_SOCKET_TIMEOUT, + socket_timeout=config.REDIS_SOCKET_TIMEOUT, + decode_responses=True, + ) + await client.ping() + return client + + +async def close() -> None: + global client + if client is not None: + await client.close() + client = None + + +def require() -> redis.Redis: + if client is None: + raise AppError(503, "service unavailable") + return client + + +async def ping() -> None: + try: + await require().ping() + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None diff --git a/app/routes.py b/app/routes.py new file mode 100644 index 00000000..0b7bb307 --- /dev/null +++ b/app/routes.py @@ -0,0 +1,284 @@ +import base64 +import binascii + +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +from app import cache, config, db, rediscli, sessions, store +from app.cursor_codec import decode_cursor, encode_cursor +from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response +from app.passwords import dummy_hash, hash_password, verify_password +from app.validation import ( + parse_json_body, + require_exact_keys, + validate_display_name, + validate_limit, + validate_idempotency_key, + validate_message_text, + validate_password, + validate_title, + validate_username, + validate_uuid, +) + + +def _json(data: dict, request_id: str, status: int = 200) -> JSONResponse: + return JSONResponse(status_code=status, headers={"X-Request-Id": request_id}, content=data) + + +def _empty(request_id: str) -> Response: + return Response(status_code=204, headers={"X-Request-Id": request_id}) + + +async def _bearer_user(request: Request) -> dict: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "authentication required", BEARER_CHALLENGE) + user_id = str(await sessions.resolve(rest.strip())) + user = await store.get_user(user_id) + if user is None: + raise AppError(401, "authentication required", BEARER_CHALLENGE) + return user + + +async def health_live(request: Request) -> Response: + return _json({"status": "ok"}, request.state.request_id) + + +async def health_ready(request: Request) -> Response: + rid = request.state.request_id + if request.app.state.shutting_down: + return error_response(503, "service is shutting down", rid) + await db.ping() + await rediscli.ping() + return _json({"status": "ok"}, rid) + + +async def register_user(request: Request) -> Response: + rid = request.state.request_id + if request.headers.get("content-type", "").split(";")[0].strip().lower() not in ("application/json", ""): + if request.headers.get("content-type"): + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}): + return error_response(400, "invalid user payload", rid) + if validate_username(body["username"]) or validate_display_name(body["display_name"]) or validate_password(body["password"]): + return error_response(400, "invalid user fields", rid) + user, conflict = await store.create_user(body["username"], body["display_name"], hash_password(body["password"])) + if conflict: + return error_response(409, "username already taken", rid) + assert user is not None + return _json(user, rid, 201) + + +async def current_user(request: Request) -> Response: + user = await _bearer_user(request) + return _json(user, request.state.request_id) + + +async def create_session(request: Request) -> Response: + rid = request.state.request_id + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "basic" or not rest.strip(): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + try: + decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeError, ValueError): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + username, sep, password = decoded.partition(":") + if not sep: + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + row = await store.get_user_by_username(username) + password_hash = row["password_hash"] if row else dummy_hash() + if row is None or not verify_password(password_hash, password): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + token, expires_at = await sessions.create(row["id"]) + return _json( + { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": store.user_dict(row), + }, + rid, + 201, + ) + + +async def revoke_session(request: Request) -> Response: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "authentication required", BEARER_CHALLENGE) + await sessions.revoke(rest.strip()) + return _empty(request.state.request_id) + + +async def create_chat(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if ct and ct != "application/json": + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"title", "member_ids"}, {"title", "member_ids"}): + return error_response(400, "invalid chat payload", rid) + if validate_title(body["title"]): + return error_response(400, "invalid title", rid) + members = body["member_ids"] + if not isinstance(members, list) or not members: + return error_response(400, "invalid member_ids", rid) + if len(members) > 100: + return error_response(400, "too many members", rid) + for mid in members: + if validate_uuid(mid): + return error_response(400, "invalid member id", rid) + chat, create_err = await store.create_chat(body["title"], members, user["id"]) + if create_err == "caller not in members": + return error_response(400, "caller must be a member", rid) + if create_err in ("duplicate members", "unknown member"): + return error_response(400, "invalid members", rid) + assert chat is not None + return _json(chat, rid, 201) + + +async def list_chats(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + limit, limit_err = validate_limit(request.query_params.get("limit")) + if limit_err: + return error_response(400, limit_err, rid) + assert limit is not None + cursor_raw = request.query_params.get("cursor") + after = None + if cursor_raw is not None: + if not cursor_raw.strip(): + return error_response(400, "invalid cursor", rid) + try: + created_at, entity_id = decode_cursor(cursor_raw, user["id"]) + except ValueError: + return error_response(400, "invalid cursor", rid) + after = (created_at, entity_id) + items, next_key = await store.list_chats(user["id"], limit, after) + next_cursor = None + if next_key is not None: + next_cursor = encode_cursor(user["id"], next_key[0], next_key[1]) + return _json({"items": items, "next_cursor": next_cursor}, rid) + + +async def get_chat(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + chat = await store.get_chat(chat_id, user["id"]) + if chat is None: + return error_response(404, "chat not found", rid) + return _json(chat, rid) + + +async def create_message(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if ct and ct != "application/json": + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"text"}, {"text"}): + return error_response(400, "invalid message payload", rid) + if validate_message_text(body["text"]): + return error_response(400, "invalid message text", rid) + if validate_idempotency_key(request.headers.get("Idempotency-Key")): + return error_response(400, "invalid idempotency key", rid) + message, msg_err = await store.add_message( + chat_id, + user["id"], + body["text"], + request.headers.get("Idempotency-Key") or "", + request.url.path, + ) + if msg_err == "not_found": + return error_response(404, "chat not found", rid) + if msg_err == "conflict": + return error_response(409, "idempotency key reused with a different body", rid) + assert message is not None + return _json(message, rid, 201) + + +async def list_messages(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + limit, limit_err = validate_limit(request.query_params.get("limit")) + if limit_err: + return error_response(400, limit_err, rid) + assert limit is not None + cursor_raw = request.query_params.get("cursor") + after = None + if cursor_raw is not None: + if not cursor_raw.strip(): + return error_response(400, "invalid cursor", rid) + try: + created_at, entity_id = decode_cursor(cursor_raw, chat_id) + except ValueError: + return error_response(400, "invalid cursor", rid) + after = (created_at, entity_id) + page, next_key, list_err = await store.list_messages(chat_id, user["id"], limit, after) + if list_err == "not_found": + return error_response(404, "chat not found", rid) + assert page is not None + next_cursor = None + if next_key is not None: + next_cursor = encode_cursor(chat_id, next_key[0], next_key[1]) + return _json({"items": page, "next_cursor": next_cursor}, rid) + + +async def lab_metrics(request: Request) -> Response: + if not config.lab_mode(): + return error_response(404, "not found", request.state.request_id) + return _json( + { + "singleflight": await cache.singleflight_enabled(), + "invalidate": await cache.invalidate_enabled(), + "history_sql": await cache.history_sql_count(), + }, + request.state.request_id, + ) + + +async def lab_flags(request: Request) -> Response: + if not config.lab_mode(): + return error_response(404, "not found", request.state.request_id) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, request.state.request_id) + assert body is not None + if body.get("reset_sql"): + await cache.reset_history_sql_count() + flags = await cache.set_flags( + singleflight=body.get("singleflight"), + invalidate=body.get("invalidate"), + delay_ms=body.get("delay_ms"), + ) + return _json(flags, request.state.request_id) diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..b0e30b11 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,56 @@ +import hashlib +import json +import secrets +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from app import config, rediscli +from app.errors import AppError, BEARER_CHALLENGE + + +def new_token() -> str: + return secrets.token_urlsafe(32) + + +def _key(token: str) -> str: + return "sess:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _fmt(dt: datetime) -> str: + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +async def create(user_id: UUID) -> tuple[str, str]: + token = new_token() + expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + payload = json.dumps({"user_id": str(user_id), "expires_at": _fmt(expires)}) + try: + await rediscli.require().set(_key(token), payload, ex=config.SESSION_TTL_SECONDS) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + return token, _fmt(expires) + + +async def resolve(token: str) -> UUID: + try: + raw = await rediscli.require().get(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if not raw: + raise AppError(401, "authentication required", BEARER_CHALLENGE) + return UUID(json.loads(raw)["user_id"]) + + +async def revoke(token: str) -> None: + try: + deleted = await rediscli.require().delete(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if not deleted: + raise AppError(401, "authentication required", BEARER_CHALLENGE) diff --git a/app/store.py b/app/store.py new file mode 100644 index 00000000..04e37555 --- /dev/null +++ b/app/store.py @@ -0,0 +1,361 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import List, Optional, Tuple +from uuid import UUID, uuid4 + +import asyncpg + +from app import cache, config, db +from app.errors import AppError + + +def format_ts(dt: datetime) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def user_dict(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_ts(row["created_at"]), + } + + +def chat_dict(row, member_ids: List[str]) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": format_ts(row["created_at"]), + } + + +def message_dict(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_ts(row["created_at"]), + } + + +async def _run(coro): + try: + return await coro + except AppError: + raise + except db.INTEGRITY: + raise + except Exception: + raise AppError(503, "service unavailable") from None + + +async def create_user(username: str, display_name: str, password_hash: str) -> Tuple[Optional[dict], Optional[str]]: + user_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + await conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + user_id, + username, + display_name, + password_hash, + created, + ) + except asyncpg.UniqueViolationError: + return None, "conflict" + except Exception: + raise AppError(503, "service unavailable") from None + return user_dict({"id": user_id, "username": username, "display_name": display_name, "created_at": created}), None + + +async def get_user(user_id: str) -> Optional[dict]: + row = await _run( + _fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", UUID(user_id)) + ) + return user_dict(row) if row else None + + +async def get_user_by_username(username: str): + return await _run( + _fetchrow( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1", + username, + ) + ) + + +async def _fetchrow(sql: str, *args): + async with db.require().acquire() as conn: + return await conn.fetchrow(sql, *args) + + +async def _fetch(sql: str, *args): + async with db.require().acquire() as conn: + return await conn.fetch(sql, *args) + + +async def member_ids_of(chat_id: UUID) -> List[str]: + rows = await _run(_fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id)) + return [str(r["user_id"]) for r in rows] + + +async def create_chat(title: str, member_ids: List[str], caller_id: str) -> Tuple[Optional[dict], Optional[str]]: + if caller_id not in member_ids: + return None, "caller not in members" + if len(set(member_ids)) != len(member_ids): + return None, "duplicate members" + ids = [UUID(x) for x in member_ids] + chat_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", ids) + if len(existing) != len(ids): + return None, "unknown member" + await conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", + chat_id, + title, + created, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, mid) for mid in ids], + ) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + return chat_dict({"id": chat_id, "title": title, "created_at": created}, member_ids), None + + +async def get_chat(chat_id: str, user_id: str) -> Optional[dict]: + row = await load_chat_for_member(chat_id, user_id) + if row is None: + return None + return chat_dict(row, await member_ids_of(UUID(chat_id))) + + +async def load_chat_for_member(chat_id: str, user_id: str): + return await _run( + _fetchrow( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 + WHERE c.id = $1 + """, + UUID(chat_id), + UUID(user_id), + ) + ) + + +async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] = None): + args: list = [UUID(user_id)] + clause = "" + if after is not None: + clause = "AND (c.created_at, c.id) > ($2, $3)" + args.extend([datetime.fromisoformat(after[0].replace("Z", "+00:00")), UUID(after[1])]) + rows = await _run( + _fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1 + WHERE 1=1 {clause} + ORDER BY c.created_at, c.id + LIMIT {int(limit) + 1} + """, + *args, + ) + ) + page = rows[:limit] + items = [chat_dict(row, await member_ids_of(row["id"])) for row in page] + next_key = None + if len(rows) > limit: + last = page[-1] + next_key = (format_ts(last["created_at"]), str(last["id"])) + return items, next_key + + +def _canonical_message(text: str) -> str: + # Missing attachment_ids and [] are the same message. Key order does not matter. + return json.dumps( + {"attachment_ids": [], "text": text}, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ) + + +async def add_message( + chat_id: str, + sender_id: str, + text: str, + idem_key: str, + path: str, +) -> Tuple[Optional[dict], Optional[str]]: + """Insert one message or replay the stored 201. The row and the result commit together. + + Same user+method+path+key is serialized with a Postgres advisory lock so two + replicas cannot insert two messages. A failed transaction stores nothing. + """ + digest = hashlib.sha256(_canonical_message(text).encode("utf-8")).hexdigest() + lock_key = f"{sender_id}|POST|{path}|{idem_key}" + payload: Optional[dict] = None + created_new = False + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))", lock_key) + member = await conn.fetchrow( + """ + SELECT c.id + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 + WHERE c.id = $1 + """, + UUID(chat_id), + UUID(sender_id), + ) + if member is None: + return None, "not_found" + row = await conn.fetchrow( + """ + SELECT request_hash, response_json, created_at + FROM idempotency_keys + WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3 + """, + UUID(sender_id), + path, + idem_key, + ) + if row is not None: + created_at = row["created_at"] + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + age = (datetime.now(timezone.utc) - created_at).total_seconds() + if age <= config.IDEMPOTENCY_TTL_SECONDS: + if row["request_hash"] != digest: + return None, "conflict" + return json.loads(row["response_json"]), None + await conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3 + """, + UUID(sender_id), + path, + idem_key, + ) + message_id = uuid4() + created = datetime.now(timezone.utc) + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + message_id, + UUID(chat_id), + UUID(sender_id), + text, + created, + ) + payload = message_dict( + { + "id": message_id, + "chat_id": UUID(chat_id), + "sender_id": UUID(sender_id), + "text": text, + "created_at": created, + } + ) + await conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, http_method, path, idem_key, request_hash, response_json, created_at) + VALUES ($1, 'POST', $2, $3, $4, $5, $6) + """, + UUID(sender_id), + path, + idem_key, + digest, + json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + created, + ) + created_new = True + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if created_new: + await cache.invalidate_history(chat_id) + return payload, None + + +async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[Tuple[str, str]] = None): + if await load_chat_for_member(chat_id, user_id) is None: + return None, None, "not_found" + cursor = "" + after_ts = after_id = None + if after is not None: + cursor = after[0] + ":" + after[1] + after_ts = datetime.fromisoformat(after[0].replace("Z", "+00:00")) + after_id = UUID(after[1]) + + async def loader() -> dict: + await cache.incr_history_sql() + if after_ts is None: + rows = await _run( + _fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id + LIMIT $2 + """, + UUID(chat_id), + limit + 1, + ) + ) + else: + rows = await _run( + _fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT $4 + """, + UUID(chat_id), + after_ts, + after_id, + limit + 1, + ) + ) + page = rows[:limit] + items = [message_dict(r) for r in page] + next_key = None + if len(rows) > limit: + last = page[-1] + next_key = (format_ts(last["created_at"]), str(last["id"])) + return {"items": items, "next_key": next_key} + + page = await cache.get_history_page(chat_id, cursor, limit, loader) + return page["items"], page.get("next_key"), None diff --git a/app/tools/__init__.py b/app/tools/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/app/tools/__init__.py @@ -0,0 +1 @@ + diff --git a/app/tools/explain.py b/app/tools/explain.py new file mode 100644 index 00000000..e649ad14 --- /dev/null +++ b/app/tools/explain.py @@ -0,0 +1,73 @@ +"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for keyset pagination.""" +import asyncio +import os +from uuid import uuid4 + +import asyncpg + +from app.passwords import hash_password + + +async def main() -> None: + conn = await asyncpg.connect(dsn=os.environ["DATABASE_URL"], timeout=10, command_timeout=120) + try: + alice = uuid4() + bob = uuid4() + chat = uuid4() + pwd = hash_password("Example-only-not-a-real-secret-123") + await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)", + alice, + "explain_a_" + alice.hex[:8], + "Alice", + pwd, + bob, + "explain_b_" + bob.hex[:8], + "Bob", + ) + await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat") + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)", + chat, + alice, + bob, + ) + n = int(os.environ.get("EXPLAIN_MESSAGES", "10000")) + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval + FROM generate_series(1, $3) AS g + """, + chat, + alice, + n, + ) + count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat) + print("seeded_messages", count, "chat_id", chat) + first = await conn.fetchrow( + "SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1", + chat, + ) + query = """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT 50 + """ + print("\n=== WITH index messages_chat_created_id_idx ===") + plan = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"]) + print("\n".join(row[0] for row in plan)) + await conn.execute("SET enable_indexscan = off") + await conn.execute("SET enable_bitmapscan = off") + await conn.execute("SET enable_indexonlyscan = off") + print("\n=== WITHOUT index scans (seq scan baseline) ===") + plan2 = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"]) + print("\n".join(row[0] for row in plan2)) + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..9723267b --- /dev/null +++ b/app/validation.py @@ -0,0 +1,102 @@ +import json +import re +from typing import Any, Dict, Optional, Tuple +from uuid import UUID + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def parse_json_body(raw: bytes) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + if not raw: + return None, "empty body" + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None, "malformed JSON" + if not isinstance(data, dict): + return None, "JSON body must be an object" + return data, None + + +def require_exact_keys(data: Dict[str, Any], allowed: set, required: set) -> Optional[str]: + keys = set(data.keys()) + if keys - allowed: + return "unexpected fields" + if required - keys: + return "missing required fields" + return None + + +def validate_username(value: Any) -> Optional[str]: + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + return "invalid username" + return None + + +def validate_display_name(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid display_name" + if not value.strip(): + return "display_name must not be blank" + if len(value) > 100: + return "display_name too long" + return None + + +def validate_password(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid password" + if not value.strip(): + return "password must not be blank" + if len(value) < 12 or len(value) > 128: + return "password length" + return None + + +def validate_title(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid title" + if not value.strip(): + return "title must not be blank" + if len(value) > 100: + return "title too long" + return None + + +def validate_uuid(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid uuid" + try: + UUID(value) + except ValueError: + return "invalid uuid" + return None + + +def validate_idempotency_key(value: Optional[str]) -> Optional[str]: + if value is None or IDEMPOTENCY_RE.fullmatch(value) is None: + return "invalid idempotency key" + return None + + +def validate_message_text(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid text" + if not value.strip(): + return "text must not be blank" + if len(value) > 4000: + return "text too long" + return None + + +def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]: + if value is None: + return default, None + try: + limit = int(value) + except ValueError: + return None, "invalid limit" + if limit < 1 or limit > 100: + return None, "limit out of range" + return limit, None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..8a85ffbe --- /dev/null +++ b/compose.yaml @@ -0,0 +1,131 @@ +# Port of the lab3 stand. Client -> lb:8080 -> api (scale) -> shared Postgres and Redis. +# API replicas do not publish host ports. Migrations run once in `migrate`. +# Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead +# or not-ready replica is about 2s, plus the 1s DNS hold. API on SIGTERM +# fails /health/ready immediately and exits after LB_DRAIN_SECONDS (default 2). + +services: + migrate: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab4 + command: ["python", "-m", "app.migrate"] + environment: + APP_ENV: lab + LOG_LEVEL: info + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + DB_POOL_MIN: "1" + DB_POOL_MAX: "2" + MIGRATIONS_ON_START: "1" + depends_on: + postgres: + condition: service_healthy + restart: "no" + mem_limit: 256m + cpus: "0.50" + + api: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab4 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + LOG_LEVEL: info + SHUTDOWN_GRACE_SECONDS: "3" + LB_DRAIN_SECONDS: ${LB_DRAIN_SECONDS:-2} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: redis + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + REDIS_URL: redis://redis:6379/0 + DB_POOL_MIN: "1" + DB_POOL_MAX: ${DB_POOL_MAX:-4} + DB_COMMAND_TIMEOUT: "5" + DB_CONNECT_TIMEOUT: "5" + MIGRATIONS_ON_START: "0" + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + expose: + - "8080" + restart: unless-stopped + mem_limit: ${API_MEM:-256m} + cpus: ${API_CPUS:-0.50} + stop_grace_period: 8s + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)", + ] + interval: 5s + timeout: 3s + retries: 6 + start_period: 5s + + lb: + image: haproxy:2.8.14-alpine + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + api: + condition: service_started + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" + + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: messenger + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: ${POSTGRES_MEM:-512m} + cpus: ${POSTGRES_CPUS:-1.0} + + redis: + image: redis:7.4.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --save + - "60" + - "1" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 128m + cpus: "0.50" + +volumes: + pg_data: + redis_data: diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..24c99b80 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 info + maxconn 200 + +defaults + log global + mode http + option httplog + timeout connect 2s + timeout client 10s + timeout server 10s + timeout http-request 5s + timeout check 2s + timeout queue 2s + retries 1 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + accepted_payload_size 8192 + +frontend fe_http + bind *:8080 + default_backend be_api + +backend be_api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr host api + http-check expect status 200 + default-server inter 1s fall 2 rise 1 + server-template srv 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..2501deea --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,49 @@ +-- Lab2 schema (written before application queries). +-- Source of truth: PostgreSQL for users, chats, membership, messages. +-- Transaction boundary: INSERT chat + INSERT members in one transaction (rollback together). +-- Sessions are stored in Redis with AOF, not in this database. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE UNIQUE INDEX users_username_key ON users (username); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id); +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset history: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..acc555c6 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,14 @@ +-- Shared idempotency for POST /messages. Scope: user + method + path + key. +-- The message row and this result are committed in one transaction. +-- A rolled-back request leaves no row, so a later retry with the same key can insert. + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users (id), + http_method TEXT NOT NULL, + path TEXT NOT NULL, + idem_key TEXT NOT NULL, + request_hash TEXT NOT NULL, + response_json TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, http_method, path, idem_key) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4a028ebb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +starlette==0.41.3 +uvicorn[standard]==0.32.1 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==23.1.0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..6a3be3ad --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Restart API replicas and every store they use. Volumes are kept. +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +if [ -f .env ]; then + docker compose --env-file .env restart postgres redis api +else + docker compose restart postgres redis api +fi diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..d22d4a10 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Replace API replicas one at a time. The other replica keeps serving. +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +compose() { + if [ -f .env ]; then + docker compose --env-file .env "$@" + else + docker compose "$@" + fi +} + +if command -v python3 >/dev/null 2>&1; then + PY=python3 +else + PY=python +fi + +ids=$(compose ps -q api) +count=0 +for id in $ids; do + count=$((count + 1)) +done +if [ "$count" -lt 2 ]; then + echo "rolling update needs at least 2 running API replicas" >&2 + exit 1 +fi + +for id in $ids; do + echo "replacing $(docker inspect -f '{{.Config.Hostname}}' "$id")" + docker stop -t 8 "$id" >/dev/null + docker rm "$id" >/dev/null + compose up -d --scale "api=$count" --no-deps --no-recreate api >/dev/null + ready=0 + i=0 + while [ "$i" -lt 40 ]; do + running=0 + for cid in $(compose ps -q api); do + running=$((running + 1)) + done + if [ "$running" -ge "$count" ] && "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" >/dev/null 2>&1; then + ready=1 + break + fi + i=$((i + 1)) + sleep 1 + done + if [ "$ready" -ne 1 ]; then + echo "replacement did not become ready" >&2 + exit 1 + fi +done +echo "rolling update finished; replicas=$count" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..707e422d --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Stop the single API replica whose X-Instance-Id is TARGET_INSTANCE_ID. +# Leaves it stopped. Does not touch the database, Redis, the load balancer, or volumes. +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +compose() { + if [ -f .env ]; then + docker compose --env-file .env "$@" + else + docker compose "$@" + fi +} + +found=0 +for id in $(compose ps -q api); do + iid=$(docker exec "$id" cat /tmp/instance-id 2>/dev/null | tr -d '\r\n' || true) + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + if [ "$iid" = "$TARGET_INSTANCE_ID" ] || [ "$host" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 5 "$id" >/dev/null + found=1 + break + fi +done + +if [ "$found" -ne 1 ]; then + echo "no running API replica matched TARGET_INSTANCE_ID" >&2 + exit 1 +fi -- 2.54.0 From 00445bc41be6c5f4279400c99a94a6f5d8ba6e9c Mon Sep 17 00:00:00 2001 From: David Date: Mon, 9 Nov 2026 00:25:18 +0300 Subject: [PATCH 2/5] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BF=D1=80=D0=B8=D0=B2=D0=B0=D1=82=D0=BD=D0=BE=D0=B5?= =?UTF-8?q?=20=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB=D0=B8=D1=89=D0=B5=20?= =?UTF-8?q?=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8=D0=B9=20=D0=B8=20?= =?UTF-8?q?=D0=BE=D1=82=D0=B4=D0=B0=D1=87=D1=83=20=D1=84=D0=B0=D0=B9=D0=BB?= =?UTF-8?q?=D0=B0=20=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20API.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Байты лежат в SeaweedFS, метаданные в Postgres. Другая реплика скачивает тот же объект, потому что локальный диск API не является источником файла. --- .env.example | 4 +- .gitattributes | 2 + app/attachments.py | 214 +++++++++++++++++++++++++++++++++ app/config.py | 9 ++ app/images.py | 107 +++++++++++++++++ app/main.py | 12 +- app/middleware.py | 65 ++++++---- app/routes.py | 99 ++++++++++++++- app/s3.py | 109 +++++++++++++++++ app/s3bootstrap.py | 48 ++++++++ app/store.py | 69 +++++++++-- app/validation.py | 35 +++++- compose.yaml | 56 ++++++++- infra/README.md | 2 + infra/haproxy.cfg | 6 +- infra/s3/seaweed-entrypoint.sh | 48 ++++++++ migrations/003_attachments.sql | 50 ++++++++ requirements.txt | 2 + scripts/restart.sh | 4 +- 19 files changed, 889 insertions(+), 52 deletions(-) create mode 100644 .gitattributes create mode 100644 app/attachments.py create mode 100644 app/images.py create mode 100644 app/s3.py create mode 100644 app/s3bootstrap.py create mode 100755 infra/s3/seaweed-entrypoint.sh create mode 100644 migrations/003_attachments.sql diff --git a/.env.example b/.env.example index 9b4f9ff3..955c92df 100644 --- a/.env.example +++ b/.env.example @@ -26,9 +26,7 @@ REDIS_URL=redis://redis:6379/0 S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -# Root is used only by bucket bootstrap. The API uses the app key below. -MINIO_ROOT_USER=lab4-minio-root -MINIO_ROOT_PASSWORD=lab4-dev-only-change-me +# SeaweedFS identity for the API only. Not a cluster admin. Must match the running .env. S3_ACCESS_KEY_ID=lab4appkey00000001 S3_SECRET_ACCESS_KEY=lab4-dev-only-change-me MAX_UPLOAD_BYTES=10485760 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..67613ca3 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +*.sh text eol=lf +infra/s3/* text eol=lf diff --git a/app/attachments.py b/app/attachments.py new file mode 100644 index 00000000..9caaf439 --- /dev/null +++ b/app/attachments.py @@ -0,0 +1,214 @@ +"""Upload path: validate bytes, PUT, then commit metadata. See migration 003.""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from app import config, db, s3 +from app.errors import AppError +from app.store import format_ts + +logger = logging.getLogger("messenger.attachments") +_sweep_task: asyncio.Task | None = None + + +def _public(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": None, + "created_at": format_ts(row["created_at"]), + "updated_at": format_ts(row["updated_at"]), + } + + +async def _drop_intent(key: str) -> None: + async with db.require().acquire() as conn: + await conn.execute( + "DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed", + key, + ) + + +async def _compensate(key: str) -> None: + try: + await s3.delete_key(key) + except AppError: + logger.warning("orphan object left for sweeper key=%s", key) + return + try: + await _drop_intent(key) + except Exception: + logger.warning("orphan intent left for sweeper key=%s", key) + + +async def create_attachment( + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + data: bytes, +) -> dict: + attachment_id = uuid4() + key = f"originals/{attachment_id}" + created = datetime.now(timezone.utc) + digest = hashlib.sha256(data).hexdigest() + async with db.require().acquire() as conn: + await conn.execute( + """ + INSERT INTO blob_writes (object_key, committed, created_at) + VALUES ($1, FALSE, $2) + """, + key, + created, + ) + try: + await s3.put_bytes(key, data, content_type) + except Exception: + await _compensate(key) + raise + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + intent = await conn.fetchrow( + "SELECT committed FROM blob_writes WHERE object_key = $1 FOR UPDATE", + key, + ) + if intent is None or intent["committed"]: + raise AppError(503, "service unavailable") + await conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, status, error_code, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ready', NULL, $9, $9) + """, + attachment_id, + UUID(chat_id), + UUID(owner_id), + filename, + content_type, + len(data), + digest, + key, + created, + ) + await conn.execute( + "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", + key, + ) + except AppError: + await _compensate(key) + raise + except Exception: + await _compensate(key) + raise AppError(503, "service unavailable") from None + return _public( + { + "id": attachment_id, + "chat_id": UUID(chat_id), + "owner_id": UUID(owner_id), + "filename": filename, + "content_type": content_type, + "size_bytes": len(data), + "sha256": digest, + "status": "ready", + "created_at": created, + "updated_at": created, + } + ) + + +async def get_for_member(attachment_id: str, user_id: str): + async with db.require().acquire() as conn: + row = await conn.fetchrow( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.object_key, a.status, a.created_at, a.updated_at + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 + WHERE a.id = $1 + """, + UUID(attachment_id), + UUID(user_id), + ) + return row + + +async def sweep_orphans() -> int: + """Delete uncommitted objects older than the grace period. In-use rows stay.""" + deleted = 0 + async with db.require().acquire() as conn: + async with conn.transaction(): + rows = await conn.fetch( + """ + SELECT object_key + FROM blob_writes + WHERE NOT committed + AND created_at < now() - ($1::int * interval '1 second') + ORDER BY created_at + LIMIT 50 + FOR UPDATE SKIP LOCKED + """, + config.ORPHAN_GRACE_SECONDS, + ) + for row in rows: + key = row["object_key"] + linked = await conn.fetchval( + "SELECT 1 FROM attachments WHERE object_key = $1", + key, + ) + if linked: + await conn.execute( + "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", + key, + ) + continue + await s3.delete_key(key) + await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", key) + deleted += 1 + if deleted: + logger.info("removed %s uncommitted objects past grace", deleted) + return deleted + + +async def _sweep_loop() -> None: + while True: + try: + await sweep_orphans() + except asyncio.CancelledError: + raise + except Exception: + logger.warning("orphan sweep failed") + await asyncio.sleep(config.S3_SWEEP_INTERVAL_SECONDS) + + +def start_sweeper() -> None: + global _sweep_task + if _sweep_task is None or _sweep_task.done(): + _sweep_task = asyncio.create_task(_sweep_loop()) + + +async def stop_sweeper() -> None: + global _sweep_task + task = _sweep_task + _sweep_task = None + if task is None: + return + task.cancel() + try: + await task + except asyncio.CancelledError: + pass diff --git a/app/config.py b/app/config.py index b6d1f3b1..005c455e 100644 --- a/app/config.py +++ b/app/config.py @@ -36,6 +36,15 @@ LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2")) # Stable for the life of this process. Compose leaves it unset so each # container uses its own hostname; stop-one.sh maps that value to one replica. INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname()) +S3_ENDPOINT = _env("S3_ENDPOINT", "http://s3:9000") +S3_REGION = _env("S3_REGION", "us-east-1") +S3_BUCKET = _env("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _file_or_env("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _file_or_env("S3_SECRET_ACCESS_KEY") +MAX_UPLOAD_BYTES = min(int(_env("MAX_UPLOAD_BYTES", "10485760")), 10_485_760) +MAX_IMAGE_PIXELS = min(int(_env("MAX_IMAGE_PIXELS", "40000000")), 40_000_000) +ORPHAN_GRACE_SECONDS = int(_env("ORPHAN_GRACE_SECONDS", "120")) +S3_SWEEP_INTERVAL_SECONDS = int(_env("S3_SWEEP_INTERVAL_SECONDS", "30")) def lab_mode() -> bool: diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..634a711a --- /dev/null +++ b/app/images.py @@ -0,0 +1,107 @@ +"""PNG/JPEG checks that reject bad files before a full pixel decode.""" + +from __future__ import annotations + +import io +from typing import Optional, Tuple + +from PIL import Image, ImageFile, UnidentifiedImageError + +from app import config + +ImageFile.LOAD_TRUNCATED_IMAGES = False +_PNG = b"\x89PNG\r\n\x1a\n" +_JPEG = b"\xff\xd8\xff" +_SOF = {0xC0, 0xC1, 0xC2} + + +class ImageRejected(Exception): + def __init__(self, status: int, message: str) -> None: + super().__init__(message) + self.status = status + self.message = message + + +def _reject(status: int, message: str) -> None: + raise ImageRejected(status, message) + + +def sniff(data: bytes) -> Optional[str]: + if data.startswith(_PNG): + return "image/png" + if data.startswith(_JPEG): + return "image/jpeg" + return None + + +def _png_size(data: bytes) -> Tuple[int, int]: + if len(data) < 24 or data[12:16] != b"IHDR": + _reject(415, "unsupported or damaged image") + length = int.from_bytes(data[8:12], "big") + if length != 13 or len(data) < 33: + _reject(415, "unsupported or damaged image") + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + return width, height + + +def _jpeg_size(data: bytes) -> Tuple[int, int]: + index = 2 + size = len(data) + while index < size: + if data[index] != 0xFF: + _reject(415, "unsupported or damaged image") + while index < size and data[index] == 0xFF: + index += 1 + if index >= size: + break + marker = data[index] + index += 1 + if marker in (0xD8, 0xD9, 0x01) or 0xD0 <= marker <= 0xD7: + continue + if index + 1 >= size: + break + segment = int.from_bytes(data[index : index + 2], "big") + if segment < 2 or index + segment > size: + _reject(415, "unsupported or damaged image") + if marker in _SOF: + if segment < 7: + _reject(415, "unsupported or damaged image") + height = int.from_bytes(data[index + 3 : index + 5], "big") + width = int.from_bytes(data[index + 5 : index + 7], "big") + return width, height + if marker == 0xDA: + break + index += segment + _reject(415, "unsupported or damaged image") + return 0, 0 + + +def _check_pixels(width: int, height: int) -> None: + limit = config.MAX_IMAGE_PIXELS + if width <= 0 or height <= 0: + _reject(415, "unsupported or damaged image") + if width > limit or height > limit or width * height > limit: + _reject(413, "image dimensions exceed the limit") + + +def _verify(data: bytes) -> None: + Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS + try: + with Image.open(io.BytesIO(data)) as image: + image.verify() + except Image.DecompressionBombError: + _reject(413, "image dimensions exceed the limit") + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + _reject(415, "unsupported or damaged image") + + +def inspect_image(data: bytes, declared: str) -> str: + """Return the sniffed content type or raise ImageRejected (413 or 415).""" + found = sniff(data) + if found is None or found != declared: + _reject(415, "unsupported or damaged image") + width, height = _png_size(data) if found == "image/png" else _jpeg_size(data) + _check_pixels(width, height) + _verify(data) + return found diff --git a/app/main.py b/app/main.py index c59ca752..b4ef0743 100644 --- a/app/main.py +++ b/app/main.py @@ -12,7 +12,7 @@ from starlette.requests import Request from starlette.responses import Response from starlette.routing import Route -from app import config, db, rediscli +from app import attachments, config, db, rediscli from app.docs import openapi_json, swagger_ui from app.errors import AppError, error_response, from_app_error from app.middleware import RequestContextMiddleware @@ -21,15 +21,19 @@ from app.routes import ( create_message, create_session, current_user, + download_attachment, + get_attachment, get_chat, health_live, health_ready, lab_flags, lab_metrics, + lab_sweep, list_chats, list_messages, register_user, revoke_session, + upload_attachment, ) @@ -63,9 +67,11 @@ async def startup() -> None: if config.MIGRATIONS_ON_START: await db.apply_migrations() await rediscli.connect() + attachments.start_sweeper() async def shutdown() -> None: + await attachments.stop_sweeper() await rediscli.close() await db.close() @@ -86,8 +92,12 @@ def build_app() -> Starlette: Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]), + Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]), + Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]), Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]), Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]), + Route("/api/v1/_lab/sweep", lab_sweep, methods=["POST"]), ], exception_handlers={404: not_found_handler, AppError: app_error_handler}, on_startup=[startup], diff --git a/app/middleware.py b/app/middleware.py index 4b05cca3..f79e98e1 100644 --- a/app/middleware.py +++ b/app/middleware.py @@ -1,11 +1,10 @@ import logging import time import uuid -from typing import Callable -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.requests import Request +from starlette.datastructures import MutableHeaders from starlette.responses import Response +from starlette.types import ASGIApp, Message, Receive, Scope, Send from app.config import INSTANCE_ID from app.errors import AppError, error_response, from_app_error @@ -13,36 +12,52 @@ from app.errors import AppError, error_response, from_app_error logger = logging.getLogger("messenger.access") -def _stamp(response: Response, request_id: str) -> Response: - response.headers["X-Request-Id"] = request_id - response.headers["X-Instance-Id"] = INSTANCE_ID - return response +class RequestContextMiddleware: + """Pure ASGI middleware so upload handlers can read the body stream themselves.""" + def __init__(self, app: ASGIApp) -> None: + self.app = app -class RequestContextMiddleware(BaseHTTPMiddleware): - async def dispatch(self, request: Request, call_next: Callable) -> Response: - request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) - request.state.request_id = request_id - - if request.app.state.shutting_down and request.url.path not in ("/health/live", "/health/ready"): - return _stamp(error_response(503, "service is shutting down", request_id), request_id) - + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + headers = {key.decode("latin1").lower(): value.decode("latin1") for key, value in scope.get("headers", [])} + request_id = headers.get("x-request-id") or str(uuid.uuid4()) + raw_state = scope.get("state") + if not isinstance(raw_state, dict): + raw_state = {} + scope["state"] = raw_state + raw_state["request_id"] = request_id + path = scope.get("path", "") started = time.perf_counter() + + async def send_wrapper(message: Message) -> None: + if message["type"] == "http.response.start": + response_headers = MutableHeaders(scope=message) + response_headers["X-Request-Id"] = request_id + response_headers["X-Instance-Id"] = INSTANCE_ID + await send(message) + + app = scope.get("app") + shutting_down = bool(app and getattr(app.state, "shutting_down", False)) + if shutting_down and path not in ("/health/live", "/health/ready"): + response: Response = error_response(503, "service is shutting down", request_id) + await response(scope, receive, send_wrapper) + return try: - response = await call_next(request) + await self.app(scope, receive, send_wrapper) except AppError as exc: - return _stamp(from_app_error(exc, request_id), request_id) + await from_app_error(exc, request_id)(scope, receive, send_wrapper) except Exception: - logger.exception("request_id=%s unhandled error path=%s", request_id, request.url.path) - return _stamp(error_response(500, "internal error", request_id), request_id) + logger.exception("request_id=%s unhandled error path=%s", request_id, path) + await error_response(500, "internal error", request_id)(scope, receive, send_wrapper) + return elapsed_ms = (time.perf_counter() - started) * 1000 - _stamp(response, request_id) logger.info( - "request_id=%s method=%s path=%s status=%s duration_ms=%.2f", + "request_id=%s method=%s path=%s duration_ms=%.2f", request_id, - request.method, - request.url.path, - response.status_code, + scope.get("method", ""), + path, elapsed_ms, ) - return response diff --git a/app/routes.py b/app/routes.py index 0b7bb307..bb6d1292 100644 --- a/app/routes.py +++ b/app/routes.py @@ -4,14 +4,16 @@ import binascii from starlette.requests import Request from starlette.responses import JSONResponse, Response -from app import cache, config, db, rediscli, sessions, store +from app import attachments, cache, config, db, images, rediscli, s3, sessions, store from app.cursor_codec import decode_cursor, encode_cursor from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response from app.passwords import dummy_hash, hash_password, verify_password from app.validation import ( parse_json_body, require_exact_keys, + parse_attachment_ids, validate_display_name, + validate_filename, validate_limit, validate_idempotency_key, validate_message_text, @@ -52,6 +54,7 @@ async def health_ready(request: Request) -> Response: return error_response(503, "service is shutting down", rid) await db.ping() await rediscli.ping() + await s3.head_bucket() return _json({"status": "ok"}, rid) @@ -202,9 +205,13 @@ async def create_message(request: Request) -> Response: if err: return error_response(400, err, rid) assert body is not None - if require_exact_keys(body, {"text"}, {"text"}): + if require_exact_keys(body, {"text", "attachment_ids"}, {"text"}): return error_response(400, "invalid message payload", rid) - if validate_message_text(body["text"]): + attachment_ids, attachment_err = parse_attachment_ids(body.get("attachment_ids")) + if attachment_err: + return error_response(400, attachment_err, rid) + assert attachment_ids is not None + if validate_message_text(body["text"], allow_blank=bool(attachment_ids)): return error_response(400, "invalid message text", rid) if validate_idempotency_key(request.headers.get("Idempotency-Key")): return error_response(400, "invalid idempotency key", rid) @@ -214,11 +221,14 @@ async def create_message(request: Request) -> Response: body["text"], request.headers.get("Idempotency-Key") or "", request.url.path, + attachment_ids, ) if msg_err == "not_found": return error_response(404, "chat not found", rid) if msg_err == "conflict": return error_response(409, "idempotency key reused with a different body", rid) + if msg_err == "not_ready": + return error_response(409, "attachment is not ready", rid) assert message is not None return _json(message, rid, 201) @@ -253,6 +263,89 @@ async def list_messages(request: Request) -> Response: return _json({"items": page, "next_cursor": next_cursor}, rid) +def _media_type(value: str) -> str: + return value.split(";", 1)[0].strip().lower() + + +async def _read_limited(request: Request, limit: int) -> bytes: + received = bytearray() + async for chunk in request.stream(): + if len(received) + len(chunk) > limit: + raise AppError(413, "file exceeds upload limit") + received.extend(chunk) + return bytes(received) + + +async def upload_attachment(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + if validate_filename(request.query_params.get("filename")): + return error_response(400, "invalid filename", rid) + filename = request.query_params.get("filename") or "" + declared = _media_type(request.headers.get("content-type", "")) + if declared not in ("image/png", "image/jpeg"): + return error_response(415, "unsupported media type", rid) + content_length = request.headers.get("content-length") + if content_length is not None: + try: + announced = int(content_length) + except ValueError: + return error_response(400, "invalid content length", rid) + if announced > config.MAX_UPLOAD_BYTES: + return error_response(413, "file exceeds upload limit", rid) + if announced <= 0: + return error_response(415, "unsupported or damaged image", rid) + if await store.load_chat_for_member(chat_id, user["id"]) is None: + return error_response(404, "chat not found", rid) + data = await _read_limited(request, config.MAX_UPLOAD_BYTES) + if not data: + return error_response(415, "unsupported or damaged image", rid) + try: + content_type = images.inspect_image(data, declared) + except images.ImageRejected as exc: + return error_response(exc.status, exc.message, rid) + created = await attachments.create_attachment(chat_id, user["id"], filename, content_type, data) + return _json(created, rid, 201) + + +async def get_attachment(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + attachment_id = request.path_params["attachment_id"] + if validate_uuid(attachment_id): + return error_response(400, "invalid attachment id", rid) + row = await attachments.get_for_member(attachment_id, user["id"]) + if row is None: + return error_response(404, "attachment not found", rid) + return _json(attachments._public(row), rid) + + +async def download_attachment(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + attachment_id = request.path_params["attachment_id"] + if validate_uuid(attachment_id): + return error_response(400, "invalid attachment id", rid) + variant = request.query_params.get("variant") + if variant not in (None, "original"): + return error_response(400, "invalid variant", rid) + row = await attachments.get_for_member(attachment_id, user["id"]) + if row is None: + return error_response(404, "attachment not found", rid) + payload = await s3.get_bytes(row["object_key"]) + return Response(payload, status_code=200, media_type=row["content_type"], headers={"X-Request-Id": rid}) + + +async def lab_sweep(request: Request) -> Response: + if not config.lab_mode(): + return error_response(404, "not found", request.state.request_id) + deleted = await attachments.sweep_orphans() + return _json({"deleted": deleted}, request.state.request_id) + + async def lab_metrics(request: Request) -> Response: if not config.lab_mode(): return error_response(404, "not found", request.state.request_id) diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 00000000..913c4a22 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,109 @@ +"""Private bucket access. The API never redirects a client to this endpoint.""" + +from __future__ import annotations + +import asyncio +import logging + +import boto3 +from botocore.client import Config +from botocore.exceptions import BotoCoreError, ClientError + +from app import config +from app.errors import AppError + +logger = logging.getLogger("messenger.s3") +_client = None + + +def _client_or_create(): + global _client + if _client is None: + if not config.S3_ACCESS_KEY_ID or not config.S3_SECRET_ACCESS_KEY: + raise AppError(503, "service unavailable") + _client = boto3.client( + "s3", + endpoint_url=config.S3_ENDPOINT, + aws_access_key_id=config.S3_ACCESS_KEY_ID, + aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, + region_name=config.S3_REGION, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=20, + retries={"max_attempts": 2}, + ), + ) + return _client + + +def _unavailable(exc: Exception) -> AppError: + code = "" + if isinstance(exc, ClientError): + code = exc.response.get("Error", {}).get("Code", "") + logger.warning("object storage call failed code=%s", code or type(exc).__name__) + return AppError(503, "service unavailable") + + +async def head_bucket() -> None: + def _head() -> None: + _client_or_create().head_bucket(Bucket=config.S3_BUCKET) + + try: + await asyncio.to_thread(_head) + except AppError: + raise + except (BotoCoreError, ClientError) as exc: + raise _unavailable(exc) from None + + +async def put_bytes(key: str, data: bytes, content_type: str) -> None: + def _put() -> None: + _client_or_create().put_object( + Bucket=config.S3_BUCKET, + Key=key, + Body=data, + ContentType=content_type, + ContentLength=len(data), + ) + + try: + await asyncio.to_thread(_put) + except AppError: + raise + except (BotoCoreError, ClientError) as exc: + raise _unavailable(exc) from None + + +async def get_bytes(key: str) -> bytes: + def _get() -> bytes: + response = _client_or_create().get_object(Bucket=config.S3_BUCKET, Key=key) + body = response["Body"].read() + response["Body"].close() + return body + + try: + return await asyncio.to_thread(_get) + except AppError: + raise + except (BotoCoreError, ClientError) as exc: + raise _unavailable(exc) from None + + +async def delete_key(key: str) -> None: + def _delete() -> None: + try: + _client_or_create().delete_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code in {"NoSuchKey", "404", "NoSuchBucket"}: + return + raise + + try: + await asyncio.to_thread(_delete) + except AppError: + raise + except (BotoCoreError, ClientError) as exc: + raise _unavailable(exc) from None diff --git a/app/s3bootstrap.py b/app/s3bootstrap.py new file mode 100644 index 00000000..bc1290df --- /dev/null +++ b/app/s3bootstrap.py @@ -0,0 +1,48 @@ +"""Create the private bucket if it is missing. Safe to run again.""" + +import os +import sys +import time + +import boto3 +from botocore.client import Config +from botocore.exceptions import BotoCoreError, ClientError + + +def main() -> None: + endpoint = os.environ["S3_ENDPOINT"] + bucket = os.environ["S3_BUCKET"] + client = boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"], + region_name=os.environ.get("S3_REGION", "us-east-1"), + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, retries={"max_attempts": 2}), + ) + last_error = "not attempted" + for _ in range(20): + try: + client.head_bucket(Bucket=bucket) + print("bucket ready") + return + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in {"404", "NoSuchBucket", "NotFound"}: + last_error = code or "ClientError" + else: + try: + client.create_bucket(Bucket=bucket) + print("bucket created") + return + except ClientError as create_exc: + last_error = create_exc.response.get("Error", {}).get("Code", "create failed") + except BotoCoreError: + last_error = "connection failed" + time.sleep(1) + print("bucket bootstrap failed:", last_error, file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/app/store.py b/app/store.py index 04e37555..2942d3e8 100644 --- a/app/store.py +++ b/app/store.py @@ -37,12 +37,16 @@ def chat_dict(row, member_ids: List[str]) -> dict: def message_dict(row) -> dict: + raw_ids = row["attachment_ids"] if "attachment_ids" in row.keys() else [] + if raw_ids is None: + raw_ids = [] return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": format_ts(row["created_at"]), + "attachment_ids": [str(item) for item in raw_ids], } @@ -192,10 +196,10 @@ async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] return items, next_key -def _canonical_message(text: str) -> str: +def _canonical_message(text: str, attachment_ids: List[str]) -> str: # Missing attachment_ids and [] are the same message. Key order does not matter. return json.dumps( - {"attachment_ids": [], "text": text}, + {"attachment_ids": attachment_ids, "text": text}, sort_keys=True, ensure_ascii=False, separators=(",", ":"), @@ -208,13 +212,15 @@ async def add_message( text: str, idem_key: str, path: str, + attachment_ids: Optional[List[str]] = None, ) -> Tuple[Optional[dict], Optional[str]]: """Insert one message or replay the stored 201. The row and the result commit together. Same user+method+path+key is serialized with a Postgres advisory lock so two replicas cannot insert two messages. A failed transaction stores nothing. """ - digest = hashlib.sha256(_canonical_message(text).encode("utf-8")).hexdigest() + attachment_ids = list(attachment_ids or []) + digest = hashlib.sha256(_canonical_message(text, attachment_ids).encode("utf-8")).hexdigest() lock_key = f"{sender_id}|POST|{path}|{idem_key}" payload: Optional[dict] = None created_new = False @@ -262,6 +268,24 @@ async def add_message( path, idem_key, ) + resolved: List[str] = [] + if attachment_ids: + rows = await conn.fetch( + """ + SELECT id, chat_id, status + FROM attachments + WHERE id = ANY($1::uuid[]) + """, + [UUID(item) for item in attachment_ids], + ) + found = {str(row["id"]): row for row in rows} + if len(found) != len(attachment_ids) or any( + str(found[item]["chat_id"]) != chat_id for item in attachment_ids + ): + return None, "not_found" + if any(found[item]["status"] != "ready" for item in attachment_ids): + return None, "not_ready" + resolved = attachment_ids message_id = uuid4() created = datetime.now(timezone.utc) await conn.execute( @@ -275,6 +299,14 @@ async def add_message( text, created, ) + if resolved: + await conn.executemany( + """ + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES ($1, $2, $3) + """, + [(message_id, UUID(item), index) for index, item in enumerate(resolved)], + ) payload = message_dict( { "id": message_id, @@ -282,6 +314,7 @@ async def add_message( "sender_id": UUID(sender_id), "text": text, "created_at": created, + "attachment_ids": resolved, } ) await conn.execute( @@ -323,10 +356,17 @@ async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[ rows = await _run( _fetch( """ - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 - ORDER BY created_at, id + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + array_agg(ma.attachment_id ORDER BY ma.position) + FILTER (WHERE ma.attachment_id IS NOT NULL), + ARRAY[]::uuid[] + ) AS attachment_ids + FROM messages m + LEFT JOIN message_attachments ma ON ma.message_id = m.id + WHERE m.chat_id = $1 + GROUP BY m.id + ORDER BY m.created_at, m.id LIMIT $2 """, UUID(chat_id), @@ -337,10 +377,17 @@ async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[ rows = await _run( _fetch( """ - SELECT id, chat_id, sender_id, text, created_at - FROM messages - WHERE chat_id = $1 AND (created_at, id) > ($2, $3) - ORDER BY created_at, id + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE( + array_agg(ma.attachment_id ORDER BY ma.position) + FILTER (WHERE ma.attachment_id IS NOT NULL), + ARRAY[]::uuid[] + ) AS attachment_ids + FROM messages m + LEFT JOIN message_attachments ma ON ma.message_id = m.id + WHERE m.chat_id = $1 AND (m.created_at, m.id) > ($2, $3) + GROUP BY m.id + ORDER BY m.created_at, m.id LIMIT $4 """, UUID(chat_id), diff --git a/app/validation.py b/app/validation.py index 9723267b..7adb169b 100644 --- a/app/validation.py +++ b/app/validation.py @@ -80,16 +80,45 @@ def validate_idempotency_key(value: Optional[str]) -> Optional[str]: return None -def validate_message_text(value: Any) -> Optional[str]: +def validate_message_text(value: Any, *, allow_blank: bool = False) -> Optional[str]: if not isinstance(value, str): return "invalid text" - if not value.strip(): - return "text must not be blank" if len(value) > 4000: return "text too long" + if not allow_blank and not value.strip(): + return "text must not be blank" return None +def validate_filename(value: Optional[str]) -> Optional[str]: + if not isinstance(value, str) or not value.strip() or len(value) > 255: + return "invalid filename" + for char in value: + code = ord(char) + if code < 32 or code == 127 or char in "/\\": + return "invalid filename" + return None + + +def parse_attachment_ids(value: Any) -> Tuple[Optional[list], Optional[str]]: + if value is None: + return [], None + if not isinstance(value, list): + return None, "invalid attachment_ids" + if len(value) > 10: + return None, "too many attachments" + seen = set() + ids = [] + for item in value: + if validate_uuid(item): + return None, "invalid attachment id" + if item in seen: + return None, "duplicate attachment_ids" + seen.add(item) + ids.append(item) + return ids, None + + def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]: if value is None: return default, None diff --git a/compose.yaml b/compose.yaml index 8a85ffbe..e5010b2d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,5 @@ -# Port of the lab3 stand. Client -> lb:8080 -> api (scale) -> shared Postgres and Redis. +# Client -> lb:8080 -> api x2 -> Postgres + Redis + private SeaweedFS S3. +# API replicas do not publish host ports. MinIO console is not published. # API replicas do not publish host ports. Migrations run once in `migrate`. # Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead # or not-ready replica is about 2s, plus the 1s DNS hold. API on SIGTERM @@ -43,6 +44,15 @@ services: SESSION_STORE: redis DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-120} + S3_SWEEP_INTERVAL_SECONDS: ${S3_SWEEP_INTERVAL_SECONDS:-30} DB_POOL_MIN: "1" DB_POOL_MAX: ${DB_POOL_MAX:-4} DB_COMMAND_TIMEOUT: "5" @@ -55,6 +65,8 @@ services: condition: service_healthy redis: condition: service_healthy + s3-init: + condition: service_completed_successfully expose: - "8080" restart: unless-stopped @@ -104,6 +116,47 @@ services: mem_limit: ${POSTGRES_MEM:-512m} cpus: ${POSTGRES_CPUS:-1.0} + s3: + image: chrislusf/seaweedfs:3.85 + entrypoint: ["/bin/sh", "/seaweed-entrypoint.sh"] + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + volumes: + - s3_data:/data + - ./infra/s3/seaweed-entrypoint.sh:/seaweed-entrypoint.sh:ro + healthcheck: + test: + [ + "CMD-SHELL", + "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status || exit 1; wget -q -O /dev/null http://127.0.0.1:9000/ || test $$? -ne 4", + ] + interval: 3s + timeout: 5s + retries: 20 + start_period: 8s + restart: unless-stopped + mem_limit: 256m + cpus: "0.50" + + s3-init: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab4 + command: ["python", "-m", "app.s3bootstrap"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + depends_on: + s3: + condition: service_healthy + restart: "no" + redis: image: redis:7.4.2-alpine command: @@ -129,3 +182,4 @@ services: volumes: pg_data: redis_data: + s3_data: diff --git a/infra/README.md b/infra/README.md index 5a39e862..8641aeca 100644 --- a/infra/README.md +++ b/infra/README.md @@ -5,3 +5,5 @@ В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. + +Выбран SeaweedFS `chrislusf/seaweedfs:3.85` (S3 gateway на порту 9000 внутри сети Compose). Образ MinIO на Docker Hub в этом окружении недоступен. Порты наружу не опубликованы, UI filer выключен (`-filer.exposeDirectoryData=false`). `infra/s3/seaweed-entrypoint.sh` поднимает одну ноду и выдаёт API-пользователю права только на bucket `messenger-attachments`. Анонимная identity без actions. `s3-init` идемпотентно создаёт bucket теми же ограниченными ключами. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg index 24c99b80..d73e4b04 100644 --- a/infra/haproxy.cfg +++ b/infra/haproxy.cfg @@ -7,9 +7,9 @@ defaults mode http option httplog timeout connect 2s - timeout client 10s - timeout server 10s - timeout http-request 5s + timeout client 120s + timeout server 120s + timeout http-request 15s timeout check 2s timeout queue 2s retries 1 diff --git a/infra/s3/seaweed-entrypoint.sh b/infra/s3/seaweed-entrypoint.sh new file mode 100755 index 00000000..1f3a9156 --- /dev/null +++ b/infra/s3/seaweed-entrypoint.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# One SeaweedFS process: master, volume, filer, S3. The filer UI is off. +# Anonymous identity has no actions. The API user is limited to one bucket. +set -eu +: "${S3_BUCKET:?}" +: "${S3_ACCESS_KEY_ID:?}" +: "${S3_SECRET_ACCESS_KEY:?}" + +cat > /tmp/s3.json < 'ready' OR error_code IS NULL) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments (id), + position INTEGER NOT NULL CHECK (position >= 0 AND position < 10), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); + +CREATE TABLE blob_writes ( + object_key TEXT PRIMARY KEY, + committed BOOLEAN NOT NULL DEFAULT FALSE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX blob_writes_pending_idx ON blob_writes (created_at) WHERE NOT committed; diff --git a/requirements.txt b/requirements.txt index 4a028ebb..d764a41d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ uvicorn[standard]==0.32.1 asyncpg==0.30.0 redis==5.2.1 argon2-cffi==23.1.0 +boto3==1.35.99 +pillow==11.1.0 diff --git a/scripts/restart.sh b/scripts/restart.sh index 6a3be3ad..c3ad93f7 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -4,7 +4,7 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) cd "$ROOT" if [ -f .env ]; then - docker compose --env-file .env restart postgres redis api + docker compose --env-file .env restart postgres redis s3 api else - docker compose restart postgres redis api + docker compose restart postgres redis s3 api fi -- 2.54.0 From 8faf08d8eb27bcddca29682a3f06cecc3e904488 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 9 Nov 2026 00:25:42 +0300 Subject: [PATCH 3/5] =?UTF-8?q?=D0=97=D0=B0=D1=84=D0=B8=D0=BA=D1=81=D0=B8?= =?UTF-8?q?=D1=80=D0=BE=D0=B2=D0=B0=D1=82=D1=8C=20=D0=BF=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D1=80=D0=BA=D0=B8=20=D1=83=D1=80=D0=BE=D0=B2=D0=BD=D1=8F?= =?UTF-8?q?=204=20=D0=B8=20=D0=BE=D1=82=D1=87=D1=91=D1=82.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Негативные загрузки, приватный bucket, очистка orphan и пик памяти при десяти одновременных upload лежат рядом с командами, которыми это воспроизводится. --- .github/pull_request_template.md | 28 +-- Makefile | 4 + REPORT.md | 87 +++++++-- evidence/grade4-api.txt | Bin 0 -> 1660 bytes evidence/grade4-infra.json | 17 ++ scripts/grade4_checks.py | 304 +++++++++++++++++++++++++++++++ scripts/grade4_infra.py | 231 +++++++++++++++++++++++ 7 files changed, 643 insertions(+), 28 deletions(-) create mode 100644 evidence/grade4-api.txt create mode 100644 evidence/grade4-infra.json create mode 100644 scripts/grade4_checks.py create mode 100644 scripts/grade4_infra.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..878aaa81 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Булацев, БПИ2301 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: нет -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: стенд lab3 (Postgres, Redis, HAProxy, две реплики API, идемпотентность) и вложения lab4 в SeaweedFS. Метаданные в Postgres, байты в приватном bucket, ключ назначает сервер. +Команды запуска из чистого clone: `cp .env.example .env`, затем `docker compose --env-file .env up -d --build --scale api=2`. Подробности в REPORT.md. +Результаты проверок и ссылка на REPORT.md: smoke, persistence и failover прошли; негативные загрузки и orphan/память — в `evidence/`. См. REPORT.md. +Ограничения / известные ошибки: один Docker-хост; трек 5 не делался; образ MinIO на Docker Hub недоступен, используется SeaweedFS 3.85. +Источники и использование ИИ: контракт репозитория, документация SeaweedFS, boto3, Pillow. Код и прогон проверок — с помощью Cursor. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Makefile b/Makefile index 5d86d171..89192443 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,7 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +grade4: + $(PYTHON) scripts/grade4_checks.py + $(PYTHON) scripts/grade4_infra.py diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..c6216d77 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,96 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Булацев, БПИ2301 +- Целевая оценка; выбранный трек 5, если нужен: 4. Трек 5 не заявляется. +- Commit/PR: локальные коммиты после переноса lab3; PR не публиковался в этой сессии +- Использованные источники, библиотеки и помощь ИИ: контракт репозитория, документация SeaweedFS S3, boto3, Pillow. Реализация и прогон проверок сделаны с помощью Cursor. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose v2 и Python 3.9+ на хосте. Образы закреплены: `python:3.12.8-slim-bookworm`, `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`, `chrislusf/seaweedfs:3.85` (`sha256:49312939c00c01e5ee6afbd7d728b18027821d3764c35a797a72acd4fdf3296a`). + +```sh +cp .env.example .env +# заменить синтетические пароли и S3-ключи в локальном .env; файл не коммитить +docker compose --env-file .env up -d --build --scale api=2 +``` + +`migrate` применяет SQL один раз. `s3-init` создаёт bucket, если его ещё нет, и при повторе только проверяет его. Порт снаружи один: `127.0.0.1:8080` у HAProxy. Реплики API, Postgres, Redis и SeaweedFS наружу не опубликованы. + +Готовность: `GET http://127.0.0.1:8080/health/ready` отвечает 200, когда доступны Postgres, Redis и bucket. Недоступный S3 даёт 503. + +```sh +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +python -X utf8 scripts/grade4_checks.py +python -X utf8 scripts/grade4_infra.py +``` + +Остановка без удаления данных: `docker compose --env-file .env stop`. Полное удаление учебных volumes: `docker compose --env-file .env down -v`. + +На Windows `*.sh` запускались через Git Bash: оболочка не умеет стартовать shell-скрипт как Win32-процесс. Сами `scripts/restart.sh` и `scripts/stop-one.sh` не менялись под Windows. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент ходит только в HAProxy. Две реплики API без публичных портов читают общие Postgres, Redis и SeaweedFS. Из lab3 перенесены сессии в Redis с AOF, идемпотентность POST сообщения, drain по SIGTERM и миграции отдельным контейнером. В этой работе добавлены байты вложений. + +Источник истины для пользователей, чатов, сообщений и метаданных вложений — Postgres. Байты лежат в приватном bucket `messenger-attachments`, ключ назначает сервер: `originals/`. Имя файла только отображается. Публичный API не отдаёт ключ, адрес хранилища и credentials. + +`201` возвращается после PUT и COMMIT метаданных со `status=ready`. Если COMMIT не прошёл, процесс удаляет объект. Намерение пишется в `blob_writes` до PUT. Строки со `committed=false` старше `ORPHAN_GRACE_SECONDS` (120) снимает фоновый обход: свежий незаконченный upload не трогается, привязанный к сообщению объект не удаляется, потому что у него `committed=true`. Обрыв клиента до конца тела не создаёт объект: в bucket пишется уже проверенный буфер. + +Лимит 10 MiB смотрится по `Content-Length` до чтения тела и повторно по фактическому потоку. Число пикселей берётся из заголовка PNG/JPEG до `Pillow.verify()`. MIME сверяется с сигнатурой, а не с заявленным `Content-Type` в одиночку. + +`/health/ready` зависит от Postgres, Redis и `HeadBucket`. Сессии по-прежнему в Redis (`SESSION_STORE=redis`), не в Postgres. + +| Роль | Сервис | Образ | Проверка | Volume | Порт | +| --- | --- | --- | --- | --- | --- | +| api | api ×2 | сборка `mtusi-messenger:lab4` | `/health/ready` | нет | только внутри сети | +| postgres | postgres | `postgres:16.6-alpine` | `pg_isready` | `pg_data` | не опубликован | +| redis | redis | `redis:7.4.2-alpine` | `redis-cli ping` | `redis_data` | не опубликован | +| load_balancer | lb | `haproxy:2.8.14-alpine` | backend `/health/ready` | нет | `127.0.0.1:8080` | +| s3 | s3 | `chrislusf/seaweedfs:3.85` | master status и ответ порта 9000 | `s3_data` | не опубликован | + +Образ `minio/minio` в этом окружении с Docker Hub не скачивается, поэтому взято одно из разрешённых хранилищ — SeaweedFS. UI filer выключен. Анонимная identity без прав. Ключ API может читать, писать и администрировать только свой bucket; создание другого bucket отвечает `AccessDenied`. ## Доказательства критериев +Стенд: 12th Gen Intel Core i5-12450H, 16 ГБ RAM. Лимиты контейнера API: 0.50 CPU и 256 МиБ. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` | PASS, SHA-256 вложения совпал, посторонний получил 404 | вывод команды | +| Сохранность lab3 и пустой `attachment_ids` | тот же smoke и `scripts/grade4_checks.py` | текстовый POST остаётся 201, в истории `attachment_ids: []` | `evidence/grade4-api.txt` | +| Оригинал PNG и JPEG | `scripts/grade4_checks.py` | 201 ready, скачивание побайтно, без `Location` | `evidence/grade4-api.txt` | +| Опасное имя, подмена MIME, обрезка, пиксели, >10 МиБ | `scripts/grade4_checks.py` | 400 / 415 / 413, ни одного 500 | `evidence/grade4-api.txt` | +| Чужой чат и чужое вложение | `scripts/grade4_checks.py` | 404 и для метаданных, и для байтов | `evidence/grade4-api.txt` | +| Restart с volumes | `tests/resilience.py --mode persistence` через `scripts/restart.sh` | PASS: сессия, сообщение и исходный файл на месте | вывод команды | +| Остановка загрузившей реплики | `tests/resilience.py --mode failover` через `scripts/stop-one.sh` | 2 реплики до отказа; запись вернулась за 3.583 с; 105 успешных POST, 1 ошибка переходного периода; файл скачался снова | вывод команды | +| Приватный bucket вне API | `scripts/grade4_infra.py` | анонимный GET bucket — 403; порт 9000 не опубликован | `evidence/grade4-infra.json` | +| Ограниченные ключи | тот же скрипт | `create_bucket` другого имени — `AccessDenied` | `evidence/grade4-infra.json` | +| Компенсация orphan | тот же скрипт, grace 120 с | старый незакоммиченный объект удалён, свежий оставлен | `evidence/grade4-infra.json` | +| Память 10 загрузок | тот же скрипт, docker stats | 10×230778 байт, все 201, пик API 61.7 МиБ при лимите 256 | `evidence/grade4-infra.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +`Content-Length: 10485761` при коротком теле отвечает 413 до дочитывания заявленного размера. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза уровня 4: десять одновременных загрузок около 230 КиБ не поднимают RSS реплики к лимиту 256 МиБ, потому что тело ограничено 10 МиБ и не декодируется целиком до проверки пикселей. Один прогон, без прогрева отдельной фазой. Это не профиль RPS сообщений. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `scripts/restart.sh` (postgres, redis, s3, api), volumes сохранены | те же сессия, сообщение и байты | PASS публичного persistence | готовность вернулась в пределах ожидания теста (90 с) | +| `scripts/stop-one.sh` одной реплики | другая реплика принимает POST и отдаёт тот же файл | recovery 3.583 с, дублей нет, вложение совпало | остановленная реплика сама не поднималась | +| Незакоммиченный объект старше grace | объект исчезает, свежий нет | удалена 1 старая запись, свежая осталась | без потери ready-вложений | +| Анонимный S3 GET | отказ | 403 | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Один Docker-хост. Отказ машины гасит и API, и Postgres, и SeaweedFS. Несколько реплик API этого не лечат. + +Скачивание и загрузка держат в памяти процесса не больше лимита 10 МиБ плюс проверку Pillow. Это граница, а не потоковая отдача клиенту минуя буфер. Эксперимент памяти сделан на файлах ~226 КиБ, не на десяти одновременных файлах по 10 МиБ. + +Окно между PUT и COMMIT закрывается немедленным удалением при ошибке и обходом раз в 30 с. Объект, умерший внутри grace 120 с, временно занимает место. Прямой presigned upload и отдельный reconciler всего bucket не делались: это трек 5. + +Случайный ключ и закрытый bucket не заменяют проверку участника чата: скачивание идёт через API и для постороннего даёт 404, даже если он угадает UUID. diff --git a/evidence/grade4-api.txt b/evidence/grade4-api.txt new file mode 100644 index 0000000000000000000000000000000000000000..68a4692681fd4e5ea339ed9f01deba6f88a0951d GIT binary patch literal 1660 zcmb7_Z%)EM5XI-SiFeow7(wv>`rX9D6O{6=(o)hEh0CkIw+lhawy7b;?K1Dp%$qmm z=lji`?VXRa%I3DT%xY^mRoo<&^PKYJ?7>Dhwp+WgGa|RrBQ^tKui2-drvsS#;;vce z_Bp8DzPQwDk$4>({jX4>>8SKFwj^$KRIeW=jrX3l;8gQT(6USEt!?<$+0M4iq$GCM zQIFUPbPW@n5&H%95;JPe@dH-PehB{pN906rs6n|L{F(3C(L>H{jRMXl_@$kv51Ufy z2A+!lIryMn1z*Z-=qAQ_Qf6ugA6J8&5uf^6<~(Opa>kZsuo@aPvR> zJ<|(53cAJdn(~^zup3gd@mb|LRUpNsnH&(yf7xN?<4a@6_hF;8muvX-JMs%0 Cqyvip literal 0 HcmV?d00001 diff --git a/evidence/grade4-infra.json b/evidence/grade4-infra.json new file mode 100644 index 00000000..5537af22 --- /dev/null +++ b/evidence/grade4-infra.json @@ -0,0 +1,17 @@ +{ + "anonymous_bucket_status": "403", + "second_bucket": "AccessDenied", + "sweep": { + "deleted": 1, + "old_removed": true, + "fresh_kept": true + }, + "s3_port_published": false, + "memory": { + "uploads": 10, + "bytes_each": 230778, + "success": 10, + "peak_api_mib": 61.7, + "samples": 8 + } +} diff --git a/scripts/grade4_checks.py b/scripts/grade4_checks.py new file mode 100644 index 00000000..5789b101 --- /dev/null +++ b/scripts/grade4_checks.py @@ -0,0 +1,304 @@ +#!/usr/bin/env python3 +"""Negative attachment cases. Prints statuses only, never tokens.""" + +import base64 +import hashlib +import json +import socket +import struct +import uuid +import urllib.error +import urllib.request +import zlib +from pathlib import Path + +BASE = "http://127.0.0.1:8080" +ROOT = Path(__file__).resolve().parents[1] +JPEG = base64.b64decode( + "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5soooroMD/9k=" +) + + +def header_only_png(width: int, height: int) -> bytes: + signature = b"\x89PNG\r\n\x1a\n" + + def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return signature + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"") + + +def png(width: int, height: int) -> bytes: + signature = b"\x89PNG\r\n\x1a\n" + + def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + raw = b"".join(b"\x00" + (b"\x10\x20\x30" * width) for _ in range(height)) + return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +class Api: + def __init__(self) -> None: + self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def call(self, method, path, body=None, token=None, headers=None, raw=None, content_type=None): + hdrs = {} + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + hdrs["Content-Type"] = "application/json" + if content_type: + hdrs["Content-Type"] = content_type + if token: + hdrs["Authorization"] = "Bearer " + token + if headers: + hdrs.update(headers) + request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method) + try: + response = self.opener.open(request, timeout=30) + except urllib.error.HTTPError as error: + response = error + with response: + payload = response.read() + parsed = None + if payload and response.headers.get_content_type() == "application/json": + parsed = json.loads(payload) + return response.status, response.headers, parsed, payload + + +def expect(status, got, label): + if got == 500: + raise SystemExit("FAIL %s returned 500" % label) + if got != status: + raise SystemExit("FAIL %s expected %s got %s" % (label, status, got)) + print("PASS", label, got) + + +def user(api: Api, label: str): + username = "g4_" + label + "_" + uuid.uuid4().hex[:10] + password = "T3st-" + uuid.uuid4().hex + status, _, body, _ = api.call( + "POST", + "/api/v1/users", + {"username": username, "display_name": "Grade " + label, "password": password}, + ) + expect(201, status, "register " + label) + basic = base64.b64encode((username + ":" + password).encode()).decode() + status, _, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + expect(201, status, "login " + label) + return body["id"], session["token"] + + +def declared_length(token, chat_id): + request = ( + "POST /api/v1/chats/%s/attachments?filename=big.png HTTP/1.1\r\n" + "Host: 127.0.0.1:8080\r\n" + "Authorization: Bearer %s\r\n" + "Content-Type: image/png\r\n" + "Content-Length: 10485761\r\n" + "Connection: close\r\n\r\n" % (chat_id, token) + ).encode() + b"\x89PNG" + sock = socket.create_connection(("127.0.0.1", 8080), timeout=5) + try: + sock.sendall(request) + data = b"" + while b"\r\n" not in data: + chunk = sock.recv(256) + if not chunk: + break + data += chunk + finally: + sock.close() + line = data.split(b"\r\n", 1)[0].decode("latin1", "replace") + code = int(line.split()[1]) + expect(413, code, "content-length above 10MiB before body is read") + + +def main(): + api = Api() + alice, alice_token = user(api, "alice") + bob, bob_token = user(api, "bob") + eve, eve_token = user(api, "eve") + status, _, chat, _ = api.call( + "POST", + "/api/v1/chats", + {"title": "grade4", "member_ids": [alice, bob]}, + token=alice_token, + ) + expect(201, status, "create chat") + chat_id = chat["id"] + status, _, other, _ = api.call( + "POST", + "/api/v1/chats", + {"title": "other", "member_ids": [alice, eve]}, + token=alice_token, + ) + expect(201, status, "create other chat") + + status, _, message, _ = api.call( + "POST", + "/api/v1/chats/%s/messages" % chat_id, + {"text": "только текст"}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + expect(201, status, "text message") + if message["attachment_ids"] != []: + raise SystemExit("FAIL text message lost empty attachment_ids") + + good = png(32, 32) + status, _, attachment, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=ok.png" % chat_id, + token=alice_token, + raw=good, + content_type="image/png", + ) + expect(201, status, "png upload") + if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "ready": + raise SystemExit("FAIL png metadata") + status, headers, _, downloaded = api.call( + "GET", + "/api/v1/attachments/%s/content" % attachment["id"], + token=bob_token, + ) + expect(200, status, "member download") + if downloaded != good or headers.get_content_type() != "image/png": + raise SystemExit("FAIL download bytes") + if headers.get("Location"): + raise SystemExit("FAIL download redirected") + + status, _, jpeg_meta, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=ok.jpg" % chat_id, + token=alice_token, + raw=JPEG, + content_type="image/jpeg", + ) + expect(201, status, "jpeg upload") + if jpeg_meta["content_type"] != "image/jpeg" or jpeg_meta["sha256"] != hashlib.sha256(JPEG).hexdigest(): + raise SystemExit("FAIL jpeg metadata") + + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=a/b.png" % chat_id, + token=alice_token, + raw=good, + content_type="image/png", + ) + expect(400, status, "filename with slash") + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s" % (chat_id, "bad%0Aname.png"), + token=alice_token, + raw=good, + content_type="image/png", + ) + expect(400, status, "filename with control character") + + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=lie.png" % chat_id, + token=alice_token, + raw=good, + content_type="image/jpeg", + ) + expect(415, status, "png bytes declared as jpeg") + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=cut.png" % chat_id, + token=alice_token, + raw=b"\x89PNG\r\n\x1a\ntruncated", + content_type="image/png", + ) + expect(415, status, "truncated png") + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=cut.jpg" % chat_id, + token=alice_token, + raw=JPEG[:-40], + content_type="image/jpeg", + ) + expect(415, status, "truncated jpeg") + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=huge.png" % chat_id, + token=alice_token, + raw=header_only_png(10000, 5000), + content_type="image/png", + ) + expect(413, status, "pixel limit") + declared_length(alice_token, chat_id) + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=big.png" % chat_id, + token=alice_token, + raw=b"\x89PNG" + b"\x00" * (10_485_760), + content_type="image/png", + ) + expect(413, status, "body above 10MiB") + + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=nope.png" % chat_id, + token=eve_token, + raw=good, + content_type="image/png", + ) + expect(404, status, "outsider upload") + for suffix in ("", "/content"): + status, _, _, _ = api.call("GET", "/api/v1/attachments/%s%s" % (attachment["id"], suffix), token=eve_token) + expect(404, status, "outsider get " + (suffix or "metadata")) + + missing = str(uuid.uuid4()) + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/messages" % chat_id, + {"text": "", "attachment_ids": [missing]}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + expect(404, status, "unknown attachment id") + status, _, foreign, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=other.png" % other["id"], + token=alice_token, + raw=good, + content_type="image/png", + ) + expect(201, status, "upload into other chat") + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/messages" % chat_id, + {"text": "", "attachment_ids": [foreign["id"]]}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + expect(404, status, "attachment from another chat") + ids = [str(uuid.uuid4()) for _ in range(11)] + status, _, _, _ = api.call( + "POST", + "/api/v1/chats/%s/messages" % chat_id, + {"text": "x", "attachment_ids": ids}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + expect(400, status, "more than 10 attachment ids") + status, _, linked, _ = api.call( + "POST", + "/api/v1/chats/%s/messages" % chat_id, + {"text": "", "attachment_ids": [attachment["id"]]}, + token=bob_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + expect(201, status, "message with attachment") + if linked["attachment_ids"] != [attachment["id"]] or linked["text"] != "": + raise SystemExit("FAIL linked message") + print("PASS grade4 api checks") + + +if __name__ == "__main__": + main() diff --git a/scripts/grade4_infra.py b/scripts/grade4_infra.py new file mode 100644 index 00000000..2c403a34 --- /dev/null +++ b/scripts/grade4_infra.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Private bucket, orphan grace, and a bounded concurrent-upload memory sample.""" + +import json +import subprocess +import threading +import time +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" / "grade4-infra.json" + + +def compose(*args, input_text=None): + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + input=input_text, + text=True, + capture_output=True, + ) + if result.returncode != 0: + raise SystemExit(result.stderr[-500:] or result.stdout[-500:] or "compose failed") + return result.stdout + + +def anonymous_status(): + script = r""" +code=$(wget -S -O /dev/null http://127.0.0.1:9000/messenger-attachments/ 2>&1 | awk 'BEGIN{c=""} /^[[:space:]]*HTTP\//{c=$2} END{print c}') +printf '%s' "$code" +""" + output = compose("exec", "-T", "s3", "sh", "-c", script).strip() + if output != "403": + raise SystemExit("anonymous bucket status %s" % output) + return output + + +def other_bucket_denied(): + code = r""" +import os +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError +client = boto3.client( + "s3", + endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["S3_SECRET_ACCESS_KEY"], + region_name=os.environ.get("S3_REGION", "us-east-1"), + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), +) +try: + client.create_bucket(Bucket="not-the-messenger-bucket") + print("allowed") +except ClientError as exc: + print(exc.response.get("Error", {}).get("Code", "denied")) +""" + output = compose("exec", "-T", "api", "python", "-c", code).strip() + if output == "allowed": + raise SystemExit("app credentials created a second bucket") + return output + + +def plant_and_sweep(): + code = r""" +import asyncio +from datetime import datetime, timedelta, timezone +from app import config, db, s3 + +async def main(): + await db.connect() + old_key = "originals/orphan-evidence" + fresh_key = "originals/fresh-evidence" + payload = b"not-a-real-image" + await s3.put_bytes(old_key, payload, "application/octet-stream") + await s3.put_bytes(fresh_key, payload, "application/octet-stream") + old = datetime.now(timezone.utc) - timedelta(seconds=config.ORPHAN_GRACE_SECONDS + 60) + fresh = datetime.now(timezone.utc) + async with db.require().acquire() as conn: + await conn.execute("DELETE FROM blob_writes WHERE object_key = ANY($1::text[])", [old_key, fresh_key]) + await conn.execute( + "INSERT INTO blob_writes (object_key, committed, created_at) VALUES ($1, FALSE, $2), ($3, FALSE, $4)", + old_key, old, fresh_key, fresh, + ) + from app.attachments import sweep_orphans + deleted = await sweep_orphans() + async def exists(key): + try: + await s3.get_bytes(key) + return True + except Exception: + return False + print("%s %s %s" % (deleted, int(await exists(old_key)), int(await exists(fresh_key)))) + await s3.delete_key(fresh_key) + async with db.require().acquire() as conn: + await conn.execute("DELETE FROM blob_writes WHERE object_key = $1 AND NOT committed", fresh_key) + await db.close() + +asyncio.run(main()) +""" + output = compose("exec", "-T", "api", "python", "-c", code).strip().split() + deleted, old_exists, fresh_exists = int(output[0]), int(output[1]), int(output[2]) + if deleted < 1 or old_exists != 0 or fresh_exists != 1: + raise SystemExit("sweep result deleted=%s old=%s fresh=%s" % (deleted, old_exists, fresh_exists)) + return {"deleted": deleted, "old_removed": True, "fresh_kept": True} + + +def memory_sample(): + import base64 + + username = "mem_" + uuid.uuid4().hex[:10] + password = "T3st-" + uuid.uuid4().hex + body = json.dumps({"username": username, "display_name": "Mem", "password": password}).encode() + + def call(method, path, data=None, headers=None): + request = urllib.request.Request("http://127.0.0.1:8080" + path, data=data, headers=headers or {}, method=method) + try: + response = urllib.request.urlopen(request, timeout=60) + except urllib.error.HTTPError as error: + response = error + with response: + return response.status, response.read() + + status, raw = call("POST", "/api/v1/users", body, {"Content-Type": "application/json"}) + if status != 201: + raise SystemExit("memory user %s" % status) + user = json.loads(raw) + basic = base64.b64encode((username + ":" + password).encode()).decode() + status, raw = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + if status != 201: + raise SystemExit("memory login %s" % status) + token = json.loads(raw)["token"] + status, raw = call( + "POST", + "/api/v1/chats", + json.dumps({"title": "mem", "member_ids": [user["id"]]}).encode(), + {"Content-Type": "application/json", "Authorization": "Bearer " + token}, + ) + if status != 201: + raise SystemExit("memory chat %s" % status) + chat_id = json.loads(raw)["id"] + import os + import struct + import zlib + + def png(): + signature = b"\x89PNG\r\n\x1a\n" + + def chunk(kind, data): + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + width, height = 320, 240 + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + raw_rows = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw_rows, 1)) + chunk(b"IEND", b"") + + blob = png() + codes = [] + + def upload(): + status, _ = call( + "POST", + "/api/v1/chats/%s/attachments?filename=m.png" % chat_id, + blob, + {"Content-Type": "image/png", "Authorization": "Bearer " + token}, + ) + codes.append(status) + + samples = [] + + def watch(): + end = time.monotonic() + 8 + while time.monotonic() < end: + stats = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, + text=True, + check=False, + ) + for line in stats.stdout.splitlines(): + if "mtusi2026lab4-api-" in line: + samples.append(line.strip()) + time.sleep(0.4) + + watcher = threading.Thread(target=watch) + watcher.start() + threads = [threading.Thread(target=upload) for _ in range(10)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + watcher.join() + if codes.count(201) != 10: + raise SystemExit("concurrent uploads %s" % codes) + + def mib(line): + value = line.split()[1] + number = float(value[:-3]) if value.lower().endswith("mib") else float(value[:-3]) / 1024 + return number + + peak = max(mib(line) for line in samples) if samples else 0 + return {"uploads": 10, "bytes_each": len(blob), "success": 10, "peak_api_mib": round(peak, 1), "samples": len(samples)} + + +def main(): + published = subprocess.run( + ["docker", "compose", "--env-file", ".env", "port", "s3", "9000"], + cwd=ROOT, + capture_output=True, + text=True, + ) + if published.returncode == 0 and published.stdout.strip(): + raise SystemExit("s3 port is published: " + published.stdout.strip()) + summary = { + "anonymous_bucket_status": anonymous_status(), + "second_bucket": other_bucket_denied(), + "sweep": plant_and_sweep(), + "s3_port_published": False, + "memory": memory_sample(), + } + OUT.parent.mkdir(exist_ok=True) + OUT.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + import urllib.error + + main() -- 2.54.0 From 4654de5837e69760370d185acc4611feac01f5f1 Mon Sep 17 00:00:00 2001 From: David Date: Mon, 9 Nov 2026 00:36:52 +0300 Subject: [PATCH 4/5] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BF=D1=80=D1=8F=D0=BC=D1=83=D1=8E=20=D0=B7=D0=B0?= =?UTF-8?q?=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D1=83=20=D0=BF=D0=BE=20=D0=BA?= =?UTF-8?q?=D0=BE=D1=80=D0=BE=D1=82=D0=BA=D0=BE=D0=B9=20=D0=BF=D0=BE=D0=B4?= =?UTF-8?q?=D0=BF=D0=B8=D1=81=D0=B8.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Proxy endpoint сохранён. Complete сверяет владельца, размер и checksum, а сравнение трафика API на двух размерах лежит в evidence. --- .env.example | 3 + .github/pull_request_template.md | 8 +- Makefile | 3 + REPORT.md | 11 +- app/attachments.py | 8 + app/config.py | 2 + app/direct.py | 199 ++++++++++++++++ app/main.py | 4 + app/routes.py | 54 ++++- app/s3.py | 54 +++++ compose.yaml | 4 + contracts/extensions.openapi.json | 21 ++ evidence/track1.json | 26 ++ examples/requests.http | 15 ++ infra/README.md | 2 +- migrations/004_direct_uploads.sql | 22 ++ scripts/grade4_infra.py | 7 +- scripts/track1_experiment.py | 380 ++++++++++++++++++++++++++++++ 18 files changed, 812 insertions(+), 11 deletions(-) create mode 100644 app/direct.py create mode 100644 contracts/extensions.openapi.json create mode 100644 evidence/track1.json create mode 100644 migrations/004_direct_uploads.sql create mode 100644 scripts/track1_experiment.py diff --git a/.env.example b/.env.example index 955c92df..30e53cfb 100644 --- a/.env.example +++ b/.env.example @@ -32,3 +32,6 @@ S3_SECRET_ACCESS_KEY=lab4-dev-only-change-me MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 ORPHAN_GRACE_SECONDS=120 +# Host the client uses for a presigned PUT. Loopback only; the API itself uses S3_ENDPOINT. +S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000 +PRESIGN_TTL_SECONDS=60 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 878aaa81..d9f72c6b 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,13 +1,13 @@ ## Лабораторная 4 Автор / группа: Булацев, БПИ2301 -Целевая оценка (3/4/5): 4 -Трек 5, если заявлен: нет +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 1, прямой upload -Что реализовано и что перенесено: стенд lab3 (Postgres, Redis, HAProxy, две реплики API, идемпотентность) и вложения lab4 в SeaweedFS. Метаданные в Postgres, байты в приватном bucket, ключ назначает сервер. +Что реализовано и что перенесено: стенд lab3, вложения в SeaweedFS и отдельный presigned upload. Proxy `POST /attachments` сохранён. Метаданные в Postgres, байты в приватном bucket. Команды запуска из чистого clone: `cp .env.example .env`, затем `docker compose --env-file .env up -d --build --scale api=2`. Подробности в REPORT.md. Результаты проверок и ссылка на REPORT.md: smoke, persistence и failover прошли; негативные загрузки и orphan/память — в `evidence/`. См. REPORT.md. -Ограничения / известные ошибки: один Docker-хост; трек 5 не делался; образ MinIO на Docker Hub недоступен, используется SeaweedFS 3.85. +Ограничения / известные ошибки: один Docker-хост; complete прямой загрузки читает объект обратно в API; образ MinIO на Docker Hub недоступен, используется SeaweedFS 3.85. S3 открыт только на 127.0.0.1:9000. Источники и использование ИИ: контракт репозитория, документация SeaweedFS, boto3, Pillow. Код и прогон проверок — с помощью Cursor. - [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. diff --git a/Makefile b/Makefile index 89192443..d709820b 100644 --- a/Makefile +++ b/Makefile @@ -32,3 +32,6 @@ test-failover: grade4: $(PYTHON) scripts/grade4_checks.py $(PYTHON) scripts/grade4_infra.py + +track1: + $(PYTHON) scripts/track1_experiment.py diff --git a/REPORT.md b/REPORT.md index c6216d77..6c48267a 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Булацев, БПИ2301 -- Целевая оценка; выбранный трек 5, если нужен: 4. Трек 5 не заявляется. +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 «Прямой upload». - Commit/PR: локальные коммиты после переноса lab3; PR не публиковался в этой сессии - Использованные источники, библиотеки и помощь ИИ: контракт репозитория, документация SeaweedFS S3, boto3, Pillow. Реализация и прогон проверок сделаны с помощью Cursor. @@ -26,6 +26,7 @@ python -X utf8 tests/resilience.py --mode persistence --action-script scripts/re python -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 python -X utf8 scripts/grade4_checks.py python -X utf8 scripts/grade4_infra.py +python -X utf8 scripts/track1_experiment.py ``` Остановка без удаления данных: `docker compose --env-file .env stop`. Полное удаление учебных volumes: `docker compose --env-file .env down -v`. @@ -44,6 +45,8 @@ python -X utf8 scripts/grade4_infra.py `/health/ready` зависит от Postgres, Redis и `HeadBucket`. Сессии по-прежнему в Redis (`SESSION_STORE=redis`), не в Postgres. +Прямая загрузка не заменяет `POST /attachments`. `POST /chats/{id}/uploads` выдаёт PUT-подпись на 60 секунд и ключ `originals/`. В JSON ключа нет, он есть только внутри подписанного URL. `POST /uploads/{id}/complete` принимает загрузку, только если её начал тот же пользователь, подпись не истекла, размер совпал с заявленным, sha256 совпал с байтами в bucket, а содержимое — целый PNG или JPEG. Чужой complete — 404. Повторный complete, complete без PUT и просроченная подпись — 409. Неверная сумма удаляет объект и больше не даёт дозавершить загрузку. Незаконченный ключ снимает тот же sweeper, что и orphan после grace. S3 с хоста доступен только на `127.0.0.1:9000`, чтобы клиент мог сделать PUT мимо API; анонимный запрос по-прежнему 403. + | Роль | Сервис | Образ | Проверка | Volume | Порт | | --- | --- | --- | --- | --- | --- | | api | api ×2 | сборка `mtusi-messenger:lab4` | `/health/ready` | нет | только внутри сети | @@ -71,6 +74,8 @@ python -X utf8 scripts/grade4_infra.py | Ограниченные ключи | тот же скрипт | `create_bucket` другого имени — `AccessDenied` | `evidence/grade4-infra.json` | | Компенсация orphan | тот же скрипт, grace 120 с | старый незакоммиченный объект удалён, свежий оставлен | `evidence/grade4-infra.json` | | Память 10 загрузок | тот же скрипт, docker stats | 10×230778 байт, все 201, пик API 61.7 МиБ при лимите 256 | `evidence/grade4-infra.json` | +| Трек 1, негативные сценарии | `scripts/track1_experiment.py` | подделанная подпись 403; complete без PUT, чужой пользователь, повтор и просрочка — 404/409; неверная сумма 400 и объект больше не завершается; proxy endpoint по-прежнему 201 | `evidence/track1.json` | +| Трек 1, трафик API | тот же скрипт, сумма RX `eth0` обеих реплик | файл 90243 байт: proxy RX 96204, прямой PUT RX 0, complete RX 94131. Файл 1891338 байт: proxy RX 1913606, прямой PUT RX 960, complete RX 1900090 | `evidence/track1.json` | `Content-Length: 10485761` при коротком теле отвечает 413 до дочитывания заявленного размера. @@ -78,6 +83,8 @@ python -X utf8 scripts/grade4_infra.py Гипотеза уровня 4: десять одновременных загрузок около 230 КиБ не поднимают RSS реплики к лимиту 256 МиБ, потому что тело ограничено 10 МиБ и не декодируется целиком до проверки пикселей. Один прогон, без прогрева отдельной фазой. Это не профиль RPS сообщений. +Гипотеза трека 1: прямой PUT не заходит в API, поэтому RX API почти не зависит от размера файла, а proxy RX растёт вместе с телом. Complete один раз скачивает объект, чтобы сверить sha256, и этот трафик остаётся. Пик RSS по `docker stats` на файлах 90 КиБ и 1.9 МиБ остался около 61–62.5 МиБ: базовая память процесса больше короткого буфера, и по RSS выигрыш на этих размерах не виден. Выигрыш, который измерен, — это RX на фазе PUT. + | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | | `scripts/restart.sh` (postgres, redis, s3, api), volumes сохранены | те же сессия, сообщение и байты | PASS публичного persistence | готовность вернулась в пределах ожидания теста (90 с) | @@ -91,6 +98,6 @@ python -X utf8 scripts/grade4_infra.py Скачивание и загрузка держат в памяти процесса не больше лимита 10 МиБ плюс проверку Pillow. Это граница, а не потоковая отдача клиенту минуя буфер. Эксперимент памяти сделан на файлах ~226 КиБ, не на десяти одновременных файлах по 10 МиБ. -Окно между PUT и COMMIT закрывается немедленным удалением при ошибке и обходом раз в 30 с. Объект, умерший внутри grace 120 с, временно занимает место. Прямой presigned upload и отдельный reconciler всего bucket не делались: это трек 5. +Окно между PUT и COMMIT закрывается немедленным удалением при ошибке и обходом раз в 30 с. Объект, умерший внутри grace 120 с, временно занимает место. Прямой upload добавлен отдельными маршрутами и не отменяет proxy. Complete всё равно читает объект в API, поэтому экономия трафика относится к фазе PUT, а не ко всей операции. Отдельный reconciler всего bucket не делался. Случайный ключ и закрытый bucket не заменяют проверку участника чата: скачивание идёт через API и для постороннего даёт 404, даже если он угадает UUID. diff --git a/app/attachments.py b/app/attachments.py index 9caaf439..ca1e726f 100644 --- a/app/attachments.py +++ b/app/attachments.py @@ -178,6 +178,14 @@ async def sweep_orphans() -> int: continue await s3.delete_key(key) await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", key) + await conn.execute( + """ + UPDATE direct_uploads + SET status = 'aborted' + WHERE object_key = $1 AND status = 'pending' + """, + key, + ) deleted += 1 if deleted: logger.info("removed %s uncommitted objects past grace", deleted) diff --git a/app/config.py b/app/config.py index 005c455e..ee6d6b0b 100644 --- a/app/config.py +++ b/app/config.py @@ -37,6 +37,8 @@ LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2")) # container uses its own hostname; stop-one.sh maps that value to one replica. INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname()) S3_ENDPOINT = _env("S3_ENDPOINT", "http://s3:9000") +S3_PUBLIC_ENDPOINT = _env("S3_PUBLIC_ENDPOINT", "http://127.0.0.1:9000") +PRESIGN_TTL_SECONDS = min(max(int(_env("PRESIGN_TTL_SECONDS", "60")), 15), 300) S3_REGION = _env("S3_REGION", "us-east-1") S3_BUCKET = _env("S3_BUCKET", "messenger-attachments") S3_ACCESS_KEY_ID = _file_or_env("S3_ACCESS_KEY_ID") diff --git a/app/direct.py b/app/direct.py new file mode 100644 index 00000000..78caa96b --- /dev/null +++ b/app/direct.py @@ -0,0 +1,199 @@ +"""Presigned PUT that does not replace the proxy upload endpoint.""" + +from __future__ import annotations + +import asyncio +import hashlib +from datetime import datetime, timedelta, timezone +from uuid import UUID, uuid4 + +from app import config, db, images, s3 +from app.attachments import _compensate, _public +from app.errors import AppError +from app.store import format_ts + + +def _aware(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value + + +async def begin_upload( + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + size_bytes: int, +) -> dict: + upload_id = uuid4() + key = f"originals/{upload_id}" + created = datetime.now(timezone.utc) + expires_at = created + timedelta(seconds=config.PRESIGN_TTL_SECONDS) + async with db.require().acquire() as conn: + async with conn.transaction(): + await conn.execute( + """ + INSERT INTO blob_writes (object_key, committed, created_at) + VALUES ($1, FALSE, $2) + """, + key, + created, + ) + await conn.execute( + """ + INSERT INTO direct_uploads ( + id, chat_id, owner_id, filename, content_type, size_bytes, + object_key, status, expires_at, created_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, 'pending', $8, $9) + """, + upload_id, + UUID(chat_id), + UUID(owner_id), + filename, + content_type, + size_bytes, + key, + expires_at, + created, + ) + try: + url = await asyncio.to_thread(s3.presign_put, key, content_type, size_bytes) + except Exception: + await _compensate(key) + async with db.require().acquire() as conn: + await conn.execute( + "UPDATE direct_uploads SET status = 'aborted' WHERE id = $1 AND status = 'pending'", + upload_id, + ) + raise + return { + "upload_id": str(upload_id), + "method": "PUT", + "url": url, + "headers": {"Content-Type": content_type}, + "expires_at": format_ts(expires_at), + "size_bytes": size_bytes, + } + + +async def _abort_pending(upload_id: UUID, key: str) -> None: + async with db.require().acquire() as conn: + changed = await conn.fetchval( + """ + UPDATE direct_uploads + SET status = 'aborted' + WHERE id = $1 AND status = 'pending' + RETURNING id + """, + upload_id, + ) + if changed is not None: + await _compensate(key) + + +async def complete_upload(upload_id: str, owner_id: str, claimed_sha: str) -> dict: + async with db.require().acquire() as conn: + row = await conn.fetchrow( + """ + SELECT id, chat_id, owner_id, filename, content_type, size_bytes, + object_key, status, expires_at + FROM direct_uploads + WHERE id = $1 AND owner_id = $2 + """, + UUID(upload_id), + UUID(owner_id), + ) + if row is None: + raise AppError(404, "upload not found") + if row["status"] != "pending": + raise AppError(409, "upload is already finished") + if _aware(row["expires_at"]) <= datetime.now(timezone.utc): + await _abort_pending(row["id"], row["object_key"]) + raise AppError(409, "upload signature expired") + + actual_size = await s3.head_size(row["object_key"]) + if actual_size is None: + raise AppError(409, "upload is incomplete") + if actual_size != row["size_bytes"] or actual_size > config.MAX_UPLOAD_BYTES: + await _abort_pending(row["id"], row["object_key"]) + raise AppError(400, "uploaded size does not match") + + data = await s3.get_bytes(row["object_key"]) + digest = hashlib.sha256(data).hexdigest() + if len(data) != row["size_bytes"] or digest != claimed_sha: + await _abort_pending(row["id"], row["object_key"]) + raise AppError(400, "checksum does not match") + try: + images.inspect_image(data, row["content_type"]) + except images.ImageRejected as exc: + await _abort_pending(row["id"], row["object_key"]) + raise AppError(exc.status, exc.message) from None + + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + locked = await conn.fetchrow( + """ + SELECT status, object_key + FROM direct_uploads + WHERE id = $1 AND owner_id = $2 + FOR UPDATE + """, + row["id"], + UUID(owner_id), + ) + if locked is None or locked["status"] != "pending": + raise AppError(409, "upload is already finished") + intent = await conn.fetchrow( + "SELECT committed FROM blob_writes WHERE object_key = $1 FOR UPDATE", + row["object_key"], + ) + if intent is None or intent["committed"]: + raise AppError(409, "upload is already finished") + await conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, status, error_code, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ready', NULL, $9, $9) + """, + row["id"], + row["chat_id"], + row["owner_id"], + row["filename"], + row["content_type"], + len(data), + digest, + row["object_key"], + created, + ) + await conn.execute( + "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", + row["object_key"], + ) + await conn.execute( + "UPDATE direct_uploads SET status = 'completed' WHERE id = $1", + row["id"], + ) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + return _public( + { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": len(data), + "sha256": digest, + "status": "ready", + "created_at": created, + "updated_at": created, + } + ) diff --git a/app/main.py b/app/main.py index b4ef0743..2c56af8e 100644 --- a/app/main.py +++ b/app/main.py @@ -17,6 +17,8 @@ from app.docs import openapi_json, swagger_ui from app.errors import AppError, error_response, from_app_error from app.middleware import RequestContextMiddleware from app.routes import ( + begin_direct_upload, + complete_direct_upload, create_chat, create_message, create_session, @@ -93,6 +95,8 @@ def build_app() -> Starlette: Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]), + Route("/api/v1/chats/{chat_id}/uploads", begin_direct_upload, methods=["POST"]), + Route("/api/v1/uploads/{upload_id}/complete", complete_direct_upload, methods=["POST"]), Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]), Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]), Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]), diff --git a/app/routes.py b/app/routes.py index bb6d1292..a536aeb0 100644 --- a/app/routes.py +++ b/app/routes.py @@ -1,10 +1,11 @@ import base64 import binascii +import re from starlette.requests import Request from starlette.responses import JSONResponse, Response -from app import attachments, cache, config, db, images, rediscli, s3, sessions, store +from app import attachments, cache, config, db, direct, images, rediscli, s3, sessions, store from app.cursor_codec import decode_cursor, encode_cursor from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response from app.passwords import dummy_hash, hash_password, verify_password @@ -311,6 +312,57 @@ async def upload_attachment(request: Request) -> Response: return _json(created, rid, 201) +async def begin_direct_upload(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"filename", "content_type", "size_bytes"}, {"filename", "content_type", "size_bytes"}): + return error_response(400, "invalid upload payload", rid) + if validate_filename(body["filename"]): + return error_response(400, "invalid filename", rid) + content_type = body["content_type"] + if content_type not in ("image/png", "image/jpeg"): + return error_response(415, "unsupported media type", rid) + size = body["size_bytes"] + if isinstance(size, bool) or not isinstance(size, int): + return error_response(400, "invalid size", rid) + if size > config.MAX_UPLOAD_BYTES: + return error_response(413, "file exceeds upload limit", rid) + if size < 1: + return error_response(400, "invalid size", rid) + if await store.load_chat_for_member(chat_id, user["id"]) is None: + return error_response(404, "chat not found", rid) + started = await direct.begin_upload(chat_id, user["id"], body["filename"], content_type, size) + return _json(started, rid, 201) + + +async def complete_direct_upload(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + upload_id = request.path_params["upload_id"] + if validate_uuid(upload_id): + return error_response(400, "invalid upload id", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"sha256"}, {"sha256"}): + return error_response(400, "invalid upload payload", rid) + digest = body["sha256"] + if not isinstance(digest, str) or re.fullmatch(r"[a-f0-9]{64}", digest) is None: + return error_response(400, "invalid checksum", rid) + created = await direct.complete_upload(upload_id, user["id"], digest) + return _json(created, rid, 201) + + async def get_attachment(request: Request) -> Response: rid = request.state.request_id user = await _bearer_user(request) diff --git a/app/s3.py b/app/s3.py index 913c4a22..f887b46d 100644 --- a/app/s3.py +++ b/app/s3.py @@ -14,6 +14,7 @@ from app.errors import AppError logger = logging.getLogger("messenger.s3") _client = None +_signer = None def _client_or_create(): @@ -38,6 +39,26 @@ def _client_or_create(): return _client +def _signer_or_create(): + """Signs URLs for the host the browser/client can reach. It does not send traffic.""" + global _signer + if _signer is None: + if not config.S3_ACCESS_KEY_ID or not config.S3_SECRET_ACCESS_KEY: + raise AppError(503, "service unavailable") + _signer = boto3.client( + "s3", + endpoint_url=config.S3_PUBLIC_ENDPOINT, + aws_access_key_id=config.S3_ACCESS_KEY_ID, + aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, + region_name=config.S3_REGION, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + ), + ) + return _signer + + def _unavailable(exc: Exception) -> AppError: code = "" if isinstance(exc, ClientError): @@ -91,6 +112,39 @@ async def get_bytes(key: str) -> bytes: raise _unavailable(exc) from None +def presign_put(key: str, content_type: str, size: int) -> str: + return _signer_or_create().generate_presigned_url( + "put_object", + Params={ + "Bucket": config.S3_BUCKET, + "Key": key, + "ContentType": content_type, + "ContentLength": size, + }, + ExpiresIn=config.PRESIGN_TTL_SECONDS, + HttpMethod="PUT", + ) + + +async def head_size(key: str) -> int | None: + def _head() -> int | None: + try: + response = _client_or_create().head_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code in {"404", "NoSuchKey", "NotFound"}: + return None + raise + return int(response["ContentLength"]) + + try: + return await asyncio.to_thread(_head) + except AppError: + raise + except (BotoCoreError, ClientError) as exc: + raise _unavailable(exc) from None + + async def delete_key(key: str) -> None: def _delete() -> None: try: diff --git a/compose.yaml b/compose.yaml index e5010b2d..f1a337b6 100644 --- a/compose.yaml +++ b/compose.yaml @@ -53,6 +53,8 @@ services: MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-120} S3_SWEEP_INTERVAL_SECONDS: ${S3_SWEEP_INTERVAL_SECONDS:-30} + S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:9000} + PRESIGN_TTL_SECONDS: ${PRESIGN_TTL_SECONDS:-60} DB_POOL_MIN: "1" DB_POOL_MAX: ${DB_POOL_MAX:-4} DB_COMMAND_TIMEOUT: "5" @@ -123,6 +125,8 @@ services: S3_BUCKET: ${S3_BUCKET:-messenger-attachments} S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + ports: + - "127.0.0.1:9000:9000" volumes: - s3_data:/data - ./infra/s3/seaweed-entrypoint.sh:/seaweed-entrypoint.sh:ro diff --git a/contracts/extensions.openapi.json b/contracts/extensions.openapi.json new file mode 100644 index 00000000..a91a973b --- /dev/null +++ b/contracts/extensions.openapi.json @@ -0,0 +1,21 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Messenger lab4 optional direct upload", + "version": "1.4.0-track1" + }, + "paths": { + "/api/v1/chats/{chat_id}/uploads": { + "post": { + "operationId": "beginDirectUpload", + "summary": "Issue a 60s PUT signature for one new object key. Does not replace POST /attachments." + } + }, + "/api/v1/uploads/{upload_id}/complete": { + "post": { + "operationId": "completeDirectUpload", + "summary": "Check owner, size and sha256 of the object that arrived, then store ready metadata." + } + } + } +} diff --git a/evidence/track1.json b/evidence/track1.json new file mode 100644 index 00000000..3a94c1a5 --- /dev/null +++ b/evidence/track1.json @@ -0,0 +1,26 @@ +{ + "hypothesis": "A presigned PUT does not enter the API, so API RX stays near zero at both sizes. complete reads the object once to check the checksum, and the proxy endpoint still receives the whole body.", + "presign_ttl_seconds": 60, + "sizes": [ + { + "label": "small", + "bytes": 90243, + "proxy_rx_bytes": 96204, + "proxy_peak_api_mib": 61.0, + "direct_put_rx_bytes": 0, + "direct_put_peak_api_mib": 61.4, + "complete_rx_bytes": 94131, + "complete_peak_api_mib": 61.4 + }, + { + "label": "large", + "bytes": 1891338, + "proxy_rx_bytes": 1913606, + "proxy_peak_api_mib": 62.4, + "direct_put_rx_bytes": 960, + "direct_put_peak_api_mib": 62.5, + "complete_rx_bytes": 1900090, + "complete_peak_api_mib": 62.1 + } + ] +} diff --git a/examples/requests.http b/examples/requests.http index 052497b1..a63d1f61 100644 --- a/examples/requests.http +++ b/examples/requests.http @@ -51,3 +51,18 @@ Authorization: Bearer {{sessionToken}} Content-Type: image/png < ../tests/fixtures/sample.png + +### Optional direct upload. PUT the bytes to the returned url, then complete with the real sha256. +POST {{baseUrl}}/api/v1/chats/{{chatId}}/uploads +Authorization: Bearer {{sessionToken}} +Content-Type: application/json + +{"filename":"sample.png","content_type":"image/png","size_bytes":1} + +### Complete only after the PUT. Replace the id and checksum locally. +POST {{baseUrl}}/api/v1/uploads/REPLACE_WITH_UPLOAD_ID/complete +Authorization: Bearer {{sessionToken}} +Content-Type: application/json + +{"sha256":"replace-with-lowercase-sha256-of-the-bytes"} + diff --git a/infra/README.md b/infra/README.md index 8641aeca..3182b208 100644 --- a/infra/README.md +++ b/infra/README.md @@ -6,4 +6,4 @@ Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. -Выбран SeaweedFS `chrislusf/seaweedfs:3.85` (S3 gateway на порту 9000 внутри сети Compose). Образ MinIO на Docker Hub в этом окружении недоступен. Порты наружу не опубликованы, UI filer выключен (`-filer.exposeDirectoryData=false`). `infra/s3/seaweed-entrypoint.sh` поднимает одну ноду и выдаёт API-пользователю права только на bucket `messenger-attachments`. Анонимная identity без actions. `s3-init` идемпотентно создаёт bucket теми же ограниченными ключами. +Выбран SeaweedFS `chrislusf/seaweedfs:3.85` (S3 gateway на порту 9000 внутри сети Compose). Образ MinIO на Docker Hub в этом окружении недоступен. Порты наружу не опубликованы, UI filer выключен (`-filer.exposeDirectoryData=false`). `infra/s3/seaweed-entrypoint.sh` поднимает одну ноду и выдаёт API-пользователю права только на bucket `messenger-attachments`. Анонимная identity без actions. `s3-init` идемпотентно создаёт bucket теми же ограниченными ключами. Для прямой загрузки S3 API слушает только `127.0.0.1:9000`; UI и остальные порты наружу не открыты. Подпись живёт `PRESIGN_TTL_SECONDS` (60). diff --git a/migrations/004_direct_uploads.sql b/migrations/004_direct_uploads.sql new file mode 100644 index 00000000..7db4663a --- /dev/null +++ b/migrations/004_direct_uploads.sql @@ -0,0 +1,22 @@ +-- Optional presigned upload. The signed URL dies quickly. The row stays pending +-- until complete checks the object that actually landed in the bucket. +-- Incomplete rows are aborted by the same grace sweeper as other uncommitted keys. + +CREATE TABLE direct_uploads ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users (id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT direct_uploads_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT direct_uploads_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT direct_uploads_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT direct_uploads_status CHECK (status IN ('pending', 'completed', 'aborted')) +); + +CREATE INDEX direct_uploads_pending_idx ON direct_uploads (expires_at) WHERE status = 'pending'; diff --git a/scripts/grade4_infra.py b/scripts/grade4_infra.py index 2c403a34..83a088be 100644 --- a/scripts/grade4_infra.py +++ b/scripts/grade4_infra.py @@ -211,13 +211,14 @@ def main(): capture_output=True, text=True, ) - if published.returncode == 0 and published.stdout.strip(): - raise SystemExit("s3 port is published: " + published.stdout.strip()) + binding = published.stdout.strip() + if published.returncode == 0 and binding and not binding.startswith("127.0.0.1:"): + raise SystemExit("s3 port is published outside loopback: " + binding) summary = { "anonymous_bucket_status": anonymous_status(), "second_bucket": other_bucket_denied(), "sweep": plant_and_sweep(), - "s3_port_published": False, + "s3_port_published": binding or False, "memory": memory_sample(), } OUT.parent.mkdir(exist_ok=True) diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py new file mode 100644 index 00000000..ff0fde80 --- /dev/null +++ b/scripts/track1_experiment.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +"""Track 1: presigned upload, negative cases, API traffic and RAM at two sizes. + +The signed URL is never written to the evidence file. +""" + +import base64 +import hashlib +import json +import os +import struct +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" / "track1.json" +BASE = "http://127.0.0.1:8080" + + +def expect(status, got, label): + if got == 500: + raise SystemExit("FAIL %s returned 500" % label) + if got != status: + raise SystemExit("FAIL %s expected %s got %s" % (label, status, got)) + print("PASS", label, got) + + +class Api: + def __init__(self) -> None: + self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def call(self, method, path, body=None, token=None, raw=None, content_type=None): + headers = {} + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + headers["Content-Type"] = "application/json" + if content_type: + headers["Content-Type"] = content_type + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + try: + response = self.opener.open(request, timeout=60) + except urllib.error.HTTPError as error: + response = error + with response: + payload = response.read() + parsed = None + if payload and response.headers.get_content_type() == "application/json": + parsed = json.loads(payload) + return response.status, parsed, payload + + +def png(width: int, height: int) -> bytes: + signature = b"\x89PNG\r\n\x1a\n" + + def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height)) + return signature + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"") + + +def put_signed(url: str, data: bytes, content_type: str) -> int: + request = urllib.request.Request(url, data=data, method="PUT", headers={"Content-Type": content_type}) + try: + response = urllib.request.urlopen(request, timeout=60) + except urllib.error.HTTPError as error: + response = error + with response: + response.read() + return response.status + + +def api_rx() -> int: + ids = subprocess.check_output( + ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], + cwd=ROOT, + text=True, + ).split() + total = 0 + for container_id in ids: + raw = subprocess.check_output(["docker", "exec", container_id, "cat", "/proc/net/dev"], text=True) + for line in raw.splitlines(): + if "eth0" not in line: + continue + total += int(line.split()[1]) + return total + + +def peak_during(action): + peaks = [] + stop = threading.Event() + + def watch(): + while not stop.is_set(): + stats = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, + text=True, + check=False, + ) + for line in stats.stdout.splitlines(): + if "mtusi2026lab4-api-" not in line: + continue + value = line.split()[1] + number = float(value[:-3]) + if not value.lower().endswith("mib"): + number = number / 1024 + peaks.append(number) + time.sleep(0.25) + + thread = threading.Thread(target=watch) + thread.start() + before = api_rx() + result = action() + after = api_rx() + time.sleep(0.3) + stop.set() + thread.join() + return {"rx_bytes": after - before, "peak_api_mib": round(max(peaks) if peaks else 0, 1), "result": result} + + +def compose_psql(sql: str) -> None: + subprocess.run( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", sql], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + + +def negatives(api, alice_token, bob_token, eve_token, chat_id): + small = png(16, 16) + status, _, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": "a/b.png", "content_type": "image/png", "size_bytes": len(small)}, + token=alice_token, + ) + expect(400, status, "direct filename") + status, _, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": "big.png", "content_type": "image/png", "size_bytes": 10485761}, + token=alice_token, + ) + expect(413, status, "direct declared size") + status, _, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": "x.png", "content_type": "image/png", "size_bytes": len(small)}, + token=eve_token, + ) + expect(404, status, "outsider presign") + + status, started, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": "wait.png", "content_type": "image/png", "size_bytes": len(small)}, + token=alice_token, + ) + expect(201, status, "presign for incomplete") + status, _, _ = api.call( + "POST", + "/api/v1/uploads/%s/complete" % started["upload_id"], + {"sha256": hashlib.sha256(small).hexdigest()}, + token=alice_token, + ) + expect(409, status, "complete before PUT") + status, _, _ = api.call( + "POST", + "/api/v1/uploads/%s/complete" % started["upload_id"], + {"sha256": hashlib.sha256(small).hexdigest()}, + token=bob_token, + ) + expect(404, status, "complete by another member") + + tampered = put_signed(started["url"] + "x", small, "image/png") + if tampered == 200: + raise SystemExit("FAIL tampered signature was accepted") + print("PASS tampered signature", tampered) + + status, started, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": "badsum.png", "content_type": "image/png", "size_bytes": len(small)}, + token=alice_token, + ) + expect(201, status, "presign for bad checksum") + if put_signed(started["url"], small, "image/png") != 200: + raise SystemExit("FAIL signed PUT") + status, _, _ = api.call( + "POST", + "/api/v1/uploads/%s/complete" % started["upload_id"], + {"sha256": "0" * 64}, + token=alice_token, + ) + expect(400, status, "wrong checksum") + status, _, _ = api.call( + "POST", + "/api/v1/uploads/%s/complete" % started["upload_id"], + {"sha256": hashlib.sha256(small).hexdigest()}, + token=alice_token, + ) + expect(409, status, "complete after rejected checksum") + + status, started, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": "late.png", "content_type": "image/png", "size_bytes": len(small)}, + token=alice_token, + ) + expect(201, status, "presign for expiry") + compose_psql( + "UPDATE direct_uploads SET expires_at = now() - interval '5 minutes' WHERE id = '%s'" + % started["upload_id"] + ) + status, _, _ = api.call( + "POST", + "/api/v1/uploads/%s/complete" % started["upload_id"], + {"sha256": hashlib.sha256(small).hexdigest()}, + token=alice_token, + ) + expect(409, status, "expired upload") + + digest = hashlib.sha256(small).hexdigest() + status, started, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": "ok.png", "content_type": "image/png", "size_bytes": len(small)}, + token=alice_token, + ) + expect(201, status, "presign happy path") + if "originals/" in json.dumps({k: v for k, v in started.items() if k != "url"}): + raise SystemExit("FAIL object key leaked outside the signed URL") + if put_signed(started["url"], small, "image/png") != 200: + raise SystemExit("FAIL happy PUT") + status, attachment, _ = api.call( + "POST", + "/api/v1/uploads/%s/complete" % started["upload_id"], + {"sha256": digest}, + token=alice_token, + ) + expect(201, status, "complete happy path") + if attachment["sha256"] != digest or attachment["status"] != "ready": + raise SystemExit("FAIL complete metadata") + status, _, downloaded = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token) + expect(200, status, "member downloads direct upload") + if downloaded != small: + raise SystemExit("FAIL direct bytes") + status, _, _ = api.call( + "POST", + "/api/v1/uploads/%s/complete" % started["upload_id"], + {"sha256": digest}, + token=alice_token, + ) + expect(409, status, "second complete") + status, proxy, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=proxy.png" % chat_id, + token=alice_token, + raw=small, + content_type="image/png", + ) + expect(201, status, "proxy endpoint still works") + return proxy["id"] + + +def compare(api, token, chat_id): + rows = [] + for label, image in (("small", png(200, 150)), ("large", png(900, 700))): + digest = hashlib.sha256(image).hexdigest() + proxy = peak_during( + lambda image=image: api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s.png" % (chat_id, label), + token=token, + raw=image, + content_type="image/png", + ) + ) + if proxy["result"][0] != 201: + raise SystemExit("FAIL proxy %s %s" % (label, proxy["result"][0])) + status, started, _ = api.call( + "POST", + "/api/v1/chats/%s/uploads" % chat_id, + {"filename": label + "-direct.png", "content_type": "image/png", "size_bytes": len(image)}, + token=token, + ) + expect(201, status, "presign " + label) + direct_put = peak_during(lambda image=image, url=started["url"]: put_signed(url, image, "image/png")) + if direct_put["result"] != 200: + raise SystemExit("FAIL direct PUT %s %s" % (label, direct_put["result"])) + direct_complete = peak_during( + lambda digest=digest, upload_id=started["upload_id"]: api.call( + "POST", + "/api/v1/uploads/%s/complete" % upload_id, + {"sha256": digest}, + token=token, + ) + ) + if direct_complete["result"][0] != 201: + raise SystemExit("FAIL complete %s %s" % (label, direct_complete["result"][0])) + rows.append( + { + "label": label, + "bytes": len(image), + "proxy_rx_bytes": proxy["rx_bytes"], + "proxy_peak_api_mib": proxy["peak_api_mib"], + "direct_put_rx_bytes": direct_put["rx_bytes"], + "direct_put_peak_api_mib": direct_put["peak_api_mib"], + "complete_rx_bytes": direct_complete["rx_bytes"], + "complete_peak_api_mib": direct_complete["peak_api_mib"], + } + ) + print("MEASURED", rows[-1]) + return rows + + +def main(): + api = Api() + + def register(label): + username = "t1_" + label + "_" + uuid.uuid4().hex[:10] + password = "T3st-" + uuid.uuid4().hex + status, body, _ = api.call( + "POST", + "/api/v1/users", + {"username": username, "display_name": "Track " + label, "password": password}, + ) + expect(201, status, "register " + label) + return username, password, body + + def login(username, password): + basic = base64.b64encode((username + ":" + password).encode()).decode() + request = urllib.request.Request( + BASE + "/api/v1/auth/sessions", + method="POST", + headers={"Authorization": "Basic " + basic}, + ) + with urllib.request.urlopen(request, timeout=30) as response: + return json.loads(response.read())["token"] + + alice_name, alice_password, alice = register("alice") + bob_name, bob_password, bob = register("bob") + eve_name, eve_password, eve = register("eve") + alice_token = login(alice_name, alice_password) + bob_token = login(bob_name, bob_password) + eve_token = login(eve_name, eve_password) + status, chat, _ = api.call( + "POST", + "/api/v1/chats", + {"title": "track1", "member_ids": [alice["id"], bob["id"]]}, + token=alice_token, + ) + expect(201, status, "chat") + negatives(api, alice_token, bob_token, eve_token, chat["id"]) + rows = compare(api, alice_token, chat["id"]) + summary = { + "hypothesis": "A presigned PUT does not enter the API, so API RX stays near zero at both sizes. complete reads the object once to check the checksum, and the proxy endpoint still receives the whole body.", + "presign_ttl_seconds": 60, + "sizes": rows, + } + OUT.parent.mkdir(exist_ok=True) + OUT.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + print(json.dumps(summary, indent=2)) + + +if __name__ == "__main__": + main() -- 2.54.0 From 41d6acde1554e40b3260bce7db2583a72125d67d Mon Sep 17 00:00:00 2001 From: David Date: Mon, 9 Nov 2026 00:38:47 +0300 Subject: [PATCH 5/5] =?UTF-8?q?=D0=A3=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D1=8C?= =?UTF-8?q?=20=D1=81=D1=81=D1=8B=D0=BB=D0=BA=D1=83=20=D0=BD=D0=B0=20pull?= =?UTF-8?q?=20request=20=D0=B2=20=D0=BE=D1=82=D1=87=D1=91=D1=82=D0=B5.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 6c48267a..e1127aff 100644 --- a/REPORT.md +++ b/REPORT.md @@ -2,7 +2,7 @@ - Автор, группа: Булацев, БПИ2301 - Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 «Прямой upload». -- Commit/PR: локальные коммиты после переноса lab3; PR не публиковался в этой сессии +- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/94 - Использованные источники, библиотеки и помощь ИИ: контракт репозитория, документация SeaweedFS S3, boto3, Pillow. Реализация и прогон проверок сделаны с помощью Cursor. ## Запуск из чистого clone -- 2.54.0