From 3601ca54432e87f4b52752a57f67574f0fa32b83 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 7 Nov 2026 12:46:37 +0300 Subject: [PATCH 1/4] =?UTF-8?q?=D0=9F=D0=B5=D1=80=D0=B5=D0=BD=D0=B5=D1=81?= =?UTF-8?q?=D1=82=D0=B8=20API=20=D1=81=20=D0=BE=D0=B1=D1=89=D0=B8=D0=BC?= =?UTF-8?q?=D0=B8=20=D1=81=D0=B5=D1=81=D1=81=D0=B8=D1=8F=D0=BC=D0=B8=20?= =?UTF-8?q?=D0=B8=20=D0=B8=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5=D0=BD?= =?UTF-8?q?=D1=82=D0=BD=D0=BE=D0=B9=20=D0=BE=D1=82=D0=BF=D1=80=D0=B0=D0=B2?= =?UTF-8?q?=D0=BA=D0=BE=D0=B9=20=D1=81=D0=BE=D0=BE=D0=B1=D1=89=D0=B5=D0=BD?= =?UTF-8?q?=D0=B8=D1=8F.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Реплика больше не держит корректность в своей памяти: сессия в Redis, а повтор POST сообщения фиксируется в Postgres вместе с X-Instance-Id. --- Dockerfile | 29 +++ Makefile | 32 ++- app/__init__.py | 1 + app/cache.py | 151 +++++++++++++ app/config.py | 42 ++++ app/cursor_codec.py | 112 ++++++++++ app/db.py | 87 ++++++++ app/docs.py | 52 +++++ app/errors.py | 49 +++++ app/main.py | 146 +++++++++++++ app/middleware.py | 48 ++++ app/migrate.py | 29 +++ app/passwords.py | 23 ++ app/rediscli.py | 40 ++++ app/routes.py | 284 ++++++++++++++++++++++++ app/sessions.py | 56 +++++ app/store.py | 361 +++++++++++++++++++++++++++++++ app/tools/__init__.py | 1 + app/tools/explain.py | 73 +++++++ app/validation.py | 102 +++++++++ migrations/001_init.sql | 49 +++++ migrations/002_idempotency.sql | 14 ++ pytest.ini | 6 + requirements-test.txt | 2 + requirements.txt | 5 + scripts/run_expiry.py | 55 +++++ student_tests/__init__.py | 0 student_tests/conftest.py | 24 ++ student_tests/test_api.py | 195 +++++++++++++++++ student_tests/test_concurrent.py | 48 ++++ student_tests/test_expiry.py | 17 ++ student_tests/test_grade4.py | 44 ++++ student_tests/test_infra.py | 89 ++++++++ student_tests/test_track1.py | 83 +++++++ 34 files changed, 2342 insertions(+), 7 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/cache.py create mode 100644 app/config.py create mode 100644 app/cursor_codec.py create mode 100644 app/db.py create mode 100644 app/docs.py create mode 100644 app/errors.py create mode 100644 app/main.py create mode 100644 app/middleware.py create mode 100644 app/migrate.py create mode 100644 app/passwords.py create mode 100644 app/rediscli.py create mode 100644 app/routes.py create mode 100644 app/sessions.py create mode 100644 app/store.py create mode 100644 app/tools/__init__.py create mode 100644 app/tools/explain.py create mode 100644 app/validation.py create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 pytest.ini create mode 100644 requirements-test.txt create mode 100644 requirements.txt create mode 100644 scripts/run_expiry.py create mode 100644 student_tests/__init__.py create mode 100644 student_tests/conftest.py create mode 100644 student_tests/test_api.py create mode 100644 student_tests/test_concurrent.py create mode 100644 student_tests/test_expiry.py create mode 100644 student_tests/test_grade4.py create mode 100644 student_tests/test_infra.py create mode 100644 student_tests/test_track1.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..65e21fb0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12.8-slim-bookworm AS runtime + +WORKDIR /app + +RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app \ + && apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations +COPY contracts/openapi.json ./contracts/openapi.json + +USER app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + PYTHONUNBUFFERED=1 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=12 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + +CMD ["python", "-m", "app.main"] diff --git a/Makefile b/Makefile index 5d86d171..27e00482 100644 --- a/Makefile +++ b/Makefile @@ -4,27 +4,45 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-student test-infra test-expiry explain evidence-grade4 evidence-track1 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" check: - $(PYTHON) scripts/check_contract.py + $(PYTHON) -X utf8 scripts/check_contract.py validate: - $(PYTHON) scripts/validate_openapi.py + $(PYTHON) -X utf8 scripts/validate_openapi.py test: - $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) -X utf8 tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) load: - $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) -X utf8 tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) - $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) -X utf8 tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) - $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) -X utf8 tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-student: + $(PYTHON) -m pytest student_tests -q -m "not infra and not expiry" + +test-infra: + RUN_INFRA=1 $(PYTHON) -m pytest student_tests/test_infra.py -q -m infra + +test-expiry: + $(PYTHON) scripts/run_expiry.py + +explain: + docker compose --env-file .env exec -T api python -m app.tools.explain + +evidence-grade4: + $(PYTHON) -X utf8 scripts/grade4_evidence.py + +evidence-track1: + $(PYTHON) -X utf8 scripts/track1_experiment.py diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..eeb0eb82 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger API (lab 2): PostgreSQL source of truth, Redis sessions/cache.""" diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..ed9261df --- /dev/null +++ b/app/cache.py @@ -0,0 +1,151 @@ +import asyncio +import json +from collections.abc import Awaitable, Callable +from typing import TypeVar + +from app import config, rediscli +from app.errors import AppError + +T = TypeVar("T") +CFG_SINGLEFLIGHT = "cfg:singleflight" +CFG_INVALIDATE = "cfg:invalidate" +CFG_DELAY_MS = "cfg:delay_ms" +SQL_HISTORY = "metrics:sql:messages_select" + + +class SingleFlight: + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._inflight: dict[str, asyncio.Task] = {} + + async def do(self, key: str, factory: Callable[[], Awaitable[T]]) -> T: + async with self._lock: + task = self._inflight.get(key) + if task is None: + task = asyncio.create_task(factory()) + self._inflight[key] = task + owner = True + else: + owner = False + try: + return await asyncio.shield(task) + finally: + if owner: + async with self._lock: + self._inflight.pop(key, None) + + +_sf = SingleFlight() + + +async def _flag(name: str, default: bool = True) -> bool: + if not config.lab_mode(): + return default + try: + raw = await rediscli.require().get(name) + except Exception: + return default + if raw is None: + return default + return raw not in {"0", "false", "off"} + + +async def singleflight_enabled() -> bool: + return await _flag(CFG_SINGLEFLIGHT, True) + + +async def invalidate_enabled() -> bool: + return await _flag(CFG_INVALIDATE, True) + + +async def incr_history_sql() -> None: + try: + await rediscli.require().incr(SQL_HISTORY) + except Exception: + pass + + +async def history_sql_count() -> int: + try: + return int(await rediscli.require().get(SQL_HISTORY) or 0) + except Exception: + return 0 + + +async def reset_history_sql_count() -> None: + try: + await rediscli.require().set(SQL_HISTORY, 0) + except Exception: + pass + + +async def load_delay_seconds() -> float: + if not config.lab_mode(): + return 0.0 + try: + raw = await rediscli.require().get(CFG_DELAY_MS) + except Exception: + return 0.0 + try: + return max(0.0, float(raw or 0) / 1000.0) + except (TypeError, ValueError): + return 0.0 + + +async def set_flags( + *, + singleflight: bool | None = None, + invalidate: bool | None = None, + delay_ms: int | None = None, +) -> dict: + r = rediscli.require() + if singleflight is not None: + await r.set(CFG_SINGLEFLIGHT, "1" if singleflight else "0") + if invalidate is not None: + await r.set(CFG_INVALIDATE, "1" if invalidate else "0") + if delay_ms is not None: + await r.set(CFG_DELAY_MS, str(int(delay_ms))) + return { + "singleflight": await singleflight_enabled(), + "invalidate": await invalidate_enabled(), + "history_sql": await history_sql_count(), + "delay_ms": int((await load_delay_seconds()) * 1000), + } + + +async def get_history_page(chat_id: str, cursor: str, limit: int, loader: Callable[[], Awaitable[dict]]) -> dict: + r = rediscli.require() + try: + version = await r.get("histver:" + chat_id) or "0" + key = f"hist:{chat_id}:{version}:{cursor}:{limit}" + cached = await r.get(key) + if cached: + return json.loads(cached) + + async def load_and_store() -> dict: + delay = await load_delay_seconds() + if delay: + await asyncio.sleep(delay) + page = await loader() + try: + await r.set(key, json.dumps(page, ensure_ascii=False), ex=config.CACHE_TTL_SECONDS) + except Exception: + pass + return page + + if await singleflight_enabled(): + return await _sf.do(key, load_and_store) + return await load_and_store() + except AppError: + raise + except Exception: + return await loader() + + +async def invalidate_history(chat_id: str) -> None: + if not await invalidate_enabled(): + return + try: + await rediscli.require().incr("histver:" + chat_id) + except Exception: + pass diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..b6d1f3b1 --- /dev/null +++ b/app/config.py @@ -0,0 +1,42 @@ +import os +import socket +from pathlib import Path + + +def _env(name: str, default: str = "") -> str: + value = os.environ.get(name) + return default if value is None or value == "" else value + + +def _file_or_env(name: str, default: str = "") -> str: + path = os.environ.get(name + "_FILE") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return _env(name, default) + + +HOST = _env("APP_HOST", "0.0.0.0") +PORT = int(_env("APP_PORT", "8080")) +ENV = _env("APP_ENV", "lab") +LOG_LEVEL = _env("LOG_LEVEL", "info").lower() +SHUTDOWN_GRACE_SECONDS = float(_env("SHUTDOWN_GRACE_SECONDS", "10")) +SESSION_TTL_SECONDS = int(_env("SESSION_TTL_SECONDS", "3600")) +CACHE_TTL_SECONDS = int(_env("CACHE_TTL_SECONDS", "60")) +DATABASE_URL = _file_or_env("DATABASE_URL") +REDIS_URL = _file_or_env("REDIS_URL", "redis://redis:6379/0") +SESSION_STORE = _env("SESSION_STORE", "redis") +DB_POOL_MIN = int(_env("DB_POOL_MIN", "1")) +DB_POOL_MAX = int(_env("DB_POOL_MAX", "10")) +DB_COMMAND_TIMEOUT = float(_env("DB_COMMAND_TIMEOUT", "5")) +DB_CONNECT_TIMEOUT = float(_env("DB_CONNECT_TIMEOUT", "5")) +REDIS_SOCKET_TIMEOUT = float(_env("REDIS_SOCKET_TIMEOUT", "2")) +IDEMPOTENCY_TTL_SECONDS = int(_env("IDEMPOTENCY_TTL_SECONDS", "86400")) +MIGRATIONS_ON_START = _env("MIGRATIONS_ON_START", "1") == "1" +LB_DRAIN_SECONDS = float(_env("LB_DRAIN_SECONDS", "2")) +# Stable for the life of this process. Compose leaves it unset so each +# container uses its own hostname; stop-one.sh maps that value to one replica. +INSTANCE_ID = _env("INSTANCE_ID", socket.gethostname()) + + +def lab_mode() -> bool: + return ENV == "lab" diff --git a/app/cursor_codec.py b/app/cursor_codec.py new file mode 100644 index 00000000..40e6ce20 --- /dev/null +++ b/app/cursor_codec.py @@ -0,0 +1,112 @@ +import base64 + +import json + +from datetime import datetime + +from typing import Any + +from uuid import UUID + + + + + +def encode_cursor(chat_id: str, created_at: str, entity_id: str) -> str: + + payload = json.dumps({"c": chat_id, "t": created_at, "i": entity_id}, separators=(",", ":")) + + return base64.urlsafe_b64encode(payload.encode("utf-8")).decode("ascii").rstrip("=") + + + + + +def _validate_cursor_timestamp(value: str) -> None: + + if not value.endswith("Z"): + + raise ValueError("invalid cursor timestamp") + + try: + + datetime.fromisoformat(value[:-1] + "+00:00") + + except ValueError as exc: + + raise ValueError("invalid cursor timestamp") from exc + + + + + +def _require_str_field(data: dict, key: str) -> str: + + if key not in data: + + raise ValueError("invalid cursor") + + value = data[key] + + if not isinstance(value, str): + + raise ValueError("invalid cursor") + + return value + + + + + +def decode_cursor(cursor: str, expected_chat_id: str) -> tuple[str, str]: + + pad = "=" * (-len(cursor) % 4) + + try: + + raw = base64.urlsafe_b64decode((cursor + pad).encode("ascii")) + + data: Any = json.loads(raw.decode("utf-8")) + + except (ValueError, json.JSONDecodeError, UnicodeDecodeError) as exc: + + raise ValueError("invalid cursor") from exc + + if not isinstance(data, dict): + + raise ValueError("invalid cursor") + + try: + + chat_id = _require_str_field(data, "c") + + created_at = _require_str_field(data, "t") + + entity_id = _require_str_field(data, "i") + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + if chat_id != expected_chat_id: + + raise ValueError("cursor chat mismatch") + + try: + + UUID(entity_id) + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + try: + + _validate_cursor_timestamp(created_at) + + except ValueError as exc: + + raise ValueError("invalid cursor") from exc + + return created_at, entity_id + diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..f98fd341 --- /dev/null +++ b/app/db.py @@ -0,0 +1,87 @@ +from pathlib import Path + +import asyncpg + +from app import config +from app.errors import AppError + +pool: asyncpg.Pool | None = None +INTEGRITY = ( + asyncpg.UniqueViolationError, + asyncpg.ForeignKeyViolationError, + asyncpg.CheckViolationError, +) + + +def _migrations_dir() -> Path: + here = Path(__file__).resolve().parent + for candidate in (here.parent / "migrations", Path("/app/migrations")): + if candidate.is_dir(): + return candidate + raise RuntimeError("migrations directory not found") + + +async def connect() -> asyncpg.Pool: + global pool + if not config.DATABASE_URL: + raise RuntimeError("DATABASE_URL is not set") + pool = await asyncpg.create_pool( + dsn=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + command_timeout=config.DB_COMMAND_TIMEOUT, + timeout=config.DB_CONNECT_TIMEOUT, + ) + return pool + + +async def close() -> None: + global pool + if pool is not None: + await pool.close() + pool = None + + +def require() -> asyncpg.Pool: + if pool is None: + raise AppError(503, "service unavailable") + return pool + + +async def ping() -> None: + try: + async with require().acquire() as conn: + await conn.execute("SELECT 1") + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + + +_MIGRATION_LOCK = 87133103 + + +async def apply_migrations() -> None: + """Apply pending SQL files once. A session advisory lock stops replica races.""" + directory = _migrations_dir() + async with require().acquire() as conn: + await conn.execute("SELECT pg_advisory_lock($1)", _MIGRATION_LOCK) + try: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = {row["filename"] for row in await conn.fetch("SELECT filename FROM schema_migrations")} + for path in sorted(p for p in directory.glob("*.sql") if p.is_file()): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute("INSERT INTO schema_migrations(filename) VALUES($1)", path.name) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", _MIGRATION_LOCK) diff --git a/app/docs.py b/app/docs.py new file mode 100644 index 00000000..7f68e1d9 --- /dev/null +++ b/app/docs.py @@ -0,0 +1,52 @@ +import json +from functools import lru_cache +from pathlib import Path + +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse, Response + +_SPEC_CANDIDATES = ( + Path(__file__).resolve().parents[1] / "contracts" / "openapi.json", + Path("/app/contracts/openapi.json"), +) + +_SWAGGER_HTML = """ + + + + MTUSI Messenger — Swagger (lab2) + + + +
+ + + + +""" + + +@lru_cache(maxsize=1) +def _load_spec() -> dict: + for path in _SPEC_CANDIDATES: + if path.is_file(): + return json.loads(path.read_text(encoding="utf-8")) + return {"openapi": "3.1.0", "info": {"title": "MTUSI Messenger", "version": "1.2.0"}, "paths": {}} + + +async def openapi_json(_: Request) -> Response: + return JSONResponse(_load_spec()) + + +async def swagger_ui(_: Request) -> Response: + return HTMLResponse(_SWAGGER_HTML) diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..105e8ca3 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,49 @@ +from typing import Dict, Optional + +from starlette.responses import JSONResponse + + +ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + 500: "internal_error", +} + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class AppError(Exception): + def __init__(self, status: int, message: str, www_authenticate: str | None = None) -> None: + super().__init__(message) + self.status = status + self.message = message + self.www_authenticate = www_authenticate + + +def error_response( + status: int, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, +) -> JSONResponse: + code = ERROR_CODES.get(status, "internal_error") + headers = {"X-Request-Id": request_id, "Content-Type": "application/json"} + if extra_headers: + headers.update(extra_headers) + return JSONResponse( + status_code=status, + headers=headers, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + ) + + +def from_app_error(exc: AppError, request_id: str) -> JSONResponse: + extra = {"WWW-Authenticate": exc.www_authenticate} if exc.www_authenticate else None + return error_response(exc.status, exc.message, request_id, extra) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..c59ca752 --- /dev/null +++ b/app/main.py @@ -0,0 +1,146 @@ +import asyncio +import logging +import signal +import sys +import uuid +from pathlib import Path + +import uvicorn +from starlette.applications import Starlette +from starlette.exceptions import HTTPException +from starlette.requests import Request +from starlette.responses import Response +from starlette.routing import Route + +from app import config, db, rediscli +from app.docs import openapi_json, swagger_ui +from app.errors import AppError, error_response, from_app_error +from app.middleware import RequestContextMiddleware +from app.routes import ( + create_chat, + create_message, + create_session, + current_user, + get_chat, + health_live, + health_ready, + lab_flags, + lab_metrics, + list_chats, + list_messages, + register_user, + revoke_session, +) + + +def _request_id(request: Request) -> str: + existing = getattr(request.state, "request_id", None) + if existing: + return existing + return request.headers.get("X-Request-Id") or str(uuid.uuid4()) + + +async def not_found_handler(request: Request, exc: HTTPException) -> Response: + return error_response(404, "not found", _request_id(request)) + + +async def app_error_handler(request: Request, exc: AppError) -> Response: + return from_app_error(exc, _request_id(request)) + + +def publish_instance_id() -> None: + path = Path(config._env("INSTANCE_ID_FILE", "/tmp/instance-id")) + try: + path.write_text(config.INSTANCE_ID, encoding="utf-8") + except OSError: + logging.getLogger("messenger").warning("instance id file was not written") + + +async def startup() -> None: + publish_instance_id() + logging.getLogger("messenger").info("instance_id=%s", config.INSTANCE_ID) + await db.connect() + if config.MIGRATIONS_ON_START: + await db.apply_migrations() + await rediscli.connect() + + +async def shutdown() -> None: + await rediscli.close() + await db.close() + + +def build_app() -> Starlette: + app = Starlette( + routes=[ + Route("/docs", swagger_ui, methods=["GET"]), + Route("/openapi.json", openapi_json, methods=["GET"]), + Route("/health/live", health_live, methods=["GET"]), + Route("/health/ready", health_ready, methods=["GET"]), + Route("/api/v1/users", register_user, methods=["POST"]), + Route("/api/v1/users/me", current_user, methods=["GET"]), + Route("/api/v1/auth/sessions", create_session, methods=["POST"]), + Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]), + Route("/api/v1/chats", create_chat, methods=["POST"]), + Route("/api/v1/chats", list_chats, methods=["GET"]), + Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), + Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]), + Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]), + ], + exception_handlers={404: not_found_handler, AppError: app_error_handler}, + on_startup=[startup], + on_shutdown=[shutdown], + ) + app.state.shutting_down = False + app.add_middleware(RequestContextMiddleware) + return app + + +def configure_logging() -> None: + logging.basicConfig( + level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + ) + + +def main() -> None: + configure_logging() + app = build_app() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + server = uvicorn.Server( + uvicorn.Config( + app, + host=config.HOST, + port=config.PORT, + log_level=config.LOG_LEVEL, + access_log=False, + timeout_graceful_shutdown=int(config.SHUTDOWN_GRACE_SECONDS), + ) + ) + + def _begin_shutdown() -> None: + if app.state.shutting_down: + return + logging.getLogger("messenger").info( + "shutdown signal received; readiness fails now, exit in %.1fs", + config.LB_DRAIN_SECONDS, + ) + app.state.shutting_down = True + loop.call_later(config.LB_DRAIN_SECONDS, lambda: setattr(server, "should_exit", True)) + + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, _begin_shutdown) + except NotImplementedError: + signal.signal(sig, lambda *_: _begin_shutdown()) + + loop.run_until_complete(server.serve()) + + +if __name__ == "__main__": + main() diff --git a/app/middleware.py b/app/middleware.py new file mode 100644 index 00000000..4b05cca3 --- /dev/null +++ b/app/middleware.py @@ -0,0 +1,48 @@ +import logging +import time +import uuid +from typing import Callable + +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request +from starlette.responses import Response + +from app.config import INSTANCE_ID +from app.errors import AppError, error_response, from_app_error + +logger = logging.getLogger("messenger.access") + + +def _stamp(response: Response, request_id: str) -> Response: + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +class RequestContextMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next: Callable) -> Response: + request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = request_id + + if request.app.state.shutting_down and request.url.path not in ("/health/live", "/health/ready"): + return _stamp(error_response(503, "service is shutting down", request_id), request_id) + + started = time.perf_counter() + try: + response = await call_next(request) + except AppError as exc: + return _stamp(from_app_error(exc, request_id), request_id) + except Exception: + logger.exception("request_id=%s unhandled error path=%s", request_id, request.url.path) + return _stamp(error_response(500, "internal error", request_id), request_id) + elapsed_ms = (time.perf_counter() - started) * 1000 + _stamp(response, request_id) + logger.info( + "request_id=%s method=%s path=%s status=%s duration_ms=%.2f", + request_id, + request.method, + request.url.path, + response.status_code, + elapsed_ms, + ) + return response diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..03bc8db8 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,29 @@ +"""One-shot schema apply. API replicas do not run this on startup.""" +import asyncio +import logging +import sys + +from app import config, db + + +async def _main() -> None: + if not config.DATABASE_URL: + raise SystemExit("DATABASE_URL is not set") + await db.connect() + try: + await db.apply_migrations() + finally: + await db.close() + + +def main() -> None: + logging.basicConfig( + level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + ) + asyncio.run(_main()) + + +if __name__ == "__main__": + main() diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..8a640cd0 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,23 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError + +hasher = PasswordHasher(time_cost=2, memory_cost=8192, parallelism=1) +_dummy = None + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def dummy_hash() -> str: + global _dummy + if _dummy is None: + _dummy = hash_password("timing-dummy-password") + return _dummy + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return hasher.verify(password_hash, password) + except (VerifyMismatchError, InvalidHashError, TypeError, ValueError): + return False diff --git a/app/rediscli.py b/app/rediscli.py new file mode 100644 index 00000000..de173117 --- /dev/null +++ b/app/rediscli.py @@ -0,0 +1,40 @@ +import redis.asyncio as redis + +from app import config +from app.errors import AppError + +client: redis.Redis | None = None + + +async def connect() -> redis.Redis: + global client + client = redis.from_url( + config.REDIS_URL, + socket_connect_timeout=config.REDIS_SOCKET_TIMEOUT, + socket_timeout=config.REDIS_SOCKET_TIMEOUT, + decode_responses=True, + ) + await client.ping() + return client + + +async def close() -> None: + global client + if client is not None: + await client.close() + client = None + + +def require() -> redis.Redis: + if client is None: + raise AppError(503, "service unavailable") + return client + + +async def ping() -> None: + try: + await require().ping() + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None diff --git a/app/routes.py b/app/routes.py new file mode 100644 index 00000000..0b7bb307 --- /dev/null +++ b/app/routes.py @@ -0,0 +1,284 @@ +import base64 +import binascii + +from starlette.requests import Request +from starlette.responses import JSONResponse, Response + +from app import cache, config, db, rediscli, sessions, store +from app.cursor_codec import decode_cursor, encode_cursor +from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response +from app.passwords import dummy_hash, hash_password, verify_password +from app.validation import ( + parse_json_body, + require_exact_keys, + validate_display_name, + validate_limit, + validate_idempotency_key, + validate_message_text, + validate_password, + validate_title, + validate_username, + validate_uuid, +) + + +def _json(data: dict, request_id: str, status: int = 200) -> JSONResponse: + return JSONResponse(status_code=status, headers={"X-Request-Id": request_id}, content=data) + + +def _empty(request_id: str) -> Response: + return Response(status_code=204, headers={"X-Request-Id": request_id}) + + +async def _bearer_user(request: Request) -> dict: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "authentication required", BEARER_CHALLENGE) + user_id = str(await sessions.resolve(rest.strip())) + user = await store.get_user(user_id) + if user is None: + raise AppError(401, "authentication required", BEARER_CHALLENGE) + return user + + +async def health_live(request: Request) -> Response: + return _json({"status": "ok"}, request.state.request_id) + + +async def health_ready(request: Request) -> Response: + rid = request.state.request_id + if request.app.state.shutting_down: + return error_response(503, "service is shutting down", rid) + await db.ping() + await rediscli.ping() + return _json({"status": "ok"}, rid) + + +async def register_user(request: Request) -> Response: + rid = request.state.request_id + if request.headers.get("content-type", "").split(";")[0].strip().lower() not in ("application/json", ""): + if request.headers.get("content-type"): + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}): + return error_response(400, "invalid user payload", rid) + if validate_username(body["username"]) or validate_display_name(body["display_name"]) or validate_password(body["password"]): + return error_response(400, "invalid user fields", rid) + user, conflict = await store.create_user(body["username"], body["display_name"], hash_password(body["password"])) + if conflict: + return error_response(409, "username already taken", rid) + assert user is not None + return _json(user, rid, 201) + + +async def current_user(request: Request) -> Response: + user = await _bearer_user(request) + return _json(user, request.state.request_id) + + +async def create_session(request: Request) -> Response: + rid = request.state.request_id + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "basic" or not rest.strip(): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + try: + decoded = base64.b64decode(rest.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeError, ValueError): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + username, sep, password = decoded.partition(":") + if not sep: + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + row = await store.get_user_by_username(username) + password_hash = row["password_hash"] if row else dummy_hash() + if row is None or not verify_password(password_hash, password): + return error_response(401, "authentication required", rid, {"WWW-Authenticate": BASIC_CHALLENGE}) + token, expires_at = await sessions.create(row["id"]) + return _json( + { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": store.user_dict(row), + }, + rid, + 201, + ) + + +async def revoke_session(request: Request) -> Response: + header = request.headers.get("Authorization") or "" + scheme, _, rest = header.partition(" ") + if scheme.lower() != "bearer" or not rest.strip(): + raise AppError(401, "authentication required", BEARER_CHALLENGE) + await sessions.revoke(rest.strip()) + return _empty(request.state.request_id) + + +async def create_chat(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if ct and ct != "application/json": + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"title", "member_ids"}, {"title", "member_ids"}): + return error_response(400, "invalid chat payload", rid) + if validate_title(body["title"]): + return error_response(400, "invalid title", rid) + members = body["member_ids"] + if not isinstance(members, list) or not members: + return error_response(400, "invalid member_ids", rid) + if len(members) > 100: + return error_response(400, "too many members", rid) + for mid in members: + if validate_uuid(mid): + return error_response(400, "invalid member id", rid) + chat, create_err = await store.create_chat(body["title"], members, user["id"]) + if create_err == "caller not in members": + return error_response(400, "caller must be a member", rid) + if create_err in ("duplicate members", "unknown member"): + return error_response(400, "invalid members", rid) + assert chat is not None + return _json(chat, rid, 201) + + +async def list_chats(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + limit, limit_err = validate_limit(request.query_params.get("limit")) + if limit_err: + return error_response(400, limit_err, rid) + assert limit is not None + cursor_raw = request.query_params.get("cursor") + after = None + if cursor_raw is not None: + if not cursor_raw.strip(): + return error_response(400, "invalid cursor", rid) + try: + created_at, entity_id = decode_cursor(cursor_raw, user["id"]) + except ValueError: + return error_response(400, "invalid cursor", rid) + after = (created_at, entity_id) + items, next_key = await store.list_chats(user["id"], limit, after) + next_cursor = None + if next_key is not None: + next_cursor = encode_cursor(user["id"], next_key[0], next_key[1]) + return _json({"items": items, "next_cursor": next_cursor}, rid) + + +async def get_chat(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + chat = await store.get_chat(chat_id, user["id"]) + if chat is None: + return error_response(404, "chat not found", rid) + return _json(chat, rid) + + +async def create_message(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if ct and ct != "application/json": + return error_response(415, "unsupported media type", rid) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, rid) + assert body is not None + if require_exact_keys(body, {"text"}, {"text"}): + return error_response(400, "invalid message payload", rid) + if validate_message_text(body["text"]): + return error_response(400, "invalid message text", rid) + if validate_idempotency_key(request.headers.get("Idempotency-Key")): + return error_response(400, "invalid idempotency key", rid) + message, msg_err = await store.add_message( + chat_id, + user["id"], + body["text"], + request.headers.get("Idempotency-Key") or "", + request.url.path, + ) + if msg_err == "not_found": + return error_response(404, "chat not found", rid) + if msg_err == "conflict": + return error_response(409, "idempotency key reused with a different body", rid) + assert message is not None + return _json(message, rid, 201) + + +async def list_messages(request: Request) -> Response: + rid = request.state.request_id + user = await _bearer_user(request) + chat_id = request.path_params["chat_id"] + if validate_uuid(chat_id): + return error_response(400, "invalid chat id", rid) + limit, limit_err = validate_limit(request.query_params.get("limit")) + if limit_err: + return error_response(400, limit_err, rid) + assert limit is not None + cursor_raw = request.query_params.get("cursor") + after = None + if cursor_raw is not None: + if not cursor_raw.strip(): + return error_response(400, "invalid cursor", rid) + try: + created_at, entity_id = decode_cursor(cursor_raw, chat_id) + except ValueError: + return error_response(400, "invalid cursor", rid) + after = (created_at, entity_id) + page, next_key, list_err = await store.list_messages(chat_id, user["id"], limit, after) + if list_err == "not_found": + return error_response(404, "chat not found", rid) + assert page is not None + next_cursor = None + if next_key is not None: + next_cursor = encode_cursor(chat_id, next_key[0], next_key[1]) + return _json({"items": page, "next_cursor": next_cursor}, rid) + + +async def lab_metrics(request: Request) -> Response: + if not config.lab_mode(): + return error_response(404, "not found", request.state.request_id) + return _json( + { + "singleflight": await cache.singleflight_enabled(), + "invalidate": await cache.invalidate_enabled(), + "history_sql": await cache.history_sql_count(), + }, + request.state.request_id, + ) + + +async def lab_flags(request: Request) -> Response: + if not config.lab_mode(): + return error_response(404, "not found", request.state.request_id) + raw = await request.body() + body, err = parse_json_body(raw) + if err: + return error_response(400, err, request.state.request_id) + assert body is not None + if body.get("reset_sql"): + await cache.reset_history_sql_count() + flags = await cache.set_flags( + singleflight=body.get("singleflight"), + invalidate=body.get("invalidate"), + delay_ms=body.get("delay_ms"), + ) + return _json(flags, request.state.request_id) diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..b0e30b11 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,56 @@ +import hashlib +import json +import secrets +from datetime import datetime, timedelta, timezone +from uuid import UUID + +from app import config, rediscli +from app.errors import AppError, BEARER_CHALLENGE + + +def new_token() -> str: + return secrets.token_urlsafe(32) + + +def _key(token: str) -> str: + return "sess:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _fmt(dt: datetime) -> str: + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +async def create(user_id: UUID) -> tuple[str, str]: + token = new_token() + expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + payload = json.dumps({"user_id": str(user_id), "expires_at": _fmt(expires)}) + try: + await rediscli.require().set(_key(token), payload, ex=config.SESSION_TTL_SECONDS) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + return token, _fmt(expires) + + +async def resolve(token: str) -> UUID: + try: + raw = await rediscli.require().get(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if not raw: + raise AppError(401, "authentication required", BEARER_CHALLENGE) + return UUID(json.loads(raw)["user_id"]) + + +async def revoke(token: str) -> None: + try: + deleted = await rediscli.require().delete(_key(token)) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if not deleted: + raise AppError(401, "authentication required", BEARER_CHALLENGE) diff --git a/app/store.py b/app/store.py new file mode 100644 index 00000000..04e37555 --- /dev/null +++ b/app/store.py @@ -0,0 +1,361 @@ +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone +from typing import List, Optional, Tuple +from uuid import UUID, uuid4 + +import asyncpg + +from app import cache, config, db +from app.errors import AppError + + +def format_ts(dt: datetime) -> str: + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def user_dict(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_ts(row["created_at"]), + } + + +def chat_dict(row, member_ids: List[str]) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": format_ts(row["created_at"]), + } + + +def message_dict(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_ts(row["created_at"]), + } + + +async def _run(coro): + try: + return await coro + except AppError: + raise + except db.INTEGRITY: + raise + except Exception: + raise AppError(503, "service unavailable") from None + + +async def create_user(username: str, display_name: str, password_hash: str) -> Tuple[Optional[dict], Optional[str]]: + user_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + await conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + user_id, + username, + display_name, + password_hash, + created, + ) + except asyncpg.UniqueViolationError: + return None, "conflict" + except Exception: + raise AppError(503, "service unavailable") from None + return user_dict({"id": user_id, "username": username, "display_name": display_name, "created_at": created}), None + + +async def get_user(user_id: str) -> Optional[dict]: + row = await _run( + _fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", UUID(user_id)) + ) + return user_dict(row) if row else None + + +async def get_user_by_username(username: str): + return await _run( + _fetchrow( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = $1", + username, + ) + ) + + +async def _fetchrow(sql: str, *args): + async with db.require().acquire() as conn: + return await conn.fetchrow(sql, *args) + + +async def _fetch(sql: str, *args): + async with db.require().acquire() as conn: + return await conn.fetch(sql, *args) + + +async def member_ids_of(chat_id: UUID) -> List[str]: + rows = await _run(_fetch("SELECT user_id FROM chat_members WHERE chat_id = $1 ORDER BY user_id", chat_id)) + return [str(r["user_id"]) for r in rows] + + +async def create_chat(title: str, member_ids: List[str], caller_id: str) -> Tuple[Optional[dict], Optional[str]]: + if caller_id not in member_ids: + return None, "caller not in members" + if len(set(member_ids)) != len(member_ids): + return None, "duplicate members" + ids = [UUID(x) for x in member_ids] + chat_id = uuid4() + created = datetime.now(timezone.utc) + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + existing = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", ids) + if len(existing) != len(ids): + return None, "unknown member" + await conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)", + chat_id, + title, + created, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, mid) for mid in ids], + ) + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + return chat_dict({"id": chat_id, "title": title, "created_at": created}, member_ids), None + + +async def get_chat(chat_id: str, user_id: str) -> Optional[dict]: + row = await load_chat_for_member(chat_id, user_id) + if row is None: + return None + return chat_dict(row, await member_ids_of(UUID(chat_id))) + + +async def load_chat_for_member(chat_id: str, user_id: str): + return await _run( + _fetchrow( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 + WHERE c.id = $1 + """, + UUID(chat_id), + UUID(user_id), + ) + ) + + +async def list_chats(user_id: str, limit: int, after: Optional[Tuple[str, str]] = None): + args: list = [UUID(user_id)] + clause = "" + if after is not None: + clause = "AND (c.created_at, c.id) > ($2, $3)" + args.extend([datetime.fromisoformat(after[0].replace("Z", "+00:00")), UUID(after[1])]) + rows = await _run( + _fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1 + WHERE 1=1 {clause} + ORDER BY c.created_at, c.id + LIMIT {int(limit) + 1} + """, + *args, + ) + ) + page = rows[:limit] + items = [chat_dict(row, await member_ids_of(row["id"])) for row in page] + next_key = None + if len(rows) > limit: + last = page[-1] + next_key = (format_ts(last["created_at"]), str(last["id"])) + return items, next_key + + +def _canonical_message(text: str) -> str: + # Missing attachment_ids and [] are the same message. Key order does not matter. + return json.dumps( + {"attachment_ids": [], "text": text}, + sort_keys=True, + ensure_ascii=False, + separators=(",", ":"), + ) + + +async def add_message( + chat_id: str, + sender_id: str, + text: str, + idem_key: str, + path: str, +) -> Tuple[Optional[dict], Optional[str]]: + """Insert one message or replay the stored 201. The row and the result commit together. + + Same user+method+path+key is serialized with a Postgres advisory lock so two + replicas cannot insert two messages. A failed transaction stores nothing. + """ + digest = hashlib.sha256(_canonical_message(text).encode("utf-8")).hexdigest() + lock_key = f"{sender_id}|POST|{path}|{idem_key}" + payload: Optional[dict] = None + created_new = False + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock(hashtextextended($1::text, 0))", lock_key) + member = await conn.fetchrow( + """ + SELECT c.id + FROM chats c + JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 + WHERE c.id = $1 + """, + UUID(chat_id), + UUID(sender_id), + ) + if member is None: + return None, "not_found" + row = await conn.fetchrow( + """ + SELECT request_hash, response_json, created_at + FROM idempotency_keys + WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3 + """, + UUID(sender_id), + path, + idem_key, + ) + if row is not None: + created_at = row["created_at"] + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + age = (datetime.now(timezone.utc) - created_at).total_seconds() + if age <= config.IDEMPOTENCY_TTL_SECONDS: + if row["request_hash"] != digest: + return None, "conflict" + return json.loads(row["response_json"]), None + await conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3 + """, + UUID(sender_id), + path, + idem_key, + ) + message_id = uuid4() + created = datetime.now(timezone.utc) + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + """, + message_id, + UUID(chat_id), + UUID(sender_id), + text, + created, + ) + payload = message_dict( + { + "id": message_id, + "chat_id": UUID(chat_id), + "sender_id": UUID(sender_id), + "text": text, + "created_at": created, + } + ) + await conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, http_method, path, idem_key, request_hash, response_json, created_at) + VALUES ($1, 'POST', $2, $3, $4, $5, $6) + """, + UUID(sender_id), + path, + idem_key, + digest, + json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + created, + ) + created_new = True + except AppError: + raise + except Exception: + raise AppError(503, "service unavailable") from None + if created_new: + await cache.invalidate_history(chat_id) + return payload, None + + +async def list_messages(chat_id: str, user_id: str, limit: int, after: Optional[Tuple[str, str]] = None): + if await load_chat_for_member(chat_id, user_id) is None: + return None, None, "not_found" + cursor = "" + after_ts = after_id = None + if after is not None: + cursor = after[0] + ":" + after[1] + after_ts = datetime.fromisoformat(after[0].replace("Z", "+00:00")) + after_id = UUID(after[1]) + + async def loader() -> dict: + await cache.incr_history_sql() + if after_ts is None: + rows = await _run( + _fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id + LIMIT $2 + """, + UUID(chat_id), + limit + 1, + ) + ) + else: + rows = await _run( + _fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT $4 + """, + UUID(chat_id), + after_ts, + after_id, + limit + 1, + ) + ) + page = rows[:limit] + items = [message_dict(r) for r in page] + next_key = None + if len(rows) > limit: + last = page[-1] + next_key = (format_ts(last["created_at"]), str(last["id"])) + return {"items": items, "next_key": next_key} + + page = await cache.get_history_page(chat_id, cursor, limit, loader) + return page["items"], page.get("next_key"), None diff --git a/app/tools/__init__.py b/app/tools/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/app/tools/__init__.py @@ -0,0 +1 @@ + diff --git a/app/tools/explain.py b/app/tools/explain.py new file mode 100644 index 00000000..e649ad14 --- /dev/null +++ b/app/tools/explain.py @@ -0,0 +1,73 @@ +"""Seed ≥10 000 messages and print EXPLAIN (ANALYZE, BUFFERS) for keyset pagination.""" +import asyncio +import os +from uuid import uuid4 + +import asyncpg + +from app.passwords import hash_password + + +async def main() -> None: + conn = await asyncpg.connect(dsn=os.environ["DATABASE_URL"], timeout=10, command_timeout=120) + try: + alice = uuid4() + bob = uuid4() + chat = uuid4() + pwd = hash_password("Example-only-not-a-real-secret-123") + await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1,$2,$3,$4),($5,$6,$7,$4)", + alice, + "explain_a_" + alice.hex[:8], + "Alice", + pwd, + bob, + "explain_b_" + bob.hex[:8], + "Bob", + ) + await conn.execute("INSERT INTO chats (id, title) VALUES ($1,$2)", chat, "Explain chat") + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1,$2),($1,$3)", + chat, + alice, + bob, + ) + n = int(os.environ.get("EXPLAIN_MESSAGES", "10000")) + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), $1, $2, 'seed-' || g, now() - (g || ' milliseconds')::interval + FROM generate_series(1, $3) AS g + """, + chat, + alice, + n, + ) + count = await conn.fetchval("SELECT count(*) FROM messages WHERE chat_id = $1", chat) + print("seeded_messages", count, "chat_id", chat) + first = await conn.fetchrow( + "SELECT created_at, id FROM messages WHERE chat_id = $1 ORDER BY created_at, id LIMIT 1", + chat, + ) + query = """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 AND (created_at, id) > ($2, $3) + ORDER BY created_at, id + LIMIT 50 + """ + print("\n=== WITH index messages_chat_created_id_idx ===") + plan = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"]) + print("\n".join(row[0] for row in plan)) + await conn.execute("SET enable_indexscan = off") + await conn.execute("SET enable_bitmapscan = off") + await conn.execute("SET enable_indexonlyscan = off") + print("\n=== WITHOUT index scans (seq scan baseline) ===") + plan2 = await conn.fetch("EXPLAIN (ANALYZE, BUFFERS) " + query, chat, first["created_at"], first["id"]) + print("\n".join(row[0] for row in plan2)) + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..9723267b --- /dev/null +++ b/app/validation.py @@ -0,0 +1,102 @@ +import json +import re +from typing import Any, Dict, Optional, Tuple +from uuid import UUID + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def parse_json_body(raw: bytes) -> Tuple[Optional[Dict[str, Any]], Optional[str]]: + if not raw: + return None, "empty body" + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None, "malformed JSON" + if not isinstance(data, dict): + return None, "JSON body must be an object" + return data, None + + +def require_exact_keys(data: Dict[str, Any], allowed: set, required: set) -> Optional[str]: + keys = set(data.keys()) + if keys - allowed: + return "unexpected fields" + if required - keys: + return "missing required fields" + return None + + +def validate_username(value: Any) -> Optional[str]: + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + return "invalid username" + return None + + +def validate_display_name(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid display_name" + if not value.strip(): + return "display_name must not be blank" + if len(value) > 100: + return "display_name too long" + return None + + +def validate_password(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid password" + if not value.strip(): + return "password must not be blank" + if len(value) < 12 or len(value) > 128: + return "password length" + return None + + +def validate_title(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid title" + if not value.strip(): + return "title must not be blank" + if len(value) > 100: + return "title too long" + return None + + +def validate_uuid(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid uuid" + try: + UUID(value) + except ValueError: + return "invalid uuid" + return None + + +def validate_idempotency_key(value: Optional[str]) -> Optional[str]: + if value is None or IDEMPOTENCY_RE.fullmatch(value) is None: + return "invalid idempotency key" + return None + + +def validate_message_text(value: Any) -> Optional[str]: + if not isinstance(value, str): + return "invalid text" + if not value.strip(): + return "text must not be blank" + if len(value) > 4000: + return "text too long" + return None + + +def validate_limit(value: Optional[str], default: int = 50) -> Tuple[Optional[int], Optional[str]]: + if value is None: + return default, None + try: + limit = int(value) + except ValueError: + return None, "invalid limit" + if limit < 1 or limit > 100: + return None, "limit out of range" + return limit, None diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..2501deea --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,49 @@ +-- Lab2 schema (written before application queries). +-- Source of truth: PostgreSQL for users, chats, membership, messages. +-- Transaction boundary: INSERT chat + INSERT members in one transaction (rollback together). +-- Sessions are stored in Redis with AOF, not in this database. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + filename TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE UNIQUE INDEX users_username_key ON users (username); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id); +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset history: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..acc555c6 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,14 @@ +-- Shared idempotency for POST /messages. Scope: user + method + path + key. +-- The message row and this result are committed in one transaction. +-- A rolled-back request leaves no row, so a later retry with the same key can insert. + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users (id), + http_method TEXT NOT NULL, + path TEXT NOT NULL, + idem_key TEXT NOT NULL, + request_hash TEXT NOT NULL, + response_json TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, http_method, path, idem_key) +); diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..ce023a9a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = student_tests +pythonpath = . +markers = + infra: compose stop/restart of redis or postgres + expiry: requires SESSION_TTL_SECONDS<=3 diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..f7f80320 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,2 @@ +pytest==8.3.4 +httpx==0.28.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..4a028ebb --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +starlette==0.41.3 +uvicorn[standard]==0.32.1 +asyncpg==0.30.0 +redis==5.2.1 +argon2-cffi==23.1.0 diff --git a/scripts/run_expiry.py b/scripts/run_expiry.py new file mode 100644 index 00000000..42d87200 --- /dev/null +++ b/scripts/run_expiry.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Recreate API with SESSION_TTL_SECONDS=2, test expiry, restore TTL.""" +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(args, extra_env=None): + env = os.environ.copy() + if extra_env: + env.update(extra_env) + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, env=env) + if proc.returncode != 0: + raise SystemExit(proc.returncode) + + +def wait_ready(): + import urllib.request + + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2) + return + except Exception: + time.sleep(0.5) + raise SystemExit("API did not become ready") + + +def main(): + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "2"}) + wait_ready() + env = os.environ.copy() + env["SESSION_TTL_SECONDS"] = "2" + env["RUN_EXPIRY"] = "1" + proc = subprocess.run( + [sys.executable, "-m", "pytest", "student_tests/test_expiry.py", "-q"], + cwd=ROOT, + env=env, + ) + compose(["up", "-d", "--no-deps", "--force-recreate", "api"], {"SESSION_TTL_SECONDS": "3600"}) + wait_ready() + raise SystemExit(proc.returncode) + + +if __name__ == "__main__": + main() diff --git a/student_tests/__init__.py b/student_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/student_tests/conftest.py b/student_tests/conftest.py new file mode 100644 index 00000000..ae6a3291 --- /dev/null +++ b/student_tests/conftest.py @@ -0,0 +1,24 @@ +import os +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +@pytest.fixture(scope="session") +def base_url(): + return os.environ.get("BASE_URL", "http://127.0.0.1:8080") + + +@pytest.fixture(scope="session") +def live(base_url): + c = Client(base_url, timeout=15) + try: + c.call("GET", "/health/ready", schema="Health") + except Failure as exc: + pytest.skip("API is not ready: %s" % exc) + return c diff --git a/student_tests/test_api.py b/student_tests/test_api.py new file mode 100644 index 00000000..60971ea1 --- /dev/null +++ b/student_tests/test_api.py @@ -0,0 +1,195 @@ +"""Negative, pagination and auth tests for lab 2.""" +import uuid +from urllib.parse import quote + +from client import Client + + +PASSWORD = "Example-only-not-a-real-secret-123" + + +def register(c: Client, username=None): + username = username or ("u_" + uuid.uuid4().hex[:10]) + user = c.call( + "POST", + "/api/v1/users", + 201, + body={"username": username, "display_name": "Test User", "password": PASSWORD}, + schema="User", + )["json"] + return user + + +def login(c: Client, username, password=PASSWORD): + import base64 + + cred = base64.b64encode((username + ":" + password).encode()).decode() + session = c.call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + cred}, + schema="Session", + )["json"] + return {"id": session["user"]["id"], "user": session["user"], "token": session["token"], "registration": {"username": username, "password": password}} + + +def actor_of(c: Client, username=None): + user = register(c, username) + return login(c, user["username"]) + + +def test_malformed_json(live: Client): + response = live.call( + "POST", + "/api/v1/users", + 400, + raw=b"{not-json", + headers={"Content-Type": "application/json"}, + ) + assert response["json"]["error"]["code"] == "invalid_request" + assert response["headers"].get("X-Request-Id") + + +def test_duplicate_username(live: Client): + user = register(live, username="dup_" + uuid.uuid4().hex[:8]) + again = live.call( + "POST", + "/api/v1/users", + 409, + body={"username": user["username"], "display_name": "Other", "password": PASSWORD}, + ) + assert again["json"]["error"]["code"] == "conflict" + + +def test_empty_message_rejected(live: Client): + alice = actor_of(live) + bob = actor_of(live) + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + response = live.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=alice, + body={"text": " "}, + ) + assert response["status"] == 400 + + +def test_outsider_chat_is_404(live: Client): + alice = actor_of(live) + bob = actor_of(live) + eve = actor_of(live) + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "secret", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + response = live.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve) + assert response["json"]["error"]["code"] == "not_found" + + +def test_message_pagination_multiple_pages(live: Client): + alice = actor_of(live) + bob = actor_of(live) + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "pages", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + created = [] + for idx in range(5): + created.append(live.message(alice, chat, "m%s" % idx)["json"]) + collected = [] + cursor = None + for _ in range(10): + path = "/api/v1/chats/%s/messages?limit=2" % chat["id"] + if cursor: + path += "&cursor=" + quote(cursor, safe="") + page = live.call("GET", path, actor=bob, schema="MessagePage")["json"] + collected.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + assert len(collected) == 5 + ordered = sorted(created, key=lambda x: (x["created_at"], x["id"])) + assert collected == ordered + + +def test_cursor_from_other_chat_rejected(live: Client): + alice = actor_of(live) + bob = actor_of(live) + chat_a = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "a", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + chat_b = live.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "b", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + msg = live.message(alice, chat_a, "only in a")["json"] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=1" % chat_a["id"], + actor=alice, + schema="MessagePage", + )["json"] + assert page["items"][0]["id"] == msg["id"] + cursor = page["next_cursor"] + # If history is a single item, next_cursor is null; encode a foreign cursor via first page of A after a dummy extra message. + if cursor is None: + live.message(alice, chat_a, "second") + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=1" % chat_a["id"], + actor=alice, + schema="MessagePage", + )["json"] + cursor = page["next_cursor"] + response = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=10&cursor=%s" % (chat_b["id"], quote(cursor, safe="")), + 400, + actor=alice, + ) + assert response["status"] == 400 + + +def test_x_user_id_does_not_authenticate(live: Client): + alice = actor_of(live) + live.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]}) + + +def test_wrong_password(live: Client): + user = register(live) + import base64 + + bad = base64.b64encode((user["username"] + ":wrong-password-xx").encode()).decode() + live.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + bad}) + + +def test_logout(live: Client): + alice = actor_of(live) + live.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + live.call("GET", "/api/v1/users/me", 401, actor=alice) diff --git a/student_tests/test_concurrent.py b/student_tests/test_concurrent.py new file mode 100644 index 00000000..5265031e --- /dev/null +++ b/student_tests/test_concurrent.py @@ -0,0 +1,48 @@ +"""10 clients × 10 messages = 100 unique messages.""" +import threading +import uuid + +from client import Client + +from student_tests.test_api import actor_of + + +def test_concurrent_messages(live: Client): + users = [actor_of(live) for _ in range(10)] + member_ids = [u["id"] for u in users] + chat = live.call( + "POST", + "/api/v1/chats", + 201, + actor=users[0], + body={"title": "load", "member_ids": member_ids}, + schema="Chat", + )["json"] + + errors = [] + lock = threading.Lock() + + def worker(user): + local = Client(live.base, timeout=15) + for _ in range(10): + try: + local.message(user, chat, uuid.uuid4().hex) + except Exception as exc: + with lock: + errors.append(str(exc)) + + threads = [threading.Thread(target=worker, args=(u,)) for u in users] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=60) + + assert not errors, "unexpected failures: %s" % errors[:5] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=users[0], + schema="MessagePage", + )["json"] + ids = [item["id"] for item in page["items"]] + assert len(ids) == len(set(ids)) == 100 diff --git a/student_tests/test_expiry.py b/student_tests/test_expiry.py new file mode 100644 index 00000000..da31f3c9 --- /dev/null +++ b/student_tests/test_expiry.py @@ -0,0 +1,17 @@ +import os +import time + +import pytest +from client import Client + + +@pytest.mark.expiry +def test_session_expires(live: Client): + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + if os.environ.get("RUN_EXPIRY") != "1": + pytest.skip("set RUN_EXPIRY=1 and SESSION_TTL_SECONDS<=3") + assert ttl <= 3 + actor = live.user("ttl") + live.call("GET", "/api/v1/users/me", actor=actor, schema="User") + time.sleep(ttl + 1.5) + live.call("GET", "/api/v1/users/me", 401, actor=actor) diff --git a/student_tests/test_grade4.py b/student_tests/test_grade4.py new file mode 100644 index 00000000..92ceb895 --- /dev/null +++ b/student_tests/test_grade4.py @@ -0,0 +1,44 @@ +"""Grade-4: username race and atomic chat rollback.""" +import threading +import uuid + +from client import Client + +from student_tests.test_api import PASSWORD, actor_of + + +def test_register_race(live: Client): + username = "r_" + uuid.uuid4().hex[:12] + body = {"username": username, "display_name": "Race", "password": PASSWORD} + results = [] + + def once(): + local = Client(live.base, timeout=20) + try: + results.append(local.call("POST", "/api/v1/users", [201, 409], body=body)["status"]) + except Exception as exc: + results.append(str(exc)) + + threads = [threading.Thread(target=once) for _ in range(16)] + for t in threads: + t.start() + for t in threads: + t.join() + assert results.count(201) == 1, results + assert results.count(409) == 15, results + + +def test_chat_rollback(live: Client): + alice = actor_of(live) + before = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + missing = str(uuid.uuid4()) + live.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + body={"title": "rollback-should-not-exist", "member_ids": [alice["id"], missing]}, + ) + after = live.call("GET", "/api/v1/chats?limit=100", actor=alice, schema="ChatPage")["json"]["items"] + assert len(after) == len(before) + assert all(x["title"] != "rollback-should-not-exist" for x in after) diff --git a/student_tests/test_infra.py b/student_tests/test_infra.py new file mode 100644 index 00000000..fe1a022e --- /dev/null +++ b/student_tests/test_infra.py @@ -0,0 +1,89 @@ +"""Dependency failure and Redis restart (AOF + cache rebuild).""" +import os +import subprocess +import time +from pathlib import Path + +import pytest +from client import Client, Failure + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args): + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + if proc.returncode != 0: + raise AssertionError("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or "")[-500:])) + return proc + + +def wait_ready(c: Client, seconds=90): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = exc + time.sleep(0.5) + raise AssertionError("ready not restored: %s" % last) + + +def wait_status(c: Client, path, expected, seconds=30, actor=None): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + schema = "Health" if expected == 200 and path.startswith("/health") else None + c.call("GET", path, expected, actor=actor, schema=schema) + return + except Failure as exc: + last = exc + time.sleep(0.4) + raise AssertionError("did not observe %s for %s: %s" % (expected, path, last)) + + +@pytest.mark.infra +def test_redis_and_postgres_unavailable(live: Client): + if os.environ.get("RUN_INFRA") != "1": + pytest.skip("set RUN_INFRA=1 to stop compose services") + alice = live.user("infra") + compose("stop", "redis") + wait_status(live, "/health/live", 200) + wait_status(live, "/health/ready", 503) + wait_status(live, "/api/v1/users/me", 503, actor=alice) + compose("start", "redis") + wait_ready(live) + live.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + compose("stop", "postgres") + wait_status(live, "/health/live", 200) + wait_status(live, "/health/ready", 503) + compose("start", "postgres") + wait_ready(live) + live.call("GET", "/api/v1/users/me", actor=alice, schema="User") + + +@pytest.mark.infra +def test_redis_restart_keeps_session_and_history(live: Client): + if os.environ.get("RUN_INFRA") != "1": + pytest.skip("set RUN_INFRA=1 to restart redis") + alice, bob, _eve, chat = live.bootstrap() + msg = live.message(alice, chat, "before-redis-restart")["json"] + compose("restart", "redis") + wait_ready(live) + me = live.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"] + assert me["id"] == alice["id"] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=bob, + schema="MessagePage", + )["json"] + assert any(m["id"] == msg["id"] for m in page["items"]) diff --git a/student_tests/test_track1.py b/student_tests/test_track1.py new file mode 100644 index 00000000..d71c3192 --- /dev/null +++ b/student_tests/test_track1.py @@ -0,0 +1,83 @@ +"""Track 1: cache consistency — stampede, stale read, read-after-write.""" +import json +import threading +import uuid +from concurrent.futures import ThreadPoolExecutor + +from client import Client + + +def _lab_metrics(c: Client): + return json.loads(c.call("GET", "/api/v1/_lab/metrics")["bytes"].decode()) + + +def _lab_flags(c: Client, body): + return json.loads(c.call("POST", "/api/v1/_lab/flags", 200, body=body)["bytes"].decode()) + + +def test_read_after_write(live: Client): + alice, bob, _eve, chat = live.bootstrap() + _lab_flags(live, {"invalidate": True, "singleflight": True, "reset_sql": True}) + first = live.message(alice, chat, "raw-visible-" + uuid.uuid4().hex)["json"] + page = live.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=bob, + schema="MessagePage", + )["json"] + assert any(m["id"] == first["id"] for m in page["items"]) + + +def test_stale_without_invalidation_then_fixed(live: Client): + alice, bob, _eve, chat = live.bootstrap() + path = "/api/v1/chats/%s/messages?limit=50" % chat["id"] + live.call("GET", path, actor=alice, schema="MessagePage") + _lab_flags(live, {"invalidate": False, "singleflight": True}) + try: + stale_text = "stale-probe-" + uuid.uuid4().hex + live.message(alice, chat, stale_text) + page = live.call("GET", path, actor=bob, schema="MessagePage")["json"] + assert all(m["text"] != stale_text for m in page["items"]) + finally: + _lab_flags(live, {"invalidate": True}) + fresh_text = "fresh-probe-" + uuid.uuid4().hex + live.message(alice, chat, fresh_text) + page2 = live.call("GET", path, actor=bob, schema="MessagePage")["json"] + assert any(m["text"] == fresh_text for m in page2["items"]) + + +def test_stampede_singleflight_reduces_sql(live: Client): + alice, _bob, _eve, chat = live.bootstrap() + live.message(alice, chat, "stampede-seed") + path = "/api/v1/chats/%s/messages?limit=50" % chat["id"] + + def burst(n=24): + barrier = threading.Barrier(n) + + def one(_i): + local = Client(live.base) + barrier.wait() + return local.call("GET", path, actor=alice, schema="MessagePage") + + with ThreadPoolExecutor(max_workers=n) as pool: + list(pool.map(one, range(n))) + + try: + _lab_flags(live, {"singleflight": False, "invalidate": True, "reset_sql": True, "delay_ms": 250}) + live.message(alice, chat, "miss-" + uuid.uuid4().hex) + _lab_flags(live, {"reset_sql": True, "singleflight": False, "delay_ms": 250}) + burst() + unprotected = _lab_metrics(live)["history_sql"] + + _lab_flags(live, {"singleflight": True, "reset_sql": True, "delay_ms": 250}) + live.message(alice, chat, "miss2-" + uuid.uuid4().hex) + _lab_flags(live, {"reset_sql": True, "singleflight": True, "delay_ms": 250}) + burst() + protected = _lab_metrics(live)["history_sql"] + finally: + _lab_flags(live, {"invalidate": True, "singleflight": True, "delay_ms": 0}) + + print(json.dumps({"stampede_sql_without_singleflight": unprotected, "stampede_sql_with_singleflight": protected})) + assert unprotected >= 8, unprotected + assert protected < unprotected, (protected, unprotected) + assert protected <= 4, protected -- 2.54.0 From fd54119d3fbd436229d14bcd40d64731086ed058 Mon Sep 17 00:00:00 2001 From: David Date: Sat, 7 Nov 2026 12:46:37 +0300 Subject: [PATCH 2/4] =?UTF-8?q?=D0=9F=D0=BE=D1=81=D1=82=D0=B0=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D0=B8=20API?= =?UTF-8?q?=20=D0=B7=D0=B0=20HAProxy=20=D0=B8=20=D0=BE=D0=BF=D0=B8=D1=81?= =?UTF-8?q?=D0=B0=D1=82=D1=8C=20=D0=BE=D1=81=D1=82=D0=B0=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D0=BA=D1=83=20=D0=BE=D0=B4=D0=BD=D0=BE=D0=B9=20=D0=B8=D0=B7=20?= =?UTF-8?q?=D0=BD=D0=B8=D1=85.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Внешний порт остаётся только у балансировщика, миграции применяются один раз, а stop-one оставляет выбранную реплику остановленной. --- .env.example | 16 ++++- compose.yaml | 131 ++++++++++++++++++++++++++++++++++++++ infra/haproxy.cfg | 35 ++++++++++ scripts/restart.bat | 3 + scripts/restart.py | 20 ++++++ scripts/restart.sh | 10 +++ scripts/rolling-update.sh | 55 ++++++++++++++++ scripts/stop-one.sh | 32 ++++++++++ 8 files changed, 301 insertions(+), 1 deletion(-) create mode 100644 compose.yaml create mode 100644 infra/haproxy.cfg create mode 100644 scripts/restart.bat create mode 100644 scripts/restart.py create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh diff --git a/.env.example b/.env.example index d32b9be0..f6739e98 100644 --- a/.env.example +++ b/.env.example @@ -3,11 +3,25 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +LOG_LEVEL=info +SHUTDOWN_GRACE_SECONDS=10 SESSION_TTL_SECONDS=3600 +CACHE_TTL_SECONDS=60 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 +API_CPUS=0.50 +API_MEM=256m +POSTGRES_CPUS=1.0 +POSTGRES_MEM=512m +DB_POOL_MAX=4 +LB_DRAIN_SECONDS=2 IDEMPOTENCY_TTL_SECONDS=86400 + +# Compose interpolates these. Synthetic local values only; replace after copy. +POSTGRES_PASSWORD=lab2-dev-only-change-me +DATABASE_URL=postgresql://messenger:lab2-dev-only-change-me@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..a1ee4f8c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,131 @@ +# Client -> lb:8080 -> api (scale) -> shared Postgres and Redis. +# API replicas do not publish host ports. Migrations run once in `migrate`. +# Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead +# or not-ready replica is about 2s, plus the 1s DNS hold. API on SIGTERM +# fails /health/ready immediately and exits after LB_DRAIN_SECONDS (default 2). + +services: + migrate: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab3 + command: ["python", "-m", "app.migrate"] + environment: + APP_ENV: lab + LOG_LEVEL: info + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + DB_POOL_MIN: "1" + DB_POOL_MAX: "2" + MIGRATIONS_ON_START: "1" + depends_on: + postgres: + condition: service_healthy + restart: "no" + mem_limit: 256m + cpus: "0.50" + + api: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab3 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + LOG_LEVEL: info + SHUTDOWN_GRACE_SECONDS: "3" + LB_DRAIN_SECONDS: ${LB_DRAIN_SECONDS:-2} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-60} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: redis + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + REDIS_URL: redis://redis:6379/0 + DB_POOL_MIN: "1" + DB_POOL_MAX: ${DB_POOL_MAX:-4} + DB_COMMAND_TIMEOUT: "5" + DB_CONNECT_TIMEOUT: "5" + MIGRATIONS_ON_START: "0" + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + expose: + - "8080" + restart: unless-stopped + mem_limit: ${API_MEM:-256m} + cpus: ${API_CPUS:-0.50} + stop_grace_period: 8s + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)", + ] + interval: 5s + timeout: 3s + retries: 6 + start_period: 5s + + lb: + image: haproxy:2.8.14-alpine + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + api: + condition: service_started + restart: unless-stopped + mem_limit: 128m + cpus: "0.25" + + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: messenger + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: ${POSTGRES_MEM:-512m} + cpus: ${POSTGRES_CPUS:-1.0} + + redis: + image: redis:7.4.2-alpine + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --save + - "60" + - "1" + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + mem_limit: 128m + cpus: "0.50" + +volumes: + pg_data: + redis_data: diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..24c99b80 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 info + maxconn 200 + +defaults + log global + mode http + option httplog + timeout connect 2s + timeout client 10s + timeout server 10s + timeout http-request 5s + timeout check 2s + timeout queue 2s + retries 1 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + accepted_payload_size 8192 + +frontend fe_http + bind *:8080 + default_backend be_api + +backend be_api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr host api + http-check expect status 200 + default-server inter 1s fall 2 rise 1 + server-template srv 8 api:8080 check resolvers docker init-addr none resolve-prefer ipv4 diff --git a/scripts/restart.bat b/scripts/restart.bat new file mode 100644 index 00000000..d96ae9d7 --- /dev/null +++ b/scripts/restart.bat @@ -0,0 +1,3 @@ +@echo off +python "%~dp0restart.py" +exit /b %ERRORLEVEL% diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..639651e8 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Restart API, PostgreSQL and Redis without removing named volumes.""" +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def main() -> int: + env_file = ROOT / ".env" + cmd = ["docker", "compose"] + if env_file.is_file(): + cmd.extend(["--env-file", str(env_file)]) + cmd.extend(["restart", "postgres", "redis", "api"]) + return subprocess.run(cmd, cwd=ROOT).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..6a3be3ad --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Restart API replicas and every store they use. Volumes are kept. +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +if [ -f .env ]; then + docker compose --env-file .env restart postgres redis api +else + docker compose restart postgres redis api +fi diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..d22d4a10 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Replace API replicas one at a time. The other replica keeps serving. +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +compose() { + if [ -f .env ]; then + docker compose --env-file .env "$@" + else + docker compose "$@" + fi +} + +if command -v python3 >/dev/null 2>&1; then + PY=python3 +else + PY=python +fi + +ids=$(compose ps -q api) +count=0 +for id in $ids; do + count=$((count + 1)) +done +if [ "$count" -lt 2 ]; then + echo "rolling update needs at least 2 running API replicas" >&2 + exit 1 +fi + +for id in $ids; do + echo "replacing $(docker inspect -f '{{.Config.Hostname}}' "$id")" + docker stop -t 8 "$id" >/dev/null + docker rm "$id" >/dev/null + compose up -d --scale "api=$count" --no-deps --no-recreate api >/dev/null + ready=0 + i=0 + while [ "$i" -lt 40 ]; do + running=0 + for cid in $(compose ps -q api); do + running=$((running + 1)) + done + if [ "$running" -ge "$count" ] && "$PY" -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" >/dev/null 2>&1; then + ready=1 + break + fi + i=$((i + 1)) + sleep 1 + done + if [ "$ready" -ne 1 ]; then + echo "replacement did not become ready" >&2 + exit 1 + fi +done +echo "rolling update finished; replicas=$count" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..707e422d --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Stop the single API replica whose X-Instance-Id is TARGET_INSTANCE_ID. +# Leaves it stopped. Does not touch the database, Redis, the load balancer, or volumes. +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +compose() { + if [ -f .env ]; then + docker compose --env-file .env "$@" + else + docker compose "$@" + fi +} + +found=0 +for id in $(compose ps -q api); do + iid=$(docker exec "$id" cat /tmp/instance-id 2>/dev/null | tr -d '\r\n' || true) + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + if [ "$iid" = "$TARGET_INSTANCE_ID" ] || [ "$host" = "$TARGET_INSTANCE_ID" ]; then + docker stop -t 5 "$id" >/dev/null + found=1 + break + fi +done + +if [ "$found" -ne 1 ]; then + echo "no running API replica matched TARGET_INSTANCE_ID" >&2 + exit 1 +fi -- 2.54.0 From 99ce1b1391a6bf6736e0eb7c7ae694e6f85ff83b Mon Sep 17 00:00:00 2001 From: David Date: Sat, 7 Nov 2026 12:46:37 +0300 Subject: [PATCH 3/4] =?UTF-8?q?=D0=98=D0=B7=D0=BC=D0=B5=D1=80=D0=B8=D1=82?= =?UTF-8?q?=D1=8C=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=20=D0=BA=D0=B0=D0=B6?= =?UTF-8?q?=D0=B4=D0=BE=D0=B9=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D0=B8?= =?UTF-8?q?=20=D0=B8=20=D1=86=D0=B5=D0=BD=D1=83=20=D0=B4=D0=BE=D0=B1=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=D0=B8=D1=8F=20=D0=BF=D1=80=D0=BE=D1=86?= =?UTF-8?q?=D0=B5=D1=81=D1=81=D0=BE=D0=B2.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Один и тот же профиль нагрузки сравнивает 1, 2 и 4 реплики при фиксированном и растущем бюджете CPU. --- evidence/grade4/rolling-load.json | 17 ++ evidence/grade4/summary.json | 70 ++++++ evidence/public-checks.txt | 29 +++ evidence/track1/A1.json | 17 ++ evidence/track1/A1.point.json | 38 ++++ evidence/track1/A2.json | 17 ++ evidence/track1/A2.point.json | 39 ++++ evidence/track1/A4.json | 17 ++ evidence/track1/A4.point.json | 41 ++++ evidence/track1/B1.json | 17 ++ evidence/track1/B1.point.json | 38 ++++ evidence/track1/B2.json | 17 ++ evidence/track1/B2.point.json | 39 ++++ evidence/track1/B4.json | 17 ++ evidence/track1/B4.point.json | 41 ++++ evidence/track1/C4-pg2.json | 17 ++ evidence/track1/C4-pg2.point.json | 41 ++++ evidence/track1/D4-pool2.json | 17 ++ evidence/track1/D4-pool2.point.json | 41 ++++ evidence/track1/rps.svg | 29 +++ evidence/track1/summary.json | 320 ++++++++++++++++++++++++++++ scripts/grade4_evidence.py | 229 ++++++++++++++++++++ scripts/track1_experiment.py | 217 +++++++++++++++++++ 23 files changed, 1365 insertions(+) create mode 100644 evidence/grade4/rolling-load.json create mode 100644 evidence/grade4/summary.json create mode 100644 evidence/public-checks.txt create mode 100644 evidence/track1/A1.json create mode 100644 evidence/track1/A1.point.json create mode 100644 evidence/track1/A2.json create mode 100644 evidence/track1/A2.point.json create mode 100644 evidence/track1/A4.json create mode 100644 evidence/track1/A4.point.json create mode 100644 evidence/track1/B1.json create mode 100644 evidence/track1/B1.point.json create mode 100644 evidence/track1/B2.json create mode 100644 evidence/track1/B2.point.json create mode 100644 evidence/track1/B4.json create mode 100644 evidence/track1/B4.point.json create mode 100644 evidence/track1/C4-pg2.json create mode 100644 evidence/track1/C4-pg2.point.json create mode 100644 evidence/track1/D4-pool2.json create mode 100644 evidence/track1/D4-pool2.point.json create mode 100644 evidence/track1/rps.svg create mode 100644 evidence/track1/summary.json create mode 100644 scripts/grade4_evidence.py create mode 100644 scripts/track1_experiment.py diff --git a/evidence/grade4/rolling-load.json b/evidence/grade4/rolling-load.json new file mode 100644 index 00000000..2e6941a8 --- /dev/null +++ b/evidence/grade4/rolling-load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 50.0, + "elapsed_seconds": 50.101, + "concurrency": 4, + "successful_requests": 8361, + "failed_requests": 18, + "error_ratio": 0.002148, + "successful_rps": 166.884, + "successful_latency_ms": { + "p50": 13.481, + "p95": 57.817, + "p99": 96.441 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/grade4/summary.json b/evidence/grade4/summary.json new file mode 100644 index 00000000..7893768b --- /dev/null +++ b/evidence/grade4/summary.json @@ -0,0 +1,70 @@ +{ + "instances_before": [ + "03c3a0545628", + "b555f8327eee" + ], + "parallel_idempotency": { + "workers": 16, + "successes": 16, + "errors": 0, + "distinct_message_ids": 1, + "history_copies": 1 + }, + "failovers": [ + { + "target": "03c3a0545628", + "hook_exit": 0, + "hook_seconds": 2.373, + "hook_stderr": "", + "failed_attempts": 2, + "successes_after_hook_start": 130, + "first_recovery_seconds": 0.034, + "tail_instance_ids": [ + "b555f8327eee", + "b555f8327eee", + "b555f8327eee", + "b555f8327eee", + "b555f8327eee" + ], + "target_still_in_tail": false + }, + { + "target": "b555f8327eee", + "hook_exit": 0, + "hook_seconds": 1.744, + "hook_stderr": "", + "failed_attempts": 1, + "successes_after_hook_start": 132, + "first_recovery_seconds": 0.038, + "tail_instance_ids": [ + "03c3a0545628", + "03c3a0545628", + "03c3a0545628", + "03c3a0545628", + "03c3a0545628" + ], + "target_still_in_tail": false + } + ], + "rolling": { + "rolling_seconds": 5.929, + "rolling_stdout": "replacing b555f8327eee\nreplacing 03c3a0545628\nrolling update finished; replicas=2\n", + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 50.0, + "elapsed_seconds": 50.101, + "concurrency": 4, + "successful_requests": 8361, + "failed_requests": 18, + "error_ratio": 0.002148, + "successful_rps": 166.884, + "successful_latency_ms": { + "p50": 13.481, + "p95": 57.817, + "p99": 96.441 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + } + } +} diff --git a/evidence/public-checks.txt b/evidence/public-checks.txt new file mode 100644 index 00000000..711ae329 --- /dev/null +++ b/evidence/public-checks.txt @@ -0,0 +1,29 @@ +Date: 2026-11-07 +Host: 12th Gen Intel Core i5-12450H, Docker Desktop ncpu=12, MemTotal=8181665792 +Engine: Docker 29.5.3, Compose 5.1.4 +Stack: haproxy:2.8.14-alpine, mtusi-messenger:lab3 (python:3.12.8-slim-bookworm), postgres:16.6-alpine, redis:7.4.2-alpine + +python -X utf8 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. + +python -X utf8 -m pytest student_tests -q -m "not infra and not expiry" +15 passed, 3 deselected + +Official persistence and failover were executed by Linux python3 in docker:27-cli +with the host Docker socket and BASE_URL=http://host.docker.internal:8080, +because Windows CreateProcess cannot run scripts/restart.sh (WinError 193). +The hooks and the API containers are the host stack. + +python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh +{"observed_instances_before": 2, "successful_writes": 109, "failed_attempts": 1, "first_recovery_seconds": 2.465} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +After that failover, docker inspect showed mtusi2026lab3-api-1 Exited (0), +hostname b555f8327eee. mtusi2026lab3-api-2 stayed up, hostname 03c3a0545628. +Exit code 0 means the process left on SIGTERM before the stop timeout. diff --git a/evidence/track1/A1.json b/evidence/track1/A1.json new file mode 100644 index 00000000..81e232fe --- /dev/null +++ b/evidence/track1/A1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.109, + "concurrency": 8, + "successful_requests": 5197, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 343.964, + "successful_latency_ms": { + "p50": 19.732, + "p95": 42.061, + "p99": 81.001 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/A1.point.json b/evidence/track1/A1.point.json new file mode 100644 index 00000000..3a986495 --- /dev/null +++ b/evidence/track1/A1.point.json @@ -0,0 +1,38 @@ +{ + "tag": "A1", + "replicas": 1, + "api_cpus_each": "1.0", + "api_mem_each": "512m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 8, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.109, + "concurrency": 8, + "successful_requests": 5197, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 343.964, + "successful_latency_ms": { + "p50": 19.732, + "p95": 42.061, + "p99": 81.001 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB", + "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/A2.json b/evidence/track1/A2.json new file mode 100644 index 00000000..f228ca31 --- /dev/null +++ b/evidence/track1/A2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.206, + "concurrency": 8, + "successful_requests": 2985, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 196.3, + "successful_latency_ms": { + "p50": 26.613, + "p95": 89.893, + "p99": 123.339 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/A2.point.json b/evidence/track1/A2.point.json new file mode 100644 index 00000000..bb805c1c --- /dev/null +++ b/evidence/track1/A2.point.json @@ -0,0 +1,39 @@ +{ + "tag": "A2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.206, + "concurrency": 8, + "successful_requests": 2985, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 196.3, + "successful_latency_ms": { + "p50": 26.613, + "p95": 89.893, + "p99": 123.339 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB", + "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB", + "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB", + "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/A4.json b/evidence/track1/A4.json new file mode 100644 index 00000000..7eefffc6 --- /dev/null +++ b/evidence/track1/A4.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.224, + "concurrency": 8, + "successful_requests": 1895, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 124.476, + "successful_latency_ms": { + "p50": 39.136, + "p95": 185.654, + "p99": 267.62 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/A4.point.json b/evidence/track1/A4.point.json new file mode 100644 index 00000000..87e0fcf0 --- /dev/null +++ b/evidence/track1/A4.point.json @@ -0,0 +1,41 @@ +{ + "tag": "A4", + "replicas": 4, + "api_cpus_each": "0.25", + "api_mem_each": "128m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.224, + "concurrency": 8, + "successful_requests": 1895, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 124.476, + "successful_latency_ms": { + "p50": 39.136, + "p95": 185.654, + "p99": 267.62 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB", + "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB", + "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB", + "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB", + "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB", + "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB", + "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/B1.json b/evidence/track1/B1.json new file mode 100644 index 00000000..af7b2ad4 --- /dev/null +++ b/evidence/track1/B1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.139, + "concurrency": 8, + "successful_requests": 1485, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 98.094, + "successful_latency_ms": { + "p50": 85.074, + "p95": 167.209, + "p99": 201.857 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/B1.point.json b/evidence/track1/B1.point.json new file mode 100644 index 00000000..20b66ffc --- /dev/null +++ b/evidence/track1/B1.point.json @@ -0,0 +1,38 @@ +{ + "tag": "B1", + "replicas": 1, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 0.5, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 4, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.139, + "concurrency": 8, + "successful_requests": 1485, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 98.094, + "successful_latency_ms": { + "p50": 85.074, + "p95": 167.209, + "p99": 201.857 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB", + "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB", + "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB", + "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB" + ], + "postgres_sessions": "5" + } +} diff --git a/evidence/track1/B2.json b/evidence/track1/B2.json new file mode 100644 index 00000000..393fe4c9 --- /dev/null +++ b/evidence/track1/B2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.136, + "concurrency": 8, + "successful_requests": 3388, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 223.842, + "successful_latency_ms": { + "p50": 24.1, + "p95": 81.009, + "p99": 142.445 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/B2.point.json b/evidence/track1/B2.point.json new file mode 100644 index 00000000..10cd6d82 --- /dev/null +++ b/evidence/track1/B2.point.json @@ -0,0 +1,39 @@ +{ + "tag": "B2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.136, + "concurrency": 8, + "successful_requests": 3388, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 223.842, + "successful_latency_ms": { + "p50": 24.1, + "p95": 81.009, + "p99": 142.445 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB", + "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB", + "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB", + "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB", + "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/B4.json b/evidence/track1/B4.json new file mode 100644 index 00000000..fb3cd66c --- /dev/null +++ b/evidence/track1/B4.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.181, + "concurrency": 8, + "successful_requests": 3592, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.616, + "successful_latency_ms": { + "p50": 22.559, + "p95": 80.196, + "p99": 131.013 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/B4.point.json b/evidence/track1/B4.point.json new file mode 100644 index 00000000..7bed65df --- /dev/null +++ b/evidence/track1/B4.point.json @@ -0,0 +1,41 @@ +{ + "tag": "B4", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.181, + "concurrency": 8, + "successful_requests": 3592, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.616, + "successful_latency_ms": { + "p50": 22.559, + "p95": 80.196, + "p99": 131.013 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB", + "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB", + "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB", + "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB", + "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB" + ], + "postgres_sessions": "17" + } +} diff --git a/evidence/track1/C4-pg2.json b/evidence/track1/C4-pg2.json new file mode 100644 index 00000000..53f3215b --- /dev/null +++ b/evidence/track1/C4-pg2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.128, + "concurrency": 8, + "successful_requests": 3800, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 251.189, + "successful_latency_ms": { + "p50": 19.749, + "p95": 77.161, + "p99": 101.17 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/C4-pg2.point.json b/evidence/track1/C4-pg2.point.json new file mode 100644 index 00000000..7a43b388 --- /dev/null +++ b/evidence/track1/C4-pg2.point.json @@ -0,0 +1,41 @@ +{ + "tag": "C4-pg2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "2.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.128, + "concurrency": 8, + "successful_requests": 3800, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 251.189, + "successful_latency_ms": { + "p50": 19.749, + "p95": 77.161, + "p99": 101.17 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB", + "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB", + "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "17" + } +} diff --git a/evidence/track1/D4-pool2.json b/evidence/track1/D4-pool2.json new file mode 100644 index 00000000..c9e683e4 --- /dev/null +++ b/evidence/track1/D4-pool2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.156, + "concurrency": 8, + "successful_requests": 3622, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 238.98, + "successful_latency_ms": { + "p50": 22.721, + "p95": 74.859, + "p99": 98.063 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1/D4-pool2.point.json b/evidence/track1/D4-pool2.point.json new file mode 100644 index 00000000..80c5703c --- /dev/null +++ b/evidence/track1/D4-pool2.point.json @@ -0,0 +1,41 @@ +{ + "tag": "D4-pool2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.156, + "concurrency": 8, + "successful_requests": 3622, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 238.98, + "successful_latency_ms": { + "p50": 22.721, + "p95": 74.859, + "p99": 98.063 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB", + "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB", + "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB", + "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "9" + } +} diff --git a/evidence/track1/rps.svg b/evidence/track1/rps.svg new file mode 100644 index 00000000..d97a031a --- /dev/null +++ b/evidence/track1/rps.svg @@ -0,0 +1,29 @@ + + + +Successful RPS, closed-loop POST, concurrency 8, 15s + +344.0 +A1 + +196.3 +A2 + +124.5 +A4 + +98.1 +B1 + +223.8 +B2 + +236.6 +B4 + +251.2 +C4-pg2 + +239.0 +D4-pool2 + diff --git a/evidence/track1/summary.json b/evidence/track1/summary.json new file mode 100644 index 00000000..f4916581 --- /dev/null +++ b/evidence/track1/summary.json @@ -0,0 +1,320 @@ +[ + { + "tag": "A1", + "replicas": 1, + "api_cpus_each": "1.0", + "api_mem_each": "512m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 8, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.109, + "concurrency": 8, + "successful_requests": 5197, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 343.964, + "successful_latency_ms": { + "p50": 19.732, + "p95": 42.061, + "p99": 81.001 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-4\t98.33%\t58.76MiB / 512MiB", + "mtusi2026lab3-postgres-1\t33.09%\t45.24MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.03%\t16.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.00%\t4.875MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.81%\t28.43MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.16%\t7.27MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "A2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.206, + "concurrency": 8, + "successful_requests": 2985, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 196.3, + "successful_latency_ms": { + "p50": 26.613, + "p95": 89.893, + "p99": 123.339 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t49.40%\t45.96MiB / 256MiB", + "mtusi2026lab3-api-4\t47.59%\t45.53MiB / 256MiB", + "mtusi2026lab3-postgres-1\t28.18%\t43.48MiB / 512MiB", + "mtusi2026lab3-lb-1\t19.93%\t17.4MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.55%\t4.996MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.34%\t6.312MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "A4", + "replicas": 4, + "api_cpus_each": "0.25", + "api_mem_each": "128m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.224, + "concurrency": 8, + "successful_requests": 1895, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 124.476, + "successful_latency_ms": { + "p50": 39.136, + "p95": 185.654, + "p99": 267.62 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t24.74%\t41.77MiB / 128MiB", + "mtusi2026lab3-api-7\t19.78%\t39.59MiB / 128MiB", + "mtusi2026lab3-api-6\t21.43%\t47.42MiB / 128MiB", + "mtusi2026lab3-api-4\t19.72%\t39.31MiB / 128MiB", + "mtusi2026lab3-postgres-1\t15.99%\t43.09MiB / 512MiB", + "mtusi2026lab3-lb-1\t8.92%\t17.54MiB / 128MiB", + "mtusi2026lab3-redis-1\t5.78%\t5.379MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.82MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t6.484MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "B1", + "replicas": 1, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 0.5, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 4, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.139, + "concurrency": 8, + "successful_requests": 1485, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 98.094, + "successful_latency_ms": { + "p50": 85.074, + "p95": 167.209, + "p99": 201.857 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-5\t48.36%\t45.55MiB / 256MiB", + "mtusi2026lab3-postgres-1\t17.04%\t34.82MiB / 512MiB", + "mtusi2026lab3-lb-1\t7.60%\t18.01MiB / 128MiB", + "mtusi2026lab3-redis-1\t2.68%\t5.312MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.47%\t27.95MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.52%\t7.203MiB / 128MiB" + ], + "postgres_sessions": "5" + } + }, + { + "tag": "B2", + "replicas": 2, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 1.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.136, + "concurrency": 8, + "successful_requests": 3388, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 223.842, + "successful_latency_ms": { + "p50": 24.1, + "p95": 81.009, + "p99": 142.445 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t49.03%\t46.91MiB / 256MiB", + "mtusi2026lab3-api-5\t47.65%\t46.8MiB / 256MiB", + "mtusi2026lab3-postgres-1\t29.21%\t44.29MiB / 512MiB", + "mtusi2026lab3-lb-1\t15.82%\t17.21MiB / 128MiB", + "mtusi2026lab3-redis-1\t4.54%\t4.777MiB / 128MiB", + "mtusi2026lab2-postgres-1\t2.25%\t27.84MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.39%\t7.191MiB / 128MiB" + ], + "postgres_sessions": "9" + } + }, + { + "tag": "B4", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.181, + "concurrency": 8, + "successful_requests": 3592, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.616, + "successful_latency_ms": { + "p50": 22.559, + "p95": 80.196, + "p99": 131.013 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t35.20%\t44.7MiB / 256MiB", + "mtusi2026lab3-api-7\t39.00%\t43.7MiB / 256MiB", + "mtusi2026lab3-api-8\t41.56%\t43.51MiB / 256MiB", + "mtusi2026lab3-api-5\t46.11%\t36.77MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.04%\t62.7MiB / 512MiB", + "mtusi2026lab3-lb-1\t20.75%\t17.96MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.94%\t5.91MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.77%\t5.637MiB / 128MiB" + ], + "postgres_sessions": "17" + } + }, + { + "tag": "C4-pg2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "2.0", + "db_pool_max_each": 4, + "db_pool_max_total": 16, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.128, + "concurrency": 8, + "successful_requests": 3800, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 251.189, + "successful_latency_ms": { + "p50": 19.749, + "p95": 77.161, + "p99": 101.17 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-8\t40.49%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-6\t41.67%\t43.55MiB / 256MiB", + "mtusi2026lab3-api-5\t42.60%\t44.36MiB / 256MiB", + "mtusi2026lab3-api-7\t40.77%\t44.83MiB / 256MiB", + "mtusi2026lab3-postgres-1\t42.82%\t61.07MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.88%\t18.09MiB / 128MiB", + "mtusi2026lab3-redis-1\t6.56%\t4.895MiB / 128MiB", + "mtusi2026lab2-postgres-1\t0.00%\t26.62MiB / 512MiB", + "mtusi2026lab2-redis-1\t0.50%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "17" + } + }, + { + "tag": "D4-pool2", + "replicas": 4, + "api_cpus_each": "0.50", + "api_mem_each": "256m", + "api_cpu_total": 2.0, + "postgres_cpus": "1.0", + "db_pool_max_each": 2, + "db_pool_max_total": 8, + "load": { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.156, + "concurrency": 8, + "successful_requests": 3622, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 238.98, + "successful_latency_ms": { + "p50": 22.721, + "p95": 74.859, + "p99": 98.063 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." + }, + "resources": { + "docker_stats": [ + "mtusi2026lab3-api-6\t48.17%\t43.79MiB / 256MiB", + "mtusi2026lab3-api-7\t46.75%\t43.41MiB / 256MiB", + "mtusi2026lab3-api-5\t44.85%\t43.35MiB / 256MiB", + "mtusi2026lab3-api-8\t45.64%\t43.84MiB / 256MiB", + "mtusi2026lab3-postgres-1\t48.72%\t45.38MiB / 512MiB", + "mtusi2026lab3-lb-1\t21.91%\t17.46MiB / 128MiB", + "mtusi2026lab3-redis-1\t7.26%\t4.801MiB / 128MiB", + "mtusi2026lab2-postgres-1\t3.50%\t26.67MiB / 512MiB", + "mtusi2026lab2-redis-1\t3.15%\t5.887MiB / 128MiB" + ], + "postgres_sessions": "9" + } + } +] diff --git a/scripts/grade4_evidence.py b/scripts/grade4_evidence.py new file mode 100644 index 00000000..4ea1029a --- /dev/null +++ b/scripts/grade4_evidence.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +"""Grade 4: parallel idempotency, stop each API replica, rolling replacement under load.""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure # noqa: E402 + +OUT = ROOT / "evidence" / "grade4" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") + + +def compose(*args, check=True): + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + if check and proc.returncode != 0: + raise SystemExit("docker compose %s failed: %s" % (" ".join(args), (proc.stderr or proc.stdout)[-800:])) + return proc + + +def bash_argv(script: Path): + if os.name != "nt": + return [str(script)] + git_bash = Path(r"C:\Program Files\Git\bin\bash.exe") + if git_bash.is_file(): + return [str(git_bash), str(script)] + return ["bash", str(script)] + + +def hook_cmd(): + return bash_argv(ROOT / "scripts" / "stop-one.sh") + + +def wait_ready(seconds=60): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = exc + time.sleep(0.4) + raise SystemExit("API did not become ready: %s" % last) + + +def instance_ids(actor, samples=24): + found = [] + client = Client(BASE, timeout=3) + for _ in range(samples): + found.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"]) + return sorted(set(found)) + + +def parallel_idempotency(): + client = Client(BASE, timeout=20) + alice, _bob, _eve, chat = client.bootstrap() + key = uuid.uuid4().hex + text = "parallel-" + key + ids = [] + errors = [] + barrier = threading.Barrier(16) + + def one(): + local = Client(BASE, timeout=20) + barrier.wait() + try: + ids.append(local.message(alice, chat, text, key=key)["json"]["id"]) + except Exception as exc: + errors.append(type(exc).__name__) + + threads = [threading.Thread(target=one) for _ in range(16)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + unique = sorted(set(ids)) + report = { + "workers": 16, + "successes": len(ids), + "errors": len(errors), + "distinct_message_ids": len(unique), + } + if len(unique) != 1 or errors: + raise SystemExit("parallel idempotency failed: %s" % report) + page = client.call( + "GET", + "/api/v1/chats/%s/messages?limit=100" % chat["id"], + actor=alice, + schema="MessagePage", + )["json"] + copies = [item for item in page["items"] if item["text"] == text] + report["history_copies"] = len(copies) + if len(copies) != 1: + raise SystemExit("history has %s copies" % len(copies)) + return report + + +def failover_one(actor, chat, instance_id): + key = "failover-" + uuid.uuid4().hex + text = "during-" + key + successes = [] + failures = 0 + started_box = {} + + def traffic(): + nonlocal failures + local = Client(BASE, timeout=2) + while "t0" not in started_box: + time.sleep(0.01) + deadline = started_box["t0"] + 12 + recovered = False + while time.monotonic() < deadline: + try: + if not recovered: + response = local.message(actor, chat, text, key=key) + recovered = True + else: + response = local.message(actor, chat, "after-" + uuid.uuid4().hex) + successes.append((time.monotonic(), response["headers"]["X-Instance-Id"], response["json"]["id"])) + except Failure: + failures += 1 + time.sleep(0.05) + + worker = threading.Thread(target=traffic) + worker.start() + time.sleep(0.2) + started = time.monotonic() + started_box["t0"] = started + env = os.environ.copy() + env["TARGET_INSTANCE_ID"] = instance_id + proc = subprocess.run(hook_cmd(), cwd=ROOT, env=env, capture_output=True, text=True) + hook_s = time.monotonic() - started + worker.join() + after = [item for item in successes if item[0] >= started] + recovery = None + if after: + recovery = round(after[0][0] - started, 3) + tail = [item[1] for item in after[-5:]] + return { + "target": instance_id, + "hook_exit": proc.returncode, + "hook_seconds": round(hook_s, 3), + "hook_stderr": (proc.stderr or "")[-300:], + "failed_attempts": failures, + "successes_after_hook_start": len(after), + "first_recovery_seconds": recovery, + "tail_instance_ids": tail, + "target_still_in_tail": instance_id in tail, + } + + +def restore_and_wait(actor): + compose("start", "api") + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + try: + if len(instance_ids(actor, samples=16)) >= 2: + return + except Failure: + pass + time.sleep(0.5) + raise SystemExit("second replica did not return") + + +def rolling_under_load(): + out = OUT / "rolling-load.json" + load = subprocess.Popen( + [sys.executable, "-X", "utf8", str(ROOT / "tests" / "load.py"), + "--base-url", BASE, "--duration", "50", "--concurrency", "4", "--output", str(out)], + cwd=ROOT, + ) + time.sleep(3) + cmd = bash_argv(ROOT / "scripts" / "rolling-update.sh") + started = time.monotonic() + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + elapsed = round(time.monotonic() - started, 3) + load.wait(timeout=90) + metrics = json.loads(out.read_text(encoding="utf-8")) if out.is_file() else {} + if proc.returncode != 0: + raise SystemExit("rolling update failed: %s" % (proc.stderr or proc.stdout)[-800:]) + return { + "rolling_seconds": elapsed, + "rolling_stdout": (proc.stdout or "")[-400:], + "load": metrics, + } + + +def main(): + OUT.mkdir(parents=True, exist_ok=True) + wait_ready() + client = Client(BASE, timeout=10) + alice, _bob, _eve, chat = client.bootstrap() + before = instance_ids(alice) + if len(before) < 2: + raise SystemExit("need 2 instance ids before grade-4 runs, saw %s" % before) + idem = parallel_idempotency() + failovers = [] + for instance_id in before: + item = failover_one(alice, chat, instance_id) + failovers.append(item) + recovery = item["first_recovery_seconds"] + if item["hook_exit"] != 0 or item["target_still_in_tail"] or recovery is None or recovery > 10 or item["successes_after_hook_start"] < 5: + raise SystemExit("failover criteria failed: %s" % item) + restore_and_wait(alice) + rolling = rolling_under_load() + report = { + "instances_before": before, + "parallel_idempotency": idem, + "failovers": failovers, + "rolling": rolling, + } + (OUT / "summary.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py new file mode 100644 index 00000000..dcf96bf3 --- /dev/null +++ b/scripts/track1_experiment.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +"""Track 1: fixed API budget, then more replicas with more CPU, then one DB change. + +The load profile is the public closed-loop POST generator. Same duration and +concurrency on every point. Raw JSON is written under evidence/track1/. +""" +import json +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" / "track1" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +DURATION = float(os.environ.get("TRACK_DURATION", "15")) +WARMUP = float(os.environ.get("TRACK_WARMUP", "5")) +CONCURRENCY = int(os.environ.get("TRACK_CONCURRENCY", "8")) + + +def compose(*args): + cmd = ["docker", "compose"] + if (ROOT / ".env").is_file(): + cmd.extend(["--env-file", str(ROOT / ".env")]) + cmd.extend(args) + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, env=os.environ.copy()) + if proc.returncode != 0: + raise SystemExit((proc.stderr or proc.stdout)[-1000:]) + return proc + + +def wait_ready(seconds=90): + import urllib.request + + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as response: + if response.status == 200: + return + except Exception: + time.sleep(0.5) + raise SystemExit("ready timeout") + + +def apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool): + os.environ["API_CPUS"] = api_cpus + os.environ["API_MEM"] = api_mem + os.environ["POSTGRES_CPUS"] = postgres_cpus + os.environ["DB_POOL_MAX"] = str(pool) + compose("up", "-d", "--no-deps", "--force-recreate", "postgres") + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + probe = subprocess.run( + ["docker", "compose", "--env-file", str(ROOT / ".env"), "exec", "-T", "postgres", "pg_isready", "-U", "messenger", "-d", "messenger"], + cwd=ROOT, + capture_output=True, + text=True, + ) + if probe.returncode == 0: + break + time.sleep(0.5) + else: + raise SystemExit("postgres did not become ready") + compose( + "up", + "-d", + "--no-deps", + "--force-recreate", + "--scale", + "api=%s" % replicas, + "api", + ) + wait_ready() + running = [line for line in compose("ps", "-q", "api").stdout.splitlines() if line.strip()] + if len(running) != replicas: + raise SystemExit("expected %s api containers, saw %s" % (replicas, len(running))) + + +def run_load(tag, warmup=False): + args = [ + sys.executable, + "-X", + "utf8", + str(ROOT / "tests" / "load.py"), + "--base-url", + BASE, + "--duration", + str(WARMUP if warmup else DURATION), + "--concurrency", + str(CONCURRENCY), + ] + dest = None + if not warmup: + dest = OUT / ("%s.json" % tag) + args.extend(["--output", str(dest)]) + proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True) + if proc.returncode != 0: + raise SystemExit("load %s failed: %s" % (tag, (proc.stderr or proc.stdout)[-800:])) + if dest is None: + return None + return json.loads(dest.read_text(encoding="utf-8")) + + +def sample_resources(): + stats = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"], + capture_output=True, + text=True, + ) + activity = compose( + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-tAc", + "SELECT count(*) FROM pg_stat_activity WHERE datname='messenger'", + ) + return { + "docker_stats": [line for line in stats.stdout.splitlines() if line.strip()], + "postgres_sessions": activity.stdout.strip(), + } + + +def measure(tag, replicas, api_cpus, api_mem, postgres_cpus, pool): + print("profile %s" % tag, flush=True) + apply_profile(replicas, api_cpus, api_mem, postgres_cpus, pool) + run_load(tag, warmup=True) + box = {} + + def snapshot(): + time.sleep(DURATION / 2) + box["resources"] = sample_resources() + + thread = threading.Thread(target=snapshot) + thread.start() + metrics = run_load(tag) + thread.join() + point = { + "tag": tag, + "replicas": replicas, + "api_cpus_each": api_cpus, + "api_mem_each": api_mem, + "api_cpu_total": round(replicas * float(api_cpus), 3), + "postgres_cpus": postgres_cpus, + "db_pool_max_each": pool, + "db_pool_max_total": replicas * pool, + "load": metrics, + "resources": box.get("resources"), + } + (OUT / ("%s.point.json" % tag)).write_text(json.dumps(point, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"tag": tag, "rps": metrics["successful_rps"], "errors": metrics["failed_requests"], "p95": metrics["successful_latency_ms"]["p95"]}), flush=True) + return point + + +def svg(points, path): + width, height = 720, 320 + labels = [p["tag"] for p in points] + values = [p["load"]["successful_rps"] for p in points] + peak = max(values) or 1 + left, top, plot_w, plot_h = 48, 20, width - 70, height - 70 + bar_w = plot_w / len(points) * 0.62 + gap = plot_w / len(points) + parts = [ + '', + '' % (width, height), + '', + 'Successful RPS, closed-loop POST, concurrency %d, %ss' + % (CONCURRENCY, int(DURATION)), + ] + for index, (label, value) in enumerate(zip(labels, values)): + h = plot_h * value / peak + x = left + index * gap + (gap - bar_w) / 2 + y = top + 12 + (plot_h - h) + parts.append('' % (x, y, bar_w, h)) + parts.append( + '%.1f' + % (x + bar_w / 2, y - 4, value) + ) + parts.append( + '%s' + % (x + bar_w / 2, height - 18, label) + ) + parts.append("") + path.write_text("\n".join(parts) + "\n", encoding="utf-8") + + +def main(): + OUT.mkdir(parents=True, exist_ok=True) + points = [] + # Series A: total API budget fixed at 1.0 CPU and 512 MiB. Pool total stays 8. + points.append(measure("A1", 1, "1.0", "512m", "1.0", 8)) + points.append(measure("A2", 2, "0.50", "256m", "1.0", 4)) + points.append(measure("A4", 4, "0.25", "128m", "1.0", 2)) + # Series B: each new replica brings the same 0.50 CPU / 256 MiB. Postgres stays at 1 CPU. + points.append(measure("B1", 1, "0.50", "256m", "1.0", 4)) + points.append(measure("B2", 2, "0.50", "256m", "1.0", 4)) + points.append(measure("B4", 4, "0.50", "256m", "1.0", 4)) + # Candidate improvement: give the database the extra CPU the API side already has. + points.append(measure("C4-pg2", 4, "0.50", "256m", "2.0", 4)) + # Candidate improvement: stop the 4 pools from opening 16 server connections. + points.append(measure("D4-pool2", 4, "0.50", "256m", "1.0", 2)) + (OUT / "summary.json").write_text(json.dumps(points, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + svg(points, OUT / "rps.svg") + # Return the stand to the documented 2 x 0.50 CPU layout. + apply_profile(2, "0.50", "256m", "1.0", 4) + print("wrote %s" % (OUT / "summary.json")) + + +if __name__ == "__main__": + main() -- 2.54.0 From 20385513429356c9e0a4541acd1eef1d411ae95f Mon Sep 17 00:00:00 2001 From: David Date: Sat, 7 Nov 2026 12:46:37 +0300 Subject: [PATCH 4/4] =?UTF-8?q?=D0=9E=D0=BF=D0=B8=D1=81=D0=B0=D1=82=D1=8C?= =?UTF-8?q?=20=D1=80=D0=B5=D1=88=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BD=D0=B0=20?= =?UTF-8?q?5=20=D0=B8=20=D0=BE=D1=82=D0=B2=D0=B5=D1=82=D1=8B=20=D0=BD?= =?UTF-8?q?=D0=B0=20=D0=B2=D0=BE=D0=BF=D1=80=D0=BE=D1=81=D1=8B=20=D0=B7?= =?UTF-8?q?=D0=B0=D1=89=D0=B8=D1=82=D1=8B.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/pull_request_template.md | 28 ++++----- REPORT.md | 100 ++++++++++++++++++++++++++----- solve.md | 71 ++++++++++++++++++++++ 3 files changed, 171 insertions(+), 28 deletions(-) create mode 100644 solve.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e6e63eb2..b93e3208 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 3 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Булацев / БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 1, цена масштабирования -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: API lab2 (PostgreSQL, Redis-сессии, кэш). Добавлены HAProxy, масштабируемый сервис API без внешнего порта, одноразовые миграции, общий Idempotency-Key и X-Instance-Id. +Команды запуска из чистого clone: `cp .env.example .env`, затем `docker compose --env-file .env up -d --build --scale api=2`, дождаться `http://127.0.0.1:8080/health/ready`. Подробности в REPORT.md. +Результаты проверок и ссылка на REPORT.md: smoke PASS; persistence PASS; failover — 2 реплики, 1 ошибка, восстановление записи 2.465 с; трек 1 в `evidence/track1/`. См. REPORT.md. +Ограничения / известные ошибки: один хост; LB, Postgres и Redis остаются единой точкой отказа. На Windows `.sh` не запускается напрямую (`WinError 193`); hook рассчитан на POSIX. +Источники и использование ИИ: README и контракт курса, документация HAProxy и Compose. Сборку решения вёл агент Cursor (Grok). -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/REPORT.md b/REPORT.md index c2679005..c8f2f39e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,109 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: **Булацев**, **БПИ2301** (ветка `solution/bpi2301-Bulatsev`) +- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена масштабирования»** +- Перенос: API lab2 (PostgreSQL, Redis-сессии, кэш истории, миграции). В этой работе добавлены HAProxy, две и более реплики API без внешнего порта, общий `Idempotency-Key` в PostgreSQL и `X-Instance-Id`. +- Commit/PR: ещё не опубликован +- Источники и помощь ИИ: контракт и README курса; образы Postgres 16.6, Redis 7.4.2, HAProxy 2.8. Документация HAProxy `server-template` и Docker Compose. Реализацию и прогоны помог собрать агент Cursor (Grok). На защите нужно объяснять схему своими словами. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Стенд: Windows 11, CPU 12th Gen Intel Core i5-12450H, Docker Desktop 29.5.3 / Compose 5.1.4, лимит Docker 12 vCPU и 7.62 ГиБ. Образы зафиксированы тегами: `python:3.12.8-slim-bookworm` (сборка `mtusi-messenger:lab3`), `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`. + +```sh +cp .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 +``` + +Дождаться `200` от `http://127.0.0.1:8080/health/ready`. Этот адрес — HAProxy. Реплики API наружу порты не публикуют. + +Миграции выполняет одноразовый сервис `migrate` (`python -m app.migrate`) после healthy Postgres и до старта API. Повторный запуск пропускает уже записанные файлы. Дополнительно `pg_advisory_lock` не даёт двум миграторам применить один файл. У API `MIGRATIONS_ON_START=0`. + +Проверки: + +```sh +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 -m pytest student_tests -q -m "not infra and not expiry" +``` + +`scripts/restart.sh` и `scripts/stop-one.sh` — POSIX. На Linux: `chmod +x` и `make test-persistence` / `make test-failover`. На этом Windows `CreateProcess` не запускает `.sh` (`WinError 193`); те же команды прогнаны интерпретатором `python3` в контейнере `docker:27-cli` с сокетом того же Docker. Скрипты при этом управляли контейнерами стенда. + +Остановка без удаления данных: `docker compose --env-file .env stop`. Учебные тома удаляет только явная команда `docker compose --env-file .env down -v`. + +Чтобы освободить `127.0.0.1:8080`, перед запуском был остановлен контейнер `mtusi2026lab2-api-1`. Postgres и Redis lab2 оставались запущены; в срезах `docker stats` их CPU был около 0–4%. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент ходит только в `lb` (HAProxy, `127.0.0.1:8080`). Backend — DNS-имя `api` сервиса Compose: `server-template` на 8 слотов, `resolvers docker` (`127.0.0.11`), `hold valid 1s`. Балансировка `roundrobin`, без sticky session. Сессии лежат в Redis (AOF + RDB), пользователи, чаты, сообщения и результаты идемпотентности — в PostgreSQL. Локальная память реплики не нужна для корректности следующего запроса. + +`X-Instance-Id` — hostname контейнера, он не меняется, пока жив процесс. `scripts/stop-one.sh` сравнивает `TARGET_INSTANCE_ID` с `/tmp/instance-id` или hostname и делает `docker stop -t 5` одного контейнера. `restart: unless-stopped` не поднимает его обратно. + +`POST /messages` требует `Idempotency-Key` (`[A-Za-z0-9._:-]{1,128}`). Область — пользователь + `POST` + путь чата + ключ. Канонический JSON (`text` и `attachment_ids`, пустой список равен отсутствию поля) хешируется SHA-256. Тот же хеш в течение `IDEMPOTENCY_TTL_SECONDS` (86400) возвращает сохранённый `201` и то же сообщение. Другой текст — `409`. Вставка сообщения и строки результата — одна транзакция; ошибка откатывает обе. Одновременные повторы сериализует `pg_advisory_xact_lock` в Postgres, поэтому две реплики не создают два сообщения. Неуспешный запрос строку не пишет, повтор с тем же ключом безопасен. + +Проверки готовности. `GET /health/live` — только процесс. `GET /health/ready` — Postgres и Redis; во время SIGTERM сразу `503`. HAProxy: `inter 1s`, `fall 2`, `rise 1`, `timeout check 2s`. Неготовая реплика выпадает примерно за 2 с. Если контейнер уже исчез, DNS-кэш HAProxy живёт ещё до 1 с. На SIGTERM API перестаёт принимать бизнес-запросы, ready становится 503, процесс завершается через `LB_DRAIN_SECONDS` (2). `stop_grace_period` сервиса — 8 с. Таймауты прокси: connect 2 с, client/server 10 с. HAProxy не повторяет POST. + +Соответствие `.env.example`: `DATABASE_URL` и `REDIS_URL` собираются в Compose из `POSTGRES_PASSWORD`; `API_CPUS`, `API_MEM`, `POSTGRES_CPUS`, `DB_POOL_MAX`, `LB_DRAIN_SECONDS`, `IDEMPOTENCY_TTL_SECONDS` интерполируются в `compose.yaml`. Рабочая точка стенда — 2 реплики по 0.50 CPU и 256 МиБ, Postgres 1.0 CPU / 512 МиБ, пул по 4 соединения. `container_name` нет. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт offline | `python -X utf8 scripts/check_contract.py` | PASS | `evidence/public-checks.txt` | +| Smoke через LB | `tests/smoke.py` | PASS, lab3 | `evidence/public-checks.txt` | +| Две реплики, одна сессия | 12× `GET /health/ready` | Чередование `03c3a0545628` и `b555f8327eee` | наблюдение до отказов; после экспериментов новые id, см. ниже | +| Общая БД и сессии | persistence hook | PASS, старый токен жив | `evidence/public-checks.txt` | +| Отказ одной реплики ≤10 с | `tests/resilience.py --mode failover` | 2 реплики до отказа; 109 успешных записей; 1 ошибка; первое подтверждение после hook 2.465 с; дублей нет | `evidence/public-checks.txt` | +| Остановлен именно целевой контейнер | `docker inspect` после failover | `mtusi2026lab3-api-1` Exited (0), hostname `b555f8327eee`; вторая реплика Up | `evidence/public-checks.txt` | +| Идемпотентность 16 параллельных запросов | `scripts/grade4_evidence.py` | 16×201, 1 id, в истории 1 копия | `evidence/grade4/summary.json` | +| Поочерёдный отказ каждой реплики | тот же скрипт | hook 2.373 с и 1.744 с; в последних ответах только живая реплика; 2 и 1 ошибка | `evidence/grade4/summary.json` | +| Rolling update под нагрузкой | `scripts/rolling-update.sh` во время `tests/load.py` 50 с, concurrency 4 | 8361 успех, 18 ошибок (0.21%), RPS 166.9, p50/p95/p99 13.5/57.8/96.4 мс, замена 5.9 с | `evidence/grade4/summary.json`, `evidence/grade4/rolling-load.json` | +| Лимиты, нет обхода LB | `docker ps` | API `8080/tcp` без host-порта; наружу только `127.0.0.1:8080` у lb | `compose.yaml` | +| Трек 1 | `scripts/track1_experiment.py` | серии A/B и проверка двух правок, см. ниже | `evidence/track1/summary.json`, `evidence/track1/rps.svg` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Студенческие тесты lab2 (гонки, откат чата, кэш, конкурентные сообщения): 15 passed. `RUN_INFRA` и expiry в этот прогон не входили. + +После трека 1 стенд возвращён к 2 репликам по 0.50 CPU. Новые hostname на ready: `0868e9d2ba45` и `14cb087ebb10`. Старые id исчезли вместе с пересозданными контейнерами; это не sticky session. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза. При фиксированном бюджете API 1.0 CPU и 512 МиБ дробление на 2 и 4 реплики не поднимет RPS: каждая упрётся в свой cgroup, а Postgres ещё свободен. Если давать каждой новой реплике те же 0.50 CPU, RPS вырастет, пока реплика упирается в квоту, и почти остановится, когда клиентский closed-loop и единственный Postgres перестанут получать пользу от лишних процессов. + +Методика. Генератор `tests/load.py`: closed-loop POST сообщений, concurrency 8, 5 с прогрев (результат выбрасывается), 15 с замер. Ошибок автоматических retry нет. На каждую точку один прогон. В середине замера один срез `docker stats` и `pg_stat_activity`. В `docker stats` 100% — одно ядро. Профиль A держит сумму API равной 1.0 CPU / 512 МиБ и сумму пулов равной 8. Профиль B добавляет реплики по 0.50 CPU / 256 МиБ при Postgres 1.0 CPU. + +| Точка | Реплики | CPU API суммарно | Успешный RPS | Ошибки | p50 / p95 / p99, мс | CPU API по stats | CPU Postgres | Сессии Postgres | CPU LB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| A1 | 1 × 1.00 | 1.0 | 344.0 | 0 | 19.7 / 42.1 / 81.0 | 98% | 33% | 9 | 21% | +| A2 | 2 × 0.50 | 1.0 | 196.3 | 0 | 26.6 / 89.9 / 123.3 | 49% и 48% | 28% | 9 | 20% | +| A4 | 4 × 0.25 | 1.0 | 124.5 | 0 | 39.1 / 185.7 / 267.6 | 20–25% | 16% | 9 | 9% | +| B1 | 1 × 0.50 | 0.5 | 98.1 | 0 | 85.1 / 167.2 / 201.9 | 48% | 17% | 5 | 8% | +| B2 | 2 × 0.50 | 1.0 | 223.8 | 0 | 24.1 / 81.0 / 142.4 | 49% и 48% | 29% | 9 | 16% | +| B4 | 4 × 0.50 | 2.0 | 236.6 | 0 | 22.6 / 80.2 / 131.0 | 35–46% | 48% | 17 | 21% | +| C4-pg2 | как B4, Postgres 2.0 CPU | 2.0 | 251.2 | 0 | 19.7 / 77.2 / 101.2 | 40–43% | 43% | 17 | 22% | +| D4-pool2 | как B4, пул 2 на реплику | 2.0 | 239.0 | 0 | 22.7 / 74.9 / 98.1 | 45–48% | 49% | 9 | 22% | + +График: `evidence/track1/rps.svg`. Сырые точки: `evidence/track1/*.point.json`. + +Что из этого следует. В серии A Postgres не насыщен (16–33%), память API далеко от лимита. Падает сам API: одна реплика с целым ядром даёт 344 RPS, те же ресурсы четырьмя квотами по 0.25 CPU — 124 RPS, p95 с 42 до 186 мс. Рост числа процессов при фиксированном CPU вреден из-за квот CFS и накладных расходов, а не из-за нехватки БД. + +В серии B первое ядро API ещё нужно: B1 упирается в квоту 0.50 CPU (stats около 50%, p95 167 мс). Вторая такая реплика — подтверждённое улучшение: RPS 98.1 → 223.8, p95 167 → 81 мс, Postgres только с 17% до 29%. Четыре реплики против двух — уже площадка: +1.0 CPU API даёт 223.8 → 236.6 RPS (+6%), зато сессий Postgres 9 → 17 и CPU Postgres 29% → 48%, LB 16% → 21%. Восемь замкнутых воркеров не заполняют четыре процесса, а каждый INSERT всё равно идёт в один Postgres. Пул здесь не узкое место: D4 снижает сессии с 17 до 9 и почти не меняет RPS (239 против 237). Дополнительное ядро Postgres тоже не подтвердилось: 251 против 237 на одном 15-секундном прогоне, срез CPU Postgres так и не упёрся в 100%. + +Ограничение измерения. Один прогон на точку, closed-loop и coordinated omission, перцентили только успешных запросов, таблица `messages` растёт от точки к точке, `docker stats` — один срез. Это не паспорт максимальной пропускной способности машины. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `scripts/restart.sh` (api, postgres, redis), тома на месте | те же сообщения и тот же токен | PASS публичного persistence | готовность вернулась внутри бюджета теста 90 с; данных не теряли | +| `scripts/stop-one.sh` по `X-Instance-Id` | живая реплика принимает запись ≤10 с, дубля нет | 2 id до отказа; 1 ошибка; 109 записей; целевой контейнер Exited (0) | 2.465 с до первого подтверждения после возврата hook | +| Та же остановка второй реплики после старта первой | хвост ответов только от оставшейся | оба прогона: target отсутствует в последних 5 id | hook 1.7–2.4 с; ошибок 1 и 2 | +| Поочерёдная замена обеих реплик под нагрузкой | короткие ошибки, запись продолжается | 18 ошибок на 8379 попыток | скрипт замены 5.9 с | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд переживает остановку одного процесса API и перезапуск контейнеров с сохранёнными томами. Он не переживает потерю единственного хоста, HAProxy, Postgres или Redis: это оставшиеся SPOF. Трек 2 не делался, выживание узла БД не заявляется. + +Sticky session нет и не нужен: сессия в Redis, идемпотентность в Postgres. Повторно безопасно отправлять POST сообщения с тем же ключом. Повтор POST пользователя или чата по-прежнему может создать второй объект; прокси такие POST не повторяет. + +Миграции совместимы с уже работающей схемой lab2 (новый файл `002_idempotency.sql`, старые таблицы не переписываются). Одновременный запуск двух версий кода с разной схемой отдельно не гонялся. + +Подтверждённое улучшение — вторая реплика, когда первая исчерпала квоту 0.50 CPU (B1→B2). Четвёртая реплика, удвоение CPU Postgres и урезание пула на этом профиле прироста не показали. Следующий осмысленный шаг — поднять concurrency клиента выше 8 и повторить B2/B4 несколькими прогонами, прежде чем добавлять реплики Postgres. diff --git a/solve.md b/solve.md new file mode 100644 index 00000000..ca0a1603 --- /dev/null +++ b/solve.md @@ -0,0 +1,71 @@ +# Разбор лабораторной 3 + +Автор: Булацев, БПИ2301. Целевая оценка: 5, трек «Цена масштабирования». + +## Суть + +Нужно запустить несколько одинаковых процессов API за одной точкой входа и не потерять корректность, когда один из них останавливают. + +Клиент больше не обращается к конкретному процессу. Он ходит в балансировщик, а тот выбирает живую реплику. Любая реплика должна узнать ту же сессию, увидеть уже подтверждённое сообщение и не создать второе сообщение, если клиент повторил запрос с тем же ключом. + +Источник истины для пользователей, чатов и сообщений — PostgreSQL. Сессии — Redis с AOF. Память процесса для этого не используется. + +## Какие проблемы решались + +### Один процесс нельзя просто скопировать + +Если сессия или кэш лежат в памяти процесса, второй экземпляр пользователя не узнает. Повторный запрос попадёт на другую реплику и получит 401 либо устаревшую историю. + +Решение: сессия хранится в общем Redis, история читается из общего Postgres. Кэш истории тоже в Redis и сбрасывается после успешной записи, поэтому следующая реплика не отдаёт старую страницу. Клиента не приклеиваем к одной реплике. + +### Клиент не должен выбирать процесс сам + +Нужна одна публичная точка входа и при этом несколько процессов. `depends_on` только задаёт порядок запуска контейнеров. Healthcheck контейнера говорит Docker, жив ли он, но не перекладывает HTTP-запрос на другой адрес. + +Решение: HAProxy публикует `127.0.0.1:8080` и сам ходит в DNS-имя `api`. У сервиса Compose нет `container_name`, поэтому реплик может быть сколько угодно. Порты API наружу не опубликованы. HAProxy держит до 8 слотов через `server-template` и обновляет адреса из DNS Docker раз в секунду. Проверка — `GET /health/ready` каждые 1 с, реплика выпадает после двух неудач. Это около 2 с плюс до 1 с устаревшего DNS, если контейнер уже исчез. + +### Остановка процесса не должна обрывать стенд надолго + +Docker посылает SIGTERM. Если процесс сразу умирает, балансировщик ещё считает его живым и шлёт туда запросы. + +Решение согласовано с проверкой готовности. По SIGTERM реплика сразу отвечает 503 на `/health/ready` и на новые бизнес-запросы, а завершается через 2 с (`LB_DRAIN_SECONDS`). За это HAProxy успевает увидеть неготовый backend. `scripts/stop-one.sh` останавливает только контейнер с нужным `X-Instance-Id` и оставляет его остановленным. `X-Instance-Id` равен hostname и не меняется, пока жив процесс. На публичной проверке запись восстановилась за 2.465 с, подтверждённые сообщения и сессия сохранились. + +### Повтор POST после таймаута может создать второе сообщение + +Клиент не знает, успел ли сервер записать сообщение, если ответ потерялся. Повтор без защиты создаёт дубль. Две реплики могут принять один и тот же повтор одновременно. + +Решение: для `POST /messages` обязателен `Idempotency-Key`. Область ключа — пользователь, метод, путь чата и сам ключ. Текст приводится к одному JSON (порядок полей не важен, пустой список вложений равен отсутствию поля) и хешируется. Сообщение и сохранённый ответ 201 пишутся одной транзакцией. Пока транзакция не закоммичена, строки результата нет, поэтому неуспешный запрос можно повторить. Одновременные повторы сериализует advisory lock в Postgres: побеждает одна вставка, остальные получают то же сообщение. Другой текст с тем же ключом — 409. Проверено 16 параллельными запросами: в истории одна копия. + +### Миграции нельзя применять гонкой всех реплик + +Несколько процессов при старте могут одновременно выполнить один и тот же SQL. + +Решение: схему применяет отдельный одноразовый сервис `migrate` до старта API. У реплик `MIGRATIONS_ON_START=0`. Повторный запуск пропускает файлы, уже записанные в `schema_migrations`. На время применения берётся `pg_advisory_lock`. + +### Больше реплик не означает больше пропускной способности + +Гипотеза трека 1: при фиксированном CPU API дробление на процессы не поможет, а при росте числа реплик выигрыш кончится, когда упрёмся не в CPU отдельного API. + +Один и тот же closed-loop, 8 параллельных POST, 15 с. При бюджете 1 CPU / 512 МиБ результат 344, 196 и 124 RPS для 1, 2 и 4 реплик. Postgres был занят на 16–33%: мешала квота CPU на каждом процессе. Когда одна реплика упиралась в 0.50 CPU (98 RPS, p95 167 мс), вторая такая же реплика подняла результат до 224 RPS и p95 81 мс. Это подтверждённое улучшение. Четвёртая реплика дала только 237 RPS, зато сессий Postgres стало 17 вместо 9. Удвоение CPU Postgres и уменьшение пула на этом профиле заметного прироста не дали. Подробные таблицы — в `REPORT.md`. + +## Ответы на вопросы README + +### Чем отказ процесса отличается от отказа узла? + +Отказ процесса — это остановка одного API-контейнера. Балансировщик перестаёт слать ему запросы, вторая реплика на том же хосте продолжает читать и писать в общие Postgres и Redis. Данные не пропадают. + +Отказ узла — это потеря всей машины. Вместе с ней пропадают все реплики API, HAProxy, Postgres и Redis, потому что они запущены на одном Docker-хосте. Несколько процессов API такую потерю не переживают. Отдельно живая реплика базы в этой работе не поднималась, поэтому выживание узла БД не заявляется. + +### Почему depends_on и healthcheck сами по себе не являются балансировщиком? + +`depends_on` лишь откладывает старт контейнера, пока другой не стал healthy. Он не принимает соединения клиентов и не выбирает, какому процессу отдать конкретный HTTP-запрос. Healthcheck Docker перезапускает или помечает свой контейнер, но не знает о соседних репликах и не проксирует трафик. Пока снаружи опубликован сам API, клиент попадает в один процесс в обход остальных. Балансировщик — отдельный процесс: он слушает единственный адрес, смотрит готовность backend-ов и перекладывает запрос на живой. + +### Какие POST можно повторять после timeout и что защищает Idempotency-Key? + +После таймаута можно повторять POST сообщения с тем же ключом и тем же телом. Если запись успела закоммититься, сервер вернёт тот же `201` и то же сообщение. Если не успела, повтор создаст его один раз. Другое тело с тем же ключом получит 409. + +Защищает общая строка в Postgres на пару «пользователь + метод + путь + ключ» вместе с advisory lock и одной транзакцией на сообщение и сохранённый ответ. Ключ не защищает создание пользователя и создание чата: их повтор может создать второй объект, поэтому балансировщик такие POST сам не повторяет. + +### Что сломается при увеличении числа API, если сессии находятся в локальной памяти? + +Вход создаст сессию только в памяти той реплики, которая обработала логин. Следующий запрос балансировщик отдаст другой реплике, и она ответит 401, потому что токена у неё нет. Пользователю придётся входить заново, а при sticky session он останется привязан к одному процессу: отказ этого процесса снова потеряет сессию. Общие сообщения в Postgres это не чинят. Поэтому сессия лежит в Redis, который видят все реплики. -- 2.54.0