From de55dd1d8a981078a906d41934a6c142b477a47e Mon Sep 17 00:00:00 2001 From: Sona Date: Tue, 10 Nov 2026 01:25:06 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=B0?= =?UTF-8?q?=D1=86=D0=B8=D1=8F=20=D0=9B=D0=A0=203=20=D0=BD=D0=B0=20=D0=BE?= =?UTF-8?q?=D1=86=D0=B5=D0=BD=D0=BA=D1=83=203:=20HAProxy,=202=20=D1=80?= =?UTF-8?q?=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=D0=B8=20API,=20=D0=B8=D0=B4?= =?UTF-8?q?=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5=D0=BD=D1=82=D0=BD=D0=BE?= =?UTF-8?q?=D1=81=D1=82=D1=8C,=20evidence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 26 ++ compose.yaml | 88 +++++ evidence/failover-1-ps.txt | 7 + evidence/failover-1.txt | 4 + evidence/failover-2-ps.txt | 7 + evidence/failover-2.txt | 4 + evidence/instances.txt | 2 + evidence/persistence.txt | 3 + evidence/ps-before.txt | 7 + evidence/smoke.txt | 2 + infra/haproxy/haproxy.cfg | 35 ++ migrations/001_init.sql | 63 ++++ migrations/002_idempotency.sql | 13 + requirements.txt | 16 + scripts/restart.sh | 7 + scripts/show-instances.sh | 12 + scripts/stop-one.sh | 14 + src/auth.py | 87 +++++ src/cache.py | 82 +++++ src/chats.py | 255 ++++++++++++++ src/common.py | 30 ++ src/config.py | 29 ++ src/db.py | 56 +++ src/main.py | 187 ++++++++++ src/migrate.py | 19 ++ student_tests/test_lab2.py | 585 ++++++++++++++++++++++++++++++++ student_tests/test_level4.py | 195 +++++++++++ student_tests/test_lifecycle.py | 68 ++++ student_tests/test_sigterm.py | 129 +++++++ 29 files changed, 2032 insertions(+) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/failover-1-ps.txt create mode 100644 evidence/failover-1.txt create mode 100644 evidence/failover-2-ps.txt create mode 100644 evidence/failover-2.txt create mode 100644 evidence/instances.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/ps-before.txt create mode 100644 evidence/smoke.txt create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 requirements.txt create mode 100755 scripts/restart.sh create mode 100755 scripts/show-instances.sh create mode 100755 scripts/stop-one.sh create mode 100644 src/auth.py create mode 100644 src/cache.py create mode 100644 src/chats.py create mode 100644 src/common.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 student_tests/test_lab2.py create mode 100644 student_tests/test_level4.py create mode 100644 student_tests/test_lifecycle.py create mode 100644 student_tests/test_sigterm.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..52a4ede3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM python:3.12.7-slim@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +RUN groupadd --system app && \ + useradd --system --gid app --create-home app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --timeout 60 -r requirements.txt + +COPY --chown=app:app src/ /app/src/ +COPY --chown=app:app migrations/ /app/migrations/ + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2)" || exit 1 + +CMD ["sh", "-c", "exec uvicorn src.main:app --host \"${APP_HOST}\" --port \"${APP_PORT}\" --timeout-graceful-shutdown 8"] \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..6f6756c8 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,88 @@ +name: mtusi-lab3 + +services: + postgres: + image: postgres:16.6-alpine + restart: unless-stopped + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + + valkey: + image: valkey/valkey:8.0.1-alpine + restart: unless-stopped + command: ["valkey-server", "--save", "", "--appendonly", "no", "--maxmemory", "64mb", "--maxmemory-policy", "allkeys-lru"] + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + + # Одноразовая задача: применяет миграции и завершается. Реплики API миграции не запускают + migrate: + build: . + image: mtusi-messenger:lab3 + restart: "no" + command: ["python", "-m", "src.migrate"] + depends_on: + postgres: + condition: service_healthy + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + + # Реплики: число задаётся API_REPLICAS (по умолчанию 2). Без container_name и без published-портов + api: + image: mtusi-messenger:lab3 + pull_policy: never + restart: unless-stopped + deploy: + replicas: ${API_REPLICAS:-2} + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_started + migrate: + condition: service_completed_successfully + environment: + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://valkey:6379/0 + secrets: + - database_url + expose: + - "8080" + + # Единственная публичная точка входа + lb: + image: haproxy:3.0.7-alpine + restart: unless-stopped + depends_on: + - api + volumes: + - ./infra/haproxy:/usr/local/etc/haproxy:ro + ports: + - "127.0.0.1:8080:8080" + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + +volumes: + pgdata: \ No newline at end of file diff --git a/evidence/failover-1-ps.txt b/evidence/failover-1-ps.txt new file mode 100644 index 00000000..9e0785a4 --- /dev/null +++ b/evidence/failover-1-ps.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About an hour ago Exited (0) 28 seconds ago +mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About an hour ago Up About an hour (healthy) 8080/tcp +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb About an hour ago Up About an hour 127.0.0.1:8080->8080/tcp +mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate About an hour ago Exited (0) About an hour ago +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres About an hour ago Up About an hour (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey About an hour ago Up About an hour (healthy) 6379/tcp diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..d0501d8d --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 122, "failed_attempts": 0, "first_recovery_seconds": 2.036} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/failover-2-ps.txt b/evidence/failover-2-ps.txt new file mode 100644 index 00000000..8199d3f6 --- /dev/null +++ b/evidence/failover-2-ps.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 58 seconds (healthy) 8080/tcp +mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (0) 21 seconds ago +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb About an hour ago Up About an hour 127.0.0.1:8080->8080/tcp +mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 58 seconds ago +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres About an hour ago Up About an hour (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up About an hour (healthy) 6379/tcp diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..4d93eeac --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 1.569} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/instances.txt b/evidence/instances.txt new file mode 100644 index 00000000..dce0509a --- /dev/null +++ b/evidence/instances.txt @@ -0,0 +1,2 @@ + 5 x-instance-id: 98cca609117d + 5 x-instance-id: bb44a95c7f25 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/ps-before.txt b/evidence/ps-before.txt new file mode 100644 index 00000000..f20313a2 --- /dev/null +++ b/evidence/ps-before.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 16 minutes ago Up 16 minutes (healthy) 8080/tcp +mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 16 minutes ago Up 16 minutes (healthy) 8080/tcp +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb About a minute ago Up About a minute 127.0.0.1:8080->8080/tcp +mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 16 minutes ago Exited (0) About a minute ago +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 12 minutes ago Up 12 minutes (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 16 minutes ago Up 16 minutes (healthy) 6379/tcp diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..44e41deb --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..58bbf906 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + timeout connect 1s + timeout client 30s + timeout server 10s + retries 3 + option redispatch + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold nx 1s + hold timeout 1s + accepted_payload_size 8192 + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-8 api:8080 diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..a43b5358 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,63 @@ +-- 001_init: базовая схема лабы 2. Повторный запуск безопасен (IF NOT EXISTS). + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, -- Argon2id, никогда не отдаётся в API + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK ( + char_length(display_name) BETWEEN 1 AND 100 AND btrim(display_name) <> '' + ) +); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT chats_title_len CHECK ( + char_length(title) BETWEEN 1 AND 100 AND btrim(title) <> '' + ) +); + +-- Порядок списка чатов: (created_at, id) +CREATE INDEX IF NOT EXISTS chats_created_idx ON chats (created_at, id); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, -- порядок member_ids как в запросе + PRIMARY KEY (chat_id, user_id) +); + +-- Список чатов пользователя и проверка членства +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT messages_text_len CHECK ( + char_length(text) BETWEEN 1 AND 4000 AND btrim(text) <> '' + ) +); + +-- Keyset-пагинация истории: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id LIMIT n -> index scan без сортировки и полного прохода +CREATE INDEX IF NOT EXISTS messages_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash bytea PRIMARY KEY, -- SHA-256 токена; сам токен не хранится + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); + +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); +CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions (expires_at); \ No newline at end of file diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..ba0c7db7 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,13 @@ +-- 002: общая идемпотентность отправки сообщений +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + request_hash bytea NOT NULL, -- SHA-256 канонического тела + message_id uuid REFERENCES messages (id), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, idem_key) +); + +CREATE INDEX IF NOT EXISTS idempotency_created_idx ON idempotency_keys (created_at); \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..23aaa3e5 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,16 @@ +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +fastapi==0.115.2 +h11==0.16.0 +idna==3.20 +pydantic==2.14.1 +pydantic-core==2.50.1 +starlette==0.40.0 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +uvicorn==0.32.0 +asyncpg==0.30.0 +argon2-cffi==23.1.0 +argon2-cffi-bindings==21.2.0 +redis==5.2.1 \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..edf94b95 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Перезапуск API и хранилищ с сохранением volume (никаких down -v). +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop api +docker compose --env-file .env restart postgres valkey +docker compose --env-file .env start api \ No newline at end of file diff --git a/scripts/show-instances.sh b/scripts/show-instances.sh new file mode 100755 index 00000000..15c74d5c --- /dev/null +++ b/scripts/show-instances.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Одна сессия, 10 запросов через LB: в ответах должны быть >=2 разных X-Instance-Id. +set -eu +BASE="${BASE_URL:-http://localhost:8080}" +U="demo_$(date +%s)" +curl -sf -X POST "$BASE/api/v1/users" -H 'Content-Type: application/json' \ + -d "{\"username\":\"$U\",\"display_name\":\"Demo\",\"password\":\"demo-password-123\"}" >/dev/null +TOKEN=$(curl -sf -X POST "$BASE/api/v1/auth/sessions" -u "$U:demo-password-123" \ + | python3 -c 'import sys,json; print(json.load(sys.stdin)["token"])') +for i in 1 2 3 4 5 6 7 8 9 10; do + curl -si "$BASE/api/v1/users/me" -H "Authorization: Bearer $TOKEN" | grep -i '^x-instance-id' +done | sort | uniq -c \ No newline at end of file diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..6939ede0 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Останавливает ОДНУ API-реплику по TARGET_INSTANCE_ID (= hostname контейнера = короткий ID) +# Реплика остаётся остановленной. БД, LB и volumes не затрагиваются +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + ""|*[!A-Za-z0-9_.-]*) echo "bad instance id" >&2; exit 2 ;; +esac +cd "$(dirname "$0")/.." +CID=$(docker ps --filter "label=com.docker.compose.service=api" \ + --filter "id=$TARGET_INSTANCE_ID" --format '{{.ID}}') +[ -n "$CID" ] || { echo "no running api container for this instance" >&2; exit 1; } +docker stop -t 2 "$CID" >/dev/null +echo "stopped api container $CID" >&2 \ No newline at end of file diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..4653429d --- /dev/null +++ b/src/auth.py @@ -0,0 +1,87 @@ +import asyncio +import base64 +import binascii +import hashlib +import secrets +from typing import Optional, Tuple + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from fastapi import HTTPException, Request + +_hasher = PasswordHasher() +# Хеш-заглушка: для несуществующего пользователя тратим столько же времени, сколько для существующего. +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing") + +BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'} +BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + + +async def hash_password(password: str) -> str: + return await asyncio.to_thread(_hasher.hash, password) # Argon2 грузит CPU, не блокируем event loop + + +async def verify_password(stored_hash: Optional[str], password: str) -> bool: + try: + return await asyncio.to_thread(_hasher.verify, stored_hash or _DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(32) # 256 бит случайности + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode()).digest() + + +def parse_basic(header: Optional[str]) -> Optional[Tuple[str, str]]: + if not header: + return None + scheme, _, value = header.partition(" ") + if scheme.lower() != "basic" or not value: + return None + try: + raw = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + return None + username, sep, password = raw.partition(":") # двоеточие в пароле допустимо + return (username, password) if sep else None + + +def parse_bearer(header: Optional[str]) -> Optional[str]: + if not header: + return None + scheme, _, value = header.partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512: + return None + return value + + +_SESSION_SQL = """ +SELECT u.id, u.username, u.display_name, u.created_at, + EXTRACT(EPOCH FROM (s.expires_at - now())) AS remaining +FROM sessions s JOIN users u ON u.id = s.user_id +WHERE s.token_hash = $1 AND s.revoked_at IS NULL AND s.expires_at > now() +""" + + +async def current_user(request: Request): + # Только Bearer. Сначала кэш, при промахе или сбое - PostgreSQL (источник истины). + token = parse_bearer(request.headers.get("Authorization")) + if token is None: + raise HTTPException(401, headers=BEARER_CHALLENGE) + digest = token_hash(token) + cache = request.app.state.cache + + user = await cache.get(digest) + if user is None: + row = await request.app.state.pool.fetchrow(_SESSION_SQL, digest) + if row is None: + raise HTTPException(401, headers=BEARER_CHALLENGE) + user = {k: row[k] for k in ("id", "username", "display_name", "created_at")} + await cache.put(digest, user, int(row["remaining"])) + request.state.token_hash = digest + return user \ No newline at end of file diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..a2f14c8d --- /dev/null +++ b/src/cache.py @@ -0,0 +1,82 @@ +import asyncio +import json +import logging +import time +import uuid +from datetime import datetime +from typing import Optional + +import redis.asyncio as redis +from redis.exceptions import RedisError + +from . import config +from .common import iso + +logger = logging.getLogger("messenger") + +_FAILURES = (RedisError, OSError, asyncio.TimeoutError) +_RETRY_AFTER = 5.0 # после сбоя 5 секунд не обращаемся к кэшу, чтобы не ждать таймаут на каждом запросе + + +def _key(digest: bytes) -> str: + return "sess:" + digest.hex() + + +class SessionCache: + """Кэш проверенных сессий. Любой сбой = промах: ответ берётся из PostgreSQL.""" + + def __init__(self, url: str): + self._client = ( + redis.Redis.from_url( + url, decode_responses=True, + socket_timeout=config.CACHE_TIMEOUT, socket_connect_timeout=config.CACHE_TIMEOUT, + ) if url else None + ) + self._down_until = 0.0 + + def _usable(self) -> bool: + return self._client is not None and time.monotonic() >= self._down_until + + def _mark_down(self, exc: Exception) -> None: + self._down_until = time.monotonic() + _RETRY_AFTER + logger.warning("Cache unavailable, bypassing: %s", type(exc).__name__) + + async def get(self, digest: bytes) -> Optional[dict]: + if not self._usable(): + return None + try: + raw = await self._client.get(_key(digest)) + except _FAILURES as exc: + self._mark_down(exc) + return None + if raw is None: + return None + try: + d = json.loads(raw) + return {"id": uuid.UUID(d["id"]), "username": d["username"], + "display_name": d["display_name"], "created_at": datetime.fromisoformat(d["created_at"])} + except (ValueError, KeyError): + return None + + async def put(self, digest: bytes, user: dict, remaining_seconds: int) -> None: + ttl = min(remaining_seconds, config.CACHE_TTL_SECONDS) # кэш никогда не живёт дольше сессии + if ttl < 1 or not self._usable(): + return + value = json.dumps({"id": str(user["id"]), "username": user["username"], + "display_name": user["display_name"], "created_at": iso(user["created_at"])}) + try: + await self._client.set(_key(digest), value, ex=ttl) + except _FAILURES as exc: + self._mark_down(exc) + + async def delete(self, digest: bytes) -> None: + if not self._usable(): + return + try: + await self._client.delete(_key(digest)) + except _FAILURES as exc: + self._mark_down(exc) + + async def close(self) -> None: + if self._client is not None: + await self._client.aclose() \ No newline at end of file diff --git a/src/chats.py b/src/chats.py new file mode 100644 index 00000000..267b4e46 --- /dev/null +++ b/src/chats.py @@ -0,0 +1,255 @@ +import base64 +import json +import uuid +from datetime import datetime +from typing import List, Optional, Tuple + +import asyncpg +from fastapi import APIRouter, Depends, Query, Request +from pydantic import BaseModel, ConfigDict, Field + +from .auth import current_user +from .common import iso, make_error_response + +import hashlib +import re +from . import config + +router = APIRouter() + +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + +# Захват ключа. Параллельный INSERT ждёт коммита первого; просроченный ключ (старше TTL) перехватывается +_CLAIM_SQL = """ +INSERT INTO idempotency_keys AS k (user_id, method, path, idem_key, request_hash, created_at) +VALUES ($1, 'POST', $2, $3, $4, now()) +ON CONFLICT (user_id, method, path, idem_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, message_id = NULL, created_at = EXCLUDED.created_at + WHERE k.created_at < now() - make_interval(secs => $5) +RETURNING 1 +""" + + +class StrictRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateChatRequest(StrictRequest): + title: str + member_ids: List[str] = Field(min_length=1, max_length=100) + + +class CreateMessageRequest(StrictRequest): + text: str + + +def _req_id(request: Request) -> str: + return request.state.request_id + + +# курсор: непрозрачная строка, привязанная к ресурсу (scope) +def encode_cursor(scope: str, created_at: datetime, row_id: uuid.UUID) -> str: + raw = json.dumps([scope, iso(created_at), str(row_id)]).encode() + return base64.urlsafe_b64encode(raw).decode() + + +def decode_cursor(cursor: str, scope: str) -> Tuple[datetime, uuid.UUID]: + """ValueError, если курсор повреждён или выдан для другого ресурса.""" + try: + sc, ts, rid = json.loads(base64.urlsafe_b64decode(cursor.encode())) + if sc != scope: + raise ValueError("scope mismatch") + return datetime.fromisoformat(ts), uuid.UUID(rid) + except Exception as exc: + raise ValueError("invalid cursor") from exc + + +def _bad_text(value: str) -> bool: + return not value.strip() or "\x00" in value # NUL нельзя хранить в PostgreSQL text + + +_CHAT_COLUMNS = ( + "c.id, c.title, c.created_at, " + "array(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.position) AS member_ids" +) + + +def chat_dict(row) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m) for m in row["member_ids"]], + "created_at": iso(row["created_at"]), + } + + +def message_dict(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso(row["created_at"]), + } + + +def _check_limit(limit: int) -> bool: + return 1 <= limit <= 100 + + +# чаты +@router.post("/chats", status_code=201) +async def create_chat(req: CreateChatRequest, request: Request, user=Depends(current_user)): + req_id = _req_id(request) + if not (1 <= len(req.title) <= 100) or _bad_text(req.title): + return make_error_response(400, "invalid_request", "Invalid title", req_id) + try: + members = [uuid.UUID(m) for m in req.member_ids] + except ValueError: + return make_error_response(400, "invalid_request", "Invalid member id", req_id) + if len(members) != len(set(members)): + return make_error_response(400, "invalid_request", "Duplicate members found", req_id) + if user["id"] not in members: + return make_error_response(400, "invalid_request", "Caller must be in member_ids", req_id) + + chat_id = uuid.uuid4() + try: + async with request.app.state.pool.acquire() as conn: + # Чат и все участники - одна транзакция: неизвестный участник откатывает всё. + async with conn.transaction(): + created_at = await conn.fetchval( + "INSERT INTO chats (id, title, created_at) VALUES ($1, $2, clock_timestamp()) RETURNING created_at", + chat_id, req.title, + ) + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position) " + "SELECT $1, t.u, (t.p - 1)::smallint FROM unnest($2::uuid[]) WITH ORDINALITY AS t(u, p)", + chat_id, members, + ) + except asyncpg.ForeignKeyViolationError: + return make_error_response(400, "invalid_request", "Unknown member", req_id) + + return {"id": str(chat_id), "title": req.title, "member_ids": [str(m) for m in members], + "created_at": iso(created_at)} + + +@router.get("/chats") +async def list_chats(request: Request, limit: int = 50, + cursor: Optional[str] = Query(default=None, min_length=1, max_length=512), + user=Depends(current_user)): + req_id = _req_id(request) + if not _check_limit(limit): + return make_error_response(400, "invalid_request", "Limit must be between 1 and 100", req_id) + scope = "chats:" + str(user["id"]) + base = (f"SELECT {_CHAT_COLUMNS} FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $1 ") + if cursor is None: + rows = await request.app.state.pool.fetch( + base + "ORDER BY c.created_at, c.id LIMIT $2", user["id"], limit + 1) + else: + try: + ts, rid = decode_cursor(cursor, scope) + except ValueError: + return make_error_response(400, "invalid_request", "Invalid cursor", req_id) + rows = await request.app.state.pool.fetch( + base + "WHERE (c.created_at, c.id) > ($2::timestamptz, $3::uuid) " + "ORDER BY c.created_at, c.id LIMIT $4", + user["id"], ts, rid, limit + 1) + items = rows[:limit] + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if len(rows) > limit else None + return {"items": [chat_dict(r) for r in items], "next_cursor": next_cursor} + + +@router.get("/chats/{chat_id}") +async def get_chat(chat_id: uuid.UUID, request: Request, user=Depends(current_user)): + row = await request.app.state.pool.fetchrow( + f"SELECT {_CHAT_COLUMNS} FROM chats c WHERE c.id = $1 " + "AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = $2)", + chat_id, user["id"], + ) + if row is None: + return make_error_response(404, "not_found", "Chat not found", _req_id(request)) + return chat_dict(row) + + +# сообщения +@router.post("/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: uuid.UUID, req: CreateMessageRequest, request: Request, + user=Depends(current_user)): + req_id = _req_id(request) + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key): + return make_error_response(400, "invalid_request", "Missing or invalid Idempotency-Key", req_id) + if _bad_text(req.text): + return make_error_response(400, "invalid_request", "Empty message text", req_id) + if len(req.text) > 4000: + return make_error_response(400, "invalid_request", "Message text exceeds 4000 characters", req_id) + + path = f"/api/v1/chats/{chat_id}/messages" + request_hash = hashlib.sha256( + json.dumps({"text": req.text}, sort_keys=True, ensure_ascii=False).encode()).digest() + + # Захват ключа, вставка сообщения и привязка результата - одна транзакция: + # при сбое откатывается всё, и ключ не фиксирует неудачный результат. + async with request.app.state.pool.acquire() as conn: + async with conn.transaction(): + # Повтор тоже проверяет доступ. + member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user["id"]) + if not member: + return make_error_response(404, "not_found", "Chat not found", req_id) + + claimed = await conn.fetchval( + _CLAIM_SQL, user["id"], path, key, request_hash, float(config.IDEMPOTENCY_TTL_SECONDS)) + if claimed is None: # ключ уже использован: повтор или конфликт + saved = await conn.fetchrow( + "SELECT request_hash, message_id FROM idempotency_keys " + "WHERE user_id = $1 AND method = 'POST' AND path = $2 AND idem_key = $3", + user["id"], path, key) + if saved["request_hash"] != request_hash: + return make_error_response(409, "conflict", "Idempotency-Key reused with another body", req_id) + row = await conn.fetchrow( + "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = $1", + saved["message_id"]) + return message_dict(row) + + row = await conn.fetchrow( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) " + "VALUES ($1, $2, $3, $4, clock_timestamp()) " + "RETURNING id, chat_id, sender_id, text, created_at", + uuid.uuid4(), chat_id, user["id"], req.text) + await conn.execute( + "UPDATE idempotency_keys SET message_id = $1 " + "WHERE user_id = $2 AND method = 'POST' AND path = $3 AND idem_key = $4", + row["id"], user["id"], path, key) + return message_dict(row) + + +@router.get("/chats/{chat_id}/messages") +async def get_messages(chat_id: uuid.UUID, request: Request, limit: int = 50, + cursor: Optional[str] = Query(default=None, min_length=1, max_length=512), + user=Depends(current_user)): + req_id = _req_id(request) + if not _check_limit(limit): + return make_error_response(400, "invalid_request", "Limit must be between 1 and 100", req_id) + pool = request.app.state.pool + is_member = await pool.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user["id"]) + if not is_member: + return make_error_response(404, "not_found", "Chat not found", req_id) + + scope = "msgs:" + str(chat_id) + base = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 " + if cursor is None: + rows = await pool.fetch(base + "ORDER BY created_at, id LIMIT $2", chat_id, limit + 1) + else: + try: + ts, rid = decode_cursor(cursor, scope) + except ValueError: + return make_error_response(400, "invalid_request", "Invalid cursor", req_id) + rows = await pool.fetch( + base + "AND (created_at, id) > ($2::timestamptz, $3::uuid) ORDER BY created_at, id LIMIT $4", + chat_id, ts, rid, limit + 1) + items = rows[:limit] + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if len(rows) > limit else None + return {"items": [message_dict(r) for r in items], "next_cursor": next_cursor} \ No newline at end of file diff --git a/src/common.py b/src/common.py new file mode 100644 index 00000000..ee8e73ad --- /dev/null +++ b/src/common.py @@ -0,0 +1,30 @@ +from datetime import datetime, timezone +from typing import Optional + +from fastapi.responses import JSONResponse +from . import config + + +def iso(dt: datetime) -> str: + """RFC 3339 UTC с Z; микросекунды нужны, чтобы курсор был точным.""" + return dt.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def make_error_response(status_code: int, code: str, message: str, req_id: str, headers: Optional[dict] = None): + h = {"X-Request-Id": req_id, "X-Instance-Id": config.INSTANCE_ID} + if headers: + h.update(headers) + return JSONResponse( + status_code=status_code, + content={"error": {"code": code, "message": message, "request_id": req_id}}, + headers=h, + ) + + +def user_dict(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso(row["created_at"]), + } \ No newline at end of file diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..521c0342 --- /dev/null +++ b/src/config.py @@ -0,0 +1,29 @@ +import os +import socket +from pathlib import Path + + +def _read_secret(file_var: str, direct_var: str) -> str: + """Секрет из файла (DATABASE_URL_FILE), с запасным вариантом из переменной окружения.""" + path = os.environ.get(file_var) + if path: + return Path(path).read_text().strip() + return os.environ.get(direct_var, "") + + +DATABASE_URL = _read_secret("DATABASE_URL_FILE", "DATABASE_URL") +SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + +DB_POOL_MIN = int(os.environ.get("DB_POOL_MIN", "2")) +DB_POOL_MAX = int(os.environ.get("DB_POOL_MAX", "10")) +DB_CONNECT_TIMEOUT = float(os.environ.get("DB_CONNECT_TIMEOUT", "5")) +DB_COMMAND_TIMEOUT = float(os.environ.get("DB_COMMAND_TIMEOUT", "5")) + +MIGRATIONS_DIR = Path(os.environ.get("MIGRATIONS_DIR", Path(__file__).resolve().parent.parent / "migrations")) +CACHE_URL = _read_secret("CACHE_URL_FILE", "CACHE_URL") +CACHE_TTL_SECONDS = int(os.environ.get("CACHE_TTL_SECONDS", "60")) +CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.5")) + +# Идентификатор реплики: hostname контейнера = короткий ID контейнера (по нему stop-one.sh находит контейнер) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) \ No newline at end of file diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..d7292bdc --- /dev/null +++ b/src/db.py @@ -0,0 +1,56 @@ +import asyncio +import logging + +import asyncpg + +from . import config + +logger = logging.getLogger("messenger") + +# Произвольный константный ключ: один экземпляр выполняет миграции, остальные ждут. +MIGRATION_LOCK_KEY = 727_001 + + +async def create_pool() -> asyncpg.Pool: + # Пул ограничен по размеру; у соединений и команд есть таймауты. + return await asyncpg.create_pool( + dsn=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + timeout=config.DB_CONNECT_TIMEOUT, + command_timeout=config.DB_COMMAND_TIMEOUT, + ) + + +async def create_pool_with_retry(attempts: int = 30, delay: float = 1.0) -> asyncpg.Pool: + # БД в compose может подняться позже API. + for attempt in range(1, attempts + 1): + try: + return await create_pool() + except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc: + logger.warning("DB not ready (%d/%d): %s", attempt, attempts, type(exc).__name__) + await asyncio.sleep(delay) + raise RuntimeError("database is unavailable") + + +async def run_migrations(pool: asyncpg.Pool) -> None: + # Применяет migrations/*.sql по порядку имён, каждую один раз. + files = sorted(config.MIGRATIONS_DIR.glob("*.sql")) + async with pool.acquire() as conn: + await conn.execute("SELECT pg_advisory_lock($1)", MIGRATION_LOCK_KEY) + try: + await conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY," + " applied_at timestamptz NOT NULL DEFAULT now())" + ) + applied = {r["version"] for r in await conn.fetch("SELECT version FROM schema_migrations")} + for path in files: + if path.name in applied: + continue + async with conn.transaction(): + await conn.execute(path.read_text()) + await conn.execute("INSERT INTO schema_migrations (version) VALUES ($1)", path.name) + logger.info("Applied migration %s", path.name) + finally: + await conn.execute("SELECT pg_advisory_unlock($1)", MIGRATION_LOCK_KEY) \ No newline at end of file diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..deb01747 --- /dev/null +++ b/src/main.py @@ -0,0 +1,187 @@ +import asyncio +import logging +import re +import sys +import uuid +from contextlib import asynccontextmanager + +import asyncpg +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from pydantic import BaseModel, ConfigDict +from starlette.exceptions import HTTPException + +from . import config +from .auth import (BASIC_CHALLENGE, current_user, hash_password, new_token, + parse_basic, token_hash, verify_password) +from .chats import router as chats_router +from .common import iso, make_error_response, user_dict +from .db import create_pool_with_retry +from .cache import SessionCache + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] [req_id=%(request_id)s] %(message)s", + handlers=[logging.StreamHandler(sys.stdout)], +) + + +class RequestIdFilter(logging.Filter): + def filter(self, record): + if not hasattr(record, "request_id"): + record.request_id = "-" + return True + + +logger = logging.getLogger("messenger") +logger.addFilter(RequestIdFilter()) + +is_shutting_down = False +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + app.state.pool = await create_pool_with_retry() + app.state.cache = SessionCache(config.CACHE_URL) + yield + global is_shutting_down + is_shutting_down = True + await app.state.cache.close() + await app.state.pool.close() + logger.info("Application shutdown complete") + + +app = FastAPI(title="MTUSI Messenger - Lab 2", lifespan=lifespan) +app.include_router(chats_router, prefix="/api/v1") + + +@app.middleware("http") +async def middleware_pipeline(request: Request, call_next): + req_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = req_id + if is_shutting_down and request.url.path != "/health/live": + return make_error_response(503, "unavailable", "Server is shutting down", req_id) + logger.info(f"Incoming request {request.method} {request.url.path}", extra={"request_id": req_id}) + response: Response = await call_next(request) + response.headers["X-Request-Id"] = req_id + response.headers["X-Instance-Id"] = config.INSTANCE_ID + logger.info(f"Completed {request.method} {request.url.path} with status {response.status_code}", + extra={"request_id": req_id}) + return response + + +def _req_id(request: Request) -> str: + return getattr(request.state, "request_id", str(uuid.uuid4())) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + return make_error_response(400, "invalid_request", "Validation error", _req_id(request)) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + code = { + 400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict", + 413: "payload_too_large", 415: "unsupported_media_type", 429: "rate_limited", 503: "unavailable", + }.get(exc.status_code, "internal_error" if exc.status_code >= 500 else "invalid_request") + return make_error_response(exc.status_code, code, "Request failed", _req_id(request), exc.headers) + + +async def dependency_unavailable_handler(request: Request, exc: Exception): + """БД недоступна или не ответила в срок: ограниченная 503, без деталей и без доверия анониму.""" + logger.error("Dependency failure: %s", type(exc).__name__, extra={"request_id": _req_id(request)}) + return make_error_response(503, "unavailable", "Service temporarily unavailable", _req_id(request)) + + +for _exc in (asyncpg.PostgresError, asyncpg.InterfaceError, OSError, asyncio.TimeoutError): + app.add_exception_handler(_exc, dependency_unavailable_handler) + + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + logger.error("Unhandled exception: %s", type(exc).__name__, extra={"request_id": _req_id(request)}) + return make_error_response(500, "internal_error", "Internal server error", _req_id(request)) + + +class StrictRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class CreateUserRequest(StrictRequest): + username: str + display_name: str + password: str + + +@app.get("/health/live") +async def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def health_ready(request: Request): + # Критичная зависимость для готовности - PostgreSQL. + await request.app.state.pool.fetchval("SELECT 1", timeout=2) + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def register_user(req: CreateUserRequest, request: Request): + req_id = _req_id(request) + if not USERNAME_RE.fullmatch(req.username): + return make_error_response(400, "invalid_request", "Invalid username format", req_id) + if not (1 <= len(req.display_name) <= 100) or not req.display_name.strip() or "\x00" in req.display_name: + return make_error_response(400, "invalid_request", "Invalid display_name", req_id) + if not (12 <= len(req.password) <= 128): + return make_error_response(400, "invalid_request", "Invalid password length", req_id) + + pw_hash = await hash_password(req.password) + try: + row = await request.app.state.pool.fetchrow( + "INSERT INTO users (id, username, display_name, password_hash, created_at) " + "VALUES ($1, $2, $3, $4, clock_timestamp()) " + "RETURNING id, username, display_name, created_at", + uuid.uuid4(), req.username, req.display_name, pw_hash, + ) + except asyncpg.UniqueViolationError: # гонку одинаковых username решает UNIQUE, а не SELECT + return make_error_response(409, "conflict", "Username already exists", req_id) + return user_dict(row) + +@app.get("/api/v1/users/me") +async def get_current_user(user=Depends(current_user)): + return user_dict(user) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + req_id = _req_id(request) + creds = parse_basic(request.headers.get("Authorization")) + if creds is None: + return make_error_response(401, "unauthorized", "Unauthorized", req_id, BASIC_CHALLENGE) + username, password = creds + pool = request.app.state.pool + row = await pool.fetchrow( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1", username + ) + ok = await verify_password(row["password_hash"] if row else None, password) + if row is None or not ok: + return make_error_response(401, "unauthorized", "Unauthorized", req_id, BASIC_CHALLENGE) + + token = new_token() + expires_at = await pool.fetchval( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at) " + "VALUES ($1, $2, now(), now() + make_interval(secs => $3)) RETURNING expires_at", + token_hash(token), row["id"], float(config.SESSION_TTL_SECONDS), + ) + return {"token": token, "token_type": "Bearer", "expires_at": iso(expires_at), "user": user_dict(row)} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request, user=Depends(current_user)): + await request.app.state.pool.execute( + "UPDATE sessions SET revoked_at = now() WHERE token_hash = $1", request.state.token_hash + ) + await request.app.state.cache.delete(request.state.token_hash) + return Response(status_code=204) \ No newline at end of file diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..52541c51 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,19 @@ +import asyncio +import logging +import sys + +from .db import create_pool_with_retry, run_migrations + +logging.basicConfig(level=logging.INFO, stream=sys.stdout) + + +async def main() -> None: + pool = await create_pool_with_retry() + try: + await run_migrations(pool) + finally: + await pool.close() + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/student_tests/test_lab2.py b/student_tests/test_lab2.py new file mode 100644 index 00000000..37dd02fc --- /dev/null +++ b/student_tests/test_lab2.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 + +import concurrent.futures +import json +import os +import sys +import uuid +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime +import base64 + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") +PASSWORD = "correct-horse-battery" + + +def request(method, path, expected, body=None, headers=None, raw_body=None): + url = BASE_URL + path + + request_headers = dict(headers or {}) + + if raw_body is not None: + data = raw_body + elif body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + else: + data = None + + if body is not None or raw_body is not None: + request_headers.setdefault("Content-Type", "application/json") + + req = urllib.request.Request( + url, + data=data, + method=method, + headers=request_headers, + ) + + try: + with urllib.request.urlopen(req, timeout=10) as response: + status = response.status + response_headers = response.headers + response_body = response.read() + + except urllib.error.HTTPError as exc: + status = exc.code + response_headers = exc.headers + response_body = exc.read() + + if status != expected: + raise AssertionError( + f"{method} {path}: expected HTTP {expected}, got {status}. " + f"Body={response_body!r}" + ) + + request_id = response_headers.get("X-Request-Id") + if not request_id: + raise AssertionError( + f"{method} {path}: missing X-Request-Id" + ) + + parsed = None + + if response_body: + content_type = response_headers.get_content_type() + + if content_type == "application/json": + parsed = json.loads(response_body.decode("utf-8")) + + if status >= 400: + codes = { + 400: "invalid_request", 401: "unauthorized", 404: "not_found", + 409: "conflict", 413: "payload_too_large", 415: "unsupported_media_type", + 429: "rate_limited", 500: "internal_error", 503: "unavailable", + } + if not isinstance(parsed, dict) or set(parsed) != {"error"}: + raise AssertionError(f"Invalid error response: {parsed!r}") + error = parsed["error"] + if not isinstance(error, dict) or set(error) != {"code", "message", "request_id"}: + raise AssertionError(f"Invalid error fields: {error!r}") + if error["code"] != codes[status] or error["request_id"] != request_id: + raise AssertionError(f"Wrong error code or request ID: {error!r}") + if not isinstance(error["message"], str) or not error["message"].strip(): + raise AssertionError("Error message must be a nonempty string") + + return { + "status": status, + "headers": response_headers, + "json": parsed, + "bytes": response_body, + } + + +def unique_username(prefix): + # username должен соответствовать [a-z0-9_]{3,32} + suffix = uuid.uuid4().hex[:12] + return f"{prefix}_{suffix}" + + +def basic(username, password=PASSWORD): + raw = f"{username}:{password}".encode("utf-8") + return "Basic " + base64.b64encode(raw).decode() + + +def create_user(prefix="usr", login=True): + user = request("POST", "/api/v1/users", 201, body={ + "username": unique_username(prefix), + "display_name": f"Test {prefix}", + "password": PASSWORD, + })["json"] + if login: + session = request("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(user["username"])})["json"] + user["token"] = session["token"] + return user + + +def auth(user): + return {"Authorization": "Bearer " + user["token"]} + + +def create_chat(owner, members, title="Student test chat"): + result = request( + "POST", + "/api/v1/chats", + 201, + headers=auth(owner), + body={ + "title": title, + "member_ids": [u["id"] for u in members], + }, + ) + + return result["json"] + + +def create_message(user, chat, text): + return request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + headers=auth(user), + body={"text": text}, + )["json"] + + +def test_malformed_json(): + for body in (b'{"username": "broken",', b"\xff"): + request("POST", "/api/v1/users", 400, raw_body=body) + + print("PASS malformed JSON and invalid UTF-8 -> 400 with error envelope") + + +def test_duplicate_username(): + username = unique_username("dup") + + body = { + "username": username, + "display_name": "First", + "password": PASSWORD, + } + + request( + "POST", + "/api/v1/users", + 201, + body=body, + ) + + request( + "POST", + "/api/v1/users", + 409, + body={ + "username": username, + "display_name": "Second", + "password": PASSWORD, + }, + ) + + print("PASS duplicate username -> 409") + + +def test_outsider_chat(): + alice = create_user("alice") + bob = create_user("bob") + eve = create_user("eve") + + chat = create_chat( + alice, + [alice, bob], + "Private chat", + ) + + request( + "GET", + f"/api/v1/chats/{chat['id']}", + 404, + headers=auth(eve), + ) + + request( + "GET", + f"/api/v1/chats/{chat['id']}/messages", + 404, + headers=auth(eve), + ) + + request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 404, + headers=auth(eve), + body={"text": "Not allowed"}, + ) + page = request("GET", "/api/v1/chats", 200, headers=auth(eve))["json"] + if page != {"items": [], "next_cursor": None}: + raise AssertionError("Outsider can see a private chat in the list") + + print("PASS outsider cannot access chat -> 404") + + +def test_empty_and_whitespace_message(): + alice = create_user("emptya") + bob = create_user("emptyb") + + chat = create_chat( + alice, + [alice, bob], + "Empty message test", + ) + + request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + headers=auth(alice), + body={"text": ""}, + ) + + request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + headers=auth(alice), + body={"text": " \t\n"}, + ) + + print("PASS empty/whitespace message -> 400") + + +def test_extra_json_field(): + request( + "POST", + "/api/v1/users", + 400, + body={ + "username": unique_username("extra"), + "display_name": "Extra", + "password": PASSWORD, + "admin": True, + }, + ) + + print("PASS extra JSON field -> 400") + + +def test_multi_page_history(): + alice = create_user("pagea") + bob = create_user("pageb") + + chat = create_chat( + alice, + [alice, bob], + "Pagination test", + ) + + expected = [] + + for i in range(7): + message = create_message( + alice, + chat, + f"pagination-message-{i}", + ) + + expected.append(message) + + expected.sort( + key=lambda message: ( + datetime.fromisoformat( + message["created_at"].replace("Z", "+00:00") + ), + uuid.UUID(message["id"]).int, + ) + ) + + expected_ids = [ + message["id"] + for message in expected + ] + + collected = [] + cursor = None + + for _ in range(len(expected) + 1): + path = ( + f"/api/v1/chats/{chat['id']}/messages" + f"?limit=2" + ) + + if cursor is not None: + path += "&cursor=" + urllib.parse.quote( + cursor, + safe="", + ) + + page = request( + "GET", + path, + 200, + headers=auth(bob), + )["json"] + + ids = [item["id"] for item in page["items"]] + collected.extend(ids) + + cursor = page["next_cursor"] + + if cursor is None: + break + else: + raise AssertionError("Pagination did not reach the last page") + + if collected != expected_ids: + raise AssertionError( + "Pagination lost, duplicated or reordered messages.\n" + f"Expected: {expected_ids}\n" + f"Received: {collected}" + ) + + if len(collected) != len(set(collected)): + raise AssertionError( + "Pagination returned duplicate messages" + ) + + print( + f"PASS pagination: {len(collected)} messages " + "read through multiple pages" + ) + + +def test_concurrent_100_messages(): + users = [ + create_user(f"c{i}") + for i in range(10) + ] + + chat = create_chat( + users[0], + users, + "Concurrency test", + ) + + def worker(client_index): + sent = [] + + for message_index in range(10): + text = ( + f"client-{client_index}-" + f"message-{message_index}" + ) + + result = create_message( + users[client_index], + chat, + text, + ) + + sent.append(result) + + return sent + + all_sent = [] + + with concurrent.futures.ThreadPoolExecutor( + max_workers=10 + ) as executor: + futures = [ + executor.submit(worker, i) + for i in range(10) + ] + + for future in concurrent.futures.as_completed( + futures + ): + all_sent.extend(future.result()) + + if len(all_sent) != 100: + raise AssertionError( + f"Expected 100 successful POSTs, got {len(all_sent)}" + ) + + page = request( + "GET", + f"/api/v1/chats/{chat['id']}/messages?limit=100", + 200, + headers=auth(users[0]), + )["json"] + + messages = page["items"] + + if len(messages) != 100: + raise AssertionError( + f"Expected exactly 100 stored messages, " + f"got {len(messages)}" + ) + + ids = [message["id"] for message in messages] + + if len(set(ids)) != 100: + raise AssertionError( + "Expected 100 unique message IDs" + ) + + if set(ids) != {message["id"] for message in all_sent}: + raise AssertionError("Stored messages differ from successful POST responses") + + texts = [message["text"] for message in messages] + + if len(set(texts)) != 100: + raise AssertionError( + "Expected 100 unique message texts" + ) + + if page["next_cursor"] is not None: + raise AssertionError( + "100 messages with limit=100 should fit in one page" + ) + + print( + "PASS concurrency: 10 clients sent 100 messages; " + "exactly 100 unique messages stored" + ) + + +def test_invalid_chat_ids(): + user = create_user("uuid") + for chat_id, expected in (("not-a-uuid", 400), (str(uuid.uuid4()), 404)): + path = f"/api/v1/chats/{chat_id}" + request("GET", path, expected, headers=auth(user)) + request("GET", path + "/messages", expected, headers=auth(user)) + request("POST", path + "/messages", expected, + headers=auth(user), body={"text": "Test"}) + print("PASS invalid chat UUID -> 400; unknown valid UUID -> 404") + + +def test_chat_member_limits(): + users = [create_user("member", login=(i == 0)) for i in range(101)] + owner = users[0] + for members in ([], users, [owner, owner], users[1:3]): + request("POST", "/api/v1/chats", 400, headers=auth(owner), body={ + "title": "Invalid members", "member_ids": [user["id"] for user in members], + }) + request("POST", "/api/v1/chats", 400, headers=auth(owner), body={ + "title": "Unknown member", "member_ids": [owner["id"], str(uuid.uuid4())], + }) + chat = create_chat(owner, users[:100]) + if len(chat["member_ids"]) != 100: + raise AssertionError("Chat must accept exactly 100 members") + print("PASS chat members: 1..100, unique, existing, caller included") + + +def test_invalid_pagination(): + user = create_user("cursor") + chat = create_chat(user, [user]) + other_chat = create_chat(user, [user]) + other_message = create_message(user, other_chat, "Other history") + paths = ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages") + for path in paths: + for query in ("cursor=", "cursor=unknown", "limit=0", "limit=101", "limit=abc", "limit=1.5"): + request("GET", path + "?" + query, 400, headers=auth(user)) + request("GET", paths[1] + "?cursor=" + other_message["id"], 400, headers=auth(user)) + print("PASS invalid cursor/limit and cursor from another chat -> 400") + + +def test_multi_page_chats(): + user = create_user("chatpage") + chats = [create_chat(user, [user], f"Chat {i}") for i in range(7)] + chats.sort(key=lambda chat: (chat["created_at"], uuid.UUID(chat["id"]).int)) + collected = [] + cursor = None + for page_number in range(len(chats) + 1): + query = {"limit": 2 if page_number == 0 else 3} + if cursor is not None: + query["cursor"] = cursor + page = request("GET", "/api/v1/chats?" + urllib.parse.urlencode(query), + 200, headers=auth(user))["json"] + collected.extend(chat["id"] for chat in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + else: + raise AssertionError("Chat pagination did not reach the last page") + if collected != [chat["id"] for chat in chats]: + raise AssertionError("Chat pagination lost, duplicated or reordered chats") + print("PASS pagination: 7 chats, changing limit between pages") + + +def test_validation_and_unicode(): + for display_name in ("", " \t", "x" * 101, 123, None): + request("POST", "/api/v1/users", 400, body={ + "username": unique_username("invalid"), "display_name": display_name, "password": PASSWORD, + }) + request("POST", "/api/v1/users", 400, + body={"username": True, "display_name": "Test", "password": PASSWORD}) + for password in ("short", "x" * 129, 12345678901234, None): + request("POST", "/api/v1/users", 400, body={ + "username": unique_username("badpw"), "display_name": "Test", "password": password, + }) + request("POST", "/api/v1/users", 400, + body={"username": unique_username("nopw"), "display_name": "Test"}) + + unicode_password = "\U00020000" * 12 # пароль не из ASCII проходит через Basic/UTF-8 и KDF + user = request("POST", "/api/v1/users", 201, body={ + "username": unique_username("unicode"), "display_name": "\U00020000" * 100, + "password": unicode_password, + })["json"] + session = request("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(user["username"], unicode_password)})["json"] + user["token"] = session["token"] + + chat = create_chat(user, [user]) + path = f"/api/v1/chats/{chat['id']}/messages" + text = "\U00020000" * 4000 + message = create_message(user, chat, text) + if message["text"] != text: + raise AssertionError("Unicode message was altered") + for invalid in (text + "x", 123, None): + request("POST", path, 400, headers=auth(user), body={"text": invalid}) + + request("GET", "/api/v1/users/me", 401) + request("GET", "/api/v1/users/me", 401, headers={"X-User-Id": user["id"]}) + request("GET", "/api/v1/users/me", 401, + headers={"Authorization": basic(user["username"], unicode_password)}) + print("PASS strict types, Unicode, password rules, Bearer-only identification") + + +def main(): + tests = [ + test_malformed_json, + test_duplicate_username, + test_outsider_chat, + test_empty_and_whitespace_message, + test_extra_json_field, + test_multi_page_history, + test_concurrent_100_messages, + test_invalid_chat_ids, + test_chat_member_limits, + test_invalid_pagination, + test_multi_page_chats, + test_validation_and_unicode, + ] + + print(f"Testing {BASE_URL}") + + for test in tests: + test() + + print() + print( + f"PASS: {len(tests)} student test scenarios completed" + ) + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print( + f"FAIL: {type(exc).__name__}: {exc}", + file=sys.stderr, + ) + sys.exit(1) diff --git a/student_tests/test_level4.py b/student_tests/test_level4.py new file mode 100644 index 00000000..d523e36b --- /dev/null +++ b/student_tests/test_level4.py @@ -0,0 +1,195 @@ +"""Проверки уровня 4. Нужен запущенный стенд (docker compose up -d --build). +Тесты отказов сами останавливают и запускают valkey/postgres/api, volumes не удаляются. +Запуск: python3 -m unittest discover -s student_tests -p "test_level4.py" -v +""" +import base64 +import json +import os +import subprocess +import threading +import time +import unittest +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +BASE = os.environ.get("BASE_URL", "http://localhost:8080") +ROOT = Path(__file__).resolve().parents[1] +PASSWORD = "correct-horse-battery" + + +def call(method, path, body=None, headers=None): + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json"} if body is not None else {} + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + raw = resp.read() + return resp.status, (json.loads(raw) if raw else None) + except urllib.error.HTTPError as exc: + raw = exc.read() + try: + return exc.code, (json.loads(raw) if raw else None) + except ValueError: + return exc.code, None + + +def register(username=None): + username = username or "u" + uuid.uuid4().hex[:12] + status, user = call("POST", "/api/v1/users", + {"username": username, "display_name": "Test", "password": PASSWORD}) + assert status == 201, status + return user + + +def login(username, password=PASSWORD): + basic = base64.b64encode(f"{username}:{password}".encode()).decode() + return call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + + +def bearer(token): + return {"Authorization": "Bearer " + token} + + +def compose(*args, env=None): + full_env = dict(os.environ) + full_env.update(env or {}) + result = subprocess.run(["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, env=full_env, check=True, capture_output=True, text=True) + return result.stdout + + +def wait_ready(timeout=90): + deadline = time.time() + timeout + while time.time() < deadline: + try: + if call("GET", "/health/ready")[0] == 200: + return + except OSError: + pass + time.sleep(1) + raise AssertionError("API did not become ready") + + +def new_session(): + user = register() + status, session = login(user["username"]) + assert status == 201 + return user, session["token"] + + +class Level4(unittest.TestCase): + @classmethod + def setUpClass(cls): + wait_ready() + + def test_wrong_password(self): + user = register() + status, body = login(user["username"], "definitely-wrong-password") + self.assertEqual(status, 401) + self.assertEqual(body["error"]["code"], "unauthorized") + status, _ = login("no_such_user_" + uuid.uuid4().hex[:8]) + self.assertEqual(status, 401) + + def test_logout_revokes_session_and_cache(self): + _, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) # прогрев кэша + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", headers=bearer(token))[0], 204) + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 401) + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", headers=bearer(token))[0], 401) + + def test_x_user_id_gives_no_access(self): + user = register() + status, _ = call("GET", "/api/v1/users/me", headers={"X-User-Id": user["id"]}) + self.assertEqual(status, 401) + + def test_register_race_same_username(self): + username = "race" + uuid.uuid4().hex[:10] + results = [] + + def worker(): + status, _ = call("POST", "/api/v1/users", + {"username": username, "display_name": "Race", "password": PASSWORD}) + results.append(status) + + threads = [threading.Thread(target=worker) for _ in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + self.assertEqual(results.count(201), 1, results) + self.assertEqual(results.count(409), 9, results) + + def test_chat_creation_rolls_back(self): + user, token = new_session() + title = "rollback-" + uuid.uuid4().hex + status, _ = call("POST", "/api/v1/chats", + {"title": title, "member_ids": [user["id"], str(uuid.uuid4())]}, + headers=bearer(token)) + self.assertEqual(status, 400) + # Проверяем саму БД: после отката не осталось ни чата, ни участников. + count = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-tAc", f"SELECT count(*) FROM chats WHERE title = '{title}'").strip() + self.assertEqual(count, "0") + + def test_valkey_down_falls_back_to_postgres(self): + user, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + compose("stop", "valkey") + try: + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + self.assertEqual(login(user["username"])[0], 201) + self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer("x" * 43))[0], 401) + finally: + compose("start", "valkey") + + def test_valkey_flush_keeps_sessions(self): + _, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + compose("exec", "-T", "valkey", "valkey-cli", "FLUSHALL") + # кэш пуст, но сессия живёт в PostgreSQL; ответ не зависит от состояния кэша + deadline = time.time() + 10 + refilled = False + while time.time() < deadline: + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + keys = compose("exec", "-T", "valkey", "valkey-cli", "keys", "sess:*") + if keys.strip(): + refilled = True + break + time.sleep(1) + self.assertTrue(refilled, "cache should be refilled after a miss") + + def test_postgres_down_returns_bounded_error(self): + compose("stop", "postgres") + try: + status, body = call("GET", "/health/ready") + self.assertEqual(status, 503) + self.assertEqual(body["error"]["code"], "unavailable") + # неизвестный токен нельзя ни подтвердить, ни отвергнуть без БД: 503, не 200 + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer("y" * 43))[0], 503) + self.assertEqual(call("GET", "/api/v1/users/me")[0], 401) # аноним не доверенный + self.assertEqual(call("POST", "/api/v1/users", + {"username": "down" + uuid.uuid4().hex[:8], "display_name": "X", + "password": PASSWORD})[0], 503) + finally: + compose("start", "postgres") + wait_ready() + + def test_zz_session_expiry(self): + compose("up", "-d", "api", env={"SESSION_TTL_SECONDS": "2"}) + try: + wait_ready() + _, token = new_session() + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 200) + time.sleep(3.5) + self.assertEqual(call("GET", "/api/v1/users/me", headers=bearer(token))[0], 401) + finally: + compose("up", "-d", "api") # обратно к TTL из .env + wait_ready() + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/student_tests/test_lifecycle.py b/student_tests/test_lifecycle.py new file mode 100644 index 00000000..5b87eff8 --- /dev/null +++ b/student_tests/test_lifecycle.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Жизненный цикл ЛР 2: данные и сессии переживают перезапуск всего стенда (API, PostgreSQL, Valkey). +Нужен запущенный compose-стенд; перезапуск делает scripts/restart.sh (volumes сохраняются).""" +import os +import sys +import uuid + +import test_lab2 as api +from test_sigterm import ROOT, run, wait_ready + + +def compose_logs(): + return run("docker", "compose", "--project-directory", str(ROOT), "--env-file", str(ROOT / ".env"), + "logs", "api", "--no-color").stdout + + +def main(): + api.BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + wait_ready(api.BASE_URL) + + alice = api.create_user("lifealice") + bob = api.create_user("lifebob") + outsider = api.create_user("lifeoutsider") + chat = api.create_chat(alice, [alice, bob], "Lifecycle test") + texts = ["private-lifecycle-" + uuid.uuid4().hex for _ in range(2)] + sent = [api.create_message(user, chat, text) for user, text in zip((alice, bob), texts)] + path = f"/api/v1/chats/{chat['id']}/messages" + page = api.request("GET", path, 200, headers=api.auth(bob))["json"] + if [m["id"] for m in page["items"]] != [m["id"] for m in sent]: + raise AssertionError("Created messages are missing or reordered") + print("PASS two users created and read two messages: HTTP 201, then HTTP 200") + + api.request("GET", path, 404, headers=api.auth(outsider)) + api.request("POST", path, 404, headers=api.auth(outsider), body={"text": "Forbidden"}) + print("PASS third user cannot read or write the chat: HTTP 404") + + logs = compose_logs() + if any(text in logs for text in texts): + raise AssertionError("Message contents leaked into container logs") + if "req_id=" not in logs or "with status 201" not in logs: + raise AssertionError("Request logs are missing") + print("PASS request logs include IDs and statuses, without message contents") + + print("Command: scripts/restart.sh (API + PostgreSQL + Valkey, volumes kept)", flush=True) + run(str(ROOT / "scripts" / "restart.sh"), timeout=120) + wait_ready(api.BASE_URL, timeout=90) + + # те же токены, те же данные + api.request("GET", "/api/v1/users/me", 200, headers=api.auth(alice)) + after = api.request("GET", path, 200, headers=api.auth(bob))["json"] + if after["items"] != page["items"]: + raise AssertionError("History changed after restart") + api.request("GET", path, 404, headers=api.auth(outsider)) + print("PASS after restart: same tokens read the same history") + + api.request("DELETE", "/api/v1/auth/sessions/current", 204, headers=api.auth(alice)) + api.request("GET", path, 401, headers=api.auth(alice)) + api.request("GET", path, 200, headers=api.auth(bob)) + print("PASS logout revokes only alice's session; bob's session still works") + print("PASS persistent lifecycle: data and sessions survive full restart") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"FAIL: {type(exc).__name__}: {exc}", file=sys.stderr) + sys.exit(1) \ No newline at end of file diff --git a/student_tests/test_sigterm.py b/student_tests/test_sigterm.py new file mode 100644 index 00000000..384cfc36 --- /dev/null +++ b/student_tests/test_sigterm.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 + +import concurrent.futures +import os +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab2") +NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab2_default") +PORT = int(os.environ.get("SIGTERM_PORT", "18080")) + + +def run(*args, check=True, timeout=60): + return subprocess.run( + args, check=check, text=True, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, timeout=timeout, + ) + +def docker_run_args(name, port): + # Отдельный контейнер API в сети compose: те же PostgreSQL и Valkey, секрет только для чтения + return ["docker", "run", "-d", "--name", name, "--network", NETWORK, + "-p", f"127.0.0.1:{port}:8080", + "-v", f"{ROOT}/secrets/database_url:/run/secrets/database_url:ro", + "-e", "DATABASE_URL_FILE=/run/secrets/database_url", + "-e", "CACHE_URL=redis://valkey:6379/0", + IMAGE] + + +def wait_ready(base_url, timeout=60): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=1) as response: + if response.status == 200: + return + except (urllib.error.URLError, ConnectionError, TimeoutError): + pass + time.sleep(0.2) + raise RuntimeError("Test container did not become ready") + + +def connection_rejected(base_url): + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=0.5): + return False + except urllib.error.HTTPError: + return False + except (urllib.error.URLError, ConnectionError, TimeoutError): + return True + + +def test_shutdown(incomplete_request): + name = "mtusi-messenger-sigterm-" + uuid.uuid4().hex[:8] + base_url = f"http://127.0.0.1:{PORT}" + scenario = "incomplete POST" if incomplete_request else "idle" + pending = None + try: + run(*docker_run_args(name, PORT)) + wait_ready(base_url) + + if incomplete_request: + pending = socket.create_connection(("127.0.0.1", PORT), timeout=3) + pending.sendall( + b"POST /api/v1/users HTTP/1.1\r\n" + b"Host: localhost\r\nContent-Type: application/json\r\n" + b"Content-Length: 100\r\nExpect: 100-continue\r\n\r\n" + ) + response = b"" + while b"\r\n\r\n" not in response: + part = pending.recv(1024) + if not part: + raise AssertionError("Connection closed before 100 Continue") + response += part + if b"100 Continue" not in response: + raise AssertionError(f"Expected 100 Continue, received {response!r}") + pending.sendall(b"{") + + print(f"Scenario: {scenario}; docker stop --timeout 10 {name}", flush=True) + start = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: + stopped = executor.submit(run, "docker", "stop", "--timeout", "10", name) + deadline = start + 2 + while not connection_rejected(base_url): + if time.monotonic() >= deadline: + raise AssertionError("New connections are still accepted during shutdown") + time.sleep(0.05) + rejected_after = time.monotonic() - start + stopped.result(timeout=12) + elapsed = time.monotonic() - start + + exit_code = int(run("docker", "inspect", name, "--format", "{{.State.ExitCode}}").stdout) + if exit_code != 0: + raise AssertionError(f"Container exited with code {exit_code}, expected 0") + if elapsed > 10: + raise AssertionError(f"Shutdown exceeded 10 seconds: {elapsed:.3f}s") + if not connection_rejected(base_url): + raise AssertionError("API still accepts connections after shutdown") + + logs = run("docker", "logs", name).stdout + if "Application shutdown complete" not in logs: + raise AssertionError("Application did not complete its shutdown handler") + print(f"PASS {scenario}: exit code 0; shutdown {elapsed:.3f}s; " + f"new connections rejected after {rejected_after:.3f}s") + finally: + if pending is not None: + pending.close() + run("docker", "rm", "-f", name, check=False) + + +def main(): + print(f"Image: {IMAGE}") + test_shutdown(False) + test_shutdown(True) + print("PASS SIGTERM: both scenarios completed within 10 seconds") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"FAIL: {type(exc).__name__}: {exc}", file=sys.stderr) + sys.exit(1) -- 2.54.0 From 32442d06db778650dadcf50be961704431679931 Mon Sep 17 00:00:00 2001 From: Sona Date: Tue, 10 Nov 2026 04:17:24 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=9B=D0=A0=203=20=D1=83=D1=80=D0=BE=D0=B2?= =?UTF-8?q?=D0=B5=D0=BD=D1=8C=204:=20drain=20=D0=BF=D1=80=D0=B8=20SIGTERM,?= =?UTF-8?q?=20=D0=BB=D0=B8=D0=BC=D0=B8=D1=82=D1=8B=20=D1=80=D0=B5=D1=81?= =?UTF-8?q?=D1=83=D1=80=D1=81=D0=BE=D0=B2,=20=D0=BD=D0=B0=D0=B3=D1=80?= =?UTF-8?q?=D1=83=D0=B7=D0=BA=D0=B0=20=D1=81=20=D0=BE=D1=82=D0=BA=D0=B0?= =?UTF-8?q?=D0=B7=D0=B0=D0=BC=D0=B8,=20rolling=20update?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile | 3 + compose.yaml | 34 +- evidence/docker-stats.txt | 6 + evidence/failover-level4.txt | 4 + evidence/idempotency-parallel.txt | 2 + evidence/load-baseline.json | 144 ++++++ evidence/load-rolling.json | 526 +++++++++++++++++++++ evidence/load-stop-0-ps.txt | 7 + evidence/load-stop-0.json | 195 ++++++++ evidence/load-stop-1-ps.txt | 7 + evidence/load-stop-1.json | 195 ++++++++ evidence/persistence-level4.txt | 3 + evidence/rolling-ps.txt | 6 + evidence/sigterm-level4.txt | 6 + evidence/smoke-level4.txt | 2 + infra/haproxy/haproxy.cfg | 4 + scripts/rolling-update.sh | 31 ++ scripts/run-failover-series.sh | 25 + src/config.py | 6 +- src/main.py | 26 +- student_tests/common.py | 34 ++ student_tests/loadgen.py | 106 +++++ student_tests/test_idempotency_parallel.py | 43 ++ student_tests/test_sigterm.py | 27 +- 24 files changed, 1429 insertions(+), 13 deletions(-) create mode 100644 evidence/docker-stats.txt create mode 100644 evidence/failover-level4.txt create mode 100644 evidence/idempotency-parallel.txt create mode 100644 evidence/load-baseline.json create mode 100644 evidence/load-rolling.json create mode 100644 evidence/load-stop-0-ps.txt create mode 100644 evidence/load-stop-0.json create mode 100644 evidence/load-stop-1-ps.txt create mode 100644 evidence/load-stop-1.json create mode 100644 evidence/persistence-level4.txt create mode 100644 evidence/rolling-ps.txt create mode 100644 evidence/sigterm-level4.txt create mode 100644 evidence/smoke-level4.txt create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/run-failover-series.sh create mode 100644 student_tests/common.py create mode 100644 student_tests/loadgen.py create mode 100644 student_tests/test_idempotency_parallel.py diff --git a/Dockerfile b/Dockerfile index 52a4ede3..570e3c9e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,9 @@ RUN pip install --no-cache-dir --timeout 60 -r requirements.txt COPY --chown=app:app src/ /app/src/ COPY --chown=app:app migrations/ /app/migrations/ +ARG APP_VERSION=v1 +ENV APP_VERSION=${APP_VERSION} + USER app EXPOSE 8080 diff --git a/compose.yaml b/compose.yaml index 6f6756c8..715e5ed2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -17,6 +17,11 @@ services: interval: 5s timeout: 3s retries: 10 + deploy: + resources: + limits: + cpus: "${PG_CPUS:-1.0}" + memory: ${PG_MEM:-512M} valkey: image: valkey/valkey:8.0.1-alpine @@ -27,10 +32,18 @@ services: interval: 5s timeout: 3s retries: 10 + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M # Одноразовая задача: применяет миграции и завершается. Реплики API миграции не запускают migrate: - build: . + build: + context: . + args: + APP_VERSION: ${APP_VERSION:-v1} image: mtusi-messenger:lab3 restart: "no" command: ["python", "-m", "src.migrate"] @@ -41,14 +54,24 @@ services: DATABASE_URL_FILE: /run/secrets/database_url secrets: - database_url + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M - # Реплики: число задаётся API_REPLICAS (по умолчанию 2). Без container_name и без published-портов + # Реплики: число API_REPLICAS, ресурсы каждой API_CPUS / API_MEM. Без container_name и published-портов api: image: mtusi-messenger:lab3 pull_policy: never restart: unless-stopped + stop_grace_period: 20s deploy: replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPUS:-0.5}" + memory: ${API_MEM:-256M} depends_on: postgres: condition: service_healthy @@ -60,6 +83,8 @@ services: APP_ENV: ${APP_ENV:-lab} SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DRAIN_SECONDS: ${DRAIN_SECONDS:-5} + DB_POOL_MAX: ${DB_POOL_MAX:-10} DATABASE_URL_FILE: /run/secrets/database_url CACHE_URL: redis://valkey:6379/0 secrets: @@ -77,6 +102,11 @@ services: - ./infra/haproxy:/usr/local/etc/haproxy:ro ports: - "127.0.0.1:8080:8080" + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M secrets: postgres_password: diff --git a/evidence/docker-stats.txt b/evidence/docker-stats.txt new file mode 100644 index 00000000..28e6d2fb --- /dev/null +++ b/evidence/docker-stats.txt @@ -0,0 +1,6 @@ +CONTAINER ID NAME CPU % MEM USAGE / LIMIT MEM % NET I/O BLOCK I/O PIDS +a66867a76907 mtusi-lab3-lb-1 0.28% 101.8MiB / 128MiB 79.52% 2.66MB / 1.7MB 0B / 0B 11 +f67725f7d715 mtusi-lab3-api-2 0.67% 40.29MiB / 256MiB 15.74% 1.69MB / 2.47MB 0B / 0B 5 +8c67bad5eac5 mtusi-lab3-api-1 0.58% 39.65MiB / 256MiB 15.49% 1.59MB / 2.33MB 0B / 0B 8 +c7d5c8d7b77a mtusi-lab3-postgres-1 2.26% 26.25MiB / 512MiB 5.13% 2.36MB / 1.77MB 0B / 614kB 8 +c36affad0631 mtusi-lab3-valkey-1 2.22% 10.35MiB / 128MiB 8.08% 57.3kB / 50.8kB 0B / 0B 5 diff --git a/evidence/failover-level4.txt b/evidence/failover-level4.txt new file mode 100644 index 00000000..ca7ca55c --- /dev/null +++ b/evidence/failover-level4.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 2.223} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/idempotency-parallel.txt b/evidence/idempotency-parallel.txt new file mode 100644 index 00000000..028bd497 --- /dev/null +++ b/evidence/idempotency-parallel.txt @@ -0,0 +1,2 @@ +{"rounds": 5, "parallel_requests_per_round": 30, "messages_in_history": 5, "replicas_that_answered": ["8c67bad5eac5", "f67725f7d715"]} +PASS: same key in parallel -> exactly one message; different body -> 409 diff --git a/evidence/load-baseline.json b/evidence/load-baseline.json new file mode 100644 index 00000000..a3e55d3b --- /dev/null +++ b/evidence/load-baseline.json @@ -0,0 +1,144 @@ +{ + "summary": { + "duration_s": 30.0, + "concurrency": 8, + "replicas_seen": [ + "8c67bad5eac5", + "f67725f7d715" + ], + "requests": 25691, + "ok": 25691, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 856.4, + "latency_ms": { + "p50": 6.4, + "p95": 28.7, + "p99": 61.7 + }, + "error_statuses": {}, + "action": {} + }, + "per_second": { + "0": { + "ok": 833, + "err": 0 + }, + "1": { + "ok": 1006, + "err": 0 + }, + "2": { + "ok": 1001, + "err": 0 + }, + "3": { + "ok": 656, + "err": 0 + }, + "4": { + "ok": 959, + "err": 0 + }, + "5": { + "ok": 577, + "err": 0 + }, + "6": { + "ok": 1010, + "err": 0 + }, + "7": { + "ok": 934, + "err": 0 + }, + "8": { + "ok": 973, + "err": 0 + }, + "9": { + "ok": 966, + "err": 0 + }, + "10": { + "ok": 976, + "err": 0 + }, + "11": { + "ok": 1011, + "err": 0 + }, + "12": { + "ok": 939, + "err": 0 + }, + "13": { + "ok": 660, + "err": 0 + }, + "14": { + "ok": 865, + "err": 0 + }, + "15": { + "ok": 686, + "err": 0 + }, + "16": { + "ok": 740, + "err": 0 + }, + "17": { + "ok": 632, + "err": 0 + }, + "18": { + "ok": 748, + "err": 0 + }, + "19": { + "ok": 919, + "err": 0 + }, + "20": { + "ok": 909, + "err": 0 + }, + "21": { + "ok": 932, + "err": 0 + }, + "22": { + "ok": 957, + "err": 0 + }, + "23": { + "ok": 953, + "err": 0 + }, + "24": { + "ok": 666, + "err": 0 + }, + "25": { + "ok": 993, + "err": 0 + }, + "26": { + "ok": 650, + "err": 0 + }, + "27": { + "ok": 859, + "err": 0 + }, + "28": { + "ok": 688, + "err": 0 + }, + "29": { + "ok": 993, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-rolling.json b/evidence/load-rolling.json new file mode 100644 index 00000000..ae6fcfb8 --- /dev/null +++ b/evidence/load-rolling.json @@ -0,0 +1,526 @@ +{ + "summary": { + "duration_s": 120.0, + "concurrency": 8, + "replicas_seen": [ + "4d6691ac2fdf", + "71148d9265a0" + ], + "requests": 95152, + "ok": 95152, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 792.9, + "latency_ms": { + "p50": 6.6, + "p95": 34.3, + "p99": 72.1 + }, + "error_statuses": {}, + "action": { + "target": "4d6691ac2fdf", + "started_at": 10.0, + "finished_at": 47.43, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.18, + "instances_in_last_5s": [ + "3ca50eabc36a", + "722fb646fbb5" + ], + "versions_first_last_seen_s": { + "v1": [ + 0.0, + 31.4 + ], + "v2": [ + 21.2, + 120.0 + ] + } + }, + "per_second": { + "0": { + "ok": 865, + "err": 0 + }, + "1": { + "ok": 709, + "err": 0 + }, + "2": { + "ok": 778, + "err": 0 + }, + "3": { + "ok": 657, + "err": 0 + }, + "4": { + "ok": 988, + "err": 0 + }, + "5": { + "ok": 854, + "err": 0 + }, + "6": { + "ok": 874, + "err": 0 + }, + "7": { + "ok": 823, + "err": 0 + }, + "8": { + "ok": 912, + "err": 0 + }, + "9": { + "ok": 909, + "err": 0 + }, + "10": { + "ok": 699, + "err": 0 + }, + "11": { + "ok": 680, + "err": 0 + }, + "12": { + "ok": 659, + "err": 0 + }, + "13": { + "ok": 514, + "err": 0 + }, + "14": { + "ok": 601, + "err": 0 + }, + "15": { + "ok": 563, + "err": 0 + }, + "16": { + "ok": 574, + "err": 0 + }, + "17": { + "ok": 580, + "err": 0 + }, + "18": { + "ok": 404, + "err": 0 + }, + "19": { + "ok": 590, + "err": 0 + }, + "20": { + "ok": 587, + "err": 0 + }, + "21": { + "ok": 683, + "err": 0 + }, + "22": { + "ok": 831, + "err": 0 + }, + "23": { + "ok": 582, + "err": 0 + }, + "24": { + "ok": 605, + "err": 0 + }, + "25": { + "ok": 842, + "err": 0 + }, + "26": { + "ok": 847, + "err": 0 + }, + "27": { + "ok": 881, + "err": 0 + }, + "28": { + "ok": 813, + "err": 0 + }, + "29": { + "ok": 831, + "err": 0 + }, + "30": { + "ok": 849, + "err": 0 + }, + "31": { + "ok": 588, + "err": 0 + }, + "32": { + "ok": 414, + "err": 0 + }, + "33": { + "ok": 549, + "err": 0 + }, + "34": { + "ok": 311, + "err": 0 + }, + "35": { + "ok": 419, + "err": 0 + }, + "36": { + "ok": 458, + "err": 0 + }, + "37": { + "ok": 583, + "err": 0 + }, + "38": { + "ok": 809, + "err": 0 + }, + "39": { + "ok": 972, + "err": 0 + }, + "40": { + "ok": 798, + "err": 0 + }, + "41": { + "ok": 594, + "err": 0 + }, + "42": { + "ok": 909, + "err": 0 + }, + "43": { + "ok": 902, + "err": 0 + }, + "44": { + "ok": 765, + "err": 0 + }, + "45": { + "ok": 699, + "err": 0 + }, + "46": { + "ok": 650, + "err": 0 + }, + "47": { + "ok": 809, + "err": 0 + }, + "48": { + "ok": 972, + "err": 0 + }, + "49": { + "ok": 800, + "err": 0 + }, + "50": { + "ok": 914, + "err": 0 + }, + "51": { + "ok": 655, + "err": 0 + }, + "52": { + "ok": 928, + "err": 0 + }, + "53": { + "ok": 900, + "err": 0 + }, + "54": { + "ok": 897, + "err": 0 + }, + "55": { + "ok": 702, + "err": 0 + }, + "56": { + "ok": 921, + "err": 0 + }, + "57": { + "ok": 891, + "err": 0 + }, + "58": { + "ok": 843, + "err": 0 + }, + "59": { + "ok": 921, + "err": 0 + }, + "60": { + "ok": 920, + "err": 0 + }, + "61": { + "ok": 856, + "err": 0 + }, + "62": { + "ok": 651, + "err": 0 + }, + "63": { + "ok": 889, + "err": 0 + }, + "64": { + "ok": 843, + "err": 0 + }, + "65": { + "ok": 690, + "err": 0 + }, + "66": { + "ok": 823, + "err": 0 + }, + "67": { + "ok": 978, + "err": 0 + }, + "68": { + "ok": 901, + "err": 0 + }, + "69": { + "ok": 911, + "err": 0 + }, + "70": { + "ok": 654, + "err": 0 + }, + "71": { + "ok": 758, + "err": 0 + }, + "72": { + "ok": 451, + "err": 0 + }, + "73": { + "ok": 612, + "err": 0 + }, + "74": { + "ok": 887, + "err": 0 + }, + "75": { + "ok": 915, + "err": 0 + }, + "76": { + "ok": 618, + "err": 0 + }, + "77": { + "ok": 983, + "err": 0 + }, + "78": { + "ok": 933, + "err": 0 + }, + "79": { + "ok": 944, + "err": 0 + }, + "80": { + "ok": 927, + "err": 0 + }, + "81": { + "ok": 960, + "err": 0 + }, + "82": { + "ok": 901, + "err": 0 + }, + "83": { + "ok": 593, + "err": 0 + }, + "84": { + "ok": 969, + "err": 0 + }, + "85": { + "ok": 911, + "err": 0 + }, + "86": { + "ok": 727, + "err": 0 + }, + "87": { + "ok": 836, + "err": 0 + }, + "88": { + "ok": 926, + "err": 0 + }, + "89": { + "ok": 920, + "err": 0 + }, + "90": { + "ok": 962, + "err": 0 + }, + "91": { + "ok": 940, + "err": 0 + }, + "92": { + "ok": 979, + "err": 0 + }, + "93": { + "ok": 691, + "err": 0 + }, + "94": { + "ok": 817, + "err": 0 + }, + "95": { + "ok": 967, + "err": 0 + }, + "96": { + "ok": 887, + "err": 0 + }, + "97": { + "ok": 570, + "err": 0 + }, + "98": { + "ok": 967, + "err": 0 + }, + "99": { + "ok": 934, + "err": 0 + }, + "100": { + "ok": 927, + "err": 0 + }, + "101": { + "ok": 947, + "err": 0 + }, + "102": { + "ok": 916, + "err": 0 + }, + "103": { + "ok": 997, + "err": 0 + }, + "104": { + "ok": 682, + "err": 0 + }, + "105": { + "ok": 902, + "err": 0 + }, + "106": { + "ok": 964, + "err": 0 + }, + "107": { + "ok": 691, + "err": 0 + }, + "108": { + "ok": 849, + "err": 0 + }, + "109": { + "ok": 924, + "err": 0 + }, + "110": { + "ok": 987, + "err": 0 + }, + "111": { + "ok": 935, + "err": 0 + }, + "112": { + "ok": 958, + "err": 0 + }, + "113": { + "ok": 804, + "err": 0 + }, + "114": { + "ok": 669, + "err": 0 + }, + "115": { + "ok": 806, + "err": 0 + }, + "116": { + "ok": 917, + "err": 0 + }, + "117": { + "ok": 969, + "err": 0 + }, + "118": { + "ok": 632, + "err": 0 + }, + "119": { + "ok": 918, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-stop-0-ps.txt b/evidence/load-stop-0-ps.txt new file mode 100644 index 00000000..47ba02e5 --- /dev/null +++ b/evidence/load-stop-0-ps.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago +mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 2 hours (healthy) 8080/tcp +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp +mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 2 hours ago +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp diff --git a/evidence/load-stop-0.json b/evidence/load-stop-0.json new file mode 100644 index 00000000..8cf9433d --- /dev/null +++ b/evidence/load-stop-0.json @@ -0,0 +1,195 @@ +{ + "summary": { + "duration_s": 40.0, + "concurrency": 8, + "replicas_seen": [ + "8c67bad5eac5", + "f67725f7d715" + ], + "requests": 25211, + "ok": 25211, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 630.3, + "latency_ms": { + "p50": 6.9, + "p95": 54.8, + "p99": 85.4 + }, + "error_statuses": {}, + "action": { + "target": "8c67bad5eac5", + "started_at": 10.0, + "finished_at": 12.29, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.17, + "instances_in_last_5s": [ + "f67725f7d715" + ] + }, + "per_second": { + "0": { + "ok": 957, + "err": 0 + }, + "1": { + "ok": 947, + "err": 0 + }, + "2": { + "ok": 918, + "err": 0 + }, + "3": { + "ok": 986, + "err": 0 + }, + "4": { + "ok": 608, + "err": 0 + }, + "5": { + "ok": 928, + "err": 0 + }, + "6": { + "ok": 680, + "err": 0 + }, + "7": { + "ok": 918, + "err": 0 + }, + "8": { + "ok": 913, + "err": 0 + }, + "9": { + "ok": 926, + "err": 0 + }, + "10": { + "ok": 878, + "err": 0 + }, + "11": { + "ok": 897, + "err": 0 + }, + "12": { + "ok": 503, + "err": 0 + }, + "13": { + "ok": 582, + "err": 0 + }, + "14": { + "ok": 488, + "err": 0 + }, + "15": { + "ok": 299, + "err": 0 + }, + "16": { + "ok": 575, + "err": 0 + }, + "17": { + "ok": 581, + "err": 0 + }, + "18": { + "ok": 601, + "err": 0 + }, + "19": { + "ok": 586, + "err": 0 + }, + "20": { + "ok": 585, + "err": 0 + }, + "21": { + "ok": 550, + "err": 0 + }, + "22": { + "ok": 579, + "err": 0 + }, + "23": { + "ok": 579, + "err": 0 + }, + "24": { + "ok": 579, + "err": 0 + }, + "25": { + "ok": 308, + "err": 0 + }, + "26": { + "ok": 378, + "err": 0 + }, + "27": { + "ok": 487, + "err": 0 + }, + "28": { + "ok": 561, + "err": 0 + }, + "29": { + "ok": 595, + "err": 0 + }, + "30": { + "ok": 602, + "err": 0 + }, + "31": { + "ok": 599, + "err": 0 + }, + "32": { + "ok": 548, + "err": 0 + }, + "33": { + "ok": 548, + "err": 0 + }, + "34": { + "ok": 578, + "err": 0 + }, + "35": { + "ok": 570, + "err": 0 + }, + "36": { + "ok": 330, + "err": 0 + }, + "37": { + "ok": 498, + "err": 0 + }, + "38": { + "ok": 484, + "err": 0 + }, + "39": { + "ok": 482, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/load-stop-1-ps.txt b/evidence/load-stop-1-ps.txt new file mode 100644 index 00000000..bbcccbfc --- /dev/null +++ b/evidence/load-stop-1-ps.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Up 56 seconds (healthy) 8080/tcp +mtusi-lab3-api-2 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api 2 hours ago Exited (137) 28 seconds ago +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 2 hours ago Up 2 hours 127.0.0.1:8080->8080/tcp +mtusi-lab3-migrate-1 mtusi-messenger:lab3 "python -m src.migra…" migrate 2 hours ago Exited (0) 56 seconds ago +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp diff --git a/evidence/load-stop-1.json b/evidence/load-stop-1.json new file mode 100644 index 00000000..a050253f --- /dev/null +++ b/evidence/load-stop-1.json @@ -0,0 +1,195 @@ +{ + "summary": { + "duration_s": 40.0, + "concurrency": 8, + "replicas_seen": [ + "8c67bad5eac5", + "f67725f7d715" + ], + "requests": 25380, + "ok": 25380, + "errors": 0, + "error_rate": 0.0, + "rps_ok": 634.5, + "latency_ms": { + "p50": 6.9, + "p95": 54.6, + "p99": 81.2 + }, + "error_statuses": {}, + "action": { + "target": "f67725f7d715", + "started_at": 10.0, + "finished_at": 12.29, + "exit_code": 0 + }, + "errors_after_action": 0, + "recovery_s": 0.0, + "max_stall_s": 0.22, + "instances_in_last_5s": [ + "8c67bad5eac5" + ] + }, + "per_second": { + "0": { + "ok": 866, + "err": 0 + }, + "1": { + "ok": 945, + "err": 0 + }, + "2": { + "ok": 991, + "err": 0 + }, + "3": { + "ok": 987, + "err": 0 + }, + "4": { + "ok": 781, + "err": 0 + }, + "5": { + "ok": 967, + "err": 0 + }, + "6": { + "ok": 911, + "err": 0 + }, + "7": { + "ok": 959, + "err": 0 + }, + "8": { + "ok": 996, + "err": 0 + }, + "9": { + "ok": 611, + "err": 0 + }, + "10": { + "ok": 569, + "err": 0 + }, + "11": { + "ok": 743, + "err": 0 + }, + "12": { + "ok": 546, + "err": 0 + }, + "13": { + "ok": 501, + "err": 0 + }, + "14": { + "ok": 570, + "err": 0 + }, + "15": { + "ok": 600, + "err": 0 + }, + "16": { + "ok": 589, + "err": 0 + }, + "17": { + "ok": 557, + "err": 0 + }, + "18": { + "ok": 485, + "err": 0 + }, + "19": { + "ok": 491, + "err": 0 + }, + "20": { + "ok": 333, + "err": 0 + }, + "21": { + "ok": 571, + "err": 0 + }, + "22": { + "ok": 559, + "err": 0 + }, + "23": { + "ok": 565, + "err": 0 + }, + "24": { + "ok": 567, + "err": 0 + }, + "25": { + "ok": 551, + "err": 0 + }, + "26": { + "ok": 570, + "err": 0 + }, + "27": { + "ok": 536, + "err": 0 + }, + "28": { + "ok": 572, + "err": 0 + }, + "29": { + "ok": 519, + "err": 0 + }, + "30": { + "ok": 424, + "err": 0 + }, + "31": { + "ok": 415, + "err": 0 + }, + "32": { + "ok": 587, + "err": 0 + }, + "33": { + "ok": 541, + "err": 0 + }, + "34": { + "ok": 569, + "err": 0 + }, + "35": { + "ok": 584, + "err": 0 + }, + "36": { + "ok": 591, + "err": 0 + }, + "37": { + "ok": 587, + "err": 0 + }, + "38": { + "ok": 551, + "err": 0 + }, + "39": { + "ok": 523, + "err": 0 + } + } +} \ No newline at end of file diff --git a/evidence/persistence-level4.txt b/evidence/persistence-level4.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence-level4.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/rolling-ps.txt b/evidence/rolling-ps.txt new file mode 100644 index 00000000..f7318c27 --- /dev/null +++ b/evidence/rolling-ps.txt @@ -0,0 +1,6 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-3 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi-lab3-api-4 mtusi-messenger:lab3 "sh -c 'exec uvicorn…" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi-lab3-lb-1 haproxy:3.0.7-alpine "docker-entrypoint.s…" lb 3 minutes ago Up 3 minutes 127.0.0.1:8080->8080/tcp +mtusi-lab3-postgres-1 postgres:16.6-alpine "docker-entrypoint.s…" postgres 2 hours ago Up 2 hours (healthy) 5432/tcp +mtusi-lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 hours ago Up 2 hours (healthy) 6379/tcp diff --git a/evidence/sigterm-level4.txt b/evidence/sigterm-level4.txt new file mode 100644 index 00000000..29e91c01 --- /dev/null +++ b/evidence/sigterm-level4.txt @@ -0,0 +1,6 @@ +Image: mtusi-messenger:lab3 +Scenario: idle; docker stop --timeout 20 mtusi-messenger-sigterm-99ecee4b +PASS idle: exit code 0; shutdown 5.353s; new connections rejected after 0.064s +Scenario: incomplete POST; docker stop --timeout 20 mtusi-messenger-sigterm-7da6202d +PASS incomplete POST: exit code 0; shutdown 13.414s; new connections rejected after 0.063s +PASS SIGTERM: both scenarios completed within 10 seconds diff --git a/evidence/smoke-level4.txt b/evidence/smoke-level4.txt new file mode 100644 index 00000000..44e41deb --- /dev/null +++ b/evidence/smoke-level4.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 58bbf906..a422f48f 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -8,6 +8,8 @@ defaults timeout connect 1s timeout client 30s timeout server 10s + timeout check 2s + timeout queue 5s retries 3 option redispatch @@ -28,6 +30,8 @@ frontend public backend api balance roundrobin + # Проверка /health/ready раз в 1 с; backend исключается после 2 неудач подряд, + # возвращается после 2 успехов. Худшее время обнаружения: 2 x 1 с + timeout check 2 с = 4 с. option httpchk http-check send meth GET uri /health/ready http-check expect status 200 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..cce3bf45 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Поочерёдная замена API-реплик новой версией образа (APP_VERSION=v2 scripts/rolling-update.sh). +# Миграции запускаются ОДИН раз до замены; migrations/002 только добавляет таблицу, поэтому старая версия с ней совместима. +set -eu +cd "$(dirname "$0")/.." +: "${APP_VERSION:?set new version, e.g. APP_VERSION=v2}" +export APP_VERSION +COMPOSE="docker compose --env-file .env" + +$COMPOSE build migrate +$COMPOSE run --rm migrate +N=$($COMPOSE ps -q api | wc -l | tr -d ' ') + +wait_healthy() { + i=0 + while [ "$i" -lt 90 ]; do + c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true) + [ "$c" -ge "$N" ] && return 0 + i=$((i + 1)); sleep 1 + done + echo "replicas did not become healthy" >&2; return 1 +} + +for cid in $($COMPOSE ps -q api); do + echo "replacing $cid" >&2 + docker stop -t 20 "$cid" >/dev/null # SIGTERM: readiness 503, drain 5 с, затем завершение + docker rm "$cid" >/dev/null + $COMPOSE up -d --no-deps --no-recreate --scale api="$N" api # создаётся реплика из нового образа + wait_healthy + sleep 5 # LB добавляет новую реплику после 2 успешных проверок +done \ No newline at end of file diff --git a/scripts/run-failover-series.sh b/scripts/run-failover-series.sh new file mode 100755 index 00000000..7b9841ed --- /dev/null +++ b/scripts/run-failover-series.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Базовая нагрузка, затем по очереди отказ каждой API-реплики под нагрузкой с восстановлением между ними. +set -eu +cd "$(dirname "$0")/.." +COMPOSE="docker compose --env-file .env" + +wait_healthy() { + i=0 + while [ "$i" -lt 90 ]; do + c=$($COMPOSE ps api --format '{{.Health}}' | grep -c '^healthy$' || true) + [ "$c" -ge 2 ] && return 0 + i=$((i + 1)); sleep 1 + done + echo "replicas did not become healthy" >&2; return 1 +} + +python3 student_tests/loadgen.py --duration 30 --output evidence/load-baseline.json +for idx in 0 1; do + python3 student_tests/loadgen.py --duration 40 --action-script scripts/stop-one.sh \ + --action-at 10 --target-index "$idx" --output "evidence/load-stop-$idx.json" + $COMPOSE ps -a > "evidence/load-stop-$idx-ps.txt" + $COMPOSE start api + wait_healthy + sleep 10 # LB возвращает реплику после 2 успешных проверок +done \ No newline at end of file diff --git a/src/config.py b/src/config.py index 521c0342..0d5cec5b 100644 --- a/src/config.py +++ b/src/config.py @@ -26,4 +26,8 @@ CACHE_TIMEOUT = float(os.environ.get("CACHE_TIMEOUT", "0.5")) # Идентификатор реплики: hostname контейнера = короткий ID контейнера (по нему stop-one.sh находит контейнер) INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() -IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) \ No newline at end of file +IDEMPOTENCY_TTL_SECONDS = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + +# SIGTERM: сколько секунд реплика отвечает 503 на /health/ready, пока LB её исключает, и продолжает обслуживать запросы. +DRAIN_SECONDS = float(os.environ.get("DRAIN_SECONDS", "5")) +APP_VERSION = os.environ.get("APP_VERSION", "dev") \ No newline at end of file diff --git a/src/main.py b/src/main.py index deb01747..766ed492 100644 --- a/src/main.py +++ b/src/main.py @@ -6,6 +6,7 @@ import uuid from contextlib import asynccontextmanager import asyncpg +import signal from fastapi import Depends, FastAPI, Request, Response from fastapi.exceptions import RequestValidationError from pydantic import BaseModel, ConfigDict @@ -39,11 +40,31 @@ logger.addFilter(RequestIdFilter()) is_shutting_down = False USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +def _install_drain_handler(app: FastAPI) -> None: + """SIGTERM: сначала /health/ready -> 503, и только через DRAIN_SECONDS обычное завершение uvicorn.""" + loop = asyncio.get_running_loop() + original = signal.getsignal(signal.SIGTERM) + if not callable(original): + return + + def on_sigterm(signum, frame): + if app.state.draining: + return + app.state.draining = True + logger.info("SIGTERM received: draining for %s s", config.DRAIN_SECONDS) + loop.call_soon_threadsafe(loop.call_later, config.DRAIN_SECONDS, original, signum, frame) + + try: + signal.signal(signal.SIGTERM, on_sigterm) + except ValueError: # не главный поток (например, в тестах) + pass @asynccontextmanager async def lifespan(app: FastAPI): + app.state.draining = False app.state.pool = await create_pool_with_retry() app.state.cache = SessionCache(config.CACHE_URL) + _install_drain_handler(app) yield global is_shutting_down is_shutting_down = True @@ -66,6 +87,7 @@ async def middleware_pipeline(request: Request, call_next): response: Response = await call_next(request) response.headers["X-Request-Id"] = req_id response.headers["X-Instance-Id"] = config.INSTANCE_ID + response.headers["X-App-Version"] = config.APP_VERSION logger.info(f"Completed {request.method} {request.url.path} with status {response.status_code}", extra={"request_id": req_id}) return response @@ -122,7 +144,9 @@ async def health_live(): @app.get("/health/ready") async def health_ready(request: Request): - # Критичная зависимость для готовности - PostgreSQL. + # Критичная зависимость для готовности - PostgreSQL. При SIGTERM реплика сразу «не готова». + if request.app.state.draining: + return make_error_response(503, "unavailable", "Draining", _req_id(request)) await request.app.state.pool.fetchval("SELECT 1", timeout=2) return {"status": "ok"} diff --git a/student_tests/common.py b/student_tests/common.py new file mode 100644 index 00000000..b649bc94 --- /dev/null +++ b/student_tests/common.py @@ -0,0 +1,34 @@ +import base64 +import json +import urllib.error +import urllib.request +import uuid + + +def call(base, method, path, body=None, headers=None, timeout=3.0): + """Возвращает (status, headers с ключами в нижнем регистре, bytes). Сетевые ошибки поднимаются наружу.""" + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json"} if data else {} + h.update(headers or {}) + req = urllib.request.Request(base + path, data=data, method=method, headers=h) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, {k.lower(): v for k, v in r.headers.items()}, r.read() + except urllib.error.HTTPError as e: + return e.code, {k.lower(): v for k, v in e.headers.items()}, e.read() + + +def setup(base): + """Синтетический пользователь, сессия и чат из одного участника. Возвращает (auth_headers, chat_id).""" + name = "load_" + uuid.uuid4().hex[:12] + password = "load-password-" + uuid.uuid4().hex[:8] + s, _, b = call(base, "POST", "/api/v1/users", {"username": name, "display_name": "Load", "password": password}) + assert s == 201, f"register failed: {s}" + uid = json.loads(b)["id"] + cred = base64.b64encode(f"{name}:{password}".encode()).decode() + s, _, b = call(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert s == 201, f"login failed: {s}" + auth = {"Authorization": "Bearer " + json.loads(b)["token"]} + s, _, b = call(base, "POST", "/api/v1/chats", {"title": "load", "member_ids": [uid]}, auth) + assert s == 201, f"chat failed: {s}" + return auth, json.loads(b)["id"] \ No newline at end of file diff --git a/student_tests/loadgen.py b/student_tests/loadgen.py new file mode 100644 index 00000000..aca9941d --- /dev/null +++ b/student_tests/loadgen.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Closed-loop нагрузка POST /messages через LB; во время неё выполняется action-скрипт (остановка реплики). +Без прозрачных retry: каждая неудача = ошибка. Токены в вывод не попадают.""" +import argparse +import json +import os +import subprocess +import threading +import time +import uuid + +from common import call, setup + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--duration", type=float, default=40) + ap.add_argument("--concurrency", type=int, default=8) + ap.add_argument("--action-script") + ap.add_argument("--action-at", type=float, default=10) + ap.add_argument("--target-index", type=int, default=0) + ap.add_argument("--output") + a = ap.parse_args() + + auth, chat = setup(a.base_url) + path = f"/api/v1/chats/{chat}/messages" + ids = set() + for _ in range(40): # какие реплики видны через LB + _, h, _ = call(a.base_url, "GET", "/api/v1/users/me", headers=auth) + ids.add(h.get("x-instance-id")) + ids = sorted(i for i in ids if i) + + records, lock = [], threading.Lock() # (t, status, latency, instance) + t_start = time.monotonic() + t_end = t_start + a.duration + + def worker(): + while time.monotonic() < t_end: + t0 = time.monotonic() + try: + s, h, _ = call(a.base_url, "POST", path, {"text": "load"}, + {**auth, "Idempotency-Key": uuid.uuid4().hex}, timeout=3) + inst, ver = h.get("x-instance-id"), h.get("x-app-version") + except Exception: + s, inst, ver = 0, None, None + with lock: + records.append((t0 - t_start, s, time.monotonic() - t0, inst, ver)) + + threads = [threading.Thread(target=worker) for _ in range(a.concurrency)] + for t in threads: + t.start() + + action = {} + if a.action_script: + time.sleep(a.action_at) + target = ids[a.target_index % len(ids)] + t0 = time.monotonic() - t_start + rc = subprocess.run([a.action_script], env={**os.environ, "TARGET_INSTANCE_ID": target}, + capture_output=True, timeout=300).returncode + action = {"target": target, "started_at": round(t0, 2), + "finished_at": round(time.monotonic() - t_start, 2), "exit_code": rc} + for t in threads: + t.join() + + ok = [r for r in records if r[1] == 201] + bad = [r for r in records if r[1] != 201] + lat = sorted(r[2] for r in ok) + pct = lambda p: round(lat[min(len(lat) - 1, int(len(lat) * p))] * 1000, 1) if lat else None + summary = { + "duration_s": a.duration, "concurrency": a.concurrency, "replicas_seen": ids, + "requests": len(records), "ok": len(ok), "errors": len(bad), + "error_rate": round(len(bad) / max(1, len(records)), 4), + "rps_ok": round(len(ok) / a.duration, 1), + "latency_ms": {"p50": pct(0.50), "p95": pct(0.95), "p99": pct(0.99)}, + "error_statuses": {str(s): sum(1 for r in bad if r[1] == s) for s in {r[1] for r in bad}}, + "action": action, + } + if action: + errs_after = [r[0] for r in bad if r[0] >= action["started_at"]] + times = sorted(r[0] for r in ok if r[0] >= action["started_at"]) + gaps = [b - a_ for a_, b in zip([action["started_at"]] + times, times)] + summary["errors_after_action"] = len(errs_after) + summary["recovery_s"] = round(max(errs_after) - action["started_at"], 2) if errs_after else 0.0 + summary["max_stall_s"] = round(max(gaps), 2) if gaps else None + summary["instances_in_last_5s"] = sorted({r[3] for r in ok if r[0] >= a.duration - 5 and r[3]}) + + vers = {} + for r in ok: + if r[4]: + v = vers.setdefault(r[4], [r[0], r[0]]) + v[0], v[1] = min(v[0], r[0]), max(v[1], r[0]) + summary["versions_first_last_seen_s"] = {k: [round(x, 1) for x in v] for k, v in vers.items()} + + timeline = {} + for t, s, *_ in records: + b = timeline.setdefault(int(t), {"ok": 0, "err": 0}) + b["ok" if s == 201 else "err"] += 1 + print(json.dumps(summary, ensure_ascii=False, indent=2)) + if a.output: + with open(a.output, "w") as f: + json.dump({"summary": summary, "per_second": timeline}, f, ensure_ascii=False, indent=1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student_tests/test_idempotency_parallel.py b/student_tests/test_idempotency_parallel.py new file mode 100644 index 00000000..56706014 --- /dev/null +++ b/student_tests/test_idempotency_parallel.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""30 параллельных POST с одним Idempotency-Key через LB: ровно одно сообщение, везде 201 и один id.""" +import json +import sys +import threading +import uuid + +from common import call, setup + +BASE = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8080" +THREADS, ROUNDS = 30, 5 + +auth, chat = setup(BASE) +path = f"/api/v1/chats/{chat}/messages" +replicas = set() + +for rnd in range(ROUNDS): + key = uuid.uuid4().hex + barrier = threading.Barrier(THREADS) + results = [None] * THREADS + + def work(i, rnd=rnd, key=key, barrier=barrier, results=results): + barrier.wait() + s, h, b = call(BASE, "POST", path, {"text": f"parallel {rnd}"}, {**auth, "Idempotency-Key": key}, timeout=10) + results[i] = (s, h.get("x-instance-id"), json.loads(b) if s == 201 else None) + + ts = [threading.Thread(target=work, args=(i,)) for i in range(THREADS)] + [t.start() for t in ts] + [t.join() for t in ts] + codes = {r[0] for r in results} + msg_ids = {r[2]["id"] for r in results if r[0] == 201} + replicas |= {r[1] for r in results} + assert codes == {201}, f"round {rnd}: codes {codes}" + assert len(msg_ids) == 1, f"round {rnd}: {len(msg_ids)} different messages" + s, _, _ = call(BASE, "POST", path, {"text": "other body"}, {**auth, "Idempotency-Key": key}) + assert s == 409, f"round {rnd}: other body gave {s}" + +s, _, b = call(BASE, "GET", path + "?limit=100", headers=auth) +count = len(json.loads(b)["items"]) +assert count == ROUNDS, f"history has {count} messages, expected {ROUNDS}" +print(json.dumps({"rounds": ROUNDS, "parallel_requests_per_round": THREADS, + "messages_in_history": count, "replicas_that_answered": sorted(r for r in replicas if r)})) +print("PASS: same key in parallel -> exactly one message; different body -> 409") \ No newline at end of file diff --git a/student_tests/test_sigterm.py b/student_tests/test_sigterm.py index 384cfc36..4651f00a 100644 --- a/student_tests/test_sigterm.py +++ b/student_tests/test_sigterm.py @@ -12,8 +12,8 @@ import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab2") -NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab2_default") +IMAGE = os.environ.get("IMAGE", "mtusi-messenger:lab3") +NETWORK = os.environ.get("COMPOSE_NETWORK", "mtusi-lab3_default") PORT = int(os.environ.get("SIGTERM_PORT", "18080")) @@ -55,6 +55,15 @@ def connection_rejected(base_url): except (urllib.error.URLError, ConnectionError, TimeoutError): return True +def not_ready(base_url): + # Во время drain /health/ready отвечает 503 (или соединение уже закрыто) + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=0.5): + return False + except urllib.error.HTTPError as exc: + return exc.code == 503 + except (urllib.error.URLError, ConnectionError, TimeoutError): + return True def test_shutdown(incomplete_request): name = "mtusi-messenger-sigterm-" + uuid.uuid4().hex[:8] @@ -82,24 +91,24 @@ def test_shutdown(incomplete_request): raise AssertionError(f"Expected 100 Continue, received {response!r}") pending.sendall(b"{") - print(f"Scenario: {scenario}; docker stop --timeout 10 {name}", flush=True) + print(f"Scenario: {scenario}; docker stop --timeout 20 {name}", flush=True) start = time.monotonic() with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: - stopped = executor.submit(run, "docker", "stop", "--timeout", "10", name) + stopped = executor.submit(run, "docker", "stop", "--timeout", "20", name) deadline = start + 2 - while not connection_rejected(base_url): + while not not_ready(base_url): if time.monotonic() >= deadline: - raise AssertionError("New connections are still accepted during shutdown") + raise AssertionError("Readiness still OK after SIGTERM") time.sleep(0.05) rejected_after = time.monotonic() - start - stopped.result(timeout=12) + stopped.result(timeout=25) elapsed = time.monotonic() - start exit_code = int(run("docker", "inspect", name, "--format", "{{.State.ExitCode}}").stdout) if exit_code != 0: raise AssertionError(f"Container exited with code {exit_code}, expected 0") - if elapsed > 10: - raise AssertionError(f"Shutdown exceeded 10 seconds: {elapsed:.3f}s") + if elapsed > 15: + raise AssertionError(f"Shutdown exceeded 15 seconds: {elapsed:.3f}s") if not connection_rejected(base_url): raise AssertionError("API still accepts connections after shutdown") -- 2.54.0 From a35183f49245c478b12275a6afd8d87c7f3017c8 Mon Sep 17 00:00:00 2001 From: Sona Date: Tue, 10 Nov 2026 04:29:57 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=9B=D0=A0=203:=20=D0=BE=D1=82=D1=87?= =?UTF-8?q?=D1=91=D1=82=20=D0=B8=20=D0=B8=D1=82=D0=BE=D0=B3=D0=BE=D0=B2?= =?UTF-8?q?=D1=8B=D0=B5=20=D0=B4=D0=BE=D0=BA=D0=B0=D0=B7=D0=B0=D1=82=D0=B5?= =?UTF-8?q?=D0=BB=D1=8C=D1=81=D1=82=D0=B2=D0=B0=20(=D1=83=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B5=D0=BD=D1=8C=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 124 +++++++++++++++++++++++++++++++----- evidence/restart-after.txt | 7 ++ evidence/restart-before.txt | 7 ++ 3 files changed, 123 insertions(+), 15 deletions(-) create mode 100644 evidence/restart-after.txt create mode 100644 evidence/restart-before.txt diff --git a/REPORT.md b/REPORT.md index c2679005..6bfb16c9 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,131 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Саргсян Сона, БВТ2302 +- Целевая оценка: 4 +- Commit/PR: заполнить после публикации +- Использованные источники, библиотеки и помощь ИИ: Claude Code. Библиотеки: FastAPI, Pydantic, Uvicorn, asyncpg, argon2-cffi, redis-py (клиент Valkey). Код перенесён из собственной ветки `solution/bvt2302-sargsyan` лабораторной работы 2. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, Docker (Compose v2) с запущенным daemon, Make, Python 3.9+ (публичные проверки и собственные тесты используют только стандартную библиотеку), `openssl`. Порт 8080 на localhost должен быть свободен. + +Версии: Python 3.12.7 (образ `python:3.12.7-slim`, закреплён digest), FastAPI 0.115.2, Uvicorn 0.32.0, asyncpg 0.30.0, argon2-cffi 23.1.0, redis-py 5.2.1, PostgreSQL 16.6 (`postgres:16.6-alpine`), Valkey 8.0.1 (`valkey/valkey:8.0.1-alpine`), HAProxy 3.0.7 (`haproxy:3.0.7-alpine`). Железо: ноутбук MacBook на Docker Desktop, точные параметры хоста и VM не фиксировались; нагрузочный генератор работает на той же машине, что и стенд. + +Действия оператора (один раз, секреты в Git не попадают): + +```sh +mkdir -p secrets +PW=$(openssl rand -hex 24) +printf '%s' "$PW" > secrets/postgres_password +printf 'postgresql://messenger:%s@postgres:5432/messenger' "$PW" > secrets/database_url +chmod 644 secrets/* +cp .env.example .env +chmod +x scripts/*.sh +``` + +Права 644 нужны, чтобы пользователь `app` в контейнере прочитал секрет; допустимо только для локального стенда с синтетическими данными. + +Сборка, миграции, запуск (одна последовательность): + +```sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +curl --fail --retry 30 --retry-connrefused --retry-delay 1 http://localhost:8080/health/ready +``` + +Порядок старта задаёт Compose: PostgreSQL (healthy) → одноразовая задача `migrate` (завершается с кодом 0) → реплики `api` → `lb`. Миграции применяет только `migrate` (`python -m src.migrate`); реплики API миграции не запускают. + +Проверки: + +```sh +make check +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # затем: docker compose --env-file .env start api +scripts/show-instances.sh +python3 student_tests/test_idempotency_parallel.py +scripts/run-failover-series.sh +APP_VERSION=v1 docker compose --env-file .env up -d --build +APP_VERSION=v2 docker compose --env-file .env build migrate +APP_VERSION=v2 python3 student_tests/loadgen.py --duration 120 --action-script scripts/rolling-update.sh --action-at 10 --output evidence/load-rolling.json +python3 student_tests/test_sigterm.py +``` + +`make test-failover` останавливает реплику, которую тест выбрал по первому ответу, и оставляет её остановленной. Восстановление: `docker compose --env-file .env start api`. Тест выбирает цель детерминированно (roundrobin), чтобы проверить вторую реплику, перед запуском сдвигается чётность одним запросом `curl http://localhost:8080/health/live`. + +Остановка с сохранением данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +Клиент ──> HAProxy (127.0.0.1:8080, единственный опубликованный порт) + │ проверка GET /health/ready раз в 1 с, адреса из Docker DNS + ├──> api (реплика 1) ──┐ + └──> api (реплика N) ──┼──> PostgreSQL 16 (volume pgdata) — источник истины + └──> Valkey 8 (кэш сессий, без persistence) +migrate (одноразово) ──> PostgreSQL +``` + +| Роль | Service | Версия | Health/readiness | Volume | Сеть/порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| API | `api` (N реплик) | собственный образ `mtusi-messenger:lab3` | Dockerfile HEALTHCHECK и `/health/ready` | нет (stateless) | `expose 8080`, наружу не опубликован | `database_url` (Compose secret, файл) | +| Миграции | `migrate` | тот же образ | код завершения 0 | нет | внутренняя сеть | `database_url` | +| БД | `postgres` | 16.6-alpine | `pg_isready` | `pgdata` | 5432 внутри сети | `postgres_password` (secret) | +| Кэш сессий | `valkey` | 8.0.1-alpine | `valkey-cli ping` | нет | 6379 внутри сети | без пароля, только внутренняя сеть | +| LB | `lb` | HAProxy 3.0.7 | активные проверки backend | нет | `127.0.0.1:8080` | нет | + +**Источник истины.** PostgreSQL хранит пользователей, чаты, сообщения, сессии (SHA-256 токена, срок, отзыв) и ключи идемпотентности. Сессии общие для всех реплик, потому что лежат в PostgreSQL; Valkey лишь ускоряет проверку и недоступность кэша не ломает API (любой сбой считается промахом). Локального состояния в процессе API нет, sticky session не используются. Кэширования истории сообщений нет, поэтому подтверждённое сообщение видно следующему чтению через любую реплику. + +**Идемпотентность отправки сообщения.** Таблица `idempotency_keys` с ключом `(user_id, method, path, idem_key)` и хешем тела. В одной транзакции: проверка членства, захват ключа (`INSERT ... ON CONFLICT`), вставка сообщения и привязка `message_id`. Параллельный запрос с тем же ключом ждёт коммита первого на уникальном индексе и затем возвращает то же сообщение. Другое тело при том же ключе даёт 409. При сбое транзакция откатывается вместе с ключом, поэтому 5xx не фиксируется как результат. Повтор заново проверяет сессию и доступ к чату. Ключи старше `IDEMPOTENCY_TTL_SECONDS` (24 ч) считаются свободными. + +**Балансировка и отказ.** HAProxy проверяет `GET /health/ready` раз в секунду (`inter 1s`), исключает реплику после 2 неудач подряд (`fall 2`), возвращает после 2 успехов (`rise 2`), `timeout check 2s`. Максимальное время обнаружения отказа: 2 × 1 с + 2 с = 4 с. Адреса backend берутся из Docker DNS (`server-template api 1-8 api:8080`, обновление раз в секунду), поэтому новые и пересозданные реплики подхватываются без перезапуска LB. `option redispatch` и `retries 3` перенаправляют запрос на другую реплику, если соединение с выбранной не установилось. + +**Readiness, таймауты, SIGTERM.** Критичная зависимость readiness — PostgreSQL (Valkey не критичен). При SIGTERM реплика сразу отвечает 503 на `/health/ready` и ещё `DRAIN_SECONDS=5` обслуживает запросы, пока LB её исключает; затем uvicorn завершается штатно (`--timeout-graceful-shutdown 8`). `stop_grace_period: 20s` вмещает drain и graceful-завершение (5 + до 8 с). Таймауты: пул БД и команды 5 с, кэш 0,5 с, LB: connect 1 с, server 10 с. + +**Ресурсы и безопасность.** Лимиты (Compose `deploy.resources.limits`): API 0,5 CPU / 256 МБ на реплику (переменные `API_CPUS`, `API_MEM`), PostgreSQL 1,0 / 512 МБ, Valkey 0,25 / 128 МБ, LB 0,5 / 256 МБ, migrate 0,5 / 256 МБ. У API нет `container_name` и published-портов, число реплик задаёт `API_REPLICAS`. Пароль БД доступен только `postgres`, строка подключения только `migrate` и `api`. + +**Миграции и rolling update.** `scripts/rolling-update.sh` собирает новый образ, один раз запускает `migrate`, затем по очереди: `docker stop -t 20` (graceful) → удаление → создание реплики из нового образа → ожидание healthy → пауза 5 с. Миграция `002_idempotency.sql` только добавляет таблицу, поэтому совместима со старой версией (подход expand/contract). Несовместимые изменения схемы (удаление или переименование колонок) потребовали бы разделения на два релиза; здесь они не выполнялись. + +**Что перенесено из lab2.** Приложение, миграция `001`, Argon2id, сессии, Valkey-кэш, Dockerfile, собственные тесты. **Что изменено.** Миграции вынесены в отдельный сервис; добавлены `X-Instance-Id`, обязательный `Idempotency-Key`, drain и `X-App-Version`; compose с LB и репликами; `test_sigterm.py` приведён к новой семантике (readiness 503, затем закрытие порта). + +**Переменные `.env.example`.** `APP_ENV`, `SESSION_TTL_SECONDS`, `API_REPLICAS` передаются в compose. `DATABASE_URL_FILE` задан в compose как путь к secret, `CACHE_URL` в compose как `redis://valkey:6379/0`. Переменные `APP_HOST`, `APP_PORT` заданы в Dockerfile. `SESSION_STORE=postgres` соответствует реализации (источник истины PostgreSQL, Valkey лишь кэш). ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `make test` | PASS | `evidence/smoke.txt`, `evidence/smoke-level4.txt` | +| Compose запускает ≥2 API, один LB, общие БД и сессии | `docker compose up -d --build`, `ps` | 2 реплики api, lb, postgres, valkey, migrate Exited (0) | `evidence/ps-before.txt` | +| Разные реплики для одной сессии | `scripts/show-instances.sh` | 2 разных `X-Instance-Id`, по 5 запросов | `evidence/instances.txt` | +| Отказ одной реплики ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, для каждой реплики | 0 ошибок, восстановление 2,0 / 1,6 / 2,2 с | `evidence/failover-1.txt`, `failover-2.txt`, `failover-*-ps.txt`, `failover-level4.txt` | +| Идемпотентность, повтор и переключение реплики | smoke и `student_tests/test_idempotency_parallel.py` | 5 раундов × 30 параллельных, 1 сообщение, другое тело → 409 | `evidence/idempotency-parallel.txt` | +| Сохранность после рестарта | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | данные и токен сохранены | `evidence/persistence.txt`, `persistence-level4.txt`, `restart-before.txt`, `restart-after.txt` | +| Уровень 4: health/readiness, SIGTERM, период и время обнаружения | `student_tests/test_sigterm.py`, конфиг HAProxy | readiness 503 за 0,06 с; выход за 5,3 с (idle) и 13,4 с (незавершённый POST), код 0; обнаружение ≤4 с | `evidence/sigterm-level4.txt`, `infra/haproxy/haproxy.cfg` | +| Уровень 4: поочерёдная остановка с RPS, ошибками, восстановлением | `scripts/run-failover-series.sh` | см. таблицу ниже | `evidence/load-baseline.json`, `load-stop-0.json`, `load-stop-1.json` | +| Уровень 4: лимиты, нет published-портов и container_name | `docker stats`, `compose.yaml` | лимиты применены | `evidence/docker-stats.txt` | +| Уровень 4: rolling update под нагрузкой | `scripts/rolling-update.sh` под `loadgen.py` | 95 152 запроса, 0 ошибок | `evidence/load-rolling.json`, `rolling-ps.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Профиль нагрузки (`student_tests/loadgen.py`): closed-loop, 8 потоков, POST сообщения с уникальным `Idempotency-Key` в один чат через LB, без прозрачных retry, таймаут 3 с. Базовая нагрузка 30 с, серия отказов по 40 с (отказ на 10-й секунде), rolling update 120 с. Прогрева отдельно не было, повторов по одному на сценарий. Сырые сводки сохранены в `evidence/*.json`, токены в них не попадают. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Базовая нагрузка, 2 реплики | стабильный RPS | 856 RPS, 25 691 запрос, 0 ошибок, p50/p95/p99 = 6,4 / 28,7 / 61,7 мс | — | +| Остановка реплики 1 (`docker stop -t 2`) | запись продолжается на реплике 2 | 0 ошибок, 630 RPS за 40 с, p99 85,4 мс, макс. пауза 0,17 с | 0 с, данные не потеряны | +| Остановка реплики 2 | то же | 0 ошибок, 635 RPS за 40 с, p99 81,2 мс, макс. пауза 0,22 с | 0 с, данные не потеряны | +| `make test-failover` по каждой реплике | запись ≤10 с | 0 ошибок, восстановление 2,0 / 1,6 / 2,2 с | данные сохранены | +| Rolling update v1 → v2 под нагрузкой | без потерь | 95 152 запроса, 0 ошибок, 793 RPS, p99 72,1 мс, макс. пауза 0,18 с; v1 виден до 31,4 с, v2 с 21,2 с | потерь нет | +| 30 параллельных POST с одним ключом × 5 раундов | одно сообщение | все 201, один `id`, в истории 5 сообщений | дублей нет | + +После потери реплики пропускная способность падает примерно с 856 до 560–630 RPS (одна реплика с лимитом 0,5 CPU). Нулевые ошибки объясняются тем, что остановленный контейнер перестаёт принимать соединения сразу, а HAProxy делает redispatch; переходный период обнаружения в этих прогонах клиент не заметил, но по настройкам он может достигать 4 с. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host: потеря узла останавливает всё. PostgreSQL, Valkey и HAProxy в единственном экземпляре, это SPOF; выдерживается только отказ одной API-реплики и её плановая замена. Отказ БД, LB и узла не заявляется. +- Нагрузочный генератор closed-loop, на той же машине, что и стенд; возможна coordinated omission, один запуск на сценарий без повторов и доверительных интервалов, параметры хоста не зафиксированы. RPS нельзя переносить на другие машины. Измерялось только время ответа успешных запросов, ошибки считались отдельно. +- HAProxy повторяет только неудачное соединение. Прерванный POST клиент повторяет сам с тем же `Idempotency-Key`; повторы создания чата и других POST без ключа не безопасны и не выполняются. +- Просроченные записи `idempotency_keys` не удаляются (таблица растёт); для эксплуатации нужна фоновая очистка по `created_at`. +- Пароль к Valkey не задан (допустимо только во внутренней сети учебного стенда), секреты имеют права 644. +- Rolling update проверен для обратно совместимой миграции; несовместимые схемы не проверялись. +- Трек уровня 5 не выполнялся. \ No newline at end of file diff --git a/evidence/restart-after.txt b/evidence/restart-after.txt new file mode 100644 index 00000000..1d1af790 --- /dev/null +++ b/evidence/restart-after.txt @@ -0,0 +1,7 @@ +NAME STATUS +mtusi-lab3-api-3 Up 17 seconds (healthy) +mtusi-lab3-api-4 Up 17 seconds (healthy) +mtusi-lab3-lb-1 Up 23 minutes +mtusi-lab3-migrate-1 Exited (0) 17 seconds ago +mtusi-lab3-postgres-1 Up 23 seconds (healthy) +mtusi-lab3-valkey-1 Up 23 seconds (healthy) diff --git a/evidence/restart-before.txt b/evidence/restart-before.txt new file mode 100644 index 00000000..72576c3e --- /dev/null +++ b/evidence/restart-before.txt @@ -0,0 +1,7 @@ +NAME STATUS +mtusi-lab3-api-3 Up 4 minutes (healthy) +mtusi-lab3-api-4 Up 3 minutes (healthy) +mtusi-lab3-lb-1 Up 23 minutes +mtusi-lab3-migrate-1 Exited (0) 3 minutes ago +mtusi-lab3-postgres-1 Up 4 minutes (healthy) +mtusi-lab3-valkey-1 Up 4 minutes (healthy) -- 2.54.0