From 3b52b91fb76d8acdeb7cbec010fd5542fd8990f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Tue, 20 Oct 2026 15:13:41 +0300 Subject: [PATCH 1/5] add lab3 --- .env.example | 1 + Dockerfile | 23 ++ REPORT.md | 65 +++- compose.yaml | 114 ++++++ infra/haproxy.cfg | 35 ++ migrations/001_initial.sql | 37 ++ migrations/002_history_cache_version.sql | 1 + migrations/003_idempotency.sql | 12 + pyproject.toml | 12 + scripts/restart.sh | 7 + scripts/stop-one.sh | 18 + src/__init__.py | 1 + src/config.py | 117 +++++++ src/cursor.py | 81 +++++ src/errors.py | 12 + src/main.py | 161 +++++++++ src/migrate.py | 18 + src/models.py | 116 ++++++ src/routes.py | 181 ++++++++++ src/storage/__init__.py | 2 + src/storage/postgres.py | 288 +++++++++++++++ student-tests/idempotency_concurrency.py | 32 ++ student-tests/readiness_failover.py | 89 +++++ student-tests/readiness_unavailable.py | 16 + student-tests/review_regressions.py | 32 ++ student-tests/session_expiry.py | 20 ++ uv.lock | 426 +++++++++++++++++++++++ 27 files changed, 1916 insertions(+), 1 deletion(-) create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 infra/haproxy.cfg create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_history_cache_version.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 pyproject.toml create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 src/__init__.py create mode 100644 src/config.py create mode 100644 src/cursor.py create mode 100644 src/errors.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/models.py create mode 100644 src/routes.py create mode 100644 src/storage/__init__.py create mode 100644 src/storage/postgres.py create mode 100755 student-tests/idempotency_concurrency.py create mode 100644 student-tests/readiness_failover.py create mode 100644 student-tests/readiness_unavailable.py create mode 100644 student-tests/review_regressions.py create mode 100644 student-tests/session_expiry.py create mode 100644 uv.lock diff --git a/.env.example b/.env.example index d99d26a2..217c94be 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,7 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +CURSOR_SECRET_FILE=/run/secrets/cursor_secret S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ab272d34 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.12.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN pip install --no-cache-dir uv==0.11.28 + +COPY pyproject.toml uv.lock ./ +RUN uv sync --frozen --no-dev --no-install-project + +COPY src ./src +COPY migrations ./migrations + +USER 65534:65534 + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + +CMD ["/app/.venv/bin/python", "-m", "src.main"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..02c40569 100644 --- a/REPORT.md +++ b/REPORT.md @@ -7,10 +7,73 @@ ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +На текущем этапе перенесена база lab3; S3 и вложения ещё не реализованы. +Инструкция ниже запускает только эту базу. + +Нужны Docker Engine с Compose v2 и Python 3 для создания локальных секретов. +Версии: Python 3.12.12 и uv 0.11.28 в Dockerfile, PostgreSQL 17.6-alpine, +HAProxy 3.0.11-alpine в compose.yaml; зависимости Python закреплены в uv.lock. + +Из корня репозитория: + +```sh +python3 - <<'PY' +from pathlib import Path +import secrets + +directory = Path("secrets") +directory.mkdir(mode=0o700, exist_ok=True) +names = ("postgres_password", "database_url", "cursor_secret") +if any((directory / name).exists() for name in names): + raise SystemExit("Секреты уже существуют: сохраните их для существующей БД.") +password = secrets.token_hex(32) +values = ( + password, + f"postgresql://messenger:{password}@postgres:5432/messenger", + secrets.token_hex(32), +) +for name, value in zip(names, values): + path = directory / name + with path.open("x", encoding="utf-8") as stream: + path.chmod(0o600) + stream.write(value + "\n") +PY +docker compose --env-file .env.example config --quiet +docker compose --env-file .env.example up -d --build --wait +curl --fail http://localhost:8080/health/ready +make check +python3 student-tests/review_regressions.py +``` + +Сервис `migrate` применяет SQL-миграции до запуска API. +Вход через HAProxy: `http://localhost:8080`; порты API и PostgreSQL на host не опубликованы. +Порт 8080 должен быть свободен. Compose использует отдельный проект и volume lab4; +секреты и данные lab3 не переносились. + +Остановка с сохранением данных: `docker compose down`. +Намеренное удаление данных этого стенда: `docker compose down -v`. + +Полный `make test` пока не проходит: модель Message перенесена из lab3 без +`attachment_ids`, endpoint вложений отсутствуют. Публичные тесты и контракт lab4 +сохранены. Проверки `test-persistence`, `test-failover` и собственная проверка +`idempotency_concurrency.py` используют контракт lab4 и требуют дальнейшей +реализации. Скрипты restart/stop-one перенесены для текущей топологии; +при добавлении S3 restart необходимо дополнить перезапуском этого хранилища. ## Архитектура и решения +Перенесено из локального проекта `mtusi2026lab3`: `src/`, миграции +`001`–`003`, Dockerfile, зависимости и lockfile, основной Compose, конфигурация +HAProxy, restart/failover hooks и собственные проверки сессий, идемпотентности, +readiness и ошибок API. Исследовательский benchmark lab3 и старые результаты +экспериментов не входят в этот перенос. + +Текущая схема: клиент → HAProxy → api-a / api-b → PostgreSQL. +PostgreSQL хранит пользователей, сессии, чаты, сообщения и записи идемпотентности. +Общий `CURSOR_SECRET_FILE` задаёт ключ подписи курсоров для обеих реплик. +Исходники API пока сохраняют поведение и версию 1.3.0 из lab3. +Параметры S3 из `.env.example` будут подключены на следующем этапе. + TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. ## Доказательства критериев diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f447ec0e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,114 @@ +x-api: &api + build: + context: . + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + APP_ENV: lab4 + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + secrets: + - database_url + - cursor_secret + depends_on: + migrate: + condition: service_completed_successfully + expose: + - "8080" + stop_grace_period: 10s + healthcheck: + test: ["CMD", "/app/.venv/bin/python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + +services: + postgres: + image: postgres:17.6-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.75" + memory: 512M + + migrate: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + secrets: + - database_url + - cursor_secret + depends_on: + postgres: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + api-a: + <<: *api + + api-b: + <<: *api + + load_balancer: + image: haproxy:3.0.11-alpine + ports: + - "8080:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api-a: + condition: service_healthy + api-b: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cursor_secret: + file: ./secrets/cursor_secret + +volumes: + postgres_data: diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..00f0b6c7 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,35 @@ +global + log stdout format raw local0 + +defaults + mode http + log global + option httplog + option forwardfor + timeout connect 1s + timeout client 30s + timeout server 10s + timeout check 2s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + +frontend http + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server check inter 2s fall 1 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none + server api-a api-a:8080 + server api-b api-b:8080 + diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..c03d5fd9 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,37 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (char_length(btrim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (char_length(btrim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL CHECK (char_length(btrim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/002_history_cache_version.sql b/migrations/002_history_cache_version.sql new file mode 100644 index 00000000..7b7972c1 --- /dev/null +++ b/migrations/002_history_cache_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0; diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 00000000..7fcf6650 --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL, + path text NOT NULL, + idempotency_key varchar(128) NOT NULL, + request_hash char(64) NOT NULL, + message_id uuid REFERENCES messages(id) ON DELETE RESTRICT, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, method, path, idempotency_key) +); + +CREATE INDEX idempotency_records_created_at_idx ON idempotency_records (created_at); diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..7de48cee --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,12 @@ +[project] +name = "mtusi2026lab4" +version = "0.1.0" +description = "MTUSI high-load messenger, laboratory 4" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "argon2-cffi==25.1.0", + "asyncpg==0.31.0", + "fastapi==0.141.1", + "uvicorn==0.40.0", +] diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..e0d9abaa --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres +docker compose up -d --wait postgres +docker compose restart api-a api-b load_balancer +docker compose up -d --wait api-a api-b load_balancer diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..befd6cfb --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." + +for service in api-a api-b; do + container="$(docker compose ps -q "$service")" + [ -n "$container" ] || continue + hostname="$(docker inspect --format '{{.Config.Hostname}}' "$container")" + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker compose stop "$service" + exit 0 + fi +done + +printf '%s\n' 'TARGET_INSTANCE_ID does not match a running API replica' >&2 +exit 2 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ + diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..f68a8b0c --- /dev/null +++ b/src/config.py @@ -0,0 +1,117 @@ +from dataclasses import dataclass +import os +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +_CONFIGURATION_KEYS = { + "APP_HOST", + "APP_PORT", + "APP_ENV", + "DATABASE_URL", + "DATABASE_URL_FILE", + "SESSION_TTL_SECONDS", + "IDEMPOTENCY_TTL_SECONDS", + "INSTANCE_ID", + "CURSOR_SECRET", + "CURSOR_SECRET_FILE", +} + + +@dataclass(frozen=True, slots=True) +class Settings: + """Проверенная конфигурация запуска HTTP-сервера.""" + + host: str + port: int + environment: str + database_url: str + session_ttl_seconds: int + idempotency_ttl_seconds: int + instance_id: str + cursor_secret: bytes + + +def load_settings(env_file: Path = ROOT / ".env") -> Settings: + """Читает разрешённые ключи .env, не заменяя явно переданное окружение.""" + + if env_file.is_file(): + for raw_line in env_file.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + key, separator, value = line.partition("=") + if not separator or key not in _CONFIGURATION_KEYS: + continue + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + os.environ.setdefault(key, value) + + def value(name: str, default: str = "") -> str: + return os.environ.get(name, default).strip() + + def number(name: str, default: int) -> int: + try: + result = int(value(name, str(default))) + except ValueError as error: + raise ValueError(f"{name} должен быть целым числом") from error + if result < 1: + raise ValueError(f"{name} должен быть положительным") + return result + + def url(name: str, file_name: str, prefixes: tuple[str, ...]) -> str: + result = value(name) + path = value(file_name) + if path: + try: + result = Path(path).read_text(encoding="utf-8").strip() + except OSError as error: + raise ValueError(f"{file_name} недоступен") from error + if result and not result.startswith(prefixes): + raise ValueError(f"{name} содержит неверный URL") + return result + + def secret(name: str, file_name: str) -> bytes: + result = value(name) + path = value(file_name) + if path: + try: + result = Path(path).read_text(encoding="utf-8").strip() + except OSError as error: + raise ValueError(f"{file_name} недоступен") from error + encoded = result.encode("utf-8") + if len(encoded) < 32: + raise ValueError(f"{name} должен содержать не менее 32 байт") + return encoded + + host = value("APP_HOST", "0.0.0.0") + environment = value("APP_ENV", "lab") + if not host: + raise ValueError("APP_HOST не может быть пустым") + if not environment: + raise ValueError("APP_ENV не может быть пустым") + port = number("APP_PORT", 8080) + if port > 65535: + raise ValueError("APP_PORT должен находиться в диапазоне 1..65535") + database_url = url( + "DATABASE_URL", "DATABASE_URL_FILE", ("postgresql://", "postgres://") + ) + if not database_url: + raise ValueError("Нужен PostgreSQL DATABASE_URL или DATABASE_URL_FILE") + instance_id = value("INSTANCE_ID") or os.uname().nodename + if not instance_id: + raise ValueError("INSTANCE_ID не может быть пустым") + idempotency_ttl_seconds = number("IDEMPOTENCY_TTL_SECONDS", 86400) + if idempotency_ttl_seconds < 86400: + raise ValueError("IDEMPOTENCY_TTL_SECONDS должен быть не менее 86400") + return Settings( + host=host, + port=port, + environment=environment, + database_url=database_url, + session_ttl_seconds=number("SESSION_TTL_SECONDS", 3600), + idempotency_ttl_seconds=idempotency_ttl_seconds, + instance_id=instance_id, + cursor_secret=secret("CURSOR_SECRET", "CURSOR_SECRET_FILE"), + ) diff --git a/src/cursor.py b/src/cursor.py new file mode 100644 index 00000000..5dd8eab3 --- /dev/null +++ b/src/cursor.py @@ -0,0 +1,81 @@ +import base64 +from datetime import UTC, datetime +import hashlib +import hmac +import json +from uuid import UUID + + +_secret: bytes | None = None + + +class CursorError(ValueError): + """Недопустимый, повреждённый или применённый не к тому ресурсу курсор.""" + + +def configure_secret(secret: bytes) -> None: + """Устанавливает общий для всех реплик ключ подписи курсоров.""" + global _secret + if len(secret) < 32: + raise ValueError("cursor secret must contain at least 32 bytes") + _secret = secret + + +def signing_secret() -> bytes: + if _secret is None: + raise RuntimeError("cursor secret has not been configured") + return _secret + + +def _encode_bytes(value: bytes) -> str: + return base64.urlsafe_b64encode(value).decode().rstrip("=") + + +def _decode_bytes(value: str) -> bytes: + try: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + except (ValueError, UnicodeEncodeError) as error: + raise CursorError from error + + +def encode_cursor(scope: str, created_at: datetime, item_id: str) -> str: + payload = json.dumps( + { + "scope": scope, + "created_at": created_at.astimezone(UTC) + .isoformat() + .replace("+00:00", "Z"), + "id": item_id, + }, + separators=(",", ":"), + sort_keys=True, + ).encode() + signature = hmac.new(signing_secret(), payload, hashlib.sha256).digest() + return _encode_bytes(payload) + "." + _encode_bytes(signature) + + +def decode_cursor(cursor: str, expected_scope: str) -> tuple[datetime, str]: + try: + encoded_payload, encoded_signature = cursor.split(".") + payload = _decode_bytes(encoded_payload) + signature = _decode_bytes(encoded_signature) + except ValueError as error: + raise CursorError from error + + expected_signature = hmac.new(signing_secret(), payload, hashlib.sha256).digest() + if not hmac.compare_digest(signature, expected_signature): + raise CursorError + + try: + data = json.loads(payload) + if set(data) != {"scope", "created_at", "id"}: + raise CursorError + if data["scope"] != expected_scope: + raise CursorError + item_id = str(UUID(data["id"])) + created_at = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + if created_at.tzinfo is None: + raise CursorError + return created_at.astimezone(UTC), item_id + except (TypeError, ValueError, KeyError) as error: + raise CursorError from error diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..258d6996 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,12 @@ +"""Ошибки, которые преобразуются в контрактные HTTP-ответы.""" + + +class ApiError(Exception): + + def __init__( + self, status_code: int, code: str, message: str, headers: dict[str, str] | None = None + ) -> None: + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..cd876651 --- /dev/null +++ b/src/main.py @@ -0,0 +1,161 @@ +import logging +import sys +import time +from contextlib import asynccontextmanager +from uuid import uuid4 + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +import uvicorn +from starlette.responses import JSONResponse, Response +from starlette.exceptions import HTTPException + +from .errors import ApiError +from .cursor import configure_secret +from .models import ErrorDetails, ErrorResponse +from .config import load_settings +from .routes import router +from .storage.postgres import PostgresStore + + +def logger(name: str) -> logging.Logger: + result = logging.getLogger(name) + if result.handlers: + return result + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s")) + result.addHandler(handler) + result.setLevel(logging.INFO) + result.propagate = False + return result + + +access_logger = logger("messenger.access") + + +@asynccontextmanager +async def lifespan(application: FastAPI): + settings = load_settings() + configure_secret(settings.cursor_secret) + store = PostgresStore(settings.database_url) + await store.connect() + application.state.store = store + application.state.settings = settings + application.state.accepting_requests = True + try: + yield + finally: + application.state.accepting_requests = False + await store.close() + + +app = FastAPI( + title="MTUSI Messenger", + version="1.3.0", + redoc_url=None, + lifespan=lifespan, +) + + +@app.middleware("http") +async def add_request_id(request: Request, call_next) -> Response: + """Добавляет ID запроса и пишет безопасную строку access-лога.""" + + request.state.request_id = str(uuid4()) + started_at = time.monotonic() + status_code = 500 + try: + response = await call_next(request) + status_code = response.status_code + response.headers["X-Request-Id"] = request.state.request_id + response.headers["X-Instance-Id"] = request.app.state.settings.instance_id + return response + finally: + access_logger.info( + "request_id=%s method=%s path=%s status=%s elapsed_ms=%.3f", + request.state.request_id, + request.method, + request.url.path, + status_code, + (time.monotonic() - started_at) * 1000, + ) + + +def error_response( + request: Request, status_code: int, code: str, message: str, headers: dict[str, str] | None = None +) -> JSONResponse: + request_id = request.state.request_id + body = ErrorResponse( + error=ErrorDetails(code=code, message=message, request_id=request_id) + ) + # Неожиданная ошибка обрабатывается за пределами HTTP-middleware. + # Поэтому её ответ тоже должен сам получить заголовок из контракта. + return JSONResponse( + status_code=status_code, + content=body.model_dump(), + headers={ + "X-Request-Id": request_id, + "X-Instance-Id": request.app.state.settings.instance_id, + **(headers or {}), + }, + ) + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, error: ApiError) -> JSONResponse: + """Преобразует контролируемую ошибку API в стандартный JSON-ответ.""" + + return error_response(request, error.status_code, error.code, error.message, error.headers) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error( + request: Request, error: RequestValidationError +) -> JSONResponse: + """Преобразует ошибки JSON и Pydantic в обязательный ответ 400.""" + + return error_response(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(HTTPException) +async def handle_http_error(request: Request, error: HTTPException) -> JSONResponse: + """Включает ошибки декодирования тела, возникающие до Pydantic.""" + codes = { + 400: "invalid_request", + 401: "unauthorized", + 403: "forbidden", + 404: "not_found", + 405: "method_not_allowed", + 503: "unavailable", + } + code = codes.get(error.status_code, "internal_error") + return error_response( + request, error.status_code, code, + "Некорректный запрос" if error.status_code < 500 else "Ошибка сервера", + error.headers, + ) + + +@app.exception_handler(Exception) +async def handle_unexpected_error(request: Request, error: Exception) -> JSONResponse: + """Не раскрывает внутренние детали при непредвиденной ошибке приложения.""" + + return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") + + +app.include_router(router) + + +def main() -> None: + settings = load_settings() + uvicorn.run( + app, + host=settings.host, + port=settings.port, + access_log=False, + timeout_graceful_shutdown=8, + ) + + +if __name__ == "__main__": + main() diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..4376e8d6 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,18 @@ +import asyncio + +from .config import load_settings +from .storage.postgres import PostgresStore + + +async def run() -> None: + settings = load_settings() + store = PostgresStore(settings.database_url) + await store.connect() + try: + await store.migrate() + finally: + await store.close() + + +if __name__ == "__main__": + asyncio.run(run()) diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..095ec1a2 --- /dev/null +++ b/src/models.py @@ -0,0 +1,116 @@ +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +class CreateUserRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + username: str = Field(min_length=3, max_length=32, pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def display_name_must_not_be_whitespace(cls, value: str) -> str: + """Не допускает обязательное имя, состоящее только из пробелов.""" + + if not value.strip(): + raise ValueError("display_name не может состоять только из пробелов") + return value + + +class CreateChatRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_must_not_be_whitespace(cls, value: str) -> str: + """Не допускает обязательное название, состоящее только из пробелов.""" + + if not value.strip(): + raise ValueError("title не может состоять только из пробелов") + return value + + +class ChatPageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list["Chat"] + next_cursor: str | None + + +class CreateMessageRequest(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + text: str = Field(min_length=1, max_length=4000) + + @field_validator("text") + @classmethod + def text_must_not_be_whitespace(cls, value: str) -> str: + """Не допускает сообщение, состоящее только из пробельных символов.""" + + if not value.strip(): + raise ValueError("text не может состоять только из пробелов") + return value + + +class MessagePageResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list["Message"] + next_cursor: str | None + + +class ErrorDetails(BaseModel): + model_config = ConfigDict(extra="forbid") + + code: str + message: str + request_id: str + + +class ErrorResponse(BaseModel): + model_config = ConfigDict(extra="forbid") + + error: ErrorDetails + + +class User(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + username: str + display_name: str + created_at: datetime + + +class Chat(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class Message(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + + +class Session(BaseModel): + model_config = ConfigDict(extra="forbid") + + token: str = Field(min_length=32) + token_type: str = "Bearer" + expires_at: datetime + user: User diff --git a/src/routes.py b/src/routes.py new file mode 100644 index 00000000..c07e4056 --- /dev/null +++ b/src/routes.py @@ -0,0 +1,181 @@ +import base64 +import re +from datetime import datetime +from uuid import UUID + +from fastapi import APIRouter, Depends, Header, Query, Request, Response + +from .cursor import CursorError, decode_cursor, encode_cursor +from .errors import ApiError +from .storage.postgres import PostgresStore +from .models import ( + Chat, + ChatPageResponse, + CreateChatRequest, + CreateMessageRequest, + CreateUserRequest, + Message, + MessagePageResponse, + Session, + User, +) + +router = APIRouter() +BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'} +BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} +IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +def get_store(request: Request) -> PostgresStore: + return request.app.state.store + + +async def get_current_user( + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), +) -> dict: + if authorization is None or not authorization.startswith("Bearer "): + raise ApiError(401, "unauthorized", "Требуется Bearer-сессия", BEARER_CHALLENGE) + token = authorization[7:] + if not token or not token.isascii() or token.strip() != token: + raise ApiError(401, "unauthorized", "Требуется Bearer-сессия", BEARER_CHALLENGE) + user = await store.user_for_token(token) + if user is None: + raise ApiError(401, "unauthorized", "Недействительная сессия", BEARER_CHALLENGE) + return user + +def parse_basic(authorization: str | None) -> tuple[str, str]: + if authorization is None or not authorization.startswith("Basic "): + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + try: + decoded = base64.b64decode(authorization[6:], validate=True) + username_bytes, password_bytes = decoded.split(b":", 1) + username = username_bytes.decode("ascii") + password = password_bytes.decode("utf-8") + except (UnicodeDecodeError, ValueError) as error: + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) from error + if not username or not (12 <= len(password) <= 128): + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + return username, password + + +def cursor_position(cursor: str | None, scope: str) -> tuple[datetime, UUID] | None: + if cursor is None: + return None + try: + created_at, item_id = decode_cursor(cursor, scope) + return created_at, UUID(item_id) + except (CursorError, ValueError) as error: + raise ApiError(400, "invalid_request", "Некорректный cursor") from error + + +def next_page_cursor(rows: list[dict], limit: int, scope: str) -> str | None: + if len(rows) <= limit: + return None + last = rows[limit - 1] + return encode_cursor(scope, last["created_at"], str(last["id"])) + +@router.get("/health/live") +async def liveness() -> dict[str, str]: + return {"status": "ok"} + +@router.get("/health/ready") +async def readiness(request: Request, store: PostgresStore = Depends(get_store)) -> dict[str, str]: + if not request.app.state.accepting_requests: + raise ApiError(503, "unavailable", "API завершает работу") + await store.ready() + return {"status": "ok"} + +@router.post("/api/v1/users", response_model=User, status_code=201) +async def register_user( + payload: CreateUserRequest, store: PostgresStore = Depends(get_store) +) -> dict: + return await store.create_user(payload.username, payload.display_name, payload.password) + +@router.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +async def create_session( + request: Request, + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), +) -> dict: + if await request.body(): + raise ApiError(400, "invalid_request", "Тело запроса не требуется") + username, password = parse_basic(authorization) + user = await store.authenticate(username, password) + if user is None: + raise ApiError(401, "unauthorized", "Неверные учётные данные", BASIC_CHALLENGE) + return await store.create_session(user, request.app.state.settings.session_ttl_seconds) + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session( + authorization: str | None = Header(default=None), + store: PostgresStore = Depends(get_store), + user: dict = Depends(get_current_user), +) -> Response: + del user + await store.revoke_session(authorization[7:]) + return Response(status_code=204) + +@router.get("/api/v1/users/me", response_model=User) +async def current_user(user: dict = Depends(get_current_user)) -> dict: + return user + +@router.post("/api/v1/chats", response_model=Chat, status_code=201) +async def create_chat( + payload: CreateChatRequest, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + return await store.create_chat(user["id"], payload.title, payload.member_ids) + +@router.get("/api/v1/chats", response_model=ChatPageResponse) +async def list_chats( + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> ChatPageResponse: + scope = "chats:" + str(user["id"]) + rows = await store.list_chats_for_member(user["id"], cursor_position(cursor, scope), limit) + page = rows[:limit] + return ChatPageResponse(items=page, next_cursor=next_page_cursor(rows, limit, scope)) + +@router.get("/api/v1/chats/{chat_id}", response_model=Chat) +async def get_chat( + chat_id: UUID, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + return await store.get_chat_for_member(chat_id, user["id"]) + +@router.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: UUID, + payload: CreateMessageRequest, + request: Request, + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), +) -> dict: + if idempotency_key is None or IDEMPOTENCY_KEY.fullmatch(idempotency_key) is None: + raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key") + return await store.create_message_idempotent( + chat_id, + user["id"], + payload.text, + idempotency_key, + request.app.state.settings.idempotency_ttl_seconds, + ) + +@router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePageResponse) +async def list_messages( + chat_id: UUID, + user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> MessagePageResponse: + scope = "messages:" + str(chat_id) + after = cursor_position(cursor, scope) + rows = await store.list_messages_for_member(chat_id, user["id"], after, limit) + page = rows[:limit] + return MessagePageResponse(items=page, next_cursor=next_page_cursor(rows, limit, scope)) diff --git a/src/storage/__init__.py b/src/storage/__init__.py new file mode 100644 index 00000000..e2e04193 --- /dev/null +++ b/src/storage/__init__.py @@ -0,0 +1,2 @@ +"""Слой in-memory хранения для приложения первой лабораторной работы.""" +"""Persistent storage adapters.""" diff --git a/src/storage/postgres.py b/src/storage/postgres.py new file mode 100644 index 00000000..67b8f815 --- /dev/null +++ b/src/storage/postgres.py @@ -0,0 +1,288 @@ +from datetime import datetime +from hashlib import sha256 +import json +from pathlib import Path +from secrets import token_urlsafe +from uuid import UUID, uuid4 + +import asyncpg +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + +from ..errors import ApiError + + +ROOT = Path(__file__).resolve().parents[2] +MIGRATIONS = ROOT / "migrations" +PASSWORD_HASHER = PasswordHasher() +DB_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError) + + +class PostgresStore: + def __init__(self, database_url: str) -> None: + self._database_url = database_url + self._pool: asyncpg.Pool | None = None + + async def connect(self) -> None: + try: + self._pool = await asyncpg.create_pool( + self._database_url, min_size=1, max_size=10, command_timeout=5 + ) + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + + async def close(self) -> None: + if self._pool is not None: + await self._pool.close() + + @property + def pool(self) -> asyncpg.Pool: + if self._pool is None: + raise RuntimeError("PostgreSQL pool has not been initialized") + return self._pool + + async def migrate(self) -> None: + async with self.pool.acquire() as connection: + await connection.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + for migration in sorted(MIGRATIONS.glob("*.sql")): + applied = await connection.fetchval( + "SELECT 1 FROM schema_migrations WHERE version = $1", migration.name + ) + if applied: + continue + async with connection.transaction(): + await connection.execute(migration.read_text(encoding="utf-8")) + await connection.execute( + "INSERT INTO schema_migrations (version) VALUES ($1)", migration.name + ) + + @staticmethod + def _user(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "username": row["username"], + "display_name": row["display_name"], + "created_at": row["created_at"], + } + + @staticmethod + def _chat(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "title": row["title"], + "member_ids": row["member_ids"], + "created_at": row["created_at"], + } + + @staticmethod + def _message(row: asyncpg.Record) -> dict: + return { + "id": row["id"], + "chat_id": row["chat_id"], + "sender_id": row["sender_id"], + "text": row["text"], + "created_at": row["created_at"], + } + + async def ready(self) -> None: + try: + await self.pool.fetchval("SELECT 1") + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_user(self, username: str, display_name: str, password: str) -> dict: + try: + row = await self.pool.fetchrow( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) RETURNING id, username, display_name, created_at", + uuid4(), + username, + display_name, + PASSWORD_HASHER.hash(password), + ) + except asyncpg.UniqueViolationError as error: + raise ApiError(409, "conflict", "Username уже занят") from error + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._user(row) + + async def authenticate(self, username: str, password: str) -> dict | None: + try: + row = await self.pool.fetchrow( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = $1", + username, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + return None + try: + valid = PASSWORD_HASHER.verify(row["password_hash"], password) + except (InvalidHashError, VerificationError): + valid = False + return self._user(row) if valid else None + + async def create_session(self, user: dict, ttl_seconds: int) -> dict: + token = token_urlsafe(32) + token_hash = sha256(token.encode("ascii")).hexdigest() + try: + row = await self.pool.fetchrow( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, now() + $3 * interval '1 second') RETURNING expires_at", + token_hash, + user["id"], + ttl_seconds, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return {"token": token, "token_type": "Bearer", "expires_at": row["expires_at"], "user": user} + + async def user_for_token(self, token: str) -> dict | None: + if not token.isascii(): + return None + token_hash = sha256(token.encode("ascii")).hexdigest() + try: + row = await self.pool.fetchrow( + "SELECT u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token_hash = $1 AND s.expires_at > now()", + token_hash, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._user(row) if row else None + + async def revoke_session(self, token: str) -> None: + try: + await self.pool.execute( + "DELETE FROM sessions WHERE token_hash = $1", sha256(token.encode("ascii")).hexdigest() + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_chat(self, creator_id: UUID, title: str, member_ids: list[UUID]) -> dict: + if len(member_ids) != len(set(member_ids)) or creator_id not in member_ids: + raise ApiError(400, "invalid_request", "Некорректный список участников") + try: + async with self.pool.acquire() as connection, connection.transaction(): + existing = await connection.fetchval( + "SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", member_ids + ) + if existing != len(member_ids): + raise ApiError(400, "invalid_request", "Один или несколько участников не существуют") + row = await connection.fetchrow( + "INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING id, title, created_at", + uuid4(), + title, + ) + await connection.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(row["id"], member_id) for member_id in member_ids], + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return { + "id": row["id"], + "title": row["title"], + "member_ids": member_ids, + "created_at": row["created_at"], + } + + async def get_chat_for_member(self, chat_id: UUID, user_id: UUID) -> dict: + try: + row = await self.pool.fetchrow( + "SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2 JOIN chat_members cm_all ON cm_all.chat_id = c.id WHERE c.id = $1 GROUP BY c.id", + chat_id, + user_id, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + return self._chat(row) + + async def list_chats_for_member( + self, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int + ) -> list[dict]: + query = "SELECT c.id, c.title, c.created_at, array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids FROM chats c JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 JOIN chat_members cm_all ON cm_all.chat_id = c.id" + args: list = [user_id] + if after is not None: + query += " WHERE (c.created_at, c.id) > ($2, $3)" + args.extend(after) + query += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT $" + str(len(args) + 1) + args.append(limit + 1) + try: + rows = await self.pool.fetch(query, *args) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return [self._chat(row) for row in rows] + + async def create_message_idempotent( + self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int + ) -> dict: + """Создаёт сообщение и фиксирует результат retry в одной транзакции.""" + path = f"/api/v1/chats/{chat_id}/messages" + request_hash = sha256( + json.dumps({"text": text}, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + ).hexdigest() + try: + async with self.pool.acquire() as connection, connection.transaction(): + member = await connection.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, sender_id + ) + if member is None: + raise ApiError(404, "not_found", "Чат не найден") + await connection.execute( + "DELETE FROM idempotency_records WHERE created_at <= now() - $1 * interval '1 second'", ttl_seconds + ) + claimed = await connection.fetchrow( + "INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) " + "VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id", + sender_id, path, idempotency_key, request_hash, + ) + if claimed is None: + recorded = await connection.fetchrow( + "SELECT request_hash, message_id FROM idempotency_records " + "WHERE user_id = $1 AND method = 'POST' AND path = $2 AND idempotency_key = $3 FOR UPDATE", + sender_id, path, idempotency_key, + ) + if recorded is None: + raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса") + if recorded["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом") + if recorded["message_id"] is None: + raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса") + row = await connection.fetchrow( + "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = $1", recorded["message_id"] + ) + if row is None: + raise ApiError(503, "unavailable", "Не удалось получить сохранённое сообщение") + return self._message(row) + row = await connection.fetchrow( + "INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4) " + "RETURNING id, chat_id, sender_id, text, created_at", + uuid4(), chat_id, sender_id, text, + ) + await connection.execute( + "UPDATE idempotency_records SET message_id = $1 WHERE user_id = $2 AND method = 'POST' " + "AND path = $3 AND idempotency_key = $4", + row["id"], sender_id, path, idempotency_key, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return self._message(row) + + async def list_messages_for_member( + self, chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int + ) -> list[dict]: + await self.get_chat_for_member(chat_id, user_id) + query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1" + args: list = [chat_id] + if after is not None: + query += " AND (created_at, id) > ($2, $3)" + args.extend(after) + query += " ORDER BY created_at, id LIMIT $" + str(len(args) + 1) + args.append(limit + 1) + try: + rows = await self.pool.fetch(query, *args) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return [self._message(row) for row in rows] diff --git a/student-tests/idempotency_concurrency.py b/student-tests/idempotency_concurrency.py new file mode 100755 index 00000000..fcf958f4 --- /dev/null +++ b/student-tests/idempotency_concurrency.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Один ключ одновременно через LB создаёт ровно одно сообщение.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import sys +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, run # noqa: E402 + + +def main() -> None: + client = Client() + alice, _bob, _eve, chat = client.bootstrap() + key = uuid.uuid4().hex + text = "concurrent idempotency " + key + + def send(_number: int): + return Client().message(alice, chat, text, key=key)["json"] + + with ThreadPoolExecutor(max_workers=16) as pool: + replies = list(pool.map(send, range(16))) + check(all(reply == replies[0] for reply in replies), "replays returned different messages") + + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + history = client.call("GET", path, actor=alice, schema="MessagePage")["json"]["items"] + check(sum(message["id"] == replies[0]["id"] for message in history) == 1, "concurrent retries created duplicates") + print("PASS: concurrent idempotency through load balancer") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/readiness_failover.py b/student-tests/readiness_failover.py new file mode 100644 index 00000000..cdb6c0a6 --- /dev/null +++ b/student-tests/readiness_failover.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Запрет БД только для api-a через pg_hba; восстановление в finally.""" +import ipaddress +import json +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, run + + +def compose(*args): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True).strip() + + +def sql(statement): + return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", + "messenger", "-v", "ON_ERROR_STOP=1", "-Atc", statement) + + +def ready(service): + code = """import urllib.request, urllib.error +try: + print(urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=7).status) +except urllib.error.HTTPError as error: + print(error.code) +""" + return compose("exec", "-T", service, "/app/.venv/bin/python", "-c", code) + + +def main(): + client = Client() + container = json.loads(subprocess.check_output( + ["docker", "inspect", compose("ps", "-q", "api-a")], text=True))[0] + addresses = list(container["NetworkSettings"]["Networks"].values()) + check(len(addresses) == 1, "test expects one API network") + address = str(ipaddress.IPv4Address(addresses[0]["IPAddress"])) + bad_id = container["Config"]["Hostname"] + before = {client.call("GET", "/health/live")["headers"]["X-Instance-Id"] for _ in range(40)} + check(len(before) == 2 and bad_id in before, "both API replicas must be reachable") + check(ready("api-a") == ready("api-b") == "200", "both APIs must be ready") + # Сохраняем исходный pg_hba; чужую оставшуюся резервную копию не перезаписываем. + compose("exec", "-T", "postgres", "sh", "-ec", + 'test ! -e "$PGDATA/pg_hba.conf.readiness-backup"; ' + 'cp -p "$PGDATA/pg_hba.conf" "$PGDATA/pg_hba.conf.readiness-backup"') + try: + compose("exec", "-T", "postgres", "sh", "-ec", + '{ printf "host all all %s/32 reject\\n" "$1"; ' + 'cat "$PGDATA/pg_hba.conf.readiness-backup"; } > "$PGDATA/pg_hba.conf"', + "sh", address) + sql("SELECT pg_reload_conf()") + time.sleep(0.5) + sql(f"SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE client_addr = '{address}'::inet") + check(ready("api-a") == "503", "api-a must be unready") + check(ready("api-b") == "200", "api-b must remain ready") + # 2s interval + 2s check timeout, с запасом на планировщик Docker Desktop. + time.sleep(6) + for _ in range(40): + result = client.call("GET", "/health/live", 200) + check(result["headers"]["X-Instance-Id"] != bad_id, "LB selected unready api-a") + for _ in range(20): + result = client.call("POST", "/api/v1/users", 201, body={ + "username": "r_" + uuid.uuid4().hex[:16], + "display_name": "Readiness regression", + "password": "Test-password-" + uuid.uuid4().hex, + }, schema="User") + check(result["headers"]["X-Instance-Id"] != bad_id, "registration reached api-a") + print("PASS: api-a ready=503, api-b ready=200; 40/40 live and 20/20 registrations use api-b") + finally: + compose("exec", "-T", "postgres", "sh", "-ec", + 'mv "$PGDATA/pg_hba.conf.readiness-backup" "$PGDATA/pg_hba.conf"') + sql("SELECT pg_reload_conf()") + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + observed = {client.call("GET", "/health/live")["headers"]["X-Instance-Id"] for _ in range(10)} + if observed == before: + check(ready("api-a") == "200", "restored api-a must be ready") + print("PASS: database access restored; both API replicas returned to LB") + return + time.sleep(0.5) + raise AssertionError("api-a did not return to LB") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/readiness_unavailable.py b/student-tests/readiness_unavailable.py new file mode 100644 index 00000000..cf4bffa1 --- /dev/null +++ b/student-tests/readiness_unavailable.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 +"""Проверка 503 readiness при остановленном PostgreSQL.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, run # noqa: E402 + + +def main() -> None: + Client().call("GET", "/health/ready", 503) + print("PASS: readiness rejects unavailable PostgreSQL") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/review_regressions.py b/student-tests/review_regressions.py new file mode 100644 index 00000000..7e2bba78 --- /dev/null +++ b/student-tests/review_regressions.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Регрессии ревью: не-ASCII Bearer и невалидная кодировка JSON через LB.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + for path in ("/api/v1/users/me", "/api/v1/auth/sessions/current"): + method = "DELETE" if path.endswith("/current") else "GET" + result = client.call(method, path, 401, + headers={"Authorization": "Bearer é"}, schema="Error") + check(result["headers"].get("WWW-Authenticate", "").startswith("Bearer"), + "missing Bearer challenge") + check(result["json"]["error"]["code"] == "unauthorized", "wrong error code") + check(result["json"]["error"]["request_id"] == result["headers"]["X-Request-Id"], + "request IDs differ") + for raw in (b"\xff", b"{", b'{"username":'): + result = client.call("POST", "/api/v1/users", 400, raw=raw, + headers={"Content-Type": "application/json"}, schema="Error") + check(result["json"]["error"]["code"] == "invalid_request", "wrong JSON error code") + check(result["json"]["error"]["request_id"] == result["headers"]["X-Request-Id"], + "request IDs differ") + print("PASS: non-ASCII Bearer returns 401/challenge; malformed JSON returns contract Error/400") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/session_expiry.py b/student-tests/session_expiry.py new file mode 100644 index 00000000..37e29fb8 --- /dev/null +++ b/student-tests/session_expiry.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Проверка expiry; запускается при SESSION_TTL_SECONDS=2.""" +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, check, run # noqa: E402 + + +def main() -> None: + client = Client() + actor = client.user("expiry") + time.sleep(3) + client.call("GET", "/api/v1/users/me", 401, actor=actor) + print("PASS: expired Bearer session was rejected") + + +if __name__ == "__main__": + run(main) diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..3ee491eb --- /dev/null +++ b/uv.lock @@ -0,0 +1,426 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version < '3.14'", +] + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions", marker = "python_full_version < '3.15'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0b/43/bb8b6e8708d49a5ab36781333af092d9f483b198a2710d01281204640055/argon2_cffi_bindings-26.1.0.tar.gz", hash = "sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d", size = 1790807, upload-time = "2026-08-20T07:44:22.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/d2/0ae991f1b2181e5be49007c574710a800ad36c2978683addb3e67c474e55/argon2_cffi_bindings-26.1.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2", size = 25521, upload-time = "2026-08-20T07:32:43.019Z" }, + { url = "https://files.pythonhosted.org/packages/7e/e4/ad91d8297638aa2258aad4501c306aca99480dfe76ccd638173fa3702db9/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69", size = 27177, upload-time = "2026-08-20T07:32:44.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/86/5363df11b86d02cf3662208e7406496327649cc90eb365bf6f4e8a54a41f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29", size = 26597, upload-time = "2026-08-20T07:32:45.172Z" }, + { url = "https://files.pythonhosted.org/packages/f4/b5/a14dcc592652347dad23ee93b278a4da5d2a25c9ed3ebd10d68eea823a4f/argon2_cffi_bindings-26.1.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d", size = 27403, upload-time = "2026-08-20T07:32:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/b3/81/b4a20d4902af7f796390bf9245ff83c5217dfa7367efa1d14986956c482b/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728", size = 27132, upload-time = "2026-08-20T07:32:47.13Z" }, + { url = "https://files.pythonhosted.org/packages/7e/1b/c8de358af07b1c490e0fcb863ef98e46ddb486e45567aca5a60bd68d9daa/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81", size = 27588, upload-time = "2026-08-20T07:32:48.087Z" }, + { url = "https://files.pythonhosted.org/packages/48/2f/7ee62a6e79f9309f9d9982d301b22a00010adb580c05c8109b94d7b33de0/argon2_cffi_bindings-26.1.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4", size = 26785, upload-time = "2026-08-20T07:32:48.977Z" }, + { url = "https://files.pythonhosted.org/packages/e9/10/960d0ee93d4897741bcaf4799c697dae2d81499f66fd1ed042a7dd54c1f4/argon2_cffi_bindings-26.1.0-cp310-abi3-win32.whl", hash = "sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb", size = 23898, upload-time = "2026-08-20T07:32:50.114Z" }, + { url = "https://files.pythonhosted.org/packages/6d/3a/0cc14a05810e6add9bce5e87693334baa2222de5f647fa31781885b6573f/argon2_cffi_bindings-26.1.0-cp310-abi3-win_amd64.whl", hash = "sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e", size = 25730, upload-time = "2026-08-20T07:32:51.091Z" }, + { url = "https://files.pythonhosted.org/packages/4e/db/d83cf2af140547f0b9cdaece05b2dc2dcbf991be4667331d073eff771435/argon2_cffi_bindings-26.1.0-cp310-abi3-win_arm64.whl", hash = "sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638", size = 24478, upload-time = "2026-08-20T07:32:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/bb/5f/f652055e18d2627e2eed94c7f31a792127cfe38df786635395d742321674/argon2_cffi_bindings-26.1.0-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083", size = 15434, upload-time = "2026-08-20T07:32:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/76/38/de696045960f5b846d428c0fb6c130ed3da87aac2af209b05c193815404c/argon2_cffi_bindings-26.1.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e", size = 15449, upload-time = "2026-08-20T07:32:54.075Z" }, + { url = "https://files.pythonhosted.org/packages/91/0a/c25af768f6b75a5a71e31207f87c540656b2808c015260444a22763221ad/argon2_cffi_bindings-26.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31", size = 25683, upload-time = "2026-08-20T07:32:55.05Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7e/be212c751ab0bcea7f646615f933bf262e8e50b3f7bef32f861d0a2d066b/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f", size = 27311, upload-time = "2026-08-20T07:32:56.166Z" }, + { url = "https://files.pythonhosted.org/packages/a6/ee/f84b28e4afd13d3cac36c1d8fa8c239d2dc2c51cd978d02ee5d5ad98d9bb/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98", size = 26771, upload-time = "2026-08-20T07:32:57.206Z" }, + { url = "https://files.pythonhosted.org/packages/21/c3/95c07a023691ecd529da9cb6a8f0779e13ebc1bdfaa86d145fdc1c6e7e79/argon2_cffi_bindings-26.1.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605", size = 27568, upload-time = "2026-08-20T07:32:58.361Z" }, + { url = "https://files.pythonhosted.org/packages/e6/31/3a18e31406d8694b4d6a31573c3e572fff6bed318bb744453eb653766d22/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2", size = 27280, upload-time = "2026-08-20T07:32:59.343Z" }, + { url = "https://files.pythonhosted.org/packages/0b/39/d4be4577e178b2397aa5b5575c8a309bf0da2afe05fe0c72c8f398662d63/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a", size = 27776, upload-time = "2026-08-20T07:33:00.325Z" }, + { url = "https://files.pythonhosted.org/packages/71/47/78f4dd96f7411339f723b96fe24039c1bd5835102b8a5ba71ac4ec712ac7/argon2_cffi_bindings-26.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a", size = 26932, upload-time = "2026-08-20T07:33:01.272Z" }, + { url = "https://files.pythonhosted.org/packages/3b/cd/96bfd37434cc0a848a9066c291d84b28846c4c9ea289ed9866b1164d622b/argon2_cffi_bindings-26.1.0-cp314-cp314t-win32.whl", hash = "sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35", size = 24878, upload-time = "2026-08-20T07:33:02.189Z" }, + { url = "https://files.pythonhosted.org/packages/f1/42/d8b6810abd9b1bd2f47ebbccf460da59c9f32e94888bea4f7b137d998797/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8", size = 26656, upload-time = "2026-08-20T07:33:03.222Z" }, + { url = "https://files.pythonhosted.org/packages/a9/d1/095d95eaf2ed1d9f77268cf3291bde148c6cd56121f8db2c74c1ba618a0e/argon2_cffi_bindings-26.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1", size = 25378, upload-time = "2026-08-20T07:33:04.332Z" }, + { url = "https://files.pythonhosted.org/packages/66/cb/214092c39c4dbcb72cf98b12234ddac2221f8fe2c0acf29c6a70fa83be53/argon2_cffi_bindings-26.1.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb", size = 25683, upload-time = "2026-08-20T07:33:05.337Z" }, + { url = "https://files.pythonhosted.org/packages/83/e5/02015b83e9b05ccb85ff2ced424cf6e83a12d3810bc7f66d679a92b69ffb/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6", size = 27310, upload-time = "2026-08-20T07:33:06.344Z" }, + { url = "https://files.pythonhosted.org/packages/c3/4a/85e612787d0796878b3b4f6bd53dcd5484b6fe7b64cc6fc7b6e6a04cf835/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990", size = 26771, upload-time = "2026-08-20T07:33:07.429Z" }, + { url = "https://files.pythonhosted.org/packages/f6/84/ccb003b6f9969820e87656398f4d49c857def71a85ca1588a0e809afd7ce/argon2_cffi_bindings-26.1.0-cp315-cp315t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08", size = 27569, upload-time = "2026-08-20T07:33:08.598Z" }, + { url = "https://files.pythonhosted.org/packages/88/07/c26b76debf0998ee08fbe947ab2058ac5de37d4b9d46b06c17abaa6c4ce9/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca", size = 27279, upload-time = "2026-08-20T07:33:09.518Z" }, + { url = "https://files.pythonhosted.org/packages/ee/0d/ead6ddc029f91bc9b9390686dad3c808ab08100d348f6266b5f93f8970ee/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1", size = 27774, upload-time = "2026-08-20T07:33:10.728Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/c108530d9eb86036b78d3af4de28b83b4a2d9a70512bd10ff8e59966aab4/argon2_cffi_bindings-26.1.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36", size = 26933, upload-time = "2026-08-20T07:33:11.661Z" }, + { url = "https://files.pythonhosted.org/packages/a9/02/0bfc59e781c89acf64c31c388aade9d9d1c1ea38aa1ba1292fe07f607fe9/argon2_cffi_bindings-26.1.0-cp315-cp315t-win32.whl", hash = "sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210", size = 24875, upload-time = "2026-08-20T07:33:12.616Z" }, + { url = "https://files.pythonhosted.org/packages/61/c7/c3e46068cddffccecb8ad94d71135e9bf62bbc789589e7dfadc7c6f59214/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_amd64.whl", hash = "sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4", size = 26655, upload-time = "2026-08-20T07:33:13.521Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ca/18b9c8c45fecf34b9100ec6d7946057f14a158f2eaa20ea123a3e82351cb/argon2_cffi_bindings-26.1.0-cp315-cp315t-win_arm64.whl", hash = "sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440", size = 25376, upload-time = "2026-08-20T07:33:14.491Z" }, +] + +[[package]] +name = "asyncpg" +version = "0.31.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/cc/d18065ce2380d80b1bcce927c24a2642efd38918e33fd724bc4bca904877/asyncpg-0.31.0.tar.gz", hash = "sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735", size = 993667, upload-time = "2025-11-24T23:27:00.812Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/a6/59d0a146e61d20e18db7396583242e32e0f120693b67a8de43f1557033e2/asyncpg-0.31.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad", size = 662042, upload-time = "2025-11-24T23:25:49.578Z" }, + { url = "https://files.pythonhosted.org/packages/36/01/ffaa189dcb63a2471720615e60185c3f6327716fdc0fc04334436fbb7c65/asyncpg-0.31.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d", size = 638504, upload-time = "2025-11-24T23:25:51.501Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/3f699ba45d8bd24c5d65392190d19656d74ff0185f42e19d0bbd973bb371/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a", size = 3426241, upload-time = "2025-11-24T23:25:53.278Z" }, + { url = "https://files.pythonhosted.org/packages/8c/d1/a867c2150f9c6e7af6462637f613ba67f78a314b00db220cd26ff559d532/asyncpg-0.31.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671", size = 3520321, upload-time = "2025-11-24T23:25:54.982Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1a/cce4c3f246805ecd285a3591222a2611141f1669d002163abef999b60f98/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec", size = 3316685, upload-time = "2025-11-24T23:25:57.43Z" }, + { url = "https://files.pythonhosted.org/packages/40/ae/0fc961179e78cc579e138fad6eb580448ecae64908f95b8cb8ee2f241f67/asyncpg-0.31.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20", size = 3471858, upload-time = "2025-11-24T23:25:59.636Z" }, + { url = "https://files.pythonhosted.org/packages/52/b2/b20e09670be031afa4cbfabd645caece7f85ec62d69c312239de568e058e/asyncpg-0.31.0-cp312-cp312-win32.whl", hash = "sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8", size = 527852, upload-time = "2025-11-24T23:26:01.084Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f0/f2ed1de154e15b107dc692262395b3c17fc34eafe2a78fc2115931561730/asyncpg-0.31.0-cp312-cp312-win_amd64.whl", hash = "sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186", size = 597175, upload-time = "2025-11-24T23:26:02.564Z" }, + { url = "https://files.pythonhosted.org/packages/95/11/97b5c2af72a5d0b9bc3fa30cd4b9ce22284a9a943a150fdc768763caf035/asyncpg-0.31.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b", size = 661111, upload-time = "2025-11-24T23:26:04.467Z" }, + { url = "https://files.pythonhosted.org/packages/1b/71/157d611c791a5e2d0423f09f027bd499935f0906e0c2a416ce712ba51ef3/asyncpg-0.31.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e", size = 636928, upload-time = "2025-11-24T23:26:05.944Z" }, + { url = "https://files.pythonhosted.org/packages/2e/fc/9e3486fb2bbe69d4a867c0b76d68542650a7ff1574ca40e84c3111bb0c6e/asyncpg-0.31.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403", size = 3424067, upload-time = "2025-11-24T23:26:07.957Z" }, + { url = "https://files.pythonhosted.org/packages/12/c6/8c9d076f73f07f995013c791e018a1cd5f31823c2a3187fc8581706aa00f/asyncpg-0.31.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4", size = 3518156, upload-time = "2025-11-24T23:26:09.591Z" }, + { url = "https://files.pythonhosted.org/packages/ae/3b/60683a0baf50fbc546499cfb53132cb6835b92b529a05f6a81471ab60d0c/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2", size = 3319636, upload-time = "2025-11-24T23:26:11.168Z" }, + { url = "https://files.pythonhosted.org/packages/50/dc/8487df0f69bd398a61e1792b3cba0e47477f214eff085ba0efa7eac9ce87/asyncpg-0.31.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602", size = 3472079, upload-time = "2025-11-24T23:26:13.164Z" }, + { url = "https://files.pythonhosted.org/packages/13/a1/c5bbeeb8531c05c89135cb8b28575ac2fac618bcb60119ee9696c3faf71c/asyncpg-0.31.0-cp313-cp313-win32.whl", hash = "sha256:f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696", size = 527606, upload-time = "2025-11-24T23:26:14.78Z" }, + { url = "https://files.pythonhosted.org/packages/91/66/b25ccb84a246b470eb943b0107c07edcae51804912b824054b3413995a10/asyncpg-0.31.0-cp313-cp313-win_amd64.whl", hash = "sha256:dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab", size = 596569, upload-time = "2025-11-24T23:26:16.189Z" }, + { url = "https://files.pythonhosted.org/packages/3c/36/e9450d62e84a13aea6580c83a47a437f26c7ca6fa0f0fd40b6670793ea30/asyncpg-0.31.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44", size = 660867, upload-time = "2025-11-24T23:26:17.631Z" }, + { url = "https://files.pythonhosted.org/packages/82/4b/1d0a2b33b3102d210439338e1beea616a6122267c0df459ff0265cd5807a/asyncpg-0.31.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5", size = 638349, upload-time = "2025-11-24T23:26:19.689Z" }, + { url = "https://files.pythonhosted.org/packages/41/aa/e7f7ac9a7974f08eff9183e392b2d62516f90412686532d27e196c0f0eeb/asyncpg-0.31.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2", size = 3410428, upload-time = "2025-11-24T23:26:21.275Z" }, + { url = "https://files.pythonhosted.org/packages/6f/de/bf1b60de3dede5c2731e6788617a512bc0ebd9693eac297ee74086f101d7/asyncpg-0.31.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2", size = 3471678, upload-time = "2025-11-24T23:26:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/46/78/fc3ade003e22d8bd53aaf8f75f4be48f0b460fa73738f0391b9c856a9147/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218", size = 3313505, upload-time = "2025-11-24T23:26:25.235Z" }, + { url = "https://files.pythonhosted.org/packages/bf/e9/73eb8a6789e927816f4705291be21f2225687bfa97321e40cd23055e903a/asyncpg-0.31.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d", size = 3434744, upload-time = "2025-11-24T23:26:26.944Z" }, + { url = "https://files.pythonhosted.org/packages/08/4b/f10b880534413c65c5b5862f79b8e81553a8f364e5238832ad4c0af71b7f/asyncpg-0.31.0-cp314-cp314-win32.whl", hash = "sha256:cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b", size = 532251, upload-time = "2025-11-24T23:26:28.404Z" }, + { url = "https://files.pythonhosted.org/packages/d3/2d/7aa40750b7a19efa5d66e67fc06008ca0f27ba1bd082e457ad82f59aba49/asyncpg-0.31.0-cp314-cp314-win_amd64.whl", hash = "sha256:04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be", size = 604901, upload-time = "2025-11-24T23:26:30.34Z" }, + { url = "https://files.pythonhosted.org/packages/ce/fe/b9dfe349b83b9dee28cc42360d2c86b2cdce4cb551a2c2d27e156bcac84d/asyncpg-0.31.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2", size = 702280, upload-time = "2025-11-24T23:26:32Z" }, + { url = "https://files.pythonhosted.org/packages/6a/81/e6be6e37e560bd91e6c23ea8a6138a04fd057b08cf63d3c5055c98e81c1d/asyncpg-0.31.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31", size = 682931, upload-time = "2025-11-24T23:26:33.572Z" }, + { url = "https://files.pythonhosted.org/packages/a6/45/6009040da85a1648dd5bc75b3b0a062081c483e75a1a29041ae63a0bf0dc/asyncpg-0.31.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7", size = 3581608, upload-time = "2025-11-24T23:26:35.638Z" }, + { url = "https://files.pythonhosted.org/packages/7e/06/2e3d4d7608b0b2b3adbee0d0bd6a2d29ca0fc4d8a78f8277df04e2d1fd7b/asyncpg-0.31.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e", size = 3498738, upload-time = "2025-11-24T23:26:37.275Z" }, + { url = "https://files.pythonhosted.org/packages/7d/aa/7d75ede780033141c51d83577ea23236ba7d3a23593929b32b49db8ed36e/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c", size = 3401026, upload-time = "2025-11-24T23:26:39.423Z" }, + { url = "https://files.pythonhosted.org/packages/ba/7a/15e37d45e7f7c94facc1e9148c0e455e8f33c08f0b8a0b1deb2c5171771b/asyncpg-0.31.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a", size = 3429426, upload-time = "2025-11-24T23:26:41.032Z" }, + { url = "https://files.pythonhosted.org/packages/13/d5/71437c5f6ae5f307828710efbe62163974e71237d5d46ebd2869ea052d10/asyncpg-0.31.0-cp314-cp314t-win32.whl", hash = "sha256:1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d", size = 614495, upload-time = "2025-11-24T23:26:42.659Z" }, + { url = "https://files.pythonhosted.org/packages/3c/d7/8fb3044eaef08a310acfe23dae9a8e2e07d305edc29a53497e52bc76eca7/asyncpg-0.31.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3", size = 706062, upload-time = "2025-11-24T23:26:44.086Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, + { url = "https://files.pythonhosted.org/packages/9d/f4/035513d4117049066b4779dc3b7c0c0fdad175fa13731c9f4003f1cd1478/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e", size = 194248, upload-time = "2026-08-03T21:19:59.399Z" }, + { url = "https://files.pythonhosted.org/packages/76/af/2aeb4dbb5fc41a04161ae9ff1518de7cec08e164f44a8ce6a4cf7fd2cd1d/cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c", size = 196908, upload-time = "2026-08-03T21:20:00.746Z" }, + { url = "https://files.pythonhosted.org/packages/a7/46/2e5fdde8555706dd98139a910ca11be02809f3f605ce956f655d0214e100/cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6", size = 184805, upload-time = "2026-08-03T21:20:02.02Z" }, + { url = "https://files.pythonhosted.org/packages/55/41/4c7042f317b9217502988f0873af87e16ad606dc20f84e546e3e6ce9764c/cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971", size = 184764, upload-time = "2026-08-03T21:20:03.141Z" }, + { url = "https://files.pythonhosted.org/packages/43/1f/1c3d90d91811c8f86ced9ed637956c54bfe5b79ca98fe976d7f8c8979f6b/cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c", size = 214722, upload-time = "2026-08-03T21:20:04.377Z" }, + { url = "https://files.pythonhosted.org/packages/37/6f/3b5ce4c3b2192d250f04908f2bfd91ef34552ec8f7716a5d4abdb8d67bb2/cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125", size = 222369, upload-time = "2026-08-03T21:20:05.544Z" }, + { url = "https://files.pythonhosted.org/packages/02/10/4b3c75dde3d9663c9e02ba05c2668b954f671d4bbe346413ca8c696b295a/cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264", size = 210175, upload-time = "2026-08-03T21:20:06.75Z" }, + { url = "https://files.pythonhosted.org/packages/df/62/14f74b9543e605d17701dc797b815958b8bb70b7624ce1b832ddad48ed6c/cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3", size = 208670, upload-time = "2026-08-03T21:20:08.04Z" }, + { url = "https://files.pythonhosted.org/packages/95/95/86342356ff5953b3fb06f7ef7c5bee212d45e770abc7218d451b9148313c/cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2", size = 221824, upload-time = "2026-08-03T21:20:09.274Z" }, + { url = "https://files.pythonhosted.org/packages/eb/ff/7b3429ff53aafe931ed8a5fc69f481bbef7ba6de87ddcbb63d08f483f613/cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b", size = 225148, upload-time = "2026-08-03T21:20:10.7Z" }, + { url = "https://files.pythonhosted.org/packages/34/34/a95870b9221e09cf4f2ce3178b1a210abdfe63a1bd357da940418d7b8d15/cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7", size = 223564, upload-time = "2026-08-03T21:20:12.165Z" }, + { url = "https://files.pythonhosted.org/packages/70/ea/839b50531021a647fb5e929f72cf97bc1ff702b5472166164b5b6e76b851/cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac", size = 175263, upload-time = "2026-08-03T21:20:13.559Z" }, + { url = "https://files.pythonhosted.org/packages/60/a6/8b149b2c3f2e11aaa1618ef64500b45f50f22c57a977a4dff1aff1f91042/cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d", size = 185688, upload-time = "2026-08-03T21:20:14.69Z" }, + { url = "https://files.pythonhosted.org/packages/01/9a/11f687cb39d6a3504060d5242f04f48c735afb4d3d533958a20594890cb2/cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973", size = 180078, upload-time = "2026-08-03T21:20:15.917Z" }, + { url = "https://files.pythonhosted.org/packages/d3/7b/d6bbf82b8b96e7391438898c42f5bd96dd02030fd5b64937d248220003e2/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c", size = 194064, upload-time = "2026-08-03T21:20:17.148Z" }, + { url = "https://files.pythonhosted.org/packages/94/e6/bcc91b283be94735e268487a054004f0aa19947b6348fa367db53230abc8/cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb", size = 196720, upload-time = "2026-08-03T21:20:18.268Z" }, + { url = "https://files.pythonhosted.org/packages/d9/99/c4b0c17cacdc9c3b8f280026286a9826d6a208c0f047591a3c3ce99b91fd/cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54", size = 184964, upload-time = "2026-08-03T21:20:19.708Z" }, + { url = "https://files.pythonhosted.org/packages/b3/a9/9db617d05d7367c1ad0ab00b3aa6e6f9281edd689b4ee9ea0e5a84e89c97/cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72", size = 184962, upload-time = "2026-08-03T21:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/67/b8/b42132ca113dc567d37684437b46ca1dafc885902b02a110a02d5b511857/cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1", size = 222328, upload-time = "2026-08-03T21:20:22.118Z" }, + { url = "https://files.pythonhosted.org/packages/80/10/c5c0cbf0a657aecf59ef511409734230bf556f05a0d6c9eed7aa5c0a0166/cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062", size = 209985, upload-time = "2026-08-03T21:20:23.401Z" }, + { url = "https://files.pythonhosted.org/packages/d5/6c/bfa0b87b03b9238148beca990292843c9396ba069b54496596594173de7b/cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03", size = 208530, upload-time = "2026-08-03T21:20:24.628Z" }, + { url = "https://files.pythonhosted.org/packages/e9/02/4e7d553a7ac4b4238b38b3c1b80d486e9d4436f8d2acbf87a0997fe3f402/cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96", size = 221525, upload-time = "2026-08-03T21:20:25.758Z" }, + { url = "https://files.pythonhosted.org/packages/82/1d/a4aaf9babd75acb4d5f223bff71533bee748dd770a382619a798960ee9ba/cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527", size = 225053, upload-time = "2026-08-03T21:20:26.985Z" }, + { url = "https://files.pythonhosted.org/packages/81/10/5dc0e7bdd18e22107054288283380fc97a06ae3f1656a106908d666a3c88/cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13", size = 223213, upload-time = "2026-08-03T21:20:28.277Z" }, + { url = "https://files.pythonhosted.org/packages/0b/e9/d0061c364cde06ee43168a0d076ac1da512cbc380d44767b844ba34fe2b6/cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c", size = 177682, upload-time = "2026-08-03T21:20:44.288Z" }, + { url = "https://files.pythonhosted.org/packages/a7/06/1c3e01e3ba14c39f6d10bfbac52753b7e22259e38088e5cfe1d704918690/cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48", size = 187949, upload-time = "2026-08-03T21:20:45.623Z" }, + { url = "https://files.pythonhosted.org/packages/87/5b/da4e39efe18eeb89cf580ea9cfc66b6a7c3eadb808fc0cc1d3a295cb5a5d/cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836", size = 182947, upload-time = "2026-08-03T21:20:46.955Z" }, + { url = "https://files.pythonhosted.org/packages/23/59/40338bf421c5accea1d45158170c87006ef1cd371b05c077e76476949728/cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3", size = 188504, upload-time = "2026-08-03T21:20:29.495Z" }, + { url = "https://files.pythonhosted.org/packages/7d/47/5ecf1023850036e674c77ec4de86182d309ae344e39e7cba984b7df5d647/cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2", size = 188259, upload-time = "2026-08-03T21:20:31.291Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9c/92934c3bea9f785b23eba304538c0b4d37a2a96d2431eb3a1bc87a11aa19/cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94", size = 223864, upload-time = "2026-08-03T21:20:32.571Z" }, + { url = "https://files.pythonhosted.org/packages/4d/45/ba4c93527bc38616a8bd36488acb69a2212d60486794f0c1f318949bbb76/cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc", size = 211538, upload-time = "2026-08-03T21:20:33.808Z" }, + { url = "https://files.pythonhosted.org/packages/80/e9/b6ef565e452acb932fb0cb5443f44a78efbd1233e566f02b5a83855e9115/cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29", size = 210688, upload-time = "2026-08-03T21:20:34.974Z" }, + { url = "https://files.pythonhosted.org/packages/9a/95/eff5f0cee78d2eabc7eebffec40d3fc1876b5f3c95582e018bb4b99601f2/cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676", size = 223803, upload-time = "2026-08-03T21:20:36.564Z" }, + { url = "https://files.pythonhosted.org/packages/fa/01/579d39fb8bef00a335a23d83757b44feb24cd6345a2c451b64cb67b9c362/cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e", size = 226763, upload-time = "2026-08-03T21:20:37.816Z" }, + { url = "https://files.pythonhosted.org/packages/8d/b0/0b44f47c60b01b57b6e2bbd92343f13a85a1d93bc46ccf6e47e244acd99c/cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f", size = 225688, upload-time = "2026-08-03T21:20:38.959Z" }, + { url = "https://files.pythonhosted.org/packages/eb/d2/3b7176cb570a1d3e27faf67b72f591af508036e0d8b2be2ef9af9e8c84bb/cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4", size = 182868, upload-time = "2026-08-03T21:20:40.388Z" }, + { url = "https://files.pythonhosted.org/packages/56/78/31f00c1bcd97c9bbf55f1bfdf5bc809a5de8887473e90bb9960dca825e80/cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e", size = 194104, upload-time = "2026-08-03T21:20:41.725Z" }, + { url = "https://files.pythonhosted.org/packages/7b/1b/58496f2ed0a35de575250c02a43ab3cc2c04d494a88fed31c1cabc0fd176/cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5", size = 186402, upload-time = "2026-08-03T21:20:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/c1/8f/9ebe220eab48a093d1a5a5e339ab0dc7316eef3bb04d63c42f0251b61f50/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d", size = 194043, upload-time = "2026-08-03T21:20:48.179Z" }, + { url = "https://files.pythonhosted.org/packages/ff/69/844bad3ece306c4782c2ecb93597035b6690d48704b803914c199da1e8b3/cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b", size = 196737, upload-time = "2026-08-03T21:20:49.457Z" }, + { url = "https://files.pythonhosted.org/packages/1b/8a/af668013284634733f02d683458a0728739c7d6ddb5e14cb0c20832266fe/cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4", size = 184933, upload-time = "2026-08-03T21:20:50.639Z" }, + { url = "https://files.pythonhosted.org/packages/0c/75/2f5207ff6d1a613133b23a5203cc0c2a628313b5eb3974d7956ae3c57950/cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8", size = 185002, upload-time = "2026-08-03T21:20:52.173Z" }, + { url = "https://files.pythonhosted.org/packages/e2/31/9e1313b0a6e30e91b3b3d3fff51ae99c857c07738e3afcce1f7334e1b7ab/cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6", size = 222271, upload-time = "2026-08-03T21:20:53.462Z" }, + { url = "https://files.pythonhosted.org/packages/50/e3/f6234a833e6e08c7007003074723c406559eecf9b48dfc97471e5a8eb7a0/cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80", size = 209919, upload-time = "2026-08-03T21:20:54.783Z" }, + { url = "https://files.pythonhosted.org/packages/0d/fc/5f74e293fced6edb51af3a46c4ccf6c23c9943774ecb375ddbd522c76add/cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779", size = 208529, upload-time = "2026-08-03T21:20:56.066Z" }, + { url = "https://files.pythonhosted.org/packages/44/16/29e6d01b388bef055ecd6ca8244b3f4d336bd09e92d5d892187b9601084e/cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399", size = 221630, upload-time = "2026-08-03T21:20:57.336Z" }, + { url = "https://files.pythonhosted.org/packages/a4/18/fa7f1f6857d5eb88a4ca99ffcbfb7c387a287ccc154c64a73e86314745d7/cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688", size = 225134, upload-time = "2026-08-03T21:20:58.675Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9f/e8e3dfa04a1b4c241f8c91faacad872b4d4efd051d49764ad4e2fd4b9fea/cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7", size = 223197, upload-time = "2026-08-03T21:20:59.968Z" }, + { url = "https://files.pythonhosted.org/packages/f8/7e/8debeb04f1ab9fe2a6963964cd6f1aaf7192627b83926586a6a4e089c9fa/cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac", size = 177683, upload-time = "2026-08-03T21:21:14.901Z" }, + { url = "https://files.pythonhosted.org/packages/e0/31/5158704cc474ab65c1647932e88be78dc0873f47130e253be38bcaf13d01/cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960", size = 187897, upload-time = "2026-08-03T21:21:16.108Z" }, + { url = "https://files.pythonhosted.org/packages/cc/4b/b3a2da8570c704ffc0f9762cdc3ec0f02c8573798e0b5cf7f11c82bbb70f/cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1", size = 182935, upload-time = "2026-08-03T21:21:17.271Z" }, + { url = "https://files.pythonhosted.org/packages/d0/ef/5443574510a1207e6f6bc38ba6e1f1de36cb48fef07b2728bb896a21f430/cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc", size = 188464, upload-time = "2026-08-03T21:21:01.163Z" }, + { url = "https://files.pythonhosted.org/packages/7e/ae/a56fa8c4686ad50e148fcbc8d3ae0d03915ff5c30d795058988c24118cef/cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab", size = 188262, upload-time = "2026-08-03T21:21:02.382Z" }, + { url = "https://files.pythonhosted.org/packages/53/b2/6187f46f2912276a3ae284076109cc5c8680482f11f766ccf26db4a86427/cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e", size = 223779, upload-time = "2026-08-03T21:21:03.553Z" }, + { url = "https://files.pythonhosted.org/packages/8a/f6/c3ad28bd19f77047a03084424fbd4cbe997303267c14423737324be0385d/cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358", size = 211520, upload-time = "2026-08-03T21:21:04.863Z" }, + { url = "https://files.pythonhosted.org/packages/a0/cd/ccac9013a5bd9fd764de118674ab9c805b5ca10c19270d90ee273f8b2240/cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231", size = 210673, upload-time = "2026-08-03T21:21:06.223Z" }, + { url = "https://files.pythonhosted.org/packages/52/86/2976131c639aead931c5bee5aba67e4b09fbeb8018b6f282f70803f923a7/cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6", size = 223835, upload-time = "2026-08-03T21:21:07.539Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0c/33a7aeab2f9c76918c52e084beb39c570db3588133412929e8ec06fab90b/cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94", size = 226705, upload-time = "2026-08-03T21:21:08.774Z" }, + { url = "https://files.pythonhosted.org/packages/e3/26/2cde30fdde421130bfc18f70395731a6e6b2053c6a1978a5258ff04e72fa/cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5", size = 225539, upload-time = "2026-08-03T21:21:09.911Z" }, + { url = "https://files.pythonhosted.org/packages/6d/cd/a361394c94b2129d604bb846f624a8e88255a3ee33129c434a00d715e64f/cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66", size = 182707, upload-time = "2026-08-03T21:21:11.226Z" }, + { url = "https://files.pythonhosted.org/packages/9b/b5/ba2b299993c26577d529b6ae29841f9e15b9fcf004d65f423f4fcf94ade9/cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3", size = 193772, upload-time = "2026-08-03T21:21:12.39Z" }, + { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "fastapi" +version = "0.141.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8a/02/91e3416a8fdd715abb903a952a6bec7cdd8d14eed55d415fc8595524c319/fastapi-0.141.1.tar.gz", hash = "sha256:e8822fc40db1e1858054d7a949a888695bc9bdce70139178e33bd2871a453ca1", size = 425799, upload-time = "2026-07-29T17:18:05.568Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "mtusi2026lab4" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "argon2-cffi" }, + { name = "asyncpg" }, + { name = "fastapi" }, + { name = "uvicorn" }, +] + +[package.metadata] +requires-dist = [ + { name = "argon2-cffi", specifier = "==25.1.0" }, + { name = "asyncpg", specifier = "==0.31.0" }, + { name = "fastapi", specifier = "==0.141.1" }, + { name = "uvicorn", specifier = "==0.40.0" }, +] + +[[package]] +name = "pycparser" +version = "3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/d3/eb1d3bc30dda12f7e69640ae2ac8cb10240b71fb73024ad528b7d2ae73da/pycparser-3.1.tar.gz", hash = "sha256:b3fc6dec06a8b2fefa0ed4ff92285306a5e3be9987bc5603c9edbdc4e492418f", size = 105946, upload-time = "2026-10-08T12:26:34.349Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/ce/b3ae9ee0324d991c860187be2a6ee436d27a3d02397eaddbb101ec901f3d/pycparser-3.1-py3-none-any.whl", hash = "sha256:f09d358c840bd147b79e55f2bc494f18ea869dc897f5852a8f5766b74f787882", size = 48709, upload-time = "2026-10-08T12:26:32.97Z" }, +] + +[[package]] +name = "pydantic" +version = "2.13.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/ef/fc4f868f4e2cee79f863883abffceff107875f569b848507319842d2a681/pydantic-2.13.5.tar.gz", hash = "sha256:51a9c5f7b2f8e636f04c6cada605d9b6a3bf1348fdf945a3d8869b19bba0ee08", size = 845750, upload-time = "2026-08-28T14:04:00.916Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/47/c95ffc2009878c7aac0c5e08528022dcb885933252a88b5f170058014464/pydantic-2.13.5-py3-none-any.whl", hash = "sha256:346a034f080da3755d8e9cb5e00e8b07de1d39e4f6e2c87d8ab7cafa0b269a73", size = 472589, upload-time = "2026-08-28T14:03:59.136Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/af/f9/8a06bea35ef8daf588f707784c973a7046e0034c8d8cfb08828eeffb8b75/pydantic_core-2.46.5.tar.gz", hash = "sha256:10416c15b8839ecc4ef4d0885da76da6fd0f67333a0eb8aff6d93c4b8f2910fc", size = 472262, upload-time = "2026-08-28T10:01:31.677Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3f/76358795aa7a8c6d4f36e2cb828ad1c90ee118e1393a9281664f5aade9d4/pydantic_core-2.46.5-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b9fe6fb92520e3fd61f2e49000b6911b188824f089b75973ea06d6267f0b476d", size = 2076516, upload-time = "2026-08-28T09:58:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/db/50/26b091836076ce4cb2fac264186936acc069e0595772cfd02a563bc4761a/pydantic_core-2.46.5-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:a39ac25a9a2fa4072efdb429833c4a4c8009a51ff9eea3eeae131713cd27991e", size = 1922874, upload-time = "2026-08-28T09:58:23.766Z" }, + { url = "https://files.pythonhosted.org/packages/09/f0/2a8ce3849e299d44e2d2c196b6082643a3235565a735cb51db7a6261f614/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fdc8b93a41521988916eeaa271173fcca7fa0803d62f87675aac8dcec1c8e29", size = 1951772, upload-time = "2026-08-28T09:58:25.435Z" }, + { url = "https://files.pythonhosted.org/packages/87/46/ac0dc8bdd9e6048183a14eb127764e7ad9240021c17513074a4711b0e31e/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b98134087d9de723658d17a42c7d0da8d6e2ef08015dee7dc93889047315f5e4", size = 2031832, upload-time = "2026-08-28T09:58:27.102Z" }, + { url = "https://files.pythonhosted.org/packages/c4/c2/339de5bef7be36301a2231eaa52e62163742c2281f11b5f4892bc79785cd/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e652ab17569c94bff5475520f907b7148b8c24036a8ebbe5cf7cf7493d28579a", size = 2208645, upload-time = "2026-08-28T09:58:28.948Z" }, + { url = "https://files.pythonhosted.org/packages/7b/a0/9ff22b797724262da14427abaed4dd1d864a139693fc5e7809114376a716/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d925f3d9afd05a8c0fb3a1031463a8d59ebe5e2afad297e29c78be19e13b4e62", size = 2265935, upload-time = "2026-08-28T09:58:30.625Z" }, + { url = "https://files.pythonhosted.org/packages/c0/a4/eb9409ec0736e50aa70a412f16c204ed149516846912f7e6724d4c73ee53/pydantic_core-2.46.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fc5be0abd4a407e200d844b404e33639a554e7bd0d448e7b9ae181be4789ac2", size = 2066284, upload-time = "2026-08-28T09:58:32.289Z" }, + { url = "https://files.pythonhosted.org/packages/c0/02/7f6156ffc926857f1c37c07d9a388682865a81830ab6a1b637082c25e399/pydantic_core-2.46.5-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:816ff0a6550ffc06c098ccd2e0698600f9aa7da192a79eaa6f9af504a35db869", size = 2105889, upload-time = "2026-08-28T09:58:33.986Z" }, + { url = "https://files.pythonhosted.org/packages/92/b1/e781d357ebe09fc929f995700f1b3503e8897f1cece183ecb1300d4d67e9/pydantic_core-2.46.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c7ea57fc63aa7da93a1bd2d644e6577befae10c52c4e36377635eea1056a74f5", size = 2158006, upload-time = "2026-08-28T09:58:35.647Z" }, + { url = "https://files.pythonhosted.org/packages/70/0a/644597d84ab400e50609c192120b85c9681c22d3a20461b9060a79be0a7a/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:efd62a42486f1bda5d24cb4f63d15a3c7768375fe83d36f9417b4ad7a2fb20b3", size = 2158408, upload-time = "2026-08-28T09:58:37.38Z" }, + { url = "https://files.pythonhosted.org/packages/1e/ee/ca3b7b3a4b3769ffe9ce9432a7c9be755de9593a46d3b0d54d0409323e44/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:2bc9419666990c06d7397831f2126a1ecc3594aaa3ff7de5bf2d066802f4e07b", size = 2309609, upload-time = "2026-08-28T09:58:39.22Z" }, + { url = "https://files.pythonhosted.org/packages/ce/52/39fa1f451486019524ca685020390e7ca351832fd874530ba30c8628e6dc/pydantic_core-2.46.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:18a09e1e1011b462f2e32774f25859ef1223d5c2b0546a633cf56654710721e0", size = 2342618, upload-time = "2026-08-28T09:58:40.89Z" }, + { url = "https://files.pythonhosted.org/packages/81/5e/468fc630568c61dcef3cd47ad32ffbeed9af643f49208d1ea86ab4f890c4/pydantic_core-2.46.5-cp312-cp312-win32.whl", hash = "sha256:5cb482e9e84c851f4e623fe4acc1ced89168cf1fe18f7089db4548c8f5bbb65b", size = 1939475, upload-time = "2026-08-28T09:58:42.591Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c9/4c19f41b84cf6b622a72fbeed7665b25d47a187d68d47d0d430c07f23268/pydantic_core-2.46.5-cp312-cp312-win_amd64.whl", hash = "sha256:5e81740c09e310f5aa5cbd3e434a01c154d4bef93241c7877b39f211d2b78ba8", size = 2043140, upload-time = "2026-08-28T09:58:44.272Z" }, + { url = "https://files.pythonhosted.org/packages/af/dd/0c1a050299147c746e5256db16d645ab5efd4f78c59937d581a0524e74a2/pydantic_core-2.46.5-cp312-cp312-win_arm64.whl", hash = "sha256:f7b0ec93a2893de856652154d73b7ba622f26fa97726487dcac373de5f4c6084", size = 1997729, upload-time = "2026-08-28T09:58:46.13Z" }, + { url = "https://files.pythonhosted.org/packages/f5/37/5abe39a8372a61d3dc3c1338fc504281c01b32fdb3169cd7187153b56d3e/pydantic_core-2.46.5-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:b7ca9034437b6022f941f4857459562ee00a560b97e7cce8a0ec5a74fc6766e0", size = 2075885, upload-time = "2026-08-28T09:58:47.856Z" }, + { url = "https://files.pythonhosted.org/packages/21/43/6323b1f8b217780454c61304bcd2b38ae4762f50754414124603ccc90bb2/pydantic_core-2.46.5-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f332f0e72a5a0400141f830744e141bf9f97917878dbe968669e8a7fefea78ff", size = 1922768, upload-time = "2026-08-28T09:58:49.58Z" }, + { url = "https://files.pythonhosted.org/packages/0f/a3/c05ca796e1197618a774b01e596aeedfefc2f7d8c01ae3054e910b120e8a/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:193375f3548919d3f0b60936ca113ada3e38f264f91b9b8e0508efaad57be931", size = 1951241, upload-time = "2026-08-28T09:58:51.511Z" }, + { url = "https://files.pythonhosted.org/packages/68/32/33bc39ac705c52cffc908e8389f9754fdb208aea5c69cceddf4eb3ce99af/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:79bdfa52f843137045b2d081cc05c120ba6665d29b7559c2c47690906f39279f", size = 2031975, upload-time = "2026-08-28T09:58:53.166Z" }, + { url = "https://files.pythonhosted.org/packages/b0/70/2333e885c0f6a67bc105c5916965dac9b57f2718ee20d81d1a06a4ebdc13/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:24922243639cbdac66c75fcb6fd6495a9cb52b213d62f9a0d16f0310b1ff8038", size = 2208542, upload-time = "2026-08-28T09:58:55.017Z" }, + { url = "https://files.pythonhosted.org/packages/f7/ea/296debfb4264207bbda5936133892e027c0a58875ad53ebd512fba8ec3a2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c76fe65e607be28c7fd4d56fc3c42b1583aa058ce3408b7ad0fd540171d31f9f", size = 2264692, upload-time = "2026-08-28T09:58:56.767Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/9e4de77a6271e07a76d2d58b11c091a979c191ed2939bf80067568b369d2/pydantic_core-2.46.5-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f7b393a8b3da82f5c1fc0751e6d01ac6c55b93c18226a60bdfba4a724efafd1", size = 2066633, upload-time = "2026-08-28T09:58:58.531Z" }, + { url = "https://files.pythonhosted.org/packages/8d/db/f9e9d0c97445987b2084823d5c240de88087338f04fc2cfaa2df186b8049/pydantic_core-2.46.5-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:7ac031912d54f3d83ef3b3eb98dfabc1608802e2202263d25957eeed40b94761", size = 2105235, upload-time = "2026-08-28T09:59:00.421Z" }, + { url = "https://files.pythonhosted.org/packages/07/c5/79169b047b3b2c3e99e04bc76372af9637e0bf6db638274fa927df96369e/pydantic_core-2.46.5-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:837b396ca3d7b74091ca623f6cbd8351bd42d670a79c2683e79fb089f06a2de5", size = 2157367, upload-time = "2026-08-28T09:59:02.442Z" }, + { url = "https://files.pythonhosted.org/packages/26/b5/ba6057afb7c291bd449f51b867f95aef2072941c4ce4e5c31d6ffd132d3b/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:5ee239d575f80b08eca11f6e20f90c4c695de7825c67eefe6091fbf20dda648e", size = 2158420, upload-time = "2026-08-28T09:59:04.2Z" }, + { url = "https://files.pythonhosted.org/packages/6e/28/2057abecaafdc22912afa819603a51f0a62d40643b7c4871c51721fea9be/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:e80675d75ae2cd14372cb65cad5400d9347a3d3f6c13000183f22dfd027283ed", size = 2309588, upload-time = "2026-08-28T09:59:06.048Z" }, + { url = "https://files.pythonhosted.org/packages/71/9d/881156dc404e27479c4246128d73538464cab4a239bec61995e227644c30/pydantic_core-2.46.5-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:9c4b71f10dd532fb7a5cbc8f58707779e64f03a258c2bf8bfbaecfcd9970b519", size = 2341866, upload-time = "2026-08-28T09:59:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/5a/38/d66f443a259f84d13babdceae568e572b0ed26da17ca5d0a649ebb110a67/pydantic_core-2.46.5-cp313-cp313-win32.whl", hash = "sha256:97bf8de4d541598c94a59344eeb988a94c08ff76b5723c41f6567ec18c7892ea", size = 1938580, upload-time = "2026-08-28T09:59:10.402Z" }, + { url = "https://files.pythonhosted.org/packages/2c/1e/1d5371213f4cc9a7ed70c0bfcc7911de22311ee99a662a56077d7292d2ac/pydantic_core-2.46.5-cp313-cp313-win_amd64.whl", hash = "sha256:15f4a94963c95accac15b7b657bb177d3ad82bb90b0d0526d9a9b85079925db5", size = 2041980, upload-time = "2026-08-28T09:59:12.396Z" }, + { url = "https://files.pythonhosted.org/packages/5a/48/4222d90b1c67568bace4dec6dca6271449c66de3595d72b6d098f5fde597/pydantic_core-2.46.5-cp313-cp313-win_arm64.whl", hash = "sha256:d22a945598fb91236b4dd793a6e42e4f3dd7740bb5aace5ebd7d4c08d13bb575", size = 1997213, upload-time = "2026-08-28T09:59:14.245Z" }, + { url = "https://files.pythonhosted.org/packages/8e/8a/14596f2a8367da50cf7cbac48169ee5d9c8e11d486a3b527082384630c72/pydantic_core-2.46.5-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:c1c43ad4339643d70ebb8124e1305a7dab423001eff58bb41a0f731adbc98355", size = 2074081, upload-time = "2026-08-28T09:59:16.141Z" }, + { url = "https://files.pythonhosted.org/packages/ae/d5/d8a4eb6d6c7f66b91dd37c576d76e9e60fba900caf5372c17bcf949febc2/pydantic_core-2.46.5-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1a353f84de772f423b5ffb11d7ae352fbbef0f446f3c0b0af0f8236d7233606e", size = 1920497, upload-time = "2026-08-28T09:59:18.065Z" }, + { url = "https://files.pythonhosted.org/packages/8e/26/092079428f86e927e030b2c0ced87df69dbb1c875cdeaa67bf42ea2be746/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5086029a57366b8cf81b130a43908738095c270c21a8d7f0e8bdfdb89718e2f3", size = 1952130, upload-time = "2026-08-28T09:59:20.476Z" }, + { url = "https://files.pythonhosted.org/packages/08/c3/8ec0e290a9ebaebd64047bf5fda94be835c6b1551b02437e4b76778fbcd7/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:46c25dda9d092a06c08db76ffe0a197107904d0dfac653f7d5306bbcd6d6119c", size = 2026371, upload-time = "2026-08-28T09:59:22.227Z" }, + { url = "https://files.pythonhosted.org/packages/01/72/4fd20ad520fb8da0157f95b27a7eb05a72790ef08138e7701ac972c342ea/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:37ea7b83c935e5b0d68c9449b82651accf78a10828b2c02b2f2d9e9496446c21", size = 2202822, upload-time = "2026-08-28T09:59:24.277Z" }, + { url = "https://files.pythonhosted.org/packages/31/b0/d16e0771206b29314f0d52198b720be21e8a99ab2bf11e3bc0d7c9cebdff/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e64e88d5585bea9ce95861079de72006c7fa6d3df4e3a3b65ba31eb979c15c9f", size = 2262756, upload-time = "2026-08-28T09:59:26.608Z" }, + { url = "https://files.pythonhosted.org/packages/2c/9b/59634b7ac631c63b2a37760eb6943af3e29573d6b59a4abc5e7f019d4cee/pydantic_core-2.46.5-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:54d510bac3ee52247af28ed4bb18a1e799f040ac60fd2bf5ccd4c92f1fbe786f", size = 2068352, upload-time = "2026-08-28T09:59:29.044Z" }, + { url = "https://files.pythonhosted.org/packages/08/7c/570abb1ad2155348dc754ea91be22e5aaa18eb6d69a6068f7c6f2679a6ed/pydantic_core-2.46.5-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:a2a5e1d0ff29adddc9f6d6821a66302e4493f8ca898b715b6b1182c2c201ea0a", size = 2104777, upload-time = "2026-08-28T09:59:30.95Z" }, + { url = "https://files.pythonhosted.org/packages/8e/25/5bf74adc65a1ac5b7be3f6cb0bcb5433615c1598a801c19d830d84c98ded/pydantic_core-2.46.5-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03b9666e41e35d8909852ba191a0607520f81b74eaf12ccf8737005dbb313821", size = 2156312, upload-time = "2026-08-28T09:59:32.604Z" }, + { url = "https://files.pythonhosted.org/packages/90/6a/2ef38830675e050121040618135564ed56b860b45433b02d9b4ebece46f3/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:a91c17edf6eea2402cb5457b4c89e99bc5ed1004aa34c4adf1d4258c1a5c22c2", size = 2150067, upload-time = "2026-08-28T09:59:34.453Z" }, + { url = "https://files.pythonhosted.org/packages/90/ef/a7dbb03a14a64c2a4621f989c615ed9a892535a6cad938fc27079f919d80/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b49924c73a235e969511bf2aabdff3beebf9820931f646c80274d5d780010c47", size = 2304516, upload-time = "2026-08-28T09:59:36.194Z" }, + { url = "https://files.pythonhosted.org/packages/68/f8/6bb4c4b80e8a6fde1904c64a51c62a1d04fcdfa3ea521a66b2ddefa1d885/pydantic_core-2.46.5-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:2cbd9a5eff05e51c447c34dfa4632145b26b09120cf04bd0c871e44c1a5e1c9a", size = 2335223, upload-time = "2026-08-28T09:59:37.931Z" }, + { url = "https://files.pythonhosted.org/packages/2a/80/f46b8c681195190b2c1f1c7c0a81abce60663e987613e09ef64d433dd96b/pydantic_core-2.46.5-cp314-cp314-win32.whl", hash = "sha256:2d5d76654becf5efd62c9e51c3756c67b49498b0c9a40884934c40807adbd074", size = 1934827, upload-time = "2026-08-28T09:59:39.836Z" }, + { url = "https://files.pythonhosted.org/packages/f7/3c/60674207246bc0a4009d2391b7c7251c7159f279c8d2ab8aae8ef46f3dee/pydantic_core-2.46.5-cp314-cp314-win_amd64.whl", hash = "sha256:fa10ef4112775900e7a0661068635eb67b2ab824fbde764de6e0e21982a93db0", size = 2042648, upload-time = "2026-08-28T09:59:41.792Z" }, + { url = "https://files.pythonhosted.org/packages/69/0c/117c562c7c1babdf44576b72a5e496906506c93690387ecfbca7c729ae2e/pydantic_core-2.46.5-cp314-cp314-win_arm64.whl", hash = "sha256:045ab3b6d308439e32b81cc173bba5b9018bc6ed896afd0c65b3b009b1699af5", size = 1989652, upload-time = "2026-08-28T09:59:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/e8/66/9336ae58f9eb68c41d121894e52c4c89eccb07eb8f602a04ee9c3f37736a/pydantic_core-2.46.5-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8816f3d218beb4b787de5c9759c259b8fa61f9dec42dc7811f320a33771778b7", size = 2065829, upload-time = "2026-08-28T09:59:45.364Z" }, + { url = "https://files.pythonhosted.org/packages/c5/02/bc19b47a96c2d3109760711acf22369e56bd7e405ca52f7ade164d2ead57/pydantic_core-2.46.5-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:bce57638e08ac148e5778cce7feb968307a727d66f8e2274a543d0cf0c9ad6a3", size = 1905716, upload-time = "2026-08-28T09:59:47.18Z" }, + { url = "https://files.pythonhosted.org/packages/52/a4/70b47c0509923dd98ccfed04fb3e32ea3849c82a0ff2205bb41009b43c00/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:976e1128455aa595ea04c79ccfedff1aaeab96ee013fcc916bed120c4f0ad94f", size = 1934216, upload-time = "2026-08-28T09:59:49.241Z" }, + { url = "https://files.pythonhosted.org/packages/52/ab/aa03b65f7bb198585edf806b906c3223ecf1795543e39e23aec4cce27ad2/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e7b891faeedeafba41b2983e5001a81b6a915b69544c7e7570d1989ce1c36ac7", size = 2010635, upload-time = "2026-08-28T09:59:51.692Z" }, + { url = "https://files.pythonhosted.org/packages/3c/8b/0da06343f30b84ec549aafd309c6456223d5dc8bd36af504c573faad561d/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5f194189415698233dd1114a093a9b56e61e2c57e11b469be3b0506f46f0771c", size = 2209369, upload-time = "2026-08-28T09:59:53.582Z" }, + { url = "https://files.pythonhosted.org/packages/d6/5b/844c4defaa34a3df66eb9257087d121d70c201298b96abdf9f492fc2f1bf/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:82a36973cf8a2ef5406f4fe2edbf8ed0c99629535d959e0b100c76a32535a111", size = 2253238, upload-time = "2026-08-28T09:59:55.484Z" }, + { url = "https://files.pythonhosted.org/packages/f4/64/a4e536cb16d7f61a7fd3120b46c577fc7fa7325992f69c4f52bc786d77d8/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cdbb78909f52b981d3b2d56b97328d71eb0b974c36bd77c920123a7ebb192829", size = 2065740, upload-time = "2026-08-28T09:59:58.038Z" }, + { url = "https://files.pythonhosted.org/packages/5f/75/aaa38c6bc2d085f6605b34eabdc6a8a4e0b2e61fc9c8e6e52b28e97b3125/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:52e24eacdb536cade636aa90fb851835222becff8484b7001fdc78cb0290f2aa", size = 2087425, upload-time = "2026-08-28T09:59:59.898Z" }, + { url = "https://files.pythonhosted.org/packages/55/ae/fcab4cfc39aba3689e1d20c8b5250ad280957022c09af2ed9cd585602a5e/pydantic_core-2.46.5-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:37ae34309d7bd8c0d61ab839668058f2a7962ea1fc51d105d2db228fe0618034", size = 2139306, upload-time = "2026-08-28T10:00:03.057Z" }, + { url = "https://files.pythonhosted.org/packages/2d/f4/f1d03a4bc9d9acbc62f4d742b8a319af52f71885079868b2ff8e48a651ee/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:0cdbada856a1c69a7624a64d3d9aefe79300bd6ef827b43a4f265010b9b55184", size = 2144589, upload-time = "2026-08-28T10:00:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/83/f3/7a53bb1356de514a4cd295f25b6ac39237895620c0462d2592b76c16e114/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:545f26c504b27c3758439a5e6d9349931f0a04f855668d5fe323c89e82300a38", size = 2288882, upload-time = "2026-08-28T10:00:07.931Z" }, + { url = "https://files.pythonhosted.org/packages/cd/94/5a81583660c175c59d49ffb09f4b3a44debeaf86a19fca664ae1cdd9ee32/pydantic_core-2.46.5-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:ff218293c9c806138dca139765e3b067621be52bcd93cdc14c7711be7ddc90a9", size = 2335210, upload-time = "2026-08-28T10:00:10.177Z" }, + { url = "https://files.pythonhosted.org/packages/5a/9f/5d685c2693b972d1a59c998586e8823712b66603aeff47ee60a4bdaafd37/pydantic_core-2.46.5-cp314-cp314t-win32.whl", hash = "sha256:97cf3eb53a8cccacf9d46686a0926186c9bfb5574f2ed66d3639d5fe117cd3a9", size = 1921180, upload-time = "2026-08-28T10:00:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/70/12/5c94ee16d65a37a15f9e869f5e6256df111154491173801a4c5e800ab548/pydantic_core-2.46.5-cp314-cp314t-win_amd64.whl", hash = "sha256:d2f9fc07a8042a8f95925b35c4f04f469707c981fc33245b6ca187cf5d2dd290", size = 2020515, upload-time = "2026-08-28T10:00:14.774Z" }, + { url = "https://files.pythonhosted.org/packages/63/19/67830dda664e6bdf9285ee2e40f355d0d7d6b92aa0c42e8d217bb8d33d36/pydantic_core-2.46.5-cp314-cp314t-win_arm64.whl", hash = "sha256:acf8a67ba51f4ca9ddbd0e6b3000a65ac51ab734661778b3e7ba64d99a710f2f", size = 1989276, upload-time = "2026-08-28T10:00:16.984Z" }, + { url = "https://files.pythonhosted.org/packages/df/dd/053c2e4303f791f3b8f8a14ab0b22008e8eb21d868c0c90b4f9be705b76a/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:013d6f3483d81e02e7c328831808f336c8596ee33b4bd4026b9ffb1e960b8942", size = 2062540, upload-time = "2026-08-28T10:01:00.318Z" }, + { url = "https://files.pythonhosted.org/packages/d7/dd/a18df751a5e37dd51bfad7f68e766999125bebe68c9e1d10a493ad01bd63/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:e9c134bb666dd54b778b9fc0d2b50cbb7f979b9e3716f26a88c9ab3b6fc1dd0f", size = 1902040, upload-time = "2026-08-28T10:01:02.529Z" }, + { url = "https://files.pythonhosted.org/packages/b7/13/01d40f9d07ce8a779fd6e0bd8ad4fba91309500dd67b869e2e219d261a6d/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:347ec774390c87326a2e4929d58d3f7e8763a104d5d35f4cd595a4c952366433", size = 1967479, upload-time = "2026-08-28T10:01:05.004Z" }, + { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" }, +] + +[[package]] +name = "starlette" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7b/2b/3850dc6bf7ef71b088962eba31dafc6cffd2f96e577ebb0bb316df96da3e/starlette-1.7.0.tar.gz", hash = "sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d", size = 2736246, upload-time = "2026-09-23T07:30:26.35Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/d6/1ec1b290f9e0fb067899b61e1d37a30c923068bad260b216dbe37a7d2967/starlette-1.7.0-py3-none-any.whl", hash = "sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e", size = 78980, upload-time = "2026-09-23T07:30:24.567Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.40.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c3/d1/8f3c683c9561a4e6689dd3b1d345c815f10f86acd044ee1fb9a4dcd0b8c5/uvicorn-0.40.0.tar.gz", hash = "sha256:839676675e87e73694518b5574fd0f24c9d97b46bea16df7b8c05ea1a51071ea", size = 81761, upload-time = "2025-12-21T14:16:22.45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/d8/2083a1daa7439a66f3a48589a57d576aa117726762618f6bb09fe3798796/uvicorn-0.40.0-py3-none-any.whl", hash = "sha256:c6c8f55bc8bf13eb6fa9ff87ad62308bbbc33d0b67f84293151efe87e0d5f2ee", size = 68502, upload-time = "2025-12-21T14:16:21.041Z" }, +] -- 2.54.0 From e9ef559c349cf8b6fc32138fed0ea7a7600ce1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Tue, 20 Oct 2026 17:24:32 +0300 Subject: [PATCH 2/5] add level 3 --- .env.example | 2 +- Makefile | 13 ++ REPORT.md | 172 ++++++++++++++----------- compose.yaml | 61 ++++++++- evidence/attachment-failover.txt | 1 + evidence/attachments.txt | 4 + evidence/bootstrap.txt | 3 + evidence/contracts.txt | 4 + evidence/environment.json | 107 +++++++++++++++ evidence/failover.txt | 4 + evidence/lab3-regressions.txt | 2 + evidence/persistence.txt | 5 + evidence/private-bucket.txt | 1 + evidence/s3-outage.txt | 2 + evidence/smoke.txt | 2 + infra/filer.toml | 3 + migrations/004_attachments.sql | 18 +++ pyproject.toml | 2 + scripts/init-secrets.py | 34 +++++ scripts/record-environment.py | 44 +++++++ scripts/restart.sh | 4 +- scripts/verify-persistence.py | 27 ++++ src/attachments.py | 104 +++++++++++++++ src/bootstrap_s3.py | 38 ++++++ src/config.py | 4 + src/main.py | 9 +- src/models.py | 40 ++++-- src/routes.py | 2 + src/storage/postgres.py | 52 +++++++- src/storage/s3.py | 85 ++++++++++++ student-tests/attachment_resilience.py | 115 +++++++++++++++++ student-tests/attachments.py | 111 ++++++++++++++++ uv.lock | 159 +++++++++++++++++++++++ 33 files changed, 1138 insertions(+), 96 deletions(-) create mode 100644 evidence/attachment-failover.txt create mode 100644 evidence/attachments.txt create mode 100644 evidence/bootstrap.txt create mode 100644 evidence/contracts.txt create mode 100644 evidence/environment.json create mode 100644 evidence/failover.txt create mode 100644 evidence/lab3-regressions.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/private-bucket.txt create mode 100644 evidence/s3-outage.txt create mode 100644 evidence/smoke.txt create mode 100644 infra/filer.toml create mode 100644 migrations/004_attachments.sql create mode 100644 scripts/init-secrets.py create mode 100644 scripts/record-environment.py create mode 100644 scripts/verify-persistence.py create mode 100644 src/attachments.py create mode 100644 src/bootstrap_s3.py create mode 100644 src/storage/s3.py create mode 100644 student-tests/attachment_resilience.py create mode 100644 student-tests/attachments.py diff --git a/.env.example b/.env.example index 217c94be..21836d11 100644 --- a/.env.example +++ b/.env.example @@ -12,7 +12,7 @@ CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 CURSOR_SECRET_FILE=/run/secrets/cursor_secret -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id diff --git a/Makefile b/Makefile index 5d86d171..fc7400ef 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,16 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: test-attachments test-private test-upload-failover test-s3-outage +test-attachments: + $(PYTHON) student-tests/attachments.py --base-url "$(BASE_URL)" + +test-private: + $(PYTHON) student-tests/attachment_resilience.py --mode private --base-url "$(BASE_URL)" + +test-upload-failover: + $(PYTHON) student-tests/attachment_resilience.py --mode failover --base-url "$(BASE_URL)" + +test-s3-outage: + $(PYTHON) student-tests/attachment_resilience.py --mode s3-outage --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index 02c40569..b14dfe68 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,100 +1,122 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Матвей Козырев, БВТ2301. +- Целевая оценка: **3 — работающий минимум**. +- Ветка: `solution/BVT2301-Kozyrev`. +- Проверки: 20 октября 2026 года. Базовый commit и SHA-256 исходников проверенного рабочего дерева — [evidence/environment.json](evidence/environment.json). +- Использованные библиотеки: FastAPI, Uvicorn, asyncpg, argon2-cffi, boto3, Pillow. Помощь ИИ: Codex — перенос основы lab3, реализация вложений, тесты и документация. ## Запуск из чистого clone -На текущем этапе перенесена база lab3; S3 и вложения ещё не реализованы. -Инструкция ниже запускает только эту базу. - -Нужны Docker Engine с Compose v2 и Python 3 для создания локальных секретов. -Версии: Python 3.12.12 и uv 0.11.28 в Dockerfile, PostgreSQL 17.6-alpine, -HAProxy 3.0.11-alpine в compose.yaml; зависимости Python закреплены в uv.lock. - -Из корня репозитория: +Нужны Docker Engine с Compose v2, Python 3.12+ и свободный порт 8080. ```sh -python3 - <<'PY' -from pathlib import Path -import secrets - -directory = Path("secrets") -directory.mkdir(mode=0o700, exist_ok=True) -names = ("postgres_password", "database_url", "cursor_secret") -if any((directory / name).exists() for name in names): - raise SystemExit("Секреты уже существуют: сохраните их для существующей БД.") -password = secrets.token_hex(32) -values = ( - password, - f"postgresql://messenger:{password}@postgres:5432/messenger", - secrets.token_hex(32), -) -for name, value in zip(names, values): - path = directory / name - with path.open("x", encoding="utf-8") as stream: - path.chmod(0o600) - stream.write(value + "\n") -PY -docker compose --env-file .env.example config --quiet -docker compose --env-file .env.example up -d --build --wait +python3 scripts/init-secrets.py +docker compose config --quiet +docker compose up -d --build --wait curl --fail http://localhost:8080/health/ready -make check -python3 student-tests/review_regressions.py +make test ``` -Сервис `migrate` применяет SQL-миграции до запуска API. -Вход через HAProxy: `http://localhost:8080`; порты API и PostgreSQL на host не опубликованы. -Порт 8080 должен быть свободен. Compose использует отдельный проект и volume lab4; -секреты и данные lab3 не переносились. +`init-secrets.py` создаёт локальные случайные credentials в игнорируемом каталоге `secrets/`, не выводит значения и сохраняет существующие файлы при повторном запуске. `.env` не требуется для стандартного Compose: параметры и пути к secrets заданы в [compose.yaml](compose.yaml). Для переопределения TTL можно скопировать `.env.example` в `.env`. S3 endpoint, bucket и лимиты стандартного стенда заданы непосредственно в Compose; `API_REPLICAS`, `SESSION_STORE` и `CACHE_URL_FILE` из шаблона не управляют этой топологией. Реплик две, сессии в PostgreSQL, Redis не используется. -Остановка с сохранением данных: `docker compose down`. -Намеренное удаление данных этого стенда: `docker compose down -v`. +Последовательность запуска: PostgreSQL → `migrate` → API; SeaweedFS → `s3-init` → API; здоровые API → HAProxy. Миграции и bootstrap — одноразовые сервисы; их успешное состояние — `Exited (0)`. Bucket `messenger-attachments` создаётся bootstrap-сервисом, ему устанавливается private ACL и удаляется публичная policy, если она существовала. Повторный bootstrap безопасен: -Полный `make test` пока не проходит: модель Message перенесена из lab3 без -`attachment_ids`, endpoint вложений отсутствуют. Публичные тесты и контракт lab4 -сохранены. Проверки `test-persistence`, `test-failover` и собственная проверка -`idempotency_concurrency.py` используют контракт lab4 и требуют дальнейшей -реализации. Скрипты restart/stop-one перенесены для текущей топологии; -при добавлении S3 restart необходимо дополнить перезапуском этого хранилища. +```sh +docker compose run --rm --no-deps s3-init +``` + +Вход только через `http://127.0.0.1:8080`. Порты PostgreSQL, API и SeaweedFS на host не публикуются. Остановка с сохранением данных: `docker compose down`; повторный запуск — `docker compose up -d --wait`. **Намеренное удаление всех учебных данных lab4:** `docker compose down -v`. Секреты следует сохранять вместе с volumes, иначе существующая БД может перестать принимать подключение. + +Версии runtime и зависимостей закреплены в [Dockerfile](Dockerfile), [pyproject.toml](pyproject.toml) и [uv.lock](uv.lock). Python контейнера — 3.12.12, uv — 0.11.28; boto3 — 1.43.115, Pillow — 13.0.0. ## Архитектура и решения -Перенесено из локального проекта `mtusi2026lab3`: `src/`, миграции -`001`–`003`, Dockerfile, зависимости и lockfile, основной Compose, конфигурация -HAProxy, restart/failover hooks и собственные проверки сессий, идемпотентности, -readiness и ошибок API. Исследовательский benchmark lab3 и старые результаты -экспериментов не входят в этот перенос. +```text +Клиент → HAProxy → api-a / api-b → PostgreSQL (метаданные) + → SeaweedFS S3 (оригиналы) +``` -Текущая схема: клиент → HAProxy → api-a / api-b → PostgreSQL. -PostgreSQL хранит пользователей, сессии, чаты, сообщения и записи идемпотентности. -Общий `CURSOR_SECRET_FILE` задаёт ключ подписи курсоров для обеих реплик. -Исходники API пока сохраняют поведение и версию 1.3.0 из lab3. -Параметры S3 из `.env.example` будут подключены на следующем этапе. +| Роль / service | Версия | Готовность | Persistent storage | Доступ и credentials | +| --- | --- | --- | --- | --- | +| API / api-a, api-b | Python 3.12.12, исходники репозитория | `/health/ready`: PostgreSQL SELECT + S3 HeadBucket | Нет | Внутренний 8080; DB URL, cursor secret, отдельные S3 credentials из файлов | +| PostgreSQL / postgres | 17.6-alpine | pg_isready | postgres_data | Внутренний 5432; password file | +| LB / load_balancer | HAProxy 3.0.11-alpine | readiness API, активные backend checks | Нет | Host 127.0.0.1:8080 | +| S3 / s3 | SeaweedFS 4.17, digest в Compose | master cluster status; bootstrap дополнительно проверяет S3 | s3_data: файлы и LevelDB filer | Внутренний 8333; identity config из secret | +| Bootstrap / s3-init | Тот же Python-образ | exit 0 | Нет | Отдельные административные S3 credentials | -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +SeaweedFS закреплён digest `sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9`. Настройка filer LevelDB в [infra/filer.toml](infra/filer.toml) сохраняет его метаданные в `/data/filer` на том же volume. S3 identity `api` имеет Read/Write/List для одного bucket; anonymous identity отсутствует. Административные credentials доступны bootstrap, но не API. Внутренняя Docker-сеть считается доверенной; это не защита от компрометации контейнера. -## Доказательства критериев +Лимиты Compose: каждая API — 0,5 CPU / 512 MiB; PostgreSQL — 0,75 CPU / 512 MiB; SeaweedFS — 1 CPU / 512 MiB; HAProxy — 0,25 CPU / 128 MiB. Это учебный стенд на одном Docker-host, без высокой доступности БД, S3, LB и самого host. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +### Перенос lab3 и модель вложений -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Из lab3 сохранены пользователи, чаты, общие PostgreSQL-сессии, Argon2 password hashing, keyset-пагинация, подпись cursor общим ключом, `X-Request-Id`, `X-Instance-Id`, идемпотентность сообщений и две реплики за HAProxy. Исходные публичные `tests/` и `contracts/` lab4 не изменены. -## Эксперименты и отказы +Миграция [004_attachments.sql](migrations/004_attachments.sql) добавляет таблицу Attachment: ID, чат, владелец, отображаемое имя, MIME, размер, SHA-256, внутренний object key и время. Байты в PostgreSQL не записываются. `messages.attachment_ids` — массив UUID, по умолчанию пустой. При создании сообщения в одной транзакции проверяются членство в чате и принадлежность всех вложений этому чату, затем записываются сообщение и результат идемпотентности. У массива нет отдельного внешнего ключа: связь проверяет приложение; удаления вложений и изменения состава чата в API нет. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Пустой текст разрешён только с вложением. Пропущенное `attachment_ids` и `[]` эквивалентны; hash текстовых сообщений сохранён совместимым с lab3. Для сообщения с вложениями hash также включает упорядоченный список нормализованных UUID. История и повтор POST возвращают те же attachment IDs. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +### Загрузка и скачивание -## Ограничения и следующие шаги +[attachments.py](src/attachments.py) принимает raw PNG/JPEG. Сначала проверяются Bearer-сессия и членство в чате. `filename` не используется как путь; запрещены `/`, `\`, управляющие символы и недопустимая длина. Сервер генерирует ключ `originals/`. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Лимит **10 MiB = 10 × 1024 × 1024 = 10 485 760 байт** проверяется по Content-Length и повторно при чтении каждого блока, в том числе для chunked upload. До декодирования Pillow определяет фактический формат и размеры; максимум **40 000 000 пикселей**. Проверяются целостность PNG и декодирование пикселей PNG/JPEG. MIME должен совпадать с содержимым. Анимированный PNG не поддерживается: ответ 415. + +Во время приёма используется удаляемый временный файл; он не является хранилищем оригиналов и не используется при скачивании. На одну API-реплику допускается одна одновременная загрузка/проверка изображения, ожидание вместе с загрузкой ограничено 30 секундами. SDK и декодирование выполняются вне event loop. S3 connect timeout — 2 с, read timeout — 5 с, автоматические повторы выключены. Это ограничения отдельного запроса, не доказательство устойчивости при произвольной нагрузке. + +Порядок фиксации: валидация → S3 PUT → INSERT метаданных PostgreSQL → **201, status=ready, variants=[], error_code=null**. GET метаданных и оригинала снова проверяет членство. Для постороннего — 404; без сессии — 401. GET content читает оригинал из S3, проверяет размер и SHA-256 и возвращает байты через API без redirect. Object key и credentials не входят в публичную модель. + +Недоступность S3: upload/download и readiness возвращают 503 в формате Error; liveness остаётся 200. HAProxy исключает unready backend, поэтому при недоступности обеих реплик сам LB может вернуть собственный 503. Проверка формата ошибки API при таком отказе выполняется напрямую внутри Docker-сети. + +## Проверки и доказательства + +Для собственных image-тестов и полной OpenAPI-валидации нужны локальные зависимости: + +```sh +uv sync --frozen +uv pip install -r requirements-dev.txt +make check validate PYTHON=.venv/bin/python +make test +make test-attachments PYTHON=.venv/bin/python +python3 student-tests/idempotency_concurrency.py +python3 student-tests/review_regressions.py +make test-private +make test-upload-failover +make test-s3-outage +python3 scripts/verify-persistence.py +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose start api-a api-b +``` + +Сценарии отказов выполняются последовательно на учебном стенде: они временно останавливают контейнеры. Собственные проверки возвращают остановленные сервисы в работу; публичный `test-failover` оставляет реплику остановленной, поэтому последняя команда нужна для восстановления. Все данные синтетические, токены не сохраняются. Тесты добавляют данные; автоматического удаления нет. + +| Критерий | Команда / наблюдение | Результат | +| --- | --- | --- | +| Структура и OpenAPI | `make check validate PYTHON=.venv/bin/python` | [PASS](evidence/contracts.txt) | +| Обязательный API lab4, checksum, ACL | `make test` | [PASS](evidence/smoke.txt) | +| PNG/JPEG, сообщение с вложением, чужой чат, неверный MIME/filename, лимиты, chunked | `make test-attachments PYTHON=.venv/bin/python` | [PASS](evidence/attachments.txt) | +| Приватность bucket вне API | `make test-private`: anonymous GET существующего объекта → 403 | [PASS](evidence/private-bucket.txt) | +| Отказ именно загрузившей API | `make test-upload-failover`: загрузка через api-b, остановка api-b, те же байты через другую реплику | [PASS](evidence/attachment-failover.txt) | +| Недоступность S3 | `make test-s3-outage`: 503, восстановление, прежний оригинал | [PASS](evidence/s3-outage.txt) | +| Перезапуск всех компонентов, сохранение действующей сессии, метаданных и файла | `python3 scripts/verify-persistence.py`; StartedAt каждого сервиса изменился | [PASS](evidence/persistence.txt) | +| Публичный failover, сообщения без дублей и сохранение файла | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | [Результат](evidence/failover.txt) | +| Конкурентная идемпотентность и регрессии lab3 | Собственные проверки lab3 | [Результат](evidence/lab3-regressions.txt) | +| Повторный bootstrap bucket | `docker compose run --rm --no-deps s3-init` | [Результат](evidence/bootstrap.txt) | + +Точные границы проверены отдельно: PNG с trailing bytes доведён до 10 485 760 байт и скачан без изменений; PNG 8000 × 5000 = 40 000 000 пикселей принят; заголовок 8001 × 5000 = 40 005 000 пикселей отклонён до декодирования. Это функциональные тесты, не исследование RPS или памяти уровня 4/5. + +## Ограничения + +- Между S3 PUT и COMMIT PostgreSQL нет общей транзакции. Сбой после PUT может оставить orphan object. Автоматическая сверка/очистка с grace period относится к следующему уровню и здесь не реализована. При неопределённом исходе COMMIT объект намеренно не удаляется, чтобы не повредить возможную подтверждённую запись. +- API буферизует скачивание не более 10 MiB, чтобы проверить checksum до ответа. Ограничение общего количества скачиваний и нагрузочный эксперимент для 10 одновременных upload не выполнены; оценка 4 не заявляется. +- Readiness проверяет доступность БД и bucket, но не наличие каждого отдельного объекта. Потеря одного объекта при доступном S3 обнаруживается при скачивании как 503. +- Перезапуск с volumes и отказ API подтверждены на одном Docker-host. Потеря volumes/host не покрывается, резервное копирование не реализовано. +- HTTP предназначен для локального учебного стенда. Presigned flow, обработка thumbnail и брокер не реализованы. + +## Источники + +- [Контракт API](contracts/README.md), [OpenAPI](contracts/openapi.json), [критерии задания](README.md). +- [SeaweedFS исходники и запуск](https://github.com/seaweedfs/seaweedfs), [релиз 4.17](https://github.com/seaweedfs/seaweedfs/releases/tag/4.17). Параметры закреплённого образа дополнительно проверены через `docker run --rm chrislusf/seaweedfs:4.17 server -help`. +- [Pillow Image API](https://pillow.readthedocs.io/en/stable/reference/Image.html): lazy open, размеры, verify/load и decompression bomb limits. +- [Boto3 S3 PutObject](https://docs.aws.amazon.com/boto3/latest/reference/services/s3/client/put_object.html): запись оригинала в S3. diff --git a/compose.yaml b/compose.yaml index f447ec0e..4e5a0c42 100644 --- a/compose.yaml +++ b/compose.yaml @@ -9,10 +9,20 @@ x-api: &api SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" secrets: - database_url - cursor_secret + - s3_access_key_id + - s3_secret_access_key depends_on: + s3-init: + condition: service_completed_successfully migrate: condition: service_completed_successfully expose: @@ -28,9 +38,45 @@ x-api: &api resources: limits: cpus: "0.50" - memory: 256M + memory: 512M services: + s3: + image: chrislusf/seaweedfs:4.17@sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9 + command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.config=/run/secrets/s3_config", "-master.volumeSizeLimitMB=128", "-volume.max=8", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] + secrets: + - s3_config + volumes: + - s3_data:/data + - ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://localhost:9333/cluster/status"] + interval: 5s + timeout: 3s + retries: 20 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + + s3-init: + build: + context: . + command: ["/app/.venv/bin/python", "-m", "src.bootstrap_s3"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: + - s3_admin_key + - s3_admin_secret + depends_on: + s3: + condition: service_healthy + restart: "no" + postgres: image: postgres:17.6-alpine environment: @@ -82,7 +128,7 @@ services: load_balancer: image: haproxy:3.0.11-alpine ports: - - "8080:8080" + - "127.0.0.1:8080:8080" volumes: - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro depends_on: @@ -103,6 +149,16 @@ services: memory: 128M secrets: + s3_config: + file: ./secrets/s3.json + s3_admin_key: + file: ./secrets/s3_admin_key + s3_admin_secret: + file: ./secrets/s3_admin_secret + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key postgres_password: file: ./secrets/postgres_password database_url: @@ -111,4 +167,5 @@ secrets: file: ./secrets/cursor_secret volumes: + s3_data: postgres_data: diff --git a/evidence/attachment-failover.txt b/evidence/attachment-failover.txt new file mode 100644 index 00000000..d353ffc9 --- /dev/null +++ b/evidence/attachment-failover.txt @@ -0,0 +1 @@ +PASS: uploaded via api-b; stopped that container; original downloaded through another replica diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 00000000..0c7d7fe1 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,4 @@ +PASS: PNG/JPEG originals, metadata, attachment-only messages and ACL +PASS: filename, MIME, malformed image, byte/pixel and chunked limits +PASS: attachment scope, duplicates and idempotency normalization +PASS: inclusive 10 MiB and 40 million pixel boundaries diff --git a/evidence/bootstrap.txt b/evidence/bootstrap.txt new file mode 100644 index 00000000..99b2bb8a --- /dev/null +++ b/evidence/bootstrap.txt @@ -0,0 +1,3 @@ + Container mtusi2026lab4-s3-init-run-459a847a8091 Creating + Container mtusi2026lab4-s3-init-run-459a847a8091 Created +PASS: private S3 bucket ready diff --git a/evidence/contracts.txt b/evidence/contracts.txt new file mode 100644 index 00000000..ed9cd475 --- /dev/null +++ b/evidence/contracts.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..6394fdf7 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,107 @@ +{ + "recorded_at_utc": "2026-10-20T14:23:50.123531+00:00", + "base_commit": "3b52b91fb76d8acdeb7cbec010fd5542fd8990f7", + "branch": "solution/BVT2301-Kozyrev", + "note": "Results apply to this working tree, not only the base commit. Functional tests; no load benchmark.", + "host": "macOS-26.1-arm64-arm-64bit-Mach-O", + "docker": { + "server": "29.8.0", + "cpus": 10, + "memory_bytes": 10416644096 + }, + "compose_version": "5.5.1", + "sources_sha256": { + "Dockerfile": "efede941e2ccf21cd1e70ebbf8af7647209bd030b2962f9446bee497a2ed6a8c", + "Makefile": "bdb2dade89d84f516b20097ee8f33db7b85dd6bfa431c41881177a323cea845e", + "compose.yaml": "47d7c77d070097cef515279c3e46435bf0925d9b82dd96ef88fa38534e217328", + "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", + "infra/filer.toml": "ae3c01bf7708f5d4a319fdb454f753be0f137fe7c31269f1c413cbccc860f5c7", + "infra/haproxy.cfg": "23c9b869170633e3c6a8c41bbd3f1102bc97bf213e0d7cfed9d461af91216fd3", + "migrations/001_initial.sql": "062e58f5535d1ab6cd89d2347479a2a6b08a09e04665f7428ce41f89b39a19d1", + "migrations/002_history_cache_version.sql": "826c8243bc148caaa7339070826a7c2928073315a40dba7fa5a3d7421d980a4d", + "migrations/003_idempotency.sql": "d650ea7c1379928a013e035977a77026d2b8b09ad5b2ea97eecb545e7450353a", + "migrations/004_attachments.sql": "d760453ba130c94c403f5b04222649fc131ab4d091edd0267ede901fdf82a79e", + "pyproject.toml": "d3962edc7a683a62c915c8edec91e85c647c8ae6fe6432603e30b595bd6c3804", + "scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671", + "scripts/init-secrets.py": "fc89fc740da189b65be1b69980ff512e2c233dd09997c8ffb0198d9368118386", + "scripts/record-environment.py": "716fa7b8edd05d7ab6781fc346338b2254f82186953c6dbf525785c723eee4a1", + "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", + "scripts/restart.sh": "0a64716273eb5c79011d99e4734b18ed5a3a28f41b8b1d4ea6b24d0a4ffc702f", + "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", + "scripts/stop-one.sh": "c69fdb1891e0c49f41d3892f2d108ecd38b2f59d505c8fe991023bb5c9b08a5e", + "scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54", + "scripts/verify-persistence.py": "6c366bf4d85802bcffe65ae3d45d00736b184033313dabb60b80e1270d8874f1", + "src/__init__.py": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", + "src/attachments.py": "b377227a623bc41585f2eca487356bbd38c59a0ada56f425b8fca11abd93c085", + "src/bootstrap_s3.py": "bd0bafad4f5443bd37ab0d14c7b49453ae77fda059076f59cc02e9abc08bbc65", + "src/config.py": "6cb85b839e2947c3334c52f24d94ceb5290103e323ba33e6ab59f10f3ae3a53a", + "src/cursor.py": "a9192f2445042b8d4ddac57031718ef558990a52eb2922827074479b7f686775", + "src/errors.py": "07f8860b45cadd242d0d9f268c4b806e35353f33b11bbde0ab3d50acda86a80a", + "src/main.py": "abb5d7a856bff6d19ada7cf17addd59ef7400a26cf795ec01ea8b7d76f44527a", + "src/migrate.py": "1d164c8921df2b5369e30b1f8936c2b71858c5279070f0fd593f62f66e1f0b7d", + "src/models.py": "b94e6b26559afc5fd90f58d34d906c77bf4b4a78c716ac552358795496ad1d1b", + "src/routes.py": "3c21149a30ea7c2f695547c42133dcebcfbacbd6544bcb73c353be46fa35a2e6", + "src/storage/__init__.py": "05ef66318c4c436eaafc493072640817219ab37bce464a265a2306233389064a", + "src/storage/postgres.py": "2ff296490b06729c8e56eb97dbdc8d1ca91c94e313375ea5a88ffec99b6f838d", + "src/storage/s3.py": "433247b86837a9a1212900da9d3241105e191b36d8fcac8a92c50100f5f4b946", + "student-tests/attachment_resilience.py": "c72c431b355dda919c1b6b854d260f8c7420672ca9f86bf0848692da35dcb0ac", + "student-tests/attachments.py": "a948fabfe83b73eb8e494c80fb9d1f2a949e2a95b7a21809a05975d65d68fc15", + "student-tests/idempotency_concurrency.py": "db5d139009e4660ac1801af45d10e24665aa81cba737731e527c0f5b4152231e", + "student-tests/readiness_failover.py": "0ef8355d85be66597d7d44eb7308dcb58c6c9a174d18f360b2d6d22fdcfbed14", + "student-tests/readiness_unavailable.py": "46e3f680a3ff9cb79e95568004d97f1752dbdcd88fd4b5f160a9cc38af65a237", + "student-tests/review_regressions.py": "de58f63b2c794c2775247ddd3fccc523870ccbe673ffa404fdd566274d1d41de", + "student-tests/session_expiry.py": "ba8ead4356e4d12c4047ac2b5704d3d3342f71c37b3c13f65a0a9d7c37186ad6", + "uv.lock": "fdbeb5ba0c5eedced1a6e5b62c6a57b2d22f293cf77b9487a806c6e1a8b992a0" + }, + "services": { + "postgres": { + "image": "postgres:17.6-alpine", + "image_id": "sha256:ef257d85f76e48da1c64832459b59fcaba1a4dac97bf5d7450c77753542eee94", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus_limit": 750000000, + "volumes": [ + "mtusi2026lab4_postgres_data" + ] + }, + "s3": { + "image": "chrislusf/seaweedfs:4.17@sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9", + "image_id": "sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus_limit": 1000000000, + "volumes": [ + "mtusi2026lab4_s3_data" + ] + }, + "api-a": { + "image": "mtusi2026lab4-api-a", + "image_id": "sha256:3a32a3f312aff9b46c6de73815627556af6f7cbee4b22783c1f4185d407f01b3", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus_limit": 500000000, + "volumes": [] + }, + "api-b": { + "image": "mtusi2026lab4-api-b", + "image_id": "sha256:0e3c794618677c4a6305586bf44165fb1b3dbe668ce77b583df97e9e72721aa1", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 536870912, + "nano_cpus_limit": 500000000, + "volumes": [] + }, + "load_balancer": { + "image": "haproxy:3.0.11-alpine", + "image_id": "sha256:e49b7beb96f4dba79f9a671515b469a6e516a2b88ff0681a820b184bba1effbd", + "state": "running", + "health": "healthy", + "memory_limit_bytes": 134217728, + "nano_cpus_limit": 250000000, + "volumes": [] + } + } +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..ee4b8fed --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 111, "failed_attempts": 1, "first_recovery_seconds": 2.791} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/lab3-regressions.txt b/evidence/lab3-regressions.txt new file mode 100644 index 00000000..9abf5c19 --- /dev/null +++ b/evidence/lab3-regressions.txt @@ -0,0 +1,2 @@ +PASS: concurrent idempotency through load balancer +PASS: non-ASCII Bearer returns 401/challenge; malformed JSON returns contract Error/400 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..0d58f0c1 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,5 @@ +Before: {"api-a": {"container": "eca67c29fa27", "started_at": "2026-10-20T14:17:13.97672451Z"}, "api-b": {"container": "71c5a224f05f", "started_at": "2026-10-20T14:19:07.971967673Z"}, "load_balancer": {"container": "3e69e9c37346", "started_at": "2026-10-20T14:17:19.597909846Z"}, "postgres": {"container": "b756a119c924", "started_at": "2026-10-20T14:17:06.843762048Z"}, "s3": {"container": "59fd3a3bca31", "started_at": "2026-10-20T14:19:38.783873424Z"}} +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +After: {"api-a": {"container": "eca67c29fa27", "started_at": "2026-10-20T14:20:18.95842697Z"}, "api-b": {"container": "71c5a224f05f", "started_at": "2026-10-20T14:20:18.82218322Z"}, "load_balancer": {"container": "3e69e9c37346", "started_at": "2026-10-20T14:20:18.598004429Z"}, "postgres": {"container": "b756a119c924", "started_at": "2026-10-20T14:20:10.227233258Z"}, "s3": {"container": "59fd3a3bca31", "started_at": "2026-10-20T14:20:13.232762718Z"}} +PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 00000000..c64957f4 --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1 @@ +PASS: anonymous S3 GET returns 403 for an existing object diff --git a/evidence/s3-outage.txt b/evidence/s3-outage.txt new file mode 100644 index 00000000..89f5204a --- /dev/null +++ b/evidence/s3-outage.txt @@ -0,0 +1,2 @@ +PASS: S3 outage gives 503 for readiness, download and upload; liveness stays 200 +PASS: original available again after S3 restart diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/filer.toml b/infra/filer.toml new file mode 100644 index 00000000..85b2149c --- /dev/null +++ b/infra/filer.toml @@ -0,0 +1,3 @@ +[leveldb2] +enabled = true +dir = "/data/filer" diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..2fe0f9e2 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,18 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + owner_id uuid NOT NULL REFERENCES users(id), + filename varchar(255) NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (id, chat_id) +); +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}'; +ALTER TABLE messages ADD CONSTRAINT messages_content_check + CHECK (char_length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0); +CREATE INDEX attachments_chat_idx ON attachments(chat_id); diff --git a/pyproject.toml b/pyproject.toml index 7de48cee..4c35468f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,6 +7,8 @@ requires-python = ">=3.12" dependencies = [ "argon2-cffi==25.1.0", "asyncpg==0.31.0", + "boto3==1.43.115", "fastapi==0.141.1", + "pillow==13.0.0", "uvicorn==0.40.0", ] diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py new file mode 100644 index 00000000..1e1f6954 --- /dev/null +++ b/scripts/init-secrets.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +"""Create local credentials without printing or overwriting existing values.""" +import json +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] / "secrets" +root.mkdir(mode=0o700, exist_ok=True) + + +def write(name, value): + path = root / name + if path.exists(): + return path.read_text().strip() + with path.open("x") as stream: + path.chmod(0o600) + stream.write(value + "\n") + return value + + +password = write("postgres_password", secrets.token_hex(32)) +write("database_url", f"postgresql://messenger:{password}@postgres:5432/messenger") +write("cursor_secret", secrets.token_hex(32)) +admin_key = write("s3_admin_key", secrets.token_hex(16)) +admin_secret = write("s3_admin_secret", secrets.token_hex(32)) +app_key = write("s3_access_key_id", secrets.token_hex(16)) +app_secret = write("s3_secret_access_key", secrets.token_hex(32)) +write("s3.json", json.dumps({"identities": [ + {"name": "bootstrap", "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "api", "credentials": [{"accessKey": app_key, "secretKey": app_secret}], + "actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]}, +]}, indent=2)) +print("Local secrets ready; existing values preserved.") diff --git a/scripts/record-environment.py b/scripts/record-environment.py new file mode 100644 index 00000000..e983ce94 --- /dev/null +++ b/scripts/record-environment.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Sanitized environment and source fingerprint for local test evidence.""" +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path +import platform +import subprocess + +root = Path(__file__).resolve().parents[1] + + +def command(*args): + return subprocess.check_output(args, cwd=root, text=True).strip() + + +paths = [root / name for name in ('Dockerfile', 'compose.yaml', 'pyproject.toml', 'uv.lock', 'Makefile')] +for directory in ('src', 'migrations', 'infra', 'scripts', 'student-tests'): + paths.extend(path for path in (root / directory).rglob('*') + if path.is_file() and '__pycache__' not in path.parts) +report = { + 'recorded_at_utc': datetime.now(timezone.utc).isoformat(), + 'base_commit': command('git', 'rev-parse', 'HEAD'), + 'branch': command('git', 'branch', '--show-current'), + 'note': 'Results apply to this working tree, not only the base commit. Functional tests; no load benchmark.', + 'host': platform.platform(), + 'docker': json.loads(command('docker', 'info', '--format', + '{"server":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}')), + 'compose_version': command('docker', 'compose', 'version', '--short'), + 'sources_sha256': {str(path.relative_to(root)): sha256(path.read_bytes()).hexdigest() for path in sorted(paths)}, + 'services': {}, +} +for service in ('postgres', 's3', 'api-a', 'api-b', 'load_balancer'): + container = command('docker', 'compose', 'ps', '-q', service) + info = json.loads(command('docker', 'inspect', container))[0] + report['services'][service] = { + 'image': info['Config']['Image'], 'image_id': info['Image'], + 'state': info['State']['Status'], 'health': info['State'].get('Health', {}).get('Status'), + 'memory_limit_bytes': info['HostConfig']['Memory'], + 'nano_cpus_limit': info['HostConfig']['NanoCpus'], + 'volumes': [mount['Name'] for mount in info['Mounts'] if mount['Type'] == 'volume'], + } +(root / 'evidence/environment.json').write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n') +print('PASS: sanitized environment and source hashes saved') diff --git a/scripts/restart.sh b/scripts/restart.sh index e0d9abaa..aa02c3b6 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,7 +1,7 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -docker compose restart postgres -docker compose up -d --wait postgres +docker compose restart postgres s3 +docker compose up -d --wait postgres s3 docker compose restart api-a api-b load_balancer docker compose up -d --wait api-a api-b load_balancer diff --git a/scripts/verify-persistence.py b/scripts/verify-persistence.py new file mode 100644 index 00000000..5bfc5a44 --- /dev/null +++ b/scripts/verify-persistence.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Public persistence test plus proof of actual container restarts.""" +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] + + +def snapshot(): + result = {} + for service in ('postgres', 's3', 'api-a', 'api-b', 'load_balancer'): + container = subprocess.check_output(['docker', 'compose', 'ps', '-q', service], cwd=ROOT, text=True).strip() + started = subprocess.check_output(['docker', 'inspect', '--format', '{{.State.StartedAt}}', container], text=True).strip() + result[service] = {'container': container[:12], 'started_at': started} + return result + + +before = snapshot() +print('Before:', json.dumps(before, sort_keys=True), flush=True) +subprocess.run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', + '--action-script', 'scripts/restart.sh'], cwd=ROOT, check=True) +after = snapshot() +print('After:', json.dumps(after, sort_keys=True), flush=True) +assert all(before[key]['started_at'] != after[key]['started_at'] for key in before) +print('PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved') diff --git a/src/attachments.py b/src/attachments.py new file mode 100644 index 00000000..a078a7c2 --- /dev/null +++ b/src/attachments.py @@ -0,0 +1,104 @@ +import asyncio +from hashlib import sha256 +from tempfile import TemporaryFile +import unicodedata +from uuid import UUID, uuid4 +import warnings + +from fastapi import APIRouter, Depends, Query, Request, Response +from PIL import Image, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool + +from .errors import ApiError +from .models import Attachment +from .routes import get_current_user, get_store +from .storage.postgres import PostgresStore + +router = APIRouter() +Image.MAX_IMAGE_PIXELS = 40000000 +warnings.filterwarnings("error", category=Image.DecompressionBombWarning) + + +def inspect_image(file, content_type, max_pixels): + try: + file.seek(0) + with Image.open(file, formats=("PNG", "JPEG")) as image: + if image.width * image.height > max_pixels: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") + actual = {"PNG": "image/png", "JPEG": "image/jpeg"}[image.format] + if actual != content_type or getattr(image, "n_frames", 1) != 1: + raise ApiError(415, "unsupported_media_type", "Нужен обычный PNG или JPEG") + image.verify() + # verify checks PNG CRC; load also catches truncated pixel streams/JPEGs. + file.seek(0) + with Image.open(file, formats=("PNG", "JPEG")) as image: + image.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as error: + raise ApiError(413, "payload_too_large", "Слишком много пикселей") from error + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error: + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") from error + file.seek(0) + + +@router.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=201) +async def upload(chat_id: UUID, request: Request, filename: str = Query(...), + user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)): + await store.get_chat_for_member(chat_id, user["id"]) + if (not 1 <= len(filename) <= 255 or not filename.strip() or + any(c in "/\\" or unicodedata.category(c) in {"Cc", "Cs"} for c in filename)): + raise ApiError(400, "invalid_request", "Некорректное имя файла") + content_type = request.headers.get("content-type", "").lower() + if content_type not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + s3 = request.app.state.s3 + limit = s3.settings.max_bytes + if "content-length" in request.headers: + try: + length = int(request.headers["content-length"]) + if length < 0: + raise ValueError() + except ValueError: + raise ApiError(400, "invalid_request", "Некорректная длина") from None + if length > limit: + raise ApiError(413, "payload_too_large", "Файл больше лимита") + try: + async with asyncio.timeout(30): + async with request.app.state.upload_slot: + # Temporary scratch space only; deleted on every exit, never used for downloads. + with TemporaryFile() as file: + size = 0 + digest = sha256() + async for chunk in request.stream(): + size += len(chunk) + if size > limit: + raise ApiError(413, "payload_too_large", "Файл больше лимита") + digest.update(chunk) + await run_in_threadpool(file.write, chunk) + await run_in_threadpool(inspect_image, file, content_type, s3.settings.max_pixels) + attachment_id = uuid4() + key = f"originals/{attachment_id}" + await s3.put(key, file, size, content_type) + return await store.create_attachment( + attachment_id, chat_id, user["id"], filename, content_type, + size, digest.hexdigest(), key, + ) + except TimeoutError: + raise ApiError(503, "unavailable", "Время загрузки истекло") from None + + +@router.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +async def metadata(attachment_id: UUID, user: dict = Depends(get_current_user), + store: PostgresStore = Depends(get_store)): + return await store.attachment_for_member(attachment_id, user["id"]) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +async def download(attachment_id: UUID, request: Request, + variant: str = Query("original", pattern="^(original|thumbnail|crop|optimized)$"), + user: dict = Depends(get_current_user), store: PostgresStore = Depends(get_store)): + attachment = await store.attachment_for_member(attachment_id, user["id"]) + if variant != "original": + raise ApiError(404, "not_found", "Вариант не найден") + data = await request.app.state.s3.download(attachment) + return Response(data, media_type=attachment["content_type"], + headers={"X-Content-Type-Options": "nosniff", "Cache-Control": "private, no-store"}) diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py new file mode 100644 index 00000000..5f90eebe --- /dev/null +++ b/src/bootstrap_s3.py @@ -0,0 +1,38 @@ +"""Idempotent private bucket bootstrap; executed with separate admin credentials.""" +import time +from botocore.exceptions import BotoCoreError, ClientError +from .storage.s3 import S3Settings, S3Store + + +def main(): + store = S3Store(S3Settings.load()) + try: + for attempt in range(60): + try: + store.client.list_buckets() + break + except (BotoCoreError, ClientError): + if attempt == 59: + raise RuntimeError("S3 startup timed out") from None + time.sleep(1) + bucket = store.settings.bucket + try: + store.client.head_bucket(Bucket=bucket) + except ClientError as error: + if str(error.response["Error"]["Code"]) not in {"404", "NoSuchBucket", "NotFound"}: + raise + store.client.create_bucket(Bucket=bucket) + # No anonymous identity; revoke any previously configured public grants/policy. + store.client.put_bucket_acl(Bucket=bucket, ACL="private") + try: + store.client.delete_bucket_policy(Bucket=bucket) + except ClientError as error: + if error.response["Error"]["Code"] != "NoSuchBucketPolicy": + raise + print("PASS: private S3 bucket ready") + finally: + store.client.close() + + +if __name__ == "__main__": + main() diff --git a/src/config.py b/src/config.py index f68a8b0c..091c861b 100644 --- a/src/config.py +++ b/src/config.py @@ -15,6 +15,10 @@ _CONFIGURATION_KEYS = { "INSTANCE_ID", "CURSOR_SECRET", "CURSOR_SECRET_FILE", + "S3_ENDPOINT", "S3_REGION", "S3_BUCKET", + "S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE", + "S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE", + "MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS", } diff --git a/src/main.py b/src/main.py index cd876651..30df8478 100644 --- a/src/main.py +++ b/src/main.py @@ -1,3 +1,4 @@ +import asyncio import logging import sys import time @@ -15,6 +16,8 @@ from .cursor import configure_secret from .models import ErrorDetails, ErrorResponse from .config import load_settings from .routes import router +from .attachments import router as attachment_router +from .storage.s3 import S3Settings, S3Store from .storage.postgres import PostgresStore @@ -41,17 +44,20 @@ async def lifespan(application: FastAPI): await store.connect() application.state.store = store application.state.settings = settings + application.state.s3 = S3Store(S3Settings.load()) + application.state.upload_slot = asyncio.Semaphore(1) application.state.accepting_requests = True try: yield finally: application.state.accepting_requests = False + application.state.s3.client.close() await store.close() app = FastAPI( title="MTUSI Messenger", - version="1.3.0", + version="1.4.0", redoc_url=None, lifespan=lifespan, ) @@ -144,6 +150,7 @@ async def handle_unexpected_error(request: Request, error: Exception) -> JSONRes app.include_router(router) +app.include_router(attachment_router) def main() -> None: diff --git a/src/models.py b/src/models.py index 095ec1a2..8dea91cf 100644 --- a/src/models.py +++ b/src/models.py @@ -1,7 +1,7 @@ from datetime import datetime from uuid import UUID -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class CreateUserRequest(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) @@ -46,16 +46,22 @@ class ChatPageResponse(BaseModel): class CreateMessageRequest(BaseModel): model_config = ConfigDict(extra="forbid", strict=True) - text: str = Field(min_length=1, max_length=4000) + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) - @field_validator("text") + @field_validator("attachment_ids") @classmethod - def text_must_not_be_whitespace(cls, value: str) -> str: - """Не допускает сообщение, состоящее только из пробельных символов.""" + def valid_ids(cls, values: list[str]) -> list[str]: + normalized = [str(UUID(value)) for value in values] + if len(normalized) != len(set(normalized)): + raise ValueError("Повторяющиеся attachment_ids") + return normalized - if not value.strip(): - raise ValueError("text не может состоять только из пробелов") - return value + @model_validator(mode="after") + def has_content(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Сообщение без содержимого") + return self class MessagePageResponse(BaseModel): @@ -104,6 +110,7 @@ class Message(BaseModel): chat_id: UUID sender_id: UUID text: str + attachment_ids: list[UUID] = Field(default_factory=list) created_at: datetime @@ -114,3 +121,20 @@ class Session(BaseModel): token_type: str = "Bearer" expires_at: datetime user: User + + +class Attachment(BaseModel): + model_config = ConfigDict(extra="ignore") + + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str = "ready" + variants: list = Field(default_factory=list) + error_code: str | None = None + created_at: datetime + updated_at: datetime diff --git a/src/routes.py b/src/routes.py index c07e4056..5bc2a3df 100644 --- a/src/routes.py +++ b/src/routes.py @@ -83,6 +83,7 @@ async def readiness(request: Request, store: PostgresStore = Depends(get_store)) if not request.app.state.accepting_requests: raise ApiError(503, "unavailable", "API завершает работу") await store.ready() + await request.app.state.s3.ready() return {"status": "ok"} @router.post("/api/v1/users", response_model=User, status_code=201) @@ -164,6 +165,7 @@ async def create_message( payload.text, idempotency_key, request.app.state.settings.idempotency_ttl_seconds, + [UUID(value) for value in payload.attachment_ids], ) @router.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePageResponse) diff --git a/src/storage/postgres.py b/src/storage/postgres.py index 67b8f815..42695ec4 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -83,6 +83,7 @@ class PostgresStore: "chat_id": row["chat_id"], "sender_id": row["sender_id"], "text": row["text"], + "attachment_ids": row["attachment_ids"], "created_at": row["created_at"], } @@ -216,12 +217,17 @@ class PostgresStore: return [self._chat(row) for row in rows] async def create_message_idempotent( - self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int + self, chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, ttl_seconds: int, attachment_ids: list[UUID] | None = None ) -> dict: """Создаёт сообщение и фиксирует результат retry в одной транзакции.""" path = f"/api/v1/chats/{chat_id}/messages" + attachment_ids = attachment_ids or [] + # Preserve hashes of text-only requests already accepted by lab3. + canonical = {"text": text} + if attachment_ids: + canonical["attachment_ids"] = [str(value) for value in attachment_ids] request_hash = sha256( - json.dumps({"text": text}, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + json.dumps(canonical, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") ).hexdigest() try: async with self.pool.acquire() as connection, connection.transaction(): @@ -233,6 +239,13 @@ class PostgresStore: await connection.execute( "DELETE FROM idempotency_records WHERE created_at <= now() - $1 * interval '1 second'", ttl_seconds ) + if attachment_ids: + count = await connection.fetchval( + "SELECT count(*) FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[])", + chat_id, attachment_ids, + ) + if count != len(attachment_ids): + raise ApiError(404, "not_found", "Вложение не найдено") claimed = await connection.fetchrow( "INSERT INTO idempotency_records (user_id, method, path, idempotency_key, request_hash) " "VALUES ($1, 'POST', $2, $3, $4) ON CONFLICT DO NOTHING RETURNING message_id", @@ -251,15 +264,15 @@ class PostgresStore: if recorded["message_id"] is None: raise ApiError(503, "unavailable", "Не удалось получить результат идемпотентного запроса") row = await connection.fetchrow( - "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = $1", recorded["message_id"] + "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE id = $1", recorded["message_id"] ) if row is None: raise ApiError(503, "unavailable", "Не удалось получить сохранённое сообщение") return self._message(row) row = await connection.fetchrow( - "INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4) " - "RETURNING id, chat_id, sender_id, text, created_at", - uuid4(), chat_id, sender_id, text, + "INSERT INTO messages (id, chat_id, sender_id, text, attachment_ids) VALUES ($1, $2, $3, $4, $5) " + "RETURNING id, chat_id, sender_id, text, attachment_ids, created_at", + uuid4(), chat_id, sender_id, text, attachment_ids, ) await connection.execute( "UPDATE idempotency_records SET message_id = $1 WHERE user_id = $2 AND method = 'POST' " @@ -274,7 +287,7 @@ class PostgresStore: self, chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int ) -> list[dict]: await self.get_chat_for_member(chat_id, user_id) - query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1" + query = "SELECT id, chat_id, sender_id, text, attachment_ids, created_at FROM messages WHERE chat_id = $1" args: list = [chat_id] if after is not None: query += " AND (created_at, id) > ($2, $3)" @@ -286,3 +299,28 @@ class PostgresStore: except DB_ERRORS as error: raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error return [self._message(row) for row in rows] + + + async def create_attachment(self, attachment_id, chat_id, owner_id, filename, + content_type, size_bytes, digest, object_key) -> dict: + try: + row = await self.pool.fetchrow( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) " + "VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *", + attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, object_key, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + return dict(row) + + async def attachment_for_member(self, attachment_id: UUID, user_id: UUID) -> dict: + try: + row = await self.pool.fetchrow( + "SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id " + "WHERE a.id = $1 AND m.user_id = $2", attachment_id, user_id, + ) + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + if row is None: + raise ApiError(404, "not_found", "Вложение не найдено") + return dict(row) diff --git a/src/storage/s3.py b/src/storage/s3.py new file mode 100644 index 00000000..1fea6569 --- /dev/null +++ b/src/storage/s3.py @@ -0,0 +1,85 @@ +"""Bounded S3 operations; synchronous SDK calls run outside the event loop.""" +from dataclasses import dataclass +from hashlib import sha256 +import os +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError +from starlette.concurrency import run_in_threadpool + +from ..errors import ApiError + + +@dataclass(frozen=True) +class S3Settings: + endpoint: str + region: str + bucket: str + access_key: str + secret_key: str + max_bytes: int + max_pixels: int + + @classmethod + def load(cls): + def secret(name): + path = os.environ.get(name + "_FILE") + result = Path(path).read_text().strip() if path else os.environ.get(name, "") + if not result: + raise ValueError(f"Missing {name}") + return result + + max_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) + max_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) + if not 1 <= max_bytes <= 10485760 or not 1 <= max_pixels <= 40000000: + raise ValueError("Image limits exceed contract") + return cls(os.environ.get("S3_ENDPOINT", "http://s3:8333"), + os.environ.get("S3_REGION", "us-east-1"), + os.environ.get("S3_BUCKET", "messenger-attachments"), + secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"), + max_bytes, max_pixels) + + +class S3Store: + def __init__(self, settings): + self.settings = settings + self.client = boto3.client( + "s3", endpoint_url=settings.endpoint, region_name=settings.region, + aws_access_key_id=settings.access_key, aws_secret_access_key=settings.secret_key, + config=Config(signature_version="s3v4", connect_timeout=2, read_timeout=5, + retries={"total_max_attempts": 1}, + s3={"addressing_style": "path"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required"), + ) + + async def call(self, method, **kwargs): + try: + return await run_in_threadpool(getattr(self.client, method), **kwargs) + except (BotoCoreError, ClientError, OSError) as error: + raise ApiError(503, "unavailable", "S3 недоступно") from error + + async def ready(self): + await self.call("head_bucket", Bucket=self.settings.bucket) + + async def put(self, key, file, size, content_type): + file.seek(0) + await self.call("put_object", Bucket=self.settings.bucket, Key=key, + Body=file, ContentLength=size, ContentType=content_type) + + async def download(self, attachment): + response = await self.call("get_object", Bucket=self.settings.bucket, + Key=attachment["object_key"]) + body = response["Body"] + try: + data = await run_in_threadpool(body.read, self.settings.max_bytes + 1) + except (BotoCoreError, OSError) as error: + raise ApiError(503, "unavailable", "S3 недоступно") from error + finally: + body.close() + if (len(data) != attachment["size_bytes"] or + sha256(data).hexdigest() != attachment["sha256"]): + raise ApiError(503, "unavailable", "Оригинал недоступен") + return data diff --git a/student-tests/attachment_resilience.py b/student-tests/attachment_resilience.py new file mode 100644 index 00000000..83ebe368 --- /dev/null +++ b/student-tests/attachment_resilience.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Explicit local Docker tests: private bucket, uploader failover, S3 outage.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def compose(*args, input=None): + return subprocess.check_output(['docker', 'compose', *args], cwd=ROOT, input=input, + text=True, stderr=subprocess.DEVNULL) + + +def wait_ready(c): + deadline = time.monotonic() + 90 + while True: + try: + c.call('GET', '/health/ready', schema='Health') + return + except Failure: + check(time.monotonic() < deadline, 'stack did not recover') + time.sleep(1) + + +def main(): + def configure(parser): + parser.add_argument('--mode', required=True, choices=['private', 'failover', 's3-outage']) + args = arguments(__doc__, configure) + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + data = (ROOT / 'tests/fixtures/sample.png').read_bytes() + uploaded = c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + att = uploaded['json'] + content_path = '/api/v1/attachments/' + att['id'] + '/content' + if args.mode == 'private': + script = '''import sys, urllib.request, urllib.error +url = "http://s3:8333/messenger-attachments/originals/" + sys.stdin.read().strip() +try: + urllib.request.urlopen(url, timeout=5) +except urllib.error.HTTPError as error: + assert error.code == 403, error.code + print("PASS: anonymous S3 GET returns 403 for an existing object") +else: + raise AssertionError("private object leaked") +''' + print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, input=att['id']).strip()) + elif args.mode == 'failover': + target = uploaded['headers']['X-Instance-Id'] + service = None + for candidate in ('api-a', 'api-b'): + container = compose('ps', '-q', candidate).strip() + hostname = subprocess.check_output(['docker', 'inspect', '--format', '{{.Config.Hostname}}', container], text=True).strip() + if hostname == target: + service = candidate + check(service is not None, 'uploading replica not found') + try: + compose('stop', service) + deadline = time.monotonic() + 15 + while True: + try: + download = c.call('GET', content_path, actor=bob) + break + except Failure: + check(time.monotonic() < deadline, 'download did not recover') + time.sleep(0.5) + check(download['headers']['X-Instance-Id'] != target, 'download used stopped uploader') + check(download['bytes'] == data, 'original changed after stopping uploader') + print('PASS: uploaded via ' + service + '; stopped that container; original downloaded through another replica') + finally: + compose('start', service) + wait_ready(c) + else: + try: + compose('stop', 's3') + # Direct API request inside the network: LB excludes unready backends. + script = '''import json, sys, urllib.request, urllib.error +payload = json.load(sys.stdin) +for method, path, body, headers in [ + ("GET", "/health/ready", None, {}), + ("GET", payload["path"], None, {"Authorization": "Bearer " + payload["token"]}), + ("POST", payload["upload_path"], bytes.fromhex(payload["bytes"]), + {"Authorization": "Bearer " + payload["token"], "Content-Type": "image/png"}), +]: + req = urllib.request.Request("http://localhost:8080" + path, data=body, headers=headers, method=method) + try: + urllib.request.urlopen(req, timeout=15) + except urllib.error.HTTPError as error: + assert error.code == 503, error.code + document = json.load(error) + assert document["error"]["code"] == "unavailable" + assert document["error"]["request_id"] == error.headers["X-Request-Id"] + else: + raise AssertionError("S3 outage was not reported") +assert urllib.request.urlopen("http://localhost:8080/health/live").status == 200 +print("PASS: S3 outage gives 503 for readiness, download and upload; liveness stays 200") +''' + payload = {'path': content_path, 'token': bob['token'], 'bytes': data.hex(), + 'upload_path': '/api/v1/chats/' + chat['id'] + '/attachments?filename=outage.png'} + print(compose('exec', '-T', 'api-a', '/app/.venv/bin/python', '-c', script, + input=json.dumps(payload)).strip()) + finally: + compose('start', 's3') + wait_ready(c) + check(c.call('GET', content_path, actor=bob)['bytes'] == data, 'original lost after S3 outage') + print('PASS: original available again after S3 restart') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 00000000..81652326 --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Attachment contract checks against the running lab4 stack.""" +import hashlib +import http.client +from io import BytesIO +from pathlib import Path +import struct +import sys +import uuid +import zlib +from urllib.parse import quote, urlsplit + +from PIL import Image + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) +from client import Client, arguments, check, run + + +def main(): + args = arguments(__doc__) + c = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, eve, chat = c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + png = (Path(__file__).resolve().parents[1] / 'tests/fixtures/sample.png').read_bytes() + + def upload(data, mime='image/png', filename='photo.png', status=201, actor=alice): + return c.call('POST', path + quote(filename, safe=''), status, actor=actor, raw=data, + headers={'Content-Type': mime}, schema='Attachment' if status == 201 else None) + + for fmt, mime in [('PNG', 'image/png'), ('JPEG', 'image/jpeg')]: + buf = BytesIO() + Image.new('RGB', (32, 20), 'blue').save(buf, fmt) + data = buf.getvalue() + att = upload(data, mime)['json'] + check(att['variants'] == [] and att['status'] == 'ready', 'wrong attachment state') + check(att['sha256'] == hashlib.sha256(data).hexdigest(), 'wrong checksum') + content_path = '/api/v1/attachments/' + att['id'] + check(c.call('GET', content_path, actor=bob, schema='Attachment')['json'] == att, + 'metadata changed') + check(c.call('GET', content_path + '/content', actor=bob)['bytes'] == data, 'bytes changed') + c.call('GET', content_path, 404, actor=eve) + c.call('GET', content_path + '/content', 404, actor=eve) + c.call('GET', content_path + '/content', 401) + c.call('GET', content_path + '/content?variant=thumbnail', 404, actor=bob) + message = c.message(bob, chat, '', attachments=[att['id']])['json'] + history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', + actor=alice, schema='MessagePage')['json']['items'] + check(message in history, 'attachment message missing in history') + print('PASS: PNG/JPEG originals, metadata, attachment-only messages and ACL') + + for name in ('../x.png', 'x/y.png', 'x\\y.png', 'x\x00.png', 'x\n.png', 'x\x7f.png', ' ', 'x' * 256): + upload(png, filename=name, status=400) + upload(png, mime='image/jpeg', status=415) + upload(b'not an image', status=415) + upload(png[:30], status=415) + upload(png, mime='application/octet-stream', status=415) + upload(png, status=404, actor=eve) + upload(b'x' * (10485760 + 1), status=413) + + # Valid PNG header with an oversized canvas: reject before attempting pixel decoding. + def chunk(kind, data): + return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data)) + huge = b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', struct.pack('>IIBBBBB', 8001, 5000, 8, 2, 0, 0, 0)) + huge += chunk(b'IDAT', zlib.compress(b'')) + chunk(b'IEND', b'') + upload(huge, status=413) + + # No Content-Length: limits must also work for chunked requests. + url = urlsplit(c.base) + connection = http.client.HTTPConnection(url.hostname, url.port, timeout=30) + connection.request('POST', path + 'chunked.png', body=iter([b'x' * 65536] * 161), + headers={'Authorization': 'Bearer ' + alice['token'], 'Content-Type': 'image/png'}, + encode_chunked=True) + response = connection.getresponse() + check(response.status == 413, 'chunked upload bypassed byte limit') + response.read() + connection.close() + print('PASS: filename, MIME, malformed image, byte/pixel and chunked limits') + + att = upload(png)['json'] + other_chat = c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'other', 'member_ids': [alice['id']]}, schema='Chat')['json'] + for target, ids, code in ((chat, [str(uuid.uuid4())], 404), + (other_chat, [att['id']], 404), + (chat, [att['id'], att['id']], 400)): + c.call('POST', '/api/v1/chats/' + target['id'] + '/messages', code, actor=alice, + body={'text': '', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex}) + key = uuid.uuid4().hex + first = c.message(alice, chat, 'same', key=key)['json'] + check(c.message(alice, chat, 'same', key=key, attachments=[])['json'] == first, + 'omitted and empty attachment_ids differ') + key = uuid.uuid4().hex + first = c.message(alice, chat, '', key=key, attachments=[att['id']])['json'] + check(c.message(alice, chat, '', key=key, attachments=[att['id']])['json'] == first, 'replay differs') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + body={'text': 'changed', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': key}) + print('PASS: attachment scope, duplicates and idempotency normalization') + + # Exact limits are accepted; original bytes (including PNG trailing data) survive. + exact = png + b'\0' * (10485760 - len(png)) + att = upload(exact)['json'] + check(c.call('GET', '/api/v1/attachments/' + att['id'] + '/content', actor=bob)['bytes'] == exact, + '10 MiB original changed') + buf = BytesIO() + Image.new('RGB', (8000, 5000)).save(buf, 'PNG') + att = upload(buf.getvalue())['json'] + check(att['size_bytes'] == len(buf.getvalue()), '40M pixel upload failed') + print('PASS: inclusive 10 MiB and 40 million pixel boundaries') + + +if __name__ == '__main__': + run(main) diff --git a/uv.lock b/uv.lock index 3ee491eb..ace5584b 100644 --- a/uv.lock +++ b/uv.lock @@ -132,6 +132,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/3c/d7/8fb3044eaef08a310acfe23dae9a8e2e07d305edc29a53497e52bc76eca7/asyncpg-0.31.0-cp314-cp314t-win_amd64.whl", hash = "sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3", size = 706062, upload-time = "2025-11-24T23:26:44.086Z" }, ] +[[package]] +name = "boto3" +version = "1.43.115" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/4a/b0ce191492a1724be73d8aacaf70d508bc8cbb9674c046d2ff7181dc51fb/boto3-1.43.115.tar.gz", hash = "sha256:c0f35e5f9cdf0edb365195e306d5586d3970b224660a11a87a4ec361ec3d7168", size = 112677, upload-time = "2026-10-19T19:19:53.093Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/be/bc/40dffae3238ef8d3415393132751fe2c88e8663ef84a31f0950e439ed2d5/boto3-1.43.115-py3-none-any.whl", hash = "sha256:6667a0c21d48dc125dbba88444bc8a601ce881a99341a0e6169f9333898e620d", size = 140044, upload-time = "2026-10-19T19:19:51.444Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.115" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6f/10/ca525e88db50f5153ddad2a3d55908b5ace915d10f443d5e5135ba808d8d/botocore-1.43.115.tar.gz", hash = "sha256:aa4d9c4e7ef99988b7347e0cd276ba84e1eb1625510557b0e36cd57ecb19bbf2", size = 16383796, upload-time = "2026-10-19T19:19:45.382Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/2d/4b6b6f5704d1ec9b412c1dd9700bda0c315619dbac4a21418180a81a2780/botocore-1.43.115-py3-none-any.whl", hash = "sha256:9ceda640c868bc15f5399129e8cfdef8cbb4b350e8a99547c9b5f477cfb9619a", size = 16085101, upload-time = "2026-10-19T19:19:42.203Z" }, +] + [[package]] name = "cffi" version = "2.1.1" @@ -260,6 +288,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, ] +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + [[package]] name = "mtusi2026lab4" version = "0.1.0" @@ -267,7 +304,9 @@ source = { virtual = "." } dependencies = [ { name = "argon2-cffi" }, { name = "asyncpg" }, + { name = "boto3" }, { name = "fastapi" }, + { name = "pillow" }, { name = "uvicorn" }, ] @@ -275,10 +314,88 @@ dependencies = [ requires-dist = [ { name = "argon2-cffi", specifier = "==25.1.0" }, { name = "asyncpg", specifier = "==0.31.0" }, + { name = "boto3", specifier = "==1.43.115" }, { name = "fastapi", specifier = "==0.141.1" }, + { name = "pillow", specifier = "==13.0.0" }, { name = "uvicorn", specifier = "==0.40.0" }, ] +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, +] + [[package]] name = "pycparser" version = "3.1" @@ -378,6 +495,39 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/04/c81d4841331c2178b6fb09ae225425e110ed72d990c9fe556c4ec03d1013/pydantic_core-2.46.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e24d8f05fa2d28513d94e877e9c75ad66175376209b3977f916e240e623193c", size = 2111034, upload-time = "2026-08-28T10:01:07.345Z" }, ] +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + [[package]] name = "starlette" version = "1.7.0" @@ -412,6 +562,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + [[package]] name = "uvicorn" version = "0.40.0" -- 2.54.0 From 901786170792195f6dd95d0939db98236eb932ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Thu, 22 Oct 2026 11:37:09 +0300 Subject: [PATCH 3/5] add level 4 --- Makefile | 19 ++++- REPORT.md | 36 ++++++--- compose.yaml | 34 +++++++- evidence/attachments-level4.txt | 5 ++ evidence/environment.json | 32 ++++---- evidence/failover-level4.txt | 4 + evidence/persistence-level4.txt | 5 ++ evidence/reconcile.txt | 31 +++++++ evidence/s3-permissions.txt | 1 + evidence/smoke-level4.txt | 2 + evidence/upload-memory.json | 21 +++++ evidence/upload-memory.txt | 2 + scripts/init-secrets.py | 6 +- scripts/restart.sh | 1 + src/attachments.py | 10 ++- src/bootstrap_s3.py | 13 +++ src/config.py | 1 + src/main.py | 6 +- src/reconcile.py | 63 +++++++++++++++ src/storage/postgres.py | 8 ++ src/storage/s3.py | 18 +++++ student-tests/reconcile.py | 138 ++++++++++++++++++++++++++++++++ student-tests/s3_permissions.py | 54 +++++++++++++ student-tests/upload_memory.py | 92 +++++++++++++++++++++ 24 files changed, 571 insertions(+), 31 deletions(-) create mode 100644 evidence/attachments-level4.txt create mode 100644 evidence/failover-level4.txt create mode 100644 evidence/persistence-level4.txt create mode 100644 evidence/reconcile.txt create mode 100644 evidence/s3-permissions.txt create mode 100644 evidence/smoke-level4.txt create mode 100644 evidence/upload-memory.json create mode 100644 evidence/upload-memory.txt create mode 100644 src/reconcile.py create mode 100644 student-tests/reconcile.py create mode 100644 student-tests/s3_permissions.py create mode 100644 student-tests/upload_memory.py diff --git a/Makefile b/Makefile index fc7400ef..ff530d52 100644 --- a/Makefile +++ b/Makefile @@ -29,9 +29,9 @@ test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) -.PHONY: test-attachments test-private test-upload-failover test-s3-outage +.PHONY: test-attachments test-private test-upload-failover test-s3-outage reconcile reconcile-apply test-reconcile test-upload-memory test-s3-permissions test-attachments: - $(PYTHON) student-tests/attachments.py --base-url "$(BASE_URL)" + uv run python student-tests/attachments.py --base-url "$(BASE_URL)" test-private: $(PYTHON) student-tests/attachment_resilience.py --mode private --base-url "$(BASE_URL)" @@ -41,3 +41,18 @@ test-upload-failover: test-s3-outage: $(PYTHON) student-tests/attachment_resilience.py --mode s3-outage --base-url "$(BASE_URL)" + +reconcile: + docker compose --profile maintenance run --rm --no-deps reconcile --grace-seconds 3600 + +reconcile-apply: + docker compose --profile maintenance run --rm --no-deps reconcile --apply --grace-seconds 3600 + +test-reconcile: + $(PYTHON) student-tests/reconcile.py --base-url "$(BASE_URL)" + +test-upload-memory: + uv run python student-tests/upload_memory.py --base-url "$(BASE_URL)" + +test-s3-permissions: + $(PYTHON) student-tests/s3_permissions.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index b14dfe68..0fb4ad3b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,9 +1,9 @@ # Отчёт — лабораторная 4 - Автор, группа: Матвей Козырев, БВТ2301. -- Целевая оценка: **3 — работающий минимум**. +- Целевая оценка: **4 — уровень стажёра/джуна**. - Ветка: `solution/BVT2301-Kozyrev`. -- Проверки: 20 октября 2026 года. Базовый commit и SHA-256 исходников проверенного рабочего дерева — [evidence/environment.json](evidence/environment.json). +- Проверки: 22 октября 2026 года. Базовый commit и SHA-256 исходников проверенного рабочего дерева — [evidence/environment.json](evidence/environment.json). - Использованные библиотеки: FastAPI, Uvicorn, asyncpg, argon2-cffi, boto3, Pillow. Помощь ИИ: Codex — перенос основы lab3, реализация вложений, тесты и документация. ## Запуск из чистого clone @@ -45,9 +45,9 @@ docker compose run --rm --no-deps s3-init | S3 / s3 | SeaweedFS 4.17, digest в Compose | master cluster status; bootstrap дополнительно проверяет S3 | s3_data: файлы и LevelDB filer | Внутренний 8333; identity config из secret | | Bootstrap / s3-init | Тот же Python-образ | exit 0 | Нет | Отдельные административные S3 credentials | -SeaweedFS закреплён digest `sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9`. Настройка filer LevelDB в [infra/filer.toml](infra/filer.toml) сохраняет его метаданные в `/data/filer` на том же volume. S3 identity `api` имеет Read/Write/List для одного bucket; anonymous identity отсутствует. Административные credentials доступны bootstrap, но не API. Внутренняя Docker-сеть считается доверенной; это не защита от компрометации контейнера. +SeaweedFS закреплён digest `sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9`. Настройка filer LevelDB в [infra/filer.toml](infra/filer.toml) сохраняет его метаданные в `/data/filer` на том же volume. S3 identity `api` имеет Read/Write/List только для `messenger-attachments`; anonymous identity отсутствует. Она не может выполнять bucket-admin operations. Административные credentials доступны bootstrap и maintenance-reconciler, но не API. SeaweedFS классифицирует DELETE object как Write, поэтому этот провайдер не разделяет PUT и DELETE внутри одной Write-разрешённой bucket-scoped identity; reconciler вынесен в отдельный maintenance container, а API-код не содержит операции удаления. Внутренняя Docker-сеть считается доверенной; это не защита от компрометации контейнера. -Лимиты Compose: каждая API — 0,5 CPU / 512 MiB; PostgreSQL — 0,75 CPU / 512 MiB; SeaweedFS — 1 CPU / 512 MiB; HAProxy — 0,25 CPU / 128 MiB. Это учебный стенд на одном Docker-host, без высокой доступности БД, S3, LB и самого host. +Лимиты Compose: каждая API — 0,5 CPU / 512 MiB; PostgreSQL — 0,75 CPU / 512 MiB; SeaweedFS — 1 CPU / 512 MiB; HAProxy — 0,25 CPU / 128 MiB. SeaweedFS допускает до 64 локальных data volumes по 128 MiB; это предотвращает исчерпание учебного лимита при повторяемых test uploads. Это учебный стенд на одном Docker-host, без высокой доступности БД, S3, LB и самого host. ### Перенос lab3 и модель вложений @@ -63,10 +63,18 @@ SeaweedFS закреплён digest `sha256:186de7ef977a20343ee9a5544073f081976a Лимит **10 MiB = 10 × 1024 × 1024 = 10 485 760 байт** проверяется по Content-Length и повторно при чтении каждого блока, в том числе для chunked upload. До декодирования Pillow определяет фактический формат и размеры; максимум **40 000 000 пикселей**. Проверяются целостность PNG и декодирование пикселей PNG/JPEG. MIME должен совпадать с содержимым. Анимированный PNG не поддерживается: ответ 415. -Во время приёма используется удаляемый временный файл; он не является хранилищем оригиналов и не используется при скачивании. На одну API-реплику допускается одна одновременная загрузка/проверка изображения, ожидание вместе с загрузкой ограничено 30 секундами. SDK и декодирование выполняются вне event loop. S3 connect timeout — 2 с, read timeout — 5 с, автоматические повторы выключены. Это ограничения отдельного запроса, не доказательство устойчивости при произвольной нагрузке. +Во время приёма используется удаляемый временный файл; он не является хранилищем оригиналов и не используется при скачивании. На одну API-реплику допускается одна одновременная загрузка/проверка изображения (`UPLOAD_CONCURRENCY=1`), ожидание вместе с загрузкой ограничено 30 секундами (`UPLOAD_TIMEOUT_SECONDS`). SDK и декодирование выполняются вне event loop. S3 connect timeout — 2 с, read timeout — 5 с, автоматические повторы выключены. Объект и запрос ограничены 10 MiB; следовательно API не накапливает неограниченное тело запроса в памяти. Временный файл ограничивает RAM ценой bounded disk usage до 10 MiB на активную загрузку. Порядок фиксации: валидация → S3 PUT → INSERT метаданных PostgreSQL → **201, status=ready, variants=[], error_code=null**. GET метаданных и оригинала снова проверяет членство. Для постороннего — 404; без сессии — 401. GET content читает оригинал из S3, проверяет размер и SHA-256 и возвращает байты через API без redirect. Object key и credentials не входят в публичную модель. +### Согласованность PostgreSQL и S3 + +Атомарной транзакции между PostgreSQL и S3 нет. Если PUT прошёл, но процесс или БД отказали до подтверждённого INSERT, API возвращает 503 и намеренно не удаляет объект синхронно: исход COMMIT мог быть неизвестен. Вместо небезопасного удаления применяется [src.reconcile](src/reconcile.py). + +`make reconcile` запускает inventory в **dry-run**. Он сравнивает только `originals/` S3 с `attachments.object_key`, сообщает orphan, свежие unreferenced и missing objects. `make reconcile-apply` удаляет только orphan старше 3600 секунд; отсутствующие объекты и все referenced записи не удаляются. Grace period защищает объект при задержанном COMMIT/наблюдении. Для ручной аварийной проверки возможно `docker compose --profile maintenance run --rm --no-deps reconcile --apply --grace-seconds N`; значение 0 применяется только в изолированных тестах. Reconciler использует отдельные admin credentials, API не получает эти секреты. + +Missing object считается наблюдаемой ошибкой, а не поводом удалить метаданные: download вернёт 503, а maintenance output содержит attachment ID и object key. Восстановление байтов требует оператора или резервной копии; автоматическое восстановление не заявляется. Это безопаснее удаления используемого сообщения/метаданных без источника оригинала. + Недоступность S3: upload/download и readiness возвращают 503 в формате Error; liveness остаётся 200. HAProxy исключает unready backend, поэтому при недоступности обеих реплик сам LB может вернуть собственный 503. Проверка формата ошибки API при таком отказе выполняется напрямую внутри Docker-сети. ## Проверки и доказательства @@ -87,6 +95,10 @@ make test-s3-outage python3 scripts/verify-persistence.py make test-failover ACTION_SCRIPT=scripts/stop-one.sh docker compose start api-a api-b +make reconcile +make test-reconcile +make test-upload-memory +make test-s3-permissions ``` Сценарии отказов выполняются последовательно на учебном стенде: они временно останавливают контейнеры. Собственные проверки возвращают остановленные сервисы в работу; публичный `test-failover` оставляет реплику остановленной, поэтому последняя команда нужна для восстановления. Все данные синтетические, токены не сохраняются. Тесты добавляют данные; автоматического удаления нет. @@ -99,17 +111,20 @@ docker compose start api-a api-b | Приватность bucket вне API | `make test-private`: anonymous GET существующего объекта → 403 | [PASS](evidence/private-bucket.txt) | | Отказ именно загрузившей API | `make test-upload-failover`: загрузка через api-b, остановка api-b, те же байты через другую реплику | [PASS](evidence/attachment-failover.txt) | | Недоступность S3 | `make test-s3-outage`: 503, восстановление, прежний оригинал | [PASS](evidence/s3-outage.txt) | -| Перезапуск всех компонентов, сохранение действующей сессии, метаданных и файла | `python3 scripts/verify-persistence.py`; StartedAt каждого сервиса изменился | [PASS](evidence/persistence.txt) | -| Публичный failover, сообщения без дублей и сохранение файла | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | [Результат](evidence/failover.txt) | +| Перезапуск всех компонентов, сохранение действующей сессии, метаданных и файла | `python3 scripts/verify-persistence.py`; StartedAt каждого сервиса изменился | [PASS](evidence/persistence-level4.txt) | +| Публичный failover, сообщения без дублей и сохранение файла | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | [PASS](evidence/failover-level4.txt) | | Конкурентная идемпотентность и регрессии lab3 | Собственные проверки lab3 | [Результат](evidence/lab3-regressions.txt) | | Повторный bootstrap bucket | `docker compose run --rm --no-deps s3-init` | [Результат](evidence/bootstrap.txt) | +| Reconcile: dry-run, grace period, protected object, missing object, post-PUT/pre-DB отказ | `make test-reconcile` | [PASS](evidence/reconcile.txt) | +| Ограниченные S3 credentials | `make test-s3-permissions`: API не создаёт bucket; отдельный maintenance principal выполняет admin list | [PASS](evidence/s3-permissions.txt) | +| 10 одновременных потоковых upload по 5 MiB | `make test-upload-memory`, cgroup `memory.current` каждые 0,1 с | [PASS](evidence/upload-memory.txt), [raw JSON](evidence/upload-memory.json) | -Точные границы проверены отдельно: PNG с trailing bytes доведён до 10 485 760 байт и скачан без изменений; PNG 8000 × 5000 = 40 000 000 пикселей принят; заголовок 8001 × 5000 = 40 005 000 пикселей отклонён до декодирования. Это функциональные тесты, не исследование RPS или памяти уровня 4/5. +Точные границы проверены отдельно: PNG с trailing bytes доведён до 10 485 760 байт и скачан без изменений; PNG 8000 × 5000 = 40 000 000 пикселей принят; заголовок 8001 × 5000 = 40 005 000 пикселей отклонён до декодирования. В `upload-memory.json` серия из 10 клиентов по 5 242 880 bytes дала длительность 5,529 с; 22 cgroup-пробы показали peak 105 254 912 и 101 470 208 bytes на двух API. Это на 431 616 000 и 435 400 704 bytes ниже лимита 536 870 912 bytes, соответственно: `536870912 − 105254912` и `536870912 − 101470208`. Измерение дискретное, поэтому это наблюдаемый минимум пика, а не строгая верхняя оценка. ## Ограничения -- Между S3 PUT и COMMIT PostgreSQL нет общей транзакции. Сбой после PUT может оставить orphan object. Автоматическая сверка/очистка с grace period относится к следующему уровню и здесь не реализована. При неопределённом исходе COMMIT объект намеренно не удаляется, чтобы не повредить возможную подтверждённую запись. -- API буферизует скачивание не более 10 MiB, чтобы проверить checksum до ответа. Ограничение общего количества скачиваний и нагрузочный эксперимент для 10 одновременных upload не выполнены; оценка 4 не заявляется. +- Между S3 PUT и COMMIT PostgreSQL нет общей транзакции; orphan закрывает reconciler с grace period, но missing-object запись восстанавливается только оператором/backup. Автоматический repair исходного файла не реализован. +- API буферизует скачивание не более 10 MiB, чтобы проверить checksum до ответа. Эксперимент 10 upload измеряет cgroup memory.current, но не является capacity benchmark и не покрывает произвольные медленные клиенты, filesystem quota и одновременные скачивания. - Readiness проверяет доступность БД и bucket, но не наличие каждого отдельного объекта. Потеря одного объекта при доступном S3 обнаруживается при скачивании как 503. - Перезапуск с volumes и отказ API подтверждены на одном Docker-host. Потеря volumes/host не покрывается, резервное копирование не реализовано. - HTTP предназначен для локального учебного стенда. Presigned flow, обработка thumbnail и брокер не реализованы. @@ -118,5 +133,6 @@ docker compose start api-a api-b - [Контракт API](contracts/README.md), [OpenAPI](contracts/openapi.json), [критерии задания](README.md). - [SeaweedFS исходники и запуск](https://github.com/seaweedfs/seaweedfs), [релиз 4.17](https://github.com/seaweedfs/seaweedfs/releases/tag/4.17). Параметры закреплённого образа дополнительно проверены через `docker run --rm chrislusf/seaweedfs:4.17 server -help`. +- [SeaweedFS S3 authorization](https://github.com/seaweedfs/seaweedfs/blob/master/weed/s3api/auth_signature_v4.go): в используемой модели операции, отличные от GET/HEAD, проверяются как `Write`; это источник ограничения на раздельные PUT/DELETE права. - [Pillow Image API](https://pillow.readthedocs.io/en/stable/reference/Image.html): lazy open, размеры, verify/load и decompression bomb limits. - [Boto3 S3 PutObject](https://docs.aws.amazon.com/boto3/latest/reference/services/s3/client/put_object.html): запись оригинала в S3. diff --git a/compose.yaml b/compose.yaml index 4e5a0c42..68b696af 100644 --- a/compose.yaml +++ b/compose.yaml @@ -15,6 +15,9 @@ x-api: &api S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES: "10485760" MAX_IMAGE_PIXELS: "40000000" + UPLOAD_CONCURRENCY: "1" + UPLOAD_TIMEOUT_SECONDS: "30" + UPLOAD_FAULT_AFTER_S3_PUT: ${UPLOAD_FAULT_AFTER_S3_PUT:-0} secrets: - database_url - cursor_secret @@ -43,7 +46,7 @@ x-api: &api services: s3: image: chrislusf/seaweedfs:4.17@sha256:186de7ef977a20343ee9a5544073f081976a29e2d29ecf8379891e7bf177fbe9 - command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.config=/run/secrets/s3_config", "-master.volumeSizeLimitMB=128", "-volume.max=8", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] + command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.config=/run/secrets/s3_config", "-master.volumeSizeLimitMB=128", "-volume.max=64", "-s3.port.iceberg=0", "-filer.exposeDirectoryData=false"] secrets: - s3_config volumes: @@ -54,6 +57,7 @@ services: interval: 5s timeout: 3s retries: 20 + start_period: 15s deploy: resources: limits: @@ -119,6 +123,34 @@ services: cpus: "0.25" memory: 128M + reconcile: + build: + context: . + entrypoint: ["/app/.venv/bin/python", "-m", "src.reconcile"] + profiles: ["maintenance"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_key + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret + secrets: + - database_url + - cursor_secret + - s3_admin_key + - s3_admin_secret + depends_on: + postgres: + condition: service_healthy + s3: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + api-a: <<: *api diff --git a/evidence/attachments-level4.txt b/evidence/attachments-level4.txt new file mode 100644 index 00000000..95e7b374 --- /dev/null +++ b/evidence/attachments-level4.txt @@ -0,0 +1,5 @@ +uv run python student-tests/attachments.py --base-url "http://localhost:8080" +PASS: PNG/JPEG originals, metadata, attachment-only messages and ACL +PASS: filename, MIME, malformed image, byte/pixel and chunked limits +PASS: attachment scope, duplicates and idempotency normalization +PASS: inclusive 10 MiB and 40 million pixel boundaries diff --git a/evidence/environment.json b/evidence/environment.json index 6394fdf7..0f7d10ba 100644 --- a/evidence/environment.json +++ b/evidence/environment.json @@ -1,6 +1,6 @@ { - "recorded_at_utc": "2026-10-20T14:23:50.123531+00:00", - "base_commit": "3b52b91fb76d8acdeb7cbec010fd5542fd8990f7", + "recorded_at_utc": "2026-10-22T08:32:17.051591+00:00", + "base_commit": "e9ef559c349cf8b6fc32138fed0ea7a7600ce1ca", "branch": "solution/BVT2301-Kozyrev", "note": "Results apply to this working tree, not only the base commit. Functional tests; no load benchmark.", "host": "macOS-26.1-arm64-arm-64bit-Mach-O", @@ -12,8 +12,8 @@ "compose_version": "5.5.1", "sources_sha256": { "Dockerfile": "efede941e2ccf21cd1e70ebbf8af7647209bd030b2962f9446bee497a2ed6a8c", - "Makefile": "bdb2dade89d84f516b20097ee8f33db7b85dd6bfa431c41881177a323cea845e", - "compose.yaml": "47d7c77d070097cef515279c3e46435bf0925d9b82dd96ef88fa38534e217328", + "Makefile": "b417167d9d5baae110b2963a26ec12cea8ad8c96ce3d627a6c9e4fe8f51b74d3", + "compose.yaml": "b239915436be84c7b1000346669ae9a10c89e83c55555a5e9eac0e20828f6fb6", "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", "infra/filer.toml": "ae3c01bf7708f5d4a319fdb454f753be0f137fe7c31269f1c413cbccc860f5c7", "infra/haproxy.cfg": "23c9b869170633e3c6a8c41bbd3f1102bc97bf213e0d7cfed9d461af91216fd3", @@ -23,34 +23,38 @@ "migrations/004_attachments.sql": "d760453ba130c94c403f5b04222649fc131ab4d091edd0267ede901fdf82a79e", "pyproject.toml": "d3962edc7a683a62c915c8edec91e85c647c8ae6fe6432603e30b595bd6c3804", "scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671", - "scripts/init-secrets.py": "fc89fc740da189b65be1b69980ff512e2c233dd09997c8ffb0198d9368118386", + "scripts/init-secrets.py": "01232befe48b69933b5c6b7e4f29cb19a777030ffab00accc7999afb6c082cef", "scripts/record-environment.py": "716fa7b8edd05d7ab6781fc346338b2254f82186953c6dbf525785c723eee4a1", "scripts/restart.example.sh": "0328246ba75defbced992104e986fde973f63e3830d83251eced5953263e944b", - "scripts/restart.sh": "0a64716273eb5c79011d99e4734b18ed5a3a28f41b8b1d4ea6b24d0a4ffc702f", + "scripts/restart.sh": "a2710fc04976b28d0da4a8cd68bf2cc28735d01ec321ec95861f855900f9a42b", "scripts/stop-one.example.sh": "1bcf34394520b89d95b7605ec56675be799eea920e83cfbba306d5dc0a6d9811", "scripts/stop-one.sh": "c69fdb1891e0c49f41d3892f2d108ecd38b2f59d505c8fe991023bb5c9b08a5e", "scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54", "scripts/verify-persistence.py": "6c366bf4d85802bcffe65ae3d45d00736b184033313dabb60b80e1270d8874f1", "src/__init__.py": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", - "src/attachments.py": "b377227a623bc41585f2eca487356bbd38c59a0ada56f425b8fca11abd93c085", - "src/bootstrap_s3.py": "bd0bafad4f5443bd37ab0d14c7b49453ae77fda059076f59cc02e9abc08bbc65", - "src/config.py": "6cb85b839e2947c3334c52f24d94ceb5290103e323ba33e6ab59f10f3ae3a53a", + "src/attachments.py": "553936936318b8096b5a662ddd85dae4043dffb9b4fb6461e60bfc1962914325", + "src/bootstrap_s3.py": "2c43f5540e63f004a88e08961a72293546fbc5210f46bc719b90b9f5c0f3f9bd", + "src/config.py": "62e214ceef5fcc24be11d23500de0fdaf6cf5efed1d913cdda0921fd34b088c3", "src/cursor.py": "a9192f2445042b8d4ddac57031718ef558990a52eb2922827074479b7f686775", "src/errors.py": "07f8860b45cadd242d0d9f268c4b806e35353f33b11bbde0ab3d50acda86a80a", - "src/main.py": "abb5d7a856bff6d19ada7cf17addd59ef7400a26cf795ec01ea8b7d76f44527a", + "src/main.py": "6980cdbb09c3b356ff870ee7fc972d85e835c58228d3c0ea27de8a668dc41ece", "src/migrate.py": "1d164c8921df2b5369e30b1f8936c2b71858c5279070f0fd593f62f66e1f0b7d", "src/models.py": "b94e6b26559afc5fd90f58d34d906c77bf4b4a78c716ac552358795496ad1d1b", + "src/reconcile.py": "67682de753e3331a5bb5c884ab6dabaad458c49e27d4a0f2c849d79b63a7c44b", "src/routes.py": "3c21149a30ea7c2f695547c42133dcebcfbacbd6544bcb73c353be46fa35a2e6", "src/storage/__init__.py": "05ef66318c4c436eaafc493072640817219ab37bce464a265a2306233389064a", - "src/storage/postgres.py": "2ff296490b06729c8e56eb97dbdc8d1ca91c94e313375ea5a88ffec99b6f838d", - "src/storage/s3.py": "433247b86837a9a1212900da9d3241105e191b36d8fcac8a92c50100f5f4b946", + "src/storage/postgres.py": "c69fea94937fdf00b077a8f1d301e83f35e120c3a7c5153fe205725be0aa0428", + "src/storage/s3.py": "c51c8a804ca1ed869556c80252e119372696110443df30bd84f0836a01a0c305", "student-tests/attachment_resilience.py": "c72c431b355dda919c1b6b854d260f8c7420672ca9f86bf0848692da35dcb0ac", "student-tests/attachments.py": "a948fabfe83b73eb8e494c80fb9d1f2a949e2a95b7a21809a05975d65d68fc15", "student-tests/idempotency_concurrency.py": "db5d139009e4660ac1801af45d10e24665aa81cba737731e527c0f5b4152231e", "student-tests/readiness_failover.py": "0ef8355d85be66597d7d44eb7308dcb58c6c9a174d18f360b2d6d22fdcfbed14", "student-tests/readiness_unavailable.py": "46e3f680a3ff9cb79e95568004d97f1752dbdcd88fd4b5f160a9cc38af65a237", + "student-tests/reconcile.py": "298537cda6523631def290e750a651db1eace0647b99c68647c68e0fdb8f34ca", "student-tests/review_regressions.py": "de58f63b2c794c2775247ddd3fccc523870ccbe673ffa404fdd566274d1d41de", + "student-tests/s3_permissions.py": "f265a64f900952db79d545e6c058a7605870c2843ed2298f5c68ad4a177663ff", "student-tests/session_expiry.py": "ba8ead4356e4d12c4047ac2b5704d3d3342f71c37b3c13f65a0a9d7c37186ad6", + "student-tests/upload_memory.py": "a1493aaa8ce8e898291a14059826c1405d31a3577027ae926f8c80febf25d3dc", "uv.lock": "fdbeb5ba0c5eedced1a6e5b62c6a57b2d22f293cf77b9487a806c6e1a8b992a0" }, "services": { @@ -78,7 +82,7 @@ }, "api-a": { "image": "mtusi2026lab4-api-a", - "image_id": "sha256:3a32a3f312aff9b46c6de73815627556af6f7cbee4b22783c1f4185d407f01b3", + "image_id": "sha256:679085dc5870799c12f9251cccbd62a1bf4d75e80a9706bc052b57d51d8251e7", "state": "running", "health": "healthy", "memory_limit_bytes": 536870912, @@ -87,7 +91,7 @@ }, "api-b": { "image": "mtusi2026lab4-api-b", - "image_id": "sha256:0e3c794618677c4a6305586bf44165fb1b3dbe668ce77b583df97e9e72721aa1", + "image_id": "sha256:0ca4765ed83bf21e802b8b90f2687839d761519271423bc10b2c6d72e29aef48", "state": "running", "health": "healthy", "memory_limit_bytes": 536870912, diff --git a/evidence/failover-level4.txt b/evidence/failover-level4.txt new file mode 100644 index 00000000..c4565a97 --- /dev/null +++ b/evidence/failover-level4.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 93, "failed_attempts": 2, "first_recovery_seconds": 2.562} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/persistence-level4.txt b/evidence/persistence-level4.txt new file mode 100644 index 00000000..4c66f858 --- /dev/null +++ b/evidence/persistence-level4.txt @@ -0,0 +1,5 @@ +Before: {"api-a": {"container": "fd7ade45cc98", "started_at": "2026-10-22T08:24:41.206791297Z"}, "api-b": {"container": "1bcbfe033cad", "started_at": "2026-10-22T08:24:41.125348256Z"}, "load_balancer": {"container": "3e69e9c37346", "started_at": "2026-10-22T08:24:41.017094297Z"}, "postgres": {"container": "b756a119c924", "started_at": "2026-10-22T08:24:32.576392335Z"}, "s3": {"container": "fe8f16b7a24d", "started_at": "2026-10-22T08:25:56.983675472Z"}} +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +After: {"api-a": {"container": "fd7ade45cc98", "started_at": "2026-10-22T08:29:52.364601219Z"}, "api-b": {"container": "1bcbfe033cad", "started_at": "2026-10-22T08:29:52.274115094Z"}, "load_balancer": {"container": "3e69e9c37346", "started_at": "2026-10-22T08:29:52.231563844Z"}, "postgres": {"container": "b756a119c924", "started_at": "2026-10-22T08:29:42.97697609Z"}, "s3": {"container": "fe8f16b7a24d", "started_at": "2026-10-22T08:29:46.000691966Z"}} +PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved diff --git a/evidence/reconcile.txt b/evidence/reconcile.txt new file mode 100644 index 00000000..ca5fecd8 --- /dev/null +++ b/evidence/reconcile.txt @@ -0,0 +1,31 @@ + Container mtusi2026lab4-s3-init-run-a56a6944b69f Creating + Container mtusi2026lab4-s3-init-run-a56a6944b69f Created + Container mtusi2026lab4-reconcile-run-2f379f07becf Creating + Container mtusi2026lab4-reconcile-run-2f379f07becf Created + Container mtusi2026lab4-reconcile-run-32de115488fc Creating + Container mtusi2026lab4-reconcile-run-32de115488fc Created + Container mtusi2026lab4-reconcile-run-8824578f5ed8 Creating + Container mtusi2026lab4-reconcile-run-8824578f5ed8 Created + Container mtusi2026lab4-s3-init-run-849f50d3d583 Creating + Container mtusi2026lab4-s3-init-run-849f50d3d583 Created + Container mtusi2026lab4-reconcile-run-cd1eb67e6a5f Creating + Container mtusi2026lab4-reconcile-run-cd1eb67e6a5f Created + Container mtusi2026lab4-s3-init-run-a50b80f3ae78 Creating + Container mtusi2026lab4-s3-init-run-a50b80f3ae78 Created +PASS: dry-run, grace period, orphan deletion and missing-object reporting + Container mtusi2026lab4-api-a-1 Recreate + Container mtusi2026lab4-api-a-1 Recreated + Container mtusi2026lab4-api-a-1 Starting + Container mtusi2026lab4-api-a-1 Started + Container mtusi2026lab4-api-a-1 Running + Container mtusi2026lab4-api-a-1 Waiting + Container mtusi2026lab4-api-a-1 Healthy + Container mtusi2026lab4-reconcile-run-1c72990b1ddc Creating + Container mtusi2026lab4-reconcile-run-1c72990b1ddc Created + Container mtusi2026lab4-reconcile-run-2e902a56554d Creating + Container mtusi2026lab4-reconcile-run-2e902a56554d Created +PASS: post-PUT/pre-DB injected failure leaves no permanent orphan + Container mtusi2026lab4-api-a-1 Recreate + Container mtusi2026lab4-api-a-1 Recreated + Container mtusi2026lab4-api-a-1 Starting + Container mtusi2026lab4-api-a-1 Started diff --git a/evidence/s3-permissions.txt b/evidence/s3-permissions.txt new file mode 100644 index 00000000..690252b5 --- /dev/null +++ b/evidence/s3-permissions.txt @@ -0,0 +1 @@ +PASS: API identity cannot create buckets; maintenance identity has separate admin access diff --git a/evidence/smoke-level4.txt b/evidence/smoke-level4.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/smoke-level4.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..0933cb1b --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,21 @@ +{ + "uploads": 10, + "file_bytes": 5242880, + "concurrency": 10, + "api_upload_concurrency_limit": 1, + "elapsed_seconds": 5.529, + "baseline_memory_bytes": { + "7b09ecf19049": 84668416, + "1bcbfe033cad": 86630400 + }, + "peak_memory_bytes": { + "7b09ecf19049": 105254912, + "1bcbfe033cad": 101470208 + }, + "peak_growth_bytes": { + "7b09ecf19049": 20586496, + "1bcbfe033cad": 14839808 + }, + "samples": 22, + "method": "cgroup memory.current sampled through Docker every 0.1 seconds while clients send chunked 512 KiB blocks every 0.5 seconds; peak is not a continuous profiler." +} diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 00000000..2559ee4e --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,2 @@ +uv run python student-tests/upload_memory.py --base-url "http://localhost:8080" +{"uploads": 10, "file_bytes": 5242880, "concurrency": 10, "api_upload_concurrency_limit": 1, "elapsed_seconds": 5.529, "baseline_memory_bytes": {"7b09ecf19049": 84668416, "1bcbfe033cad": 86630400}, "peak_memory_bytes": {"7b09ecf19049": 105254912, "1bcbfe033cad": 101470208}, "peak_growth_bytes": {"7b09ecf19049": 20586496, "1bcbfe033cad": 14839808}, "samples": 22, "method": "cgroup memory.current sampled through Docker every 0.1 seconds while clients send chunked 512 KiB blocks every 0.5 seconds; peak is not a continuous profiler."} diff --git a/scripts/init-secrets.py b/scripts/init-secrets.py index 1e1f6954..987771d4 100644 --- a/scripts/init-secrets.py +++ b/scripts/init-secrets.py @@ -25,10 +25,12 @@ admin_key = write("s3_admin_key", secrets.token_hex(16)) admin_secret = write("s3_admin_secret", secrets.token_hex(32)) app_key = write("s3_access_key_id", secrets.token_hex(16)) app_secret = write("s3_secret_access_key", secrets.token_hex(32)) -write("s3.json", json.dumps({"identities": [ +config_path = root / "s3.json" +config_path.write_text(json.dumps({"identities": [ {"name": "bootstrap", "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}], "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, {"name": "api", "credentials": [{"accessKey": app_key, "secretKey": app_secret}], "actions": ["Read:messenger-attachments", "Write:messenger-attachments", "List:messenger-attachments"]}, -]}, indent=2)) +]}, indent=2) + "\n") +config_path.chmod(0o600) print("Local secrets ready; existing values preserved.") diff --git a/scripts/restart.sh b/scripts/restart.sh index aa02c3b6..d3d9b67e 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -3,5 +3,6 @@ set -eu cd "$(dirname "$0")/.." docker compose restart postgres s3 docker compose up -d --wait postgres s3 +docker compose run --rm --no-deps s3-init docker compose restart api-a api-b load_balancer docker compose up -d --wait api-a api-b load_balancer diff --git a/src/attachments.py b/src/attachments.py index a078a7c2..09bb9702 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -1,5 +1,6 @@ import asyncio from hashlib import sha256 +import os from tempfile import TemporaryFile import unicodedata from uuid import UUID, uuid4 @@ -62,7 +63,10 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...), if length > limit: raise ApiError(413, "payload_too_large", "Файл больше лимита") try: - async with asyncio.timeout(30): + timeout = int(os.environ.get("UPLOAD_TIMEOUT_SECONDS", "30")) + if not 1 <= timeout <= 120: + raise ApiError(503, "unavailable", "Некорректный лимит времени загрузки") + async with asyncio.timeout(timeout): async with request.app.state.upload_slot: # Temporary scratch space only; deleted on every exit, never used for downloads. with TemporaryFile() as file: @@ -78,6 +82,10 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...), attachment_id = uuid4() key = f"originals/{attachment_id}" await s3.put(key, file, size, content_type) + # Disabled by default. Used only by the failure-injection test to + # model a process crash after S3 accepts the object and before DB. + if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": + raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") return await store.create_attachment( attachment_id, chat_id, user["id"], filename, content_type, size, digest.hexdigest(), key, diff --git a/src/bootstrap_s3.py b/src/bootstrap_s3.py index 5f90eebe..28f2c8b0 100644 --- a/src/bootstrap_s3.py +++ b/src/bootstrap_s3.py @@ -1,5 +1,6 @@ """Idempotent private bucket bootstrap; executed with separate admin credentials.""" import time +from uuid import uuid4 from botocore.exceptions import BotoCoreError, ClientError from .storage.s3 import S3Settings, S3Store @@ -29,6 +30,18 @@ def main(): except ClientError as error: if error.response["Error"]["Code"] != "NoSuchBucketPolicy": raise + # The master health endpoint can become available before a volume accepts + # writes after a restart. Verify the actual data path before APIs start. + probe_key = f"_bootstrap/write-probe-{uuid4()}" + for attempt in range(30): + try: + store.client.put_object(Bucket=bucket, Key=probe_key, Body=b"ready") + store.client.delete_object(Bucket=bucket, Key=probe_key) + break + except (BotoCoreError, ClientError): + if attempt == 29: + raise RuntimeError("S3 write path startup timed out") from None + time.sleep(1) print("PASS: private S3 bucket ready") finally: store.client.close() diff --git a/src/config.py b/src/config.py index 091c861b..abb5dea6 100644 --- a/src/config.py +++ b/src/config.py @@ -19,6 +19,7 @@ _CONFIGURATION_KEYS = { "S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE", "S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE", "MAX_UPLOAD_BYTES", "MAX_IMAGE_PIXELS", + "UPLOAD_CONCURRENCY", "UPLOAD_TIMEOUT_SECONDS", "UPLOAD_FAULT_AFTER_S3_PUT", } diff --git a/src/main.py b/src/main.py index 30df8478..f3559329 100644 --- a/src/main.py +++ b/src/main.py @@ -1,5 +1,6 @@ import asyncio import logging +import os import sys import time from contextlib import asynccontextmanager @@ -45,7 +46,10 @@ async def lifespan(application: FastAPI): application.state.store = store application.state.settings = settings application.state.s3 = S3Store(S3Settings.load()) - application.state.upload_slot = asyncio.Semaphore(1) + concurrency = int(os.environ.get("UPLOAD_CONCURRENCY", "1")) + if not 1 <= concurrency <= 8: + raise ValueError("UPLOAD_CONCURRENCY must be 1..8") + application.state.upload_slot = asyncio.Semaphore(concurrency) application.state.accepting_requests = True try: yield diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..a2016448 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,63 @@ +"""Safely reconcile attachment metadata and original S3 objects. + +The command is dry-run by default. It only deletes unreferenced objects older +than a configured grace period; missing objects are reported and never cause a +metadata delete. That keeps a delayed PostgreSQL commit and active attachment +records safe from a maintenance run. +""" +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +import json + +from .config import load_settings +from .storage.postgres import PostgresStore +from .storage.s3 import S3Settings, S3Store + + +async def run(args) -> dict: + settings = load_settings() + db = PostgresStore(settings.database_url) + s3 = S3Store(S3Settings.load()) + await db.connect() + try: + objects, metadata = await s3.list("originals/"), await db.attachment_keys() + cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) + orphan = sorted(key for key, modified in objects.items() + if key not in metadata and modified <= cutoff) + fresh = sorted(key for key, modified in objects.items() + if key not in metadata and modified > cutoff) + missing = sorted({str(attachment_id): key for key, attachment_id in metadata.items() + if key not in objects}.items()) + deleted = [] + if args.apply: + for key in orphan: + await s3.delete(key) + deleted.append(key) + return { + "mode": "apply" if args.apply else "dry-run", + "grace_seconds": args.grace_seconds, + "s3_objects": len(objects), + "metadata_rows": len(metadata), + "orphan_candidates": orphan, + "fresh_unreferenced": fresh, + "missing_objects": [{"attachment_id": item[0], "object_key": item[1]} for item in missing], + "deleted": deleted, + } + finally: + s3.client.close() + await db.close() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="delete only aged orphan objects") + parser.add_argument("--grace-seconds", type=int, default=3600) + args = parser.parse_args() + if not 0 <= args.grace_seconds <= 7 * 24 * 3600: + parser.error("--grace-seconds must be 0..604800") + print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/src/storage/postgres.py b/src/storage/postgres.py index 42695ec4..465f1ba9 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -324,3 +324,11 @@ class PostgresStore: if row is None: raise ApiError(404, "not_found", "Вложение не найдено") return dict(row) + + async def attachment_keys(self) -> dict[str, UUID]: + """Inventory used only by the maintenance reconciler.""" + try: + rows = await self.pool.fetch("SELECT object_key, id FROM attachments") + except DB_ERRORS as error: + raise RuntimeError("PostgreSQL is unavailable") from error + return {row["object_key"]: row["id"] for row in rows} diff --git a/src/storage/s3.py b/src/storage/s3.py index 1fea6569..0df4c498 100644 --- a/src/storage/s3.py +++ b/src/storage/s3.py @@ -69,6 +69,24 @@ class S3Store: await self.call("put_object", Bucket=self.settings.bucket, Key=key, Body=file, ContentLength=size, ContentType=content_type) + async def list(self, prefix: str) -> dict[str, object]: + """Return every object under a controlled prefix, handling S3 pagination.""" + result: dict[str, object] = {} + token = None + while True: + args = {"Bucket": self.settings.bucket, "Prefix": prefix} + if token: + args["ContinuationToken"] = token + page = await self.call("list_objects_v2", **args) + for item in page.get("Contents", []): + result[item["Key"]] = item["LastModified"] + if not page.get("IsTruncated"): + return result + token = page["NextContinuationToken"] + + async def delete(self, key: str) -> None: + await self.call("delete_object", Bucket=self.settings.bucket, Key=key) + async def download(self, attachment): response = await self.call("get_object", Bucket=self.settings.bucket, Key=attachment["object_key"]) diff --git a/student-tests/reconcile.py b/student-tests/reconcile.py new file mode 100644 index 00000000..24782d53 --- /dev/null +++ b/student-tests/reconcile.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Inject S3/DB disagreement and prove dry-run, grace period and safe repair.""" +import json +import base64 +from pathlib import Path +import subprocess +import sys +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, arguments, check, run + + +def compose(*args, input=None, env=None) -> str: + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, + input=input, env=env) + + +def reconcile(*args) -> dict: + output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", *args) + return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{"))) + + +def s3_code(action: str, key: str, data: bytes | None = None) -> None: + code = """from src.storage.s3 import S3Settings, S3Store +import base64, json, sys +s3 = S3Store(S3Settings.load()) +payload = json.load(sys.stdin) +key = payload['key'] +try: + if sys.argv[1] == 'put': + s3.client.put_object(Bucket=s3.settings.bucket, Key=key, + Body=base64.b64decode(payload.get('data', 'b3JwaGFu'))) + else: + s3.client.delete_object(Bucket=s3.settings.bucket, Key=key) +finally: + s3.client.close() +""" + payload = {"key": key} + if data is not None: + payload["data"] = base64.b64encode(data).decode() + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action, + input=json.dumps(payload)) + + +def wait_ready(client: Client, env=None) -> None: + subprocess.run(["docker", "compose", "up", "-d", "--wait", "--no-deps", "api-a"], + cwd=ROOT, env=env, check=True, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 15 + while True: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure: + check(time.monotonic() < deadline, "stack did not recover") + time.sleep(.5) + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=30) + alice, bob, _eve, chat = client.bootstrap() + image = (ROOT / "tests/fixtures/sample.png").read_bytes() + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + + def upload(name, expected=201): + return client.call("POST", path + name, expected, actor=alice, raw=image, + headers={"Content-Type": "image/png"}, + schema="Attachment" if expected == 201 else None) + + # A reference object must survive an apply sweep, even with zero grace. + protected = upload("protected.png")["json"] + orphan = "originals/test-orphan-" + uuid.uuid4().hex + s3_code("put", orphan) + + report = reconcile("--grace-seconds", "3600") + check(orphan in report["fresh_unreferenced"], "fresh orphan missing from dry-run") + check(not report["deleted"], "dry-run deleted an object") + report = reconcile("--apply", "--grace-seconds", "3600") + check(orphan in report["fresh_unreferenced"] and not report["deleted"], "grace period ignored") + report = reconcile("--apply", "--grace-seconds", "0") + check(report["deleted"] == [orphan], "aged orphan was not deleted safely") + check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, + "reconciler deleted a referenced object") + + # This models loss after metadata has committed: report it, do not delete metadata. + missing = upload("missing.png")["json"] + missing_key = "originals/" + missing["id"] + s3_code("delete", missing_key) + report = reconcile("--grace-seconds", "0") + expected = {"attachment_id": missing["id"], "object_key": missing_key} + check(expected in report["missing_objects"], "missing metadata/object pair not reported") + client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", 503, actor=bob) + s3_code("put", missing_key, image) + check(client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", actor=bob)["bytes"] == image, + "test object was not restored") + print("PASS: dry-run, grace period, orphan deletion and missing-object reporting", flush=True) + + # Inject a process-equivalent failure after PUT directly into the configured + # replica. The public load balancer may correctly select the healthy peer, + # which would not exercise this boundary. + environment = dict(__import__("os").environ, UPLOAD_FAULT_AFTER_S3_PUT="1") + try: + subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"], + cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) + wait_ready(client, environment) + fault = """import base64, json, sys, urllib.error, urllib.request +payload = json.load(sys.stdin) +request = urllib.request.Request('http://127.0.0.1:8080/api/v1/chats/' + payload['chat_id'] + + '/attachments?filename=fault-after-put.png', data=base64.b64decode(payload['image']), + method='POST', headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'}) +try: + urllib.request.urlopen(request, timeout=30) +except urllib.error.HTTPError as error: + assert error.code == 503, error.code + assert json.load(error)['error']['code'] == 'unavailable' +else: + raise AssertionError('fault injection unexpectedly returned success') +""" + compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", fault, + input=json.dumps({"chat_id": chat["id"], "token": alice["token"], + "image": base64.b64encode(image).decode()})) + report = reconcile("--grace-seconds", "0") + injected = [key for key in report["orphan_candidates"] if key.startswith("originals/")] + check(injected, "injected post-PUT failure did not create a detectable orphan") + report = reconcile("--apply", "--grace-seconds", "0") + check(set(injected).issubset(report["deleted"]), "reconciler did not clean injected orphan") + print("PASS: post-PUT/pre-DB injected failure leaves no permanent orphan", flush=True) + finally: + environment["UPLOAD_FAULT_AFTER_S3_PUT"] = "0" + subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"], + cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/s3_permissions.py b/student-tests/s3_permissions.py new file mode 100644 index 00000000..a9291575 --- /dev/null +++ b/student-tests/s3_permissions.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Prove that the API identity is bucket-scoped and lacks admin operations.""" +import subprocess +from pathlib import Path +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def compose(*args, input=None): + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, + input=input, stderr=subprocess.DEVNULL) + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + data = (ROOT / "tests/fixtures/sample.png").read_bytes() + client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=permission.png", 201, + actor=alice, raw=data, headers={"Content-Type": "image/png"}, schema="Attachment") + bucket = "permission-probe-" + uuid.uuid4().hex + code = """from botocore.exceptions import ClientError +from src.storage.s3 import S3Settings, S3Store +import sys +s3 = S3Store(S3Settings.load()) +try: + try: + s3.client.create_bucket(Bucket=sys.stdin.read().strip()) + except ClientError as error: + assert error.response['Error']['Code'] in {'AccessDenied', 'Unauthorized'}, error.response + else: + raise AssertionError('API S3 identity unexpectedly has admin bucket creation') +finally: + s3.client.close() +""" + compose("exec", "-T", "api-a", "/app/.venv/bin/python", "-c", code, input=bucket) + admin = """from src.storage.s3 import S3Settings, S3Store +import sys +s3 = S3Store(S3Settings.load()) +try: + s3.client.list_buckets() +finally: + s3.client.close() +""" + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", admin) + print("PASS: API identity cannot create buckets; maintenance identity has separate admin access") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..79c3056b --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Ten concurrent bounded uploads, with API-container peak memory sampling.""" +from concurrent.futures import ThreadPoolExecutor +import json +import http.client +from pathlib import Path +import subprocess +import sys +import threading +import time +from urllib.parse import quote +from urllib.parse import urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, check, run + + +def api_memory() -> dict[str, int]: + ids = subprocess.check_output(["docker", "compose", "ps", "-q", "api-a", "api-b"], cwd=ROOT, text=True).split() + return {container[:12]: int(subprocess.check_output( + ["docker", "exec", container, "cat", "/sys/fs/cgroup/memory.current"], text=True)) + for container in ids} + + +def main(): + def configure(parser): + parser.add_argument("--output", default="evidence/upload-memory.json") + parser.add_argument("--size-mib", type=int, default=5) + args = arguments(__doc__, configure) + check(1 <= args.size_mib <= 10, "--size-mib must be 1..10") + client = Client(args.base_url, args.ca_file, timeout=60) + alice, _bob, _eve, chat = client.bootstrap() + seed = (ROOT / "tests/fixtures/sample.png").read_bytes() + payload = seed + b"\0" * (args.size_mib * 1024 * 1024 - len(seed)) + baseline, peak, samples, stop = api_memory(), {}, [], threading.Event() + + def sampler(): + while not stop.is_set(): + values = api_memory() + samples.append(values) + for name, value in values.items(): + peak[name] = max(peak.get(name, 0), value) + time.sleep(.1) + + def upload(index): + parsed = urlsplit(client.base) + connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=60) + + def chunks(): + for offset in range(0, len(payload), 512 * 1024): + yield payload[offset:offset + 512 * 1024] + # Keep all ten client streams open long enough for multiple memory samples. + time.sleep(.5) + + connection.request("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=" + quote(f"load-{index}.png"), + body=chunks(), encode_chunked=True, + headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"}) + response = connection.getresponse() + body = response.read() + connection.close() + check(response.status == 201, "upload %s returned %s: %s" % (index, response.status, body[:200])) + return json.loads(body) + + started = time.monotonic() + thread = threading.Thread(target=sampler, daemon=True) + thread.start() + try: + with ThreadPoolExecutor(max_workers=10) as pool: + attachments = list(pool.map(upload, range(10))) + finally: + stop.set() + thread.join(timeout=5) + elapsed = time.monotonic() - started + check(len({item["id"] for item in attachments}) == 10, "uploads were not distinct") + report = { + "uploads": 10, "file_bytes": len(payload), "concurrency": 10, + "api_upload_concurrency_limit": 1, "elapsed_seconds": round(elapsed, 3), + "baseline_memory_bytes": baseline, "peak_memory_bytes": peak, + "peak_growth_bytes": {name: peak.get(name, 0) - baseline.get(name, 0) for name in baseline}, + "samples": len(samples), + "method": "cgroup memory.current sampled through Docker every 0.1 seconds while clients send chunked 512 KiB blocks every 0.5 seconds; peak is not a continuous profiler.", + } + destination = ROOT / args.output + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(report, ensure_ascii=False)) + check(all(value <= 512 * 1024 * 1024 for value in peak.values()), "API exceeded configured memory limit") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From 483d7eecb38eb4addc99143ad718417dd00582f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Thu, 22 Oct 2026 11:55:41 +0300 Subject: [PATCH 4/5] add level 5 --- Makefile | 5 +- REPORT.md | 16 +++++- evidence/environment.json | 17 ++++--- evidence/reconcile-benchmark.txt | 2 + evidence/reconcile.txt | 4 +- src/reconcile.py | 14 +++++- src/storage/s3.py | 9 +++- student-tests/reconcile.py | 9 +++- student-tests/reconcile_benchmark.py | 74 ++++++++++++++++++++++++++++ 9 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 evidence/reconcile-benchmark.txt create mode 100644 student-tests/reconcile_benchmark.py diff --git a/Makefile b/Makefile index ff530d52..47d3331e 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) -.PHONY: test-attachments test-private test-upload-failover test-s3-outage reconcile reconcile-apply test-reconcile test-upload-memory test-s3-permissions +.PHONY: test-attachments test-private test-upload-failover test-s3-outage reconcile reconcile-apply test-reconcile test-reconcile-benchmark test-upload-memory test-s3-permissions test-attachments: uv run python student-tests/attachments.py --base-url "$(BASE_URL)" @@ -51,6 +51,9 @@ reconcile-apply: test-reconcile: $(PYTHON) student-tests/reconcile.py --base-url "$(BASE_URL)" +test-reconcile-benchmark: + $(PYTHON) student-tests/reconcile_benchmark.py --base-url "$(BASE_URL)" + test-upload-memory: uv run python student-tests/upload_memory.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index 0fb4ad3b..6ac19112 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 4 - Автор, группа: Матвей Козырев, БВТ2301. -- Целевая оценка: **4 — уровень стажёра/джуна**. +- Целевая оценка: **5 — исследование и доказанный результат; трек 2: согласованность PostgreSQL и S3**. - Ветка: `solution/BVT2301-Kozyrev`. - Проверки: 22 октября 2026 года. Базовый commit и SHA-256 исходников проверенного рабочего дерева — [evidence/environment.json](evidence/environment.json). - Использованные библиотеки: FastAPI, Uvicorn, asyncpg, argon2-cffi, boto3, Pillow. Помощь ИИ: Codex — перенос основы lab3, реализация вложений, тесты и документация. @@ -75,6 +75,16 @@ SeaweedFS закреплён digest `sha256:186de7ef977a20343ee9a5544073f081976a Missing object считается наблюдаемой ошибкой, а не поводом удалить метаданные: download вернёт 503, а maintenance output содержит attachment ID и object key. Восстановление байтов требует оператора или резервной копии; автоматическое восстановление не заявляется. Это безопаснее удаления используемого сообщения/метаданных без источника оригинала. +### Трек 2 — гипотеза, методика и результат + +**Гипотеза.** Полный inventory ключей `originals/` в S3 и `attachments.object_key` в PostgreSQL позволяет повторяемо обнаружить оба направления рассогласования. При удалении только aged orphan и при запрете автоматического удаления метаданных свежий или используемый файл не будет удалён maintenance-командой. + +**Методика.** [student-tests/reconcile.py](student-tests/reconcile.py) создаёт используемое вложение, новый orphan и запускает dry-run, apply с grace 3600 с и apply с grace 0. Затем повторяет apply и проверяет, что уже исправленный orphan не появился снова. Ошибка на стороне PostgreSQL моделируется контролируемым отказом API после успешного S3 PUT и до INSERT; следующая сверка обнаруживает orphan и удаляет его. Ошибка на стороне S3 моделируется удалением оригинала после созданной metadata; сверка сообщает пару `attachment_id`/`object_key`, не удаляет metadata, после восстановления байтов следующий обход не сообщает missing. Во всех проверках скачивается защищённое используемое вложение. + +**Стоимость обхода.** [student-tests/reconcile_benchmark.py](student-tests/reconcile_benchmark.py) добавляет 32, затем 128 свежих неиспользуемых объектов с уникальным префиксом. Для каждого размера выполняются три dry-run обхода; после измерения fixtures удаляются. Reconciler возвращает число объектов, metadata-строк, S3 list pages и длительности стадий, поэтому результат не зависит от ручного таймера. + +В зафиксированном прогоне для 32 добавленных объектов проверены 119 S3 keys и 87 metadata-строк, одна S3 page; длительности 0,299804, 0,204790 и 0,321822 с, медиана 0,299804 с. Для 128 добавленных объектов: 215 keys, те же 87 metadata-строк, одна page; 0,311197, 0,301204 и 0,220673 с, медиана 0,301204 с. Полный машинный вывод — [evidence/reconcile-benchmark.txt](evidence/reconcile-benchmark.txt). Шесть локальных замеров недостаточны для вывода о производительности при большом числе объектов или нескольких S3 pages; это измерение стоимости именно данного стенда. + Недоступность S3: upload/download и readiness возвращают 503 в формате Error; liveness остаётся 200. HAProxy исключает unready backend, поэтому при недоступности обеих реплик сам LB может вернуть собственный 503. Проверка формата ошибки API при таком отказе выполняется напрямую внутри Docker-сети. ## Проверки и доказательства @@ -97,6 +107,7 @@ make test-failover ACTION_SCRIPT=scripts/stop-one.sh docker compose start api-a api-b make reconcile make test-reconcile +make test-reconcile-benchmark make test-upload-memory make test-s3-permissions ``` @@ -116,6 +127,7 @@ make test-s3-permissions | Конкурентная идемпотентность и регрессии lab3 | Собственные проверки lab3 | [Результат](evidence/lab3-regressions.txt) | | Повторный bootstrap bucket | `docker compose run --rm --no-deps s3-init` | [Результат](evidence/bootstrap.txt) | | Reconcile: dry-run, grace period, protected object, missing object, post-PUT/pre-DB отказ | `make test-reconcile` | [PASS](evidence/reconcile.txt) | +| Трек 2: три обхода для 32 и 128 добавленных S3-объектов; метрики inventory | `make test-reconcile-benchmark` | [PASS](evidence/reconcile-benchmark.txt) | | Ограниченные S3 credentials | `make test-s3-permissions`: API не создаёт bucket; отдельный maintenance principal выполняет admin list | [PASS](evidence/s3-permissions.txt) | | 10 одновременных потоковых upload по 5 MiB | `make test-upload-memory`, cgroup `memory.current` каждые 0,1 с | [PASS](evidence/upload-memory.txt), [raw JSON](evidence/upload-memory.json) | @@ -123,7 +135,7 @@ make test-s3-permissions ## Ограничения -- Между S3 PUT и COMMIT PostgreSQL нет общей транзакции; orphan закрывает reconciler с grace period, но missing-object запись восстанавливается только оператором/backup. Автоматический repair исходного файла не реализован. +- Между S3 PUT и COMMIT PostgreSQL нет общей транзакции; orphan закрывает reconciler с grace period, но missing-object запись восстанавливается только оператором/backup. Автоматический repair исходного файла не реализован. Inventory выполняет полный обход keyspace `originals/` и таблицы metadata, поэтому его стоимость растёт с их размером. - API буферизует скачивание не более 10 MiB, чтобы проверить checksum до ответа. Эксперимент 10 upload измеряет cgroup memory.current, но не является capacity benchmark и не покрывает произвольные медленные клиенты, filesystem quota и одновременные скачивания. - Readiness проверяет доступность БД и bucket, но не наличие каждого отдельного объекта. Потеря одного объекта при доступном S3 обнаруживается при скачивании как 503. - Перезапуск с volumes и отказ API подтверждены на одном Docker-host. Потеря volumes/host не покрывается, резервное копирование не реализовано. diff --git a/evidence/environment.json b/evidence/environment.json index 0f7d10ba..21ec1b75 100644 --- a/evidence/environment.json +++ b/evidence/environment.json @@ -1,6 +1,6 @@ { - "recorded_at_utc": "2026-10-22T08:32:17.051591+00:00", - "base_commit": "e9ef559c349cf8b6fc32138fed0ea7a7600ce1ca", + "recorded_at_utc": "2026-10-22T08:46:03.568305+00:00", + "base_commit": "901786170792195f6dd95d0939db98236eb932ff", "branch": "solution/BVT2301-Kozyrev", "note": "Results apply to this working tree, not only the base commit. Functional tests; no load benchmark.", "host": "macOS-26.1-arm64-arm-64bit-Mach-O", @@ -12,7 +12,7 @@ "compose_version": "5.5.1", "sources_sha256": { "Dockerfile": "efede941e2ccf21cd1e70ebbf8af7647209bd030b2962f9446bee497a2ed6a8c", - "Makefile": "b417167d9d5baae110b2963a26ec12cea8ad8c96ce3d627a6c9e4fe8f51b74d3", + "Makefile": "3255468d0a33aae6c111e99a51bc22008171a493d69b4a01aecabc49a0a99eb3", "compose.yaml": "b239915436be84c7b1000346669ae9a10c89e83c55555a5e9eac0e20828f6fb6", "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", "infra/filer.toml": "ae3c01bf7708f5d4a319fdb454f753be0f137fe7c31269f1c413cbccc860f5c7", @@ -40,17 +40,18 @@ "src/main.py": "6980cdbb09c3b356ff870ee7fc972d85e835c58228d3c0ea27de8a668dc41ece", "src/migrate.py": "1d164c8921df2b5369e30b1f8936c2b71858c5279070f0fd593f62f66e1f0b7d", "src/models.py": "b94e6b26559afc5fd90f58d34d906c77bf4b4a78c716ac552358795496ad1d1b", - "src/reconcile.py": "67682de753e3331a5bb5c884ab6dabaad458c49e27d4a0f2c849d79b63a7c44b", + "src/reconcile.py": "db7cc30de97f62a172024e1ffa94d75f8e6e91b4e80db8b491ce5c213e37aae7", "src/routes.py": "3c21149a30ea7c2f695547c42133dcebcfbacbd6544bcb73c353be46fa35a2e6", "src/storage/__init__.py": "05ef66318c4c436eaafc493072640817219ab37bce464a265a2306233389064a", "src/storage/postgres.py": "c69fea94937fdf00b077a8f1d301e83f35e120c3a7c5153fe205725be0aa0428", - "src/storage/s3.py": "c51c8a804ca1ed869556c80252e119372696110443df30bd84f0836a01a0c305", + "src/storage/s3.py": "0779e6131c699055a1cff74e62addf207476819252e9a7aaf46c77af6d9044e6", "student-tests/attachment_resilience.py": "c72c431b355dda919c1b6b854d260f8c7420672ca9f86bf0848692da35dcb0ac", "student-tests/attachments.py": "a948fabfe83b73eb8e494c80fb9d1f2a949e2a95b7a21809a05975d65d68fc15", "student-tests/idempotency_concurrency.py": "db5d139009e4660ac1801af45d10e24665aa81cba737731e527c0f5b4152231e", "student-tests/readiness_failover.py": "0ef8355d85be66597d7d44eb7308dcb58c6c9a174d18f360b2d6d22fdcfbed14", "student-tests/readiness_unavailable.py": "46e3f680a3ff9cb79e95568004d97f1752dbdcd88fd4b5f160a9cc38af65a237", - "student-tests/reconcile.py": "298537cda6523631def290e750a651db1eace0647b99c68647c68e0fdb8f34ca", + "student-tests/reconcile.py": "25547bd5d41ff7e6650a7f30f8404cbb9dd8e61e5ab97897e065d82296b6c73a", + "student-tests/reconcile_benchmark.py": "9ae60097cb70e63baf5653559e727e02da246827a256e02580f03b5d47e1ca15", "student-tests/review_regressions.py": "de58f63b2c794c2775247ddd3fccc523870ccbe673ffa404fdd566274d1d41de", "student-tests/s3_permissions.py": "f265a64f900952db79d545e6c058a7605870c2843ed2298f5c68ad4a177663ff", "student-tests/session_expiry.py": "ba8ead4356e4d12c4047ac2b5704d3d3342f71c37b3c13f65a0a9d7c37186ad6", @@ -82,7 +83,7 @@ }, "api-a": { "image": "mtusi2026lab4-api-a", - "image_id": "sha256:679085dc5870799c12f9251cccbd62a1bf4d75e80a9706bc052b57d51d8251e7", + "image_id": "sha256:c6f7a7d42d87e0edbd0b8880e6c4b5b5594dea953154af4ff50158c78db66233", "state": "running", "health": "healthy", "memory_limit_bytes": 536870912, @@ -91,7 +92,7 @@ }, "api-b": { "image": "mtusi2026lab4-api-b", - "image_id": "sha256:0ca4765ed83bf21e802b8b90f2687839d761519271423bc10b2c6d72e29aef48", + "image_id": "sha256:466940ca873740202feb1ee727a2e0d0b2487ae8340df776639dd6f6381189fe", "state": "running", "health": "healthy", "memory_limit_bytes": 536870912, diff --git a/evidence/reconcile-benchmark.txt b/evidence/reconcile-benchmark.txt new file mode 100644 index 00000000..b88752a7 --- /dev/null +++ b/evidence/reconcile-benchmark.txt @@ -0,0 +1,2 @@ +{"method": "three dry-run inventories per controlled size", "results": [{"added_unreferenced_objects": 32, "median_seconds": 0.299804, "metadata_rows_scanned": 87, "runs": 3, "s3_list_pages": 1, "s3_objects_scanned": 119, "seconds": [0.299804, 0.20479, 0.321822]}, {"added_unreferenced_objects": 128, "median_seconds": 0.301204, "metadata_rows_scanned": 87, "runs": 3, "s3_list_pages": 1, "s3_objects_scanned": 215, "seconds": [0.311197, 0.301204, 0.220673]}]} +PASS: reconciler cost measured for 32 and 128 added S3 objects; fixtures removed diff --git a/evidence/reconcile.txt b/evidence/reconcile.txt index ca5fecd8..c84572b3 100644 --- a/evidence/reconcile.txt +++ b/evidence/reconcile.txt @@ -12,7 +12,7 @@ Container mtusi2026lab4-reconcile-run-cd1eb67e6a5f Created Container mtusi2026lab4-s3-init-run-a50b80f3ae78 Creating Container mtusi2026lab4-s3-init-run-a50b80f3ae78 Created -PASS: dry-run, grace period, orphan deletion and missing-object reporting +PASS: S3-side missing object, dry-run, grace period and idempotent orphan repair Container mtusi2026lab4-api-a-1 Recreate Container mtusi2026lab4-api-a-1 Recreated Container mtusi2026lab4-api-a-1 Starting @@ -24,7 +24,7 @@ PASS: dry-run, grace period, orphan deletion and missing-object reporting Container mtusi2026lab4-reconcile-run-1c72990b1ddc Created Container mtusi2026lab4-reconcile-run-2e902a56554d Creating Container mtusi2026lab4-reconcile-run-2e902a56554d Created -PASS: post-PUT/pre-DB injected failure leaves no permanent orphan +PASS: PostgreSQL-side post-PUT/pre-DB failure is repaired without deleting references Container mtusi2026lab4-api-a-1 Recreate Container mtusi2026lab4-api-a-1 Recreated Container mtusi2026lab4-api-a-1 Starting diff --git a/src/reconcile.py b/src/reconcile.py index a2016448..bc944b91 100644 --- a/src/reconcile.py +++ b/src/reconcile.py @@ -9,6 +9,7 @@ import argparse import asyncio from datetime import datetime, timedelta, timezone import json +from time import perf_counter from .config import load_settings from .storage.postgres import PostgresStore @@ -21,7 +22,11 @@ async def run(args) -> dict: s3 = S3Store(S3Settings.load()) await db.connect() try: - objects, metadata = await s3.list("originals/"), await db.attachment_keys() + started = perf_counter() + objects, s3_pages = await s3.list_with_pages("originals/") + listed_at = perf_counter() + metadata = await db.attachment_keys() + metadata_at = perf_counter() cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) orphan = sorted(key for key, modified in objects.items() if key not in metadata and modified <= cutoff) @@ -34,6 +39,7 @@ async def run(args) -> dict: for key in orphan: await s3.delete(key) deleted.append(key) + finished = perf_counter() return { "mode": "apply" if args.apply else "dry-run", "grace_seconds": args.grace_seconds, @@ -43,6 +49,12 @@ async def run(args) -> dict: "fresh_unreferenced": fresh, "missing_objects": [{"attachment_id": item[0], "object_key": item[1]} for item in missing], "deleted": deleted, + "scan_metrics": { + "s3_list_pages": s3_pages, + "s3_list_seconds": round(listed_at - started, 6), + "postgres_metadata_seconds": round(metadata_at - listed_at, 6), + "total_seconds": round(finished - started, 6), + }, } finally: s3.client.close() diff --git a/src/storage/s3.py b/src/storage/s3.py index 0df4c498..bb6bd2e6 100644 --- a/src/storage/s3.py +++ b/src/storage/s3.py @@ -71,17 +71,24 @@ class S3Store: async def list(self, prefix: str) -> dict[str, object]: """Return every object under a controlled prefix, handling S3 pagination.""" + objects, _ = await self.list_with_pages(prefix) + return objects + + async def list_with_pages(self, prefix: str) -> tuple[dict[str, object], int]: + """Return inventory and number of S3 ListObjectsV2 pages consumed.""" result: dict[str, object] = {} token = None + pages = 0 while True: args = {"Bucket": self.settings.bucket, "Prefix": prefix} if token: args["ContinuationToken"] = token page = await self.call("list_objects_v2", **args) + pages += 1 for item in page.get("Contents", []): result[item["Key"]] = item["LastModified"] if not page.get("IsTruncated"): - return result + return result, pages token = page["NextContinuationToken"] async def delete(self, key: str) -> None: diff --git a/student-tests/reconcile.py b/student-tests/reconcile.py index 24782d53..e2e7e3b7 100644 --- a/student-tests/reconcile.py +++ b/student-tests/reconcile.py @@ -82,6 +82,9 @@ def main(): check(orphan in report["fresh_unreferenced"] and not report["deleted"], "grace period ignored") report = reconcile("--apply", "--grace-seconds", "0") check(report["deleted"] == [orphan], "aged orphan was not deleted safely") + report = reconcile("--apply", "--grace-seconds", "0") + check(orphan not in report["deleted"] and orphan not in report["orphan_candidates"], + "second repair run was not idempotent") check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, "reconciler deleted a referenced object") @@ -96,7 +99,9 @@ def main(): s3_code("put", missing_key, image) check(client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", actor=bob)["bytes"] == image, "test object was not restored") - print("PASS: dry-run, grace period, orphan deletion and missing-object reporting", flush=True) + report = reconcile("--grace-seconds", "0") + check(expected not in report["missing_objects"], "restored object remained missing on second scan") + print("PASS: S3-side missing object, dry-run, grace period and idempotent orphan repair", flush=True) # Inject a process-equivalent failure after PUT directly into the configured # replica. The public load balancer may correctly select the healthy peer, @@ -127,7 +132,7 @@ else: check(injected, "injected post-PUT failure did not create a detectable orphan") report = reconcile("--apply", "--grace-seconds", "0") check(set(injected).issubset(report["deleted"]), "reconciler did not clean injected orphan") - print("PASS: post-PUT/pre-DB injected failure leaves no permanent orphan", flush=True) + print("PASS: PostgreSQL-side post-PUT/pre-DB failure is repaired without deleting references", flush=True) finally: environment["UPLOAD_FAULT_AFTER_S3_PUT"] = "0" subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"], diff --git a/student-tests/reconcile_benchmark.py b/student-tests/reconcile_benchmark.py new file mode 100644 index 00000000..358ba405 --- /dev/null +++ b/student-tests/reconcile_benchmark.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Measure repeatable reconciler inventory cost for two controlled S3 sizes.""" +import json +from pathlib import Path +import statistics +import subprocess +import sys +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import arguments, check, run + + +def compose(*args, input=None) -> str: + return subprocess.check_output(["docker", "compose", *args], cwd=ROOT, text=True, input=input) + + +def reconcile() -> dict: + output = compose("--profile", "maintenance", "run", "--rm", "--no-deps", "reconcile", + "--grace-seconds", "3600") + return json.loads(next(line for line in reversed(output.splitlines()) if line.startswith("{"))) + + +def mutate(action: str, keys: list[str]) -> None: + code = """from src.storage.s3 import S3Settings, S3Store +import json, sys +s3 = S3Store(S3Settings.load()) +try: + for key in json.load(sys.stdin): + if sys.argv[1] == 'put': + s3.client.put_object(Bucket=s3.settings.bucket, Key=key, Body=b'track-2-benchmark') + else: + s3.client.delete_object(Bucket=s3.settings.bucket, Key=key) +finally: + s3.client.close() +""" + compose("run", "--rm", "--no-deps", "s3-init", "/app/.venv/bin/python", "-c", code, action, + input=json.dumps(keys)) + + +def main() -> None: + arguments(__doc__) # Keeps the common --base-url/--ca-file command interface. + results = [] + for requested in (32, 128): + prefix = f"originals/track2-benchmark-{uuid.uuid4().hex}-" + keys = [prefix + str(index) for index in range(requested)] + try: + mutate("put", keys) + reports = [reconcile() for _ in range(3)] + for report in reports: + check(set(keys).issubset(report["fresh_unreferenced"]), + "benchmark objects were not included in the inventory") + check(not set(keys) & set(report["orphan_candidates"]), + "grace period classified fresh benchmark objects as deletable") + seconds = [report["scan_metrics"]["total_seconds"] for report in reports] + results.append({ + "added_unreferenced_objects": requested, + "runs": 3, + "s3_objects_scanned": reports[-1]["s3_objects"], + "metadata_rows_scanned": reports[-1]["metadata_rows"], + "s3_list_pages": reports[-1]["scan_metrics"]["s3_list_pages"], + "seconds": seconds, + "median_seconds": round(statistics.median(seconds), 6), + }) + finally: + mutate("delete", keys) + print(json.dumps({"method": "three dry-run inventories per controlled size", "results": results}, + sort_keys=True)) + print("PASS: reconciler cost measured for 32 and 128 added S3 objects; fixtures removed") + + +if __name__ == "__main__": + run(main) -- 2.54.0 From c84d112f0dccdc9c12c9435dc8f8febb228a86a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=82=D0=B2=D0=B5=D0=B9?= Date: Thu, 22 Oct 2026 16:38:54 +0300 Subject: [PATCH 5/5] refactoring --- .github/pull_request_template.md | 29 +++- Makefile | 7 + REPORT.md | 55 ++++--- evidence/audit/attachments.txt | 5 + evidence/audit/bootstrap.txt | 3 + evidence/audit/compose.txt | 0 evidence/audit/contracts.txt | 4 + evidence/audit/environment.txt | 1 + evidence/audit/final-smoke.txt | 2 + evidence/audit/idempotency.txt | 1 + evidence/audit/persistence.txt | 5 + evidence/audit/private-bucket.txt | 2 + evidence/audit/public-failover.txt | 4 + evidence/audit/reconcile-audit.txt | 8 + evidence/audit/reconcile-benchmark.txt | 23 +++ evidence/audit/reconcile.txt | 40 +++++ evidence/audit/regressions.txt | 1 + evidence/audit/restore-replicas.txt | 40 +++++ evidence/audit/results.json | 201 +++++++++++++++++++++++++ evidence/audit/s3-outage.txt | 3 + evidence/audit/s3-permissions.txt | 2 + evidence/audit/smoke.txt | 2 + evidence/audit/upload-failover.txt | 2 + evidence/audit/upload-memory.txt | 2 + evidence/environment.json | 24 +-- evidence/upload-memory.json | 48 ++++-- scripts/audit.py | 70 +++++++++ src/attachments.py | 22 +-- src/reconcile.py | 24 ++- src/storage/postgres.py | 26 +++- student-tests/attachment_resilience.py | 7 +- student-tests/reconcile.py | 35 +++-- student-tests/reconcile_audit.py | 159 +++++++++++++++++++ student-tests/upload_memory.py | 18 ++- 34 files changed, 791 insertions(+), 84 deletions(-) create mode 100644 evidence/audit/attachments.txt create mode 100644 evidence/audit/bootstrap.txt create mode 100644 evidence/audit/compose.txt create mode 100644 evidence/audit/contracts.txt create mode 100644 evidence/audit/environment.txt create mode 100644 evidence/audit/final-smoke.txt create mode 100644 evidence/audit/idempotency.txt create mode 100644 evidence/audit/persistence.txt create mode 100644 evidence/audit/private-bucket.txt create mode 100644 evidence/audit/public-failover.txt create mode 100644 evidence/audit/reconcile-audit.txt create mode 100644 evidence/audit/reconcile-benchmark.txt create mode 100644 evidence/audit/reconcile.txt create mode 100644 evidence/audit/regressions.txt create mode 100644 evidence/audit/restore-replicas.txt create mode 100644 evidence/audit/results.json create mode 100644 evidence/audit/s3-outage.txt create mode 100644 evidence/audit/s3-permissions.txt create mode 100644 evidence/audit/smoke.txt create mode 100644 evidence/audit/upload-failover.txt create mode 100644 evidence/audit/upload-memory.txt create mode 100644 scripts/audit.py create mode 100644 student-tests/reconcile_audit.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..605f8d5f 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,14 +1,27 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Матвей Козырев / БВТ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 2 — согласованность PostgreSQL и S3 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: основа lab3 (две API-реплики, общие сессии, +идемпотентные сообщения); приватные S3-вложения PNG/JPEG, ACL и лимиты; +reconciler с dry-run, grace period и блокировкой параллельного upload/COMMIT. + +Команды запуска из чистого clone: +`python3 scripts/init-secrets.py`, `docker compose up -d --build --wait`, `make test`. +Полные зависимости и порядок аудита — в [REPORT.md](../REPORT.md). + +Результаты проверок: [последовательный аудит](../evidence/audit/results.json), +логи каждого шага в `evidence/audit/`; гипотеза и ограничения трека в REPORT.md. + +Ограничения: один Docker-host; отсутствующий оригинал восстанавливается из +известной копии оператором; полный inventory требует O(N) памяти и на время +сверки задерживает PUT/COMMIT. Измерения RAM дискретные, benchmark локальный. + +Источники и использование ИИ: собственный проект lab3; контракт курса; +SeaweedFS, Pillow, boto3; Codex использован для реализации, тестов и ревью. +Ссылки и закреплённые версии — в REPORT.md. - [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [ ] Контракт и публичные тесты не ослаблены. diff --git a/Makefile b/Makefile index 47d3331e..4ae1889b 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,13 @@ test-reconcile: test-reconcile-benchmark: $(PYTHON) student-tests/reconcile_benchmark.py --base-url "$(BASE_URL)" +.PHONY: test-reconcile-audit audit +test-reconcile-audit: + $(PYTHON) student-tests/reconcile_audit.py --base-url "$(BASE_URL)" + +audit: + $(PYTHON) scripts/audit.py + test-upload-memory: uv run python student-tests/upload_memory.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index 6ac19112..ea113680 100644 --- a/REPORT.md +++ b/REPORT.md @@ -69,9 +69,9 @@ SeaweedFS закреплён digest `sha256:186de7ef977a20343ee9a5544073f081976a ### Согласованность PostgreSQL и S3 -Атомарной транзакции между PostgreSQL и S3 нет. Если PUT прошёл, но процесс или БД отказали до подтверждённого INSERT, API возвращает 503 и намеренно не удаляет объект синхронно: исход COMMIT мог быть неизвестен. Вместо небезопасного удаления применяется [src.reconcile](src/reconcile.py). +Атомарной транзакции между PostgreSQL и S3 нет. При обработанной ошибке после PUT и до подтверждённого COMMIT API возвращает 503; при аварийном завершении процесса клиент может не получить ответ. Приложение намеренно не удаляет объект синхронно: исход COMMIT мог быть неизвестен. Вместо небезопасного удаления применяется [src.reconcile](src/reconcile.py). -`make reconcile` запускает inventory в **dry-run**. Он сравнивает только `originals/` S3 с `attachments.object_key`, сообщает orphan, свежие unreferenced и missing objects. `make reconcile-apply` удаляет только orphan старше 3600 секунд; отсутствующие объекты и все referenced записи не удаляются. Grace period защищает объект при задержанном COMMIT/наблюдении. Для ручной аварийной проверки возможно `docker compose --profile maintenance run --rm --no-deps reconcile --apply --grace-seconds N`; значение 0 применяется только в изолированных тестах. Reconciler использует отдельные admin credentials, API не получает эти секреты. +`make reconcile` запускает inventory в **dry-run**. Он сравнивает только `originals/` S3 с `attachments.object_key`, сообщает orphan, свежие unreferenced и missing objects. `make reconcile-apply` удаляет только orphan старше 3600 секунд; отсутствующие объекты и все referenced записи не удаляются. Возраст объекта отсчитывается от начала обхода. Grace period дополняет общую advisory-блокировку PostgreSQL: upload удерживает shared transaction lock от PUT до COMMIT, а reconciler — exclusive lock на время inventory и удаления. Это исключает гонку с API, завершающим запись. Ожидание блокировки ограничено 5 с; при ошибке БД сверка прекращается. Для ручной аварийной проверки возможно `docker compose --profile maintenance run --rm --no-deps reconcile --apply --grace-seconds N`; значение 0 применяется только в тестах с `--object-key originals/<точный-ключ>`; этот повторяемый флаг ограничивает apply выбранными ключами, сохраняя полный inventory. Reconciler использует отдельные admin credentials, API не получает эти секреты. Missing object считается наблюдаемой ошибкой, а не поводом удалить метаданные: download вернёт 503, а maintenance output содержит attachment ID и object key. Восстановление байтов требует оператора или резервной копии; автоматическое восстановление не заявляется. Это безопаснее удаления используемого сообщения/метаданных без источника оригинала. @@ -79,16 +79,18 @@ Missing object считается наблюдаемой ошибкой, а не **Гипотеза.** Полный inventory ключей `originals/` в S3 и `attachments.object_key` в PostgreSQL позволяет повторяемо обнаружить оба направления рассогласования. При удалении только aged orphan и при запрете автоматического удаления метаданных свежий или используемый файл не будет удалён maintenance-командой. -**Методика.** [student-tests/reconcile.py](student-tests/reconcile.py) создаёт используемое вложение, новый orphan и запускает dry-run, apply с grace 3600 с и apply с grace 0. Затем повторяет apply и проверяет, что уже исправленный orphan не появился снова. Ошибка на стороне PostgreSQL моделируется контролируемым отказом API после успешного S3 PUT и до INSERT; следующая сверка обнаруживает orphan и удаляет его. Ошибка на стороне S3 моделируется удалением оригинала после созданной metadata; сверка сообщает пару `attachment_id`/`object_key`, не удаляет metadata, после восстановления байтов следующий обход не сообщает missing. Во всех проверках скачивается защищённое используемое вложение. +**Методика.** [student-tests/reconcile.py](student-tests/reconcile.py) создаёт используемое вложение, новый orphan и запускает dry-run, apply с grace 3600 с и apply с grace 0. Затем повторяет apply и проверяет, что уже исправленный orphan не появился снова. Граница между хранилищами моделируется контролируемым отказом API после успешного S3 PUT и до INSERT; следующая сверка обнаруживает orphan и удаляет его. Ошибка на стороне S3 моделируется удалением оригинала после созданной metadata; сверка сообщает пару `attachment_id`/`object_key`, не удаляет metadata, после восстановления байтов следующий обход не сообщает missing. Защищённое вложение прикреплено к сообщению; после исправления проверяются история и оригинал. Дополнительно [reconcile_audit.py](student-tests/reconcile_audit.py) временно устанавливает PostgreSQL trigger только для собственного тестового чата: реальный INSERT отвергается после PUT, API возвращает 503, dry-run находит orphan, apply удаляет только выбранный ключ, повторный apply ничего не удаляет. Тот же тест проверяет незавершённый PUT/COMMIT параллельно с apply, обрыв клиентского upload и реальную S3-пагинацию с `MaxKeys=7`. -**Стоимость обхода.** [student-tests/reconcile_benchmark.py](student-tests/reconcile_benchmark.py) добавляет 32, затем 128 свежих неиспользуемых объектов с уникальным префиксом. Для каждого размера выполняются три dry-run обхода; после измерения fixtures удаляются. Reconciler возвращает число объектов, metadata-строк, S3 list pages и длительности стадий, поэтому результат не зависит от ручного таймера. +**Стоимость обхода.** [student-tests/reconcile_benchmark.py](student-tests/reconcile_benchmark.py) добавляет 32, затем 128 свежих неиспользуемых объектов с уникальным префиксом. Для каждого размера выполняются три dry-run обхода; после измерения fixtures удаляются. Reconciler возвращает число объектов, metadata-строк, S3 list pages и длительности стадий. `total_seconds` измеряется после подключения и получения блокировки; запуск контейнера, подключение и ожидание lock в него не входят. Inventory хранит все ключи в памяти: время и RAM имеют порядок O(S3 keys + metadata rows), число LIST зависит от страниц провайдера. Это не измерение предельной пропускной способности. -В зафиксированном прогоне для 32 добавленных объектов проверены 119 S3 keys и 87 metadata-строк, одна S3 page; длительности 0,299804, 0,204790 и 0,321822 с, медиана 0,299804 с. Для 128 добавленных объектов: 215 keys, те же 87 metadata-строк, одна page; 0,311197, 0,301204 и 0,220673 с, медиана 0,301204 с. Полный машинный вывод — [evidence/reconcile-benchmark.txt](evidence/reconcile-benchmark.txt). Шесть локальных замеров недостаточны для вывода о производительности при большом числе объектов или нескольких S3 pages; это измерение стоимости именно данного стенда. +В актуальном прогоне для 32 добавленных объектов проверены 155 S3 keys и 122 metadata-строки, одна S3 page; длительности 0,405214, 0,404481 и 0,406961 с, медиана 0,405214 с. Для 128 добавленных объектов: 251 keys, те же 122 metadata-строки, одна page; 0,406231, 0,304459 и 0,403061 с, медиана 0,403061 с. Медиана — среднее по порядку значение отсортированной тройки. Полный машинный вывод — [evidence/audit/reconcile-benchmark.txt](evidence/audit/reconcile-benchmark.txt). Эти замеры не показывают монотонного роста задержки на малом наборе и не доказывают масштабируемость. Отдельная функциональная проверка пагинации прочитала 113 keys через 17 страниц с `MaxKeys=7` — [лог](evidence/audit/reconcile-audit.txt). -Недоступность S3: upload/download и readiness возвращают 503 в формате Error; liveness остаётся 200. HAProxy исключает unready backend, поэтому при недоступности обеих реплик сам LB может вернуть собственный 503. Проверка формата ошибки API при таком отказе выполняется напрямую внутри Docker-сети. +Недоступность S3: upload/download и readiness возвращают 503 в формате Error; liveness остаётся 200. HAProxy исключает unready backend, поэтому при недоступности обеих реплик сам LB может вернуть собственный 503. Проверка формата ошибки API при таком отказе выполняется напрямую внутри Docker-сети. После запуска S3 повторяется `s3-init`: реальный PUT/DELETE подтверждает восстановление записи, затем проверяются прежний download и новый upload. Один успешный HeadBucket не гарантирует готовность записи. ## Проверки и доказательства +Полный последовательный повтор: `make audit`. [scripts/audit.py](scripts/audit.py) сохраняет фактический stdout/stderr каждого шага и его exit code в [evidence/audit/results.json](evidence/audit/results.json), останавливаясь при первой ошибке. Актуальные результаты находятся в `evidence/audit/`; прежние логи вне этого каталога относятся к предыдущим версиям. При проверке исходников `git diff ec540a1 -- tests contracts README.md` пуст: обязательные публичные файлы не менялись. + Для собственных image-тестов и полной OpenAPI-валидации нужны локальные зависимости: ```sh @@ -107,34 +109,43 @@ make test-failover ACTION_SCRIPT=scripts/stop-one.sh docker compose start api-a api-b make reconcile make test-reconcile +make test-reconcile-audit make test-reconcile-benchmark make test-upload-memory make test-s3-permissions ``` -Сценарии отказов выполняются последовательно на учебном стенде: они временно останавливают контейнеры. Собственные проверки возвращают остановленные сервисы в работу; публичный `test-failover` оставляет реплику остановленной, поэтому последняя команда нужна для восстановления. Все данные синтетические, токены не сохраняются. Тесты добавляют данные; автоматического удаления нет. +Сценарии отказов выполняются последовательно на учебном стенде: они временно останавливают контейнеры. Собственные проверки возвращают остановленные сервисы в работу; публичный `test-failover` оставляет реплику остановленной, поэтому после него нужна команда восстановления реплик. Все данные синтетические, токены не сохраняются. API-тесты добавляют синтетические данные. Reconcile-тесты удаляют только собственные выбранные orphan, а benchmark удаляет свои временные объекты. | Критерий | Команда / наблюдение | Результат | | --- | --- | --- | -| Структура и OpenAPI | `make check validate PYTHON=.venv/bin/python` | [PASS](evidence/contracts.txt) | -| Обязательный API lab4, checksum, ACL | `make test` | [PASS](evidence/smoke.txt) | -| PNG/JPEG, сообщение с вложением, чужой чат, неверный MIME/filename, лимиты, chunked | `make test-attachments PYTHON=.venv/bin/python` | [PASS](evidence/attachments.txt) | -| Приватность bucket вне API | `make test-private`: anonymous GET существующего объекта → 403 | [PASS](evidence/private-bucket.txt) | -| Отказ именно загрузившей API | `make test-upload-failover`: загрузка через api-b, остановка api-b, те же байты через другую реплику | [PASS](evidence/attachment-failover.txt) | -| Недоступность S3 | `make test-s3-outage`: 503, восстановление, прежний оригинал | [PASS](evidence/s3-outage.txt) | -| Перезапуск всех компонентов, сохранение действующей сессии, метаданных и файла | `python3 scripts/verify-persistence.py`; StartedAt каждого сервиса изменился | [PASS](evidence/persistence-level4.txt) | -| Публичный failover, сообщения без дублей и сохранение файла | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | [PASS](evidence/failover-level4.txt) | -| Конкурентная идемпотентность и регрессии lab3 | Собственные проверки lab3 | [Результат](evidence/lab3-regressions.txt) | -| Повторный bootstrap bucket | `docker compose run --rm --no-deps s3-init` | [Результат](evidence/bootstrap.txt) | -| Reconcile: dry-run, grace period, protected object, missing object, post-PUT/pre-DB отказ | `make test-reconcile` | [PASS](evidence/reconcile.txt) | -| Трек 2: три обхода для 32 и 128 добавленных S3-объектов; метрики inventory | `make test-reconcile-benchmark` | [PASS](evidence/reconcile-benchmark.txt) | -| Ограниченные S3 credentials | `make test-s3-permissions`: API не создаёт bucket; отдельный maintenance principal выполняет admin list | [PASS](evidence/s3-permissions.txt) | -| 10 одновременных потоковых upload по 5 MiB | `make test-upload-memory`, cgroup `memory.current` каждые 0,1 с | [PASS](evidence/upload-memory.txt), [raw JSON](evidence/upload-memory.json) | +| Структура и OpenAPI | `make check validate PYTHON=.venv/bin/python` | [PASS](evidence/audit/contracts.txt) | +| Обязательный API lab4, checksum, ACL | `make test` | [PASS](evidence/audit/smoke.txt) | +| PNG/JPEG, сообщение с вложением, чужой чат, неверный MIME/filename, лимиты, chunked | `make test-attachments PYTHON=.venv/bin/python` | [PASS](evidence/audit/attachments.txt) | +| Приватность bucket вне API | `make test-private`: anonymous GET существующего объекта → 403 | [PASS](evidence/audit/private-bucket.txt) | +| Отказ именно загрузившей API | `make test-upload-failover`: загрузка, остановка обслужившей реплики, те же байты через другую реплику | [PASS](evidence/audit/upload-failover.txt) | +| Недоступность S3 | `make test-s3-outage`: 503, восстановление, прежний оригинал | [PASS](evidence/audit/s3-outage.txt) | +| Перезапуск всех компонентов, сохранение действующей сессии, метаданных и файла | `python3 scripts/verify-persistence.py`; StartedAt каждого сервиса изменился | [PASS](evidence/audit/persistence.txt) | +| Публичный failover, сообщения без дублей и сохранение файла | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | [PASS](evidence/audit/public-failover.txt) | +| Конкурентная идемпотентность и регрессии lab3 | Собственные проверки lab3 | [Результат](evidence/audit/idempotency.txt) | +| Повторный bootstrap bucket | `docker compose run --rm --no-deps s3-init` | [Результат](evidence/audit/bootstrap.txt) | +| Reconcile: dry-run, grace period, protected object, missing object, post-PUT/pre-DB отказ | `make test-reconcile` | [PASS](evidence/audit/reconcile.txt) | +| Реальный отказ INSERT, ошибка inventory, гонка PUT/COMMIT, обрыв upload, S3 pagination | `make test-reconcile-audit` | [Лог](evidence/audit/reconcile-audit.txt) | +| Трек 2: три обхода для 32 и 128 добавленных S3-объектов; метрики inventory | `make test-reconcile-benchmark` | [PASS](evidence/audit/reconcile-benchmark.txt) | +| Ограниченные S3 credentials | `make test-s3-permissions`: API не создаёт bucket; отдельный maintenance principal выполняет admin list | [PASS](evidence/audit/s3-permissions.txt) | +| 10 одновременных потоковых upload по 5 MiB | `make test-upload-memory`, cgroup `memory.current`, фактические интервалы опроса в JSON | [PASS](evidence/audit/upload-memory.txt), [raw JSON](evidence/upload-memory.json) | -Точные границы проверены отдельно: PNG с trailing bytes доведён до 10 485 760 байт и скачан без изменений; PNG 8000 × 5000 = 40 000 000 пикселей принят; заголовок 8001 × 5000 = 40 005 000 пикселей отклонён до декодирования. В `upload-memory.json` серия из 10 клиентов по 5 242 880 bytes дала длительность 5,529 с; 22 cgroup-пробы показали peak 105 254 912 и 101 470 208 bytes на двух API. Это на 431 616 000 и 435 400 704 bytes ниже лимита 536 870 912 bytes, соответственно: `536870912 − 105254912` и `536870912 − 101470208`. Измерение дискретное, поэтому это наблюдаемый минимум пика, а не строгая верхняя оценка. +Точные границы проверены отдельно: PNG с trailing bytes доведён до 10 485 760 байт и скачан без изменений; PNG 8000 × 5000 = 40 000 000 пикселей принят; заголовок 8001 × 5000 = 40 005 000 пикселей отклонён до декодирования. Актуальные значения RAM — в [upload-memory.json](evidence/upload-memory.json), вывод запуска — [audit/upload-memory.txt](evidence/audit/upload-memory.txt). Десять клиентов стартуют через barrier; измерение проверяет наличие обеих API, успешность sampler и минимум две пробы. Между завершением опросов есть пауза 0,1 с; реальный интервал включает Docker latency и записывается в `sample_intervals_seconds`. Измерение дискретное: максимальная проба — нижняя оценка истинного пика, а не строгая верхняя граница. + +### Итог аудита уровней 3–5 + +В [results.json](evidence/audit/results.json) завершены 20 шагов, каждый с `exit_code=0`, `complete=true`. Это число включает конфигурацию, восстановление реплик и снимок окружения, а не только тесты. Для повторения нужен запущенный стенд и зависимости из инструкции выше. + +Исправления аудита: синхронизация upload/reconcile; ограничение тестового apply точными ключами; проверка вложения, действительно прикреплённого к сообщению; реальный отказ PostgreSQL INSERT и отказ inventory; корректная обработка ClientDisconnect; write-probe при восстановлении S3; фактические интервалы измерения RAM. Заполнен шаблон PR. Аудит запускался на существующем учебном volume после пересборки образов; отдельный запуск из чистого clone с пустыми volumes в этот прогон не входил. ## Ограничения +- Advisory lock действует для писателей, использующих протокол приложения. Ручные SQL/S3-изменения администратора должны выполняться в maintenance-окне. Длительный полный обход задерживает новые PUT/COMMIT; после 5 с ожидания запрос получает 503. Отдельная общая транзакция между S3 и PostgreSQL отсутствует. - Между S3 PUT и COMMIT PostgreSQL нет общей транзакции; orphan закрывает reconciler с grace period, но missing-object запись восстанавливается только оператором/backup. Автоматический repair исходного файла не реализован. Inventory выполняет полный обход keyspace `originals/` и таблицы metadata, поэтому его стоимость растёт с их размером. - API буферизует скачивание не более 10 MiB, чтобы проверить checksum до ответа. Эксперимент 10 upload измеряет cgroup memory.current, но не является capacity benchmark и не покрывает произвольные медленные клиенты, filesystem quota и одновременные скачивания. - Readiness проверяет доступность БД и bucket, но не наличие каждого отдельного объекта. Потеря одного объекта при доступном S3 обнаруживается при скачивании как 503. diff --git a/evidence/audit/attachments.txt b/evidence/audit/attachments.txt new file mode 100644 index 00000000..95e7b374 --- /dev/null +++ b/evidence/audit/attachments.txt @@ -0,0 +1,5 @@ +uv run python student-tests/attachments.py --base-url "http://localhost:8080" +PASS: PNG/JPEG originals, metadata, attachment-only messages and ACL +PASS: filename, MIME, malformed image, byte/pixel and chunked limits +PASS: attachment scope, duplicates and idempotency normalization +PASS: inclusive 10 MiB and 40 million pixel boundaries diff --git a/evidence/audit/bootstrap.txt b/evidence/audit/bootstrap.txt new file mode 100644 index 00000000..ffc1b940 --- /dev/null +++ b/evidence/audit/bootstrap.txt @@ -0,0 +1,3 @@ + Container mtusi2026lab4-s3-init-run-2b50a47ece12 Creating + Container mtusi2026lab4-s3-init-run-2b50a47ece12 Created +PASS: private S3 bucket ready diff --git a/evidence/audit/compose.txt b/evidence/audit/compose.txt new file mode 100644 index 00000000..e69de29b diff --git a/evidence/audit/contracts.txt b/evidence/audit/contracts.txt new file mode 100644 index 00000000..ed9cd475 --- /dev/null +++ b/evidence/audit/contracts.txt @@ -0,0 +1,4 @@ +.venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/audit/environment.txt b/evidence/audit/environment.txt new file mode 100644 index 00000000..56071020 --- /dev/null +++ b/evidence/audit/environment.txt @@ -0,0 +1 @@ +PASS: sanitized environment and source hashes saved diff --git a/evidence/audit/final-smoke.txt b/evidence/audit/final-smoke.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/audit/final-smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/audit/idempotency.txt b/evidence/audit/idempotency.txt new file mode 100644 index 00000000..aca95f9d --- /dev/null +++ b/evidence/audit/idempotency.txt @@ -0,0 +1 @@ +PASS: concurrent idempotency through load balancer diff --git a/evidence/audit/persistence.txt b/evidence/audit/persistence.txt new file mode 100644 index 00000000..b2d75558 --- /dev/null +++ b/evidence/audit/persistence.txt @@ -0,0 +1,5 @@ +Before: {"api-a": {"container": "20ad79ee2eed", "started_at": "2026-10-22T09:02:05.82757442Z"}, "api-b": {"container": "1af56e498842", "started_at": "2026-10-22T09:03:29.014379Z"}, "load_balancer": {"container": "8cc83af691c9", "started_at": "2026-10-22T08:39:31.728278626Z"}, "postgres": {"container": "b756a119c924", "started_at": "2026-10-22T08:29:42.97697609Z"}, "s3": {"container": "fe8f16b7a24d", "started_at": "2026-10-22T09:05:52.697448136Z"}} +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +After: {"api-a": {"container": "20ad79ee2eed", "started_at": "2026-10-22T09:06:32.459401543Z"}, "api-b": {"container": "1af56e498842", "started_at": "2026-10-22T09:06:32.503877126Z"}, "load_balancer": {"container": "8cc83af691c9", "started_at": "2026-10-22T09:06:32.274093543Z"}, "postgres": {"container": "b756a119c924", "started_at": "2026-10-22T09:06:12.261186631Z"}, "s3": {"container": "fe8f16b7a24d", "started_at": "2026-10-22T09:06:15.296714216Z"}} +PASS: PostgreSQL, S3, both API replicas and LB actually restarted; volumes preserved diff --git a/evidence/audit/private-bucket.txt b/evidence/audit/private-bucket.txt new file mode 100644 index 00000000..2cb5ce1f --- /dev/null +++ b/evidence/audit/private-bucket.txt @@ -0,0 +1,2 @@ +python3 student-tests/attachment_resilience.py --mode private --base-url "http://localhost:8080" +PASS: anonymous S3 GET returns 403 for an existing object diff --git a/evidence/audit/public-failover.txt b/evidence/audit/public-failover.txt new file mode 100644 index 00000000..ba1d5ba1 --- /dev/null +++ b/evidence/audit/public-failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 110, "failed_attempts": 1, "first_recovery_seconds": 2.438} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/audit/reconcile-audit.txt b/evidence/audit/reconcile-audit.txt new file mode 100644 index 00000000..5829e0b8 --- /dev/null +++ b/evidence/audit/reconcile-audit.txt @@ -0,0 +1,8 @@ +python3 student-tests/reconcile_audit.py --base-url "http://localhost:8080" + Container mtusi2026lab4-reconcile-run-15577660c231 Creating + Container mtusi2026lab4-reconcile-run-15577660c231 Created +PASS: failed PostgreSQL inventory aborts apply without deleting the orphan +PASS: actual PostgreSQL INSERT rejection -> 503, dry-run detection, scoped repair, repeat no-op +PASS: apply with zero grace waits for in-flight PUT/COMMIT; committed original survives +PASS: interrupted upload leaves no metadata/object and next upload succeeds +PASS: S3 pagination: 113 keys across 17 pages, no missing/duplicate keys diff --git a/evidence/audit/reconcile-benchmark.txt b/evidence/audit/reconcile-benchmark.txt new file mode 100644 index 00000000..f333e6ef --- /dev/null +++ b/evidence/audit/reconcile-benchmark.txt @@ -0,0 +1,23 @@ +python3 student-tests/reconcile_benchmark.py --base-url "http://localhost:8080" + Container mtusi2026lab4-s3-init-run-6a8baf52639c Creating + Container mtusi2026lab4-s3-init-run-6a8baf52639c Created + Container mtusi2026lab4-reconcile-run-59b1c6ee9b52 Creating + Container mtusi2026lab4-reconcile-run-59b1c6ee9b52 Created + Container mtusi2026lab4-reconcile-run-ee05d61bcec3 Creating + Container mtusi2026lab4-reconcile-run-ee05d61bcec3 Created + Container mtusi2026lab4-reconcile-run-153fed7867b0 Creating + Container mtusi2026lab4-reconcile-run-153fed7867b0 Created + Container mtusi2026lab4-s3-init-run-5d803fce14c6 Creating + Container mtusi2026lab4-s3-init-run-5d803fce14c6 Created + Container mtusi2026lab4-s3-init-run-3c280622f4f2 Creating + Container mtusi2026lab4-s3-init-run-3c280622f4f2 Created + Container mtusi2026lab4-reconcile-run-feb4441d6df4 Creating + Container mtusi2026lab4-reconcile-run-feb4441d6df4 Created + Container mtusi2026lab4-reconcile-run-48539f3d2d44 Creating + Container mtusi2026lab4-reconcile-run-48539f3d2d44 Created + Container mtusi2026lab4-reconcile-run-52134038d85e Creating + Container mtusi2026lab4-reconcile-run-52134038d85e Created + Container mtusi2026lab4-s3-init-run-8524bd7d89a0 Creating + Container mtusi2026lab4-s3-init-run-8524bd7d89a0 Created +{"method": "three dry-run inventories per controlled size", "results": [{"added_unreferenced_objects": 32, "median_seconds": 0.405214, "metadata_rows_scanned": 122, "runs": 3, "s3_list_pages": 1, "s3_objects_scanned": 155, "seconds": [0.405214, 0.404481, 0.406961]}, {"added_unreferenced_objects": 128, "median_seconds": 0.403061, "metadata_rows_scanned": 122, "runs": 3, "s3_list_pages": 1, "s3_objects_scanned": 251, "seconds": [0.406231, 0.304459, 0.403061]}]} +PASS: reconciler cost measured for 32 and 128 added S3 objects; fixtures removed diff --git a/evidence/audit/reconcile.txt b/evidence/audit/reconcile.txt new file mode 100644 index 00000000..f6780fd2 --- /dev/null +++ b/evidence/audit/reconcile.txt @@ -0,0 +1,40 @@ +python3 student-tests/reconcile.py --base-url "http://localhost:8080" + Container mtusi2026lab4-s3-init-run-873e8d2796d3 Creating + Container mtusi2026lab4-s3-init-run-873e8d2796d3 Created + Container mtusi2026lab4-reconcile-run-c2ac112ce56e Creating + Container mtusi2026lab4-reconcile-run-c2ac112ce56e Created + Container mtusi2026lab4-reconcile-run-41147f965ded Creating + Container mtusi2026lab4-reconcile-run-41147f965ded Created + Container mtusi2026lab4-reconcile-run-c9842622f73e Creating + Container mtusi2026lab4-reconcile-run-c9842622f73e Created + Container mtusi2026lab4-reconcile-run-557b0a13c872 Creating + Container mtusi2026lab4-reconcile-run-557b0a13c872 Created + Container mtusi2026lab4-s3-init-run-73474bd49386 Creating + Container mtusi2026lab4-s3-init-run-73474bd49386 Created + Container mtusi2026lab4-reconcile-run-c34b435de6ef Creating + Container mtusi2026lab4-reconcile-run-c34b435de6ef Created + Container mtusi2026lab4-s3-init-run-ee3a098fecf9 Creating + Container mtusi2026lab4-s3-init-run-ee3a098fecf9 Created + Container mtusi2026lab4-reconcile-run-5e20e2eda1ee Creating + Container mtusi2026lab4-reconcile-run-5e20e2eda1ee Created +PASS: S3-side missing object, dry-run, grace period and idempotent orphan repair + Container mtusi2026lab4-reconcile-run-cb994093df21 Creating + Container mtusi2026lab4-reconcile-run-cb994093df21 Created + Container mtusi2026lab4-api-a-1 Recreate + Container mtusi2026lab4-api-a-1 Recreated + Container mtusi2026lab4-api-a-1 Starting + Container mtusi2026lab4-api-a-1 Started + Container mtusi2026lab4-api-a-1 Running + Container mtusi2026lab4-api-a-1 Waiting + Container mtusi2026lab4-api-a-1 Healthy + Container mtusi2026lab4-reconcile-run-9f42ecfc5dfe Creating + Container mtusi2026lab4-reconcile-run-9f42ecfc5dfe Created + Container mtusi2026lab4-reconcile-run-9bb0b9b36ed0 Creating + Container mtusi2026lab4-reconcile-run-9bb0b9b36ed0 Created +PASS: injected pre-INSERT failure repaired; message and used original survive + Container mtusi2026lab4-api-a-1 Recreate + Container mtusi2026lab4-api-a-1 Recreated + Container mtusi2026lab4-api-a-1 Starting + Container mtusi2026lab4-api-a-1 Started + Container mtusi2026lab4-api-a-1 Waiting + Container mtusi2026lab4-api-a-1 Healthy diff --git a/evidence/audit/regressions.txt b/evidence/audit/regressions.txt new file mode 100644 index 00000000..608a144d --- /dev/null +++ b/evidence/audit/regressions.txt @@ -0,0 +1 @@ +PASS: non-ASCII Bearer returns 401/challenge; malformed JSON returns contract Error/400 diff --git a/evidence/audit/restore-replicas.txt b/evidence/audit/restore-replicas.txt new file mode 100644 index 00000000..7058ed8b --- /dev/null +++ b/evidence/audit/restore-replicas.txt @@ -0,0 +1,40 @@ + Container mtusi2026lab4-postgres-1 Running + Container mtusi2026lab4-api-a-1 Running + Container mtusi2026lab4-s3-1 Running + Container mtusi2026lab4-load_balancer-1 Running + Container mtusi2026lab4-s3-1 Waiting + Container mtusi2026lab4-postgres-1 Waiting + Container mtusi2026lab4-s3-1 Healthy + Container mtusi2026lab4-postgres-1 Healthy + Container mtusi2026lab4-s3-init-1 Starting + Container mtusi2026lab4-migrate-1 Starting + Container mtusi2026lab4-s3-init-1 Started + Container mtusi2026lab4-migrate-1 Started + Container mtusi2026lab4-s3-init-1 Waiting + Container mtusi2026lab4-migrate-1 Waiting + Container mtusi2026lab4-migrate-1 Waiting + Container mtusi2026lab4-s3-init-1 Waiting + Container mtusi2026lab4-s3-init-1 Exited + Container mtusi2026lab4-s3-init-1 Exited + Container mtusi2026lab4-migrate-1 Exited + Container mtusi2026lab4-migrate-1 Exited + Container mtusi2026lab4-api-b-1 Starting + Container mtusi2026lab4-api-b-1 Started + Container mtusi2026lab4-api-a-1 Waiting + Container mtusi2026lab4-api-b-1 Waiting + Container mtusi2026lab4-api-a-1 Healthy + Container mtusi2026lab4-api-b-1 Healthy + Container mtusi2026lab4-load_balancer-1 Waiting + Container mtusi2026lab4-migrate-1 Waiting + Container mtusi2026lab4-api-b-1 Waiting + Container mtusi2026lab4-s3-init-1 Waiting + Container mtusi2026lab4-postgres-1 Waiting + Container mtusi2026lab4-api-a-1 Waiting + Container mtusi2026lab4-s3-1 Waiting + Container mtusi2026lab4-s3-1 Healthy + Container mtusi2026lab4-load_balancer-1 Healthy + Container mtusi2026lab4-migrate-1 Exited + Container mtusi2026lab4-s3-init-1 Exited + Container mtusi2026lab4-postgres-1 Healthy + Container mtusi2026lab4-api-a-1 Healthy + Container mtusi2026lab4-api-b-1 Healthy diff --git a/evidence/audit/results.json b/evidence/audit/results.json new file mode 100644 index 00000000..52d83e07 --- /dev/null +++ b/evidence/audit/results.json @@ -0,0 +1,201 @@ +{ + "recorded_at_utc": "2026-10-22T09:10:01.024409+00:00", + "checks": [ + { + "name": "contracts", + "command": [ + "make", + "check", + "validate", + "PYTHON=.venv/bin/python" + ], + "exit_code": 0, + "seconds": 0.529 + }, + { + "name": "compose", + "command": [ + "docker", + "compose", + "config", + "--quiet" + ], + "exit_code": 0, + "seconds": 0.206 + }, + { + "name": "smoke", + "command": [ + "make", + "test" + ], + "exit_code": 0, + "seconds": 2.432 + }, + { + "name": "attachments", + "command": [ + "make", + "test-attachments" + ], + "exit_code": 0, + "seconds": 4.112 + }, + { + "name": "idempotency", + "command": [ + "/Library/Frameworks/Python.framework/Versions/3.14/bin/python3", + "student-tests/idempotency_concurrency.py" + ], + "exit_code": 0, + "seconds": 2.78 + }, + { + "name": "regressions", + "command": [ + "/Library/Frameworks/Python.framework/Versions/3.14/bin/python3", + "student-tests/review_regressions.py" + ], + "exit_code": 0, + "seconds": 0.071 + }, + { + "name": "private-bucket", + "command": [ + "make", + "test-private" + ], + "exit_code": 0, + "seconds": 2.478 + }, + { + "name": "s3-permissions", + "command": [ + "make", + "test-s3-permissions" + ], + "exit_code": 0, + "seconds": 3.223 + }, + { + "name": "bootstrap", + "command": [ + "docker", + "compose", + "run", + "--rm", + "--no-deps", + "s3-init" + ], + "exit_code": 0, + "seconds": 0.749 + }, + { + "name": "upload-failover", + "command": [ + "make", + "test-upload-failover" + ], + "exit_code": 0, + "seconds": 5.878 + }, + { + "name": "public-failover", + "command": [ + "make", + "test-failover", + "ACTION_SCRIPT=scripts/stop-one.sh" + ], + "exit_code": 0, + "seconds": 18.54 + }, + { + "name": "restore-replicas", + "command": [ + "docker", + "compose", + "up", + "-d", + "--wait", + "api-a", + "api-b", + "load_balancer" + ], + "exit_code": 0, + "seconds": 7.856 + }, + { + "name": "s3-outage", + "command": [ + "make", + "test-s3-outage" + ], + "exit_code": 0, + "seconds": 23.474 + }, + { + "name": "persistence", + "command": [ + "/Library/Frameworks/Python.framework/Versions/3.14/bin/python3", + "scripts/verify-persistence.py" + ], + "exit_code": 0, + "seconds": 28.621 + }, + { + "name": "reconcile", + "command": [ + "make", + "test-reconcile" + ], + "exit_code": 0, + "seconds": 61.524 + }, + { + "name": "reconcile-audit", + "command": [ + "make", + "test-reconcile-audit" + ], + "exit_code": 0, + "seconds": 9.908 + }, + { + "name": "upload-memory", + "command": [ + "make", + "test-upload-memory" + ], + "exit_code": 0, + "seconds": 10.33 + }, + { + "name": "reconcile-benchmark", + "command": [ + "make", + "test-reconcile-benchmark" + ], + "exit_code": 0, + "seconds": 30.609 + }, + { + "name": "final-smoke", + "command": [ + "make", + "test" + ], + "exit_code": 0, + "seconds": 2.676 + }, + { + "name": "environment", + "command": [ + "/Library/Frameworks/Python.framework/Versions/3.14/bin/python3", + "scripts/record-environment.py" + ], + "exit_code": 0, + "seconds": 1.082 + } + ], + "complete": true +} diff --git a/evidence/audit/s3-outage.txt b/evidence/audit/s3-outage.txt new file mode 100644 index 00000000..bb7051a8 --- /dev/null +++ b/evidence/audit/s3-outage.txt @@ -0,0 +1,3 @@ +python3 student-tests/attachment_resilience.py --mode s3-outage --base-url "http://localhost:8080" +PASS: S3 outage gives 503 for readiness, download and upload; liveness stays 200 +PASS: original available and new upload accepted after S3 restart/bootstrap diff --git a/evidence/audit/s3-permissions.txt b/evidence/audit/s3-permissions.txt new file mode 100644 index 00000000..beaf1b27 --- /dev/null +++ b/evidence/audit/s3-permissions.txt @@ -0,0 +1,2 @@ +python3 student-tests/s3_permissions.py --base-url "http://localhost:8080" +PASS: API identity cannot create buckets; maintenance identity has separate admin access diff --git a/evidence/audit/smoke.txt b/evidence/audit/smoke.txt new file mode 100644 index 00000000..1e7b9ff4 --- /dev/null +++ b/evidence/audit/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/audit/upload-failover.txt b/evidence/audit/upload-failover.txt new file mode 100644 index 00000000..b53113bf --- /dev/null +++ b/evidence/audit/upload-failover.txt @@ -0,0 +1,2 @@ +python3 student-tests/attachment_resilience.py --mode failover --base-url "http://localhost:8080" +PASS: uploaded via api-b; stopped that container; original downloaded through another replica diff --git a/evidence/audit/upload-memory.txt b/evidence/audit/upload-memory.txt new file mode 100644 index 00000000..66ef3b49 --- /dev/null +++ b/evidence/audit/upload-memory.txt @@ -0,0 +1,2 @@ +uv run python student-tests/upload_memory.py --base-url "http://localhost:8080" +{"uploads": 10, "file_bytes": 5242880, "concurrency": 10, "api_upload_concurrency_limit": 1, "elapsed_seconds": 7.319, "baseline_memory_bytes": {"55cc3bd6323e": 82366464, "1af56e498842": 77262848}, "peak_memory_bytes": {"55cc3bd6323e": 95637504, "1af56e498842": 95907840}, "peak_growth_bytes": {"55cc3bd6323e": 13271040, "1af56e498842": 18644992}, "samples": 29, "sample_intervals_seconds": [0.251, 0.231, 0.246, 0.249, 0.256, 0.252, 0.262, 0.251, 0.232, 0.248, 0.26, 0.248, 0.261, 0.254, 0.262, 0.254, 0.267, 0.248, 0.271, 0.241, 0.236, 0.269, 0.227, 0.256, 0.26, 0.265, 0.271, 0.252], "method": "cgroup memory.current via Docker; 0.1s pause AFTER each poll, actual intervals include Docker latency; ten clients start at a barrier and send chunked 512 KiB blocks every 0.5s. Peak is sampled, not continuous."} diff --git a/evidence/environment.json b/evidence/environment.json index 21ec1b75..95af0815 100644 --- a/evidence/environment.json +++ b/evidence/environment.json @@ -1,6 +1,6 @@ { - "recorded_at_utc": "2026-10-22T08:46:03.568305+00:00", - "base_commit": "901786170792195f6dd95d0939db98236eb932ff", + "recorded_at_utc": "2026-10-22T09:09:59.977765+00:00", + "base_commit": "483d7eecb38eb4addc99143ad718417dd00582f8", "branch": "solution/BVT2301-Kozyrev", "note": "Results apply to this working tree, not only the base commit. Functional tests; no load benchmark.", "host": "macOS-26.1-arm64-arm-64bit-Mach-O", @@ -12,7 +12,7 @@ "compose_version": "5.5.1", "sources_sha256": { "Dockerfile": "efede941e2ccf21cd1e70ebbf8af7647209bd030b2962f9446bee497a2ed6a8c", - "Makefile": "3255468d0a33aae6c111e99a51bc22008171a493d69b4a01aecabc49a0a99eb3", + "Makefile": "2d8be77ed5368450e46a2ef2d67161e245bfaf3a941fd729dbc36012f241bc71", "compose.yaml": "b239915436be84c7b1000346669ae9a10c89e83c55555a5e9eac0e20828f6fb6", "infra/README.md": "0e804fa51debba4c38d3cc84abf4704897665e0335b272060363f03aadd6354f", "infra/filer.toml": "ae3c01bf7708f5d4a319fdb454f753be0f137fe7c31269f1c413cbccc860f5c7", @@ -22,6 +22,7 @@ "migrations/003_idempotency.sql": "d650ea7c1379928a013e035977a77026d2b8b09ad5b2ea97eecb545e7450353a", "migrations/004_attachments.sql": "d760453ba130c94c403f5b04222649fc131ab4d091edd0267ede901fdf82a79e", "pyproject.toml": "d3962edc7a683a62c915c8edec91e85c647c8ae6fe6432603e30b595bd6c3804", + "scripts/audit.py": "5c695c70058b24e975ceca310b2bdc879858081e5ce82b444ea9eab6b9c0d2ed", "scripts/check_contract.py": "7c4402c2a6ab80d3b4676e1866d6ecc87b6daea14c080bcb9be9a7e8b959d671", "scripts/init-secrets.py": "01232befe48b69933b5c6b7e4f29cb19a777030ffab00accc7999afb6c082cef", "scripts/record-environment.py": "716fa7b8edd05d7ab6781fc346338b2254f82186953c6dbf525785c723eee4a1", @@ -32,7 +33,7 @@ "scripts/validate_openapi.py": "4e981acbfc10dac58817bf922d5896e996798551fedc1b2385130e1cddb13d54", "scripts/verify-persistence.py": "6c366bf4d85802bcffe65ae3d45d00736b184033313dabb60b80e1270d8874f1", "src/__init__.py": "01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b", - "src/attachments.py": "553936936318b8096b5a662ddd85dae4043dffb9b4fb6461e60bfc1962914325", + "src/attachments.py": "11cc27c6687b7e54321f1224f74fa0f9f4bab47eb5e2c68db28d2bc9e9659549", "src/bootstrap_s3.py": "2c43f5540e63f004a88e08961a72293546fbc5210f46bc719b90b9f5c0f3f9bd", "src/config.py": "62e214ceef5fcc24be11d23500de0fdaf6cf5efed1d913cdda0921fd34b088c3", "src/cursor.py": "a9192f2445042b8d4ddac57031718ef558990a52eb2922827074479b7f686775", @@ -40,22 +41,23 @@ "src/main.py": "6980cdbb09c3b356ff870ee7fc972d85e835c58228d3c0ea27de8a668dc41ece", "src/migrate.py": "1d164c8921df2b5369e30b1f8936c2b71858c5279070f0fd593f62f66e1f0b7d", "src/models.py": "b94e6b26559afc5fd90f58d34d906c77bf4b4a78c716ac552358795496ad1d1b", - "src/reconcile.py": "db7cc30de97f62a172024e1ffa94d75f8e6e91b4e80db8b491ce5c213e37aae7", + "src/reconcile.py": "83ecb6c83df77a9b5edc9f3e34d9cf54d39400076fec6bc00ee2648f6f393659", "src/routes.py": "3c21149a30ea7c2f695547c42133dcebcfbacbd6544bcb73c353be46fa35a2e6", "src/storage/__init__.py": "05ef66318c4c436eaafc493072640817219ab37bce464a265a2306233389064a", - "src/storage/postgres.py": "c69fea94937fdf00b077a8f1d301e83f35e120c3a7c5153fe205725be0aa0428", + "src/storage/postgres.py": "a1009fc646c570ff181c05a00771c95a40896e0afa034eb5d8cf4d66952ca1a0", "src/storage/s3.py": "0779e6131c699055a1cff74e62addf207476819252e9a7aaf46c77af6d9044e6", - "student-tests/attachment_resilience.py": "c72c431b355dda919c1b6b854d260f8c7420672ca9f86bf0848692da35dcb0ac", + "student-tests/attachment_resilience.py": "8a4ad367263a5d5a597fcd4a2a657c80946d19a3bbedeebdfffbf75f45531e5d", "student-tests/attachments.py": "a948fabfe83b73eb8e494c80fb9d1f2a949e2a95b7a21809a05975d65d68fc15", "student-tests/idempotency_concurrency.py": "db5d139009e4660ac1801af45d10e24665aa81cba737731e527c0f5b4152231e", "student-tests/readiness_failover.py": "0ef8355d85be66597d7d44eb7308dcb58c6c9a174d18f360b2d6d22fdcfbed14", "student-tests/readiness_unavailable.py": "46e3f680a3ff9cb79e95568004d97f1752dbdcd88fd4b5f160a9cc38af65a237", - "student-tests/reconcile.py": "25547bd5d41ff7e6650a7f30f8404cbb9dd8e61e5ab97897e065d82296b6c73a", + "student-tests/reconcile.py": "9c82be28eb5a41b38c9ddd63dbf65d3edc238ca09ffa4612fc4b679d64487173", + "student-tests/reconcile_audit.py": "1bcca4da1030901e45c1369672c8363f9a5b08513c0270dc8a6f73b1f0b74221", "student-tests/reconcile_benchmark.py": "9ae60097cb70e63baf5653559e727e02da246827a256e02580f03b5d47e1ca15", "student-tests/review_regressions.py": "de58f63b2c794c2775247ddd3fccc523870ccbe673ffa404fdd566274d1d41de", "student-tests/s3_permissions.py": "f265a64f900952db79d545e6c058a7605870c2843ed2298f5c68ad4a177663ff", "student-tests/session_expiry.py": "ba8ead4356e4d12c4047ac2b5704d3d3342f71c37b3c13f65a0a9d7c37186ad6", - "student-tests/upload_memory.py": "a1493aaa8ce8e898291a14059826c1405d31a3577027ae926f8c80febf25d3dc", + "student-tests/upload_memory.py": "ed6c013b9e29a826191e1ca904e0546449f0e6ed9ecddbf5aa256d0768f64604", "uv.lock": "fdbeb5ba0c5eedced1a6e5b62c6a57b2d22f293cf77b9487a806c6e1a8b992a0" }, "services": { @@ -83,7 +85,7 @@ }, "api-a": { "image": "mtusi2026lab4-api-a", - "image_id": "sha256:c6f7a7d42d87e0edbd0b8880e6c4b5b5594dea953154af4ff50158c78db66233", + "image_id": "sha256:c658fea891d5262b78c3b417f6417737bd7dd2d204cd3b354bfbb652e0918707", "state": "running", "health": "healthy", "memory_limit_bytes": 536870912, @@ -92,7 +94,7 @@ }, "api-b": { "image": "mtusi2026lab4-api-b", - "image_id": "sha256:466940ca873740202feb1ee727a2e0d0b2487ae8340df776639dd6f6381189fe", + "image_id": "sha256:5f72ab070787d35e2f911f49b33a43da0a0f8148cb3737e0a9285846e308523e", "state": "running", "health": "healthy", "memory_limit_bytes": 536870912, diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json index 0933cb1b..ceeedb75 100644 --- a/evidence/upload-memory.json +++ b/evidence/upload-memory.json @@ -3,19 +3,49 @@ "file_bytes": 5242880, "concurrency": 10, "api_upload_concurrency_limit": 1, - "elapsed_seconds": 5.529, + "elapsed_seconds": 7.319, "baseline_memory_bytes": { - "7b09ecf19049": 84668416, - "1bcbfe033cad": 86630400 + "55cc3bd6323e": 82366464, + "1af56e498842": 77262848 }, "peak_memory_bytes": { - "7b09ecf19049": 105254912, - "1bcbfe033cad": 101470208 + "55cc3bd6323e": 95637504, + "1af56e498842": 95907840 }, "peak_growth_bytes": { - "7b09ecf19049": 20586496, - "1bcbfe033cad": 14839808 + "55cc3bd6323e": 13271040, + "1af56e498842": 18644992 }, - "samples": 22, - "method": "cgroup memory.current sampled through Docker every 0.1 seconds while clients send chunked 512 KiB blocks every 0.5 seconds; peak is not a continuous profiler." + "samples": 29, + "sample_intervals_seconds": [ + 0.251, + 0.231, + 0.246, + 0.249, + 0.256, + 0.252, + 0.262, + 0.251, + 0.232, + 0.248, + 0.26, + 0.248, + 0.261, + 0.254, + 0.262, + 0.254, + 0.267, + 0.248, + 0.271, + 0.241, + 0.236, + 0.269, + 0.227, + 0.256, + 0.26, + 0.265, + 0.271, + 0.252 + ], + "method": "cgroup memory.current via Docker; 0.1s pause AFTER each poll, actual intervals include Docker latency; ten clients start at a barrier and send chunked 512 KiB blocks every 0.5s. Peak is sampled, not continuous." } diff --git a/scripts/audit.py b/scripts/audit.py new file mode 100644 index 00000000..5c15de21 --- /dev/null +++ b/scripts/audit.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Sequential lab3-5 verification; capture actual output and exit status, fail fast.""" +from datetime import datetime, timezone +import argparse +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +DESTINATION = ROOT / "evidence/audit" +CHECKS = [ + ("contracts", ["make", "check", "validate", "PYTHON=.venv/bin/python"]), + ("compose", ["docker", "compose", "config", "--quiet"]), + ("smoke", ["make", "test"]), + ("attachments", ["make", "test-attachments"]), + ("idempotency", [sys.executable, "student-tests/idempotency_concurrency.py"]), + ("regressions", [sys.executable, "student-tests/review_regressions.py"]), + ("private-bucket", ["make", "test-private"]), + ("s3-permissions", ["make", "test-s3-permissions"]), + ("bootstrap", ["docker", "compose", "run", "--rm", "--no-deps", "s3-init"]), + ("upload-failover", ["make", "test-upload-failover"]), + ("public-failover", ["make", "test-failover", "ACTION_SCRIPT=scripts/stop-one.sh"]), + ("restore-replicas", ["docker", "compose", "up", "-d", "--wait", "api-a", "api-b", "load_balancer"]), + ("s3-outage", ["make", "test-s3-outage"]), + ("persistence", [sys.executable, "scripts/verify-persistence.py"]), + ("reconcile", ["make", "test-reconcile"]), + ("reconcile-audit", ["make", "test-reconcile-audit"]), + ("upload-memory", ["make", "test-upload-memory"]), + ("reconcile-benchmark", ["make", "test-reconcile-benchmark"]), + ("final-smoke", ["make", "test"]), + ("environment", [sys.executable, "scripts/record-environment.py"]), +] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--resume-from', choices=[name for name, _ in CHECKS]) + args = parser.parse_args() + DESTINATION.mkdir(parents=True, exist_ok=True) + results = [] + checks = CHECKS + if args.resume_from: + offset = [name for name, _ in CHECKS].index(args.resume_from) + previous = json.loads((DESTINATION / 'results.json').read_text())['checks'] + results = previous[:offset] + if [item['name'] for item in results] != [name for name, _ in CHECKS[:offset]] or any(item['exit_code'] for item in results): + parser.error('resume requires successful evidence for every earlier step') + checks = CHECKS[offset:] + for name, command in checks: + print(f"RUN {name}", flush=True) + started = time.monotonic() + with (DESTINATION / f"{name}.txt").open("w") as stream: + result = subprocess.run(command, cwd=ROOT, stdout=stream, stderr=subprocess.STDOUT) + results.append({"name": name, "command": command, "exit_code": result.returncode, + "seconds": round(time.monotonic() - started, 3)}) + (DESTINATION / "results.json").write_text(json.dumps({ + "recorded_at_utc": datetime.now(timezone.utc).isoformat(), + "checks": results, "complete": len(results) == len(CHECKS), + }, indent=2) + "\n") + print(f"{'PASS' if result.returncode == 0 else 'FAIL'} {name}: exit={result.returncode}", flush=True) + if result.returncode: + print((DESTINATION / f"{name}.txt").read_text()[-8000:], flush=True) + return result.returncode + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/attachments.py b/src/attachments.py index 09bb9702..b01e27c9 100644 --- a/src/attachments.py +++ b/src/attachments.py @@ -9,6 +9,7 @@ import warnings from fastapi import APIRouter, Depends, Query, Request, Response from PIL import Image, UnidentifiedImageError from starlette.concurrency import run_in_threadpool +from starlette.requests import ClientDisconnect from .errors import ApiError from .models import Attachment @@ -81,15 +82,18 @@ async def upload(chat_id: UUID, request: Request, filename: str = Query(...), await run_in_threadpool(inspect_image, file, content_type, s3.settings.max_pixels) attachment_id = uuid4() key = f"originals/{attachment_id}" - await s3.put(key, file, size, content_type) - # Disabled by default. Used only by the failure-injection test to - # model a process crash after S3 accepts the object and before DB. - if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": - raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") - return await store.create_attachment( - attachment_id, chat_id, user["id"], filename, content_type, - size, digest.hexdigest(), key, - ) + async with store.attachment_transaction() as connection: + await s3.put(key, file, size, content_type) + # Disabled by default; models a crash after PUT, before INSERT. + if os.environ.get("UPLOAD_FAULT_AFTER_S3_PUT") == "1": + raise ApiError(503, "unavailable", "Инъецирован отказ после S3 PUT") + attachment = await store.create_attachment( + attachment_id, chat_id, user["id"], filename, content_type, + size, digest.hexdigest(), key, connection=connection, + ) + return attachment # Transaction has committed before 201. + except ClientDisconnect: + raise ApiError(400, "invalid_request", "Загрузка прервана") from None except TimeoutError: raise ApiError(503, "unavailable", "Время загрузки истекло") from None diff --git a/src/reconcile.py b/src/reconcile.py index bc944b91..47dfb5ba 100644 --- a/src/reconcile.py +++ b/src/reconcile.py @@ -20,14 +20,23 @@ async def run(args) -> dict: settings = load_settings() db = PostgresStore(settings.database_url) s3 = S3Store(S3Settings.load()) - await db.connect() try: + await db.connect() + async with db.attachment_transaction(maintenance=True): + return await scan(args, db, s3) + finally: + s3.client.close() + await db.close() + + +async def scan(args, db, s3) -> dict: + # Age is measured at scan start, not after a potentially slow inventory. + cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) started = perf_counter() objects, s3_pages = await s3.list_with_pages("originals/") listed_at = perf_counter() metadata = await db.attachment_keys() metadata_at = perf_counter() - cutoff = datetime.now(timezone.utc) - timedelta(seconds=args.grace_seconds) orphan = sorted(key for key, modified in objects.items() if key not in metadata and modified <= cutoff) fresh = sorted(key for key, modified in objects.items() @@ -35,8 +44,11 @@ async def run(args) -> dict: missing = sorted({str(attachment_id): key for key, attachment_id in metadata.items() if key not in objects}.items()) deleted = [] + selected = set(args.object_key) if args.object_key else None if args.apply: for key in orphan: + if selected is not None and key not in selected: + continue await s3.delete(key) deleted.append(key) finished = perf_counter() @@ -49,6 +61,7 @@ async def run(args) -> dict: "fresh_unreferenced": fresh, "missing_objects": [{"attachment_id": item[0], "object_key": item[1]} for item in missing], "deleted": deleted, + "selected_keys": sorted(selected) if selected is not None else None, "scan_metrics": { "s3_list_pages": s3_pages, "s3_list_seconds": round(listed_at - started, 6), @@ -56,18 +69,19 @@ async def run(args) -> dict: "total_seconds": round(finished - started, 6), }, } - finally: - s3.client.close() - await db.close() def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--apply", action="store_true", help="delete only aged orphan objects") parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--object-key", action="append", default=[], + help="restrict apply to these exact keys; repeatable, inventory stays complete") args = parser.parse_args() if not 0 <= args.grace_seconds <= 7 * 24 * 3600: parser.error("--grace-seconds must be 0..604800") + if any(not key.startswith("originals/") for key in args.object_key): + parser.error("--object-key must start with originals/") print(json.dumps(asyncio.run(run(args)), ensure_ascii=False, sort_keys=True)) diff --git a/src/storage/postgres.py b/src/storage/postgres.py index 465f1ba9..3b707862 100644 --- a/src/storage/postgres.py +++ b/src/storage/postgres.py @@ -1,4 +1,5 @@ from datetime import datetime +from contextlib import asynccontextmanager from hashlib import sha256 import json from pathlib import Path @@ -15,7 +16,7 @@ from ..errors import ApiError ROOT = Path(__file__).resolve().parents[2] MIGRATIONS = ROOT / "migrations" PASSWORD_HASHER = PasswordHasher() -DB_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError) +DB_ERRORS = (asyncpg.PostgresError, asyncpg.InterfaceError, OSError, TimeoutError) class PostgresStore: @@ -301,10 +302,27 @@ class PostgresStore: return [self._message(row) for row in rows] - async def create_attachment(self, attachment_id, chat_id, owner_id, filename, - content_type, size_bytes, digest, object_key) -> dict: + @asynccontextmanager + async def attachment_transaction(self, *, maintenance=False): + """Shared writer / exclusive reconciler lock across all API replicas. + + Writers hold the transaction from before PUT through metadata COMMIT. + Maintenance holds the exclusive lock through inventory and deletion. + All attachment writers must participate in this protocol. + """ try: - row = await self.pool.fetchrow( + async with self.pool.acquire() as connection, connection.transaction(): + await connection.execute("SET LOCAL lock_timeout = '5s'") + lock = "pg_advisory_xact_lock" if maintenance else "pg_advisory_xact_lock_shared" + await connection.execute(f"SELECT {lock}(41004, 1)") + yield connection + except DB_ERRORS as error: + raise ApiError(503, "unavailable", "PostgreSQL недоступен") from error + + async def create_attachment(self, attachment_id, chat_id, owner_id, filename, + content_type, size_bytes, digest, object_key, *, connection=None) -> dict: + try: + row = await (connection or self.pool).fetchrow( "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key) " "VALUES ($1,$2,$3,$4,$5,$6,$7,$8) RETURNING *", attachment_id, chat_id, owner_id, filename, content_type, size_bytes, digest, object_key, diff --git a/student-tests/attachment_resilience.py b/student-tests/attachment_resilience.py index 83ebe368..4f72d9ff 100644 --- a/student-tests/attachment_resilience.py +++ b/student-tests/attachment_resilience.py @@ -106,9 +106,14 @@ print("PASS: S3 outage gives 503 for readiness, download and upload; liveness st input=json.dumps(payload)).strip()) finally: compose('start', 's3') + # HeadBucket/read recovery alone does not prove the write path is ready. + # Use the same write-probing bootstrap as the documented restart hook. + compose('run', '--rm', '--no-deps', 's3-init') wait_ready(c) check(c.call('GET', content_path, actor=bob)['bytes'] == data, 'original lost after S3 outage') - print('PASS: original available again after S3 restart') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=after-recovery.png', + 201, actor=alice, raw=data, headers={'Content-Type': 'image/png'}, schema='Attachment') + print('PASS: original available and new upload accepted after S3 restart/bootstrap') if __name__ == '__main__': diff --git a/student-tests/reconcile.py b/student-tests/reconcile.py index e2e7e3b7..04a8fd7d 100644 --- a/student-tests/reconcile.py +++ b/student-tests/reconcile.py @@ -72,17 +72,18 @@ def main(): # A reference object must survive an apply sweep, even with zero grace. protected = upload("protected.png")["json"] + message = client.message(alice, chat, "protected", attachments=[protected["id"]])["json"] orphan = "originals/test-orphan-" + uuid.uuid4().hex s3_code("put", orphan) report = reconcile("--grace-seconds", "3600") check(orphan in report["fresh_unreferenced"], "fresh orphan missing from dry-run") check(not report["deleted"], "dry-run deleted an object") - report = reconcile("--apply", "--grace-seconds", "3600") + report = reconcile("--apply", "--grace-seconds", "3600", "--object-key", orphan) check(orphan in report["fresh_unreferenced"] and not report["deleted"], "grace period ignored") - report = reconcile("--apply", "--grace-seconds", "0") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan) check(report["deleted"] == [orphan], "aged orphan was not deleted safely") - report = reconcile("--apply", "--grace-seconds", "0") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", orphan) check(orphan not in report["deleted"] and orphan not in report["orphan_candidates"], "second repair run was not idempotent") check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, @@ -92,11 +93,14 @@ def main(): missing = upload("missing.png")["json"] missing_key = "originals/" + missing["id"] s3_code("delete", missing_key) - report = reconcile("--grace-seconds", "0") - expected = {"attachment_id": missing["id"], "object_key": missing_key} - check(expected in report["missing_objects"], "missing metadata/object pair not reported") - client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", 503, actor=bob) - s3_code("put", missing_key, image) + try: + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", missing_key) + expected = {"attachment_id": missing["id"], "object_key": missing_key} + check(expected in report["missing_objects"], "missing metadata/object pair not reported") + client.call("GET", "/api/v1/attachments/" + missing["id"], actor=bob, schema="Attachment") + client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", 503, actor=bob) + finally: + s3_code("put", missing_key, image) check(client.call("GET", "/api/v1/attachments/" + missing["id"] + "/content", actor=bob)["bytes"] == image, "test object was not restored") report = reconcile("--grace-seconds", "0") @@ -107,6 +111,7 @@ def main(): # replica. The public load balancer may correctly select the healthy peer, # which would not exercise this boundary. environment = dict(__import__("os").environ, UPLOAD_FAULT_AFTER_S3_PUT="1") + before_fault = set(reconcile("--grace-seconds", "0")["orphan_candidates"]) try: subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"], cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) @@ -128,14 +133,18 @@ else: input=json.dumps({"chat_id": chat["id"], "token": alice["token"], "image": base64.b64encode(image).decode()})) report = reconcile("--grace-seconds", "0") - injected = [key for key in report["orphan_candidates"] if key.startswith("originals/")] - check(injected, "injected post-PUT failure did not create a detectable orphan") - report = reconcile("--apply", "--grace-seconds", "0") + injected = sorted(set(report["orphan_candidates"]) - before_fault) + check(len(injected) == 1, "fault must create exactly one new orphan; run without other writers") + report = reconcile("--apply", "--grace-seconds", "0", "--object-key", injected[0]) check(set(injected).issubset(report["deleted"]), "reconciler did not clean injected orphan") - print("PASS: PostgreSQL-side post-PUT/pre-DB failure is repaired without deleting references", flush=True) + history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages", actor=bob, schema="MessagePage")["json"] + check(message in history["items"], "message reference was lost") + check(client.call("GET", "/api/v1/attachments/" + protected["id"] + "/content", actor=bob)["bytes"] == image, + "used original was lost") + print("PASS: injected pre-INSERT failure repaired; message and used original survive", flush=True) finally: environment["UPLOAD_FAULT_AFTER_S3_PUT"] = "0" - subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "api-a"], + subprocess.run(["docker", "compose", "up", "-d", "--wait", "--force-recreate", "--no-deps", "api-a"], cwd=ROOT, env=environment, check=True, stdout=subprocess.DEVNULL) diff --git a/student-tests/reconcile_audit.py b/student-tests/reconcile_audit.py new file mode 100644 index 00000000..8c1e0469 --- /dev/null +++ b/student-tests/reconcile_audit.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Real database rejection, concurrent commit, interrupted upload and pagination.""" +import base64 +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, arguments, run + +PROBE = r''' +import asyncio, base64, hashlib, http.client, json, sys, urllib.request, urllib.error +from types import SimpleNamespace +from uuid import UUID, uuid4 +from src.config import load_settings +from src.storage.postgres import PostgresStore +from src.storage.s3 import S3Store, S3Settings +from src.reconcile import run as reconcile, scan + +payload = json.load(sys.stdin) +image = base64.b64decode(payload['image']) +chat_id, user_id = UUID(payload['chat_id']), UUID(payload['user_id']) + +def upload(filename): + req = urllib.request.Request('http://api-a:8080/api/v1/chats/' + str(chat_id) + + '/attachments?filename=' + filename, data=image, method='POST', + headers={'Authorization': 'Bearer ' + payload['token'], 'Content-Type': 'image/png'}) + try: + with urllib.request.urlopen(req, timeout=30) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, json.load(error) + +async def main(): + db = PostgresStore(load_settings().database_url) + s3 = S3Store(S3Settings.load()) + await db.connect() + def args(key): + return SimpleNamespace(apply=True, grace_seconds=0, object_key=[key]) + try: + # A failed metadata inventory must not authorize any S3 deletion. + key = 'originals/audit-inventory-' + uuid4().hex + await s3.call('put_object', Bucket=s3.settings.bucket, Key=key, Body=image) + class BrokenInventory: + async def attachment_keys(self): + # Real SQL failure, not an empty inventory on error. + return await db.pool.fetch('SELECT * FROM audit_missing_table_' + uuid4().hex) + try: + async with db.attachment_transaction(maintenance=True): + try: + await scan(args(key), BrokenInventory(), s3) + except Exception: + pass + else: + raise AssertionError('failed inventory did not abort') + assert key in await s3.list('originals/') + finally: + await s3.delete(key) + print('PASS: failed PostgreSQL inventory aborts apply without deleting the orphan', flush=True) + + # An actual PostgreSQL trigger rejects only this test chat's INSERT. + suffix = uuid4().hex + name = 'audit_reject_' + suffix + before = await s3.list('originals/') + await db.pool.execute(f"""CREATE FUNCTION {name}() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN IF NEW.chat_id = '{chat_id}'::uuid THEN + RAISE EXCEPTION 'synthetic audit rejection' USING ERRCODE = '23514'; + END IF; RETURN NEW; END $$""") + try: + await db.pool.execute(f'CREATE TRIGGER {name} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {name}()') + status, body = await asyncio.to_thread(upload, 'db-failure.png') + assert status == 503 and body['error']['code'] == 'unavailable', (status, body) + assert await db.pool.fetchval('SELECT count(*) FROM attachments WHERE chat_id=$1', chat_id) == 0 + finally: + await db.pool.execute(f'DROP TRIGGER IF EXISTS {name} ON attachments') + await db.pool.execute(f'DROP FUNCTION {name}()') + created = set(await s3.list('originals/')) - set(before) + assert len(created) == 1, created + key = created.pop() + report = await reconcile(SimpleNamespace(apply=False, grace_seconds=0, object_key=[key])) + assert key in report['orphan_candidates'] and not report['deleted'] + assert (await reconcile(args(key)))['deleted'] == [key] + assert not (await reconcile(args(key)))['deleted'] + print('PASS: actual PostgreSQL INSERT rejection -> 503, dry-run detection, scoped repair, repeat no-op', flush=True) + + # Writer holds a shared lock after PUT; cleanup must wait for COMMIT. + identifier = uuid4() + key = 'originals/' + str(identifier) + async with db.attachment_transaction() as connection: + await s3.call('put_object', Bucket=s3.settings.bucket, Key=key, Body=image) + sweep = asyncio.create_task(reconcile(args(key))) + for _ in range(40): + waiting = await db.pool.fetchval("SELECT count(*) FROM pg_locks WHERE locktype='advisory' AND classid=41004 AND objid=1 AND NOT granted") + if waiting: + break + await asyncio.sleep(.05) + assert waiting and not sweep.done(), 'maintenance failed to wait for active writer' + await db.create_attachment(identifier, chat_id, user_id, 'concurrent.png', 'image/png', + len(image), hashlib.sha256(image).hexdigest(), key, connection=connection) + report = await sweep + assert key not in report['deleted'] and key not in report['orphan_candidates'] + row = await db.attachment_for_member(identifier, user_id) + assert await s3.download(row) == image + print('PASS: apply with zero grace waits for in-flight PUT/COMMIT; committed original survives', flush=True) + + # Abrupt client close before all bytes arrive must not persist a file. + before = await s3.list('originals/') + rows = await db.attachment_keys() + def interrupt(): + conn = http.client.HTTPConnection('api-a', 8080, timeout=10) + conn.putrequest('POST', '/api/v1/chats/' + str(chat_id) + '/attachments?filename=interrupted.png') + conn.putheader('Authorization', 'Bearer ' + payload['token']) + conn.putheader('Content-Type', 'image/png') + conn.putheader('Content-Length', str(len(image) + 100000)) + conn.endheaders() + conn.send(image[:32]) + conn.close() + await asyncio.to_thread(interrupt) + await asyncio.sleep(1) + assert await db.attachment_keys() == rows + assert await s3.list('originals/') == before + status, body = await asyncio.to_thread(upload, 'after-disconnect.png') + assert status == 201, (status, body) + print('PASS: interrupted upload leaves no metadata/object and next upload succeeds', flush=True) + + # Exercise pagination on real S3 without adding thousands of fixtures. + original_call = s3.call + async def small_pages(method, **kwargs): + if method == 'list_objects_v2': + kwargs['MaxKeys'] = 7 + return await original_call(method, **kwargs) + expected = await s3.list('originals/') + s3.call = small_pages + actual, pages = await s3.list_with_pages('originals/') + assert actual == expected and pages > 1 + print(f'PASS: S3 pagination: {len(actual)} keys across {pages} pages, no missing/duplicate keys', flush=True) + finally: + s3.client.close() + await db.close() + +asyncio.run(main()) +''' + + +def main(): + args = arguments(__doc__) + client = Client(args.base_url, args.ca_file, timeout=30) + alice, _, _, chat = client.bootstrap() + payload = {"chat_id": chat["id"], "user_id": alice["id"], "token": alice["token"], + "image": base64.b64encode((ROOT / "tests/fixtures/sample.png").read_bytes()).decode()} + subprocess.run(["docker", "compose", "--profile", "maintenance", "run", "--rm", "--no-deps", + "--entrypoint", "/app/.venv/bin/python", "reconcile", "-c", PROBE], + cwd=ROOT, input=json.dumps(payload), text=True, check=True) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py index 79c3056b..1075a4eb 100644 --- a/student-tests/upload_memory.py +++ b/student-tests/upload_memory.py @@ -34,16 +34,27 @@ def main(): seed = (ROOT / "tests/fixtures/sample.png").read_bytes() payload = seed + b"\0" * (args.size_mib * 1024 * 1024 - len(seed)) baseline, peak, samples, stop = api_memory(), {}, [], threading.Event() + check(len(baseline) == 2, "memory experiment requires two running APIs") + sampling_errors = [] + timestamps = [] + barrier = threading.Barrier(10) def sampler(): while not stop.is_set(): - values = api_memory() + try: + values = api_memory() + check(set(values) == set(baseline), "API containers changed during measurement") + except Exception as error: + sampling_errors.append(type(error).__name__) + return + timestamps.append(time.monotonic()) samples.append(values) for name, value in values.items(): peak[name] = max(peak.get(name, 0), value) time.sleep(.1) def upload(index): + barrier.wait(timeout=15) parsed = urlsplit(client.base) connection = http.client.HTTPConnection(parsed.hostname, parsed.port, timeout=60) @@ -73,13 +84,16 @@ def main(): thread.join(timeout=5) elapsed = time.monotonic() - started check(len({item["id"] for item in attachments}) == 10, "uploads were not distinct") + check(not sampling_errors and len(samples) >= 2 and not thread.is_alive(), + "memory sampler failed or did not collect enough samples: " + repr(sampling_errors)) report = { "uploads": 10, "file_bytes": len(payload), "concurrency": 10, "api_upload_concurrency_limit": 1, "elapsed_seconds": round(elapsed, 3), "baseline_memory_bytes": baseline, "peak_memory_bytes": peak, "peak_growth_bytes": {name: peak.get(name, 0) - baseline.get(name, 0) for name in baseline}, "samples": len(samples), - "method": "cgroup memory.current sampled through Docker every 0.1 seconds while clients send chunked 512 KiB blocks every 0.5 seconds; peak is not a continuous profiler.", + "sample_intervals_seconds": [round(b-a, 3) for a, b in zip(timestamps, timestamps[1:])], + "method": "cgroup memory.current via Docker; 0.1s pause AFTER each poll, actual intervals include Docker latency; ten clients start at a barrier and send chunked 512 KiB blocks every 0.5s. Peak is sampled, not continuous.", } destination = ROOT / args.output destination.parent.mkdir(parents=True, exist_ok=True) -- 2.54.0