diff --git a/.env.example b/.env.example index d99d26a2..6358e9b7 100644 --- a/.env.example +++ b/.env.example @@ -6,8 +6,8 @@ APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Redis OR Valkey for sessions (lab2 grade 4 / lab3+): +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 @@ -18,3 +18,4 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +ORPHAN_GRACE_SECONDS=900 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..9b8bed51 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +# Закреплённый runtime-образ (Python интерпретируемый — один stage). +FROM python:3.12.7-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +RUN groupadd --system app && useradd --system --gid app --home /app --shell /usr/sbin/nologin app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=5 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + +# PID 1 = uvicorn; SIGTERM обрабатывается lifespan'ом приложения. +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/PR_BODY.md b/PR_BODY.md new file mode 100644 index 00000000..7a2f056e --- /dev/null +++ b/PR_BODY.md @@ -0,0 +1,22 @@ +## Лабораторная 4 + +Автор / группа: Соболев Даниил Романович, БПИ2302 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: нет + +Что реализовано и что перенесено: перенос lab3 (LB, ≥2 API, Postgres, Redis, идемпотентность). Добавлены RustFS (private bucket), метаданные Attachment в Postgres, upload/download, `attachment_ids` в Message, orphan compensate/reaper, own-тесты и evidence. + +Команды запуска из чистого clone: см. `REPORT.md` (`make_secrets` → `compose up --scale api=2` → smoke/own/resilience). + +Результаты проверок и ссылка на REPORT.md: `evidence/*.txt`, подробности в `REPORT.md`. + +Ограничения / известные ошибки: один host/S3/Postgres; download через API-proxy; трек 5 не заявлен. + +Источники и использование ИИ: FastAPI, boto3, Pillow, RustFS, HAProxy; ИИ при S3/compose. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..d3fa271e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,115 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Соболев Даниил Романович, БПИ2302 +- Целевая оценка; выбранный трек 5, если нужен: 4 +- Commit/PR: ветка `solution/bpi2302-sobolev` (после push) +- Использованные источники, библиотеки и помощь ИИ: FastAPI, uvicorn, psycopg, redis, argon2-cffi, boto3, Pillow, RustFS, HAProxy; перенос решения lab3; помощь ИИ при S3/вложениях и compose ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Prerequisites:** Docker Desktop / Compose; Python 3.9+ на хосте для `tests/`. + +```sh +cp .env.example .env +powershell -File scripts/make_secrets.ps1 # или sh scripts/make_secrets.sh + +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 +``` + +Публичная точка входа: **только LB** `http://127.0.0.1:8080` (порты API наружу не опубликованы). +S3 (RustFS): `127.0.0.1:9000` только для проверки private bucket с хоста; **консоль RustFS не включена** (`RUSTFS_CONSOLE_ENABLE` нет). + +Ожидание готовности: `GET http://127.0.0.1:8080/health/ready` → `{"status":"ok"}` (Postgres + Redis + S3). +Миграции — one-shot сервис `migrate` (`RUN_MIGRATIONS=true`); у API `RUN_MIGRATIONS=false`. +Bucket `messenger-attachments` создаётся идемпотентно в API (`S3Store.ensure_bucket`). + +```sh +# Windows (UTF-8 для contracts/openapi.json): +set PYTHONUTF8=1 +py -3 tests\smoke.py --base-url http://localhost:8080 +py -3 tests\own.py --base-url http://localhost:8080 +py -3 tests\resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://localhost:8080 +py -3 tests\resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://localhost:8080 +# после failover поднять остановленную реплику: +scripts\start-stopped-api.cmd +``` + +Остановка (volumes сохраняются): `docker compose --env-file .env down` +Сброс учебных данных: `docker compose --env-file .env down -v` + +**Версии образов:** `python:3.12.7-slim-bookworm`, `postgres:16.4-bookworm`, `redis:7.4.1-bookworm`, `haproxy:2.9.7-bookworm`, `rustfs/rustfs:1.0.5`. + +Ручная сверка orphan (dry-run / apply): + +```sh +docker compose --env-file .env run --rm api python -m app.reconcile +docker compose --env-file .env run --rm api python -m app.reconcile --apply +``` + +Секреты только в `secrets/` (не в Git): Postgres, Redis URL, S3 access/secret. Создание — `scripts/make_secrets.*`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +Клиент → HAProxy (:8080) → API×2+ → PostgreSQL (метаданные + idempotency + attachments) + ↘ Redis (общие сессии, AOF everysec) + ↘ RustFS private bucket (байты; object key назначает сервер) +``` + +**Перенос lab3:** LB, ≥2 API-реплики, Postgres, Redis-сессии, Argon2id, Basic→Bearer, `Idempotency-Key`, `X-Instance-Id`, hooks `restart` / `stop-one`. + +**lab4 — вложения:** +- Таблицы `attachments`, `message_attachments` (миграция `003_attachments.sql`). +- Object key: `attachments/{chat_id}/{attachment_id}`; `filename` только для отображения (запрет `/`, `\`, NUL, control). +- `POST .../attachments?filename=`: сырые PNG/JPEG, не multipart; ≤10 MiB; MIME по magic + Pillow; ≤40M пикселей → 413; подмена/мусор → 415. +- Потоковое чтение тела (`app/upload.py`); при превышении лимита поток дренируется, клиент получает 413 (не connection reset). +- HTTP middleware без буферизации всего тела (`@app.middleware("http")`, не `BaseHTTPMiddleware`). +- Порядок записи: S3 `PutObject` → INSERT метаданных; при ошибке БД — компенсирующий `DeleteObject`. +- Orphan reaper: фоновый поток + CLI `python -m app.reconcile`; удаляет ключи без строки в БД только после `ORPHAN_GRACE_SECONDS` (900 с). +- Credentials: отдельные нестандартные access/secret для API (не built-in defaults RustFS); консоль выключена; anonymous GET объекта с `:9000` → 403/404. +- `/health/ready` пингует Postgres, Redis и S3 (`head_bucket`). +- Сообщение: `attachment_ids` (0–10); `text=""` допустим при ≥1 ready-вложении того же чата; чужой чат/id → 404. + +Соответствие `.env.example`: `SESSION_STORE=redis`, `S3_ENDPOINT=http://s3:9000`, лимиты upload/pixels, `ORPHAN_GRACE_SECONDS`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Дата прогона: 2026-11-12. Стенд: Docker Desktop, `--scale api=2`, mem/cpu limits из `compose.yaml`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke lab4 | `py -3 tests\smoke.py --base-url http://localhost:8080` | `PASS: lab4 public API scenario` | `evidence/smoke.txt` | +| ≥2 instance id (lab3) | `tests\own.py` | instances `01e0dbb41bb1`, `b9570a5a3c7c` | `evidence/own.txt` | +| Upload/download/message + checksum | own happy path | PASS | `evidence/own.txt` | +| >10 MiB / битый / MIME spoof / evil filename | own limits | 413 / 415 / 415 / 400 | `evidence/own.txt` | +| Чужой пользователь / чужой chat attachment | own ACL | 404 | `evidence/own.txt` | +| Private bucket вне API | anonymous GET `127.0.0.1:9000/...` | 403/404; download через API OK | `evidence/own.txt` | +| ≥10 concurrent uploads | own | 10×201 за ~2.48 с | `evidence/own.txt` | +| Persistence (+оригинал) | `resilience.py --mode persistence` + `restart.cmd` | PASS session+данные+файл | `evidence/persistence.txt` | +| Failover (+download с другой реплики) | `resilience.py --mode failover` + `stop-one.cmd` | PASS; `first_recovery_seconds`: **6.14** | `evidence/failover.txt` | + +Перезапуск persistence: `docker compose --env-file .env restart` (все сервисы стека, volumes не удаляются). +Failover: останавливается одна API-реплика (`stop-one.cmd`); трафик обслуживает вторая. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Ожидание | Что наблюдали | Восстановление / данные | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| 10 параллельных upload | все 201 | PASS ~2.48 с; API `mem_limit` 384m | без потери | +| Файл >10 MiB | 413 без 500 | PASS | — | +| Битый PNG / MIME spoof | 415 | PASS | — | +| Eve → meta/content | 404 | PASS | — | +| attachment другого чата в message | 404 | PASS | — | +| Anonymous S3 GET | отказ | 403/404 | — | +| `restart` всего стека | сессия + attachment | persistence PASS | volumes сохранены | +| stop одной API | ≤10 с, файл с другой реплики | recovery **6.14 с** | данные OK | + +Гипотеза concurrent: при потоковом лимите и mem_limit 384m десять небольших PNG не упираются в OOM. Измерение — wall time own-теста; пик RSS контейнера смотреть `docker stats` на защите при необходимости. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host / один RustFS / один Postgres — не отказоустойчивость хранилищ. +- Download идёт прокси через API (без redirect) — нагрузка на RAM/сеть API; трек 5.1 (presigned) снизит её. +- Grace orphan: свежие объекты не удаляются; слишком короткий grace опасен при медленной записи в БД. +- Трек 5 не заявлен. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..3fca08ca --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger API lab2: PostgreSQL + Bearer sessions + Redis.""" diff --git a/app/api.py b/app/api.py new file mode 100644 index 00000000..5d33a771 --- /dev/null +++ b/app/api.py @@ -0,0 +1,258 @@ +"""HTTP-слой: тонкие обработчики (SRP: только транспорт).""" +import logging +from typing import Optional +from urllib.parse import quote + +from fastapi import APIRouter, Depends, Header, Query, Request, Response + +from app.auth import require_actor +from app.db import Database +from app.errors import unavailable, unsupported_media +from app.s3store import S3Store +from app.schemas import CreateChatBody, CreateMessageBody, CreateUserBody +from app.services import ( + AttachmentService, + AuthService, + ChatService, + MessageService, + UserService, +) +from app.sessions import SessionStore +from app.upload import read_body_limited + +logger = logging.getLogger("messenger") + +router = APIRouter() + + +def _state(request: Request): + db: Database = request.app.state.db + sessions: SessionStore = request.app.state.sessions + s3: S3Store = request.app.state.s3 + return db, sessions, s3 + + +def require_json_content_type(request: Request) -> None: + content_type = request.headers.get("content-type", "") + media = content_type.split(";")[0].strip().lower() + if media != "application/json": + raise unsupported_media() + + +def _require_accepting(db: Database) -> None: + if not db.accepting: + raise unavailable("Сервер завершает работу") + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(request: Request): + db, sessions, s3 = _state(request) + if not db.ready or not db.accepting: + raise unavailable("Не готов принимать трафик") + if not db.ping() or not sessions.ping() or not s3.ping(): + raise unavailable("Зависимость недоступна") + return {"status": "ok"} + + +@router.post("/api/v1/users", status_code=201) +def create_user( + request: Request, + body: CreateUserBody, + _: None = Depends(require_json_content_type), +): + db, _sessions, _s3 = _state(request) + _require_accepting(db) + return UserService(db).create(body.model_dump()) + + +@router.post("/api/v1/auth/sessions", status_code=201) +def create_session( + request: Request, + authorization: Optional[str] = Header(default=None), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + return AuthService(db, sessions).create_session(authorization) + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session( + request: Request, + authorization: Optional[str] = Header(default=None), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + _actor, token = require_actor(db, sessions, authorization) + AuthService(db, sessions).logout(token) + return Response(status_code=204) + + +@router.get("/api/v1/users/me") +def current_user( + request: Request, + authorization: Optional[str] = Header(default=None), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + return actor.to_dict() + + +@router.post("/api/v1/chats", status_code=201) +def create_chat( + request: Request, + body: CreateChatBody, + authorization: Optional[str] = Header(default=None), + _: None = Depends(require_json_content_type), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + return ChatService(db).create(actor, body.model_dump()) + + +@router.get("/api/v1/chats") +def list_chats( + request: Request, + authorization: Optional[str] = Header(default=None), + limit: Optional[str] = Query(default=None), + cursor: Optional[str] = Query(default=None), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + return ChatService(db).list_for_user(actor, limit, cursor) + + +@router.get("/api/v1/chats/{chat_id}") +def get_chat( + chat_id: str, + request: Request, + authorization: Optional[str] = Header(default=None), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + return ChatService(db).get_for_member(actor, chat_id).to_dict() + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + request: Request, + body: CreateMessageBody, + authorization: Optional[str] = Header(default=None), + idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"), + _: None = Depends(require_json_content_type), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + chats = ChatService(db) + chat = chats.get_for_member(actor, chat_id) + path = f"/api/v1/chats/{chat.id}/messages" + result = MessageService(db).create( + actor, + chat, + body.model_dump(), + idempotency_key=idempotency_key, + method="POST", + path=path, + ) + logger.info( + "message created chat_id=%s sender_id=%s message_id=%s", + chat.id, + actor.id, + result["id"], + ) + return result + + +@router.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: str, + request: Request, + authorization: Optional[str] = Header(default=None), + limit: Optional[str] = Query(default=None), + cursor: Optional[str] = Query(default=None), +): + db, sessions, _s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + chat = ChatService(db).get_for_member(actor, chat_id) + return MessageService(db).list_for_chat(chat, limit, cursor) + + +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: Optional[str] = Query(default=None), + authorization: Optional[str] = Header(default=None), +): + db, sessions, s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + chat = ChatService(db).get_for_member(actor, chat_id) + content_type = request.headers.get("content-type", "") + media = content_type.split(";")[0].strip().lower() + if media not in ("image/png", "image/jpeg"): + raise unsupported_media("Content-Type должен быть image/png или image/jpeg") + data = await read_body_limited(request) + result = AttachmentService(db, s3).upload( + actor, + chat, + filename_raw=filename, + content_type=media, + data=data, + ) + logger.info( + "attachment uploaded chat_id=%s attachment_id=%s size=%s", + chat.id, + result["id"], + result["size_bytes"], + ) + return result + + +@router.get("/api/v1/attachments/{attachment_id}") +def get_attachment( + attachment_id: str, + request: Request, + authorization: Optional[str] = Header(default=None), +): + db, sessions, s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + return AttachmentService(db, s3).get_for_member(actor, attachment_id).to_dict() + + +@router.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + request: Request, + authorization: Optional[str] = Header(default=None), + variant: Optional[str] = Query(default=None), +): + db, sessions, s3 = _state(request) + _require_accepting(db) + actor, _token = require_actor(db, sessions, authorization) + if variant is not None and variant != "" and variant != "original": + from app.errors import not_found + + raise not_found("Вариант не найден") + att, data = AttachmentService(db, s3).download_for_member(actor, attachment_id) + filename = quote(att.filename) + return Response( + content=data, + media_type=att.content_type, + headers={ + "Content-Length": str(len(data)), + "Content-Disposition": f"inline; filename*=UTF-8''{filename}", + }, + ) diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..a7b8e3c2 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,64 @@ +"""Разбор Basic/Bearer и проверка сессии.""" +from __future__ import annotations + +import base64 +import binascii +from typing import Optional, Tuple + +from app.db import Database +from app.errors import unauthorized +from app.models import User +from app.sessions import SessionStore +from app.validation import USERNAME_RE + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def parse_basic(authorization: Optional[str]) -> Tuple[str, str]: + if not authorization or not authorization.startswith("Basic "): + raise unauthorized("Требуется Basic-аутентификация", challenge=BASIC_CHALLENGE) + raw = authorization[6:].strip() + if not raw: + raise unauthorized("Требуется Basic-аутентификация", challenge=BASIC_CHALLENGE) + try: + decoded = base64.b64decode(raw, validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError) as exc: + raise unauthorized("Неверные учётные данные", challenge=BASIC_CHALLENGE) from exc + if ":" not in decoded: + raise unauthorized("Неверные учётные данные", challenge=BASIC_CHALLENGE) + username, password = decoded.split(":", 1) + if not USERNAME_RE.fullmatch(username): + raise unauthorized("Неверные учётные данные", challenge=BASIC_CHALLENGE) + n = len(password) + if n < 12 or n > 128: + raise unauthorized("Неверные учётные данные", challenge=BASIC_CHALLENGE) + return username, password + + +def parse_bearer(authorization: Optional[str]) -> str: + if not authorization: + raise unauthorized(challenge=BEARER_CHALLENGE) + if authorization.startswith("Basic "): + raise unauthorized(challenge=BEARER_CHALLENGE) + if not authorization.startswith("Bearer "): + raise unauthorized(challenge=BEARER_CHALLENGE) + token = authorization[7:].strip() + if not token: + raise unauthorized(challenge=BEARER_CHALLENGE) + return token + + +def require_actor( + db: Database, + sessions: SessionStore, + authorization: Optional[str], +) -> Tuple[User, str]: + token = parse_bearer(authorization) + user_id = sessions.resolve(token) + if user_id is None: + raise unauthorized(challenge=BEARER_CHALLENGE) + user = db.get_user(user_id) + if user is None: + raise unauthorized(challenge=BEARER_CHALLENGE) + return user, token diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..17279237 --- /dev/null +++ b/app/config.py @@ -0,0 +1,51 @@ +"""Конфигурация из окружения и secret-файлов.""" +from __future__ import annotations + +import os +import socket +from pathlib import Path + + +def _read_secret(file_env: str, direct_env: str) -> str: + path = os.environ.get(file_env, "").strip() + if path: + return Path(path).read_text(encoding="utf-8").strip() + value = os.environ.get(direct_env, "").strip() + if value: + return value + raise RuntimeError(f"Задайте {file_env} или {direct_env}") + + +class Settings: + def __init__(self) -> None: + self.host = os.environ.get("APP_HOST", "0.0.0.0") + self.port = int(os.environ.get("APP_PORT", "8080")) + self.env = os.environ.get("APP_ENV", "lab") + self.session_ttl_seconds = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + if self.session_ttl_seconds < 1: + raise RuntimeError("SESSION_TTL_SECONDS должен быть >= 1") + self.session_store = os.environ.get("SESSION_STORE", "redis").strip().lower() + self.database_url = _read_secret("DATABASE_URL_FILE", "DATABASE_URL") + self.cache_url = _read_secret("CACHE_URL_FILE", "CACHE_URL") + self.db_connect_timeout = float(os.environ.get("DB_CONNECT_TIMEOUT", "5")) + self.db_pool_min = int(os.environ.get("DB_POOL_MIN", "1")) + self.db_pool_max = int(os.environ.get("DB_POOL_MAX", "10")) + self.redis_socket_timeout = float(os.environ.get("REDIS_SOCKET_TIMEOUT", "3")) + self.idempotency_ttl_seconds = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + self.instance_id = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + self.run_migrations = os.environ.get("RUN_MIGRATIONS", "true").strip().lower() in { + "1", + "true", + "yes", + } + self.s3_endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").strip() + self.s3_region = os.environ.get("S3_REGION", "us-east-1").strip() + self.s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() + self.s3_access_key = _read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") + self.s3_secret_key = _read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") + self.max_upload_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) + self.max_image_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) + self.orphan_grace_seconds = int(os.environ.get("ORPHAN_GRACE_SECONDS", "900")) + + +settings = Settings() diff --git a/app/cursor.py b/app/cursor.py new file mode 100644 index 00000000..8b95223e --- /dev/null +++ b/app/cursor.py @@ -0,0 +1,27 @@ +"""Непрозрачный keyset-cursor: (created_at, id), привязан к ресурсу.""" +import base64 +import json + +from app.errors import invalid_request + + +def encode(resource_id: str, created_at: str, item_id: str) -> str: + payload = {"r": resource_id, "t": created_at, "i": item_id} + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode(cursor: str, expected_resource_id: str) -> tuple: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + payload = json.loads(raw.decode("utf-8")) + resource_id = payload["r"] + created_at = payload["t"] + item_id = payload["i"] + if not isinstance(resource_id, str) or not isinstance(created_at, str) or not isinstance(item_id, str): + raise ValueError("bad types") + except Exception as exc: + raise invalid_request("Неверный cursor") from exc + if resource_id != expected_resource_id: + raise invalid_request("Cursor относится к другому ресурсу") + return created_at, item_id diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..90bcc75d --- /dev/null +++ b/app/db.py @@ -0,0 +1,582 @@ +"""PostgreSQL pool, миграции и параметризованные запросы.""" +from __future__ import annotations + +import json +import logging +from pathlib import Path +from typing import List, Optional, Sequence, Tuple + +from psycopg import OperationalError +from psycopg.errors import UniqueViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from psycopg.types.json import Json + +from app.config import settings +from app.errors import conflict, unavailable +from app.models import Attachment, Chat, Message, User +from app.timefmt import format_utc + +logger = logging.getLogger("messenger") + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + + +class Database: + def __init__(self) -> None: + self.pool: Optional[ConnectionPool] = None + self.accepting = True + self.ready = False + + def open(self) -> None: + self.pool = ConnectionPool( + conninfo=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_connect_timeout, + kwargs={ + "connect_timeout": max(1, int(settings.db_connect_timeout)), + "row_factory": dict_row, + }, + open=True, + ) + if settings.run_migrations: + self.apply_migrations() + logger.info("postgres pool ready; migrations applied") + else: + logger.info("postgres pool ready; migrations skipped (RUN_MIGRATIONS=false)") + self.ready = True + + def close(self) -> None: + self.ready = False + self.accepting = False + if self.pool is not None: + self.pool.close() + self.pool = None + + def begin_shutdown(self) -> None: + self.accepting = False + self.ready = False + + def _require_pool(self) -> ConnectionPool: + if self.pool is None: + raise unavailable("База данных недоступна") + return self.pool + + def ping(self) -> bool: + try: + pool = self._require_pool() + with pool.connection() as conn: + conn.execute("SELECT 1") + return True + except Exception: + return False + + @staticmethod + def _split_sql(script: str) -> List[str]: + statements: List[str] = [] + buf: List[str] = [] + for line in script.splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("--"): + continue + buf.append(line) + if stripped.endswith(";"): + stmt = "\n".join(buf).strip() + if stmt: + statements.append(stmt) + buf = [] + tail = "\n".join(buf).strip() + if tail: + statements.append(tail) + return statements + + def apply_migrations(self) -> None: + pool = self._require_pool() + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + with pool.connection() as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + for path in files: + version = path.name + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (version,), + ).fetchone() + if row: + continue + statements = self._split_sql(path.read_text(encoding="utf-8")) + with conn.transaction(): + for stmt in statements: + conn.execute(stmt) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + logger.info("applied migration %s", version) + + def _user_from_row(self, row: dict) -> User: + return User( + id=str(row["id"]), + username=row["username"], + display_name=row["display_name"], + created_at=format_utc(row["created_at"]), + password_hash=row.get("password_hash", ""), + ) + + def add_user(self, user: User) -> None: + pool = self._require_pool() + try: + with pool.connection() as conn: + conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s::timestamptz) + """, + ( + user.id, + user.username, + user.display_name, + user.password_hash, + user.created_at, + ), + ) + except UniqueViolation as exc: + raise KeyError("username") from exc + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + + def get_user(self, user_id: str) -> Optional[User]: + pool = self._require_pool() + try: + with pool.connection() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE id = %s + """, + (user_id,), + ).fetchone() + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + return self._user_from_row(row) if row else None + + def get_user_by_username(self, username: str) -> Optional[User]: + pool = self._require_pool() + try: + with pool.connection() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + return self._user_from_row(row) if row else None + + def users_exist(self, user_ids: Sequence[str]) -> bool: + if not user_ids: + return True + pool = self._require_pool() + try: + with pool.connection() as conn: + row = conn.execute( + "SELECT COUNT(*) AS c FROM users WHERE id = ANY(%s::uuid[])", + (list(user_ids),), + ).fetchone() + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + return int(row["c"]) == len(set(user_ids)) + + def add_chat(self, chat: Chat) -> None: + """Атомарно: чат + участники в одной транзакции.""" + pool = self._require_pool() + try: + with pool.connection() as conn: + with conn.transaction(): + # Повторная проверка участников внутри транзакции. + row = conn.execute( + "SELECT COUNT(*) AS c FROM users WHERE id = ANY(%s::uuid[])", + (list(chat.member_ids),), + ).fetchone() + if int(row["c"]) != len(set(chat.member_ids)): + raise ValueError("unknown member") + conn.execute( + """ + INSERT INTO chats (id, title, created_at) + VALUES (%s, %s, %s::timestamptz) + """, + (chat.id, chat.title, chat.created_at), + ) + for mid in chat.member_ids: + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id) + VALUES (%s, %s) + """, + (chat.id, mid), + ) + except ValueError: + raise + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + + def get_chat(self, chat_id: str) -> Optional[Chat]: + pool = self._require_pool() + try: + with pool.connection() as conn: + row = conn.execute( + "SELECT id, title, created_at FROM chats WHERE id = %s", + (chat_id,), + ).fetchone() + if not row: + return None + members = conn.execute( + """ + SELECT user_id FROM chat_members + WHERE chat_id = %s + ORDER BY user_id + """, + (chat_id,), + ).fetchall() + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + return Chat( + id=str(row["id"]), + title=row["title"], + member_ids=[str(m["user_id"]) for m in members], + created_at=format_utc(row["created_at"]), + ) + + def list_chats_for(self, user_id: str) -> List[Chat]: + pool = self._require_pool() + try: + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = %s + ORDER BY c.created_at ASC, c.id ASC + """, + (user_id,), + ).fetchall() + result: List[Chat] = [] + for row in rows: + members = conn.execute( + """ + SELECT user_id FROM chat_members + WHERE chat_id = %s + ORDER BY user_id + """, + (row["id"],), + ).fetchall() + result.append( + Chat( + id=str(row["id"]), + title=row["title"], + member_ids=[str(m["user_id"]) for m in members], + created_at=format_utc(row["created_at"]), + ) + ) + return result + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + + def add_message(self, message: Message) -> Message: + pool = self._require_pool() + try: + with pool.connection() as conn: + conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, %s::timestamptz) + """, + ( + message.id, + message.chat_id, + message.sender_id, + message.text, + message.created_at, + ), + ) + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + return message + + def list_messages_page( + self, + chat_id: str, + limit: int, + after_created_at: Optional[str] = None, + after_id: Optional[str] = None, + ) -> List[Message]: + """Keyset page using index (chat_id, created_at, id).""" + pool = self._require_pool() + try: + with pool.connection() as conn: + if after_created_at is None: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + ORDER BY created_at ASC, id ASC + LIMIT %s + """, + (chat_id, limit), + ).fetchall() + else: + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + AND (created_at, id) > (%s::timestamptz, %s::uuid) + ORDER BY created_at ASC, id ASC + LIMIT %s + """, + (chat_id, after_created_at, after_id, limit), + ).fetchall() + result: List[Message] = [] + for r in rows: + aids = conn.execute( + """ + SELECT attachment_id FROM message_attachments + WHERE message_id = %s + ORDER BY position ASC + """, + (r["id"],), + ).fetchall() + result.append( + Message( + id=str(r["id"]), + chat_id=str(r["chat_id"]), + sender_id=str(r["sender_id"]), + text=r["text"], + created_at=format_utc(r["created_at"]), + attachment_ids=[str(a["attachment_id"]) for a in aids], + ) + ) + return result + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + + def explain_messages_page(self, chat_id: str, limit: int = 50) -> str: + pool = self._require_pool() + with pool.connection() as conn: + rows = conn.execute( + """ + EXPLAIN (ANALYZE, BUFFERS, FORMAT TEXT) + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + ORDER BY created_at ASC, id ASC + LIMIT %s + """, + (chat_id, limit), + ).fetchall() + # EXPLAIN text rows come as single-column dicts + lines = [] + for r in rows: + lines.append(next(iter(r.values()))) + return "\n".join(lines) + + def create_message_idempotent( + self, + message: Message, + *, + user_id: str, + method: str, + path: str, + idem_key: str, + request_hash: str, + ) -> Tuple[dict, bool]: + """Atomically create message or replay prior result. + + Returns (message_dict, created_new). + """ + pool = self._require_pool() + try: + with pool.connection() as conn: + with conn.transaction(): + claimed = conn.execute( + """ + INSERT INTO idempotency_keys ( + user_id, method, path, idem_key, request_hash, + response_status, response_body, created_at + ) + VALUES ( + %s, %s, %s, %s, %s, + 0, '{}'::jsonb, %s::timestamptz + ) + ON CONFLICT DO NOTHING + RETURNING idem_key + """, + ( + user_id, + method, + path, + idem_key, + request_hash, + message.created_at, + ), + ).fetchone() + if claimed is None: + existing = conn.execute( + """ + SELECT request_hash, response_status, response_body + FROM idempotency_keys + WHERE user_id = %s AND method = %s + AND path = %s AND idem_key = %s + FOR UPDATE + """, + (user_id, method, path, idem_key), + ).fetchone() + if existing is None: + raise unavailable("Идемпотентность временно недоступна") + if existing["request_hash"] != request_hash: + raise conflict("Idempotency-Key уже использован с другим телом") + if int(existing["response_status"]) != 201: + # Another replica still writing; rare — wait via conflict for client retry + raise unavailable("Идемпотентный запрос ещё обрабатывается") + body = existing["response_body"] + if isinstance(body, str): + body = json.loads(body) + return dict(body), False + + conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, %s::timestamptz) + """, + ( + message.id, + message.chat_id, + message.sender_id, + message.text, + message.created_at, + ), + ) + for pos, aid in enumerate(message.attachment_ids): + conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES (%s, %s, %s) + """, + (message.id, aid, pos), + ) + payload = message.to_dict() + conn.execute( + """ + UPDATE idempotency_keys + SET response_status = 201, response_body = %s + WHERE user_id = %s AND method = %s + AND path = %s AND idem_key = %s + """, + (Json(payload), user_id, method, path, idem_key), + ) + return payload, True + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + + def _attachment_from_row(self, row: dict) -> Attachment: + return Attachment( + id=str(row["id"]), + chat_id=str(row["chat_id"]), + owner_id=str(row["owner_id"]), + filename=row["filename"], + content_type=row["content_type"], + size_bytes=int(row["size_bytes"]), + sha256=row["sha256"], + object_key=row["object_key"], + status=row["status"], + error_code=row["error_code"], + created_at=format_utc(row["created_at"]), + updated_at=format_utc(row["updated_at"]), + variants=[], + ) + + def add_attachment(self, att: Attachment) -> None: + pool = self._require_pool() + try: + with pool.connection() as conn: + conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, status, error_code, created_at, updated_at + ) + VALUES ( + %s, %s, %s, %s, %s, %s, + %s, %s, %s, %s, %s::timestamptz, %s::timestamptz + ) + """, + ( + att.id, + att.chat_id, + att.owner_id, + att.filename, + att.content_type, + att.size_bytes, + att.sha256, + att.object_key, + att.status, + att.error_code, + att.created_at, + att.updated_at, + ), + ) + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + + def get_attachment(self, attachment_id: str) -> Optional[Attachment]: + pool = self._require_pool() + try: + with pool.connection() as conn: + row = conn.execute( + """ + SELECT id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, status, error_code, created_at, updated_at + FROM attachments WHERE id = %s + """, + (attachment_id,), + ).fetchone() + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + return self._attachment_from_row(row) if row else None + + def get_attachments_for_chat(self, chat_id: str, ids: Sequence[str]) -> List[Attachment]: + if not ids: + return [] + pool = self._require_pool() + try: + with pool.connection() as conn: + rows = conn.execute( + """ + SELECT id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, object_key, status, error_code, created_at, updated_at + FROM attachments + WHERE chat_id = %s AND id = ANY(%s::uuid[]) + """, + (chat_id, list(ids)), + ).fetchall() + except OperationalError as exc: + raise unavailable("База данных недоступна") from exc + return [self._attachment_from_row(r) for r in rows] + + def list_known_object_keys(self) -> List[str]: + pool = self._require_pool() + with pool.connection() as conn: + rows = conn.execute("SELECT object_key FROM attachments").fetchall() + return [r["object_key"] for r in rows] + diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..89fab268 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,58 @@ +"""Единый формат ошибок API.""" +from typing import Optional + + +class AppError(Exception): + def __init__( + self, + status: int, + code: str, + message: str, + headers: Optional[dict] = None, + ) -> None: + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + super().__init__(message) + + +def invalid_request(message: str = "Неверный запрос") -> AppError: + return AppError(400, "invalid_request", message) + + +def unauthorized( + message: str = "Требуется идентификация", + *, + challenge: str = 'Bearer realm="messenger"', +) -> AppError: + return AppError( + 401, + "unauthorized", + message, + headers={"WWW-Authenticate": challenge}, + ) + + +def not_found(message: str = "Объект не найден") -> AppError: + return AppError(404, "not_found", message) + + +def conflict(message: str = "Конфликт") -> AppError: + return AppError(409, "conflict", message) + + +def unsupported_media(message: str = "Неподдерживаемый Content-Type") -> AppError: + return AppError(415, "unsupported_media_type", message) + + +def payload_too_large(message: str = "Слишком большой файл") -> AppError: + return AppError(413, "payload_too_large", message) + + +def unavailable(message: str = "Сервис недоступен") -> AppError: + return AppError(503, "unavailable", message) + + +def error_body(code: str, message: str, request_id: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id}} diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..1551f53c --- /dev/null +++ b/app/images.py @@ -0,0 +1,51 @@ +"""Проверка PNG/JPEG по содержимому с лимитом пикселей.""" +from __future__ import annotations + +import io +from typing import Tuple + +from PIL import Image, UnidentifiedImageError + +from app.config import settings +from app.errors import payload_too_large, unsupported_media + +# Лимит до декодирования: Pillow прервёт слишком большие изображения. +Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + + +def detect_and_validate_image(data: bytes, declared_content_type: str) -> Tuple[str, int, int]: + """Return (content_type, width, height). Raises 413/415 AppError.""" + if declared_content_type not in ("image/png", "image/jpeg"): + raise unsupported_media("Content-Type должен быть image/png или image/jpeg") + + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + else: + raise unsupported_media("Содержимое не является PNG/JPEG") + + if detected != declared_content_type: + raise unsupported_media("Content-Type не совпадает с содержимым") + + try: + with Image.open(io.BytesIO(data)) as img: + img.load() + width, height = img.size + fmt = (img.format or "").upper() + except Image.DecompressionBombError as exc: + raise payload_too_large("Слишком много пикселей") from exc + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise unsupported_media("Повреждённое или неподдерживаемое изображение") from exc + + if detected == "image/png" and fmt != "PNG": + raise unsupported_media("Повреждённое или неподдерживаемое изображение") + if detected == "image/jpeg" and fmt not in ("JPEG", "MPO"): + raise unsupported_media("Повреждённое или неподдерживаемое изображение") + + pixels = width * height + if pixels > settings.max_image_pixels: + raise payload_too_large("Слишком много пикселей") + if width < 1 or height < 1: + raise unsupported_media("Повреждённое или неподдерживаемое изображение") + return detected, width, height diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..2c51afc8 --- /dev/null +++ b/app/main.py @@ -0,0 +1,125 @@ +"""Точка входа HTTP-приложения lab4.""" +import logging +import uuid +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from app.api import router +from app.config import settings +from app.db import Database +from app.errors import AppError, error_body +from app.reconcile import OrphanReaper +from app.s3store import S3Store +from app.sessions import SessionStore + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) +logger = logging.getLogger("messenger") + +db = Database() +sessions = SessionStore() +s3 = S3Store() +reaper: OrphanReaper | None = None +INSTANCE_ID = settings.instance_id + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + global reaper + logger.info("server starting; instance_id=%s", INSTANCE_ID) + db.open() + sessions.open() + s3.open() + reaper = OrphanReaper(db, s3, interval_seconds=300) + reaper.start() + _app.state.db = db + _app.state.sessions = sessions + _app.state.s3 = s3 + logger.info("dependencies ready") + yield + if reaper is not None: + reaper.stop() + db.begin_shutdown() + sessions.close() + s3.close() + db.close() + logger.info("SIGTERM/shutdown: stop accepting, ready=false") + + +app = FastAPI(title="MTUSI Messenger lab4", lifespan=lifespan) +app.state.db = db +app.state.sessions = sessions +app.state.s3 = s3 +app.include_router(router) + + +@app.middleware("http") +async def request_headers_middleware(request: Request, call_next): + """Чистый ASGI middleware — не буферизует тело (важно для upload ≤10 MiB).""" + request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + logger.info( + "%s %s -> %s request_id=%s instance=%s", + request.method, + request.url.path, + response.status_code, + request_id, + INSTANCE_ID, + ) + return response + + +@app.exception_handler(AppError) +async def app_error_handler(request: Request, exc: AppError): + request_id = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + headers = {"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID} + headers.update(exc.headers) + return JSONResponse( + status_code=exc.status, + content=error_body(exc.code, exc.message, request_id), + headers=headers, + ) + + +@app.exception_handler(RequestValidationError) +async def validation_handler(request: Request, _exc: RequestValidationError): + request_id = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + return JSONResponse( + status_code=400, + content=error_body("invalid_request", "Неверный запрос", request_id), + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + ) + + +@app.exception_handler(Exception) +async def internal_handler(request: Request, exc: Exception): + request_id = getattr(request.state, "request_id", None) or str(uuid.uuid4()) + logger.exception("unhandled error request_id=%s", request_id) + return JSONResponse( + status_code=500, + content=error_body("internal_error", "Внутренняя ошибка", request_id), + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + ) + + +def run() -> None: + import uvicorn + + uvicorn.run( + "app.main:app", + host=settings.host, + port=settings.port, + log_level="info", + ) + + +if __name__ == "__main__": + run() diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..e070ed16 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,18 @@ +"""One-shot migrations (run once before API replicas).""" +import logging + +from app.db import Database + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +logger = logging.getLogger("migrate") + + +def main() -> None: + db = Database() + db.open() + logger.info("migrations complete") + db.close() + + +if __name__ == "__main__": + main() diff --git a/app/models.py b/app/models.py new file mode 100644 index 00000000..41910bed --- /dev/null +++ b/app/models.py @@ -0,0 +1,93 @@ +"""Доменные сущности.""" +import uuid +from dataclasses import dataclass, field +from typing import List, Optional + + +@dataclass +class User: + id: str + username: str + display_name: str + created_at: str + password_hash: str = "" + + def to_dict(self) -> dict: + return { + "id": self.id, + "username": self.username, + "display_name": self.display_name, + "created_at": self.created_at, + } + + +@dataclass +class Chat: + id: str + title: str + member_ids: List[str] + created_at: str + + def to_dict(self) -> dict: + return { + "id": self.id, + "title": self.title, + "member_ids": list(self.member_ids), + "created_at": self.created_at, + } + + +@dataclass +class Message: + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: List[str] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "id": self.id, + "chat_id": self.chat_id, + "sender_id": self.sender_id, + "text": self.text, + "created_at": self.created_at, + "attachment_ids": list(self.attachment_ids), + } + + def sort_key(self): + return (self.created_at, uuid.UUID(self.id).int) + + +@dataclass +class Attachment: + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + object_key: str + status: str + created_at: str + updated_at: str + error_code: Optional[str] = None + variants: List[dict] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "id": self.id, + "chat_id": self.chat_id, + "owner_id": self.owner_id, + "filename": self.filename, + "content_type": self.content_type, + "size_bytes": self.size_bytes, + "sha256": self.sha256, + "status": self.status, + "variants": list(self.variants), + "error_code": self.error_code, + "created_at": self.created_at, + "updated_at": self.updated_at, + } diff --git a/app/password.py b/app/password.py new file mode 100644 index 00000000..6a72d3d7 --- /dev/null +++ b/app/password.py @@ -0,0 +1,18 @@ +"""Password KDF: Argon2id (библиотечный).""" +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return _hasher.verify(password_hash, password) + except VerifyMismatchError: + return False + except Exception: + return False diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..a1a06644 --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,100 @@ +"""Сверка orphan-объектов S3 без записи в PostgreSQL. + +После S3 PUT при сбое INSERT метаданных приложение сразу удаляет объект. +Этот reconciler — страховка: объекты старше ORPHAN_GRACE_SECONDS без строки +в attachments удаляются. Свежие объекты не трогаем (гонка upload). +""" +from __future__ import annotations + +import argparse +import logging +import threading +import time +from datetime import datetime, timedelta, timezone + +from app.config import settings +from app.db import Database +from app.s3store import S3Store + +logger = logging.getLogger("messenger.reconcile") + + +def find_orphans(db: Database, s3: S3Store, *, now: datetime | None = None): + now = now or datetime.now(timezone.utc) + grace = timedelta(seconds=settings.orphan_grace_seconds) + known = set(db.list_known_object_keys()) + orphans = [] + for key, last_modified in s3.list_objects("attachments/"): + if key in known: + continue + age = now - last_modified + if age >= grace: + orphans.append((key, last_modified, age)) + return orphans + + +def run_once(*, dry_run: bool = True) -> int: + db = Database() + s3 = S3Store() + db.open() + s3.open() + try: + orphans = find_orphans(db, s3) + for key, lm, age in orphans: + logger.info( + "orphan key=%s last_modified=%s age_s=%.0f dry_run=%s", + key, + lm.isoformat(), + age.total_seconds(), + dry_run, + ) + if not dry_run: + s3.delete_object(key) + return len(orphans) + finally: + s3.close() + db.close() + + +class OrphanReaper: + """Фоновый цикл сверки внутри API-процесса.""" + + def __init__(self, db: Database, s3: S3Store, interval_seconds: int = 300) -> None: + self.db = db + self.s3 = s3 + self.interval_seconds = interval_seconds + self._stop = threading.Event() + self._thread: threading.Thread | None = None + + def start(self) -> None: + self._thread = threading.Thread(target=self._loop, name="orphan-reaper", daemon=True) + self._thread.start() + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(timeout=5) + + def _loop(self) -> None: + while not self._stop.wait(self.interval_seconds): + try: + orphans = find_orphans(self.db, self.s3) + for key, _lm, _age in orphans: + self.s3.delete_object(key) + logger.info("reaper deleted orphan key=%s", key) + except Exception: + logger.exception("orphan reaper cycle failed") + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + parser = argparse.ArgumentParser(description="S3 orphan reconciler") + parser.add_argument("--apply", action="store_true", help="Удалить orphans (без флага — dry-run)") + args = parser.parse_args() + n = run_once(dry_run=not args.apply) + mode = "apply" if args.apply else "dry-run" + print(f"orphans={n} mode={mode} grace_s={settings.orphan_grace_seconds}") + + +if __name__ == "__main__": + main() diff --git a/app/s3store.py b/app/s3store.py new file mode 100644 index 00000000..212a4210 --- /dev/null +++ b/app/s3store.py @@ -0,0 +1,118 @@ +"""S3-совместимое хранилище (MinIO): байты вложений.""" +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from typing import List, Tuple + +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError + +from app.config import settings +from app.errors import unavailable + +logger = logging.getLogger("messenger") + + +class S3Store: + def __init__(self) -> None: + self.client = None + self.bucket = settings.s3_bucket + + def open(self) -> None: + self.client = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + aws_access_key_id=settings.s3_access_key, + aws_secret_access_key=settings.s3_secret_key, + region_name=settings.s3_region, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + self.ensure_bucket() + logger.info("s3 store ready bucket=%s", self.bucket) + + def close(self) -> None: + self.client = None + + def _require(self): + if self.client is None: + raise unavailable("Объектное хранилище недоступно") + return self.client + + def ensure_bucket(self) -> None: + """Идемпотентный bootstrap: head → create при отсутствии.""" + client = self._require() + try: + client.head_bucket(Bucket=self.bucket) + return + except ClientError: + pass + try: + client.create_bucket(Bucket=self.bucket) + except ClientError as exc: + try: + client.head_bucket(Bucket=self.bucket) + except ClientError: + raise unavailable("Не удалось подготовить bucket") from exc + + def ping(self) -> bool: + try: + self._require().head_bucket(Bucket=self.bucket) + return True + except Exception: + return False + + def put_bytes(self, object_key: str, data: bytes, content_type: str) -> None: + client = self._require() + try: + client.put_object( + Bucket=self.bucket, + Key=object_key, + Body=data, + ContentType=content_type, + ContentLength=len(data), + ) + except ClientError as exc: + raise unavailable("Объектное хранилище недоступно") from exc + + def get_bytes(self, object_key: str) -> bytes: + client = self._require() + try: + obj = client.get_object(Bucket=self.bucket, Key=object_key) + return obj["Body"].read() + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code in ("NoSuchKey", "404", "NotFound"): + raise FileNotFoundError(object_key) from exc + raise unavailable("Объектное хранилище недоступно") from exc + + def delete_object(self, object_key: str) -> None: + client = self._require() + try: + client.delete_object(Bucket=self.bucket, Key=object_key) + except ClientError: + logger.warning("failed to delete s3 object key=%s", object_key) + + def list_objects(self, prefix: str = "attachments/") -> List[Tuple[str, datetime]]: + """Список (key, last_modified UTC) под prefix.""" + client = self._require() + out: List[Tuple[str, datetime]] = [] + token = None + while True: + kwargs = {"Bucket": self.bucket, "Prefix": prefix} + if token: + kwargs["ContinuationToken"] = token + try: + resp = client.list_objects_v2(**kwargs) + except ClientError as exc: + raise unavailable("Объектное хранилище недоступно") from exc + for item in resp.get("Contents", []): + lm = item["LastModified"] + if lm.tzinfo is None: + lm = lm.replace(tzinfo=timezone.utc) + out.append((item["Key"], lm.astimezone(timezone.utc))) + if not resp.get("IsTruncated"): + break + token = resp.get("NextContinuationToken") + return out diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 00000000..d743c40d --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,30 @@ +"""Схемы тел запросов для OpenAPI / Swagger (валидация домена — в services).""" +from typing import List + +from pydantic import BaseModel, ConfigDict, Field + + +class CreateUserBody(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(..., examples=["alice"]) + display_name: str = Field(..., examples=["Алиса"]) + password: str = Field(..., examples=["Example-only-not-a-real-secret-123"]) + + +class CreateChatBody(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(..., examples=["Проект по ВНП"]) + member_ids: List[str] = Field( + ..., + examples=[["11111111-1111-4111-8111-111111111111"]], + ) + + +class CreateMessageBody(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(..., examples=["Привет, Боб!"]) + attachment_ids: List[str] = Field(default_factory=list) + diff --git a/app/services.py b/app/services.py new file mode 100644 index 00000000..6be5cdcd --- /dev/null +++ b/app/services.py @@ -0,0 +1,316 @@ +"""Бизнес-логика lab4.""" +from __future__ import annotations + +import hashlib +import json +import uuid +from typing import List, Optional + +from app import cursor as cursor_util +from app.auth import BASIC_CHALLENGE, parse_basic +from app.config import settings +from app.db import Database +from app.errors import conflict, invalid_request, not_found, unauthorized +from app.images import detect_and_validate_image +from app.models import Attachment, Chat, Message, User +from app.password import hash_password, verify_password +from app.s3store import S3Store +from app.sessions import SessionStore +from app.timefmt import now_utc +from app.validation import ( + parse_limit, + parse_uuid, + reject_unknown, + require_display_name, + require_filename, + require_idempotency_key, + require_keys, + require_message_text, + require_object, + require_password, + require_title, + require_username, +) + + +class UserService: + def __init__(self, db: Database) -> None: + self.db = db + + def create(self, body: object) -> dict: + data = require_object(body) + reject_unknown(data, {"username", "display_name", "password"}) + require_keys(data, ("username", "display_name", "password")) + username = require_username(data["username"]) + display_name = require_display_name(data["display_name"]) + password = require_password(data["password"]) + user = User( + id=str(uuid.uuid4()), + username=username, + display_name=display_name, + created_at=now_utc(), + password_hash=hash_password(password), + ) + try: + self.db.add_user(user) + except KeyError as exc: + raise conflict("username уже занят") from exc + return user.to_dict() + + +class AuthService: + def __init__(self, db: Database, sessions: SessionStore) -> None: + self.db = db + self.sessions = sessions + + def create_session(self, authorization: Optional[str]) -> dict: + username, password = parse_basic(authorization) + user = self.db.get_user_by_username(username) + if user is None or not verify_password(user.password_hash, password): + raise unauthorized("Неверные учётные данные", challenge=BASIC_CHALLENGE) + token, expires_at = self.sessions.create(user.id) + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at, + "user": user.to_dict(), + } + + def logout(self, token: str) -> None: + self.sessions.revoke(token) + + +class ChatService: + def __init__(self, db: Database) -> None: + self.db = db + + def create(self, actor: User, body: object) -> dict: + data = require_object(body) + reject_unknown(data, {"title", "member_ids"}) + require_keys(data, ("title", "member_ids")) + title = require_title(data["title"]) + raw_members = data["member_ids"] + if not isinstance(raw_members, list): + raise invalid_request("member_ids должен быть массивом") + if len(raw_members) < 1 or len(raw_members) > 100: + raise invalid_request("member_ids: от 1 до 100 участников") + member_ids: List[str] = [] + seen = set() + for item in raw_members: + mid = parse_uuid(item, "member_ids") + if mid in seen: + raise invalid_request("member_ids содержит дубликаты") + seen.add(mid) + member_ids.append(mid) + if actor.id not in seen: + raise invalid_request("Вызывающий должен быть в member_ids") + if not self.db.users_exist(member_ids): + raise invalid_request("Неизвестный участник") + chat = Chat( + id=str(uuid.uuid4()), + title=title, + member_ids=member_ids, + created_at=now_utc(), + ) + try: + self.db.add_chat(chat) + except ValueError as exc: + raise invalid_request("Неизвестный участник") from exc + return chat.to_dict() + + def get_for_member(self, actor: User, chat_id: str) -> Chat: + cid = parse_uuid(chat_id, "chat_id") + chat = self.db.get_chat(cid) + if chat is None or actor.id not in chat.member_ids: + raise not_found() + return chat + + def list_for_user(self, actor: User, limit_raw: Optional[str], cursor_raw: Optional[str]) -> dict: + limit = parse_limit(limit_raw) + chats = self.db.list_chats_for(actor.id) + start = 0 + if cursor_raw: + created_at, item_id = cursor_util.decode(cursor_raw, actor.id) + for i, chat in enumerate(chats): + if (chat.created_at, chat.id) > (created_at, item_id): + start = i + break + else: + start = len(chats) + page = chats[start : start + limit] + next_cursor = None + if start + limit < len(chats) and page: + last = page[-1] + next_cursor = cursor_util.encode(actor.id, last.created_at, last.id) + return {"items": [c.to_dict() for c in page], "next_cursor": next_cursor} + + +def _message_request_hash(text: str, attachment_ids: List[str]) -> str: + canonical = json.dumps( + {"text": text, "attachment_ids": attachment_ids}, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +class MessageService: + def __init__(self, db: Database) -> None: + self.db = db + + def create( + self, + actor: User, + chat: Chat, + body: object, + *, + idempotency_key: Optional[str], + method: str, + path: str, + ) -> dict: + data = require_object(body) + reject_unknown(data, {"text", "attachment_ids"}) + require_keys(data, ("text",)) + raw_ids = data.get("attachment_ids", []) + if raw_ids is None: + raw_ids = [] + if not isinstance(raw_ids, list): + raise invalid_request("attachment_ids должен быть массивом") + if len(raw_ids) > 10: + raise invalid_request("attachment_ids: максимум 10") + attachment_ids: List[str] = [] + seen = set() + for item in raw_ids: + aid = parse_uuid(item, "attachment_ids") + if aid in seen: + raise invalid_request("attachment_ids содержит дубликаты") + seen.add(aid) + attachment_ids.append(aid) + + allow_blank = len(attachment_ids) > 0 + text = require_message_text(data["text"], allow_blank=allow_blank) + if text.strip() == "" and not attachment_ids: + raise invalid_request("text не может быть пустым") + + if attachment_ids: + found = self.db.get_attachments_for_chat(chat.id, attachment_ids) + by_id = {a.id: a for a in found} + if len(by_id) != len(attachment_ids): + raise not_found() + for aid in attachment_ids: + att = by_id[aid] + if att.status != "ready": + raise conflict("Вложение ещё не готово или повреждено") + + key = require_idempotency_key(idempotency_key) + message = Message( + id=str(uuid.uuid4()), + chat_id=chat.id, + sender_id=actor.id, + text=text, + created_at=now_utc(), + attachment_ids=attachment_ids, + ) + payload, _created = self.db.create_message_idempotent( + message, + user_id=actor.id, + method=method, + path=path, + idem_key=key, + request_hash=_message_request_hash(text, attachment_ids), + ) + return payload + + def list_for_chat( + self, + chat: Chat, + limit_raw: Optional[str], + cursor_raw: Optional[str], + ) -> dict: + limit = parse_limit(limit_raw) + after_created_at = None + after_id = None + if cursor_raw: + created_at, item_id = cursor_util.decode(cursor_raw, chat.id) + parse_uuid(item_id, "cursor.id") + after_created_at = created_at + after_id = item_id + rows = self.db.list_messages_page( + chat.id, + limit + 1, + after_created_at=after_created_at, + after_id=after_id, + ) + page = rows[:limit] + next_cursor = None + if len(rows) > limit and page: + last = page[-1] + next_cursor = cursor_util.encode(chat.id, last.created_at, last.id) + return {"items": [m.to_dict() for m in page], "next_cursor": next_cursor} + + +class AttachmentService: + def __init__(self, db: Database, s3: S3Store) -> None: + self.db = db + self.s3 = s3 + + def upload( + self, + actor: User, + chat: Chat, + *, + filename_raw: Optional[str], + content_type: str, + data: bytes, + ) -> dict: + filename = require_filename(filename_raw) + media = content_type.split(";")[0].strip().lower() + content_type_ok, _w, _h = detect_and_validate_image(data, media) + digest = hashlib.sha256(data).hexdigest() + att_id = str(uuid.uuid4()) + object_key = f"attachments/{chat.id}/{att_id}" + ts = now_utc() + # S3 first; on DB failure compensate by deleting object (no permanent orphan). + self.s3.put_bytes(object_key, data, content_type_ok) + att = Attachment( + id=att_id, + chat_id=chat.id, + owner_id=actor.id, + filename=filename, + content_type=content_type_ok, + size_bytes=len(data), + sha256=digest, + object_key=object_key, + status="ready", + error_code=None, + created_at=ts, + updated_at=ts, + variants=[], + ) + try: + self.db.add_attachment(att) + except Exception: + self.s3.delete_object(object_key) + raise + return att.to_dict() + + def get_for_member(self, actor: User, attachment_id: str) -> Attachment: + aid = parse_uuid(attachment_id, "attachment_id") + att = self.db.get_attachment(aid) + if att is None: + raise not_found() + chat = self.db.get_chat(att.chat_id) + if chat is None or actor.id not in chat.member_ids: + raise not_found() + return att + + def download_for_member(self, actor: User, attachment_id: str) -> tuple: + att = self.get_for_member(actor, attachment_id) + if att.status != "ready": + raise conflict("Содержимое вложения недоступно") + try: + data = self.s3.get_bytes(att.object_key) + except FileNotFoundError as exc: + raise not_found() from exc + return att, data diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..6fdb9668 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,93 @@ +"""Opaque Bearer-сессии в Redis с TTL и AOF-persistence на стороне Redis.""" +from __future__ import annotations + +import json +import logging +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional, Tuple + +import redis + +from app.config import settings +from app.errors import unavailable +from app.timefmt import format_utc, parse_rfc3339 + +logger = logging.getLogger("messenger") + +TOKEN_PREFIX = "session:" + + +class SessionStore: + def __init__(self) -> None: + self.client: Optional[redis.Redis] = None + + def open(self) -> None: + self.client = redis.Redis.from_url( + settings.cache_url, + decode_responses=True, + socket_connect_timeout=settings.redis_socket_timeout, + socket_timeout=settings.redis_socket_timeout, + ) + self.client.ping() + logger.info("redis session store ready") + + def close(self) -> None: + if self.client is not None: + self.client.close() + self.client = None + + def _require(self) -> redis.Redis: + if self.client is None: + raise unavailable("Хранилище сессий недоступно") + return self.client + + def ping(self) -> bool: + try: + return bool(self._require().ping()) + except Exception: + return False + + def create(self, user_id: str) -> Tuple[str, str]: + """Возвращает (token, expires_at).""" + client = self._require() + token = secrets.token_urlsafe(32) # ≥256 бит + expires = datetime.now(timezone.utc) + timedelta(seconds=settings.session_ttl_seconds) + expires_at = format_utc(expires) + payload = json.dumps({"user_id": user_id, "expires_at": expires_at}, separators=(",", ":")) + try: + client.setex(TOKEN_PREFIX + token, settings.session_ttl_seconds, payload) + except redis.RedisError as exc: + raise unavailable("Хранилище сессий недоступно") from exc + return token, expires_at + + def resolve(self, token: str) -> Optional[str]: + """user_id или None, если нет/истекла. 503 при недоступности Redis.""" + client = self._require() + try: + raw = client.get(TOKEN_PREFIX + token) + except redis.RedisError as exc: + raise unavailable("Хранилище сессий недоступно") from exc + if not raw: + return None + try: + data = json.loads(raw) + user_id = data["user_id"] + expires_at = parse_rfc3339(data["expires_at"]) + except Exception: + return None + if expires_at <= datetime.now(timezone.utc): + try: + client.delete(TOKEN_PREFIX + token) + except redis.RedisError: + pass + return None + return str(user_id) + + def revoke(self, token: str) -> bool: + client = self._require() + try: + deleted = client.delete(TOKEN_PREFIX + token) + except redis.RedisError as exc: + raise unavailable("Хранилище сессий недоступно") from exc + return bool(deleted) diff --git a/app/timefmt.py b/app/timefmt.py new file mode 100644 index 00000000..58734b42 --- /dev/null +++ b/app/timefmt.py @@ -0,0 +1,20 @@ +"""Форматирование времени в RFC 3339 UTC с миллисекундами и Z.""" +from datetime import datetime, timezone + + +def now_utc() -> str: + return format_utc(datetime.now(timezone.utc)) + + +def format_utc(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + else: + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" + + +def parse_rfc3339(value: str) -> datetime: + if value.endswith("Z"): + value = value[:-1] + "+00:00" + return datetime.fromisoformat(value) diff --git a/app/upload.py b/app/upload.py new file mode 100644 index 00000000..66a716d0 --- /dev/null +++ b/app/upload.py @@ -0,0 +1,41 @@ +"""Потоковое чтение тела загрузки с жёстким лимитом размера.""" +from __future__ import annotations + +from fastapi import Request + +from app.config import settings +from app.errors import invalid_request, payload_too_large + + +async def read_body_limited(request: Request) -> bytes: + """Читает тело чанками; обрывает накопление при превышении MAX_UPLOAD_BYTES. + + При слишком большом Content-Length дренируем поток (без хранения), + чтобы клиент успел получить 413, а не ConnectionReset. + """ + max_bytes = settings.max_upload_bytes + declared = request.headers.get("content-length") + if declared is not None: + try: + length = int(declared) + except ValueError as exc: + raise invalid_request("Неверный Content-Length") from exc + if length > max_bytes: + async for _chunk in request.stream(): + pass + raise payload_too_large("Файл больше 10 MiB") + + chunks: list[bytes] = [] + total = 0 + async for chunk in request.stream(): + if not chunk: + continue + total += len(chunk) + if total > max_bytes: + async for _rest in request.stream(): + pass + raise payload_too_large("Файл больше 10 MiB") + chunks.append(chunk) + if total < 1: + raise invalid_request("Пустое тело запроса") + return b"".join(chunks) diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..8aefbbd6 --- /dev/null +++ b/app/validation.py @@ -0,0 +1,128 @@ +"""Проверки входных данных по контракту lab2.""" +import re +import uuid +from typing import Any, Iterable, Optional, Set + +from app.errors import invalid_request + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def require_object(body: Any) -> dict: + if not isinstance(body, dict): + raise invalid_request("Тело запроса должно быть JSON-объектом") + return body + + +def reject_unknown(body: dict, allowed: Set[str]) -> None: + extra = set(body) - allowed + if extra: + raise invalid_request("Лишние поля в запросе") + + +def require_keys(body: dict, keys: Iterable[str]) -> None: + for key in keys: + if key not in body: + raise invalid_request(f"Поле {key} обязательно") + + +def parse_uuid(value: Any, field: str = "id") -> str: + if not isinstance(value, str): + raise invalid_request(f"Поле {field} должно быть UUID-строкой") + try: + return str(uuid.UUID(value)) + except ValueError as exc: + raise invalid_request(f"Неверный UUID: {field}") from exc + + +def code_points(text: str) -> int: + return len(text) + + +def require_username(value: Any) -> str: + if not isinstance(value, str): + raise invalid_request("username должен быть строкой") + if not USERNAME_RE.fullmatch(value): + raise invalid_request("username: только [a-z0-9_] длиной 3–32") + return value + + +def require_display_name(value: Any) -> str: + if not isinstance(value, str): + raise invalid_request("display_name должен быть строкой") + if value.strip() == "": + raise invalid_request("display_name не может быть пустым") + n = code_points(value) + if n < 1 or n > 100: + raise invalid_request("display_name: 1–100 символов Unicode") + return value + + +def require_password(value: Any) -> str: + if not isinstance(value, str): + raise invalid_request("password должен быть строкой") + if value.strip() == "": + raise invalid_request("password не может быть пустым") + n = code_points(value) + if n < 12 or n > 128: + raise invalid_request("password: 12–128 символов Unicode") + return value + + +def require_title(value: Any) -> str: + if not isinstance(value, str): + raise invalid_request("title должен быть строкой") + if value.strip() == "": + raise invalid_request("title не может быть пустым") + n = code_points(value) + if n < 1 or n > 100: + raise invalid_request("title: 1–100 символов Unicode") + return value + + +def require_message_text(value: Any, *, allow_blank: bool = False) -> str: + if not isinstance(value, str): + raise invalid_request("text должен быть строкой") + if not allow_blank and value.strip() == "": + raise invalid_request("text не может быть пустым") + if code_points(value) > 4000: + raise invalid_request("text: максимум 4000 символов Unicode") + return value + + +def require_filename(value: Any) -> str: + if not isinstance(value, str): + raise invalid_request("filename должен быть строкой") + if value.strip() == "": + raise invalid_request("filename не может быть пустым") + n = code_points(value) + if n < 1 or n > 255: + raise invalid_request("filename: 1–255 символов Unicode") + for ch in value: + o = ord(ch) + if ch in "/\\" or o == 0 or o < 32: + raise invalid_request("filename содержит недопустимые символы") + return value + + + +def parse_limit(raw: Optional[str], default: int = 50) -> int: + if raw is None or raw == "": + return default + try: + value = int(raw) + except ValueError as exc: + raise invalid_request("limit должен быть целым числом") from exc + if value < 1 or value > 100: + raise invalid_request("limit должен быть от 1 до 100") + return value + + +def require_idempotency_key(value: Any) -> str: + if not isinstance(value, str) or not value: + raise invalid_request("Idempotency-Key обязателен") + if not IDEMPOTENCY_KEY_RE.fullmatch(value): + raise invalid_request("Idempotency-Key: ASCII [A-Za-z0-9._:-]{1,128}") + return value + diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..0e66eb61 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,197 @@ +# Lab4: LB + ≥2 API + Postgres + Redis + RustFS (private S3 bucket). +x-lab-number: 4 + +x-api-env: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: redis + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + DB_CONNECT_TIMEOUT: "5" + DB_POOL_MIN: "1" + DB_POOL_MAX: "8" + REDIS_SOCKET_TIMEOUT: "3" + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + RUN_MIGRATIONS: "false" + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-900} + +services: + postgres: + image: postgres:16.4-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: + - postgres_password + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + restart: unless-stopped + mem_limit: 512m + cpus: 0.50 + + redis: + image: redis:7.4.1-bookworm + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --dir + - /data + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + restart: unless-stopped + mem_limit: 256m + cpus: 0.25 + + s3: + image: rustfs/rustfs:1.0.5 + command: ["server", "/data"] + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key + RUSTFS_ADDRESS: ":9000" + # Консоль намеренно выключена (нет RUSTFS_CONSOLE_ENABLE). + secrets: + - s3_access_key_id + - s3_secret_access_key + volumes: + - s3_data:/data + # Только loopback для проверки private bucket вне API. + ports: + - "127.0.0.1:9000:9000" + healthcheck: + test: + [ + "CMD", + "wget", + "-q", + "-O", + "/dev/null", + "http://127.0.0.1:9000/minio/health/live", + ] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + mem_limit: 512m + cpus: 0.50 + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "app.migrate"] + environment: + <<: *api-env + RUN_MIGRATIONS: "true" + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + s3: + condition: service_healthy + restart: "no" + mem_limit: 256m + cpus: 0.25 + + api: + build: + context: . + dockerfile: Dockerfile + environment: + <<: *api-env + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + s3: + condition: service_healthy + expose: + - "8080" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)", + ] + interval: 2s + timeout: 2s + start_period: 20s + retries: 15 + restart: unless-stopped + stop_grace_period: 10s + mem_limit: 384m + cpus: 0.50 + + lb: + image: haproxy:2.9.7-bookworm + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + restart: unless-stopped + mem_limit: 128m + cpus: 0.25 + +volumes: + pg_data: + redis_data: + s3_data: + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/evidence/README.md b/evidence/README.md index 8df70015..8d6c1cc4 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,12 @@ -# Доказательства +# Доказательства lab4 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Дата: 2026-11-12. Целевая оценка: 4. Стенд: Docker Compose `--scale api=2`. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +| Файл | Содержание | +| --- | --- | +| `smoke.txt` | Публичный smoke lab4 — PASS | +| `own.txt` | Own-тесты: instances, ACL, 413/415, private bucket, 10 uploads, idempotency — PASS | +| `persistence.txt` | Restart hook: сессия + attachment сохранены — PASS | +| `failover.txt` | stop-one: recovery ~6.14 s, данные/файл сохранены — PASS | + +Не коммитьте `.env`, `secrets/`, токены, сырые dumps с credentials. diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..f4aff2de --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 6, "failed_attempts": 1, "first_recovery_seconds": 6.14} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/own.txt b/evidence/own.txt new file mode 100644 index 00000000..55a0ca5c --- /dev/null +++ b/evidence/own.txt @@ -0,0 +1,8 @@ +PASS: observed instances ['01e0dbb41bb1', 'b9570a5a3c7c'] +PASS: attachment upload/download/message +PASS: 413/415/400 on bad uploads +PASS: outsider 404 and foreign chat attachment 404 +PASS: private bucket / API-only download +PASS: 10 concurrent uploads in 2.48s (observe API mem_limit separately) +PASS: idempotency with attachment_ids +PASS: all own lab4 tests diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..6d631a1d --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,39 @@ +global + maxconn 4096 + log stdout format raw local0 + +defaults + mode http + log global + option httplog + option http-server-close + timeout connect 3s + # Uploads up to 10 MiB need headroom beyond small JSON APIs. + timeout client 120s + timeout server 120s + timeout http-request 30s + timeout check 2s + retries 2 + +resolvers docker + nameserver dns1 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold other 5s + hold refused 5s + hold nx 5s + hold timeout 5s + hold valid 5s + hold obsolete 5s + +frontend fe_http + bind *:8080 + default_backend be_api + +backend be_api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + # Dynamic API tasks via Compose DNS; drop failed backends within ~4s (inter 2s * fall 2). + server-template api- 8 api:8080 check resolvers docker init-addr last,libc,none resolve-prefer ipv4 inter 2s fall 2 rise 1 diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..4894343b --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,37 @@ +-- Lab2 schema: users, chats, membership, messages. + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE UNIQUE INDEX IF NOT EXISTS users_username_uq ON users (username); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_user_id_idx ON chat_members (user_id); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +-- Keyset history: (chat_id, created_at, id) — without full table scan per page. +CREATE INDEX IF NOT EXISTS messages_chat_created_id_idx + ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..ece32e1e --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,15 @@ +-- Shared idempotency for POST messages across API replicas. +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id UUID NOT NULL REFERENCES users (id), + method TEXT NOT NULL, + path TEXT NOT NULL, + idem_key TEXT NOT NULL, + request_hash TEXT NOT NULL, + response_status INT NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, method, path, idem_key) +); + +CREATE INDEX IF NOT EXISTS idempotency_keys_created_at_idx + ON idempotency_keys (created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..0174813c --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,26 @@ +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id UUID NOT NULL REFERENCES users (id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL UNIQUE, + status TEXT NOT NULL, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS attachments_chat_id_idx ON attachments (chat_id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments (id), + position INTEGER NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); + +CREATE INDEX IF NOT EXISTS message_attachments_attachment_id_idx + ON message_attachments (attachment_id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..fa33b4ec --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.6 +uvicorn==0.32.1 +pydantic==2.10.3 +starlette==0.41.3 +psycopg[binary,pool]==3.2.3 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.35.49 +Pillow==11.0.0 diff --git a/scripts/make_secrets.ps1 b/scripts/make_secrets.ps1 new file mode 100644 index 00000000..ac44b4b1 --- /dev/null +++ b/scripts/make_secrets.ps1 @@ -0,0 +1,19 @@ +# Creates local synthetic secrets for compose (never commit secrets/). +$ErrorActionPreference = "Stop" +$root = Split-Path -Parent $PSScriptRoot +$secrets = Join-Path $root "secrets" +New-Item -ItemType Directory -Force -Path $secrets | Out-Null + +$password = "lab2-pg-pass-" + [guid]::NewGuid().ToString("N").Substring(0, 12) +Set-Content -Path (Join-Path $secrets "postgres_password") -Value $password -NoNewline +$dbUrl = "postgresql://messenger:${password}@postgres:5432/messenger" +Set-Content -Path (Join-Path $secrets "database_url") -Value $dbUrl -NoNewline +Set-Content -Path (Join-Path $secrets "cache_url") -Value "redis://redis:6379/0" -NoNewline + +# Dedicated non-default S3 credentials for API (not built-in rustfs defaults). +$apiKey = "messengerapi" + [guid]::NewGuid().ToString("N").Substring(0, 8) +$apiSecret = "s3-api-" + [guid]::NewGuid().ToString("N") +Set-Content -Path (Join-Path $secrets "s3_access_key_id") -Value $apiKey -NoNewline +Set-Content -Path (Join-Path $secrets "s3_secret_access_key") -Value $apiSecret -NoNewline + +Write-Host "Wrote secrets/postgres_password, secrets/database_url, secrets/cache_url, secrets/s3_*" diff --git a/scripts/make_secrets.sh b/scripts/make_secrets.sh new file mode 100644 index 00000000..ffdf913b --- /dev/null +++ b/scripts/make_secrets.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +mkdir -p "$ROOT/secrets" +PASSWORD="lab2-pg-pass-$(openssl rand -hex 6)" +printf '%s' "$PASSWORD" > "$ROOT/secrets/postgres_password" +printf '%s' "postgresql://messenger:${PASSWORD}@postgres:5432/messenger" > "$ROOT/secrets/database_url" +printf '%s' "redis://redis:6379/0" > "$ROOT/secrets/cache_url" +printf '%s' "messengerapi$(openssl rand -hex 4)" > "$ROOT/secrets/s3_access_key_id" +printf '%s' "s3-api-$(openssl rand -hex 16)" > "$ROOT/secrets/s3_secret_access_key" +echo "Wrote secrets/postgres_password, secrets/database_url, secrets/cache_url, secrets/s3_*" diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..e7e425f2 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0.." +docker compose --env-file .env restart +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..7ccfdc7c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +# Restart LB, API replicas and state stores without removing volumes. +cd "$(dirname "$0")/.." +docker compose --env-file .env restart +exit 0 diff --git a/scripts/start-stopped-api.cmd b/scripts/start-stopped-api.cmd new file mode 100644 index 00000000..61363e08 --- /dev/null +++ b/scripts/start-stopped-api.cmd @@ -0,0 +1,5 @@ +@echo off +REM Start any stopped API containers after failover demo. +cd /d "%~dp0.." +docker compose --env-file .env start api +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..928aa533 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,17 @@ +@echo off +setlocal +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="" ( + echo TARGET_INSTANCE_ID required 1>&2 + exit /b 1 +) +for /f %%i in ('docker compose --env-file .env ps -q api') do ( + for /f %%h in ('docker inspect -f "{{.Config.Hostname}}" %%i') do ( + if "%%h"=="%TARGET_INSTANCE_ID%" ( + docker stop %%i >nul + exit /b 0 + ) + ) +) +echo API replica not found for TARGET_INSTANCE_ID 1>&2 +exit /b 1 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..9d91d201 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu +# Stop ONE API replica matching TARGET_INSTANCE_ID (= X-Instance-Id = container hostname). +# Leave it stopped; do not touch DB/LB/volumes. Return quickly. +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID required}" + +for id in $(docker compose --env-file .env ps -q api); do + [ -n "$id" ] || continue + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + if [ "$host" = "$TARGET_INSTANCE_ID" ]; then + docker stop "$id" >/dev/null + exit 0 + fi +done + +printf '%s\n' "API replica not found for TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/tests/own.py b/tests/own.py new file mode 100644 index 00000000..e274c204 --- /dev/null +++ b/tests/own.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 +"""Own lab4 tests (grade 4): attachments, limits, ACL, private bucket.""" +import hashlib +import struct +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parent +SAMPLE = (ROOT / "fixtures" / "sample.png").read_bytes() + + +def _png(width: int, height: int, rgba=(0, 128, 255, 255)) -> bytes: + """Minimal valid solid-color PNG.""" + import zlib + + def chunk(tag: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + raw = b"".join(b"\x00" + bytes(rgba) * width for _ in range(height)) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 6, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 9)) + + chunk(b"IEND", b"") + ) + + +def test_multiple_instance_ids(c: Client): + alice = c.user("inst") + seen = set() + for _ in range(40): + seen.add( + c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][ + "X-Instance-Id" + ] + ) + check(len(seen) >= 2, "expected >=2 X-Instance-Id via LB, got %r" % (seen,)) + print("PASS: observed instances %s" % sorted(seen)) + + +def test_attachment_happy_path(c: Client): + alice = c.user("attok") + bob = c.user("attokb") + chat = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Att", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + att, content = c.attachment(alice, chat) + meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")[ + "json" + ] + check(meta["sha256"] == hashlib.sha256(content).hexdigest(), "meta sha mismatch") + dl = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob) + check(dl["bytes"] == content, "download mismatch") + msg = c.message(alice, chat, "", attachments=[att["id"]])["json"] + check(msg["attachment_ids"] == [att["id"]], "message lost attachment_ids") + print("PASS: attachment upload/download/message") + + +def test_limits_and_mime(c: Client): + alice = c.user("limits") + chat = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Limits", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + path = "/api/v1/chats/" + chat["id"] + "/attachments?filename=x.png" + + huge = b"\x89PNG\r\n\x1a\n" + (b"0" * (10 * 1024 * 1024 + 1)) + c.call( + "POST", + path, + 413, + actor=alice, + raw=huge, + headers={"Content-Type": "image/png"}, + schema="Error", + ) + + c.call( + "POST", + path, + 415, + actor=alice, + raw=b"not-an-image", + headers={"Content-Type": "image/png"}, + schema="Error", + ) + + # MIME spoof: JPEG magic with image/png Content-Type + jpeg_magic = b"\xff\xd8\xff\xe0" + b"\x00" * 64 + c.call( + "POST", + path, + 415, + actor=alice, + raw=jpeg_magic, + headers={"Content-Type": "image/png"}, + schema="Error", + ) + + c.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=evil/../x.png", + 400, + actor=alice, + raw=SAMPLE, + headers={"Content-Type": "image/png"}, + schema="Error", + ) + print("PASS: 413/415/400 on bad uploads") + + +def test_acl_and_foreign_attachment(c: Client): + alice = c.user("acl_a") + bob = c.user("acl_b") + eve = c.user("acl_e") + chat_ab = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "AB", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + chat_ae = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "AE", "member_ids": [alice["id"], eve["id"]]}, + schema="Chat", + )["json"] + att, _ = c.attachment(alice, chat_ab) + c.call("GET", "/api/v1/attachments/" + att["id"], 404, actor=eve, schema="Error") + c.call( + "GET", + "/api/v1/attachments/" + att["id"] + "/content", + 404, + actor=eve, + schema="Error", + ) + # Attach AB file into AE chat → 404 + c.call( + "POST", + "/api/v1/chats/" + chat_ae["id"] + "/messages", + 404, + actor=alice, + body={"text": "", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + schema="Error", + ) + # Unknown attachment id + c.call( + "POST", + "/api/v1/chats/" + chat_ab["id"] + "/messages", + 404, + actor=alice, + body={"text": "", "attachment_ids": [str(uuid.uuid4())]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + schema="Error", + ) + print("PASS: outsider 404 and foreign chat attachment 404") + + +def test_private_bucket(c: Client): + """Unauthenticated GET to MinIO must not return the object.""" + alice = c.user("priv") + chat = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Priv", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + att, _ = c.attachment(alice, chat) + # Guess key pattern used by API; anonymous access must fail either way. + key = "attachments/%s/%s" % (chat["id"], att["id"]) + url = "http://127.0.0.1:9000/messenger-attachments/" + key + try: + urlopen(Request(url, method="GET"), timeout=5) + raise Failure("anonymous S3 GET unexpectedly succeeded") + except HTTPError as exc: + check(exc.code in (403, 404), "expected 403/404 from private bucket, got %s" % exc.code) + except URLError as exc: + # MinIO not published — still record that API download works. + print("WARN: cannot reach MinIO from host (%s); API ACL still checked" % exc.reason) + # Authorized API still works + c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=alice) + print("PASS: private bucket / API-only download") + + +def test_concurrent_uploads(c: Client): + alice = c.user("conc") + chat = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Conc", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + png = _png(64, 64) + barrier = threading.Barrier(10) + results = [] + + def worker(i): + local = Client(c.base, timeout=60) + barrier.wait() + try: + att = local.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=c%d.png" % i, + 201, + actor=alice, + raw=png, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + results.append(att["id"]) + except Failure as exc: + results.append(str(exc)) + + t0 = time.monotonic() + with ThreadPoolExecutor(max_workers=10) as pool: + list(pool.map(worker, range(10))) + elapsed = time.monotonic() - t0 + ids = [r for r in results if isinstance(r, str) and len(r) == 36] + check(len(ids) == 10, "expected 10 uploads, got %r" % (results,)) + check(len(set(ids)) == 10, "duplicate attachment ids") + print("PASS: 10 concurrent uploads in %.2fs (observe API mem_limit separately)" % elapsed) + + +def test_idempotency_with_attachments(c: Client): + alice = c.user("idematt") + chat = c.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "IdemAtt", "member_ids": [alice["id"]]}, + schema="Chat", + )["json"] + att, _ = c.attachment(alice, chat) + key = uuid.uuid4().hex + first = c.message(alice, chat, "", key=key, attachments=[att["id"]])["json"] + second = c.message(alice, chat, "", key=key, attachments=[att["id"]])["json"] + check(first == second, "idempotent attachment message replay") + print("PASS: idempotency with attachment_ids") + + +def main(): + args = arguments("Own lab4 tests") + c = Client(args.base_url, args.ca_file, timeout=120) + c.call("GET", "/health/ready", schema="Health") + test_multiple_instance_ids(c) + test_attachment_happy_path(c) + test_limits_and_mime(c) + test_acl_and_foreign_attachment(c) + test_private_bucket(c) + test_concurrent_uploads(c) + test_idempotency_with_attachments(c) + print("PASS: all own lab4 tests") + + +if __name__ == "__main__": + run(main)