diff --git a/.env.example b/.env.example index d99d26a2..4a867ac3 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,10 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 -S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 +DB_POOL_MAX_SIZE=8 +API_CPUS=1.00 +API_MEMORY=512M +UPLOAD_CONCURRENCY=1 +LB_PORT=8080 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a2df1023 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /srv/messenger +COPY requirements.txt . +RUN python -m pip install --no-cache-dir -r requirements.txt +COPY lab.json . +COPY contracts ./contracts +COPY app ./app +COPY scripts ./scripts +COPY tests ./tests + +USER 10001:10001 + +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1", "--timeout-keep-alive", "5", "--timeout-graceful-shutdown", "10"] diff --git a/Makefile b/Makefile index 5d86d171..da14338d 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,15 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help init-secrets check validate test test-grade4 test-s3-failure test-reconcile test-upload-load load test-persistence test-failover test-failover-target help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "init-secrets: create local secrets; check: offline structure; test: public smoke; test-grade4: negatives/concurrency" + @echo "test-s3-failure: stops/restarts local S3; test-reconcile: reconciliation fault injection" + @echo "test-upload-load: >=10 concurrent uploads; load: synthetic writes" + +init-secrets: + sh scripts/init-secrets.sh check: $(PYTHON) scripts/check_contract.py @@ -18,6 +23,18 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-grade4: + $(PYTHON) tests/grade4.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-s3-failure: + $(PYTHON) tests/s3_failure.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-reconcile: + @docker compose run --rm -T --no-deps api python tests/reconcile_integration.py + +test-upload-load: + $(PYTHON) tests/upload_load.py --base-url "$(BASE_URL)" --concurrency 10 $(if $(OUTPUT),--output "$(OUTPUT)",) + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -28,3 +45,8 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-failover-target: + @test -n "$(TARGET_INSTANCE_ID)" || (echo "Set TARGET_INSTANCE_ID to an observed X-Instance-Id"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + $(PYTHON) tests/replica_failover.py --target-id "$(TARGET_INSTANCE_ID)" --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) $(if $(OUTPUT),--output "$(OUTPUT)",) diff --git a/README.md b/README.md index 947094c3..32d7ef26 100644 --- a/README.md +++ b/README.md @@ -46,25 +46,39 @@ ## Как начать и проверить -```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: -make check - -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. -make test -``` - -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +Для этого решения используется Compose, PostgreSQL, Redis, два API, HAProxy и RustFS. В начале требуются Docker Compose v2 и Python 3.9+. ```sh -# Compose-вариант после реализации: +cp .env.example .env +make init-secrets docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +docker compose --env-file .env build api migrate +docker compose --env-file .env up -d postgres redis s3 +docker compose --env-file .env run --rm --no-deps s3-bootstrap +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api load_balancer + +until curl -fsS http://localhost:8080/health/ready; do sleep 1; done +make check +make test +make test-grade4 +make test-persistence ACTION_SCRIPT=scripts/restart.sh ``` +`.env.example` содержит только параметры без секретов; `make init-secrets` создаёт локальные файлы с правами `0600`. RustFS console и S3-порт не публикуются наружу. `s3-bootstrap` идемпотентно создаёт приватный bucket, scoped policy и отдельного пользователя API. Образ RustFS закреплён на версии 1.0.6. + +Тесты на восстановление и отказ: + +```sh +make test-s3-failure +make test-reconcile +make test-upload-load OUTPUT=evidence/upload-load.json +``` + +`test-reconcile` — fault-injection в отдельном учебном стенде: он создаёт синтетические записи и специально запускает `--apply --grace-hours 0`, чтобы проверить исправления; не запускайте его на полезных данных. Сам reconciler по умолчанию только dry-run и использует grace period 24 часа. + +`docker compose down` останавливает стенд, сохраняя volumes. Только намеренная команда `docker compose down -v` удаляет локальные volumes и учебные данные. Для полной OpenAPI-валидации установите `requirements-dev.txt` в venv и запустите `make validate`. + Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): @@ -76,6 +90,8 @@ make validate PYTHON=.venv/bin/python Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. +Для оценки 5 выбрана ветка исследования согласованности: `python scripts/reconcile.py` сначала выводит план; для применения нужен явный `--apply`. Объекты без метаданных и записи без объекта сверяются по уникальному server-generated key. Свежие orphans не удаляются, использованные вложения никогда не чистятся автоматически; missing object у используемого вложения остаётся отмечен в отчёте для ручного восстановления из резервной копии. Fault-injection в `tests/reconcile_integration.py` проверяет обнаружение, исправление, dry-run, grace period, повторный запуск и сохранность используемого вложения. + ## Сценарий защиты 1. Загрузить fixture, отправить message с attachment_ids, скачать как второй участник и сравнить SHA-256. diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..1c4693ef 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,100 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не указаны в исходных материалах; заполнить перед сдачей. +- Целевая оценка: 5. +- Выбранный трек: 2 — согласованность PostgreSQL/S3. +- Commit/PR: не создавались по просьбе автора. +- Основа: предоставленный skeleton лабораторной 4 и перенесённая реализация лабораторной 3. +- Библиотеки: FastAPI, psycopg, Redis, boto3, Pillow, argon2-cffi; версии заданы в `requirements.txt`. Помощь при реализации и проверках: Copilot. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Compose v2 и Python 3.9+ на хосте. Секреты генерируются локально и не добавляются в репозиторий. + +```sh +cp .env.example .env +make init-secrets +docker compose --env-file .env config --quiet +docker compose --env-file .env build api migrate +docker compose --env-file .env up -d postgres redis s3 +docker compose --env-file .env run --rm --no-deps s3-bootstrap +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api load_balancer + +until curl -fsS http://localhost:8080/health/ready; do sleep 1; done +make check +make test +make test-grade4 +make test-s3-failure +make test-upload-load OUTPUT=evidence/upload-load.json +make test-reconcile +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +``` + +Для точечного failover сначала укажите один из наблюдаемых `X-Instance-Id`: + +```sh +make test-failover-target TARGET_INSTANCE_ID= \ + ACTION_SCRIPT=scripts/stop-one.sh OUTPUT=evidence/failover-target.json +``` + +Обычная остановка сохраняет учебные volumes: `docker compose down`. Только намеренное удаление данных: `docker compose down -v`. Скрипт `scripts/restart.sh` перезапускает PostgreSQL, Redis, S3 и по очереди здоровые API-реплики; `scripts/stop-one.sh` останавливает ровно выбранную API-реплику и оставляет её остановленной. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент → HAProxy → API × 2+ + ├─ PostgreSQL: пользователи, чаты, сообщения, сессии, метаданные вложений + ├─ Redis: общие сессии + └─ приватный RustFS bucket: байты вложений +``` + +Проверенная среда: macOS 27 arm64; Docker client/server 29.1.3, Docker сообщает 12 CPU и около 7.65 GiB памяти. Образы закреплены в `compose.yaml`: Python 3.12.8 для API, PostgreSQL 16.6, Redis 7.4.2, HAProxy 3.1 и RustFS 1.0.6. Лимиты Compose: API 1 CPU/512 MiB на реплику, PostgreSQL 0.5 CPU/384 MiB, Redis 0.25 CPU/128 MiB, S3 0.75 CPU/512 MiB. Это локальный учебный стенд, не production-конфигурация. + +PostgreSQL является источником истины для прав и метаданных; объектный ключ формируется сервером как `attachments/`, а исходное имя хранится только для отображения. Составные внешние ключи в `message_attachments` не позволяют связать файл и сообщение из разных чатов. Bucket приватный; root credentials используются только bootstrap-сервисом, API получает отдельного пользователя с правами на bucket и `attachments/*`. Консоль и S3-порт не опубликованы. Bootstrap запускался повторно без ошибки. + +Загрузка проверяет членство пользователя в чате, имя и заявленный MIME, читает тело потоково с пределом 10 MiB и проверяет фактический PNG/JPEG через Pillow; лимит изображения — 40 млн пикселей. `SpooledTemporaryFile` хранит первые 1 MiB в памяти и переносит превышение во временный файл. Семафор ограничивает одновременные загрузки каждой API-реплики. Сначала выполняется S3 PUT, затем транзакция PostgreSQL; `201 ready` возвращается только после обоих шагов. Ошибка S3 даёт `503`; ошибка вставки метаданных запускает компенсирующий S3 delete. Если компенсация тоже не удалась, возможный orphan удаляется reconciler после grace period. + +Скачивание сначала проверяет авторизацию по метаданным, затем читает объект из S3 и сравнивает размер и SHA-256 с PostgreSQL. Приватный bucket не заменяет проверку доступа приложения: участники другого чата получают `404`. + +Reconciler по умолчанию только строит план. Он сканирует ключи с префиксом `attachments/`, сравнивает их с метаданными, а для удаления требует возраст не меньше 24 часов. В apply-режиме удаляются только старые orphan-объекты и старые отсутствующие metadata без ссылок в сообщениях. Отсутствующий объект у используемого вложения не удаляется автоматически: несоответствие сохраняется для ручного восстановления. Повторный запуск безопасен и идемпотентен для уже исправленного состояния. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Контракт и структура | `make check` | PASS; проверены структура, контракт и обязательные файлы | stdout при локальном запуске | +| Сквозной API, сохранение lab3 | `make test` | PASS; пользователи, сессии, чат, сообщения, права, пагинация и вложение | stdout при локальном запуске | +| Негативные проверки уровня 4 | `make test-grade4` | PASS; размер, повреждённый файл, MIME, пиксельный лимит, имя, cross-chat, чужое вложение и конкурентная идемпотентность | stdout при локальном запуске | +| S3 outage и восстановление | `make test-s3-failure` | PASS; upload вернул 503, после запуска S3 bucket и API восстановились | `evidence/s3-failure.txt` | +| Bootstrap/private bucket | повторный запуск `s3-bootstrap`; anonymous S3 GET в `make test-reconcile` | Два последовательных bootstrap завершились успешно; анонимное чтение запрещено | `evidence/bootstrap.txt`, `evidence/reconcile-integration.json` | +| Потоковая загрузка и нагрузка уровня 4 | `make test-upload-load OUTPUT=evidence/upload-load.json` | 10/10 загрузок по 4,322,578 байт, concurrency 10, время 3.05 s. Пиковая память API: 81,128,325 и 82,732,646 байт (семплирование раз в 0.5 s) | `evidence/upload-load.json` | +| Persistence/restart | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; подтверждённое сообщение, старый токен и оригинал вложения сохранились после перезапуска PostgreSQL, Redis, S3 и здоровых API | `evidence/persistence.txt` | +| Failover уровня 4 | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS; две реплики наблюдались до отказа, 113 успешных записей, 1 переходная ошибка, восстановление записи за 2.558 s, данные и сессия сохранены | `evidence/failover.txt` | +| Целевой failover | `make test-failover-target ... OUTPUT=evidence/failover-target.json` | До отказа наблюдались 2 реплики; после остановки target оставалась другая; 109 успешных записей, 1 ошибка записи, 0 ошибок чтения, восстановление за 2.454 s, дубликатов нет | `evidence/failover-target.json` | +| Трек 2: обнаружение и dry-run | `make test-reconcile` | Сканировано 38 строк metadata и 33 объекта за 0.081 s; dry-run ничего не удалил; свежий orphan сохранён; anonymous read запрещён | `evidence/reconcile-integration.json` | +| Трек 2: исправление и повтор | `make test-reconcile` | На синтетических расхождениях apply удалил 1 старый orphan и 1 старую неиспользуемую запись metadata; используемое вложение и referenced missing metadata сохранены; повтор удалил 0 объектов и 0 metadata | `evidence/reconcile-integration.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза:** ограничение размера, spooling и локальный семафор позволят обслужить не менее десяти параллельных загрузок без пропорционального удержания всех тел в RAM; reconciler с grace period не удалит свежие или связанные данные и повторный apply не внесёт повторных изменений. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**Методика загрузки:** синтетический PNG 1200×1200 размером 4,322,578 байт загружался десятью потоками через HAProxy. Во время сценария `docker stats` снимал память двух здоровых API-реплик каждые 0.5 s. Все десять запросов завершились успешно за 3.05 s; наблюдаемые пики составили 81,128,325 и 82,732,646 байт. Это абсолютные sampled peaks, не прирост относительно baseline. Один запуск и короткий профиль не дают статистической оценки; частота семплирования может пропустить кратковременный пик. Измерение исключает память PostgreSQL, S3, HAProxy и клиента. + +**Методика сверки:** интеграционный тест создал синтетические записи и удалил объекты из S3, чтобы получить два направления несогласованности; затем создал orphan. На измеренном запуске при 38 строках metadata и 33 объектах dry-run занял 0.081 s, не удалив данные. Apply с обычным grace period также ничего не удалил, потому что объект был свежим. В изолированном тесте с grace period 0 удалены один старый orphan и одна старая неиспользуемая строка metadata; missing object у связанного сообщения и существующее используемое вложение сохранены. Повторный apply удалил ноль объектов и строк. На таком малом наборе 0.081 s — только локальная оценка, а не прогноз для production. Реализация загружает список объектов и metadata в память, поэтому для больших bucket нужны измерения и обработка порциями. + +| Воздействие | Ожидание | Наблюдение | Восстановление/данные | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановить S3 во время upload | Явный 503, без ложного `ready` | PASS; после возврата S3 сервис снова стал ready | Автоматическое восстановление, persistent volumes сохранены | +| Перезапуск PostgreSQL, Redis, S3 и API | Сохранить сообщение, сессию и оригинал файла | PASS со старым токеном; hook сохранил volumes | Здоровые API перезапущены по очереди | +| Остановить одну API-реплику | LB продолжает обслуживать запросы другой; retry не дублирует запись | 113 успешных writes и 1 переходная ошибка в общем сценарии; targeted test сохранил 109 writes при 1 ошибке | В targeted test восстановление за 2.454 s; остановленная реплика оставлена остановленной, затем восстановлена вручную | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Измерения выполнены один раз на локальном Docker Desktop; для сравнения производительности нужны прогрев, несколько повторов, baseline/delta RAM и более длительная нагрузка. +- Reconciler выполняет полный проход по bucket и собирает списки в память. На больших объёмах потребуются порционная сверка, метрики, контроль времени/нагрузки и отдельное планирование запуска. +- Не автоматизируется восстановление потерянного байтового объекта, если к нему привязано сообщение; требуется backup/ручное восстановление. +- S3, PostgreSQL, Redis и HAProxy в Compose представлены одиночными экземплярами. Тестируется отказ API-реплики, но не отказ host, балансировщика, БД или самого S3-сервера. +- Upload синхронный и временные файлы используют локальный диск контейнера после 1 MiB; полноценные quotas, отдельный scratch volume и distributed concurrency limits не реализованы. +- Автор и группа не содержались в переданных материалах; их нужно вписать перед сдачей. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..90cd2ee1 --- /dev/null +++ b/app/main.py @@ -0,0 +1,1030 @@ +import base64 +import binascii +import asyncio +import hashlib +import io +import json +import logging +import os +import re +import secrets +import socket +import tempfile +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Tuple + +import boto3 +import botocore +from botocore.config import Config +import psycopg +import redis +from PIL import Image, UnidentifiedImageError +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError +from fastapi import Depends, FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator +from psycopg.errors import UniqueViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.concurrency import run_in_threadpool + +from app.settings import database_connection, read_setting, session_ttl_seconds + +logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO")) +logger = logging.getLogger("messenger") +PASSWORD_HASHER = PasswordHasher() +USERNAME_PATTERN = re.compile(r"^[a-z0-9_]{3,32}$") +CURSOR_VERSION = 1 +MAX_POOL_SIZE = int(os.environ.get("DB_POOL_MAX_SIZE", "10")) +if MAX_POOL_SIZE < 1 or MAX_POOL_SIZE > 50: + raise RuntimeError("DB_POOL_MAX_SIZE must be between 1 and 50") +MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024))) +MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) +UPLOAD_CONCURRENCY = int(os.environ.get("UPLOAD_CONCURRENCY", "1")) +if MAX_UPLOAD_BYTES != 10 * 1024 * 1024 or MAX_IMAGE_PIXELS != 40_000_000: + raise RuntimeError("Upload byte/pixel limits must match the public contract") +if UPLOAD_CONCURRENCY < 1 or UPLOAD_CONCURRENCY > 4: + raise RuntimeError("UPLOAD_CONCURRENCY must be between 1 and 4") +Image.MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS + + +class APIError(Exception): + def __init__( + self, status_code: int, code: str, message: str, challenge: Optional[str] = None + ) -> None: + self.status_code = status_code + self.code = code + self.message = message + self.challenge = challenge + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class UserInput(StrictModel): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def display_name_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("display_name must not be blank") + return value + + @field_validator("password") + @classmethod + def password_no_nul(cls, value: str) -> str: + if "\x00" in value: + raise ValueError("password contains an invalid character") + return value + + +class ChatInput(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: List[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title must not be blank") + return value + + +class MessageInput(StrictModel): + text: str = Field(max_length=4000) + attachment_ids: List[str] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def unique_attachment_ids(cls, value: List[str]) -> List[str]: + if len(set(value)) != len(value): + raise ValueError("attachment_ids must be unique") + return value + + +def utc_string(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace( + "+00:00", "Z" + ) + + +def user_json(row: Dict[str, Any]) -> Dict[str, str]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": utc_string(row["created_at"]), + } + + +def chat_json(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": utc_string(row["created_at"]), + } + + +def message_json(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": utc_string(row["created_at"]), + "attachment_ids": [str(value) for value in row.get("attachment_ids", [])], + } + + +def db_unavailable() -> APIError: + return APIError(503, "unavailable", "Сервис временно недоступен") + + +def get_connection(): + pool: ConnectionPool = app.state.pool + try: + with pool.connection(timeout=2) as connection: + yield connection + except (psycopg.OperationalError, PoolTimeout) as exc: + logger.warning("database connection unavailable: %s", type(exc).__name__) + raise db_unavailable() from None + + +@asynccontextmanager +async def lifespan(application: FastAPI): + session_ttl_seconds() + application.state.pool = ConnectionPool( + conninfo="", + kwargs={ + **database_connection(), + "autocommit": True, + "row_factory": dict_row, + }, + min_size=1, + max_size=MAX_POOL_SIZE, + timeout=2, + max_lifetime=1800, + open=True, + name="messenger-api", + ) + redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") + application.state.redis = redis.Redis.from_url( + redis_url, + socket_connect_timeout=1, + socket_timeout=1, + decode_responses=True, + health_check_interval=30, + ) + application.state.instance_id = socket.gethostname() + s3_config = Config( + connect_timeout=2, + read_timeout=5, + retries={"max_attempts": 2, "mode": "standard"}, + signature_version="s3v4", + ) + application.state.s3 = boto3.client( + "s3", + endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"), + region_name=os.environ.get("S3_REGION", "us-east-1"), + aws_access_key_id=read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"), + aws_secret_access_key=read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"), + config=s3_config, + ) + application.state.s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + application.state.upload_slots = asyncio.Semaphore(UPLOAD_CONCURRENCY) + try: + yield + finally: + application.state.redis.close() + application.state.pool.close() + application.state.s3.close() + + +app = FastAPI(title="Messenger API", version="1.0.0", lifespan=lifespan) + + +@app.middleware("http") +async def request_metadata(request: Request, call_next): + request_id = request.headers.get("X-Request-Id", "") + if not request_id or len(request_id) > 128: + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception: + logger.exception("unhandled request failure") + response = JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "Внутренняя ошибка", + "request_id": request_id, + } + }, + ) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = request.app.state.instance_id + return response + + +@app.exception_handler(APIError) +async def handle_api_error(request: Request, exc: APIError): + headers = {"WWW-Authenticate": exc.challenge} if exc.challenge else None + return JSONResponse( + status_code=exc.status_code, + content={ + "error": { + "code": exc.code, + "message": exc.message, + "request_id": request.state.request_id, + } + }, + headers=headers, + ) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error(request: Request, exc: RequestValidationError): + return JSONResponse( + status_code=400, + content={ + "error": { + "code": "invalid_request", + "message": "Некорректный запрос", + "request_id": request.state.request_id, + } + }, + ) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException): + status = 404 if exc.status_code == 404 else exc.status_code + code = "not_found" if status == 404 else "invalid_request" + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": "Объект не найден" if status == 404 else "Некорректный запрос", + "request_id": request.state.request_id, + } + }, + ) + + +@app.exception_handler(psycopg.Error) +async def handle_database_error(request: Request, exc: psycopg.Error): + logger.warning("database operation failed: %s", type(exc).__name__) + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "Сервис временно недоступен", + "request_id": request.state.request_id, + } + }, + ) + + +@app.exception_handler(PoolTimeout) +async def handle_pool_timeout(request: Request, exc: PoolTimeout): + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "Сервис временно недоступен", + "request_id": request.state.request_id, + } + }, + ) + + +def auth_error(challenge: str = 'Bearer realm="messenger"') -> APIError: + return APIError(401, "unauthorized", "Требуется действительная аутентификация", challenge) + + +def parse_uuid(value: str) -> uuid.UUID: + try: + return uuid.UUID(value) + except (ValueError, AttributeError, TypeError): + raise APIError(400, "invalid_request", "Некорректный UUID") from None + + +def authenticated_user( + request: Request, + authorization: Optional[str] = Header(default=None), + connection=Depends(get_connection), +) -> Dict[str, Any]: + if not authorization or not authorization.startswith("Bearer "): + raise auth_error() + token = authorization[7:] + if not token or len(token) > 256: + raise auth_error() + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + now = datetime.now(timezone.utc) + cache = request.app.state.redis + cache_key = "session:" + token_hash.hex() + cached: Optional[Dict[str, Any]] = None + try: + raw = cache.get(cache_key) + if raw: + cached_value = json.loads(raw) + if isinstance(cached_value, dict): + cached = cached_value + else: + logger.warning("session cache entry has an invalid shape") + except (redis.RedisError, json.JSONDecodeError): + logger.warning("session cache read failed; using PostgreSQL") + try: + session = connection.execute( + """ + SELECT s.user_id, s.expires_at, s.revoked_at + FROM sessions AS s + WHERE s.token_hash = %s + """, + (token_hash,), + ).fetchone() + except psycopg.Error as exc: + logger.warning("session lookup failed: %s", type(exc).__name__) + raise db_unavailable() from None + if not session or session["revoked_at"] is not None or session["expires_at"] <= now: + try: + cache.delete(cache_key) + except redis.RedisError: + logger.warning("expired session cache invalidation failed") + raise auth_error() + # PostgreSQL remains authoritative for expiry and revocation. Redis avoids + # a user-profile query on a cache hit. + if cached and cached.get("user_id") == str(session["user_id"]): + profile = cached.get("user") + if isinstance(profile, dict) and profile.get("id") == str(session["user_id"]): + user = profile + else: + cached = None + else: + cached = None + if not cached: + profile_row = connection.execute( + """ + SELECT id, username, display_name, created_at + FROM users WHERE id = %s + """, + (session["user_id"],), + ).fetchone() + if not profile_row: + raise auth_error() + user = user_json(profile_row) + remaining = max(1, int((session["expires_at"] - now).total_seconds())) + try: + cache.set( + cache_key, + json.dumps({"user_id": str(session["user_id"]), "user": user}), + ex=remaining, + ) + except redis.RedisError: + logger.warning("session cache write failed; continuing with PostgreSQL") + request.state.token_hash = token_hash + return user + + +@app.get("/health/live") +def live() -> Dict[str, str]: + return {"status": "ok"} + + +@app.get("/health/ready") +def ready(request: Request, connection=Depends(get_connection)) -> Dict[str, str]: + try: + connection.execute("SELECT 1") + request.app.state.s3.head_bucket(Bucket=request.app.state.s3_bucket) + except psycopg.Error: + raise db_unavailable() from None + except botocore.exceptions.BotoCoreError: + raise APIError(503, "unavailable", "S3-хранилище временно недоступно") from None + except botocore.exceptions.ClientError: + raise APIError(503, "unavailable", "S3-хранилище временно недоступно") from None + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def register_user(payload: UserInput, connection=Depends(get_connection)): + user_id = uuid.uuid4() + password_hash = PASSWORD_HASHER.hash(payload.password) + try: + with connection.transaction(): + row = connection.execute( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (user_id, payload.username, payload.display_name, password_hash), + ).fetchone() + except UniqueViolation: + raise APIError(409, "conflict", "Имя пользователя уже занято") from None + except psycopg.Error as exc: + logger.warning("user registration failed: %s", type(exc).__name__) + raise db_unavailable() from None + return user_json(row) + + +def decode_basic_credentials(authorization: Optional[str]) -> Tuple[str, str]: + challenge = 'Basic realm="messenger", charset="UTF-8"' + if not authorization or not authorization.startswith("Basic "): + raise auth_error(challenge) + try: + raw = base64.b64decode(authorization[6:], validate=True).decode("utf-8") + username, password = raw.split(":", 1) + except (binascii.Error, UnicodeDecodeError, ValueError): + raise auth_error(challenge) from None + if not USERNAME_PATTERN.fullmatch(username) or not 12 <= len(password) <= 128: + raise auth_error(challenge) + return username, password + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session( + request: Request, + authorization: Optional[str] = Header(default=None), + connection=Depends(get_connection), +): + username, password = decode_basic_credentials(authorization) + row = connection.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + if not row: + raise auth_error('Basic realm="messenger", charset="UTF-8"') + try: + PASSWORD_HASHER.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + raise auth_error('Basic realm="messenger", charset="UTF-8"') from None + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + expires_at = datetime.now(timezone.utc) + timedelta(seconds=session_ttl_seconds()) + try: + with connection.transaction(): + connection.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, %s) + """, + (token_hash, row["id"], expires_at), + ) + except psycopg.Error as exc: + logger.warning("session creation failed: %s", type(exc).__name__) + raise db_unavailable() from None + user = user_json(row) + try: + request.app.state.redis.set( + "session:" + token_hash.hex(), + json.dumps({"user_id": str(row["id"]), "user": user}), + ex=session_ttl_seconds(), + ) + except redis.RedisError: + logger.warning("session cache write failed; session remains durable in PostgreSQL") + return { + "token": token, + "token_type": "Bearer", + "expires_at": utc_string(expires_at), + "user": user, + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session( + request: Request, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + token_hash: bytes = request.state.token_hash + try: + with connection.transaction(): + result = connection.execute( + """ + UPDATE sessions SET revoked_at = clock_timestamp() + WHERE token_hash = %s AND revoked_at IS NULL + """, + (token_hash,), + ) + if result.rowcount != 1: + raise auth_error() + except APIError: + raise + except psycopg.Error as exc: + logger.warning("session revocation failed: %s", type(exc).__name__) + raise db_unavailable() from None + try: + request.app.state.redis.delete("session:" + token_hash.hex()) + except redis.RedisError: + logger.warning("revoked session cache invalidation failed") + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def current_user(user: Dict[str, Any] = Depends(authenticated_user)): + return user + + +def member_ids_for(connection, chat_id: uuid.UUID) -> List[str]: + rows = connection.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", + (chat_id,), + ).fetchall() + return [str(row["user_id"]) for row in rows] + + +def require_chat_member( + connection, chat_id: uuid.UUID, user_id: uuid.UUID +) -> Dict[str, Any]: + row = connection.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats AS c JOIN chat_members AS cm ON cm.chat_id = c.id + WHERE c.id = %s AND cm.user_id = %s + """, + (chat_id, user_id), + ).fetchone() + if not row: + raise APIError(404, "not_found", "Объект не найден") + return row + + +@app.post("/api/v1/chats", status_code=201) +def create_chat( + payload: ChatInput, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + ids = [parse_uuid(member_id) for member_id in payload.member_ids] + if len(set(ids)) != len(ids) or uuid.UUID(user["id"]) not in ids: + raise APIError(400, "invalid_request", "Некорректный список участников") + chat_id = uuid.uuid4() + try: + with connection.transaction(): + found = connection.execute( + "SELECT id FROM users WHERE id = ANY(%s)", + (ids,), + ).fetchall() + if len(found) != len(ids): + raise APIError(400, "invalid_request", "Участник не существует") + chat = connection.execute( + """ + INSERT INTO chats (id, title) VALUES (%s, %s) + RETURNING id, title, created_at + """, + (chat_id, payload.title), + ).fetchone() + for member_id in ids: + connection.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (chat_id, member_id), + ) + except APIError: + raise + except psycopg.Error as exc: + logger.warning("chat creation rolled back: %s", type(exc).__name__) + raise db_unavailable() from None + return chat_json(chat, [str(member_id) for member_id in sorted(ids)]) + + +@app.get("/api/v1/chats") +def list_chats( + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), + limit: int = 50, + cursor: Optional[str] = None, +): + if limit < 1 or limit > 100: + raise APIError(400, "invalid_request", "limit должен быть от 1 до 100") + cursor_values = decode_cursor(cursor, "chats") if cursor else None + parameters: List[Any] = [uuid.UUID(user["id"])] + clause = "" + if cursor_values: + clause = "AND (c.created_at, c.id) > (%s, %s)" + parameters.extend(cursor_values) + parameters.append(limit + 1) + rows = connection.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = %s + """ + + clause + + " ORDER BY c.created_at, c.id LIMIT %s", + parameters, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + items = [ + chat_json(row, member_ids_for(connection, row["id"])) + for row in rows + ] + next_cursor = encode_cursor("chats", rows[-1]) if has_more and rows else None + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat( + chat_id: str, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + parsed_id = parse_uuid(chat_id) + row = require_chat_member(connection, parsed_id, uuid.UUID(user["id"])) + return chat_json(row, member_ids_for(connection, parsed_id)) + + +def encode_cursor(kind: str, row: Dict[str, Any]) -> str: + payload = json.dumps( + [CURSOR_VERSION, kind, str(row["created_at"]), str(row["id"])], + separators=(",", ":"), + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def decode_cursor(value: str, kind: str) -> Tuple[datetime, uuid.UUID]: + try: + padded = value + "=" * (-len(value) % 4) + decoded = json.loads(base64.urlsafe_b64decode(padded.encode("ascii"))) + if ( + not isinstance(decoded, list) + or len(decoded) != 4 + or decoded[0] != CURSOR_VERSION + or decoded[1] != kind + ): + raise ValueError + created_at = datetime.fromisoformat(decoded[2]) + if created_at.tzinfo is None: + raise ValueError + return created_at, uuid.UUID(decoded[3]) + except (ValueError, TypeError, UnicodeError, json.JSONDecodeError): + raise APIError(400, "invalid_request", "Некорректный cursor") from None + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + payload: MessageInput, + idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"), + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + if not idempotency_key or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key): + raise APIError(400, "invalid_request", "Некорректный Idempotency-Key") + parsed_chat_id = parse_uuid(chat_id) + chat = require_chat_member(connection, parsed_chat_id, uuid.UUID(user["id"])) + if not payload.text.strip() and not payload.attachment_ids: + raise APIError(400, "invalid_request", "Сообщение не может быть пустым") + attachment_ids = [parse_uuid(value) for value in payload.attachment_ids] + sender_id = uuid.UUID(user["id"]) + request_hash = hashlib.sha256( + json.dumps( + {"text": payload.text, "attachment_ids": [str(value) for value in attachment_ids]}, + ensure_ascii=False, + sort_keys=True, + ).encode("utf-8") + ).digest() + legacy_request_hash = hashlib.sha256( + json.dumps({"text": payload.text}, ensure_ascii=False, sort_keys=True).encode("utf-8") + ).digest() + try: + with connection.transaction(): + if attachment_ids: + attachments = connection.execute( + """ + SELECT id, status FROM attachments + WHERE chat_id = %s AND id = ANY(%s) + """, + (chat["id"], attachment_ids), + ).fetchall() + if len(attachments) != len(attachment_ids): + raise APIError(404, "not_found", "Объект не найден") + if any(row["status"] != "ready" for row in attachments): + raise APIError(409, "conflict", "Вложение ещё не готово") + claim = connection.execute( + """ + INSERT INTO idempotency_keys + (user_id, chat_id, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s) + ON CONFLICT (user_id, chat_id, idempotency_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, + message_id = NULL, + created_at = clock_timestamp() + WHERE idempotency_keys.created_at < + clock_timestamp() - interval '24 hours' + RETURNING user_id + """, + (sender_id, chat["id"], idempotency_key, request_hash), + ).fetchone() + if claim: + row = connection.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at + """, + (uuid.uuid4(), chat["id"], sender_id, payload.text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + connection.execute( + """ + INSERT INTO message_attachments + (message_id, chat_id, attachment_id, ordinal) + VALUES (%s, %s, %s, %s) + """, + (row["id"], chat["id"], attachment_id, ordinal), + ) + row["attachment_ids"] = attachment_ids + result = message_json(row) + connection.execute( + """ + UPDATE idempotency_keys SET message_id = %s + WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s + """, + (row["id"], sender_id, chat["id"], idempotency_key), + ) + else: + existing = connection.execute( + """ + SELECT request_hash, message_id + FROM idempotency_keys + WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s + """, + (sender_id, chat["id"], idempotency_key), + ).fetchone() + compatible_hashes = {request_hash} + if not attachment_ids: + compatible_hashes.add(legacy_request_hash) + if not existing or existing["request_hash"] not in compatible_hashes: + raise APIError(409, "conflict", "Ключ идемпотентности уже использован") + row = connection.execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE(array_agg(ma.attachment_id ORDER BY ma.ordinal) + FILTER (WHERE ma.attachment_id IS NOT NULL), '{}') AS attachment_ids + FROM messages AS m + LEFT JOIN message_attachments AS ma ON ma.message_id = m.id + WHERE m.id = %s + GROUP BY m.id + """, + (existing["message_id"],), + ).fetchone() + if not row: + raise APIError(503, "unavailable", "Результат запроса временно недоступен") + result = message_json(row) + except APIError: + raise + except psycopg.Error as exc: + logger.warning("message write failed: %s", type(exc).__name__) + raise db_unavailable() from None + return result + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: str, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), + limit: int = 50, + cursor: Optional[str] = None, +): + if limit < 1 or limit > 100: + raise APIError(400, "invalid_request", "limit должен быть от 1 до 100") + parsed_chat_id = parse_uuid(chat_id) + chat = require_chat_member(connection, parsed_chat_id, uuid.UUID(user["id"])) + cursor_values = decode_cursor(cursor, str(parsed_chat_id)) if cursor else None + parameters: List[Any] = [chat["id"]] + clause = "" + if cursor_values: + clause = "AND (created_at, id) > (%s, %s)" + parameters.extend(cursor_values) + parameters.append(limit + 1) + rows = connection.execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + COALESCE(array_agg(ma.attachment_id ORDER BY ma.ordinal) + FILTER (WHERE ma.attachment_id IS NOT NULL), '{}') AS attachment_ids + FROM messages AS m + LEFT JOIN message_attachments AS ma ON ma.message_id = m.id + WHERE m.chat_id = %s + """ + + clause + + " GROUP BY m.id ORDER BY m.created_at, m.id LIMIT %s", + parameters, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + items = [message_json(row) for row in rows] + next_cursor = encode_cursor(str(parsed_chat_id), rows[-1]) if has_more and rows else None + return {"items": items, "next_cursor": next_cursor} + + +def attachment_json(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"].strip(), + "status": row["status"], + "variants": [], + "error_code": None, + "created_at": utc_string(row["created_at"]), + "updated_at": utc_string(row["updated_at"]), + } + + +def attachment_for_member(connection, attachment_id: uuid.UUID, user_id: uuid.UUID): + row = connection.execute( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.object_key, a.status, + a.created_at, a.updated_at + FROM attachments AS a + JOIN chat_members AS cm ON cm.chat_id = a.chat_id + WHERE a.id = %s AND cm.user_id = %s + """, + (attachment_id, user_id), + ).fetchone() + if not row: + raise APIError(404, "not_found", "Объект не найден") + return row + + +def image_details(upload, declared_type: str) -> Tuple[str, int, int]: + try: + upload.seek(0) + with Image.open(upload) as image: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) + if actual_type != declared_type: + raise APIError(415, "unsupported_media_type", "Тип файла не соответствует содержимому") + width, height = image.size + if width < 1 or height < 1 or width * height > MAX_IMAGE_PIXELS: + raise APIError(413, "payload_too_large", "Превышен лимит размера изображения") + image.verify() + upload.seek(0) + with Image.open(upload) as image: + image.load() + except APIError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise APIError(413, "payload_too_large", "Превышен лимит размера изображения") from None + except (UnidentifiedImageError, OSError, ValueError): + raise APIError(415, "unsupported_media_type", "Повреждённое или неподдерживаемое изображение") from None + upload.seek(0) + return declared_type, width, height + + +def s3_unavailable() -> APIError: + return APIError(503, "unavailable", "Файловое хранилище временно недоступно") + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + parsed_chat_id = parse_uuid(chat_id) + require_chat_member(connection, parsed_chat_id, uuid.UUID(user["id"])) + if any(char in filename for char in ("/", "\\", "\x00")) or any( + ord(char) < 32 or 127 <= ord(char) <= 159 for char in filename + ): + raise APIError(400, "invalid_request", "Некорректное имя файла") + declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if declared_type not in {"image/png", "image/jpeg"}: + raise APIError(415, "unsupported_media_type", "Поддерживаются только PNG и JPEG") + content_length = request.headers.get("content-length") + if content_length: + try: + if int(content_length) > MAX_UPLOAD_BYTES: + raise APIError(413, "payload_too_large", "Файл превышает лимит 10 MiB") + except ValueError: + raise APIError(400, "invalid_request", "Некорректный Content-Length") from None + + async with request.app.state.upload_slots: + digest = hashlib.sha256() + size = 0 + with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b") as upload: + async for chunk in request.stream(): + size += len(chunk) + if size > MAX_UPLOAD_BYTES: + raise APIError(413, "payload_too_large", "Файл превышает лимит 10 MiB") + digest.update(chunk) + upload.write(chunk) + if size == 0: + raise APIError(400, "invalid_request", "Файл не может быть пустым") + actual_type, _, _ = image_details(upload, declared_type) + attachment_id = uuid.uuid4() + object_key = "attachments/" + str(attachment_id) + s3 = request.app.state.s3 + bucket = request.app.state.s3_bucket + try: + await run_in_threadpool( + s3.upload_fileobj, + upload, + bucket, + object_key, + ExtraArgs={ + "ContentType": actual_type, + "Metadata": {"sha256": digest.hexdigest()}, + }, + ) + except botocore.exceptions.BotoCoreError: + logger.warning("S3 upload failed: %s", object_key) + raise s3_unavailable() from None + except botocore.exceptions.ClientError: + logger.warning("S3 upload failed: %s", object_key) + raise s3_unavailable() from None + try: + with connection.transaction(): + row = connection.execute( + """ + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, status, created_at, updated_at + """, + ( + attachment_id, + parsed_chat_id, + uuid.UUID(user["id"]), + filename, + actual_type, + size, + digest.hexdigest(), + object_key, + ), + ).fetchone() + except psycopg.Error as exc: + logger.warning("attachment metadata write failed: %s", type(exc).__name__) + try: + await run_in_threadpool(s3.delete_object, Bucket=bucket, Key=object_key) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError): + logger.exception("S3 compensation failed for %s", object_key) + raise db_unavailable() from None + return attachment_json(row) + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment( + attachment_id: str, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + row = attachment_for_member( + connection, parse_uuid(attachment_id), uuid.UUID(user["id"]) + ) + return attachment_json(row) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + request: Request, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), + variant: str = "original", +): + if variant != "original": + raise APIError(404, "not_found", "Объект не найден") + row = attachment_for_member( + connection, parse_uuid(attachment_id), uuid.UUID(user["id"]) + ) + try: + result = request.app.state.s3.get_object( + Bucket=request.app.state.s3_bucket, Key=row["object_key"] + ) + with result["Body"] as body: + content = body.read(MAX_UPLOAD_BYTES + 1) + except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError): + logger.warning("S3 download failed for attachment %s", row["id"]) + raise s3_unavailable() from None + if len(content) != row["size_bytes"] or hashlib.sha256(content).hexdigest() != row["sha256"].strip(): + logger.error("S3 object integrity check failed for attachment %s", row["id"]) + raise s3_unavailable() + return Response(content=content, media_type=row["content_type"]) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..559901f2 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,35 @@ +from pathlib import Path + +import psycopg + +from app.settings import database_connection + + +def main() -> None: + migration_dir = Path(__file__).with_name("migrations") + with psycopg.connect(**database_connection(), autocommit=False) as connection: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version varchar(255) PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + """ + ) + for migration in sorted(migration_dir.glob("*.sql")): + already_applied = connection.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (migration.name,), + ).fetchone() + if already_applied: + continue + connection.execute(migration.read_text(encoding="utf-8")) + connection.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (migration.name,), + ) + connection.commit() + + +if __name__ == "__main__": + main() diff --git a/app/migrations/001_initial.sql b/app/migrations/001_initial.sql new file mode 100644 index 00000000..6cccb668 --- /dev/null +++ b/app/migrations/001_initial.sql @@ -0,0 +1,54 @@ +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_nonblank CHECK (length(btrim(display_name)) > 0) +); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_nonblank CHECK (length(btrim(title)) > 0) +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_by_user + ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_nonblank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX IF NOT EXISTS messages_history_keyset + ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX IF NOT EXISTS sessions_active_by_expiry + ON sessions (expires_at) + WHERE revoked_at IS NULL; + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version varchar(255) PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/app/migrations/002_idempotency.sql b/app/migrations/002_idempotency.sql new file mode 100644 index 00000000..550e73e7 --- /dev/null +++ b/app/migrations/002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + message_id uuid REFERENCES messages(id) ON DELETE SET NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idempotency_keys_expiry + ON idempotency_keys (created_at); diff --git a/app/migrations/003_attachments.sql b/app/migrations/003_attachments.sql new file mode 100644 index 00000000..016b1b4b --- /dev/null +++ b/app/migrations/003_attachments.sql @@ -0,0 +1,34 @@ +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename varchar(255) NOT NULL, + content_type varchar(32) NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key varchar(160) NOT NULL UNIQUE, + status varchar(16) NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_filename_nonblank CHECK (length(filename) > 0), + UNIQUE (id, chat_id) +); + +CREATE INDEX IF NOT EXISTS attachments_by_chat + ON attachments (chat_id, created_at, id); + +CREATE UNIQUE INDEX IF NOT EXISTS messages_id_chat_unique + ON messages (id, chat_id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + ordinal smallint NOT NULL CHECK (ordinal BETWEEN 0 AND 9), + PRIMARY KEY (message_id, ordinal), + UNIQUE (message_id, attachment_id), + FOREIGN KEY (message_id, chat_id) + REFERENCES messages(id, chat_id) ON DELETE CASCADE, + FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments(id, chat_id) ON DELETE RESTRICT +); diff --git a/app/migrations/004_message_attachments_allow_empty_text.sql b/app/migrations/004_message_attachments_allow_empty_text.sql new file mode 100644 index 00000000..e3f53906 --- /dev/null +++ b/app/migrations/004_message_attachments_allow_empty_text.sql @@ -0,0 +1,2 @@ +ALTER TABLE messages + DROP CONSTRAINT IF EXISTS messages_text_nonblank; diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 00000000..e473c91e --- /dev/null +++ b/app/settings.py @@ -0,0 +1,54 @@ +import os +from pathlib import Path +from typing import Dict, Optional + + +def read_setting( + name: str, + file_name: str, + default: Optional[str] = None, + default_file: Optional[str] = None, +) -> str: + configured_secret_path = os.environ.get(file_name) + secret_path = configured_secret_path or default_file + if secret_path and (configured_secret_path or Path(secret_path).is_file()): + try: + return Path(secret_path).read_text(encoding="utf-8").strip() + except OSError as exc: + raise RuntimeError("Unable to read configured secret file") from exc + value = os.environ.get(name, default) + if value is None: + if default_file: + raise RuntimeError( + "Required setting is missing: " + + name + + "; set " + + file_name + + " or mount the secret at " + + default_file + ) + raise RuntimeError("Required application setting is missing: " + name) + return value + + +def database_connection() -> Dict[str, object]: + return { + "host": os.environ.get("DB_HOST", "localhost"), + "port": int(os.environ.get("DB_PORT", "5432")), + "dbname": os.environ.get("DB_NAME", "messenger"), + "user": os.environ.get("DB_USER", "messenger"), + "password": read_setting( + "DB_PASSWORD", + "DB_PASSWORD_FILE", + default_file="/run/secrets/database_password", + ), + "connect_timeout": int(os.environ.get("DB_CONNECT_TIMEOUT_SECONDS", "3")), + "options": "-c statement_timeout=5000 -c lock_timeout=3000", + } + + +def session_ttl_seconds() -> int: + value = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + if value < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be at least 1") + return value diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..a71c644c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,228 @@ +services: + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/database_password + secrets: + - database_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 2s + retries: 20 + cpus: 0.50 + mem_limit: 384m + restart: unless-stopped + networks: + - backend + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 20 + cpus: 0.25 + mem_limit: 128m + restart: unless-stopped + networks: + - backend + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "app.migrate"] + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: messenger + DB_USER: messenger + DB_PASSWORD_FILE: /run/secrets/database_password + secrets: + - database_password + depends_on: + postgres: + condition: service_healthy + cpus: 0.50 + mem_limit: 256m + networks: + - backend + profiles: + - tools + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: messenger + DB_USER: messenger + DB_PASSWORD_FILE: /run/secrets/database_password + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} + REDIS_URL: redis://redis:6379/0 + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: 10485760 + MAX_IMAGE_PIXELS: 40000000 + UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-1} + secrets: + - database_password + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + interval: 2s + timeout: 2s + retries: 3 + start_period: 8s + stop_grace_period: 12s + deploy: + replicas: ${API_REPLICAS:-2} + endpoint_mode: dnsrr + resources: + limits: + cpus: "${API_CPUS:-1.00}" + memory: ${API_MEMORY:-512M} + restart: unless-stopped + networks: + - backend + + load_balancer: + image: haproxy:3.1-alpine + ports: + - "${LB_PORT:-8080}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 5s + timeout: 2s + retries: 3 + cpus: 0.25 + mem_limit: 128m + restart: unless-stopped + networks: + - frontend + - backend + + s3: + image: rustfs/rustfs:1.0.6 + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: 0.0.0.0:9000 + RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001 + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_REGION: us-east-1 + entrypoint: ["/bin/sh", "-ec"] + command: + - | + export RUSTFS_ACCESS_KEY="$$(cat /run/secrets/s3_root_access_key)" + export RUSTFS_SECRET_KEY="$$(cat /run/secrets/s3_root_secret_key)" + exec /entrypoint.sh + secrets: + - s3_root_access_key + - s3_root_secret_key + volumes: + - s3_data:/data + restart: unless-stopped + networks: + - backend + cpus: 0.75 + mem_limit: 512m + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/health"] + interval: 3s + timeout: 2s + retries: 20 + start_period: 8s + depends_on: + s3-volume-permissions: + condition: service_completed_successfully + + s3-volume-permissions: + image: alpine:3.21.3 + user: "0:0" + command: ["sh", "-c", "chown -R 10001:10001 /data"] + volumes: + - s3_data:/data + restart: "no" + networks: + - backend + + s3-bootstrap: + build: + context: . + dockerfile: Dockerfile + command: ["python", "scripts/bootstrap-s3.py"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key + S3_ROOT_SECRET_KEY_FILE: /run/secrets/s3_root_secret_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - s3_root_access_key + - s3_root_secret_key + - s3_access_key_id + - s3_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + networks: + - backend + +secrets: + database_password: + file: ./secrets/database_password + s3_root_access_key: + file: ./secrets/s3_root_access_key + s3_root_secret_key: + file: ./secrets/s3_root_secret_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + postgres_data: + redis_data: + s3_data: + +networks: + frontend: + backend: + internal: true diff --git a/evidence/bootstrap.txt b/evidence/bootstrap.txt new file mode 100644 index 00000000..2aef3fcc --- /dev/null +++ b/evidence/bootstrap.txt @@ -0,0 +1,2 @@ +S3 bootstrap complete: private bucket and scoped API credentials are ready. +S3 bootstrap complete: private bucket and scoped API credentials are ready. diff --git a/evidence/failover-target.json b/evidence/failover-target.json new file mode 100644 index 00000000..7c3dcf4c --- /dev/null +++ b/evidence/failover-target.json @@ -0,0 +1,16 @@ +{ + "target_instance_id": "e162139af177", + "observed_instances_before": 2, + "instances_serving_after": [ + "ad823d83c069" + ], + "duration_seconds": 15.121, + "successful_writes": 109, + "failed_write_attempts": 1, + "failed_read_attempts": 0, + "successful_rps": 7.209, + "write_error_ratio": 0.009091, + "recovery_seconds": 2.454, + "confirmed_data_preserved": true, + "duplicate_messages": 0 +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..a4081c09 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 113, "failed_attempts": 1, "first_recovery_seconds": 2.558} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/reconcile-integration.json b/evidence/reconcile-integration.json new file mode 100644 index 00000000..6fcb1f7f --- /dev/null +++ b/evidence/reconcile-integration.json @@ -0,0 +1 @@ +{"apply_deleted_objects": 1, "apply_deleted_unreferenced_metadata": 1, "dry_run_elapsed_seconds": 0.081, "dry_run_orphan_candidates": 0, "fresh_orphan_preserved": 1, "metadata_scanned": 38, "objects_scanned": 33, "private_bucket_anonymous_read": "denied", "referenced_missing_metadata_preserved": true, "repeat_deleted_metadata": 0, "repeat_deleted_objects": 0, "used_attachment_preserved": true} diff --git a/evidence/s3-failure.txt b/evidence/s3-failure.txt new file mode 100644 index 00000000..a4e46552 --- /dev/null +++ b/evidence/s3-failure.txt @@ -0,0 +1 @@ +PASS: upload returned 503 during S3 outage; bucket and API recovered with volumes intact. diff --git a/evidence/upload-load.json b/evidence/upload-load.json new file mode 100644 index 00000000..3531ef0f --- /dev/null +++ b/evidence/upload-load.json @@ -0,0 +1,15 @@ +{ + "api_container_ids": [ + "21c94e568619d0d4b8048e2ed7adc7a327f399dca08df3e1c8cd561d5a28e78f", + "d9c73db60b17c0e4fb7d24145ad52955f077c3a021028413d85c3591b95ee97c" + ], + "api_container_peak_memory_bytes": { + "21c94e568619d0d4b8048e2ed7adc7a327f399dca08df3e1c8cd561d5a28e78f": 83078676, + "d9c73db60b17c0e4fb7d24145ad52955f077c3a021028413d85c3591b95ee97c": 83760250 + }, + "concurrency": 10, + "elapsed_seconds": 3.045, + "notes": "Peak sampled every 0.5s using docker stats; excludes database, LB, and client memory.", + "upload_bytes_each": 4322578, + "uploads_succeeded": 10 +} diff --git a/infra/README.md b/infra/README.md index 5a39e862..042deb7c 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,14 @@ -# Инфраструктура студента +# Инфраструктура -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Compose описан в корневом `compose.yaml`. Роли: -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +| Роль | Service | Версия | Сеть и сохранность | +| --- | --- | --- | --- | +| Входной балансировщик | `load_balancer` | HAProxy 3.1-alpine | Публикует только порт 8080 | +| API | `api` × 2 | Python 3.12.8 image | Только внутренняя сеть; stateless | +| Метаданные/сессии | `postgres`, `redis` | PostgreSQL 16.6-alpine, Redis 7.4.2-alpine | Внутренняя сеть; persistent volumes | +| Объекты | `s3` | RustFS 1.0.6 | Внутренняя сеть; persistent volume | -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +`scripts/init-secrets.sh` идемпотентно создаёт секреты с правами `0600`; `scripts/bootstrap-s3.py` создаёт bucket, API-учётную запись и bucket-scoped policy через подписанный RustFS admin API. Консоль RustFS выключена, внутренние порты не публикуются. `s3-bootstrap` можно повторить после рестарта без удаления данных. + +Миграции выполняются единожды `docker compose run --rm migrate`, до старта API. Restart hook сохраняет volumes; failover hook останавливает только выбранную API-реплику. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..34273ebe --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,30 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 30s + timeout server 30s + timeout http-request 10s + timeout queue 5s + +resolvers docker + nameserver embedded 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 2s + hold obsolete 2s + +frontend public + bind :8080 + default_backend messenger_api + +backend messenger_api + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server inter 1s fall 2 rise 1 + server-template api 1-8 api:8080 check resolvers docker init-addr libc,none diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..01a9e76a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +argon2-cffi==23.1.0 +boto3==1.36.26 +fastapi==0.115.6 +Pillow==11.1.0 +psycopg[binary,pool]==3.2.4 +redis==5.2.1 +uvicorn[standard]==0.34.0 diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/scripts/bootstrap-s3.py b/scripts/bootstrap-s3.py new file mode 100644 index 00000000..458e1c76 --- /dev/null +++ b/scripts/bootstrap-s3.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""Idempotently create the private bucket and restricted RustFS API user.""" +import json +import hashlib +import os +from pathlib import Path +import time +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode +from urllib.request import Request, urlopen + +import boto3 +import botocore +from botocore.auth import SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +from botocore.exceptions import ClientError +from botocore.config import Config + + +def secret(name): + return Path(os.environ[name + "_FILE"]).read_text(encoding="utf-8").strip() + + +def signed_admin_request(root_access, root_secret, method, path, body=None): + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000") + region = os.environ.get("S3_REGION", "us-east-1") + content = json.dumps(body, separators=(",", ":")).encode("utf-8") if body else b"" + url = endpoint.rstrip("/") + path + headers = { + "Host": url.split("/", 3)[2], + "Content-Type": "application/json", + "X-Amz-Content-SHA256": hashlib.sha256(content).hexdigest(), + } + request = AWSRequest(method=method, url=url, data=content, headers=headers) + SigV4Auth(Credentials(root_access, root_secret), "s3", region).add_auth(request) + http_request = Request( + url, + data=content if content else None, + headers=dict(request.headers), + method=method, + ) + try: + with urlopen(http_request, timeout=5) as response: + response.read() + except HTTPError as error: + detail = error.read(2048).decode("utf-8", "replace") + if error.code == 409 and "add-user" in path: + return + raise RuntimeError( + "RustFS IAM bootstrap request failed: " + + method + + " " + + path.split("?", 1)[0] + + " returned " + + str(error.code) + + ": " + + detail[:300] + ) from None + except URLError as error: + raise RuntimeError("RustFS IAM bootstrap service is unavailable") from error + + +def main(): + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000") + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + root_access = secret("S3_ROOT_ACCESS_KEY") + root_secret = secret("S3_ROOT_SECRET_KEY") + app_access = secret("S3_ACCESS_KEY_ID") + app_secret = secret("S3_SECRET_ACCESS_KEY") + config = Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=5, + retries={"max_attempts": 2, "mode": "standard"}, + ) + root = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=root_access, + aws_secret_access_key=root_secret, + config=config, + ) + for attempt in range(30): + try: + try: + root.create_bucket(Bucket=bucket) + except ClientError as error: + if error.response.get("Error", {}).get("Code") != "BucketAlreadyOwnedByYou": + raise + root.head_bucket(Bucket=bucket) + break + except (botocore.exceptions.BotoCoreError, ClientError): + if attempt == 29: + raise + time.sleep(2) + + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::" + bucket], + }, + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::" + bucket + "/attachments/*"], + }, + ], + } + signed_admin_request( + root_access, + root_secret, + "PUT", + "/rustfs/admin/v3/add-user?" + urlencode({"accessKey": app_access}), + {"secretKey": app_secret, "status": "enabled"}, + ) + signed_admin_request( + root_access, + root_secret, + "PUT", + "/rustfs/admin/v3/add-canned-policy?" + urlencode({"name": "messenger-api"}), + policy, + ) + signed_admin_request( + root_access, + root_secret, + "POST", + "/rustfs/admin/v3/idp/builtin/policy/attach", + {"policies": ["messenger-api"], "user": app_access}, + ) + + api_client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=app_access, + aws_secret_access_key=app_secret, + config=config, + ) + api_client.head_bucket(Bucket=bucket) + root.close() + api_client.close() + print("S3 bootstrap complete: private bucket and scoped API credentials are ready.") + + +if __name__ == "__main__": + main() diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..68b37cc6 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +umask 077 +mkdir -p secrets + +create_secret() { + name=$1 + length=$2 + if [ ! -s "secrets/$name" ]; then + openssl rand -hex "$length" > "secrets/$name" + chmod 600 "secrets/$name" + fi +} + +create_secret database_password 32 +create_secret s3_root_access_key 16 +create_secret s3_root_secret_key 32 +create_secret s3_access_key_id 16 +create_secret s3_secret_access_key 32 +printf '%s\n' 'Local Compose secrets are ready; values were not printed.' diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100755 index 00000000..ef949dbb --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Reconcile attachment object keys and metadata; safe by default.""" +import argparse +from datetime import datetime, timedelta, timezone +import json +import logging +import os +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app.settings import database_connection, read_setting + +LOGGER = logging.getLogger("attachment-reconciler") + + +def plan_reconciliation(metadata, objects, now, grace): + cutoff = now - grace + object_keys = {item["Key"]: item for item in objects} + metadata_by_key = {row["object_key"]: row for row in metadata} + return { + "orphan_old": sorted( + key + for key, item in object_keys.items() + if key.startswith("attachments/") + and key not in metadata_by_key + and item.get("LastModified") is not None + and item["LastModified"] <= cutoff + ), + "orphan_fresh": sorted( + key + for key, item in object_keys.items() + if key.startswith("attachments/") + and key not in metadata_by_key + and ( + item.get("LastModified") is None + or item["LastModified"] > cutoff + ) + ), + "missing_unreferenced_old": sorted( + row["id"] + for row in metadata + if row["object_key"] not in object_keys + and not row["referenced"] + and row["created_at"] <= cutoff + ), + "missing_protected": sorted( + row["id"] + for row in metadata + if row["object_key"] not in object_keys + and (row["referenced"] or row["created_at"] > cutoff) + ), + "object_count": len(object_keys), + "metadata_count": len(metadata_by_key), + } + + +def load_state(connection, s3, bucket): + metadata = connection.execute( + """ + SELECT a.id::text AS id, a.object_key, a.created_at, + EXISTS ( + SELECT 1 FROM message_attachments ma + WHERE ma.attachment_id = a.id + ) AS referenced + FROM attachments a + """ + ).fetchall() + paginator = s3.get_paginator("list_objects_v2") + objects = [] + for page in paginator.paginate(Bucket=bucket, Prefix="attachments/"): + objects.extend(page.get("Contents", [])) + return metadata, objects + + +def apply_plan(connection, s3, bucket, plan): + deleted_objects = 0 + deleted_metadata = 0 + for key in plan["orphan_old"]: + s3.delete_object(Bucket=bucket, Key=key) + deleted_objects += 1 + for attachment_id in plan["missing_unreferenced_old"]: + with connection.transaction(): + row = connection.execute( + """ + DELETE FROM attachments AS a + WHERE a.id = %s + AND NOT EXISTS ( + SELECT 1 FROM message_attachments ma + WHERE ma.attachment_id = a.id + ) + RETURNING a.id + """, + (attachment_id,), + ).fetchone() + if row: + deleted_metadata += 1 + return deleted_objects, deleted_metadata + + +def main(): + import boto3 + import botocore + import psycopg + from botocore.config import Config + from psycopg.rows import dict_row + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--apply", + action="store_true", + help="delete stale orphan objects and stale unreferenced missing metadata", + ) + parser.add_argument( + "--grace-hours", + type=float, + default=24, + help="minimum age before repair; defaults to 24 hours", + ) + args = parser.parse_args() + if args.grace_hours < 0: + parser.error("--grace-hours cannot be negative") + + logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") + started = time.monotonic() + s3 = boto3.client( + "s3", + endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"), + region_name=os.environ.get("S3_REGION", "us-east-1"), + aws_access_key_id=read_setting("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"), + aws_secret_access_key=read_setting("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"), + config=Config( + connect_timeout=2, read_timeout=10, + retries={"max_attempts": 2, "mode": "standard"}, + ), + ) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + try: + with psycopg.connect(**database_connection(), row_factory=dict_row) as connection: + metadata, objects = load_state(connection, s3, bucket) + plan = plan_reconciliation( + metadata, + objects, + datetime.now(timezone.utc), + timedelta(hours=args.grace_hours), + ) + changes = {"objects_deleted": 0, "metadata_deleted": 0} + if args.apply: + ( + changes["objects_deleted"], + changes["metadata_deleted"], + ) = apply_plan(connection, s3, bucket, plan) + except (psycopg.Error, botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError): + LOGGER.exception("reconciliation failed; no success result was produced") + raise SystemExit(1) from None + summary = { + "mode": "apply" if args.apply else "dry-run", + "grace_hours": args.grace_hours, + "objects_scanned": plan["object_count"], + "metadata_scanned": plan["metadata_count"], + "orphan_objects": len(plan["orphan_old"]) + len(plan["orphan_fresh"]), + "orphan_old_candidates": len(plan["orphan_old"]), + "orphan_fresh_preserved": len(plan["orphan_fresh"]), + "missing_metadata_rows": len(plan["missing_unreferenced_old"]) + + len(plan["missing_protected"]), + "missing_unreferenced_old_candidates": len(plan["missing_unreferenced_old"]), + "missing_referenced_or_fresh_preserved": len(plan["missing_protected"]), + **changes, + "elapsed_seconds": round(time.monotonic() - started, 3), + } + print(json.dumps(summary, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..8ca583f1 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +docker compose restart postgres redis s3 >/dev/null +deadline=$(( $(date +%s) + 60 )) +for service in postgres redis; do + container=$(docker compose ps -q "$service") + while :; do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + if [ "$status" = healthy ]; then + break + fi + if [ "$status" = unhealthy ] || [ "$(date +%s)" -ge "$deadline" ]; then + printf '%s\n' "$service did not become healthy after restart" >&2 + exit 1 + fi + sleep 1 + done +done + +docker compose run --rm --no-deps s3-bootstrap >/dev/null +"$ROOT/scripts/rolling-restart.sh" diff --git a/scripts/rolling-restart.sh b/scripts/rolling-restart.sh new file mode 100755 index 00000000..effaeff2 --- /dev/null +++ b/scripts/rolling-restart.sh @@ -0,0 +1,39 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +containers=$(docker compose ps -q api) +healthy_count=0 +for container in $containers; do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + if [ "$status" = healthy ]; then + healthy_count=$((healthy_count + 1)) + fi +done +if [ "$healthy_count" -lt 2 ]; then + printf '%s\n' 'Rolling restart requires at least two healthy API replicas' >&2 + exit 1 +fi + +for container in $containers; do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + if [ "$status" != healthy ]; then + printf '%s\n' "Skipping unhealthy API replica: $container" >&2 + continue + fi + docker restart --time 10 "$container" >/dev/null + deadline=$(( $(date +%s) + 60 )) + while :; do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + if [ "$status" = healthy ]; then + break + fi + if [ "$status" = unhealthy ] || [ "$(date +%s)" -ge "$deadline" ]; then + printf '%s\n' "API replica did not become healthy: $container" >&2 + exit 1 + fi + sleep 1 + done +done diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..73bd1cb8 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container in $(docker compose ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 2 "$container" >/dev/null + exit 0 + fi +done + +printf '%s\n' 'No running API replica matches TARGET_INSTANCE_ID' >&2 +exit 1 diff --git a/tests/README.md b/tests/README.md index 0dfa3f89..3ed3b021 100644 --- a/tests/README.md +++ b/tests/README.md @@ -55,3 +55,16 @@ make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca. ## Что ещё остаётся доказать Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production. + +## Дополнительные проверки lab4 + +```sh +make test-grade4 +make test-s3-failure +make test-upload-load OUTPUT=evidence/upload-load.json +make test-reconcile +``` + +`test-grade4` проверяет слишком большой файл, повреждение, MIME-подмену, пиксельный лимит, опасное имя, cross-chat привязку, чужое вложение и конкурентную идемпотентность. `test-s3-failure` временно останавливает локальный Compose S3 и обязательно поднимает его обратно. `test-upload-load` отправляет десять параллельных синтетических JPEG и семплирует память API через `docker stats`; частота замеров ограничивает точность пика. + +Reconciler `python scripts/reconcile.py` по умолчанию работает только в режиме dry-run и использует grace period 24 часа. Он рассматривает для удаления только старые объекты `attachments/`, для которых нет metadata, и старые missing metadata без ссылок в сообщениях. `--apply` включает исправление; уменьшать grace period до нуля можно лишь на disposable synthetic стенде. Ссылочные вложения и свежие объекты сохраняются. Missing object у используемого вложения выводится как защищённое несоответствие для ручного восстановления, автоматическое удаление metadata не выполняется. `make test-reconcile` создаёт синтетические сбои и демонстрирует dry-run, grace period, исправление и повторный запуск; команда не подходит для production или ценных данных. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/grade4.py b/tests/grade4.py new file mode 100755 index 00000000..5462f2e4 --- /dev/null +++ b/tests/grade4.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Concurrent idempotency and replica-distribution checks.""" +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +import uuid + +from client import Client, Failure, arguments, check, run +from image_fixture import solid_gray_png + + +def main(): + args = arguments("Concurrent shared-idempotency and API-replica check") + client = Client(args.base_url, args.ca_file, timeout=5) + alice, bob, eve, chat = client.bootstrap() + key = uuid.uuid4().hex + body = {"text": "parallel-idempotency-" + key} + + def send(_): + return Client(args.base_url, args.ca_file, timeout=5).call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 201, + body=body, + actor=alice, + headers={"Idempotency-Key": key}, + schema="Message", + ) + + with ThreadPoolExecutor(max_workers=12) as executor: + responses = list(executor.map(send, range(12))) + messages = [response["json"] for response in responses] + check(all(message == messages[0] for message in messages), "concurrent replay differed") + + conflict = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 409, + body={"text": "different body"}, + actor=alice, + headers={"Idempotency-Key": key}, + ) + check(conflict["status"] == 409, "same key with a different body was accepted") + + instances = { + client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][ + "X-Instance-Id" + ] + for _ in range(80) + } + check(len(instances) >= 2, "load balancer did not expose two API instances") + + upload_path = "/api/v1/chats/" + chat["id"] + "/attachments" + fixture = (Path(__file__).parent / "fixtures" / "sample.png").read_bytes() + client.call( + "POST", + upload_path + "?filename=too-large.png", + 413, + actor=alice, + raw=b"x" * (10 * 1024 * 1024 + 1), + headers={"Content-Type": "image/png"}, + ) + client.call( + "POST", + upload_path + "?filename=broken.png", + 415, + actor=alice, + raw=b"\x89PNG\r\n\x1a\nnot-an-image", + headers={"Content-Type": "image/png"}, + ) + client.call( + "POST", + upload_path + "?filename=mime.jpg", + 415, + actor=alice, + raw=fixture, + headers={"Content-Type": "image/jpeg"}, + ) + client.call( + "POST", + upload_path + "?filename=pixels.png", + 413, + actor=alice, + raw=solid_gray_png(6401, 6251), + headers={"Content-Type": "image/png"}, + ) + client.call( + "POST", + upload_path + "?filename=..%2Fprivate.png", + 400, + actor=alice, + raw=fixture, + headers={"Content-Type": "image/png"}, + ) + attachment, _ = client.attachment(alice, chat) + other_chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Другой чат", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + client.call( + "POST", + "/api/v1/chats/" + other_chat["id"] + "/messages", + 404, + actor=alice, + body={"text": "", "attachment_ids": [attachment["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + for path in ( + "/api/v1/attachments/" + attachment["id"], + "/api/v1/attachments/" + attachment["id"] + "/content", + ): + client.call("GET", path, 404, actor=eve) + print("PASS: concurrent idempotency returned one message; observed %d API replicas" % len(instances)) + print("PASS: level-4 attachment error, pixel limit, chat binding and outsider checks.") + + +if __name__ == "__main__": + run(main) diff --git a/tests/image_fixture.py b/tests/image_fixture.py new file mode 100644 index 00000000..7ebb8fea --- /dev/null +++ b/tests/image_fixture.py @@ -0,0 +1,36 @@ +"""Small deterministic PNG builders for standard-library-only acceptance tests.""" +import random +import struct +import zlib + + +def _chunk(name, data): + kind = name.encode("ascii") + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + + +def solid_gray_png(width, height): + row_size = (width + 7) // 8 + raw = (b"\x00" + bytes(row_size)) * height + image_header = struct.pack(">IIBBBBB", width, height, 1, 0, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _chunk("IHDR", image_header) + _chunk( + "IDAT", zlib.compress(raw) + ) + _chunk("IEND", b"") + + +def noisy_rgb_png(width, height, seed=42024): + image_header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + compressor = zlib.compressobj(level=1) + compressed = bytearray() + rng = random.Random(seed) + for _ in range(height): + compressed.extend(compressor.compress(b"\x00" + rng.randbytes(width * 3))) + compressed.extend(compressor.flush()) + return b"\x89PNG\r\n\x1a\n" + _chunk("IHDR", image_header) + _chunk( + "IDAT", bytes(compressed) + ) + _chunk("IEND", b"") diff --git a/tests/reconcile_integration.py b/tests/reconcile_integration.py new file mode 100755 index 00000000..786f8ba8 --- /dev/null +++ b/tests/reconcile_integration.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Inject safe synthetic S3/DB inconsistencies and verify reconciliation.""" +import json +import os +import subprocess +import sys +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +from botocore import UNSIGNED + +from client import Client, check + + +def reconcile(*options): + result = subprocess.run( + [sys.executable, "scripts/reconcile.py", *options], + capture_output=True, + text=True, + check=False, + ) + check(result.returncode == 0, "reconciler command failed: " + result.stderr.strip()) + return json.loads(result.stdout) + + +def upload(client, alice, chat): + return client.attachment(alice, chat)[0] + + +def main(): + base_url = os.environ.get("RECONCILE_TEST_BASE_URL", "http://load_balancer:8080") + client = Client(base_url) + alice, _, _, chat = client.bootstrap() + linked = upload(client, alice, chat) + client.message(alice, chat, "", attachments=[linked["id"]]) + missing_referenced = upload(client, alice, chat) + client.message(alice, chat, "", attachments=[missing_referenced["id"]]) + missing_unreferenced = upload(client, alice, chat) + + with open(os.environ["S3_ACCESS_KEY_ID_FILE"], encoding="utf-8") as file: + access_key = file.read().strip() + with open(os.environ["S3_SECRET_ACCESS_KEY_FILE"], encoding="utf-8") as file: + secret_key = file.read().strip() + s3 = boto3.client( + "s3", + endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"), + region_name=os.environ.get("S3_REGION", "us-east-1"), + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=Config(signature_version="s3v4", connect_timeout=2, read_timeout=5), + ) + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + for attachment in (missing_referenced, missing_unreferenced): + s3.delete_object(Bucket=bucket, Key="attachments/" + attachment["id"]) + fresh_orphan = "attachments/" + str(uuid.uuid4()) + with open("tests/fixtures/sample.png", "rb") as file: + fixture = file.read() + s3.put_object( + Bucket=bucket, + Key=fresh_orphan, + Body=fixture, + ContentType="image/png", + ) + + denied = boto3.client( + "s3", + endpoint_url=os.environ.get("S3_ENDPOINT", "http://s3:9000"), + region_name=os.environ.get("S3_REGION", "us-east-1"), + config=Config(signature_version=UNSIGNED, connect_timeout=2, read_timeout=5), + ) + try: + denied.get_object(Bucket=bucket, Key="attachments/" + linked["id"]) + raise AssertionError("private S3 bucket allowed an anonymous object read") + except ClientError as error: + check( + error.response["ResponseMetadata"]["HTTPStatusCode"] in {403, 404}, + "anonymous S3 request did not fail as private access", + ) + + dry_run = reconcile("--grace-hours", "24") + check(dry_run["mode"] == "dry-run", "default reconciliation must be dry-run") + check(dry_run["orphan_fresh_preserved"] >= 1, "fresh orphan was not protected") + check( + dry_run["missing_referenced_or_fresh_preserved"] >= 2, + "referenced/fresh missing metadata was not protected", + ) + applied_grace = reconcile("--apply", "--grace-hours", "24") + check(applied_grace["objects_deleted"] == 0, "grace run deleted fresh object") + check(applied_grace["metadata_deleted"] == 0, "grace run deleted fresh metadata") + s3.head_object(Bucket=bucket, Key=fresh_orphan) + s3.head_object(Bucket=bucket, Key="attachments/" + linked["id"]) + + repaired = reconcile("--apply", "--grace-hours", "0") + check(repaired["objects_deleted"] >= 1, "stale orphan object was not removed") + check(repaired["metadata_deleted"] >= 1, "stale unreferenced missing row was not removed") + check( + repaired["missing_referenced_or_fresh_preserved"] >= 1, + "missing referenced attachment was not protected", + ) + repeated = reconcile("--apply", "--grace-hours", "0") + check(repeated["objects_deleted"] == 0, "repeated run was not idempotent for objects") + check(repeated["metadata_deleted"] == 0, "repeated run was not idempotent for metadata") + check( + client.call("GET", "/api/v1/attachments/" + linked["id"], actor=alice)["status"] == 200, + "reconciler removed a used attachment", + ) + downloaded = client.call( + "GET", "/api/v1/attachments/" + linked["id"] + "/content", actor=alice + ) + check(downloaded["bytes"] == fixture, "reconciler damaged a used attachment") + client.call( + "GET", + "/api/v1/attachments/" + missing_referenced["id"], + 200, + actor=alice, + schema="Attachment", + ) + client.call( + "GET", + "/api/v1/attachments/" + missing_unreferenced["id"], + 404, + actor=alice, + ) + print( + json.dumps( + { + "private_bucket_anonymous_read": "denied", + "dry_run_elapsed_seconds": dry_run["elapsed_seconds"], + "objects_scanned": dry_run["objects_scanned"], + "metadata_scanned": dry_run["metadata_scanned"], + "dry_run_orphan_candidates": dry_run["orphan_old_candidates"], + "fresh_orphan_preserved": applied_grace["orphan_fresh_preserved"], + "apply_deleted_objects": repaired["objects_deleted"], + "apply_deleted_unreferenced_metadata": repaired["metadata_deleted"], + "referenced_missing_metadata_preserved": True, + "repeat_deleted_objects": repeated["objects_deleted"], + "repeat_deleted_metadata": repeated["metadata_deleted"], + "used_attachment_preserved": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/replica_failover.py b/tests/replica_failover.py new file mode 100644 index 00000000..26aa3f1a --- /dev/null +++ b/tests/replica_failover.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Fail one explicitly selected replica and write sanitized evidence.""" +import json +import os +from pathlib import Path +import subprocess +import time +import uuid +from urllib.parse import quote + +from client import Client, Failure, arguments, check, run + + +def history(client, actor, chat): + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + messages = [] + seen = set() + while True: + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + messages.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return messages + check(cursor not in seen, "history cursor cycle") + seen.add(cursor) + path = ( + "/api/v1/chats/" + + chat["id"] + + "/messages?limit=100&cursor=" + + quote(cursor, safe="") + ) + + +def main(): + def configure(parser): + parser.add_argument("--target-id", required=True) + parser.add_argument("--action-script", required=True) + parser.add_argument("--output") + + args = arguments("Targeted API failover evidence", configure) + target = args.target_id + script = Path(args.action_script).expanduser().resolve() + check(script.is_file() and os.access(script, os.X_OK), "action script must be executable") + client = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, _, chat = client.bootstrap() + observed_before = { + client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][ + "X-Instance-Id" + ] + for _ in range(40) + } + check(len(observed_before) >= 2, "fewer than two API replicas observed") + check(target in observed_before, "target ID is not a currently observed API replica") + + original = None + for _ in range(20): + original = client.message( + alice, chat, "pre-failure-" + uuid.uuid4().hex, key=uuid.uuid4().hex + ) + if original["headers"]["X-Instance-Id"] == target: + break + check(original["headers"]["X-Instance-Id"] == target, "could not select requested replica") + + env = dict(os.environ) + env["TARGET_INSTANCE_ID"] = target + process = subprocess.Popen( + [str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + started = time.monotonic() + recovered_at = None + successful_writes = [] + failed_writes = 0 + failed_reads = 0 + served_after = [] + key = uuid.uuid4().hex + text = "during-failover-" + key + try: + while time.monotonic() - started < 15: + status = process.poll() + check(status is None or status == 0, "action script failed") + check(status is not None or time.monotonic() - started < 10, "action script exceeded 10s") + try: + read = client.call("GET", "/api/v1/users/me", actor=alice, schema="User") + served_after.append(read["headers"]["X-Instance-Id"]) + except Failure: + failed_reads += 1 + time.sleep(0.1) + continue + try: + response = client.message(alice, chat, text, key=key) + successful_writes.append(response["json"]) + if status == 0 and recovered_at is None: + recovered_at = time.monotonic() - started + key = uuid.uuid4().hex + text = "during-failover-" + key + except Failure: + failed_writes += 1 + time.sleep(0.1) + + check(process.poll() == 0, "action script did not finish successfully") + check(recovered_at is not None and recovered_at <= 10, "writes did not recover within 10s") + check(len(successful_writes) >= 5, "too few successful writes after the failure") + check(target not in served_after[-5:], "stopped replica still serves requests") + all_messages = history(client, bob, chat) + ids = [message["id"] for message in all_messages] + texts = [message["text"] for message in all_messages] + check(len(ids) == len(set(ids)), "duplicate message IDs in history") + check(len(texts) == len(set(texts)), "retry created duplicate message bodies") + check(original["json"] in all_messages, "confirmed pre-failure message was lost") + for message in successful_writes: + check(message in all_messages, "confirmed failover write was lost") + + elapsed = time.monotonic() - started + total_attempts = len(successful_writes) + failed_writes + result = { + "target_instance_id": target, + "observed_instances_before": len(observed_before), + "instances_serving_after": sorted(set(served_after)), + "duration_seconds": round(elapsed, 3), + "successful_writes": len(successful_writes), + "failed_write_attempts": failed_writes, + "failed_read_attempts": failed_reads, + "successful_rps": round(len(successful_writes) / elapsed, 3), + "write_error_ratio": round(failed_writes / total_attempts, 6), + "recovery_seconds": round(recovered_at, 3), + "confirmed_data_preserved": True, + "duplicate_messages": 0, + } + rendered = json.dumps(result, ensure_ascii=False, indent=2) + "\n" + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(rendered, encoding="utf-8") + print(rendered, end="") + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +if __name__ == "__main__": + run(main) diff --git a/tests/s3_failure.py b/tests/s3_failure.py new file mode 100644 index 00000000..8fcc1de4 --- /dev/null +++ b/tests/s3_failure.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Verify uploads return an explicit 503 while the Compose S3 service is stopped.""" +import json +from pathlib import Path +import subprocess +import time + +from client import Client, ROOT, arguments, check, run + + +def compose(*args): + subprocess.run( + ["docker", "compose", *args], + cwd=ROOT, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def healthy_api_container(): + result = subprocess.run( + ["docker", "compose", "ps", "-q", "api"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + for container in result.stdout.split(): + health = subprocess.run( + ["docker", "inspect", "--format", "{{.State.Health.Status}}", container], + check=True, + capture_output=True, + text=True, + ) + if health.stdout.strip() == "healthy": + return container + raise AssertionError("no healthy API replica is available") + + +def main(): + args = arguments("S3 outage behavior; stops and restores the local Compose S3 service") + client = Client(args.base_url, args.ca_file, timeout=15) + alice, _, _, chat = client.bootstrap() + fixture = (Path(__file__).parent / "fixtures" / "sample.png").read_bytes() + api_container = healthy_api_container() + stopped = False + try: + compose("stop", "s3") + stopped = True + payload = json.dumps( + {"alice": alice, "chat": chat, "fixture": fixture.hex()} + ) + worker = subprocess.run( + [ + "docker", + "exec", + "-i", + api_container, + "python", + "tests/s3_failure_worker.py", + ], + cwd=ROOT, + input=payload, + text=True, + capture_output=True, + check=False, + ) + check(worker.returncode == 0, "S3 failure worker failed: " + worker.stderr.strip()) + check(worker.stdout.strip() == "PASS", "API did not return 503 for unavailable S3") + finally: + if stopped: + compose("up", "-d", "s3") + compose("run", "--rm", "--no-deps", "s3-bootstrap") + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + break + except AssertionError: + time.sleep(1) + else: + raise AssertionError("API did not become ready after S3 recovery") + print("PASS: upload returned 503 during S3 outage; bucket and API recovered with volumes intact.") + + +if __name__ == "__main__": + run(main) diff --git a/tests/s3_failure_worker.py b/tests/s3_failure_worker.py new file mode 100644 index 00000000..62c9838e --- /dev/null +++ b/tests/s3_failure_worker.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""Internal worker for tests/s3_failure.py; reads auth data only from stdin.""" +import json +import sys +import urllib.error +import urllib.request + + +def main(): + payload = json.load(sys.stdin) + alice = payload["alice"] + chat = payload["chat"] + content = bytes.fromhex(payload["fixture"]) + request = urllib.request.Request( + "http://127.0.0.1:8080/api/v1/chats/" + + chat["id"] + + "/attachments?filename=s3-down.png", + data=content, + headers={ + "Authorization": "Bearer " + alice["token"], + "Content-Type": "image/png", + }, + method="POST", + ) + try: + urllib.request.urlopen(request, timeout=15) + except urllib.error.HTTPError as response: + if response.code != 503: + raise AssertionError("expected HTTP 503, received " + str(response.code)) + body = json.loads(response.read()) + request_id = response.headers.get("X-Request-Id") + if body.get("error", {}).get("code") != "unavailable": + raise AssertionError("expected unavailable error response") + if not request_id or body["error"].get("request_id") != request_id: + raise AssertionError("error request ID does not match response header") + if not response.headers.get("X-Instance-Id"): + raise AssertionError("missing API instance ID") + else: + raise AssertionError("expected HTTP 503 during the S3 outage") + print("PASS") + + +if __name__ == "__main__": + main() diff --git a/tests/test_reconcile_plan.py b/tests/test_reconcile_plan.py new file mode 100644 index 00000000..859d96c7 --- /dev/null +++ b/tests/test_reconcile_plan.py @@ -0,0 +1,39 @@ +import unittest +from datetime import datetime, timedelta, timezone + +from scripts.reconcile import plan_reconciliation + + +class ReconcilePlanTests(unittest.TestCase): + def test_only_stale_orphans_and_old_unreferenced_rows_are_candidates(self): + now = datetime.now(timezone.utc) + old = now - timedelta(days=2) + fresh = now - timedelta(minutes=2) + metadata = [ + {"id": "used", "object_key": "attachments/used", "created_at": old, "referenced": True}, + {"id": "unreferenced", "object_key": "attachments/missing", "created_at": old, "referenced": False}, + {"id": "fresh", "object_key": "attachments/fresh-missing", "created_at": fresh, "referenced": False}, + ] + objects = [ + {"Key": "attachments/used", "LastModified": old}, + {"Key": "attachments/orphan-old", "LastModified": old}, + {"Key": "attachments/orphan-fresh", "LastModified": fresh}, + {"Key": "unrelated/keep", "LastModified": old}, + ] + + plan = plan_reconciliation(metadata, objects, now, timedelta(hours=24)) + + self.assertEqual(plan["orphan_old"], ["attachments/orphan-old"]) + self.assertEqual(plan["orphan_fresh"], ["attachments/orphan-fresh"]) + self.assertEqual(plan["missing_unreferenced_old"], ["unreferenced"]) + self.assertEqual(plan["missing_protected"], ["fresh"]) + + def test_repeat_after_repair_has_no_candidates(self): + now = datetime.now(timezone.utc) + plan = plan_reconciliation([], [], now, timedelta(hours=24)) + self.assertEqual(plan["orphan_old"], []) + self.assertEqual(plan["missing_unreferenced_old"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/upload_load.py b/tests/upload_load.py new file mode 100755 index 00000000..bb9a2cde --- /dev/null +++ b/tests/upload_load.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Upload 10+ valid images concurrently and sample API container memory.""" +import argparse +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import re +import subprocess +import threading +import time + +from client import Client, check +from image_fixture import noisy_rgb_png + + +def memory_bytes(value): + match = re.match(r"\s*([\d.]+)\s*([KMG]i?B)", value) + if not match: + return 0 + amount = float(match.group(1)) + unit = match.group(2).upper() + return int( + amount + * { + "KB": 1000, + "MB": 1000**2, + "GB": 1000**3, + "KIB": 1024, + "MIB": 1024**2, + "GIB": 1024**3, + }[unit] + ) + + +def compose_api_ids(): + result = subprocess.run( + ["docker", "compose", "ps", "-q", "api"], + check=True, + capture_output=True, + text=True, + ) + healthy_ids = [] + for container_id in result.stdout.split(): + health = subprocess.run( + ["docker", "inspect", "--format", "{{.State.Health.Status}}", container_id], + check=True, + capture_output=True, + text=True, + ) + if health.stdout.strip() == "healthy": + healthy_ids.append(container_id) + check(len(healthy_ids) >= 2, "need at least two healthy API containers for the measurement") + return healthy_ids + + +def sample_memory(ids, stop, peak): + while not stop.is_set(): + for container_id in ids: + result = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", container_id], + capture_output=True, + text=True, + check=False, + ) + if result.returncode == 0: + used = memory_bytes(result.stdout.split("/", 1)[0]) + peak[container_id] = max(peak.get(container_id, 0), used) + stop.wait(0.5) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default=None) + args = parser.parse_args() + if args.concurrency < 10: + parser.error("--concurrency must be at least 10") + + bootstrap = Client(args.base_url) + alice, _, _, chat = bootstrap.bootstrap() + content = noisy_rgb_png(1200, 1200) + check(0 < len(content) <= 10 * 1024 * 1024, "generated PNG must be within 10 MiB") + api_ids = compose_api_ids() + stop = threading.Event() + peak = {} + sampler = threading.Thread(target=sample_memory, args=(api_ids, stop, peak), daemon=True) + sampler.start() + started = time.monotonic() + attachment_ids = [] + try: + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + tasks = [ + executor.submit( + Client(args.base_url).call, + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=load.png", + 201, + None, + alice, + {"Content-Type": "image/png"}, + content, + "Attachment", + ) + for _ in range(args.concurrency) + ] + for task in as_completed(tasks): + attachment_ids.append(task.result()["json"]["id"]) + finally: + stop.set() + sampler.join() + result = { + "concurrency": args.concurrency, + "upload_bytes_each": len(content), + "uploads_succeeded": len(attachment_ids), + "elapsed_seconds": round(time.monotonic() - started, 3), + "api_container_peak_memory_bytes": peak, + "api_container_ids": api_ids, + "notes": "Peak sampled every 0.5s using docker stats; excludes database, LB, and client memory.", + } + if len(attachment_ids) != args.concurrency: + raise SystemExit("not every upload succeeded") + output = json.dumps(result, indent=2, sort_keys=True) + if args.output: + with open(args.output, "w", encoding="utf-8") as file: + file.write(output + "\n") + print(output) + + +if __name__ == "__main__": + main()