diff --git a/.dockerignore b/.dockerignore
index bc16e55..cf740db 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -18,3 +18,10 @@ __pycache__
node_modules
evidence
.DS_Store
+docs
+
+.agents
+.aws
+.codex
+.pytest_cache
+venv
diff --git a/.env.example b/.env.example
index 2b32655..6931197 100644
--- a/.env.example
+++ b/.env.example
@@ -1,30 +1,91 @@
-# Copy to .env for your LOCAL synthetic test environment; never commit .env.
-# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
+LOG_LEVEL=INFO
+
+MAX_MESSAGES=100000
+MAX_IN_FLIGHT=16
+
SESSION_TTL_SECONDS=3600
-# Choose credentials locally. File paths contain no secret values.
-DATABASE_URL_FILE=/run/secrets/database_url
-# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
-SESSION_STORE=postgres
-CACHE_URL_FILE=/run/secrets/cache_url
+SESSION_STORE=redis
+
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_DB=0
+REDIS_MAX_CONNECTIONS=20
+REDIS_POOL_TIMEOUT_SECONDS=1
+REDIS_CONNECT_TIMEOUT_SECONDS=1
+REDIS_SOCKET_TIMEOUT_SECONDS=0.5
+
+DB_HOST=postgres
+DB_PORT=5432
+DB_NAME=messenger
+DB_USER=messenger
+
+DB_POOL_SIZE=5
+DB_MAX_OVERFLOW=10
+DB_POOL_TIMEOUT_SECONDS=3
+DB_CONNECT_TIMEOUT_SECONDS=3
+DB_STATEMENT_TIMEOUT_MS=5000
+
API_REPLICAS=2
+API_IMAGE=messenger:local
+API_BUILD_REVISION=development
IDEMPOTENCY_TTL_SECONDS=86400
-S3_ENDPOINT=http://s3:9000
+SHUTDOWN_DRAIN_SECONDS=1
+GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS=5
+READINESS_TIMEOUT_SECONDS=2
+
+LB_HOST_PORT=18081
+DB_HOST_PORT=25432
+REDIS_HOST_PORT=26379
+
+S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
-MAX_UPLOAD_BYTES=10485760
-MAX_IMAGE_PIXELS=40000000
-BROKER_URL_FILE=/run/secrets/broker_url
+S3_CONNECT_TIMEOUT_SECONDS=2
+S3_READ_TIMEOUT_SECONDS=10
+S3_UPLOAD_TIMEOUT_SECONDS=30
+S3_DOWNLOAD_TIMEOUT_SECONDS=60
+S3_IMAGE_TIMEOUT_SECONDS=10
+S3_MAX_UPLOAD_BYTES=10485760
+S3_MAX_IMAGE_PIXELS=40000000
+S3_CHUNK_BYTES=65536
+S3_MAX_CONCURRENT_UPLOADS=2
+S3_CLEANUP_GRACE_SECONDS=86400
+S3_CLEANUP_INTERVAL_SECONDS=300
+
+POSTGRES_PASSWORD_FILE=/run/secrets/db_password
+MINIO_ROOT_USER_FILE=/run/secrets/minio_root_user
+MINIO_ROOT_PASSWORD_FILE=/run/secrets/minio_root_password
+MINIO_BROWSER=off
+
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
-OTEL_SERVICE_NAME=messenger-api
-# Worker must set a distinct service.name.
+BROKER_HOST=broker
+BROKER_PORT=5672
+BROKER_PASSWORD_FILE=/run/secrets/broker_password
+BROKER_QUEUE=attachment-jobs
+BROKER_USER=messenger
+BROKER_VHOST=/
+JOB_LEASE_SECONDS=90
+JOB_RETRY_SECONDS=5
+PUBLISHER_POLL_SECONDS=1
+
+IMAGE_JPEG_QUALITY=85
+IMAGE_WEBP_QUALITY=80
+IMAGE_WEBP_METHOD=4
+IMAGE_MEMORY_MIB=768
+IMAGE_CPU_SECONDS=20
+
+JOB_RETRY_MAX_SECONDS=120
+FAILED_STAGING_RETENTION_SECONDS=604800
+
+# Экспорт метрик, логов и трасс в Collector.
+OTEL_ENABLED=true
OTEL_EXPORTER_OTLP_ENDPOINT=http://otel-collector:4318
-OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf
-OTEL_RESOURCE_ATTRIBUTES=deployment.environment.name=lab
+GRAFANA_HOST_PORT=13000
diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..d3190c1
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,10 @@
+* text=auto
+*.py text eol=lf
+*.sh text eol=lf
+*.conf text eol=lf
+*.yaml text eol=lf
+*.yml text eol=lf
+*.json text eol=lf
+.env.example text eol=lf
+Dockerfile text eol=lf
+*.png binary
diff --git a/.gitignore b/.gitignore
index b5a7189..3abc769 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,3 @@
-# Local/private state
.env
.env.*
!.env.example
@@ -13,7 +12,6 @@ certs/private/
volumes/
data/
backups/
-# Runtime/build artifacts
.venv/
__pycache__/
*.py[cod]
@@ -28,5 +26,8 @@ coverage/
.DS_Store
.idea/
.vscode/
-# Keep small, sanitized evidence summaries in evidence/ under version control.
-# Inspect git diff --cached: ignore rules are not a secret scanner.
+venv/
+.pytest_cache/
+.ruff_cache/
+
+/docs/
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..889eddb
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,30 @@
+FROM python:3.12.15-slim-bookworm
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+ PYTHONUNBUFFERED=1 \
+ PYTHONUTF8=1 \
+ PIP_NO_CACHE_DIR=1 \
+ PIP_DISABLE_PIP_VERSION_CHECK=1
+
+WORKDIR /app
+
+RUN useradd --system --uid 10001 --no-create-home appuser
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+COPY alembic.ini ./
+COPY migrations ./migrations
+COPY app ./app
+
+ARG API_BUILD_REVISION
+LABEL org.opencontainers.image.revision=$API_BUILD_REVISION
+
+USER appuser
+
+EXPOSE 8080
+
+HEALTHCHECK --interval=3s --timeout=3s --start-period=10s --retries=2 \
+ CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
+
+CMD ["python", "-m", "app"]
diff --git a/Makefile b/Makefile
index 5d86d17..911b03d 100644
--- a/Makefile
+++ b/Makefile
@@ -1,10 +1,10 @@
PYTHON ?= python3
-BASE_URL ?= http://localhost:8080
+BASE_URL ?= http://127.0.0.1:18081
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
-.PHONY: help check validate test load test-persistence test-failover
+.PHONY: help check validate test load test-persistence test-failover verify
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@@ -28,3 +28,6 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+
+verify:
+ $(PYTHON) -X utf8 scripts/test.py
diff --git a/REPORT.md b/REPORT.md
index b97ae42..8fda614 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,37 +1,75 @@
# Отчёт — лабораторная 6
-- Автор, группа: TODO
-- Целевая оценка; выбранный трек 5, если нужен: TODO
-- Commit/PR: TODO после публикации
-- Использованные источники, библиотеки и помощь ИИ: TODO
+- Автор, группа: Антонова Ирина Андреевна, БПИ2301.
+- Целевая оценка: 4.
+- Commit/PR:
+- Использованные источники, библиотеки и помощь ИИ: контракт ЛР № 6, документация OpenTelemetry (Collector, propagation), W3C Trace Context, документация Prometheus, Loki, Tempo и Grafana provisioning; OpenTelemetry SDK 1.45.1. Codex помог перенести код из ЛР № 5 и адаптировать запуск.
## Запуск из чистого clone
-TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
+**Требования:** Python 3.12, Docker Desktop с Compose. Пошаговая инструкция запуска и остановки для Windows 10 — в [tests/README.md](tests/README.md).
+
+**Версии образов** закреплены в [compose.yaml](compose.yaml): OpenTelemetry Collector contrib 0.165.0, Prometheus v3.16.0, Loki 3.7.10, Tempo 2.9.0, Grafana 13.3.1, PostgreSQL 16.10, RabbitMQ 4.4.0, Redis 7.4.2, Nginx 1.28.0-alpine. Образы приложения (`api`, `server`, `bootstrap`) собираются из репозитория.
+
+
+**Порядок действий:**
+
+1. `python scripts/init_local.py` создаёт secrets в `secrets/` (существующие сохраняются; в Git они не попадают).
+2. `docker compose --env-file .env up -d --build` поднимает стенд; миграции PostgreSQL и настройку MinIO выполняют одноразовые контейнеры `migrate` и `minio-bootstrap`.
+3. Готовность: healthcheck Compose и `/health/ready`.
+4. Тесты: `python -m pytest -q`, `python tests/smoke.py --base-url <адрес API>`.
+5. Адреса: API — [порт 18081](http://127.0.0.1:18081), дашборд — [Grafana](http://127.0.0.1:13000/d/messenger-overview) (пароль в `secrets/grafana_admin_password`). Контрольный прогон шёл в проекте `messenger-test` на портах 28080 и 23000.
+
+**Остановка:** `docker compose --env-file .env down` (тома сохраняются). **Удаление учебных данных:** `docker compose --env-file .env down --volumes --remove-orphans`.
## Архитектура и решения
-TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab5, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
+Из ЛР № 5 перенесены две реплики API за Nginx, Redis, MinIO, RabbitMQ, worker и PostgreSQL; вложение и задание принимаются в одной транзакции, worker подтверждает доставку после фиксации результата. Бизнес-API не менялся.
+
+Добавлено: API и worker отправляют метрики, логи и трассы по OTLP в Collector. Prometheus хранит метрики, Loki — логи, Tempo — трассы, Grafana их отображает и связывает по `trace_id`. У каждого хранилища свой persistent volume; конфиги и provisioning (в том числе [дашборд](infra/otel/grafana/dashboards/messenger.json)) лежат в Git.
+
+- **Контекст через очередь:** API сохраняет `traceparent`/`tracestate` вместе с заданием, publisher передаёт их в RabbitMQ, worker создаёт consumer span с этим родителем. Повторы видны по `job.attempt`.
+- **Логи:** структурные, с `trace_id`, `span_id`, `request_id` (API) и `job_id` (worker).
+- **Приватность:** в labels нет `user_id`, `chat_id`, `message_id`; тела сообщений, изображения и секреты не экспортируются (проверено на 563 записях логов).
+- **Метрики:** задержка — histogram в секундах, p50/p95/p99 через `histogram_quantile` по сумме реплик; также доля ошибок, глубина и возраст очереди, пул БД, загрузка worker.
+- **SLO:** не менее 99% бизнес-запросов без 5xx за 24 ч. Оповещения в [alerts.yaml](infra/otel/alerts.yaml): `MessengerHighErrorRate`, `MessengerQueueLag`.
+- **Отказ хранилища телеметрии:** экспорт через ограниченную очередь (256 пакетов); при недоступном backend записи отбрасываются, API не блокируется.
## Доказательства критериев
+Журналы — в `evidence/lab6/`. Прогон 22.11.2026, профиль `full`, все 13 шагов с кодом 0.
+
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
-| Публичный smoke | make test | TODO | TODO |
-| Остальные пункты уровня 3 | TODO | TODO | TODO |
-| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
-| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
-
-TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
+| Стенд, контракт, OpenAPI | `init_local.py`, `check_contract.py`, `validate_openapi.py`, `docker compose up -d` | Secrets созданы, контракт и OpenAPI валидны, все сервисы healthy | [00](evidence/lab6/integration/00_init.txt), [01](evidence/lab6/integration/01_contract.txt), [02](evidence/lab6/integration/02_openapi.txt), [03](evidence/lab6/integration/03_startup.txt) |
+| Публичный smoke | `tests/smoke.py` | PASS | [06](evidence/lab6/integration/06_smoke.txt) |
+| Минимум lab5 | `python -m pytest -q` | 206 из 206 за 156,6 с | [05](evidence/lab6/05_pytest.txt), [pytest.xml](evidence/lab6/integration/pytest.xml) |
+| Метрики, логи, трассы, приватность, история (3) | `scripts.check_observability --restart-telemetry` | Данные есть во всех трёх хранилищах; лог связан с трассой API (`9021bfee…`) и worker (`80b42516…`); все проверки true, история пережила перезапуск | [07](evidence/lab6/integration/07_observability.txt), [observability.json](evidence/lab6/integration/observability.json) |
+| Trace context, цепочка до S3/БД, retry (4) | `--incidents` | Одна трасса: загрузка → `rabbitmq.publish` → `attachment.process.v1` → S3 и БД; при retry — 6 попыток (`2ee8f51d…`) | [observability.json](evidence/lab6/integration/observability.json) |
+| Дашборд, SLO, alerts (4) | Дашборд `messenger-overview`, `alerts.yaml`, `--incidents` | Панели по требованиям уровня 4; оба правила прошли firing → resolved | [Дашборд](infra/otel/grafana/dashboards/messenger.json), [alerts.yaml](infra/otel/alerts.yaml) |
+| Отказ телеметрии, два инцидента (4) | Остановка Loki, разборы ниже | API работает, очередь ограничена | [observability.json](evidence/lab6/integration/observability.json) |
+| Сохранность и failover | `tests/resilience.py` | Данные и сессия сохранены, запись восстановилась за 2,81 с | [08](evidence/lab6/integration/08_persistence.txt), [09](evidence/lab6/integration/09_failover.txt), [verification.json](evidence/lab6/integration/verification.json) |
## Эксперименты и отказы
-TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
+**Профиль нагрузки.** Один Docker-хост, 2 реплики API, 1 worker. Синтетические `POST` сообщений, 10 запр./с, параллелизм 2, 10 с, без повторов; 3 пользователя, 1 чат. Результат: 100 успешных, 0 ошибок, 10,0 запр./с, клиентские p50/p95/p99 — 26,1 / 51,4 / 80,9 мс. Под нагрузкой CPU API до 66,6%, Collector — 59,6 МиБ из 256 МиБ. Сводка: [observability.json](evidence/lab6/integration/observability.json).
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
-| TODO | TODO | TODO | TODO |
+| Остановка PostgreSQL | Рост 5xx | API вернул 503, в трассе ошибка `postgres.checkout` | Оповещение сработало через 25,9 с, снялось через 66,3 с. Данные не потеряны |
+| Остановка worker | Рост возраста заданий | Возраст 27,2 с, оповещение через 32,7 с; задание обработано с 6-й попытки, статус `ready` | Глубина очереди вернулась к 0. Задание не потеряно |
+| Остановка Loki | API продолжает работу при ограниченном буфере | 204 сообщения записаны; очередь до 48 из 256, память Collector около 230,9 МиБ | Потеряно 6 пакетов логов; после запуска Loki экспорт восстановился |
+| Остановка одной реплики API | Переключение на вторую | 115 успешных записей, 4 неуспешные попытки | Восстановление за 2,81 с, данные и сессия сохранены |
+
+**Инцидент 1: PostgreSQL недоступен.** Сработало `MessengerHighErrorRate` → на дашборде рост 5xx → в Tempo по ошибочному `POST /api/v1/users` (`6c4b9559…`) ошибка на `postgres.checkout` → в Loki по `trace_id` событие `dependency.unavailable` (`gaierror`: хост БД не разрешается) и `http.request` 503. Причина — недоступная зависимость, а не код API; после запуска БД оповещение снялось.
+
+**Инцидент 2: worker остановлен.** Сработало `MessengerQueueLag` → на панели очереди рост возраста при отсутствии завершённых заданий → в трассе `2ee8f51d…` растёт `job.attempt`: попытки 1–5 с `outcome=error`, 6-я `success`, затем `s3.write_result` и `postgres.commit_ready`. Причина — остановленный worker и временные сбои; после восстановления очередь опустела.
## Ограничения и следующие шаги
-TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
+- Один Docker-хост; нагрузка небольшая (не тест производительности); время срабатывания оповещений измерено в одном прогоне.
+- Для проверки SLO нужно полное 24-часовое окно; доставка оповещений наружу не реализована.
+- При недоступном Loki часть логов теряется намеренно ради ограниченной памяти; метрики и трассы этим сценарием отдельно не проверялись.
+- Вывод hook-скриптов `restart.sh` и `stop-one.sh` скрыт из-за секретов, охват перезапущенных процессов подтверждается логами Compose.
+- Grafana использует около 72% лимита памяти (369 из 512 МиБ).
+- Трек 5 не реализован.
\ No newline at end of file
diff --git a/alembic.ini b/alembic.ini
new file mode 100644
index 0000000..f95b20a
--- /dev/null
+++ b/alembic.ini
@@ -0,0 +1,38 @@
+[alembic]
+script_location = migrations
+prepend_sys_path = .
+path_separator = os
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARNING
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARNING
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/app/__init__.py b/app/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/app/__main__.py b/app/__main__.py
new file mode 100644
index 0000000..ee20995
--- /dev/null
+++ b/app/__main__.py
@@ -0,0 +1,4 @@
+from app.main import main
+
+if __name__ == "__main__":
+ main()
diff --git a/app/core/__init__.py b/app/core/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/app/core/background.py b/app/core/background.py
new file mode 100644
index 0000000..ad87d26
--- /dev/null
+++ b/app/core/background.py
@@ -0,0 +1,40 @@
+"""Остановка фоновых процессов и безопасные сообщения об отказах."""
+
+import asyncio
+import logging
+import signal
+
+from aio_pika.exceptions import AMQPException, ChannelInvalidStateError
+from sqlalchemy.exc import InterfaceError, OperationalError
+from sqlalchemy.exc import TimeoutError as PoolTimeoutError
+
+TRANSIENT_JOB_ERRORS = (
+ AMQPException,
+ ChannelInvalidStateError,
+ InterfaceError,
+ OperationalError,
+ PoolTimeoutError,
+ OSError,
+)
+logger = logging.getLogger(__name__)
+
+
+def shutdown_event() -> asyncio.Event:
+ stop = asyncio.Event()
+ loop = asyncio.get_running_loop()
+ for signum in (signal.SIGTERM, signal.SIGINT):
+ loop.add_signal_handler(signum, stop.set)
+ return stop
+
+
+async def wait_or_stop(stop: asyncio.Event, seconds: float) -> None:
+ try:
+ async with asyncio.timeout(seconds):
+ await stop.wait()
+ except TimeoutError:
+ pass
+
+
+def log_background_error(operation: str, error: Exception) -> None:
+
+ logger.warning("%s: %s", operation, type(error).__name__)
diff --git a/app/core/cleanup.py b/app/core/cleanup.py
new file mode 100644
index 0000000..548959f
--- /dev/null
+++ b/app/core/cleanup.py
@@ -0,0 +1,66 @@
+"""Периодическая компенсация незавершённых загрузок; отдельный Compose-сервис."""
+
+import argparse
+import asyncio
+from dataclasses import asdict
+import json
+
+from app.core.config import Settings
+from app.core.jobs import JobSettings
+from app.core.database import create_db_engine, create_session_factory
+from app.core.s3 import S3Settings
+from app.domain.attachments import reconcile_attachments
+from app.domain.attachment_jobs import cleanup_staging
+from app.errors.handlers import RECOVERABLE_STORAGE_ERRORS, log_recoverable_error
+from app.storage.s3 import create_s3_storage
+
+
+async def cleanup_once(factory, storage, apply: bool, jobs: JobSettings) -> bool:
+ completed = True
+ try:
+ async with factory() as session:
+ result = await reconcile_attachments(session, storage, dry_run=not apply)
+ staging = await cleanup_staging(session, storage, jobs, apply=apply)
+ print(
+ json.dumps({**asdict(result), "terminal_staging": staging}), flush=True
+ )
+ except RECOVERABLE_STORAGE_ERRORS as error:
+ log_recoverable_error("Reconciliation", error)
+ completed = False
+ return completed
+
+
+async def run(once: bool, apply: bool) -> None:
+ settings = Settings()
+ s3_settings = S3Settings()
+ jobs = JobSettings()
+ engine = create_db_engine(settings)
+ factory = create_session_factory(engine)
+ storage = create_s3_storage(s3_settings)
+ try:
+ while True:
+ completed = await cleanup_once(factory, storage, apply, jobs)
+ if once:
+ if not completed:
+ raise SystemExit(1)
+ break
+ await asyncio.sleep(s3_settings.cleanup_interval_seconds)
+ finally:
+ await storage.aclose()
+ await engine.dispose()
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="Сверка журнала загрузок MinIO/БД")
+ parser.add_argument("--once", action="store_true")
+ parser.add_argument(
+ "--apply",
+ action="store_true",
+ help="удалять безопасные кандидаты; по умолчанию dry-run",
+ )
+ args = parser.parse_args()
+ asyncio.run(run(args.once, args.apply))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/core/config.py b/app/core/config.py
new file mode 100644
index 0000000..bf6dd4b
--- /dev/null
+++ b/app/core/config.py
@@ -0,0 +1,128 @@
+from enum import StrEnum
+from pathlib import Path
+from typing import Literal, Self
+
+from pydantic import Field, SecretStr, model_validator
+from pydantic_settings import (
+ BaseSettings,
+ PydanticBaseSettingsSource,
+ SettingsConfigDict,
+)
+from sqlalchemy import URL
+from app.core.s3 import S3Settings
+
+ASYNC_DRIVER = "postgresql+asyncpg"
+SECRETS_DIR = next(
+ (
+ path
+ for path in (
+ Path("/run/secrets"),
+ Path(__file__).resolve().parents[2] / "secrets",
+ )
+ if (path / "db_password").is_file()
+ ),
+ None,
+)
+
+
+class LogLevel(StrEnum):
+ DEBUG = "DEBUG"
+ INFO = "INFO"
+ WARNING = "WARNING"
+ ERROR = "ERROR"
+ CRITICAL = "CRITICAL"
+
+
+class Settings(BaseSettings):
+ s3_enabled: bool = True
+ s3: S3Settings = Field(default_factory=S3Settings)
+ app_host: str
+ app_port: int = Field(ge=1, le=65535)
+ app_env: str
+ log_level: LogLevel = LogLevel.INFO
+
+ max_messages: int = Field(ge=1)
+ max_in_flight: int = Field(ge=1)
+
+ session_ttl_seconds: int = Field(default=3600, ge=1)
+ session_store: Literal["redis"] = "redis"
+
+ redis_host: str = "localhost"
+ redis_port: int = Field(default=6379, ge=1, le=65535)
+ redis_db: int = Field(default=0, ge=0)
+ redis_password: SecretStr | None = None
+
+ redis_max_connections: int = Field(default=20, ge=1)
+ redis_pool_timeout_seconds: float = Field(default=1.0, gt=0)
+ redis_connect_timeout_seconds: float = Field(default=1.0, gt=0)
+ redis_socket_timeout_seconds: float = Field(default=0.5, gt=0)
+
+ database_url: SecretStr | None = None
+ db_host: str | None = None
+ db_port: int = Field(default=5432, ge=1, le=65535)
+ db_name: str | None = None
+ db_user: str | None = None
+ db_password: SecretStr | None = None
+
+ db_pool_size: int = Field(default=5, ge=1)
+ db_max_overflow: int = Field(default=5, ge=0)
+ db_pool_timeout_seconds: float = Field(default=3.0, gt=0)
+ db_connect_timeout_seconds: int = Field(default=3, ge=1)
+ db_statement_timeout_ms: int = Field(default=5000, ge=1)
+
+ instance_id: str | None = None
+ idempotency_ttl_seconds: int = Field(default=86400, ge=1)
+ shutdown_drain_seconds: float = Field(default=1, ge=0)
+ graceful_shutdown_timeout_seconds: int = Field(default=5, ge=1)
+ readiness_timeout_seconds: float = Field(default=2, gt=0, lt=2.5)
+
+ model_config = SettingsConfigDict(
+ env_file=".env",
+ env_file_encoding="utf-8",
+ case_sensitive=False,
+ extra="ignore",
+ secrets_dir=SECRETS_DIR,
+ )
+
+ @classmethod
+ def settings_customise_sources(
+ cls,
+ settings_cls: type[BaseSettings],
+ init_settings: PydanticBaseSettingsSource,
+ env_settings: PydanticBaseSettingsSource,
+ dotenv_settings: PydanticBaseSettingsSource,
+ file_secret_settings: PydanticBaseSettingsSource,
+ ) -> tuple[PydanticBaseSettingsSource, ...]:
+ return init_settings, file_secret_settings, env_settings, dotenv_settings
+
+ @model_validator(mode="after")
+ def _require_database_source(self) -> Self:
+ if self.shutdown_drain_seconds + self.graceful_shutdown_timeout_seconds >= 8:
+ raise ValueError(
+ "Drain и graceful shutdown должны суммарно занимать менее 8 секунд"
+ )
+ if self.database_url is not None:
+ if not self.database_url.get_secret_value().startswith(ASYNC_DRIVER):
+ raise ValueError(f"DATABASE_URL должен начинаться с {ASYNC_DRIVER}://")
+ return self
+
+ if not (self.db_host and self.db_name and self.db_user and self.db_password):
+ raise ValueError(
+ "Нужны DATABASE_URL либо DB_HOST, DB_NAME, DB_USER и секрет db_password"
+ )
+ return self
+
+ def get_database_url(self) -> str:
+ if self.database_url is not None:
+ return self.database_url.get_secret_value()
+
+ assert self.db_password is not None
+ url = URL.create(
+ ASYNC_DRIVER,
+ username=self.db_user,
+ password=self.db_password.get_secret_value(),
+ host=self.db_host,
+ port=self.db_port,
+ database=self.db_name,
+ )
+ return url.render_as_string(hide_password=False)
diff --git a/app/core/database.py b/app/core/database.py
new file mode 100644
index 0000000..42650cf
--- /dev/null
+++ b/app/core/database.py
@@ -0,0 +1,61 @@
+from contextlib import nullcontext
+
+from sqlalchemy.pool import AsyncAdaptedQueuePool
+from sqlalchemy.ext.asyncio import (
+ AsyncEngine,
+ AsyncSession,
+ async_sessionmaker,
+ create_async_engine,
+)
+
+from app.core.config import Settings
+
+
+class ObservedPool(AsyncAdaptedQueuePool):
+ telemetry = None
+ waiting = 0
+
+ def __init__(self, *args, pool_size, max_overflow, **kwargs):
+ self.capacity = (
+ pool_size + max_overflow if pool_size and max_overflow >= 0 else None
+ )
+ super().__init__(
+ *args, pool_size=pool_size, max_overflow=max_overflow, **kwargs
+ )
+
+ def _do_get(self):
+ waiting = self.capacity is not None and self.checkedout() >= self.capacity
+ self.waiting += int(waiting)
+ try:
+ return super()._do_get()
+ finally:
+ self.waiting -= int(waiting)
+
+ def connect(self):
+ with (
+ self.telemetry.span("postgres.checkout")
+ if self.telemetry
+ else nullcontext()
+ ):
+ return super().connect()
+
+
+def create_db_engine(settings: Settings) -> AsyncEngine:
+ return create_async_engine(
+ settings.get_database_url(),
+ poolclass=ObservedPool,
+ pool_size=settings.db_pool_size,
+ max_overflow=settings.db_max_overflow,
+ pool_timeout=settings.db_pool_timeout_seconds,
+ pool_pre_ping=True,
+ connect_args={
+ "timeout": settings.db_connect_timeout_seconds,
+ "server_settings": {
+ "statement_timeout": str(settings.db_statement_timeout_ms)
+ },
+ },
+ )
+
+
+def create_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]:
+ return async_sessionmaker(bind=engine, expire_on_commit=False, autoflush=False)
diff --git a/app/core/image_processing.py b/app/core/image_processing.py
new file mode 100644
index 0000000..107dc42
--- /dev/null
+++ b/app/core/image_processing.py
@@ -0,0 +1,16 @@
+"""Качество вариантов и пределы отдельного процесса обработки."""
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class ImageSettings(BaseSettings):
+ model_config = SettingsConfigDict(
+ env_prefix="IMAGE_", env_file=".env", extra="ignore"
+ )
+
+ jpeg_quality: int = Field(default=85, ge=1, le=95)
+ webp_quality: int = Field(default=80, ge=1, le=100)
+ webp_method: int = Field(default=4, ge=0, le=6)
+ memory_mib: int = Field(default=768, ge=256, le=1024)
+ cpu_seconds: int = Field(default=20, ge=1, le=120)
diff --git a/app/core/image_worker.py b/app/core/image_worker.py
new file mode 100644
index 0000000..5f4f19e
--- /dev/null
+++ b/app/core/image_worker.py
@@ -0,0 +1,24 @@
+"""Дочерний процесс декодирования с пределами памяти и CPU"""
+
+from pathlib import Path
+import resource
+import sys
+
+from app.domain.image_validation import decode_image
+from app.errors.exceptions import IMAGE_WORKER_ERRORS, image_worker_error_code
+
+
+def main() -> int:
+ resource.setrlimit(resource.RLIMIT_AS, (512 * 1024 * 1024, 512 * 1024 * 1024))
+ resource.setrlimit(resource.RLIMIT_CPU, (10, 10))
+ path, content_type, maximum = sys.argv[1:]
+ status = 0
+ try:
+ decode_image(Path(path), content_type, int(maximum))
+ except IMAGE_WORKER_ERRORS as error:
+ status = image_worker_error_code(error)
+ return status
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/app/core/instance.py b/app/core/instance.py
new file mode 100644
index 0000000..8365b7c
--- /dev/null
+++ b/app/core/instance.py
@@ -0,0 +1,38 @@
+"""Идентификатор API-реплики и заголовок X-Instance-Id."""
+
+import os
+from uuid import uuid4
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+
+INSTANCE_ID_HEADER = "X-Instance-Id"
+
+
+def resolve_instance_id(configured: str | None) -> str:
+ if configured and configured.strip():
+ return configured.strip()
+
+ hostname = os.getenv("HOSTNAME", "").strip()
+ if hostname:
+ return hostname
+
+ return str(uuid4())
+
+
+class InstanceIdMiddleware(BaseHTTPMiddleware):
+
+ def __init__(self, app, instance_id: str) -> None:
+ super().__init__(app)
+ self.instance_id = instance_id
+
+ async def dispatch(
+ self,
+ request: Request,
+ call_next: RequestResponseEndpoint,
+ ) -> Response:
+ response = await call_next(request)
+ response.headers[INSTANCE_ID_HEADER] = self.instance_id
+
+ return response
diff --git a/app/core/job_admin.py b/app/core/job_admin.py
new file mode 100644
index 0000000..f75efa6
--- /dev/null
+++ b/app/core/job_admin.py
@@ -0,0 +1,82 @@
+"""Операторский интерфейс failed registry. Доступ требует credentials стенда."""
+
+import argparse
+import asyncio
+import json
+from uuid import UUID
+
+from app.core.background import log_background_error
+from app.core.config import Settings
+from app.core.database import create_db_engine, create_session_factory
+from app.core.jobs import JobSettings
+from app.domain.job_admin import (
+ REPLAY_REASONS,
+ history,
+ list_failed,
+ replay_failed_job,
+ show_job,
+)
+from app.errors.exceptions import JobOperationError
+from app.storage.s3 import create_s3_storage
+
+
+async def execute(args):
+ settings = Settings()
+ engine = create_db_engine(settings)
+ storage = None
+ try:
+ factory = create_session_factory(engine)
+ async with factory() as session:
+ if args.command == "list-failed":
+ result = await list_failed(session, args.limit)
+ elif args.command == "show":
+ result = await show_job(session, args.job_id)
+ elif args.command == "history":
+ result = await history(session, args.job_id)
+ else:
+ storage = create_s3_storage(settings.s3)
+ result = await replay_failed_job(
+ session,
+ storage,
+ JobSettings(),
+ args.job_id,
+ request_id=args.request_id,
+ actor=args.actor,
+ reason=args.reason,
+ )
+ print(json.dumps(result, default=str), flush=True)
+ finally:
+ if storage is not None:
+ await storage.aclose()
+ await engine.dispose()
+
+
+def main():
+ parser = argparse.ArgumentParser(
+ description="Реестр failed и явный replay с аудитом"
+ )
+ commands = parser.add_subparsers(dest="command", required=True)
+ listing = commands.add_parser("list-failed")
+ listing.add_argument(
+ "--limit", type=int, choices=range(1, 501), default=50, metavar="1..500"
+ )
+ for name in ("show", "history", "replay"):
+ command = commands.add_parser(name)
+ command.add_argument("job_id", type=UUID)
+ if name == "replay":
+ command.add_argument("--request-id", type=UUID, required=True)
+ command.add_argument("--actor", required=True)
+ command.add_argument("--reason", choices=REPLAY_REASONS, required=True)
+ args = parser.parse_args()
+ try:
+ asyncio.run(execute(args))
+ except JobOperationError as error:
+ print(json.dumps({"error_code": error.code}), flush=True)
+ raise SystemExit(2) from None
+ except Exception as error:
+ log_background_error("Job administration failed", error)
+ raise SystemExit(1) from None
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/core/job_worker.py b/app/core/job_worker.py
new file mode 100644
index 0000000..9d3fa43
--- /dev/null
+++ b/app/core/job_worker.py
@@ -0,0 +1,204 @@
+"""Отдельные consumers с ограниченной конкуренцией; API не выполняет resize."""
+
+import argparse
+import asyncio
+from dataclasses import dataclass
+import json
+import logging
+import os
+import signal
+from uuid import UUID, uuid4
+
+from PIL import features
+from opentelemetry.context import Context
+from opentelemetry.trace import SpanKind
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from pydantic import ValidationError
+
+from app.core.background import (
+ TRANSIENT_JOB_ERRORS,
+ log_background_error,
+ shutdown_event,
+ wait_or_stop,
+)
+from app.core.config import Settings
+from app.core.database import create_db_engine, create_session_factory
+from app.core.jobs import JobSettings
+from app.core.telemetry import Telemetry
+from app.domain.attachment_jobs import process_attachment_job
+from app.errors.exceptions import ProcessingError
+from app.schemas.job import JobMessage
+from app.storage.broker import JobBroker
+from app.storage.s3 import create_s3_storage
+
+
+@dataclass(frozen=True)
+class Faults:
+ """Только явные CLI-флаги; обычный worker не включает инъекции сбоев."""
+
+ crash_after_object: UUID | None = None
+ crash_before_ack: UUID | None = None
+ fail_transient: UUID | None = None
+ fail_permanent: UUID | None = None
+ object_name: str = "optimized"
+
+ def kill(self, attachment_id: UUID, phase: str) -> None:
+ print(
+ json.dumps({"fault": phase, "attachment_id": str(attachment_id)}),
+ flush=True,
+ )
+ os.kill(os.getpid(), signal.SIGKILL)
+
+ def after_object(self, attachment_id: UUID, name: str) -> None:
+ if attachment_id == self.crash_after_object and name == self.object_name:
+ self.kill(attachment_id, "after_object:" + name)
+
+ def before_ack(self, attachment_id: UUID) -> None:
+ if attachment_id == self.crash_before_ack:
+ self.kill(attachment_id, "before_ack")
+
+ def before_processing(self, attachment_id: UUID) -> None:
+ if attachment_id == self.fail_transient:
+ raise TimeoutError("injected transient failure")
+ if attachment_id == self.fail_permanent:
+ raise ProcessingError("injected_permanent")
+
+
+async def consume_connected(
+ factory,
+ storage,
+ broker: JobBroker,
+ settings: JobSettings,
+ stop: asyncio.Event,
+ faults: Faults,
+ telemetry: Telemetry | None = None,
+) -> None:
+ consumer = uuid4().hex
+ while not stop.is_set():
+ try:
+ await broker.ensure_ready()
+ delivery = await broker.receive(consumer)
+ if delivery is None:
+ continue
+ try:
+ message = JobMessage.model_validate_json(delivery.payload)
+ except ValidationError:
+ await broker.reject_invalid(delivery.id)
+ continue
+ try:
+ telemetry = telemetry or Telemetry("messenger-worker", enabled=False)
+ parent = TraceContextTextMapPropagator().extract(
+ delivery.id.headers or {}, context=Context()
+ )
+ with telemetry.span(
+ "attachment.process.v1",
+ context=parent,
+ kind=SpanKind.CONSUMER,
+ attributes={
+ "messaging.system": "rabbitmq",
+ "job.type": message.type,
+ "job.attempt": message.attempt,
+ },
+ ):
+ telemetry.event(
+ "job.received",
+ job_id=str(message.job_id),
+ attachment_id=str(message.attachment_id),
+ )
+ async with factory() as session:
+ acknowledge = await process_attachment_job(
+ session,
+ storage,
+ message,
+ settings,
+ after_object=faults.after_object,
+ before_processing=faults.before_processing,
+ telemetry=telemetry,
+ )
+ except ProcessingError:
+ await broker.reject_invalid(delivery.id)
+ else:
+ if acknowledge:
+ faults.before_ack(message.attachment_id)
+ await broker.acknowledge_job(delivery.id)
+ else:
+ await broker.defer_job(delivery.id)
+ except TRANSIENT_JOB_ERRORS as error:
+ log_background_error("Worker delivery deferred", error)
+ await broker.aclose()
+ await wait_or_stop(stop, settings.publisher_poll_seconds)
+
+
+async def consume(factory, storage, settings, stop, faults, telemetry=None) -> None:
+ broker = JobBroker(settings)
+ try:
+ await consume_connected(
+ factory, storage, broker, settings, stop, faults, telemetry
+ )
+ finally:
+ await broker.aclose()
+
+
+async def run_worker(faults: Faults) -> None:
+ settings = Settings()
+ jobs = JobSettings()
+ telemetry = Telemetry("messenger-worker")
+ logging.basicConfig(level=settings.log_level)
+ if not features.check("webp"):
+ logging.error("Pillow requires WebP support for the image worker")
+ raise RuntimeError("WebP codec is unavailable")
+ stop = shutdown_event()
+ engine = create_db_engine(settings)
+ telemetry.observe_pool(engine.pool)
+ telemetry.gauge("messenger.worker.busy", lambda: telemetry.busy)
+ telemetry.gauge("messenger.worker.capacity", lambda: jobs.worker_concurrency)
+ storage = None
+ try:
+ storage = create_s3_storage(settings.s3)
+ factory = create_session_factory(engine)
+ async with asyncio.TaskGroup() as group:
+ tasks = [
+ group.create_task(
+ consume(factory, storage, jobs, stop, faults, telemetry)
+ )
+ for _ in range(jobs.worker_concurrency)
+ ]
+ await stop.wait()
+ # Отмена убивает resize-процесс, дожидается SDK и не отправляет ack.
+ for task in tasks:
+ task.cancel()
+ finally:
+ if storage is not None:
+ await storage.aclose()
+ await engine.dispose()
+ await asyncio.to_thread(telemetry.shutdown)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description="Image worker с явными fault-injection hooks"
+ )
+ modes = parser.add_mutually_exclusive_group()
+ for flag in (
+ "crash-after-object",
+ "crash-before-ack",
+ "fail-transient",
+ "fail-permanent",
+ ):
+ modes.add_argument("--" + flag, type=UUID, metavar="ATTACHMENT_ID")
+ parser.add_argument(
+ "--object-name",
+ choices=("original", "thumbnail", "crop", "optimized"),
+ default="optimized",
+ )
+ args = parser.parse_args()
+ faults = Faults(**vars(args))
+ try:
+ asyncio.run(run_worker(faults))
+ except Exception as error:
+ log_background_error("Worker stopped", error)
+ raise SystemExit(1) from None
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/core/jobs.py b/app/core/jobs.py
new file mode 100644
index 0000000..a67c86d
--- /dev/null
+++ b/app/core/jobs.py
@@ -0,0 +1,37 @@
+"""Настройки очереди из .env; credentials брокера читаются отдельно из файла."""
+
+from pathlib import Path
+from typing import Self
+
+from pydantic import Field, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class JobSettings(BaseSettings):
+ model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+ broker_host: str = "broker"
+ broker_port: int = Field(default=5672, ge=1, le=65535)
+ broker_password_file: Path = Path("/run/secrets/broker_password")
+ broker_queue: str = "attachment-jobs"
+ broker_user: str = "messenger"
+ broker_vhost: str = "/"
+ worker_concurrency: int = Field(default=2, ge=1, le=4)
+ job_max_attempts: int = Field(default=5, ge=1, le=100)
+ job_timeout_seconds: int = Field(default=60, ge=10, le=300)
+ job_lease_seconds: int = Field(default=90, ge=20, le=600)
+ job_retry_seconds: int = Field(default=5, ge=1, le=300)
+ job_retry_max_seconds: int = Field(default=120, ge=1, le=3600)
+ failed_staging_retention_seconds: int = Field(default=604800, ge=300)
+ publisher_poll_seconds: float = Field(default=1, gt=0, le=10)
+ staging_retention_seconds: int = Field(default=86400, ge=300)
+
+ @model_validator(mode="after")
+ def validate_lease(self) -> Self:
+ if self.job_lease_seconds < self.job_timeout_seconds + 30:
+ raise ValueError(
+ "Lease должен превышать timeout задания минимум на 30 секунд"
+ )
+ if self.job_retry_max_seconds < self.job_retry_seconds:
+ raise ValueError("Retry maximum must be at least the base delay")
+ return self
diff --git a/app/core/lifecycle.py b/app/core/lifecycle.py
new file mode 100644
index 0000000..2308677
--- /dev/null
+++ b/app/core/lifecycle.py
@@ -0,0 +1,45 @@
+"""Готовность реплики и мягкое завершение сервера."""
+
+import asyncio
+from types import FrameType
+
+import uvicorn
+
+
+class Lifecycle:
+
+ def __init__(self) -> None:
+ self._draining = False
+
+ @property
+ def draining(self) -> bool:
+ return self._draining
+
+ def start_draining(self) -> None:
+ self._draining = True
+
+
+class GracefulServer(uvicorn.Server):
+
+ def __init__(
+ self, config: uvicorn.Config, lifecycle: Lifecycle, drain_seconds: float
+ ) -> None:
+ super().__init__(config)
+ self.lifecycle = lifecycle
+ self.drain_seconds = drain_seconds
+ self._drain_task: asyncio.Task[None] | None = None
+
+ def handle_exit(self, sig: int, frame: FrameType | None) -> None:
+ if self._drain_task is not None:
+ if not self._drain_task.done():
+ self._drain_task.cancel()
+ super().handle_exit(sig, frame)
+ return
+ self.lifecycle.start_draining()
+ self._drain_task = asyncio.get_running_loop().create_task(
+ self._finish_draining(sig, frame)
+ )
+
+ async def _finish_draining(self, sig: int, frame: FrameType | None) -> None:
+ await asyncio.sleep(self.drain_seconds)
+ super().handle_exit(sig, frame)
diff --git a/app/core/outbox_publisher.py b/app/core/outbox_publisher.py
new file mode 100644
index 0000000..4f10ffa
--- /dev/null
+++ b/app/core/outbox_publisher.py
@@ -0,0 +1,105 @@
+"""Доставка намерений из PostgreSQL; crash hook включается только явно через CLI."""
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import signal
+import time
+from uuid import UUID
+
+from app.core.background import (
+ TRANSIENT_JOB_ERRORS,
+ log_background_error,
+ shutdown_event,
+ wait_or_stop,
+)
+from app.core.config import Settings
+from app.core.database import create_db_engine, create_session_factory
+from app.core.jobs import JobSettings
+from app.domain.attachment_jobs import publish_next_job, recover_expired_jobs
+from app.storage.broker import JobBroker
+from app.storage.jobs import queued_backlog
+from app.core.telemetry import Telemetry
+
+
+def crash_before_publish(attachment_id: UUID) -> None:
+ """Реальный SIGKILL после чтения принятой job, до publisher confirm и отметки publish."""
+ print(
+ json.dumps({"fault": "before_publish", "attachment_id": str(attachment_id)}),
+ flush=True,
+ )
+ os.kill(os.getpid(), signal.SIGKILL)
+
+
+async def run_publisher(crash_attachment_id: UUID | None = None) -> None:
+ settings = Settings()
+ jobs = JobSettings()
+ logging.basicConfig(level=settings.log_level)
+ stop = shutdown_event()
+ engine = create_db_engine(settings)
+ telemetry = Telemetry("messenger-publisher")
+ telemetry.observe_pool(engine.pool)
+ backlog = (0, 0.0)
+ labels = {"queue.name": jobs.broker_queue}
+ telemetry.gauge("messenger.queue.depth", lambda: backlog[0], attributes=labels)
+ telemetry.gauge(
+ "messenger.queue.oldest", lambda: backlog[1], unit="s", attributes=labels
+ )
+ next_sample = 0.0
+ broker = None
+ try:
+ broker = JobBroker(jobs)
+ factory = create_session_factory(engine)
+ while not stop.is_set():
+ published = False
+ try:
+ async with factory() as session:
+ if telemetry.enabled and time.monotonic() >= next_sample:
+ backlog = await queued_backlog(session)
+ next_sample = time.monotonic() + 2
+ await broker.ensure_ready()
+ await recover_expired_jobs(session, jobs)
+ published = await publish_next_job(
+ session,
+ broker,
+ telemetry=telemetry,
+ attachment_id=crash_attachment_id,
+ before_publish=(
+ crash_before_publish
+ if crash_attachment_id is not None
+ else None
+ ),
+ )
+ except TRANSIENT_JOB_ERRORS as error:
+ log_background_error("Publisher deferred", error)
+ if not published:
+ await wait_or_stop(stop, jobs.publisher_poll_seconds)
+ finally:
+ if broker is not None:
+ await broker.aclose()
+ await engine.dispose()
+ await asyncio.to_thread(telemetry.shutdown)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description="Публикация устойчивых jobs в RabbitMQ"
+ )
+ parser.add_argument(
+ "--crash-before-publish",
+ type=UUID,
+ metavar="ATTACHMENT_ID",
+ help="аварийно завершить этот процесс перед публикацией только указанного вложения",
+ )
+ args = parser.parse_args()
+ try:
+ asyncio.run(run_publisher(args.crash_before_publish))
+ except Exception as error:
+ log_background_error("Publisher stopped", error)
+ raise SystemExit(1) from None
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/core/redis.py b/app/core/redis.py
new file mode 100644
index 0000000..fb61304
--- /dev/null
+++ b/app/core/redis.py
@@ -0,0 +1,24 @@
+from redis.asyncio import BlockingConnectionPool, Redis
+from redis.asyncio.retry import Retry
+from redis.backoff import NoBackoff
+
+from app.core.config import Settings
+
+
+def create_redis_client(settings: Settings) -> Redis:
+ password = (
+ settings.redis_password.get_secret_value() if settings.redis_password else None
+ )
+ pool = BlockingConnectionPool(
+ host=settings.redis_host,
+ port=settings.redis_port,
+ db=settings.redis_db,
+ password=password,
+ max_connections=settings.redis_max_connections,
+ timeout=settings.redis_pool_timeout_seconds,
+ socket_connect_timeout=settings.redis_connect_timeout_seconds,
+ socket_timeout=settings.redis_socket_timeout_seconds,
+ retry=Retry(NoBackoff(), retries=1),
+ decode_responses=True,
+ )
+ return Redis.from_pool(pool)
diff --git a/app/core/request_capacity.py b/app/core/request_capacity.py
new file mode 100644
index 0000000..31968e7
--- /dev/null
+++ b/app/core/request_capacity.py
@@ -0,0 +1,42 @@
+from collections.abc import AsyncIterator
+
+from fastapi import Request
+
+from app.errors.exceptions import rate_limited, unavailable
+
+
+class RequestCapacity:
+ def __init__(self, max_in_flight: int) -> None:
+ self._max_in_flight = max_in_flight
+ self._in_flight = 0
+
+ @property
+ def in_flight(self) -> int:
+ return self._in_flight
+
+ def try_acquire(self) -> bool:
+ if self._in_flight >= self._max_in_flight:
+ return False
+ self._in_flight += 1
+ return True
+
+ def release(self) -> None:
+ self._in_flight -= 1
+
+
+def create_request_capacity(max_in_flight: int) -> RequestCapacity:
+ return RequestCapacity(max_in_flight)
+
+
+async def acquire_request_slot(request: Request) -> AsyncIterator[None]:
+ if request.app.state.lifecycle.draining:
+ raise unavailable("Реплика завершает работу")
+ capacity: RequestCapacity = request.app.state.request_capacity
+
+ if not capacity.try_acquire():
+ raise rate_limited()
+
+ try:
+ yield
+ finally:
+ capacity.release()
diff --git a/app/core/request_id.py b/app/core/request_id.py
new file mode 100644
index 0000000..a78a860
--- /dev/null
+++ b/app/core/request_id.py
@@ -0,0 +1,22 @@
+from uuid import uuid4
+
+from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
+from starlette.requests import Request
+from starlette.responses import Response
+
+REQUEST_ID_HEADER = "X-Request-Id"
+
+
+class RequestIdMiddleware(BaseHTTPMiddleware):
+ async def dispatch(
+ self,
+ request: Request,
+ call_next: RequestResponseEndpoint,
+ ) -> Response:
+ request_id = str(uuid4())
+ request.state.request_id = request_id
+
+ response = await call_next(request)
+ response.headers[REQUEST_ID_HEADER] = request_id
+
+ return response
diff --git a/app/core/s3.py b/app/core/s3.py
new file mode 100644
index 0000000..f2de188
--- /dev/null
+++ b/app/core/s3.py
@@ -0,0 +1,47 @@
+"""Конфигурация MinIO; значения credentials читаются только из файлов."""
+
+from pathlib import Path
+from urllib.parse import urlsplit
+
+from pydantic import Field, model_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class S3Settings(BaseSettings):
+ endpoint: str = "http://minio:9000"
+ region: str = "us-east-1"
+ bucket: str = "messenger-attachments"
+ access_key_id_file: Path = Path("/run/secrets/s3_access_key_id")
+ secret_access_key_file: Path = Path("/run/secrets/s3_secret_access_key")
+ connect_timeout_seconds: float = Field(default=2, gt=0, le=5)
+ read_timeout_seconds: float = Field(default=10, gt=0, le=15)
+ upload_timeout_seconds: float = Field(default=30, gt=0, le=60)
+ download_timeout_seconds: float = Field(default=60, gt=0, le=120)
+ image_timeout_seconds: float = Field(default=10, gt=0, le=15)
+ max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760)
+ max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000)
+ chunk_bytes: int = Field(default=65_536, ge=1024, le=262_144)
+ max_concurrent_uploads: int = Field(default=2, ge=1, le=2)
+ cleanup_grace_seconds: int = Field(default=86_400, ge=300)
+ cleanup_interval_seconds: int = Field(default=300, ge=1)
+
+ model_config = SettingsConfigDict(
+ env_prefix="S3_", env_file=".env", extra="ignore", frozen=True
+ )
+
+ @model_validator(mode="after")
+ def validate_endpoint(self):
+ url = urlsplit(self.endpoint)
+ if (
+ url.scheme not in {"http", "https"}
+ or not url.hostname
+ or url.username
+ or url.password
+ or url.path not in {"", "/"}
+ or url.query
+ or url.fragment
+ ):
+ raise ValueError(
+ "S3_ENDPOINT должен быть http(s) адресом без credentials/пути"
+ )
+ return self
diff --git a/app/core/security.py b/app/core/security.py
new file mode 100644
index 0000000..5e7bb97
--- /dev/null
+++ b/app/core/security.py
@@ -0,0 +1,56 @@
+"""Хеширование паролей (Argon2id) и выпуск непрозрачных токенов сессий."""
+
+import asyncio
+import hashlib
+import os
+import secrets
+from concurrent.futures import ThreadPoolExecutor
+
+from argon2 import PasswordHasher
+from argon2.exceptions import InvalidHashError, VerificationError
+
+TOKEN_BYTES = 32
+
+_hasher = PasswordHasher()
+_executor = ThreadPoolExecutor(
+ max_workers=max(1, min(2, (os.cpu_count() or 1) // _hasher.parallelism)),
+ thread_name_prefix="argon2",
+)
+
+
+def _hash_password_sync(password: str) -> str:
+ return _hasher.hash(password)
+
+
+def _verify_password_sync(password: str, password_hash: str) -> bool:
+ try:
+ return _hasher.verify(password_hash, password)
+ except (VerificationError, InvalidHashError):
+ return False
+
+
+_DUMMY_HASH = _hash_password_sync(secrets.token_urlsafe(TOKEN_BYTES))
+
+
+async def hash_password(password: str) -> str:
+ loop = asyncio.get_running_loop()
+ return await loop.run_in_executor(_executor, _hash_password_sync, password)
+
+
+async def verify_password(password: str, password_hash: str) -> bool:
+ loop = asyncio.get_running_loop()
+ return await loop.run_in_executor(
+ _executor, _verify_password_sync, password, password_hash
+ )
+
+
+async def dummy_verify_password(password: str) -> None:
+ await verify_password(password, _DUMMY_HASH)
+
+
+def generate_token() -> str:
+ return secrets.token_urlsafe(TOKEN_BYTES)
+
+
+def hash_token(token: str) -> str:
+ return hashlib.sha256(token.encode()).hexdigest()
diff --git a/app/core/telemetry.py b/app/core/telemetry.py
new file mode 100644
index 0000000..a766583
--- /dev/null
+++ b/app/core/telemetry.py
@@ -0,0 +1,288 @@
+"""Явная OTLP-инструментация: экспортируются только выбранные технические поля."""
+
+from contextlib import contextmanager
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import socket
+import time
+
+from opentelemetry import trace
+from opentelemetry.context import Context
+from opentelemetry._logs import SeverityNumber
+from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
+from opentelemetry.exporter.otlp.proto.http.metric_exporter import OTLPMetricExporter
+from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
+from opentelemetry.sdk._logs import LoggerProvider
+from opentelemetry.sdk._logs.export import BatchLogRecordProcessor
+from opentelemetry.metrics import Observation
+from opentelemetry.sdk.metrics import MeterProvider
+from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader
+from opentelemetry.sdk.metrics.view import ExplicitBucketHistogramAggregation, View
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider
+from opentelemetry.sdk.trace.export import BatchSpanProcessor
+from opentelemetry.trace import SpanKind, Status, StatusCode
+
+HTTP_BUCKETS = (0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10)
+JOB_BUCKETS = (0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60)
+
+
+class Telemetry:
+ def __init__(self, service: str, instance: str | None = None, *, enabled=None):
+ self.enabled = (
+ os.getenv("OTEL_ENABLED", "false").lower() == "true"
+ if enabled is None
+ else enabled
+ )
+ self.service = service
+ self.busy = 0
+ if not self.enabled:
+ self.tracer = trace.NoOpTracerProvider().get_tracer("messenger")
+ return
+ endpoint = os.getenv(
+ "OTEL_EXPORTER_OTLP_ENDPOINT", "http://otel-collector:4318"
+ ).rstrip("/")
+ resource = Resource(
+ {
+ "service.name": service,
+ "service.version": "1.6.0",
+ "service.instance.id": instance or socket.gethostname(),
+ "deployment.environment.name": os.getenv("APP_ENV", "development"),
+ }
+ )
+ self.traces = TracerProvider(resource=resource)
+ self.traces.add_span_processor(
+ BatchSpanProcessor(
+ OTLPSpanExporter(endpoint=endpoint + "/v1/traces", timeout=2),
+ max_queue_size=512,
+ max_export_batch_size=128,
+ schedule_delay_millis=1000,
+ )
+ )
+ self.tracer = self.traces.get_tracer("messenger", "1.6.0")
+ self.logs = LoggerProvider(resource=resource)
+ self.logs.add_log_record_processor(
+ BatchLogRecordProcessor(
+ OTLPLogExporter(endpoint=endpoint + "/v1/logs", timeout=2),
+ max_queue_size=512,
+ max_export_batch_size=128,
+ schedule_delay_millis=1000,
+ )
+ )
+ self.logger = self.logs.get_logger("messenger", "1.6.0")
+ self.metrics = MeterProvider(
+ resource=resource,
+ metric_readers=[
+ PeriodicExportingMetricReader(
+ OTLPMetricExporter(endpoint=endpoint + "/v1/metrics", timeout=2),
+ export_interval_millis=5000,
+ export_timeout_millis=3000,
+ )
+ ],
+ views=[
+ View(
+ instrument_name="messenger.http.duration",
+ aggregation=ExplicitBucketHistogramAggregation(HTTP_BUCKETS),
+ ),
+ View(
+ instrument_name="messenger.job.time_to_ready",
+ aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
+ ),
+ View(
+ instrument_name="messenger.job.duration",
+ aggregation=ExplicitBucketHistogramAggregation(JOB_BUCKETS),
+ ),
+ ],
+ )
+ meter = self.meter = self.metrics.get_meter("messenger", "1.6.0")
+ self.http_requests = meter.create_counter("messenger.http.requests", unit="1")
+ self.http_duration = meter.create_histogram("messenger.http.duration", unit="s")
+ self.jobs = meter.create_counter("messenger.job.attempts", unit="1")
+ self.job_duration = meter.create_histogram("messenger.job.duration", unit="s")
+
+ self.time_to_ready = meter.create_histogram(
+ "messenger.job.time_to_ready", unit="s"
+ )
+ self.gauge("messenger.process.cpu", time.process_time, unit="s", counter=True)
+ self.gauge("messenger.process.rss", resident_bytes, unit="By")
+
+ def gauge(self, name, sample, *, unit="1", attributes=None, counter=False):
+ if self.enabled:
+ create = (
+ self.meter.create_observable_counter
+ if counter
+ else self.meter.create_observable_gauge
+ )
+ create(
+ name,
+ callbacks=[lambda _: [Observation(sample(), attributes)]],
+ unit=unit,
+ )
+
+ def observe_pool(self, pool):
+ if self.enabled:
+ pool.telemetry = self
+ self.gauge("messenger.db.pool.used", pool.checkedout)
+ self.gauge("messenger.db.pool.waiting", lambda: pool.waiting)
+
+ @contextmanager
+ def span(self, name, **kwargs):
+ with self.tracer.start_as_current_span(
+ name,
+ record_exception=False,
+ set_status_on_exception=False,
+ **kwargs,
+ ) as span:
+ try:
+ yield span
+ except BaseException:
+ span.set_status(Status(StatusCode.ERROR))
+ raise
+
+ def event(self, event: str, *, error=False, **fields):
+ if not self.enabled:
+ return
+ context = trace.get_current_span().get_span_context()
+ body = {
+ "timestamp": datetime.now(timezone.utc).isoformat(),
+ "severity": "ERROR" if error else "INFO",
+ "service_name": self.service,
+ "event": event,
+ "trace_id": f"{context.trace_id:032x}",
+ "span_id": f"{context.span_id:016x}",
+ **fields,
+ }
+ text = json.dumps(body, ensure_ascii=False)
+ # Не подключаем root logging: SDK/SQL-исключения могут содержать секреты.
+ self.logger.emit(
+ body=text,
+ severity_text=body["severity"],
+ severity_number=SeverityNumber.ERROR if error else SeverityNumber.INFO,
+ attributes={"event": event},
+ )
+ print(text, flush=True)
+
+ @contextmanager
+ def job_attempt(self):
+ self.busy += 1
+ started = time.monotonic()
+ result = {"outcome": "success"}
+ try:
+ yield result
+ except BaseException:
+ result["outcome"] = "error"
+ raise
+ finally:
+ self.busy -= 1
+ if self.enabled:
+ attrs = {
+ "job.type": "attachment.process.v1",
+ "outcome": result["outcome"],
+ }
+ elapsed = time.monotonic() - started
+ self.jobs.add(1, attrs)
+ self.job_duration.record(elapsed, attrs)
+ span = trace.get_current_span()
+ span.set_attribute("outcome", result["outcome"])
+ span.set_status(
+ Status(
+ StatusCode.ERROR
+ if result["outcome"] == "error"
+ else StatusCode.OK
+ )
+ )
+ self.event(
+ "job.processed",
+ error=result["outcome"] == "error",
+ job_type="attachment.process.v1",
+ duration_seconds=elapsed,
+ **result,
+ )
+
+ def shutdown(self):
+ if self.enabled:
+ self.metrics.shutdown(timeout_millis=5000)
+ self.logs.shutdown()
+ self.traces.shutdown()
+
+
+def resident_bytes():
+ return int(Path("/proc/self/statm").read_text().split()[1]) * os.sysconf(
+ "SC_PAGE_SIZE"
+ )
+
+
+class HttpTelemetryMiddleware:
+ def __init__(self, app, telemetry: Telemetry):
+ self.app = app
+ self.telemetry = telemetry
+
+ async def __call__(self, scope, receive, send):
+ if scope["type"] != "http" or not self.telemetry.enabled:
+ return await self.app(scope, receive, send)
+ telemetry = self.telemetry
+ method = scope["method"]
+ if method not in {"GET", "POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"}:
+ method = "OTHER"
+ started = time.monotonic()
+ status = 500
+ request_id = ""
+ with telemetry.tracer.start_as_current_span(
+ "HTTP",
+ context=Context(),
+ kind=SpanKind.SERVER,
+ record_exception=False,
+ set_status_on_exception=False,
+ ) as span:
+
+ async def observed_send(message):
+ nonlocal status, request_id
+ if message["type"] == "http.response.start":
+ status = message["status"]
+ headers = list(message.get("headers", []))
+ request_id = next(
+ (
+ v.decode()
+ for k, v in headers
+ if k.lower() == b"x-request-id"
+ ),
+ "",
+ )
+ headers.append(
+ (
+ b"x-trace-id",
+ f"{span.get_span_context().trace_id:032x}".encode(),
+ )
+ )
+ message = {**message, "headers": headers}
+ await send(message)
+
+ try:
+ await self.app(scope, receive, observed_send)
+ finally:
+ request_id = request_id or scope.get("state", {}).get("request_id", "")
+ route = getattr(scope.get("route"), "path", "unmatched")
+ attrs = {
+ "http.request.method": method,
+ "http.route": route,
+ "http.response.status_class": f"{status // 100}xx",
+ }
+ elapsed = time.monotonic() - started
+ telemetry.http_requests.add(1, attrs)
+ telemetry.http_duration.record(elapsed, attrs)
+ span.update_name(f"{method} {route}")
+ span.set_attributes({**attrs, "http.response.status_code": status})
+ span.set_status(
+ Status(StatusCode.ERROR if status >= 500 else StatusCode.OK)
+ )
+ telemetry.event(
+ "http.request",
+ error=status >= 500,
+ method=method,
+ route=route,
+ status_code=status,
+ request_id=request_id,
+ duration_seconds=elapsed,
+ )
diff --git a/app/core/thumbnail_worker.py b/app/core/thumbnail_worker.py
new file mode 100644
index 0000000..2c46510
--- /dev/null
+++ b/app/core/thumbnail_worker.py
@@ -0,0 +1,32 @@
+"""Один pipeline изображений в Linux-процессе с лимитами памяти и CPU."""
+
+from pathlib import Path
+import resource
+import sys
+
+from app.core.image_processing import ImageSettings
+from app.domain.image_pipeline import create_variants
+from app.errors.exceptions import IMAGE_WORKER_ERRORS, pipeline_worker_error_code
+
+
+def main() -> int:
+ settings = ImageSettings()
+ memory = settings.memory_mib * 1024 * 1024
+ resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
+ resource.setrlimit(
+ resource.RLIMIT_CPU, (settings.cpu_seconds, settings.cpu_seconds)
+ )
+ source, directory, metadata, content_type, maximum = sys.argv[1:]
+ code = 0
+ try:
+ result = create_variants(
+ Path(source), Path(directory), content_type, int(maximum), settings
+ )
+ Path(metadata).write_text(result.model_dump_json())
+ except IMAGE_WORKER_ERRORS as error:
+ code = pipeline_worker_error_code(error)
+ return code
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/app/domain/__init__.py b/app/domain/__init__.py
new file mode 100644
index 0000000..93c96d5
--- /dev/null
+++ b/app/domain/__init__.py
@@ -0,0 +1 @@
+"""Бизнес-логика: проверки доступа, транзакции и ошибки API."""
diff --git a/app/domain/attachment_jobs.py b/app/domain/attachment_jobs.py
new file mode 100644
index 0000000..c4f75a5
--- /dev/null
+++ b/app/domain/attachment_jobs.py
@@ -0,0 +1,304 @@
+"""Приём, durable публикация, попытки и фиксация результата обработки."""
+
+import asyncio
+from collections.abc import Callable
+from contextlib import nullcontext
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from uuid import UUID, uuid4
+
+from minio.error import MinioException
+from opentelemetry.context import Context
+from opentelemetry.trace import SpanKind
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from sqlalchemy.ext.asyncio import AsyncSession
+from urllib3.exceptions import HTTPError as StorageHTTPError
+
+from app.core.jobs import JobSettings
+from app.core.telemetry import Telemetry
+from app.domain.job_retry import classify_failure, retry_delay
+from app.domain.job_files import fetch_original, render_variants
+from app.errors.exceptions import ProcessingError
+from app.schemas.attachment import VARIANT_FILENAMES
+from app.schemas.job import JobMessage
+from app.storage import jobs, outbox
+from app.storage.broker import JobBroker
+from app.storage.jobs import JobWork
+from app.storage.s3 import MinioStorage
+
+
+@dataclass(frozen=True)
+class Claim:
+ work: JobWork | None = None
+ acknowledge: bool = False
+
+
+async def enqueue_attachment_job(
+ session: AsyncSession, attachment_id: UUID, staging_key: str
+) -> UUID:
+ """Включить намерение обработки в транзакцию queued metadata; без commit."""
+ return await jobs.create_job(session, attachment_id, staging_key)
+
+
+async def publish_next_job(
+ session: AsyncSession,
+ broker: JobBroker,
+ *,
+ attachment_id: UUID | None = None,
+ before_publish: Callable[[UUID], None] | None = None,
+ telemetry: Telemetry | None = None,
+) -> bool:
+ """Отметка publish следует за durable publisher confirm; crash допускает тот же job-дубль."""
+ job = await outbox.next_unpublished_job(session, attachment_id=attachment_id)
+ published = job is not None
+ if job is not None:
+ if before_publish is not None:
+ before_publish(job.attachment_id)
+ parent = TraceContextTextMapPropagator().extract(
+ job.trace_context or {}, context=Context()
+ )
+ with (
+ telemetry.span(
+ "rabbitmq.publish",
+ context=parent,
+ kind=SpanKind.PRODUCER,
+ attributes={
+ "messaging.system": "rabbitmq",
+ "job.attempt": job.attempt + 1,
+ },
+ )
+ if telemetry
+ else nullcontext()
+ ):
+ await broker.publish_job(
+ JobMessage(
+ job_id=job.id,
+ attachment_id=job.attachment_id,
+ pipeline_version=job.pipeline_version,
+ attempt=job.attempt + 1,
+ created_at=job.created_at,
+ )
+ )
+ outbox.mark_published(job)
+ await session.commit()
+ return published
+
+
+async def claim_job(
+ session: AsyncSession, message: JobMessage, settings: JobSettings
+) -> Claim:
+ """Дубликат не захватывает живую аренду и не откатывает terminal-состояние."""
+ row = await jobs.lock_job(session, message.job_id)
+ result = Claim()
+ if row is None:
+ raise ProcessingError("unknown_job")
+ job, attachment = row
+ if (
+ message.attachment_id != attachment.id
+ or message.pipeline_version != job.pipeline_version
+ ):
+ raise ProcessingError("invalid_job")
+ now = await jobs.database_time(session)
+ if job.attempt_limit is None:
+ job.attempt_limit = settings.job_max_attempts
+ if attachment.status in {"ready", "failed"}:
+ result = Claim(acknowledge=True)
+ elif message.attempt <= job.attempt and attachment.status == "queued":
+ # Предыдущая попытка уже устойчиво заменена новой в PostgreSQL.
+ result = Claim(acknowledge=True)
+ elif message.attempt < job.attempt:
+ result = Claim(acknowledge=True)
+ elif message.attempt > job.attempt + 1:
+ raise ProcessingError("invalid_attempt")
+ elif attachment.status == "queued" and job.next_attempt_at <= now:
+ if (
+ job.attempt >= (job.attempt_limit or settings.job_max_attempts)
+ or job.staging_key is None
+ ):
+ jobs.record_failure(
+ job, attachment, now, "retry_exhausted", terminal=True, retry_seconds=0
+ )
+ result = Claim(acknowledge=True)
+ else:
+ result = Claim(
+ work=jobs.start_job(
+ job, attachment, uuid4(), now, settings.job_lease_seconds
+ )
+ )
+ await session.commit()
+ return result
+
+
+async def save_failure(
+ session: AsyncSession,
+ work: JobWork,
+ settings: JobSettings,
+ code: str,
+ *,
+ permanent: bool,
+) -> None:
+ # Неоднозначный COMMIT проверяется повторным чтением: ready сохраняется.
+ await session.rollback()
+ row = await jobs.lock_job(session, work.job_id)
+ if row is not None:
+ job, attachment = row
+ if attachment.status == "processing" and job.lease_token == work.lease_token:
+ exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
+ jobs.record_failure(
+ job,
+ attachment,
+ await jobs.database_time(session),
+ "retry_exhausted" if exhausted and not permanent else code,
+ terminal=permanent or exhausted,
+ retry_seconds=retry_delay(
+ settings, job.attempt - job.cycle_start_attempt
+ ),
+ )
+ job.last_error_code = code
+ job.last_error_permanent = permanent
+ await session.commit()
+
+
+async def build_result(
+ session: AsyncSession,
+ storage: MinioStorage,
+ work: JobWork,
+ after_object: Callable[[UUID, str], None] | None = None,
+ telemetry: Telemetry | None = None,
+) -> bool:
+ """Фиксированные ключи и блокировка job защищают PUT от устаревшей попытки."""
+ with TemporaryDirectory(prefix="messenger-job-") as directory:
+ source = Path(directory) / "original"
+ with telemetry.span("s3.read_staging") if telemetry else nullcontext():
+ await fetch_original(storage, work, source)
+ result = await render_variants(storage, work, source, Path(directory))
+ row = await jobs.lock_job(session, work.job_id)
+ if row is None:
+ await session.rollback()
+ return True
+ job, attachment = row
+ now = await jobs.database_time(session)
+ if attachment.status != "processing" or job.lease_token != work.lease_token:
+ await session.rollback()
+ return True
+ if job.lease_until is None or job.lease_until <= now:
+ await session.rollback()
+ return False
+ # Блокировка остаётся до завершения PUT/COMMIT; reaper использует SKIP LOCKED.
+ # При отмене S3 adapter дожидается SDK-потока до освобождения блокировки.
+ with telemetry.span("s3.write_result") if telemetry else nullcontext():
+ await storage.put_file(
+ work.original_key,
+ source,
+ size_bytes=work.size_bytes,
+ content_type=work.content_type,
+ sha256=work.sha256,
+ )
+ if after_object is not None:
+ after_object(work.attachment_id, "original")
+ for variant in result.variants:
+ await storage.put_file(
+ work.variant_key(variant.name),
+ Path(directory) / VARIANT_FILENAMES[variant.name],
+ size_bytes=variant.size_bytes,
+ content_type=variant.content_type,
+ sha256=variant.sha256,
+ )
+ if after_object is not None:
+ after_object(work.attachment_id, variant.name)
+ with telemetry.span("postgres.commit_ready") if telemetry else nullcontext():
+ ready_at = await jobs.database_time(session)
+ accepted_at = job.created_at
+ jobs.complete_job(
+ job,
+ attachment,
+ work,
+ [variant.model_dump() for variant in result.variants],
+ ready_at,
+ )
+ await session.commit()
+ if telemetry and telemetry.enabled:
+ telemetry.time_to_ready.record(
+ max(0, (ready_at - accepted_at).total_seconds())
+ )
+ return True
+
+
+async def process_attachment_job(
+ session: AsyncSession,
+ storage: MinioStorage,
+ message: JobMessage,
+ settings: JobSettings,
+ *,
+ after_object: Callable[[UUID, str], None] | None = None,
+ before_processing: Callable[[UUID], None] | None = None,
+ telemetry: Telemetry | None = None,
+) -> bool:
+ with telemetry.span("postgres.claim_job") if telemetry else nullcontext():
+ claim = await claim_job(session, message, settings)
+ acknowledge = claim.acknowledge
+ if claim.work is not None:
+ work = claim.work
+ with telemetry.job_attempt() if telemetry else nullcontext({}) as result:
+ try:
+ async with asyncio.timeout(settings.job_timeout_seconds):
+ if before_processing is not None:
+ before_processing(work.attachment_id)
+ acknowledge = await build_result(
+ session, storage, work, after_object, telemetry
+ )
+ result["outcome"] = "success" if acknowledge else "deferred"
+ except (
+ ProcessingError,
+ MinioException,
+ StorageHTTPError,
+ OSError,
+ TimeoutError,
+ ) as error:
+ code, permanent = classify_failure(error)
+ await save_failure(session, work, settings, code, permanent=permanent)
+ result.update(outcome="error", error_code=code)
+ acknowledge = True
+ # Ошибка БД/неизвестная ошибка или отмена не подтверждают доставку.
+ return acknowledge
+
+
+async def recover_expired_jobs(session: AsyncSession, settings: JobSettings) -> int:
+ rows = await jobs.expired_jobs(session)
+ now = await jobs.database_time(session)
+ for job, attachment in rows:
+ if job.attempt_limit is None:
+ job.attempt_limit = settings.job_max_attempts
+ exhausted = job.attempt >= (job.attempt_limit or settings.job_max_attempts)
+ jobs.record_failure(
+ job,
+ attachment,
+ now,
+ "retry_exhausted" if exhausted else "worker_timeout",
+ terminal=exhausted,
+ retry_seconds=retry_delay(settings, job.attempt - job.cycle_start_attempt),
+ )
+ job.last_error_code = "worker_timeout"
+ job.last_error_permanent = False
+ await session.commit()
+ return len(rows)
+
+
+async def cleanup_staging(
+ session: AsyncSession, storage: MinioStorage, settings: JobSettings, *, apply: bool
+) -> int:
+ rows = await jobs.terminal_staging(
+ session,
+ settings.staging_retention_seconds,
+ settings.failed_staging_retention_seconds,
+ )
+ if apply:
+ for job, attachment in rows:
+ if job.staging_key != attachment.object_key:
+ await storage.delete_object(job.staging_key)
+ jobs.clear_staging(job)
+ await session.commit()
+ else:
+ await session.rollback()
+ return len(rows)
diff --git a/app/domain/attachments.py b/app/domain/attachments.py
new file mode 100644
index 0000000..3600ea6
--- /dev/null
+++ b/app/domain/attachments.py
@@ -0,0 +1,337 @@
+"""Авторизация, приём оригинала и согласованность PostgreSQL/MinIO."""
+
+from collections.abc import AsyncIterator, Iterator
+from contextlib import contextmanager, suppress
+from dataclasses import dataclass
+import hashlib
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from uuid import UUID, uuid4
+
+from anyio import CancelScope
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.request_capacity import RequestCapacity
+from app.domain.chats import ensure_chat_member
+from app.domain.attachment_jobs import enqueue_attachment_job
+from app.domain.images import validate_filename, validate_image
+from app.errors.exceptions import (
+ conflict,
+ invalid_request,
+ not_found,
+ payload_too_large,
+ rate_limited,
+ unavailable,
+ unsupported_media_type,
+)
+from app.errors.handlers import (
+ RECOVERABLE_STORAGE_ERRORS,
+ log_recoverable_error,
+ operation_timeout,
+)
+from app.schemas.attachment import Attachment, VARIANT_FILENAMES
+from app.schemas.user import User
+from app.storage import attachments as repository
+from app.storage.s3 import MinioStorage
+
+
+@dataclass(frozen=True)
+class CleanupResult:
+ examined: int
+ candidates: int
+ deleted: int
+
+
+@dataclass(frozen=True)
+class OriginalFile:
+ path: Path
+ size_bytes: int
+ sha256: str
+
+
+def require_storage(storage: MinioStorage | None) -> MinioStorage:
+ if storage is None:
+ raise unavailable("Объектное хранилище не настроено")
+ return storage
+
+
+def validate_declared_size(value: str | None, maximum: int) -> None:
+ if value is not None:
+ if not value.isascii() or not value.isdecimal() or len(value) > 20:
+ raise invalid_request("Некорректный Content-Length")
+ if int(value) > maximum:
+ raise payload_too_large()
+
+
+@contextmanager
+def upload_slot(capacity: RequestCapacity) -> Iterator[None]:
+ if not capacity.try_acquire():
+ raise rate_limited()
+ try:
+ yield
+ finally:
+ capacity.release()
+
+
+async def receive_attachment(
+ session: AsyncSession,
+ storage: MinioStorage | None,
+ user: User,
+ chat_id: UUID,
+ *,
+ capacity: RequestCapacity,
+ filename: str,
+ content_type: str,
+ declared_length: str | None,
+ chunks: AsyncIterator[bytes],
+) -> Attachment:
+ storage = require_storage(storage)
+ validate_declared_size(declared_length, storage.settings.max_upload_bytes)
+ timeout_error = invalid_request("Истекло время загрузки")
+ with upload_slot(capacity):
+ async with operation_timeout(
+ storage.settings.upload_timeout_seconds, timeout_error
+ ):
+ result = await upload_attachment(
+ session,
+ storage,
+ user,
+ chat_id,
+ filename=filename,
+ content_type=content_type,
+ chunks=chunks,
+ )
+ return result
+
+
+async def read_original(
+ chunks: AsyncIterator[bytes],
+ path: Path,
+ maximum: int,
+ chunk_bytes: int,
+) -> OriginalFile:
+ checksum = hashlib.sha256()
+ size = 0
+ with path.open("wb") as target:
+ async for chunk in chunks:
+ size += len(chunk)
+ if size > maximum:
+ raise payload_too_large()
+ checksum.update(chunk)
+ view = memoryview(chunk)
+ for offset in range(0, len(view), chunk_bytes):
+ target.write(view[offset : offset + chunk_bytes])
+ if size == 0:
+ raise unsupported_media_type("Пустой файл")
+ return OriginalFile(path, size, checksum.hexdigest())
+
+
+async def compensate_upload(
+ session: AsyncSession, storage: MinioStorage, id: UUID
+) -> None:
+ await session.rollback()
+ intent = await repository.lock_upload(session, id)
+ if intent is not None:
+ if await repository.get_by_key(session, intent.object_key) is None:
+ await storage.delete_object(intent.object_key)
+ await repository.finish_upload(session, id)
+ await session.commit()
+
+
+async def recover_upload(
+ session: AsyncSession, storage: MinioStorage, id: UUID
+) -> None:
+ try:
+ await compensate_upload(session, storage, id)
+ except RECOVERABLE_STORAGE_ERRORS as error:
+ log_recoverable_error("Upload compensation", error)
+
+ with suppress(*RECOVERABLE_STORAGE_ERRORS):
+ await session.rollback()
+
+
+async def stage_attachment(
+ session: AsyncSession,
+ storage: MinioStorage,
+ user: User,
+ chat_id: UUID,
+ *,
+ original: OriginalFile,
+ filename: str,
+ content_type: str,
+) -> Attachment:
+ id = uuid4()
+ key = f"staging/{uuid4().hex}"
+
+ await repository.reserve_upload(session, id, key)
+ await session.commit()
+ try:
+ if await repository.lock_upload(session, id) is None:
+ raise unavailable("Загрузка отменена до записи оригинала")
+ await storage.put_file(
+ key,
+ original.path,
+ size_bytes=original.size_bytes,
+ content_type=content_type,
+ sha256=original.sha256,
+ )
+ record = await repository.create_attachment(
+ session,
+ id=id,
+ chat_id=chat_id,
+ owner_id=user.id,
+ object_key=f"originals/{id.hex}/v1",
+ status="queued",
+ filename=filename,
+ content_type=content_type,
+ size_bytes=original.size_bytes,
+ sha256=original.sha256,
+ )
+ await enqueue_attachment_job(session, id, key)
+ await repository.finish_upload(session, id)
+
+ result = Attachment.model_validate(record)
+ await session.commit()
+ except BaseException:
+
+ with CancelScope(shield=True):
+ await recover_upload(session, storage, id)
+ raise
+ return result
+
+
+async def upload_attachment(
+ session: AsyncSession,
+ storage: MinioStorage,
+ user: User,
+ chat_id: UUID,
+ *,
+ filename: str,
+ content_type: str,
+ chunks: AsyncIterator[bytes],
+) -> Attachment:
+ await ensure_chat_member(session, user, chat_id)
+ validate_filename(filename)
+ if content_type not in {"image/png", "image/jpeg"}:
+ raise unsupported_media_type("Принимаются только image/png и image/jpeg")
+
+ await session.rollback()
+ settings = storage.settings
+ with TemporaryDirectory(prefix="messenger-upload-") as directory:
+ original = await read_original(
+ chunks,
+ Path(directory) / "original",
+ settings.max_upload_bytes,
+ settings.chunk_bytes,
+ )
+ await validate_image(original.path, content_type, settings)
+ result = await stage_attachment(
+ session,
+ storage,
+ user,
+ chat_id,
+ original=original,
+ filename=filename,
+ content_type=content_type,
+ )
+ return result
+
+
+async def get_attachment(
+ session: AsyncSession, user: User, attachment_id: UUID
+) -> Attachment:
+ record = await repository.get_attachment(session, attachment_id)
+ if record is None:
+ raise not_found()
+ await ensure_chat_member(session, user, record.chat_id)
+ return Attachment.model_validate(record)
+
+
+@dataclass(frozen=True)
+class DownloadInfo:
+ filename: str
+ content_type: str
+ size_bytes: int
+
+
+async def download_attachment(
+ session: AsyncSession,
+ storage: MinioStorage | None,
+ user: User,
+ attachment_id: UUID,
+ *,
+ variant: str = "original",
+) -> tuple[DownloadInfo, AsyncIterator[bytes]]:
+ result = await get_attachment(session, user, attachment_id)
+ if result.status != "ready":
+ raise conflict("Вложение пока недоступно для скачивания")
+ storage = require_storage(storage)
+ record = await repository.get_attachment(session, attachment_id)
+ if record is None:
+ raise not_found()
+ if variant == "original":
+ key = record.object_key
+ info = DownloadInfo(result.filename, result.content_type, result.size_bytes)
+ else:
+ selected = next(
+ (item for item in result.variants if item.name == variant), None
+ )
+ keys = {
+ "thumbnail": record.thumbnail_key,
+ "crop": record.crop_key,
+ "optimized": record.optimized_key,
+ }
+ key = keys.get(variant)
+ if selected is None or key is None:
+ raise not_found()
+ info = DownloadInfo(
+ VARIANT_FILENAMES[variant], selected.content_type, selected.size_bytes
+ )
+ await session.rollback()
+ stored = await storage.stat_object(key)
+ if stored.size_bytes != info.size_bytes or stored.content_type != info.content_type:
+ raise unavailable("Файл временно недоступен")
+ stream = await storage.open_object(key)
+ return info, stream
+
+
+async def validate_message_attachments(
+ session: AsyncSession,
+ user: User,
+ chat_id: UUID,
+ attachment_ids: list[UUID],
+) -> None:
+ await ensure_chat_member(session, user, chat_id)
+ records = await repository.get_for_chat(session, chat_id, attachment_ids)
+ if len(records) != len(attachment_ids):
+ raise not_found()
+ if any(record.status == "failed" for record in records):
+ raise conflict("Обработка вложения завершилась ошибкой")
+
+
+async def reconcile_attachments(
+ session: AsyncSession,
+ storage: MinioStorage,
+ *,
+ batch_size: int = 20,
+ dry_run: bool = True,
+) -> CleanupResult:
+ if not 1 <= batch_size <= 100:
+ raise invalid_request("batch_size должен быть от 1 до 100")
+ intents = await repository.stale_uploads(
+ session, storage.settings.cleanup_grace_seconds, batch_size
+ )
+ candidates = deleted = 0
+ for intent in intents:
+ if await repository.get_by_key(session, intent.object_key) is None:
+ candidates += 1
+ if not dry_run:
+ await storage.delete_object(intent.object_key)
+ deleted += 1
+ if not dry_run:
+ await repository.finish_upload(session, intent.id)
+ if dry_run:
+ await session.rollback()
+ else:
+ await session.commit()
+ return CleanupResult(len(intents), candidates, deleted)
diff --git a/app/domain/auth.py b/app/domain/auth.py
new file mode 100644
index 0000000..b05320a
--- /dev/null
+++ b/app/domain/auth.py
@@ -0,0 +1,81 @@
+import re
+from datetime import UTC, datetime, timedelta
+
+from redis.asyncio import Redis
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core import security
+from app.domain.users import to_user_schema
+from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
+from app.schemas.session import AuthSession
+from app.schemas.user import (
+ PASSWORD_MAX_LENGTH,
+ PASSWORD_MIN_LENGTH,
+ USERNAME_PATTERN,
+ User,
+)
+from app.storage import sessions as sessions_storage
+from app.storage import users as users_storage
+
+USERNAME_RE = re.compile(USERNAME_PATTERN)
+
+
+async def create_session(
+ session: AsyncSession,
+ redis: Redis,
+ username: str,
+ password: str,
+ ttl_seconds: int,
+) -> AuthSession:
+ if (
+ USERNAME_RE.fullmatch(username) is None
+ or not PASSWORD_MIN_LENGTH <= len(password) <= PASSWORD_MAX_LENGTH
+ or not password.strip()
+ ):
+ raise unauthorized_basic()
+
+ user = await users_storage.get_user_by_username(session, username)
+ await session.commit()
+
+ if user is None:
+ await security.dummy_verify_password(password)
+ raise unauthorized_basic()
+
+ if not await security.verify_password(password, user.password_hash):
+ raise unauthorized_basic()
+
+ token = security.generate_token()
+ expires_at = datetime.now(UTC) + timedelta(seconds=ttl_seconds)
+ await sessions_storage.create_session(
+ redis,
+ token_hash=security.hash_token(token),
+ user_id=user.id,
+ expires_at=expires_at,
+ )
+ auth_session = AuthSession(
+ token=token, expires_at=expires_at, user=to_user_schema(user)
+ )
+ return auth_session
+
+
+async def authenticate_token(session: AsyncSession, redis: Redis, token: str) -> User:
+ user_id = await sessions_storage.get_session_user_id(
+ redis, security.hash_token(token)
+ )
+ if user_id is None:
+ raise unauthorized_bearer()
+
+ user = await users_storage.get_user_by_id(session, user_id)
+ if user is None:
+ raise unauthorized_bearer()
+
+ current_user = to_user_schema(user)
+ return current_user
+
+
+async def revoke_session(redis: Redis, token: str) -> None:
+ is_deleted = await sessions_storage.delete_session(
+ redis, security.hash_token(token)
+ )
+ if not is_deleted:
+ raise unauthorized_bearer()
diff --git a/app/domain/chats.py b/app/domain/chats.py
new file mode 100644
index 0000000..ec884a5
--- /dev/null
+++ b/app/domain/chats.py
@@ -0,0 +1,98 @@
+from uuid import UUID
+
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.domain.pagination import (
+ KIND_CHATS,
+ build_next_cursor,
+ decode_cursor,
+ split_page,
+)
+from app.errors.exceptions import invalid_request, not_found
+from app.models.chat import Chat as ChatModel
+from app.schemas.chat import Chat, ChatPage, CreateChat
+from app.schemas.user import User
+from app.storage import chats as chats_storage
+from app.storage import users as users_storage
+from app.storage.types import Position
+
+UNKNOWN_MEMBER_MESSAGE = "member_ids содержит неизвестного пользователя"
+
+
+def to_chat_schema(chat: ChatModel) -> Chat:
+ chat_schema = Chat(
+ id=chat.id,
+ title=chat.title,
+ member_ids=[member.user_id for member in chat.members],
+ created_at=chat.created_at,
+ )
+ return chat_schema
+
+
+def validate_new_chat(creator: User, new_chat: CreateChat) -> None:
+ member_ids = new_chat.member_ids
+
+ if creator.id not in member_ids:
+ raise invalid_request("Создатель чата должен входить в member_ids")
+
+ if len(set(member_ids)) != len(member_ids):
+ raise invalid_request("member_ids содержит дубликаты")
+
+
+async def create_chat(
+ session: AsyncSession, creator: User, new_chat: CreateChat
+) -> Chat:
+ validate_new_chat(creator, new_chat)
+
+ existing_ids = await users_storage.get_existing_user_ids(
+ session, new_chat.member_ids
+ )
+ if existing_ids != set(new_chat.member_ids):
+ raise invalid_request(UNKNOWN_MEMBER_MESSAGE)
+
+ try:
+ chat = await chats_storage.create_chat(
+ session, title=new_chat.title, member_ids=new_chat.member_ids
+ )
+ await session.commit()
+ except IntegrityError:
+ await session.rollback()
+ raise invalid_request(UNKNOWN_MEMBER_MESSAGE) from None
+
+ created_chat = to_chat_schema(chat)
+ return created_chat
+
+
+async def get_chat(session: AsyncSession, user: User, chat_id: UUID) -> Chat:
+ chat = await chats_storage.get_chat_for_member(session, chat_id, user.id)
+ if chat is None:
+ raise not_found()
+
+ found_chat = to_chat_schema(chat)
+ return found_chat
+
+
+async def ensure_chat_member(session: AsyncSession, user: User, chat_id: UUID) -> None:
+ if not await chats_storage.is_chat_member(session, chat_id, user.id):
+ raise not_found()
+
+
+async def list_chats(
+ session: AsyncSession,
+ user: User,
+ cursor: str | None,
+ limit: int,
+) -> ChatPage:
+ after: Position | None = None
+ if cursor is not None:
+ after = decode_cursor(cursor, KIND_CHATS, user.id)
+
+ rows = await chats_storage.list_chats_for_user(session, user.id, after, limit + 1)
+ page_rows, has_next = split_page(rows, limit)
+ items = [to_chat_schema(chat) for chat in page_rows]
+ page = ChatPage(
+ items=items,
+ next_cursor=build_next_cursor(KIND_CHATS, user.id, items, has_next),
+ )
+ return page
diff --git a/app/domain/health.py b/app/domain/health.py
new file mode 100644
index 0000000..c5d6ac3
--- /dev/null
+++ b/app/domain/health.py
@@ -0,0 +1,36 @@
+"""Условия готовности реплики и ограниченная проверка зависимостей."""
+
+import asyncio
+
+from redis.asyncio import Redis
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from app.core.lifecycle import Lifecycle
+from app.errors.exceptions import unavailable
+from app.errors.handlers import operation_timeout
+from app.schemas.base import Health
+from app.storage.health import check_database, check_redis
+from app.storage.s3 import MinioStorage
+
+
+def liveness() -> Health:
+ return Health(status="ok")
+
+
+async def readiness(
+ engine: AsyncEngine,
+ redis: Redis,
+ storage: MinioStorage | None,
+ lifecycle: Lifecycle,
+ timeout_seconds: float,
+) -> Health:
+ if lifecycle.draining:
+ raise unavailable()
+ checks = [check_database(engine), check_redis(redis)]
+ if storage is not None:
+ checks.append(storage.check_available())
+ async with operation_timeout(timeout_seconds, unavailable()):
+ dependencies = await asyncio.gather(*checks, return_exceptions=True)
+ if not all(result is True for result in dependencies):
+ raise unavailable()
+ return Health(status="ok")
diff --git a/app/domain/idempotency.py b/app/domain/idempotency.py
new file mode 100644
index 0000000..fc1c7f4
--- /dev/null
+++ b/app/domain/idempotency.py
@@ -0,0 +1,75 @@
+"""Атомарная отправка сообщения с защитой от повторов."""
+
+import hashlib
+import json
+import re
+from uuid import UUID
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.domain.chats import ensure_chat_member
+from app.domain.attachments import validate_message_attachments
+from app.errors.exceptions import conflict, invalid_request, unavailable
+from app.schemas.message import CreateMessage, Message
+from app.schemas.user import User
+from app.storage import idempotency as key_storage
+from app.storage import messages as message_storage
+from app.storage import attachments as attachment_storage
+
+IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
+
+
+def validate_idempotency_key(key: str | None) -> str:
+ if key is None or IDEMPOTENCY_KEY_PATTERN.fullmatch(key) is None:
+ raise invalid_request("Требуется корректный Idempotency-Key")
+ return key
+
+
+def request_fingerprint(new_message: CreateMessage) -> str:
+ body = new_message.model_dump(mode="json")
+
+ if not body["attachment_ids"]:
+ body.pop("attachment_ids")
+ canonical = json.dumps(body, sort_keys=True, ensure_ascii=False)
+ return hashlib.sha256(canonical.encode()).hexdigest()
+
+
+async def send_message_idempotent(
+ session: AsyncSession,
+ sender: User,
+ chat_id: UUID,
+ new_message: CreateMessage,
+ idempotency_key: str,
+ max_messages: int,
+ ttl_seconds: int,
+) -> Message:
+ await ensure_chat_member(session, sender, chat_id)
+ scope = (sender.id, "POST", f"/api/v1/chats/{chat_id}/messages", idempotency_key)
+ fingerprint = request_fingerprint(new_message)
+
+ await key_storage.lock_key(session, scope)
+ previous = await key_storage.get_active_key(session, scope, ttl_seconds)
+ if previous is not None:
+ if previous.request_hash != fingerprint:
+ raise conflict("Idempotency-Key уже использован для другого сообщения")
+ message = await message_storage.get_message(session, previous.message_id)
+ else:
+ await validate_message_attachments(
+ session, sender, chat_id, new_message.attachment_ids
+ )
+ if not await message_storage.reserve_message_slot(session, max_messages):
+ raise unavailable("Достигнут лимит количества сообщений")
+ message = await message_storage.create_message(
+ session, chat_id=chat_id, sender_id=sender.id, text=new_message.text
+ )
+ await attachment_storage.bind_attachments(
+ session, message.id, new_message.attachment_ids
+ )
+ await key_storage.save_result(session, scope, fingerprint, message.id)
+
+ links = await attachment_storage.message_attachment_ids(session, [message.id])
+ result = Message.model_validate(message).model_copy(
+ update={"attachment_ids": links[message.id]}
+ )
+ await session.commit()
+ return result
diff --git a/app/domain/image_pipeline.py b/app/domain/image_pipeline.py
new file mode 100644
index 0000000..dee38e6
--- /dev/null
+++ b/app/domain/image_pipeline.py
@@ -0,0 +1,101 @@
+"""Thumbnail, crop и WebP; один ограниченный процесс, последовательная обработка."""
+
+import hashlib
+from pathlib import Path
+
+from PIL import Image, ImageOps
+
+from app.core.image_processing import ImageSettings
+from app.domain.image_validation import decode_image
+from app.schemas.attachment import PipelineResult, Variant
+
+
+def fitted_size(size: tuple[int, int], maximum: int) -> tuple[int, int]:
+ longest = max(size)
+ result = size
+ if longest > maximum:
+ result = (
+ max(1, size[0] * maximum // longest),
+ max(1, size[1] * maximum // longest),
+ )
+ return result
+
+
+def describe_variant(
+ name: str, content_type: str, path: Path, size: tuple[int, int]
+) -> Variant:
+ checksum = hashlib.sha256()
+ with path.open("rb") as source:
+ for chunk in iter(lambda: source.read(65536), b""):
+ checksum.update(chunk)
+ return Variant(
+ name=name,
+ content_type=content_type,
+ size_bytes=path.stat().st_size,
+ width=size[0],
+ height=size[1],
+ sha256=checksum.hexdigest(),
+ )
+
+
+def save_jpeg(image: Image.Image, destination: Path, quality: int) -> None:
+ with Image.new("RGB", image.size, "white") as clean:
+ with image.getchannel("A") as alpha:
+ clean.paste(image, mask=alpha)
+ clean.save(destination, format="JPEG", quality=quality, exif=b"")
+
+
+def create_thumbnail(image: Image.Image, destination: Path, quality: int) -> Variant:
+ size = fitted_size(image.size, 256)
+ with image.resize(size, Image.Resampling.LANCZOS) as resized:
+ save_jpeg(resized, destination, quality)
+ return describe_variant("thumbnail", "image/jpeg", destination, size)
+
+
+def create_crop(image: Image.Image, destination: Path, quality: int) -> Variant:
+ with ImageOps.fit(
+ image, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)
+ ) as crop:
+ save_jpeg(crop, destination, quality)
+ return describe_variant("crop", "image/jpeg", destination, (128, 128))
+
+
+def create_optimized(
+ image: Image.Image, destination: Path, settings: ImageSettings
+) -> Variant:
+ size = fitted_size(image.size, 1280)
+ with image.resize(size, Image.Resampling.LANCZOS) as resized:
+ with Image.new("RGBA", size) as clean:
+ clean.paste(resized)
+ clean.save(
+ destination,
+ format="WEBP",
+ lossless=False,
+ quality=settings.webp_quality,
+ method=settings.webp_method,
+ exif=b"",
+ icc_profile=b"",
+ xmp=b"",
+ )
+ return describe_variant("optimized", "image/webp", destination, size)
+
+
+def create_variants(
+ source: Path,
+ directory: Path,
+ content_type: str,
+ max_pixels: int,
+ settings: ImageSettings,
+) -> PipelineResult:
+ decode_image(source, content_type, max_pixels)
+ with Image.open(source, formats=["PNG", "JPEG"]) as image:
+ ImageOps.exif_transpose(image, in_place=True)
+ with image.convert("RGBA") as pixels:
+ variants = [
+ create_thumbnail(
+ pixels, directory / "thumbnail.jpg", settings.jpeg_quality
+ ),
+ create_crop(pixels, directory / "crop.jpg", settings.jpeg_quality),
+ create_optimized(pixels, directory / "optimized.webp", settings),
+ ]
+ return PipelineResult(variants=variants)
diff --git a/app/domain/image_validation.py b/app/domain/image_validation.py
new file mode 100644
index 0000000..eeec06a
--- /dev/null
+++ b/app/domain/image_validation.py
@@ -0,0 +1,23 @@
+"""Синхронная проверка изображения; вызывается только из ограниченного процесса."""
+
+from pathlib import Path
+import warnings
+
+from PIL import Image
+
+from app.errors.exceptions import payload_too_large, unsupported_media_type
+
+
+def decode_image(path: Path, content_type: str, max_pixels: int) -> None:
+ Image.MAX_IMAGE_PIXELS = max_pixels
+ warnings.simplefilter("error", Image.DecompressionBombWarning)
+ with Image.open(path, formats=["PNG", "JPEG"]) as image:
+ if image.width * image.height > max_pixels:
+ raise payload_too_large("Превышен лимит пикселей изображения")
+ image_format = image.format
+ if image_format is None or Image.MIME.get(image_format) != content_type:
+ raise unsupported_media_type("Содержимое не соответствует Content-Type")
+ image.verify()
+
+ with Image.open(path, formats=["PNG", "JPEG"]) as image:
+ image.load()
diff --git a/app/domain/images.py b/app/domain/images.py
new file mode 100644
index 0000000..4436656
--- /dev/null
+++ b/app/domain/images.py
@@ -0,0 +1,48 @@
+"""Проверка имени и запуск ограниченного декодера."""
+
+import asyncio
+from pathlib import Path
+import sys
+import unicodedata
+
+from anyio import CancelScope
+
+from app.core.s3 import S3Settings
+from app.errors.exceptions import (
+ image_validation_error,
+ invalid_request,
+ payload_too_large,
+)
+from app.errors.handlers import operation_timeout
+
+
+def validate_filename(filename: str) -> None:
+ if not 1 <= len(filename) <= 255 or any(
+ char in "/\\" or unicodedata.category(char) in {"Cc", "Cf", "Cs"}
+ for char in filename
+ ):
+ raise invalid_request("Недопустимое имя файла")
+
+
+async def validate_image(path: Path, content_type: str, settings: S3Settings) -> None:
+ process = await asyncio.create_subprocess_exec(
+ sys.executable,
+ "-m",
+ "app.core.image_worker",
+ str(path),
+ content_type,
+ str(settings.max_image_pixels),
+ stdout=asyncio.subprocess.DEVNULL,
+ stderr=asyncio.subprocess.DEVNULL,
+ )
+ error = payload_too_large("Превышен бюджет проверки изображения")
+ try:
+ async with operation_timeout(settings.image_timeout_seconds, error):
+ code = await process.wait()
+ finally:
+ if process.returncode is None:
+ process.kill()
+ with CancelScope(shield=True):
+ await process.wait()
+ if code != 0:
+ raise image_validation_error(code)
diff --git a/app/domain/job_admin.py b/app/domain/job_admin.py
new file mode 100644
index 0000000..15a9b35
--- /dev/null
+++ b/app/domain/job_admin.py
@@ -0,0 +1,157 @@
+"""Replay только failed с проверенным исходником и ограниченным новым бюджетом."""
+
+import asyncio
+import re
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from uuid import uuid4
+
+from minio.error import S3Error
+
+from app.domain.job_files import fetch_original
+from app.errors.exceptions import JobOperationError, ProcessingError
+from app.storage import job_admin, jobs
+from app.storage.jobs import JobWork
+
+REPLAY_REASONS = ("dependency_restored", "configuration_fixed", "manual_review")
+
+
+def snapshot(job, attachment):
+ return dict(
+ job_id=str(job.id),
+ attachment_id=str(attachment.id),
+ status=attachment.status,
+ attempt=job.attempt,
+ attempt_limit=job.attempt_limit,
+ cycle_start_attempt=job.cycle_start_attempt,
+ error_code=attachment.error_code,
+ last_error_code=job.last_error_code,
+ last_error_permanent=job.last_error_permanent,
+ staging_retained=job.staging_key is not None,
+ next_attempt_at=job.next_attempt_at,
+ lease_until=job.lease_until,
+ finished_at=job.finished_at,
+ )
+
+
+async def list_failed(session, limit):
+ return [
+ snapshot(job, attachment)
+ for job, attachment in await job_admin.failed_jobs(session, limit)
+ ]
+
+
+async def show_job(session, job_id):
+ row = await jobs.lock_job(session, job_id)
+ if row is None:
+ raise JobOperationError("unknown_job")
+ result = snapshot(*row)
+ await session.rollback()
+ return result
+
+
+async def history(session, job_id):
+ return [
+ dict(
+ request_id=str(row.id),
+ actor=row.actor,
+ reason=row.reason,
+ previous_attempt=row.previous_attempt,
+ attempt_limit=row.attempt_limit,
+ previous_error_code=row.previous_error_code,
+ created_at=row.created_at,
+ )
+ for row in await job_admin.replay_history(session, job_id)
+ ]
+
+
+def same_request(record, job_id, actor, reason):
+ if record.job_id != job_id or record.actor != actor or record.reason != reason:
+ raise JobOperationError("request_conflict")
+ return {
+ "request_id": str(record.id),
+ "job_id": str(job_id),
+ "already_applied": True,
+ }
+
+
+def validate_replay_request(actor: str, reason: str) -> None:
+ if (
+ re.fullmatch(r"[A-Za-z0-9._-]{1,64}", actor) is None
+ or reason not in REPLAY_REASONS
+ ):
+ raise JobOperationError("invalid_audit_fields")
+
+
+def validate_replay_job(job, attachment) -> None:
+ if attachment.status != "failed":
+ raise JobOperationError("job_not_failed")
+ if job.staging_key is None:
+ raise JobOperationError("source_expired")
+
+
+async def verify_replay_source(storage, settings, job, attachment) -> None:
+ work = JobWork(
+ job.id,
+ attachment.id,
+ uuid4(),
+ job.pipeline_version,
+ job.staging_key,
+ attachment.content_type,
+ attachment.size_bytes,
+ attachment.sha256,
+ )
+ try:
+ async with asyncio.timeout(settings.job_timeout_seconds):
+ with TemporaryDirectory(prefix="replay-source-") as directory:
+ await fetch_original(storage, work, Path(directory) / "original")
+ except ProcessingError as error:
+ raise JobOperationError(error.code) from None
+ except S3Error as error:
+ if error.code in {"NoSuchKey", "NoSuchObject"}:
+ raise JobOperationError("source_missing") from None
+ raise
+
+
+async def replay_failed_job(
+ session, storage, settings, job_id, *, request_id, actor, reason
+):
+ validate_replay_request(actor, reason)
+ row = await jobs.lock_job(session, job_id)
+ if row is None:
+ raise JobOperationError("unknown_job")
+ record = await job_admin.replay_record(session, request_id)
+ if record is not None:
+ result = same_request(record, job_id, actor, reason)
+ await session.rollback()
+ return result
+ job, attachment = row
+ validate_replay_job(job, attachment)
+
+ await verify_replay_source(storage, settings, job, attachment)
+ limit = job.attempt + settings.job_max_attempts
+ inserted = await job_admin.insert_replay(
+ session,
+ id=request_id,
+ job_id=job_id,
+ actor=actor,
+ reason=reason,
+ previous_attempt=job.attempt,
+ attempt_limit=limit,
+ previous_error_code=job.last_error_code,
+ )
+ if not inserted:
+ result = same_request(
+ await job_admin.replay_record(session, request_id), job_id, actor, reason
+ )
+ await session.rollback()
+ return result
+ job_admin.reset_failed(job, attachment, await jobs.database_time(session), limit)
+ await session.commit()
+ return {
+ "request_id": str(request_id),
+ "job_id": str(job_id),
+ "already_applied": False,
+ "status": "queued",
+ "attempt_limit": limit,
+ }
diff --git a/app/domain/job_files.py b/app/domain/job_files.py
new file mode 100644
index 0000000..b99bda6
--- /dev/null
+++ b/app/domain/job_files.py
@@ -0,0 +1,72 @@
+"""Получение исходника и запуск изолированного процесса обработки вариантов."""
+
+import asyncio
+import errno
+import hashlib
+from pathlib import Path
+import sys
+
+from anyio import CancelScope
+
+from app.errors.exceptions import ProcessingError
+from app.schemas.attachment import PipelineResult
+from app.storage.jobs import JobWork
+from app.storage.s3 import MinioStorage
+
+
+async def fetch_original(
+ storage: MinioStorage, work: JobWork, destination: Path
+) -> None:
+ if work.size_bytes > storage.settings.max_upload_bytes:
+ raise ProcessingError("image_limits")
+ checksum = hashlib.sha256()
+ size = 0
+ stream = await storage.open_object(work.staging_key)
+ try:
+ with destination.open("wb") as target:
+ async for chunk in stream:
+ size += len(chunk)
+ if size > work.size_bytes:
+ raise ProcessingError("source_mismatch")
+ target.write(chunk)
+ checksum.update(chunk)
+ finally:
+ await stream.aclose()
+ if size != work.size_bytes or checksum.hexdigest() != work.sha256:
+ raise ProcessingError("source_mismatch")
+
+
+async def render_variants(
+ storage: MinioStorage, work: JobWork, source: Path, directory: Path
+) -> PipelineResult:
+ metadata = directory / "variants.json"
+ process = await asyncio.create_subprocess_exec(
+ sys.executable,
+ "-m",
+ "app.core.thumbnail_worker",
+ str(source),
+ str(directory),
+ str(metadata),
+ work.content_type,
+ str(storage.settings.max_image_pixels),
+ stdout=asyncio.subprocess.DEVNULL,
+ stderr=asyncio.subprocess.DEVNULL,
+ )
+ try:
+ code = await process.wait()
+ finally:
+ if process.returncode is None:
+ process.kill()
+ with CancelScope(shield=True):
+ await process.wait()
+ if code == 2:
+ raise ProcessingError("invalid_image")
+ if code == 3 or code < 0:
+ raise ProcessingError("image_limits")
+ if code == 4:
+ raise OSError(errno.EIO, "Pipeline filesystem unavailable")
+ if code == 5:
+ raise ProcessingError("processing_configuration")
+ if code != 0:
+ raise RuntimeError("Image pipeline subprocess failed")
+ return PipelineResult.model_validate_json(metadata.read_text())
diff --git a/app/domain/job_retry.py b/app/domain/job_retry.py
new file mode 100644
index 0000000..f6a6790
--- /dev/null
+++ b/app/domain/job_retry.py
@@ -0,0 +1,51 @@
+"""Ограниченный exponential backoff: jitter сохраняется вместе с новой попыткой."""
+
+import errno
+import random
+
+from minio.error import S3Error
+
+from app.core.jobs import JobSettings
+from app.errors.exceptions import ProcessingError
+
+
+def retry_delay(settings: JobSettings, cycle_attempt: int) -> float:
+ ceiling = min(
+ settings.job_retry_max_seconds,
+ settings.job_retry_seconds * 2 ** min(30, max(0, cycle_attempt - 1)),
+ )
+ return random.uniform(ceiling / 2, ceiling)
+
+
+def classify_failure(error: Exception) -> tuple[str, bool]:
+ if isinstance(error, ProcessingError):
+ return error.code, True
+ if isinstance(error, S3Error):
+ if error.code in {"NoSuchKey", "NoSuchObject"}:
+ return "source_missing", True
+ if error.code in {
+ "AccessDenied",
+ "InvalidAccessKeyId",
+ "SignatureDoesNotMatch",
+ "NoSuchBucket",
+ }:
+ return "storage_configuration", True
+ if error.code in {
+ "InternalError",
+ "SlowDown",
+ "ServiceUnavailable",
+ "RequestTimeout",
+ "Throttling",
+ }:
+ return "storage_unavailable", False
+ return "storage_rejected", True
+ if isinstance(error, TimeoutError):
+ return "processing_timeout", False
+ if isinstance(error, OSError) and error.errno in {
+ errno.EACCES,
+ errno.EPERM,
+ errno.EROFS,
+ errno.ENOENT,
+ }:
+ return "processing_configuration", True
+ return "processing_unavailable", False
diff --git a/app/domain/messages.py b/app/domain/messages.py
new file mode 100644
index 0000000..35b13aa
--- /dev/null
+++ b/app/domain/messages.py
@@ -0,0 +1,47 @@
+from uuid import UUID
+
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.domain.chats import ensure_chat_member
+from app.domain.pagination import (
+ KIND_MESSAGES,
+ build_next_cursor,
+ decode_cursor,
+ split_page,
+)
+from app.schemas.message import Message, MessagePage
+from app.schemas.user import User
+from app.storage import messages as messages_storage
+from app.storage import attachments as attachments_storage
+from app.storage.types import Position
+
+
+async def list_messages(
+ session: AsyncSession,
+ user: User,
+ chat_id: UUID,
+ cursor: str | None,
+ limit: int,
+) -> MessagePage:
+ after: Position | None = None
+ if cursor is not None:
+ after = decode_cursor(cursor, KIND_MESSAGES, chat_id)
+
+ await ensure_chat_member(session, user, chat_id)
+
+ rows = await messages_storage.list_messages(session, chat_id, after, limit + 1)
+ page_rows, has_next = split_page(rows, limit)
+ links = await attachments_storage.message_attachment_ids(
+ session, [message.id for message in page_rows]
+ )
+ items = [
+ Message.model_validate(message).model_copy(
+ update={"attachment_ids": links[message.id]},
+ )
+ for message in page_rows
+ ]
+ page = MessagePage(
+ items=items,
+ next_cursor=build_next_cursor(KIND_MESSAGES, chat_id, items, has_next),
+ )
+ return page
diff --git a/app/domain/pagination.py b/app/domain/pagination.py
new file mode 100644
index 0000000..6c0fc3f
--- /dev/null
+++ b/app/domain/pagination.py
@@ -0,0 +1,78 @@
+import base64
+import json
+from collections.abc import Sequence
+from datetime import datetime
+from typing import Protocol, TypeVar
+from uuid import UUID
+
+from app.errors.exceptions import invalid_request
+from app.storage.types import Position
+
+KIND_CHATS = "chats"
+KIND_MESSAGES = "messages"
+
+T = TypeVar("T")
+
+
+class Positioned(Protocol):
+
+ @property
+ def id(self) -> UUID: ...
+
+ @property
+ def created_at(self) -> datetime: ...
+
+
+def encode_cursor(
+ kind: str,
+ resource_id: UUID,
+ created_at: datetime,
+ item_id: UUID,
+) -> str:
+ payload = {
+ "kind": kind,
+ "resource_id": str(resource_id),
+ "created_at": created_at.isoformat(),
+ "item_id": str(item_id),
+ }
+ raw = json.dumps(payload).encode()
+ return base64.urlsafe_b64encode(raw).decode()
+
+
+def decode_cursor(cursor: str, kind: str, resource_id: UUID) -> Position:
+ try:
+ payload = json.loads(base64.urlsafe_b64decode(cursor))
+ cursor_kind = payload["kind"]
+ cursor_resource_id = UUID(payload["resource_id"])
+ created_at = datetime.fromisoformat(payload["created_at"])
+ item_id = UUID(payload["item_id"])
+ except (ValueError, KeyError, TypeError, AttributeError):
+ raise invalid_request("Некорректный cursor") from None
+
+ if (
+ cursor_kind != kind
+ or cursor_resource_id != resource_id
+ or created_at.tzinfo is None
+ ):
+ raise invalid_request("Некорректный cursor")
+
+ return created_at, item_id
+
+
+def split_page(rows: list[T], limit: int) -> tuple[list[T], bool]:
+ has_next = len(rows) > limit
+ items = rows[:limit]
+ return items, has_next
+
+
+def build_next_cursor(
+ kind: str,
+ resource_id: UUID,
+ items: Sequence[Positioned],
+ has_next: bool,
+) -> str | None:
+ if not has_next:
+ return None
+
+ last_item = items[-1]
+ return encode_cursor(kind, resource_id, last_item.created_at, last_item.id)
diff --git a/app/domain/users.py b/app/domain/users.py
new file mode 100644
index 0000000..ec4f97a
--- /dev/null
+++ b/app/domain/users.py
@@ -0,0 +1,39 @@
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core import security
+from app.errors.exceptions import conflict
+from app.models.user import User as UserModel
+from app.schemas.user import CreateUser, User
+from app.storage import users as users_storage
+
+USERNAME_TAKEN_MESSAGE = "Имя пользователя уже занято"
+
+
+def to_user_schema(user: UserModel) -> User:
+ user_schema = User.model_validate(user)
+ return user_schema
+
+
+async def register_user(session: AsyncSession, new_user: CreateUser) -> User:
+ existing_user = await users_storage.get_user_by_username(session, new_user.username)
+ await session.commit()
+ if existing_user is not None:
+ raise conflict(USERNAME_TAKEN_MESSAGE)
+
+ password_hash = await security.hash_password(new_user.password)
+
+ try:
+ user = await users_storage.create_user(
+ session,
+ username=new_user.username,
+ display_name=new_user.display_name,
+ password_hash=password_hash,
+ )
+ await session.commit()
+ except IntegrityError:
+ await session.rollback()
+ raise conflict(USERNAME_TAKEN_MESSAGE) from None
+
+ created_user = to_user_schema(user)
+ return created_user
diff --git a/app/endpoints/__init__.py b/app/endpoints/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/app/endpoints/attachments.py b/app/endpoints/attachments.py
new file mode 100644
index 0000000..ddeddfa
--- /dev/null
+++ b/app/endpoints/attachments.py
@@ -0,0 +1,74 @@
+"""HTTP-параметры и упаковка ответов; сценарии находятся в domain."""
+
+from typing import Annotated
+from urllib.parse import quote
+from uuid import UUID
+
+from fastapi import APIRouter, Query, Request
+from fastapi.responses import StreamingResponse
+from starlette.background import BackgroundTask
+
+from app.domain import attachments
+from app.endpoints.dependencies import CurrentUser, DbSession, S3Storage, UploadCapacity
+from app.schemas.attachment import Attachment
+
+router = APIRouter(prefix="/api/v1", tags=["attachments"])
+
+
+@router.post("/chats/{chat_id}/attachments", status_code=202)
+async def upload(
+ request: Request,
+ chat_id: UUID,
+ user: CurrentUser,
+ session: DbSession,
+ storage: S3Storage,
+ capacity: UploadCapacity,
+ filename: Annotated[str, Query(min_length=1, max_length=255)],
+) -> Attachment:
+ return await attachments.receive_attachment(
+ session,
+ storage,
+ user,
+ chat_id,
+ capacity=capacity,
+ filename=filename,
+ content_type=request.headers.get("content-type", "").lower(),
+ declared_length=request.headers.get("content-length"),
+ chunks=request.stream(),
+ )
+
+
+@router.get("/attachments/{attachment_id}")
+async def metadata(
+ attachment_id: UUID, user: CurrentUser, session: DbSession
+) -> Attachment:
+ return await attachments.get_attachment(session, user, attachment_id)
+
+
+@router.get("/attachments/{attachment_id}/content")
+async def content(
+ attachment_id: UUID,
+ user: CurrentUser,
+ session: DbSession,
+ storage: S3Storage,
+ variant: Annotated[str, Query()] = "original",
+) -> StreamingResponse:
+ record, stream = await attachments.download_attachment(
+ session,
+ storage,
+ user,
+ attachment_id,
+ variant=variant,
+ )
+ return StreamingResponse(
+ stream,
+ media_type=record.content_type,
+ headers={
+ "Content-Length": str(record.size_bytes),
+ "Content-Disposition": "attachment; filename*=UTF-8''"
+ + quote(record.filename, safe=""),
+ "X-Content-Type-Options": "nosniff",
+ "Cache-Control": "private, no-store",
+ },
+ background=BackgroundTask(stream.aclose),
+ )
diff --git a/app/endpoints/auth.py b/app/endpoints/auth.py
new file mode 100644
index 0000000..69f4535
--- /dev/null
+++ b/app/endpoints/auth.py
@@ -0,0 +1,32 @@
+from fastapi import APIRouter, Response
+
+from app.domain.auth import create_session, revoke_session
+from app.endpoints.dependencies import (
+ AppSettings,
+ BasicCredentials,
+ BearerToken,
+ DbSession,
+ RedisClient,
+)
+from app.schemas.session import AuthSession
+
+router = APIRouter(prefix="/api/v1/auth", tags=["auth"])
+
+
+@router.post("/sessions", status_code=201)
+async def login(
+ credentials: BasicCredentials,
+ session: DbSession,
+ redis: RedisClient,
+ settings: AppSettings,
+) -> AuthSession:
+ username, password = credentials
+ return await create_session(
+ session, redis, username, password, settings.session_ttl_seconds
+ )
+
+
+@router.delete("/sessions/current", status_code=204)
+async def logout(token: BearerToken, redis: RedisClient) -> Response:
+ await revoke_session(redis, token)
+ return Response(status_code=204)
diff --git a/app/endpoints/chats.py b/app/endpoints/chats.py
new file mode 100644
index 0000000..91f2b21
--- /dev/null
+++ b/app/endpoints/chats.py
@@ -0,0 +1,32 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Query
+
+from app.domain.chats import create_chat, get_chat, list_chats
+from app.endpoints.dependencies import CurrentUser, DbSession
+from app.schemas.chat import Chat, ChatPage, CreateChat
+
+router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
+
+
+@router.post("", status_code=201)
+async def create_new_chat(
+ new_chat: CreateChat, creator: CurrentUser, session: DbSession
+) -> Chat:
+ return await create_chat(session, creator, new_chat)
+
+
+@router.get("")
+async def list_user_chats(
+ user: CurrentUser,
+ session: DbSession,
+ limit: Annotated[int, Query(ge=1, le=100)] = 50,
+ cursor: Annotated[str | None, Query(min_length=1)] = None,
+) -> ChatPage:
+ return await list_chats(session, user, cursor, limit)
+
+
+@router.get("/{chat_id}")
+async def get_chat_by_id(chat_id: UUID, user: CurrentUser, session: DbSession) -> Chat:
+ return await get_chat(session, user, chat_id)
diff --git a/app/endpoints/dependencies.py b/app/endpoints/dependencies.py
new file mode 100644
index 0000000..85a5bc2
--- /dev/null
+++ b/app/endpoints/dependencies.py
@@ -0,0 +1,109 @@
+import base64
+import binascii
+from collections.abc import AsyncIterator
+from typing import Annotated
+
+from fastapi import Depends, Request, Security
+from fastapi.openapi.models import HTTPBase, HTTPBearer
+from fastapi.security.base import SecurityBase
+from redis.asyncio import Redis
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.config import Settings
+from app.core.request_capacity import RequestCapacity
+from app.domain.auth import authenticate_token
+from app.errors.exceptions import unauthorized_basic, unauthorized_bearer
+from app.schemas.user import User
+from app.storage.s3 import MinioStorage
+
+
+async def get_settings(request: Request) -> Settings:
+ settings: Settings = request.app.state.settings
+ return settings
+
+
+async def get_db(request: Request) -> AsyncIterator[AsyncSession]:
+ session_factory = request.app.state.session_factory
+ async with session_factory() as session:
+ yield session
+
+
+async def get_redis(request: Request) -> Redis:
+ redis: Redis = request.app.state.redis
+ return redis
+
+
+def get_s3(request: Request) -> MinioStorage | None:
+ return request.app.state.s3
+
+
+def get_upload_capacity(request: Request) -> RequestCapacity:
+ return request.app.state.upload_capacity
+
+
+class AuthorizationHeader(SecurityBase):
+
+ def __init__(self, model: HTTPBase, scheme_name: str) -> None:
+ self.model = model
+ self.scheme_name = scheme_name
+
+ async def __call__(self, request: Request) -> str | None:
+ return request.headers.get("Authorization")
+
+
+basic_auth = AuthorizationHeader(HTTPBase(scheme="basic"), "BasicAuth")
+bearer_auth = AuthorizationHeader(HTTPBearer(), "BearerAuth")
+
+
+def _split_authorization(authorization: str | None) -> tuple[str, str] | None:
+ if authorization is None:
+ return None
+ scheme, _, value = authorization.partition(" ")
+ if not value.strip():
+ return None
+ return scheme.lower(), value.strip()
+
+
+async def get_bearer_token(
+ authorization: Annotated[str | None, Security(bearer_auth)],
+) -> str:
+ parts = _split_authorization(authorization)
+ if parts is None or parts[0] != "bearer":
+ raise unauthorized_bearer()
+ return parts[1]
+
+
+async def get_basic_credentials(
+ authorization: Annotated[str | None, Security(basic_auth)],
+) -> tuple[str, str]:
+ parts = _split_authorization(authorization)
+ if parts is None or parts[0] != "basic":
+ raise unauthorized_basic()
+
+ try:
+ decoded = base64.b64decode(parts[1], validate=True).decode("utf-8")
+ except (binascii.Error, UnicodeDecodeError, ValueError):
+ raise unauthorized_basic() from None
+
+ username, separator, password = decoded.partition(":")
+ if not separator or not username:
+ raise unauthorized_basic()
+ return username, password
+
+
+DbSession = Annotated[AsyncSession, Depends(get_db)]
+RedisClient = Annotated[Redis, Depends(get_redis)]
+AppSettings = Annotated[Settings, Depends(get_settings)]
+S3Storage = Annotated[MinioStorage | None, Depends(get_s3)]
+UploadCapacity = Annotated[RequestCapacity, Depends(get_upload_capacity)]
+BearerToken = Annotated[str, Depends(get_bearer_token)]
+BasicCredentials = Annotated[tuple[str, str], Depends(get_basic_credentials)]
+
+
+async def get_current_user(
+ token: BearerToken, session: DbSession, redis: RedisClient
+) -> User:
+ return await authenticate_token(session, redis, token)
+
+
+CurrentUser = Annotated[User, Depends(get_current_user)]
diff --git a/app/endpoints/health.py b/app/endpoints/health.py
new file mode 100644
index 0000000..2be984b
--- /dev/null
+++ b/app/endpoints/health.py
@@ -0,0 +1,23 @@
+from fastapi import APIRouter, Request
+
+from app.domain import health
+from app.schemas.base import Health
+
+router = APIRouter(prefix="/health", tags=["health"])
+
+
+@router.get("/live")
+async def liveness() -> Health:
+ return health.liveness()
+
+
+@router.get("/ready")
+async def readiness(request: Request) -> Health:
+ state = request.app.state
+ return await health.readiness(
+ state.engine,
+ state.redis,
+ state.s3,
+ state.lifecycle,
+ state.settings.readiness_timeout_seconds,
+ )
diff --git a/app/endpoints/messages.py b/app/endpoints/messages.py
new file mode 100644
index 0000000..4bfbd3f
--- /dev/null
+++ b/app/endpoints/messages.py
@@ -0,0 +1,42 @@
+from typing import Annotated
+from uuid import UUID
+
+from fastapi import APIRouter, Header, Query
+
+from app.domain.idempotency import send_message_idempotent, validate_idempotency_key
+from app.domain.messages import list_messages
+from app.endpoints.dependencies import AppSettings, CurrentUser, DbSession
+from app.schemas.message import CreateMessage, Message, MessagePage
+
+router = APIRouter(prefix="/api/v1/chats", tags=["messages"])
+
+
+@router.post("/{chat_id}/messages", status_code=201)
+async def send_new_message(
+ chat_id: UUID,
+ new_message: CreateMessage,
+ sender: CurrentUser,
+ session: DbSession,
+ settings: AppSettings,
+ idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None,
+) -> Message:
+ return await send_message_idempotent(
+ session,
+ sender,
+ chat_id,
+ new_message,
+ validate_idempotency_key(idempotency_key),
+ settings.max_messages,
+ settings.idempotency_ttl_seconds,
+ )
+
+
+@router.get("/{chat_id}/messages")
+async def get_history_messages(
+ chat_id: UUID,
+ user: CurrentUser,
+ session: DbSession,
+ limit: Annotated[int, Query(ge=1, le=100)] = 50,
+ cursor: Annotated[str | None, Query(min_length=1)] = None,
+) -> MessagePage:
+ return await list_messages(session, user, chat_id, cursor, limit)
diff --git a/app/endpoints/users.py b/app/endpoints/users.py
new file mode 100644
index 0000000..c502d01
--- /dev/null
+++ b/app/endpoints/users.py
@@ -0,0 +1,17 @@
+from fastapi import APIRouter
+
+from app.domain.users import register_user
+from app.endpoints.dependencies import CurrentUser, DbSession
+from app.schemas.user import CreateUser, User
+
+router = APIRouter(prefix="/api/v1/users", tags=["users"])
+
+
+@router.post("", status_code=201)
+async def create_user(new_user: CreateUser, session: DbSession) -> User:
+ return await register_user(session, new_user)
+
+
+@router.get("/me")
+async def get_me(user: CurrentUser) -> User:
+ return user
diff --git a/app/errors/__init__.py b/app/errors/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/app/errors/exceptions.py b/app/errors/exceptions.py
new file mode 100644
index 0000000..2189707
--- /dev/null
+++ b/app/errors/exceptions.py
@@ -0,0 +1,131 @@
+import errno
+
+from collections.abc import Mapping
+
+from PIL import Image, UnidentifiedImageError
+
+BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
+BEARER_CHALLENGE = 'Bearer realm="messenger"'
+
+
+class ApiError(Exception):
+ def __init__(
+ self,
+ status_code: int,
+ code: str,
+ message: str,
+ headers: Mapping[str, str] | None = None,
+ ) -> None:
+ super().__init__(message)
+ self.status_code = status_code
+ self.code = code
+ self.message = message
+ self.headers = headers
+
+
+def invalid_request(message: str = "Неверный запрос") -> ApiError:
+ return ApiError(400, "invalid_request", message)
+
+
+def unauthorized(
+ message: str = "Нет действительной аутентификации",
+ www_authenticate: str | None = None,
+) -> ApiError:
+ headers = {"WWW-Authenticate": www_authenticate} if www_authenticate else None
+ return ApiError(401, "unauthorized", message, headers)
+
+
+def unauthorized_basic(message: str = "Неверные учётные данные") -> ApiError:
+ return unauthorized(message, BASIC_CHALLENGE)
+
+
+def unauthorized_bearer(message: str = "Нет действительной сессии") -> ApiError:
+ return unauthorized(message, BEARER_CHALLENGE)
+
+
+def not_found(message: str = "Объект не найден") -> ApiError:
+ return ApiError(404, "not_found", message)
+
+
+def conflict(message: str = "Конфликт") -> ApiError:
+ return ApiError(409, "conflict", message)
+
+
+def payload_too_large(message: str = "Слишком большое тело запроса") -> ApiError:
+ return ApiError(413, "payload_too_large", message)
+
+
+def unsupported_media_type(
+ message: str = "Требуется Content-Type: application/json",
+) -> ApiError:
+ return ApiError(415, "unsupported_media_type", message)
+
+
+def rate_limited(retry_after: int = 1) -> ApiError:
+ return ApiError(
+ 429,
+ "rate_limited",
+ "Слишком много запросов",
+ {"Retry-After": str(max(1, retry_after))},
+ )
+
+
+def unavailable(message: str = "Сервис временно недоступен") -> ApiError:
+ return ApiError(503, "unavailable", message)
+
+
+def internal_error() -> ApiError:
+ return ApiError(500, "internal_error", "Внутренняя ошибка сервера")
+
+
+IMAGE_WORKER_ERRORS = (
+ ApiError,
+ Image.DecompressionBombError,
+ Image.DecompressionBombWarning,
+ MemoryError,
+ UnidentifiedImageError,
+ OSError,
+ ValueError,
+ SyntaxError,
+)
+
+
+def image_worker_error_code(error: Exception) -> int:
+ code = 2
+ resource_error = isinstance(
+ error,
+ (Image.DecompressionBombError, Image.DecompressionBombWarning, MemoryError),
+ )
+ declared_limit = isinstance(error, ApiError) and error.status_code == 413
+ if resource_error or declared_limit:
+ code = 3
+ return code
+
+
+def image_validation_error(code: int) -> ApiError:
+ error = unsupported_media_type("Повреждённое изображение или несовпадение MIME")
+ if code == 3 or code < 0:
+ error = payload_too_large("Превышен лимит пикселей или ресурсов изображения")
+ return error
+
+
+class ProcessingError(Exception):
+
+ def __init__(self, code: str) -> None:
+ super().__init__(code)
+ self.code = code
+
+
+class JobOperationError(Exception):
+
+ def __init__(self, code: str):
+ self.code = code
+ super().__init__(code)
+
+
+def pipeline_worker_error_code(error: Exception) -> int:
+ if isinstance(error, OSError) and error.errno is not None:
+ if error.errno in {errno.EACCES, errno.EPERM, errno.EROFS, errno.ENOENT}:
+ return 5
+ return 4
+ return image_worker_error_code(error)
diff --git a/app/errors/handlers.py b/app/errors/handlers.py
new file mode 100644
index 0000000..55a42da
--- /dev/null
+++ b/app/errors/handlers.py
@@ -0,0 +1,246 @@
+import asyncio
+import logging
+from contextlib import asynccontextmanager
+from collections.abc import Mapping
+from uuid import uuid4
+
+from fastapi import FastAPI, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+from minio.error import MinioException
+from redis.exceptions import ConnectionError as RedisConnectionError
+from redis.exceptions import TimeoutError as RedisTimeoutError
+from sqlalchemy.exc import DBAPIError, InterfaceError, OperationalError, SQLAlchemyError
+from sqlalchemy.exc import TimeoutError as PoolTimeoutError
+from starlette.exceptions import HTTPException as StarletteHTTPException
+from starlette.requests import ClientDisconnect
+from urllib3.exceptions import HTTPError as StorageHTTPError
+
+from app.core.instance import INSTANCE_ID_HEADER
+from app.core.request_id import REQUEST_ID_HEADER
+from app.errors.exceptions import (
+ ApiError,
+ internal_error,
+ invalid_request,
+ not_found,
+ unavailable,
+)
+
+logger = logging.getLogger(__name__)
+
+TRANSIENT_SQLSTATE_CLASSES = ("08", "40", "53", "57")
+TRANSIENT_SQLSTATES = frozenset({"55P03"})
+RECOVERABLE_STORAGE_ERRORS = (
+ ApiError,
+ SQLAlchemyError,
+ MinioException,
+ StorageHTTPError,
+ OSError,
+ TimeoutError,
+)
+
+
+@asynccontextmanager
+async def operation_timeout(seconds: float, error: ApiError):
+ """Единое преобразование таймаутов; владелец операции задаёт ожидаемую ошибку."""
+ try:
+ async with asyncio.timeout(seconds):
+ yield
+ except TimeoutError:
+ raise error from None
+
+
+async def client_disconnect_handler(request: Request, exc: Exception) -> JSONResponse:
+ return await api_error_handler(
+ request, invalid_request("Загрузка прервана клиентом")
+ )
+
+
+def log_recoverable_error(operation: str, error: Exception) -> None:
+ """Согласованный безопасный лог восстановления без содержимого SDK-ошибки."""
+ logger.warning("%s deferred: %s", operation, type(error).__name__)
+
+
+def _get_request_id(request: Request) -> str:
+ request_id = getattr(request.state, "request_id", None)
+
+ if request_id is None:
+ request_id = str(uuid4())
+ request.state.request_id = request_id
+
+ return request_id
+
+
+def _build_response(
+ request: Request,
+ status_code: int,
+ code: str,
+ message: str,
+ headers: Mapping[str, str] | None = None,
+) -> JSONResponse:
+ request_id = _get_request_id(request)
+
+ response_headers = {
+ **(headers or {}),
+ REQUEST_ID_HEADER: request_id,
+ }
+ instance_id = getattr(request.app.state, "instance_id", None)
+ if instance_id is not None:
+ response_headers[INSTANCE_ID_HEADER] = instance_id
+
+ body = {
+ "error": {
+ "code": code,
+ "message": message,
+ "request_id": request_id,
+ }
+ }
+
+ return JSONResponse(
+ status_code=status_code,
+ content=body,
+ headers=response_headers,
+ )
+
+
+async def api_error_handler(
+ request: Request,
+ exc: Exception,
+) -> JSONResponse:
+ assert isinstance(exc, ApiError)
+
+ return _build_response(
+ request,
+ exc.status_code,
+ exc.code,
+ exc.message,
+ exc.headers,
+ )
+
+
+async def routing_error_handler(
+ request: Request,
+ exc: Exception,
+) -> JSONResponse:
+ assert isinstance(exc, StarletteHTTPException)
+
+ error = invalid_request()
+ if exc.status_code == 404:
+ error = not_found()
+ return _build_response(
+ request,
+ exc.status_code,
+ error.code,
+ error.message,
+ exc.headers,
+ )
+
+
+async def validation_error_handler(
+ request: Request,
+ exc: Exception,
+) -> JSONResponse:
+ assert isinstance(exc, RequestValidationError)
+
+ fields = sorted({".".join(map(str, error["loc"])) for error in exc.errors()})
+ message = "Неверный запрос"
+
+ if fields:
+ message = f"{message}: {', '.join(fields)}"
+
+ return _build_response(
+ request,
+ 400,
+ "invalid_request",
+ message,
+ )
+
+
+async def dependency_unavailable_handler(
+ request: Request,
+ exc: Exception,
+) -> JSONResponse:
+ """PostgreSQL/Redis/S3 недоступны либо исчерпан пул: 503 без внутренних деталей."""
+ request_id = _get_request_id(request)
+ logger.error(
+ "Dependency unavailable request_id=%s: %s",
+ request_id,
+ type(exc).__name__,
+ )
+ telemetry = getattr(request.app.state, "telemetry", None)
+ if telemetry is not None:
+ telemetry.event(
+ "dependency.unavailable",
+ error=True,
+ request_id=request_id,
+ exception_type=type(exc).__name__,
+ )
+
+ error = unavailable()
+ return _build_response(
+ request,
+ error.status_code,
+ error.code,
+ error.message,
+ {"Retry-After": "1"},
+ )
+
+
+def _is_transient_database_error(exc: DBAPIError) -> bool:
+ sqlstate = getattr(exc.orig, "sqlstate", None) or ""
+ return (
+ sqlstate.startswith(TRANSIENT_SQLSTATE_CLASSES)
+ or sqlstate in TRANSIENT_SQLSTATES
+ )
+
+
+async def database_error_handler(
+ request: Request,
+ exc: Exception,
+) -> JSONResponse:
+ """Временные ошибки PostgreSQL (таймаут запроса, блокировки, нехватка ресурсов) — 503."""
+ assert isinstance(exc, DBAPIError)
+
+ handler = unhandled_error_handler
+ if _is_transient_database_error(exc):
+ handler = dependency_unavailable_handler
+ return await handler(request, exc)
+
+
+async def unhandled_error_handler(
+ request: Request,
+ exc: Exception,
+) -> JSONResponse:
+ request_id = _get_request_id(request)
+
+ logger.error(
+ "Unhandled error request_id=%s type=%s",
+ request_id,
+ type(exc).__name__,
+ )
+
+ error = internal_error()
+
+ return _build_response(
+ request,
+ error.status_code,
+ error.code,
+ error.message,
+ )
+
+
+def register_exception_handlers(app: FastAPI) -> None:
+ app.add_exception_handler(ApiError, api_error_handler)
+ app.add_exception_handler(ClientDisconnect, client_disconnect_handler)
+ app.add_exception_handler(MinioException, dependency_unavailable_handler)
+ app.add_exception_handler(StorageHTTPError, dependency_unavailable_handler)
+ app.add_exception_handler(StarletteHTTPException, routing_error_handler)
+ app.add_exception_handler(RequestValidationError, validation_error_handler)
+ app.add_exception_handler(OperationalError, dependency_unavailable_handler)
+ app.add_exception_handler(InterfaceError, dependency_unavailable_handler)
+ app.add_exception_handler(PoolTimeoutError, dependency_unavailable_handler)
+ app.add_exception_handler(DBAPIError, database_error_handler)
+ app.add_exception_handler(RedisConnectionError, dependency_unavailable_handler)
+ app.add_exception_handler(RedisTimeoutError, dependency_unavailable_handler)
+ app.add_exception_handler(OSError, dependency_unavailable_handler)
+ app.add_exception_handler(Exception, unhandled_error_handler)
diff --git a/app/main.py b/app/main.py
new file mode 100644
index 0000000..4882359
--- /dev/null
+++ b/app/main.py
@@ -0,0 +1,102 @@
+import asyncio
+import logging
+from collections.abc import AsyncGenerator
+from contextlib import asynccontextmanager
+
+import uvicorn
+from fastapi import Depends, FastAPI
+
+from app.core.config import Settings
+from app.core.database import create_db_engine, create_session_factory
+from app.core.instance import InstanceIdMiddleware, resolve_instance_id
+from app.core.lifecycle import GracefulServer, Lifecycle
+from app.core.redis import create_redis_client
+from app.core.request_capacity import acquire_request_slot, create_request_capacity
+from app.core.request_id import RequestIdMiddleware
+from app.core.telemetry import HttpTelemetryMiddleware, Telemetry
+from app.endpoints.auth import router as auth_router
+from app.endpoints.attachments import router as attachments_router
+from app.endpoints.chats import router as chats_router
+from app.endpoints.health import router as health_router
+from app.endpoints.messages import router as messages_router
+from app.endpoints.users import router as users_router
+from app.errors.handlers import register_exception_handlers
+from app.storage.s3 import create_s3_storage
+
+
+def configure_logging(settings: Settings) -> None:
+ logging.basicConfig(
+ level=settings.log_level,
+ format="%(asctime)s %(levelname)s %(name)s %(message)s",
+ )
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+ settings = settings or Settings() # type: ignore
+ configure_logging(settings)
+
+ engine = create_db_engine(settings)
+ redis = create_redis_client(settings)
+ s3 = create_s3_storage(settings.s3) if settings.s3_enabled else None
+
+ @asynccontextmanager
+ async def lifespan(_: FastAPI) -> AsyncGenerator[None]:
+ yield
+ if s3 is not None:
+ await s3.aclose()
+ await redis.aclose()
+ await engine.dispose()
+ await asyncio.to_thread(telemetry.shutdown)
+
+ application = FastAPI(title="Messenger", version="1.6.0", lifespan=lifespan)
+
+ application.state.settings = settings
+ application.state.engine = engine
+ application.state.session_factory = create_session_factory(engine)
+ application.state.redis = redis
+ application.state.s3 = s3
+ application.state.upload_capacity = create_request_capacity(
+ settings.s3.max_concurrent_uploads
+ )
+ application.state.request_capacity = create_request_capacity(settings.max_in_flight)
+ application.state.instance_id = resolve_instance_id(settings.instance_id)
+ telemetry = Telemetry("messenger-api", application.state.instance_id)
+ telemetry.observe_pool(engine.pool)
+ application.state.telemetry = telemetry
+ application.state.lifecycle = Lifecycle()
+ application.add_middleware(RequestIdMiddleware)
+ application.add_middleware(
+ InstanceIdMiddleware, instance_id=application.state.instance_id
+ )
+ application.add_middleware(HttpTelemetryMiddleware, telemetry=telemetry)
+ register_exception_handlers(application)
+
+ limited = [Depends(acquire_request_slot)]
+ application.include_router(health_router)
+ application.include_router(users_router, dependencies=limited)
+ application.include_router(auth_router, dependencies=limited)
+ application.include_router(chats_router, dependencies=limited)
+ application.include_router(messages_router, dependencies=limited)
+ application.include_router(attachments_router, dependencies=limited)
+
+ return application
+
+
+def main() -> None:
+ settings = Settings() # type: ignore
+ application = create_app(settings)
+ config = uvicorn.Config(
+ application,
+ host=settings.app_host,
+ port=settings.app_port,
+ log_level=settings.log_level.lower(),
+ access_log=False,
+ timeout_graceful_shutdown=settings.graceful_shutdown_timeout_seconds,
+ )
+ GracefulServer(
+ config, application.state.lifecycle, settings.shutdown_drain_seconds
+ ).run()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/app/models/__init__.py b/app/models/__init__.py
new file mode 100644
index 0000000..d9476c4
--- /dev/null
+++ b/app/models/__init__.py
@@ -0,0 +1,25 @@
+from app.models.job_replay import JobReplay
+from app.models.base import Base
+from app.models.job import AttachmentJob
+from app.models.attachment import Attachment, MessageAttachment, UploadIntent
+from app.models.chat import Chat
+from app.models.chat_member import ChatMember
+from app.models.counter import Counter
+from app.models.idempotency_key import IdempotencyKey
+from app.models.message import Message
+from app.models.user import User
+
+__all__ = [
+ "JobReplay",
+ "AttachmentJob",
+ "Base",
+ "Attachment",
+ "MessageAttachment",
+ "UploadIntent",
+ "Chat",
+ "ChatMember",
+ "Counter",
+ "IdempotencyKey",
+ "Message",
+ "User",
+]
diff --git a/app/models/attachment.py b/app/models/attachment.py
new file mode 100644
index 0000000..dfc12f1
--- /dev/null
+++ b/app/models/attachment.py
@@ -0,0 +1,70 @@
+"""Метаданные оригиналов и журнал операций до записи в S3."""
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import CheckConstraint, ForeignKey, Index, String, func, text
+from sqlalchemy.dialects.postgresql import JSONB
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.models.base import Base
+
+
+class Attachment(Base):
+ __tablename__ = "attachments"
+ __table_args__ = (
+ CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="size_bytes"),
+ CheckConstraint(
+ "content_type IN ('image/png', 'image/jpeg')", name="content_type"
+ ),
+ CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="sha256"),
+ CheckConstraint(
+ "status IN ('queued', 'processing', 'ready', 'failed')", name="status"
+ ),
+ CheckConstraint(
+ "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
+ name="error_code",
+ ),
+ Index("ix_attachments_chat_id", "chat_id"),
+ )
+
+ id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+ chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
+ owner_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
+ object_key: Mapped[str] = mapped_column(String(128), unique=True)
+ filename: Mapped[str] = mapped_column(String(255))
+ content_type: Mapped[str] = mapped_column(String(32))
+ size_bytes: Mapped[int]
+ sha256: Mapped[str] = mapped_column(String(64))
+ status: Mapped[str] = mapped_column(String(16), server_default="queued")
+ error_code: Mapped[str | None] = mapped_column(String(64))
+ thumbnail_key: Mapped[str | None] = mapped_column(String(128))
+ crop_key: Mapped[str | None] = mapped_column(String(128))
+ optimized_key: Mapped[str | None] = mapped_column(String(128))
+ variants: Mapped[list[dict]] = mapped_column(
+ JSONB, default=list, server_default=text("'[]'::jsonb")
+ )
+ created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+ updated_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+
+
+class MessageAttachment(Base):
+ __tablename__ = "message_attachments"
+
+ message_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
+ )
+ attachment_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("attachments.id"), primary_key=True
+ )
+ position: Mapped[int]
+
+
+class UploadIntent(Base):
+
+ __tablename__ = "upload_intents"
+ __table_args__ = (Index("ix_upload_intents_created_at", "created_at"),)
+
+ id: Mapped[uuid.UUID] = mapped_column(primary_key=True)
+ object_key: Mapped[str] = mapped_column(String(128), unique=True)
+ created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
diff --git a/app/models/base.py b/app/models/base.py
new file mode 100644
index 0000000..ff742f1
--- /dev/null
+++ b/app/models/base.py
@@ -0,0 +1,26 @@
+import uuid
+from datetime import datetime
+from typing import Any, ClassVar
+
+from sqlalchemy import DateTime, MetaData
+from sqlalchemy.dialects.postgresql import UUID as PG_UUID
+from sqlalchemy.orm import DeclarativeBase
+from sqlalchemy.types import TypeEngine
+
+NAMING_CONVENTION = {
+ "ix": "ix_%(column_0_label)s",
+ "uq": "uq_%(table_name)s_%(column_0_name)s",
+ "ck": "ck_%(table_name)s_%(constraint_name)s",
+ "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s",
+ "pk": "pk_%(table_name)s",
+}
+
+
+class Base(DeclarativeBase):
+
+ metadata = MetaData(naming_convention=NAMING_CONVENTION)
+
+ type_annotation_map: ClassVar[dict[Any, TypeEngine[Any]]] = {
+ uuid.UUID: PG_UUID(as_uuid=True),
+ datetime: DateTime(timezone=True),
+ }
diff --git a/app/models/chat.py b/app/models/chat.py
new file mode 100644
index 0000000..2119834
--- /dev/null
+++ b/app/models/chat.py
@@ -0,0 +1,31 @@
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from sqlalchemy import String, func
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base
+
+if TYPE_CHECKING:
+ from app.models.chat_member import ChatMember
+ from app.models.message import Message
+
+
+class Chat(Base):
+
+ __tablename__ = "chats"
+
+ id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+ title: Mapped[str] = mapped_column(String(100))
+ created_at: Mapped[datetime] = mapped_column(server_default=func.now())
+
+ members: Mapped[list["ChatMember"]] = relationship(
+ back_populates="chat",
+ cascade="all, delete-orphan",
+ passive_deletes=True,
+ lazy="selectin",
+ )
+ messages: Mapped[list["Message"]] = relationship(
+ back_populates="chat", lazy="raise"
+ )
diff --git a/app/models/chat_member.py b/app/models/chat_member.py
new file mode 100644
index 0000000..77021f5
--- /dev/null
+++ b/app/models/chat_member.py
@@ -0,0 +1,25 @@
+import uuid
+from typing import TYPE_CHECKING
+
+from sqlalchemy import ForeignKey, Index
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base
+
+if TYPE_CHECKING:
+ from app.models.chat import Chat
+ from app.models.user import User
+
+
+class ChatMember(Base):
+
+ __tablename__ = "chat_members"
+ __table_args__ = (Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"),)
+
+ chat_id: Mapped[uuid.UUID] = mapped_column(
+ ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
+ )
+ user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
+
+ chat: Mapped["Chat"] = relationship(back_populates="members", lazy="raise")
+ user: Mapped["User"] = relationship(back_populates="memberships", lazy="raise")
diff --git a/app/models/counter.py b/app/models/counter.py
new file mode 100644
index 0000000..38446c0
--- /dev/null
+++ b/app/models/counter.py
@@ -0,0 +1,12 @@
+from sqlalchemy import BigInteger, String
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.models.base import Base
+
+
+class Counter(Base):
+
+ __tablename__ = "counters"
+
+ name: Mapped[str] = mapped_column(String(64), primary_key=True)
+ value: Mapped[int] = mapped_column(BigInteger)
diff --git a/app/models/idempotency_key.py b/app/models/idempotency_key.py
new file mode 100644
index 0000000..d39dd07
--- /dev/null
+++ b/app/models/idempotency_key.py
@@ -0,0 +1,23 @@
+"""Сохранённый результат отправки сообщения."""
+
+import uuid
+from datetime import datetime
+
+from sqlalchemy import ForeignKey, Index, String, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.models.base import Base
+
+
+class IdempotencyKey(Base):
+
+ __tablename__ = "idempotency_keys"
+ __table_args__ = (Index("ix_idempotency_keys_created_at", "created_at"),)
+
+ user_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"), primary_key=True)
+ method: Mapped[str] = mapped_column(String(8), primary_key=True)
+ path: Mapped[str] = mapped_column(String(255), primary_key=True)
+ key: Mapped[str] = mapped_column(String(128), primary_key=True)
+ request_hash: Mapped[str] = mapped_column(String(64))
+ message_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("messages.id"))
+ created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
diff --git a/app/models/job.py b/app/models/job.py
new file mode 100644
index 0000000..b7723d0
--- /dev/null
+++ b/app/models/job.py
@@ -0,0 +1,41 @@
+"""Устойчивая работа и намерение публикации; один pipeline на вложение."""
+
+from datetime import datetime
+from uuid import UUID, uuid4
+
+from sqlalchemy import JSON, CheckConstraint, ForeignKey, Index, String, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.models.base import Base
+
+
+class AttachmentJob(Base):
+ __tablename__ = "attachment_jobs"
+ __table_args__ = (
+ CheckConstraint("pipeline_version = 1", name="pipeline_version"),
+ CheckConstraint("attempt >= 0 AND published_attempt >= 0", name="attempt"),
+ Index("ix_attachment_jobs_due", "next_attempt_at"),
+ Index("ix_attachment_jobs_lease", "lease_until"),
+ )
+
+ id: Mapped[UUID] = mapped_column(primary_key=True, default=uuid4)
+ attachment_id: Mapped[UUID] = mapped_column(
+ ForeignKey("attachments.id"),
+ unique=True,
+ )
+ trace_context: Mapped[dict | None] = mapped_column(JSON, nullable=True)
+ staging_key: Mapped[str | None] = mapped_column(String(128))
+ pipeline_version: Mapped[int] = mapped_column(server_default="1")
+ attempt: Mapped[int] = mapped_column(server_default="0")
+ cycle_start_attempt: Mapped[int] = mapped_column(server_default="0")
+ attempt_limit: Mapped[int | None]
+ last_error_permanent: Mapped[bool | None]
+ published_attempt: Mapped[int] = mapped_column(server_default="0")
+ next_attempt_at: Mapped[datetime] = mapped_column(
+ server_default=func.clock_timestamp()
+ )
+ lease_token: Mapped[UUID | None]
+ lease_until: Mapped[datetime | None]
+ last_error_code: Mapped[str | None] = mapped_column(String(64))
+ finished_at: Mapped[datetime | None]
+ created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
diff --git a/app/models/job_replay.py b/app/models/job_replay.py
new file mode 100644
index 0000000..7f2b27a
--- /dev/null
+++ b/app/models/job_replay.py
@@ -0,0 +1,22 @@
+"""Аудит явного повторного запуска failed; request id обеспечивает идемпотентность."""
+
+from datetime import datetime
+from uuid import UUID
+
+from sqlalchemy import ForeignKey, String, func
+from sqlalchemy.orm import Mapped, mapped_column
+
+from app.models.base import Base
+
+
+class JobReplay(Base):
+ __tablename__ = "job_replays"
+
+ id: Mapped[UUID] = mapped_column(primary_key=True)
+ job_id: Mapped[UUID] = mapped_column(ForeignKey("attachment_jobs.id"), index=True)
+ actor: Mapped[str] = mapped_column(String(64))
+ reason: Mapped[str] = mapped_column(String(32))
+ previous_attempt: Mapped[int]
+ attempt_limit: Mapped[int]
+ previous_error_code: Mapped[str | None] = mapped_column(String(64))
+ created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
diff --git a/app/models/message.py b/app/models/message.py
new file mode 100644
index 0000000..b93e289
--- /dev/null
+++ b/app/models/message.py
@@ -0,0 +1,29 @@
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from sqlalchemy import ForeignKey, Index, Text, func
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base
+
+if TYPE_CHECKING:
+ from app.models.chat import Chat
+ from app.models.user import User
+
+
+class Message(Base):
+
+ __tablename__ = "messages"
+ __table_args__ = (
+ Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"),
+ )
+
+ id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+ chat_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("chats.id"))
+ sender_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("users.id"))
+ text: Mapped[str] = mapped_column(Text)
+ created_at: Mapped[datetime] = mapped_column(server_default=func.clock_timestamp())
+
+ chat: Mapped["Chat"] = relationship(back_populates="messages", lazy="raise")
+ sender: Mapped["User"] = relationship(back_populates="messages", lazy="raise")
diff --git a/app/models/user.py b/app/models/user.py
new file mode 100644
index 0000000..aa50e4d
--- /dev/null
+++ b/app/models/user.py
@@ -0,0 +1,30 @@
+import uuid
+from datetime import datetime
+from typing import TYPE_CHECKING
+
+from sqlalchemy import String, func
+from sqlalchemy.orm import Mapped, mapped_column, relationship
+
+from app.models.base import Base
+
+if TYPE_CHECKING:
+ from app.models.chat_member import ChatMember
+ from app.models.message import Message
+
+
+class User(Base):
+
+ __tablename__ = "users"
+
+ id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
+ username: Mapped[str] = mapped_column(String(32), unique=True)
+ display_name: Mapped[str] = mapped_column(String(100))
+ password_hash: Mapped[str] = mapped_column(String(255))
+ created_at: Mapped[datetime] = mapped_column(server_default=func.now())
+
+ memberships: Mapped[list["ChatMember"]] = relationship(
+ back_populates="user", lazy="raise"
+ )
+ messages: Mapped[list["Message"]] = relationship(
+ back_populates="sender", lazy="raise"
+ )
diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py
new file mode 100644
index 0000000..2f03894
--- /dev/null
+++ b/app/schemas/__init__.py
@@ -0,0 +1,6 @@
+from app.schemas.chat import Chat
+from app.schemas.message import Message
+from app.schemas.session import AuthSession
+from app.schemas.user import User
+
+__all__ = ["AuthSession", "Chat", "Message", "User"]
diff --git a/app/schemas/attachment.py b/app/schemas/attachment.py
new file mode 100644
index 0000000..629e26b
--- /dev/null
+++ b/app/schemas/attachment.py
@@ -0,0 +1,53 @@
+"""Публичные состояния и варианты; внутренние ключи S3 не сериализуются."""
+
+from datetime import datetime
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from app.schemas.base import Base
+
+
+class Variant(BaseModel):
+ model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+ name: Literal["thumbnail", "crop", "optimized"]
+ content_type: Literal["image/jpeg", "image/png", "image/webp"]
+ size_bytes: int = Field(ge=1)
+ width: int = Field(ge=1)
+ height: int = Field(ge=1)
+ sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
+
+
+class Attachment(Base):
+ chat_id: UUID
+ owner_id: UUID
+ filename: str = Field(min_length=1, max_length=255)
+ content_type: Literal["image/png", "image/jpeg"]
+ size_bytes: int = Field(ge=1, le=10_485_760)
+ sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
+ status: Literal["queued", "processing", "ready", "failed"]
+ variants: list[Variant] = Field(default_factory=list)
+ error_code: str | None = None
+ created_at: datetime
+ updated_at: datetime
+
+
+VARIANT_FILENAMES = {
+ "thumbnail": "thumbnail.jpg",
+ "crop": "crop.jpg",
+ "optimized": "optimized.webp",
+}
+
+
+class PipelineResult(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ variants: list[Variant] = Field(min_length=3, max_length=3)
+
+ @field_validator("variants")
+ @classmethod
+ def require_all_variants(cls, variants: list[Variant]) -> list[Variant]:
+ if {item.name for item in variants} != set(VARIANT_FILENAMES):
+ raise ValueError("Нужны thumbnail, crop и optimized без повторов")
+ return variants
diff --git a/app/schemas/base.py b/app/schemas/base.py
new file mode 100644
index 0000000..6a18d6a
--- /dev/null
+++ b/app/schemas/base.py
@@ -0,0 +1,34 @@
+from typing import Literal
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, field_validator
+
+
+class Base(BaseModel):
+ model_config = ConfigDict(extra="forbid", from_attributes=True)
+
+ id: UUID
+
+
+class Health(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ status: Literal["ok"]
+
+
+class BaseCreate(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ @field_validator("*", mode="before")
+ @classmethod
+ def reject_invalid_strings(cls, value: object) -> object:
+ if isinstance(value, str):
+ if not value.strip():
+ raise ValueError("Строка не может состоять только из пробелов")
+ if "\x00" in value:
+ raise ValueError("Строка не может содержать символ NUL")
+ return value
+
+
+class BasePage(BaseModel):
+ model_config = ConfigDict(extra="forbid")
diff --git a/app/schemas/chat.py b/app/schemas/chat.py
new file mode 100644
index 0000000..c0436da
--- /dev/null
+++ b/app/schemas/chat.py
@@ -0,0 +1,22 @@
+from datetime import datetime
+from uuid import UUID
+
+from pydantic import Field
+
+from app.schemas.base import Base, BaseCreate, BasePage
+
+
+class Chat(Base):
+ title: str = Field(min_length=1, max_length=100)
+ member_ids: list[UUID] = Field(min_length=1, max_length=100)
+ created_at: datetime
+
+
+class CreateChat(BaseCreate):
+ title: str = Field(min_length=1, max_length=100)
+ member_ids: list[UUID] = Field(min_length=1, max_length=100)
+
+
+class ChatPage(BasePage):
+ items: list[Chat]
+ next_cursor: str | None
diff --git a/app/schemas/job.py b/app/schemas/job.py
new file mode 100644
index 0000000..7b78370
--- /dev/null
+++ b/app/schemas/job.py
@@ -0,0 +1,22 @@
+"""Сообщение брокера: только идентификаторы и служебные поля контракта v1."""
+
+from typing import Literal
+from uuid import UUID
+
+from pydantic import AwareDatetime, BaseModel, ConfigDict, Field
+
+
+class JobMessage(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ schema_version: Literal[1] = 1
+ type: Literal["attachment.process.v1"] = "attachment.process.v1"
+ job_id: UUID
+ attachment_id: UUID
+ pipeline_version: int = Field(default=1, ge=1)
+ attempt: int = Field(ge=1)
+ created_at: AwareDatetime
+ traceparent: str | None = Field(
+ default=None, pattern=r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$"
+ )
+ tracestate: str | None = Field(default=None, max_length=512)
diff --git a/app/schemas/message.py b/app/schemas/message.py
new file mode 100644
index 0000000..a085e03
--- /dev/null
+++ b/app/schemas/message.py
@@ -0,0 +1,41 @@
+from datetime import datetime
+from typing import Self
+from uuid import UUID
+
+from pydantic import Field, field_validator, model_validator
+
+from app.schemas.base import Base, BaseCreate, BasePage
+
+
+class Message(Base):
+ chat_id: UUID
+ sender_id: UUID
+ text: str = Field(max_length=4000)
+ attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
+ created_at: datetime
+
+
+class CreateMessage(BaseCreate):
+ text: str = Field(max_length=4000)
+ attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
+
+ @field_validator("*", mode="before")
+ @classmethod
+ def reject_invalid_strings(cls, value: object) -> object:
+ if isinstance(value, str):
+ if "\x00" in value or (value != "" and not value.strip()):
+ raise ValueError("Некорректная строка сообщения")
+ return value
+
+ @model_validator(mode="after")
+ def require_content(self) -> Self:
+ if not self.text and not self.attachment_ids:
+ raise ValueError("Нужен текст или вложение")
+ if len(self.attachment_ids) != len(set(self.attachment_ids)):
+ raise ValueError("attachment_ids содержит дубликаты")
+ return self
+
+
+class MessagePage(BasePage):
+ items: list[Message]
+ next_cursor: str | None
diff --git a/app/schemas/session.py b/app/schemas/session.py
new file mode 100644
index 0000000..4b67bd6
--- /dev/null
+++ b/app/schemas/session.py
@@ -0,0 +1,16 @@
+from datetime import datetime
+from typing import Literal
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.schemas.user import User
+
+
+class AuthSession(BaseModel):
+
+ model_config = ConfigDict(extra="forbid")
+
+ token: str = Field(min_length=32)
+ token_type: Literal["Bearer"] = "Bearer"
+ expires_at: datetime
+ user: User
diff --git a/app/schemas/user.py b/app/schemas/user.py
new file mode 100644
index 0000000..39a26dc
--- /dev/null
+++ b/app/schemas/user.py
@@ -0,0 +1,24 @@
+from datetime import datetime
+
+from pydantic import Field
+
+from app.schemas.base import Base, BaseCreate
+
+USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$"
+PASSWORD_MIN_LENGTH = 12
+PASSWORD_MAX_LENGTH = 128
+
+
+class User(Base):
+
+ username: str = Field(pattern=USERNAME_PATTERN)
+ display_name: str = Field(min_length=1, max_length=100)
+ created_at: datetime
+
+
+class CreateUser(BaseCreate):
+ username: str = Field(pattern=USERNAME_PATTERN)
+ display_name: str = Field(min_length=1, max_length=100)
+ password: str = Field(
+ min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH, repr=False
+ )
diff --git a/app/storage/__init__.py b/app/storage/__init__.py
new file mode 100644
index 0000000..c881665
--- /dev/null
+++ b/app/storage/__init__.py
@@ -0,0 +1 @@
+"""Доступ к данным PostgreSQL и Redis."""
diff --git a/app/storage/attachments.py b/app/storage/attachments.py
new file mode 100644
index 0000000..3ef1765
--- /dev/null
+++ b/app/storage/attachments.py
@@ -0,0 +1,99 @@
+"""SQL-запросы метаданных и журнала; commit выполняется в domain."""
+
+from datetime import timedelta
+from uuid import UUID
+
+from sqlalchemy import delete, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.attachment import Attachment, MessageAttachment, UploadIntent
+
+
+async def create_attachment(session: AsyncSession, **values) -> Attachment:
+ record = Attachment(**values)
+ session.add(record)
+ await session.flush()
+ return record
+
+
+async def get_attachment(
+ session: AsyncSession, attachment_id: UUID
+) -> Attachment | None:
+ return await session.get(Attachment, attachment_id)
+
+
+async def get_by_key(session: AsyncSession, key: str) -> Attachment | None:
+ return await session.scalar(select(Attachment).where(Attachment.object_key == key))
+
+
+async def get_for_chat(
+ session: AsyncSession, chat_id: UUID, attachment_ids: list[UUID]
+) -> list[Attachment]:
+ records = []
+ if attachment_ids:
+ records = list(
+ await session.scalars(
+ select(Attachment).where(
+ Attachment.id.in_(attachment_ids),
+ Attachment.chat_id == chat_id,
+ )
+ )
+ )
+ return records
+
+
+async def bind_attachments(
+ session: AsyncSession, message_id: UUID, attachment_ids: list[UUID]
+) -> None:
+ session.add_all(
+ MessageAttachment(message_id=message_id, attachment_id=id, position=position)
+ for position, id in enumerate(attachment_ids)
+ )
+ await session.flush()
+
+
+async def message_attachment_ids(
+ session: AsyncSession, message_ids: list[UUID]
+) -> dict[UUID, list[UUID]]:
+ result: dict[UUID, list[UUID]] = {id: [] for id in message_ids}
+ if not message_ids:
+ return result
+ rows = await session.execute(
+ select(MessageAttachment)
+ .where(MessageAttachment.message_id.in_(message_ids))
+ .order_by(MessageAttachment.position)
+ )
+ for link in rows.scalars():
+ result[link.message_id].append(link.attachment_id)
+ return result
+
+
+async def reserve_upload(session: AsyncSession, id: UUID, key: str) -> None:
+ session.add(UploadIntent(id=id, object_key=key))
+ await session.flush()
+
+
+async def lock_upload(session: AsyncSession, id: UUID) -> UploadIntent | None:
+ return await session.scalar(
+ select(UploadIntent).where(UploadIntent.id == id).with_for_update()
+ )
+
+
+async def finish_upload(session: AsyncSession, id: UUID) -> None:
+ await session.execute(delete(UploadIntent).where(UploadIntent.id == id))
+
+
+async def stale_uploads(
+ session: AsyncSession, grace_seconds: int, limit: int
+) -> list[UploadIntent]:
+ result = await session.scalars(
+ select(UploadIntent)
+ .where(
+ UploadIntent.created_at
+ < func.clock_timestamp() - timedelta(seconds=grace_seconds),
+ )
+ .order_by(UploadIntent.created_at)
+ .limit(limit)
+ .with_for_update(skip_locked=True)
+ )
+ return list(result)
diff --git a/app/storage/broker.py b/app/storage/broker.py
new file mode 100644
index 0000000..2f5d2e4
--- /dev/null
+++ b/app/storage/broker.py
@@ -0,0 +1,98 @@
+"""RabbitMQ: durable queue, persistent messages, publisher confirms и manual ack."""
+
+import asyncio
+import logging
+from dataclasses import dataclass
+
+import aio_pika
+from aio_pika.abc import AbstractIncomingMessage
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+
+from app.core.jobs import JobSettings
+from app.schemas.job import JobMessage
+
+
+@dataclass(frozen=True)
+class Delivery:
+ id: AbstractIncomingMessage
+ payload: bytes
+
+
+class JobBroker:
+ """Один consumer/канал на экземпляр; publisher использует отдельный экземпляр."""
+
+ def __init__(self, settings: JobSettings) -> None:
+ self.settings = settings
+ self.connection = self.channel = self.queue = self.iterator = None
+ # Transport exceptions могут содержать URL с credentials.
+ logging.getLogger("aio_pika").setLevel(logging.CRITICAL)
+ logging.getLogger("aiormq").setLevel(logging.CRITICAL)
+
+ async def ensure_ready(self) -> None:
+ if (
+ self.connection is None
+ or self.connection.is_closed
+ or self.channel is None
+ or self.channel.is_closed
+ ):
+ await self.aclose()
+ self.connection = await aio_pika.connect(
+ host=self.settings.broker_host,
+ port=self.settings.broker_port,
+ login=self.settings.broker_user,
+ password=self.settings.broker_password_file.read_text().strip(),
+ virtualhost=self.settings.broker_vhost,
+ timeout=5,
+ heartbeat=30,
+ )
+ self.channel = await self.connection.channel(
+ publisher_confirms=True, on_return_raises=True
+ )
+ await self.channel.set_qos(prefetch_count=1)
+ self.queue = await self.channel.declare_queue(
+ self.settings.broker_queue, durable=True
+ )
+
+ async def publish_job(self, message: JobMessage) -> None:
+ await self.ensure_ready()
+ headers = {}
+ TraceContextTextMapPropagator().inject(headers)
+ await self.channel.default_exchange.publish(
+ aio_pika.Message(
+ body=message.model_dump_json(exclude_none=True).encode(),
+ content_type="application/json",
+ headers=headers,
+ delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
+ message_id=f"{message.job_id}:{message.attempt}",
+ ),
+ routing_key=self.settings.broker_queue,
+ mandatory=True,
+ timeout=10,
+ )
+
+ async def receive(self, consumer: str) -> Delivery:
+ await self.ensure_ready()
+ if self.iterator is None:
+ self.iterator = self.queue.iterator(no_ack=False, consumer_tag=consumer)
+ await self.iterator.__aenter__()
+ message = await self.iterator.__anext__()
+ return Delivery(message, message.body)
+
+ async def acknowledge_job(self, delivery_id: AbstractIncomingMessage) -> None:
+ await delivery_id.ack()
+
+ async def defer_job(self, delivery_id: AbstractIncomingMessage) -> None:
+ # Живая чужая lease: не удерживать единственный prefetch slot навсегда.
+ await asyncio.sleep(self.settings.publisher_poll_seconds)
+ await delivery_id.nack(requeue=True)
+
+ async def reject_invalid(self, delivery_id: AbstractIncomingMessage) -> None:
+ # Failed jobs находятся в PostgreSQL; неизвестные payload не копируются в DLQ.
+ logging.getLogger(__name__).warning("Rejected invalid job envelope")
+ await delivery_id.reject(requeue=False)
+
+ async def aclose(self) -> None:
+ connection = self.connection
+ self.connection = self.channel = self.queue = self.iterator = None
+ if connection is not None and not connection.is_closed:
+ await connection.close()
diff --git a/app/storage/chats.py b/app/storage/chats.py
new file mode 100644
index 0000000..d313034
--- /dev/null
+++ b/app/storage/chats.py
@@ -0,0 +1,67 @@
+from collections.abc import Sequence
+from uuid import UUID
+
+from sqlalchemy import exists, select, tuple_
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.chat import Chat
+from app.models.chat_member import ChatMember
+from app.storage.types import Position
+
+
+async def create_chat(
+ session: AsyncSession,
+ *,
+ title: str,
+ member_ids: Sequence[UUID],
+) -> Chat:
+ chat = Chat(
+ title=title,
+ members=[ChatMember(user_id=user_id) for user_id in member_ids],
+ )
+ session.add(chat)
+ await session.flush()
+ return chat
+
+
+async def get_chat_for_member(
+ session: AsyncSession,
+ chat_id: UUID,
+ user_id: UUID,
+) -> Chat | None:
+ stmt = (
+ select(Chat)
+ .join(ChatMember, ChatMember.chat_id == Chat.id)
+ .where(Chat.id == chat_id, ChatMember.user_id == user_id)
+ )
+ result = await session.execute(stmt)
+ chat = result.scalar_one_or_none()
+ return chat
+
+
+async def is_chat_member(session: AsyncSession, chat_id: UUID, user_id: UUID) -> bool:
+ stmt = select(
+ exists().where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id)
+ )
+ result = await session.execute(stmt)
+ is_member = bool(result.scalar_one())
+ return is_member
+
+
+async def list_chats_for_user(
+ session: AsyncSession,
+ user_id: UUID,
+ after: Position | None,
+ limit: int,
+) -> list[Chat]:
+ stmt = (
+ select(Chat)
+ .join(ChatMember, ChatMember.chat_id == Chat.id)
+ .where(ChatMember.user_id == user_id)
+ )
+ if after is not None:
+ stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after))
+ stmt = stmt.order_by(Chat.created_at, Chat.id).limit(limit)
+ result = await session.execute(stmt)
+ chats = list(result.scalars().all())
+ return chats
diff --git a/app/storage/health.py b/app/storage/health.py
new file mode 100644
index 0000000..2d034b7
--- /dev/null
+++ b/app/storage/health.py
@@ -0,0 +1,15 @@
+"""Простые проверки соединения с общими хранилищами."""
+
+from redis.asyncio import Redis
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+
+async def check_database(engine: AsyncEngine) -> bool:
+ async with engine.connect() as connection:
+ await connection.execute(text("SELECT 1"))
+ return True
+
+
+async def check_redis(client: Redis) -> bool:
+ return bool(await client.ping())
diff --git a/app/storage/idempotency.py b/app/storage/idempotency.py
new file mode 100644
index 0000000..4f25c0e
--- /dev/null
+++ b/app/storage/idempotency.py
@@ -0,0 +1,67 @@
+"""Хранение и блокировка ключей в PostgreSQL."""
+
+from datetime import timedelta
+import hashlib
+import json
+from uuid import UUID
+
+from sqlalchemy import BigInteger, func, literal, select, tuple_
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.idempotency_key import IdempotencyKey
+
+Scope = tuple[UUID, str, str, str]
+
+
+async def lock_key(session: AsyncSession, scope: Scope) -> None:
+ raw = json.dumps([str(scope[0]), *scope[1:]]).encode()
+ lock_id = int.from_bytes(hashlib.sha256(raw).digest()[:8], signed=True)
+ await session.execute(
+ select(func.pg_advisory_xact_lock(literal(lock_id, type_=BigInteger)))
+ )
+
+
+async def get_active_key(
+ session: AsyncSession, scope: Scope, ttl_seconds: int
+) -> IdempotencyKey | None:
+ stmt = select(IdempotencyKey).where(
+ tuple_(
+ IdempotencyKey.user_id,
+ IdempotencyKey.method,
+ IdempotencyKey.path,
+ IdempotencyKey.key,
+ )
+ == tuple_(*scope),
+ IdempotencyKey.created_at
+ > func.clock_timestamp() - timedelta(seconds=ttl_seconds),
+ )
+ return await session.scalar(stmt)
+
+
+async def save_result(
+ session: AsyncSession, scope: Scope, request_hash: str, message_id: UUID
+) -> None:
+ stmt = insert(IdempotencyKey).values(
+ user_id=scope[0],
+ method=scope[1],
+ path=scope[2],
+ key=scope[3],
+ request_hash=request_hash,
+ message_id=message_id,
+ )
+ await session.execute(
+ stmt.on_conflict_do_update(
+ index_elements=[
+ IdempotencyKey.user_id,
+ IdempotencyKey.method,
+ IdempotencyKey.path,
+ IdempotencyKey.key,
+ ],
+ set_={
+ "request_hash": request_hash,
+ "message_id": message_id,
+ "created_at": func.clock_timestamp(),
+ },
+ )
+ )
diff --git a/app/storage/job_admin.py b/app/storage/job_admin.py
new file mode 100644
index 0000000..6780924
--- /dev/null
+++ b/app/storage/job_admin.py
@@ -0,0 +1,59 @@
+"""Операторские выборки и журнал replay; транзакция принадлежит domain."""
+
+from sqlalchemy import select
+from sqlalchemy.dialects.postgresql import insert
+
+from app.models.attachment import Attachment
+from app.models.job import AttachmentJob
+from app.models.job_replay import JobReplay
+
+
+async def failed_jobs(session, limit: int):
+ result = await session.execute(
+ select(AttachmentJob, Attachment)
+ .join(Attachment, Attachment.id == AttachmentJob.attachment_id)
+ .where(Attachment.status == "failed")
+ .order_by(AttachmentJob.finished_at.desc(), AttachmentJob.id)
+ .limit(limit)
+ )
+ return result.all()
+
+
+async def replay_record(session, request_id):
+ return await session.get(JobReplay, request_id)
+
+
+async def insert_replay(session, **values) -> bool:
+ result = await session.scalar(
+ insert(JobReplay)
+ .values(**values)
+ .on_conflict_do_nothing(index_elements=[JobReplay.id])
+ .returning(JobReplay.id)
+ )
+ return result is not None
+
+
+async def replay_history(session, job_id):
+ return (
+ await session.scalars(
+ select(JobReplay)
+ .where(JobReplay.job_id == job_id)
+ .order_by(JobReplay.created_at)
+ .limit(100)
+ )
+ ).all()
+
+
+def reset_failed(job, attachment, now, attempt_limit):
+ job.cycle_start_attempt = job.attempt
+ job.attempt_limit = attempt_limit
+ job.published_attempt = job.attempt
+ job.next_attempt_at = now
+ job.lease_token = None
+ job.lease_until = None
+ job.finished_at = None
+ job.last_error_code = None
+ job.last_error_permanent = None
+ attachment.status = "queued"
+ attachment.error_code = None
+ attachment.updated_at = now
diff --git a/app/storage/jobs.py b/app/storage/jobs.py
new file mode 100644
index 0000000..d81a7fe
--- /dev/null
+++ b/app/storage/jobs.py
@@ -0,0 +1,182 @@
+"""SQL и изменения jobs/attachments; commit остаётся в domain."""
+
+from dataclasses import dataclass
+from datetime import datetime, timedelta
+from uuid import UUID
+
+from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
+from sqlalchemy import and_, or_, func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.attachment import Attachment
+from app.models.job import AttachmentJob
+from app.schemas.attachment import VARIANT_FILENAMES
+
+
+@dataclass(frozen=True)
+class JobWork:
+ job_id: UUID
+ attachment_id: UUID
+ lease_token: UUID
+ pipeline_version: int
+ staging_key: str
+ content_type: str
+ size_bytes: int
+ sha256: str
+
+ @property
+ def original_key(self) -> str:
+ return f"originals/{self.attachment_id.hex}/v{self.pipeline_version}"
+
+ def variant_key(self, name: str) -> str:
+ filename = VARIANT_FILENAMES[name]
+ return f"variants/{self.attachment_id.hex}/v{self.pipeline_version}/{filename}"
+
+
+async def create_job(
+ session: AsyncSession, attachment_id: UUID, staging_key: str
+) -> UUID:
+ carrier = {}
+ TraceContextTextMapPropagator().inject(carrier)
+ job = AttachmentJob(
+ attachment_id=attachment_id,
+ staging_key=staging_key,
+ trace_context=carrier or None,
+ )
+ session.add(job)
+ await session.flush()
+ return job.id
+
+
+async def lock_job(session: AsyncSession, job_id: UUID):
+ result = await session.execute(
+ select(AttachmentJob, Attachment)
+ .join(Attachment, Attachment.id == AttachmentJob.attachment_id)
+ .where(AttachmentJob.id == job_id)
+ .with_for_update(of=AttachmentJob)
+ .execution_options(populate_existing=True)
+ )
+ return result.one_or_none()
+
+
+async def database_time(session: AsyncSession) -> datetime:
+ return await session.scalar(select(func.clock_timestamp()))
+
+
+async def queued_backlog(session: AsyncSession) -> tuple[int, float]:
+ count, oldest = (
+ await session.execute(
+ select(func.count(), func.min(AttachmentJob.created_at))
+ .join(Attachment, Attachment.id == AttachmentJob.attachment_id)
+ .where(Attachment.status == "queued")
+ )
+ ).one()
+ return count, (
+ max(0, ((await database_time(session)) - oldest).total_seconds())
+ if oldest
+ else 0.0
+ )
+
+
+async def expired_jobs(session: AsyncSession, limit: int = 20):
+ result = await session.execute(
+ select(AttachmentJob, Attachment)
+ .join(Attachment, Attachment.id == AttachmentJob.attachment_id)
+ .where(
+ Attachment.status == "processing",
+ AttachmentJob.lease_until < func.clock_timestamp(),
+ )
+ .order_by(AttachmentJob.lease_until)
+ .limit(limit)
+ .with_for_update(of=AttachmentJob, skip_locked=True)
+ )
+ return result.all()
+
+
+def start_job(
+ job, attachment, token: UUID, now: datetime, lease_seconds: int
+) -> JobWork:
+ job.attempt += 1
+ job.lease_token = token
+ job.lease_until = now + timedelta(seconds=lease_seconds)
+ attachment.status = "processing"
+ attachment.updated_at = now
+ return JobWork(
+ job.id,
+ attachment.id,
+ token,
+ job.pipeline_version,
+ job.staging_key,
+ attachment.content_type,
+ attachment.size_bytes,
+ attachment.sha256,
+ )
+
+
+def record_failure(
+ job,
+ attachment,
+ now: datetime,
+ error_code: str,
+ *,
+ terminal: bool,
+ retry_seconds: float,
+) -> None:
+ job.lease_token = None
+ job.lease_until = None
+ job.last_error_code = error_code
+ job.next_attempt_at = now + timedelta(seconds=retry_seconds)
+ job.finished_at = now if terminal else None
+ attachment.status = "failed" if terminal else "queued"
+ attachment.error_code = error_code if terminal else None
+ attachment.updated_at = now
+
+
+def complete_job(
+ job, attachment, work: JobWork, variants: list[dict], now: datetime
+) -> None:
+ attachment.object_key = work.original_key
+ attachment.thumbnail_key = work.variant_key("thumbnail")
+ attachment.crop_key = work.variant_key("crop")
+ attachment.optimized_key = work.variant_key("optimized")
+ attachment.variants = variants
+ attachment.status = "ready"
+ attachment.error_code = None
+ attachment.updated_at = now
+ job.lease_token = None
+ job.lease_until = None
+ job.last_error_code = None
+ job.last_error_permanent = None
+ job.finished_at = now
+
+
+async def terminal_staging(
+ session: AsyncSession, retention: int, failed_retention: int, limit: int = 20
+):
+ result = await session.execute(
+ select(AttachmentJob, Attachment)
+ .join(Attachment, Attachment.id == AttachmentJob.attachment_id)
+ .where(
+ AttachmentJob.staging_key.is_not(None),
+ or_(
+ and_(
+ Attachment.status == "ready",
+ AttachmentJob.finished_at
+ < func.clock_timestamp() - timedelta(seconds=retention),
+ ),
+ and_(
+ Attachment.status == "failed",
+ AttachmentJob.finished_at
+ < func.clock_timestamp() - timedelta(seconds=failed_retention),
+ ),
+ ),
+ )
+ .order_by(AttachmentJob.finished_at)
+ .limit(limit)
+ .with_for_update(of=AttachmentJob, skip_locked=True)
+ )
+ return result.all()
+
+
+def clear_staging(job) -> None:
+ job.staging_key = None
diff --git a/app/storage/messages.py b/app/storage/messages.py
new file mode 100644
index 0000000..2da9ffa
--- /dev/null
+++ b/app/storage/messages.py
@@ -0,0 +1,59 @@
+from uuid import UUID
+
+from sqlalchemy import select, tuple_
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.counter import Counter
+from app.models.message import Message
+from app.storage.types import Position
+
+MESSAGES_COUNTER = "messages"
+
+
+async def get_message(session: AsyncSession, message_id: UUID) -> Message | None:
+ return await session.get(Message, message_id)
+
+
+async def create_message(
+ session: AsyncSession,
+ *,
+ chat_id: UUID,
+ sender_id: UUID,
+ text: str,
+) -> Message:
+ message = Message(chat_id=chat_id, sender_id=sender_id, text=text)
+ session.add(message)
+ await session.flush()
+ return message
+
+
+async def list_messages(
+ session: AsyncSession,
+ chat_id: UUID,
+ after: Position | None,
+ limit: int,
+) -> list[Message]:
+ stmt = select(Message).where(Message.chat_id == chat_id)
+ if after is not None:
+ stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
+ stmt = stmt.order_by(Message.created_at, Message.id).limit(limit)
+ result = await session.execute(stmt)
+ messages = list(result.scalars().all())
+ return messages
+
+
+async def reserve_message_slot(session: AsyncSession, max_messages: int) -> bool:
+ stmt = (
+ insert(Counter)
+ .values(name=MESSAGES_COUNTER, value=1)
+ .on_conflict_do_update(
+ index_elements=[Counter.name],
+ set_={"value": Counter.value + 1},
+ where=Counter.value < max_messages,
+ )
+ .returning(Counter.value)
+ )
+ result = await session.execute(stmt)
+ is_reserved = result.scalar_one_or_none() is not None
+ return is_reserved
diff --git a/app/storage/outbox.py b/app/storage/outbox.py
new file mode 100644
index 0000000..be50c31
--- /dev/null
+++ b/app/storage/outbox.py
@@ -0,0 +1,33 @@
+"""Jobs одновременно хранят намерение публикации очередной попытки."""
+
+from uuid import UUID
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.attachment import Attachment
+from app.models.job import AttachmentJob
+
+
+async def next_unpublished_job(
+ session: AsyncSession, *, attachment_id: UUID | None = None
+) -> AttachmentJob | None:
+ statement = (
+ select(AttachmentJob)
+ .join(Attachment, Attachment.id == AttachmentJob.attachment_id)
+ .where(
+ Attachment.status == "queued",
+ AttachmentJob.next_attempt_at <= func.clock_timestamp(),
+ AttachmentJob.published_attempt <= AttachmentJob.attempt,
+ )
+ .order_by(AttachmentJob.next_attempt_at, AttachmentJob.id)
+ .limit(1)
+ .with_for_update(of=AttachmentJob, skip_locked=True)
+ )
+ if attachment_id is not None:
+ statement = statement.where(AttachmentJob.attachment_id == attachment_id)
+ return await session.scalar(statement)
+
+
+def mark_published(job: AttachmentJob) -> None:
+ job.published_attempt = job.attempt + 1
diff --git a/app/storage/s3.py b/app/storage/s3.py
new file mode 100644
index 0000000..b328ea2
--- /dev/null
+++ b/app/storage/s3.py
@@ -0,0 +1,180 @@
+"""MinIO SDK выполняется в потоках; ресурсы живут до конца операции."""
+
+import asyncio
+from collections.abc import Callable
+from contextlib import suppress
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path
+from typing import TypeVar
+import time
+from urllib.parse import urlsplit
+
+from anyio import CancelScope
+from minio import Minio
+import urllib3
+
+from app.core.s3 import S3Settings
+
+T = TypeVar("T")
+
+
+async def run_sync(
+ operation: Callable[..., T], *args, discard_result=None, **kwargs
+) -> T:
+ """При отмене дождаться потока: иначе PUT может догнать компенсацию удаления."""
+ task = asyncio.create_task(asyncio.to_thread(operation, *args, **kwargs))
+ try:
+ result = await asyncio.shield(task)
+ except asyncio.CancelledError:
+ with CancelScope(shield=True):
+ await settle_cancelled_call(task, discard_result)
+ raise
+ return result
+
+
+async def settle_cancelled_call(
+ task: asyncio.Task, discard_result: Callable | None
+) -> None:
+ """wait не переносит исключение SDK; повторная отмена не прерывает компенсацию."""
+ while not task.done():
+ with suppress(asyncio.CancelledError):
+ await asyncio.wait([task])
+ completed = not task.cancelled() and task.exception() is None
+ if completed and discard_result is not None:
+ discard_result(task.result())
+
+
+@dataclass(frozen=True)
+class ObjectInfo:
+ """Поля S3 сохраняют необязательность SDK; согласованность проверяет domain."""
+
+ key: str
+ size_bytes: int | None
+ content_type: str | None
+ last_modified: datetime | None
+
+
+def close_response(response) -> None:
+ response.close()
+ response.release_conn()
+
+
+class ObjectStream:
+ """Закрывается даже если StreamingResponse не успел начать итерацию."""
+
+ def __init__(self, storage, response):
+ self.storage = storage
+ self.response = response
+ self.deadline = time.monotonic() + storage.settings.download_timeout_seconds
+ self.closed = False
+
+ def __aiter__(self):
+ return self
+
+ async def __anext__(self):
+ if self.closed:
+ raise StopAsyncIteration
+ try:
+ remaining = self.deadline - time.monotonic()
+ if remaining <= 0:
+ raise TimeoutError
+ async with asyncio.timeout(remaining):
+ chunk = await self.storage._call(
+ lambda: self.response.read(self.storage.settings.chunk_bytes)
+ )
+ if not chunk:
+ await self.aclose()
+ raise StopAsyncIteration
+ except BaseException:
+ await self.aclose()
+ raise
+ return chunk
+
+ async def aclose(self):
+ if not self.closed:
+ self.closed = True
+ close_response(self.response)
+
+
+class MinioStorage:
+ def __init__(self, settings: S3Settings) -> None:
+ self.settings = settings
+
+ url = urlsplit(settings.endpoint)
+ self._http = urllib3.PoolManager(
+ timeout=urllib3.Timeout(
+ connect=settings.connect_timeout_seconds,
+ read=settings.read_timeout_seconds,
+ ),
+ retries=urllib3.Retry(total=0),
+ maxsize=32,
+ )
+ self._health_http = urllib3.PoolManager(
+ timeout=urllib3.Timeout(connect=0.25, read=0.5),
+ retries=urllib3.Retry(total=0),
+ )
+ self._client = Minio(
+ url.netloc,
+ secure=url.scheme == "https",
+ region=settings.region,
+ access_key=settings.access_key_id_file.read_text().strip(),
+ secret_key=settings.secret_access_key_file.read_text().strip(),
+ http_client=self._http,
+ )
+
+ async def _call(self, operation: Callable, *, discard_result=None):
+ return await run_sync(operation, discard_result=discard_result)
+
+ async def put_file(
+ self, key: str, path: Path, *, size_bytes: int, content_type: str, sha256: str
+ ) -> None:
+ def put():
+ with path.open("rb") as source:
+
+ self._client.put_object(
+ self.settings.bucket,
+ key,
+ source,
+ size_bytes,
+ content_type=content_type,
+ metadata={"sha256": sha256},
+ part_size=10_485_760,
+ num_parallel_uploads=1,
+ )
+
+ await self._call(put)
+
+ async def open_object(self, key: str) -> ObjectStream:
+ response = await self._call(
+ lambda: self._client.get_object(self.settings.bucket, key),
+ discard_result=close_response,
+ )
+ return ObjectStream(self, response)
+
+ async def stat_object(self, key: str) -> ObjectInfo:
+ def stat():
+ info = self._client.stat_object(self.settings.bucket, key)
+ return ObjectInfo(key, info.size, info.content_type, info.last_modified)
+
+ return await self._call(stat)
+
+ async def delete_object(self, key: str) -> None:
+ await self._call(lambda: self._client.remove_object(self.settings.bucket, key))
+
+ async def check_available(self) -> bool:
+ status = await self._call(
+ lambda: self._health_http.request(
+ "GET",
+ self.settings.endpoint.rstrip("/") + "/minio/health/ready",
+ ).status
+ )
+ return status == 200
+
+ async def aclose(self) -> None:
+ self._http.clear()
+ self._health_http.clear()
+
+
+def create_s3_storage(settings: S3Settings) -> MinioStorage:
+ return MinioStorage(settings)
diff --git a/app/storage/sessions.py b/app/storage/sessions.py
new file mode 100644
index 0000000..f845562
--- /dev/null
+++ b/app/storage/sessions.py
@@ -0,0 +1,34 @@
+from datetime import datetime
+from uuid import UUID
+
+from redis.asyncio import Redis
+
+SESSION_KEY_PREFIX = "session:"
+
+
+def session_key(token_hash: str) -> str:
+ return f"{SESSION_KEY_PREFIX}{token_hash}"
+
+
+async def create_session(
+ redis: Redis,
+ *,
+ token_hash: str,
+ user_id: UUID,
+ expires_at: datetime,
+) -> None:
+ await redis.set(session_key(token_hash), str(user_id), pxat=expires_at)
+
+
+async def get_session_user_id(redis: Redis, token_hash: str) -> UUID | None:
+ raw_user_id = await redis.get(session_key(token_hash))
+ if isinstance(raw_user_id, bytes):
+ raw_user_id = raw_user_id.decode()
+ user_id = UUID(raw_user_id) if raw_user_id else None
+ return user_id
+
+
+async def delete_session(redis: Redis, token_hash: str) -> bool:
+ deleted_count = await redis.delete(session_key(token_hash))
+ is_deleted = deleted_count > 0
+ return is_deleted
diff --git a/app/storage/types.py b/app/storage/types.py
new file mode 100644
index 0000000..e0357cf
--- /dev/null
+++ b/app/storage/types.py
@@ -0,0 +1,6 @@
+"""Общие типы слоя хранения."""
+
+from datetime import datetime
+from uuid import UUID
+
+Position = tuple[datetime, UUID]
diff --git a/app/storage/users.py b/app/storage/users.py
new file mode 100644
index 0000000..ae5c719
--- /dev/null
+++ b/app/storage/users.py
@@ -0,0 +1,43 @@
+from collections.abc import Collection
+from uuid import UUID
+
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.user import User
+
+
+async def create_user(
+ session: AsyncSession,
+ *,
+ username: str,
+ display_name: str,
+ password_hash: str,
+) -> User:
+ user = User(
+ username=username, display_name=display_name, password_hash=password_hash
+ )
+ session.add(user)
+ await session.flush()
+ return user
+
+
+async def get_user_by_id(session: AsyncSession, user_id: UUID) -> User | None:
+ user = await session.get(User, user_id)
+ return user
+
+
+async def get_user_by_username(session: AsyncSession, username: str) -> User | None:
+ stmt = select(User).where(User.username == username)
+ result = await session.execute(stmt)
+ user = result.scalar_one_or_none()
+ return user
+
+
+async def get_existing_user_ids(
+ session: AsyncSession, user_ids: Collection[UUID]
+) -> set[UUID]:
+ stmt = select(User.id).where(User.id.in_(user_ids))
+ result = await session.execute(stmt)
+ existing_ids = set(result.scalars().all())
+ return existing_ids
diff --git a/compose.test.yaml b/compose.test.yaml
new file mode 100644
index 0000000..d4c5a70
--- /dev/null
+++ b/compose.test.yaml
@@ -0,0 +1,7 @@
+services:
+ postgres:
+ ports:
+ - "127.0.0.1:25432:5432"
+ redis:
+ ports:
+ - "127.0.0.1:26379:6379"
diff --git a/compose.yaml b/compose.yaml
new file mode 100644
index 0000000..826d509
--- /dev/null
+++ b/compose.yaml
@@ -0,0 +1,515 @@
+name: messenger
+
+x-api-image: &api-image
+ build:
+ context: .
+ args:
+ API_BUILD_REVISION: ${API_BUILD_REVISION:?Set API_BUILD_REVISION in .env}
+ image: ${API_IMAGE:?Set API_IMAGE in .env}
+
+x-base-env: &base-env
+ APP_HOST: ${APP_HOST:?Set APP_HOST in .env}
+ APP_PORT: ${APP_PORT:?Set APP_PORT in .env}
+ APP_ENV: ${APP_ENV:?Set APP_ENV in .env}
+ OTEL_ENABLED: ${OTEL_ENABLED:-true}
+ OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
+ LOG_LEVEL: ${LOG_LEVEL:?Set LOG_LEVEL in .env}
+ MAX_MESSAGES: ${MAX_MESSAGES:?Set MAX_MESSAGES in .env}
+ MAX_IN_FLIGHT: ${MAX_IN_FLIGHT:?Set MAX_IN_FLIGHT in .env}
+ DB_HOST: ${DB_HOST:?Set DB_HOST in .env}
+ DB_PORT: ${DB_PORT:?Set DB_PORT in .env}
+ DB_NAME: ${DB_NAME:?Set DB_NAME in .env}
+ DB_USER: ${DB_USER:?Set DB_USER in .env}
+ DB_POOL_SIZE: ${DB_POOL_SIZE:?Set DB_POOL_SIZE in .env}
+ DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:?Set DB_MAX_OVERFLOW in .env}
+ DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:?Set DB_POOL_TIMEOUT_SECONDS in .env}
+ DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:?Set DB_CONNECT_TIMEOUT_SECONDS in .env}
+ DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:?Set DB_STATEMENT_TIMEOUT_MS in .env}
+ SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:?Set SHUTDOWN_DRAIN_SECONDS in .env}
+ GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS: ${GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS:?Set GRACEFUL_SHUTDOWN_TIMEOUT_SECONDS in .env}
+ READINESS_TIMEOUT_SECONDS: ${READINESS_TIMEOUT_SECONDS:?Set READINESS_TIMEOUT_SECONDS in .env}
+
+x-s3-env: &s3-env
+ S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
+ S3_REGION: ${S3_REGION:?Set S3_REGION in .env}
+ S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
+ S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
+ S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
+ S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:?Set S3_CONNECT_TIMEOUT_SECONDS in .env}
+ S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:?Set S3_READ_TIMEOUT_SECONDS in .env}
+ S3_UPLOAD_TIMEOUT_SECONDS: ${S3_UPLOAD_TIMEOUT_SECONDS:?Set S3_UPLOAD_TIMEOUT_SECONDS in .env}
+ S3_DOWNLOAD_TIMEOUT_SECONDS: ${S3_DOWNLOAD_TIMEOUT_SECONDS:?Set S3_DOWNLOAD_TIMEOUT_SECONDS in .env}
+ S3_IMAGE_TIMEOUT_SECONDS: ${S3_IMAGE_TIMEOUT_SECONDS:?Set S3_IMAGE_TIMEOUT_SECONDS in .env}
+ S3_MAX_UPLOAD_BYTES: ${S3_MAX_UPLOAD_BYTES:?Set S3_MAX_UPLOAD_BYTES in .env}
+ S3_MAX_IMAGE_PIXELS: ${S3_MAX_IMAGE_PIXELS:?Set S3_MAX_IMAGE_PIXELS in .env}
+ S3_CHUNK_BYTES: ${S3_CHUNK_BYTES:?Set S3_CHUNK_BYTES in .env}
+ S3_MAX_CONCURRENT_UPLOADS: ${S3_MAX_CONCURRENT_UPLOADS:?Set S3_MAX_CONCURRENT_UPLOADS in .env}
+ S3_CLEANUP_GRACE_SECONDS: ${S3_CLEANUP_GRACE_SECONDS:?Set S3_CLEANUP_GRACE_SECONDS in .env}
+ S3_CLEANUP_INTERVAL_SECONDS: ${S3_CLEANUP_INTERVAL_SECONDS:?Set S3_CLEANUP_INTERVAL_SECONDS in .env}
+
+x-jobs-env: &jobs-env
+ BROKER_HOST: ${BROKER_HOST:?Set BROKER_HOST in .env}
+ BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
+ BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
+ BROKER_QUEUE: ${BROKER_QUEUE:?Set BROKER_QUEUE in .env}
+ BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
+ BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
+ WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:?Set WORKER_CONCURRENCY in .env}
+ JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:?Set JOB_MAX_ATTEMPTS in .env}
+ JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:?Set JOB_TIMEOUT_SECONDS in .env}
+ JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:?Set JOB_LEASE_SECONDS in .env}
+ JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:?Set JOB_RETRY_MAX_SECONDS in .env}
+ JOB_RETRY_SECONDS: ${JOB_RETRY_SECONDS:?Set JOB_RETRY_SECONDS in .env}
+ PUBLISHER_POLL_SECONDS: ${PUBLISHER_POLL_SECONDS:?Set PUBLISHER_POLL_SECONDS in .env}
+ FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
+ STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
+ IMAGE_JPEG_QUALITY: ${IMAGE_JPEG_QUALITY:?Set IMAGE_JPEG_QUALITY in .env}
+ IMAGE_WEBP_QUALITY: ${IMAGE_WEBP_QUALITY:?Set IMAGE_WEBP_QUALITY in .env}
+ IMAGE_WEBP_METHOD: ${IMAGE_WEBP_METHOD:?Set IMAGE_WEBP_METHOD in .env}
+ IMAGE_MEMORY_MIB: ${IMAGE_MEMORY_MIB:?Set IMAGE_MEMORY_MIB in .env}
+ IMAGE_CPU_SECONDS: ${IMAGE_CPU_SECONDS:?Set IMAGE_CPU_SECONDS in .env}
+
+services:
+ minio:
+ build:
+ context: .
+ dockerfile: infra/minio/Dockerfile
+ target: server
+ image: ${MINIO_IMAGE:-messenger-minio:local}
+ command: ["server", "/data", "--console-address", "127.0.0.1:9001"]
+ restart: unless-stopped
+ environment:
+ MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
+ MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
+ MINIO_BROWSER: ${MINIO_BROWSER:?Set MINIO_BROWSER in .env}
+ secrets:
+ - source: minio_root_user
+ target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
+ - source: minio_root_password
+ target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
+ volumes:
+ - miniodata:/data
+ networks: [objects]
+ healthcheck:
+ test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:9000/minio/health/ready', timeout=2)"]
+ interval: 3s
+ timeout: 3s
+ retries: 10
+ deploy:
+ resources:
+ limits:
+ cpus: "1.0"
+ memory: 512M
+
+ minio-bootstrap:
+ build:
+ context: .
+ dockerfile: infra/minio/Dockerfile
+ target: bootstrap
+ image: ${MINIO_BOOTSTRAP_IMAGE:-messenger-minio-bootstrap:local}
+ environment:
+ S3_ENDPOINT: ${S3_ENDPOINT:?Set S3_ENDPOINT in .env}
+ S3_BUCKET: ${S3_BUCKET:?Set S3_BUCKET in .env}
+ S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
+ S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
+ MINIO_ROOT_USER_FILE: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
+ MINIO_ROOT_PASSWORD_FILE: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
+ secrets:
+ - source: minio_root_user
+ target: ${MINIO_ROOT_USER_FILE:?Set MINIO_ROOT_USER_FILE in .env}
+ - source: minio_root_password
+ target: ${MINIO_ROOT_PASSWORD_FILE:?Set MINIO_ROOT_PASSWORD_FILE in .env}
+ - source: s3_access_key_id
+ target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
+ - source: s3_secret_access_key
+ target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
+ networks: [objects]
+ restart: "no"
+ depends_on:
+ minio:
+ condition: service_healthy
+ deploy:
+ resources:
+ limits:
+ cpus: "0.25"
+ memory: 128M
+
+ postgres:
+ image: postgres:16.10-bookworm
+ restart: unless-stopped
+ environment:
+ POSTGRES_USER: ${DB_USER:?Set DB_USER in .env}
+ POSTGRES_DB: ${DB_NAME:?Set DB_NAME in .env}
+ POSTGRES_PASSWORD_FILE: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env}
+ secrets:
+ - source: db_password
+ target: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE in .env}
+ volumes:
+ - pgdata:/var/lib/postgresql/data
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+ interval: 3s
+ timeout: 3s
+ retries: 10
+ deploy:
+ resources:
+ limits:
+ cpus: "1.0"
+ memory: 512M
+
+ redis:
+ image: redis:7.4.2-bookworm
+ restart: unless-stopped
+ entrypoint: ["/bin/sh", "/usr/local/bin/start-redis.sh"]
+ secrets:
+ - redis_password
+ volumes:
+ - ./infra/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
+ - ./infra/redis/start-redis.sh:/usr/local/bin/start-redis.sh:ro
+ - redisdata:/data
+ healthcheck:
+ test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping"]
+ interval: 3s
+ timeout: 3s
+ retries: 10
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 512M
+
+ broker:
+ image: rabbitmq:4.4.0
+ hostname: broker
+ restart: unless-stopped
+ entrypoint: ["/bin/sh", "/usr/local/bin/start-broker.sh"]
+ environment:
+ BROKER_PORT: ${BROKER_PORT:?Set BROKER_PORT in .env}
+ BROKER_USER: ${BROKER_USER:?Set BROKER_USER in .env}
+ BROKER_VHOST: ${BROKER_VHOST:?Set BROKER_VHOST in .env}
+ BROKER_PASSWORD_FILE: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
+ secrets:
+ - source: broker_password
+ target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
+ volumes:
+ - ./infra/broker/start-broker.sh:/usr/local/bin/start-broker.sh:ro
+ - rabbitmqdata:/var/lib/rabbitmq
+ healthcheck:
+ test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
+ interval: 5s
+ timeout: 5s
+ retries: 20
+ start_period: 20s
+ deploy:
+ resources:
+ limits:
+ cpus: "1.0"
+ memory: 768M
+
+ publisher:
+ <<: *api-image
+ init: true
+ command: ["python", "-m", "app.core.outbox_publisher"]
+ restart: unless-stopped
+ environment:
+ <<: [*base-env, *jobs-env]
+ secrets:
+ - db_password
+ - source: broker_password
+ target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
+ depends_on:
+ postgres:
+ condition: service_healthy
+ broker:
+ condition: service_healthy
+ migrate:
+ condition: service_completed_successfully
+ healthcheck:
+ disable: true
+ stop_grace_period: 15s
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 256M
+
+ worker:
+ <<: *api-image
+ init: true
+ command: ["python", "-m", "app.core.job_worker"]
+ restart: unless-stopped
+ environment:
+ <<: [*base-env, *s3-env, *jobs-env]
+ secrets:
+ - db_password
+ - source: broker_password
+ target: ${BROKER_PASSWORD_FILE:?Set BROKER_PASSWORD_FILE in .env}
+ - source: s3_access_key_id
+ target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
+ - source: s3_secret_access_key
+ target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
+ networks: [default, objects]
+ tmpfs:
+ - /tmp:rw,noexec,nosuid,size=128m,mode=1777
+ depends_on:
+ postgres:
+ condition: service_healthy
+ broker:
+ condition: service_healthy
+ minio:
+ condition: service_healthy
+ minio-bootstrap:
+ condition: service_completed_successfully
+ migrate:
+ condition: service_completed_successfully
+ healthcheck:
+ disable: true
+ stop_signal: SIGTERM
+ stop_grace_period: 20s
+ deploy:
+ resources:
+ limits:
+ cpus: "2.0"
+ memory: 2048M
+
+ migrate:
+ <<: *api-image
+ command: ["alembic", "upgrade", "head"]
+ environment: *base-env
+ secrets:
+ - db_password
+ depends_on:
+ postgres:
+ condition: service_healthy
+ restart: "no"
+ healthcheck:
+ disable: true
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 256M
+
+ api:
+ <<: *api-image
+ restart: unless-stopped
+ environment:
+ <<: [*base-env, *s3-env]
+ REDIS_HOST: ${REDIS_HOST:?Set REDIS_HOST in .env}
+ REDIS_PORT: ${REDIS_PORT:?Set REDIS_PORT in .env}
+ REDIS_DB: ${REDIS_DB:?Set REDIS_DB in .env}
+ REDIS_MAX_CONNECTIONS: ${REDIS_MAX_CONNECTIONS:?Set REDIS_MAX_CONNECTIONS in .env}
+ REDIS_POOL_TIMEOUT_SECONDS: ${REDIS_POOL_TIMEOUT_SECONDS:?Set REDIS_POOL_TIMEOUT_SECONDS in .env}
+ REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:?Set REDIS_CONNECT_TIMEOUT_SECONDS in .env}
+ REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:?Set REDIS_SOCKET_TIMEOUT_SECONDS in .env}
+ SESSION_STORE: ${SESSION_STORE:?Set SESSION_STORE in .env}
+ SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:?Set SESSION_TTL_SECONDS in .env}
+ IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:?Set IDEMPOTENCY_TTL_SECONDS in .env}
+ secrets:
+ - db_password
+ - redis_password
+ - source: s3_access_key_id
+ target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
+ - source: s3_secret_access_key
+ target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
+ networks: [default, objects]
+ tmpfs:
+ - /tmp:rw,noexec,nosuid,size=64m,mode=1777
+ deploy:
+ replicas: ${API_REPLICAS:?Set API_REPLICAS in .env}
+ resources:
+ limits:
+ cpus: "1.0"
+ memory: 1536M
+ healthcheck:
+ test: ["CMD", "python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.environ['APP_PORT'] + '/health/ready', timeout=2.5)"]
+ interval: 3s
+ timeout: 3s
+ start_period: 10s
+ retries: 2
+ stop_signal: SIGTERM
+ stop_grace_period: 8s
+ depends_on:
+ minio:
+ condition: service_healthy
+ minio-bootstrap:
+ condition: service_completed_successfully
+ postgres:
+ condition: service_healthy
+ redis:
+ condition: service_healthy
+ migrate:
+ condition: service_completed_successfully
+
+ cleanup:
+ <<: *api-image
+ command: ["python", "-m", "app.core.cleanup", "--apply"]
+ restart: unless-stopped
+ environment:
+ <<: [*base-env, *s3-env]
+ FAILED_STAGING_RETENTION_SECONDS: ${FAILED_STAGING_RETENTION_SECONDS:?Set FAILED_STAGING_RETENTION_SECONDS in .env}
+ STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:?Set STAGING_RETENTION_SECONDS in .env}
+ secrets:
+ - db_password
+ - source: s3_access_key_id
+ target: ${S3_ACCESS_KEY_ID_FILE:?Set S3_ACCESS_KEY_ID_FILE in .env}
+ - source: s3_secret_access_key
+ target: ${S3_SECRET_ACCESS_KEY_FILE:?Set S3_SECRET_ACCESS_KEY_FILE in .env}
+ networks: [default, objects]
+ healthcheck:
+ disable: true
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ minio-bootstrap:
+ condition: service_completed_successfully
+ deploy:
+ resources:
+ limits:
+ cpus: "0.25"
+ memory: 256M
+
+ nginx:
+ image: nginx:1.28.0-alpine
+ restart: unless-stopped
+ volumes:
+ - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+ ports:
+ - "127.0.0.1:${LB_HOST_PORT:?Set LB_HOST_PORT in .env}:8080"
+ healthcheck:
+ test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/lb-health"]
+ interval: 5s
+ timeout: 3s
+ retries: 5
+ depends_on:
+ api:
+ condition: service_healthy
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 128M
+
+
+ otel-collector:
+ image: otel/opentelemetry-collector-contrib:0.165.0
+ command: ["--config=/etc/otelcol/config.yaml"]
+ restart: unless-stopped
+ volumes:
+ - ./infra/otel/collector.yaml:/etc/otelcol/config.yaml:ro
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 256M
+
+ prometheus:
+ image: prom/prometheus:v3.16.0
+ command: ["--config.file=/etc/prometheus/prometheus.yml", "--storage.tsdb.retention.time=3d"]
+ restart: unless-stopped
+ volumes:
+ - ./infra/otel/prometheus.yaml:/etc/prometheus/prometheus.yml:ro
+ - ./infra/otel/alerts.yaml:/etc/prometheus/alerts.yaml:ro
+ - prometheusdata:/prometheus
+ healthcheck:
+ test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9090/-/ready"]
+ interval: 5s
+ timeout: 3s
+ retries: 20
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 512M
+
+ loki:
+ image: grafana/loki:3.7.10
+ command: ["-config.file=/etc/loki/config.yaml"]
+ restart: unless-stopped
+ volumes:
+ - ./infra/otel/loki.yaml:/etc/loki/config.yaml:ro
+ - lokidata:/loki
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 512M
+
+ tempo:
+ image: grafana/tempo:2.9.0
+ user: "0:0"
+ command: ["-config.file=/etc/tempo/config.yaml"]
+ restart: unless-stopped
+ volumes:
+ - ./infra/otel/tempo.yaml:/etc/tempo/config.yaml:ro
+ - tempodata:/var/tempo
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 512M
+
+ grafana:
+ image: grafana/grafana:13.3.1
+ restart: unless-stopped
+ environment:
+ GF_AUTH_ANONYMOUS_ENABLED: "true"
+ GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
+ GF_SECURITY_ADMIN_PASSWORD__FILE: /run/secrets/grafana_admin_password
+ GF_USERS_ALLOW_SIGN_UP: "false"
+ GF_ANALYTICS_REPORTING_ENABLED: "false"
+ GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+ secrets:
+ - grafana_admin_password
+ ports:
+ - "127.0.0.1:${GRAFANA_HOST_PORT:-13000}:3000"
+ volumes:
+ - ./infra/otel/grafana/provisioning:/etc/grafana/provisioning:ro
+ - ./infra/otel/grafana/dashboards:/var/lib/grafana/dashboards:ro
+ - grafanadata:/var/lib/grafana
+ healthcheck:
+ # Loki/Tempo/Collector не содержат shell/HTTP-клиент: проверяем их из Grafana.
+ test:
+ - CMD-SHELL
+ - wget -q -T 2 -O /dev/null http://127.0.0.1:3000/api/health && wget -q -T 2 -O /dev/null http://loki:3100/ready && wget -q -T 2 -O /dev/null http://tempo:3200/ready && wget -q -T 2 -O /dev/null http://otel-collector:13133/
+ interval: 5s
+ timeout: 10s
+ retries: 36
+ deploy:
+ resources:
+ limits:
+ cpus: "0.5"
+ memory: 512M
+
+volumes:
+ pgdata:
+ redisdata:
+ miniodata:
+ rabbitmqdata:
+ prometheusdata:
+ lokidata:
+ tempodata:
+ grafanadata:
+
+networks:
+ default:
+ objects:
+ internal: true
+
+secrets:
+ grafana_admin_password:
+ file: ./secrets/grafana_admin_password
+ broker_password:
+ file: ./secrets/broker_password
+ db_password:
+ file: ./secrets/db_password
+ redis_password:
+ file: ./secrets/redis_password
+ minio_root_user:
+ file: ./secrets/minio_root_user
+ minio_root_password:
+ file: ./secrets/minio_root_password
+ s3_access_key_id:
+ file: ./secrets/s3_access_key_id
+ s3_secret_access_key:
+ file: ./secrets/s3_secret_access_key
diff --git a/evidence/lab6.zip b/evidence/lab6.zip
new file mode 100644
index 0000000..8b11878
Binary files /dev/null and b/evidence/lab6.zip differ
diff --git a/evidence/lab6/00_init.txt b/evidence/lab6/00_init.txt
new file mode 100644
index 0000000..edd583a
--- /dev/null
+++ b/evidence/lab6/00_init.txt
@@ -0,0 +1,10 @@
+$ python scripts/init_local.py
+Создан файл secrets/db_password
+Создан файл secrets/redis_password
+Создан файл secrets/broker_password
+Создан файл secrets/minio_root_user
+Создан файл secrets/minio_root_password
+Создан файл secrets/s3_access_key_id
+Создан файл secrets/s3_secret_access_key
+Создан файл secrets/grafana_admin_password
+Настройки готовы; существующие значения сохранены
diff --git a/evidence/lab6/01_contract.txt b/evidence/lab6/01_contract.txt
new file mode 100644
index 0000000..9180d64
--- /dev/null
+++ b/evidence/lab6/01_contract.txt
@@ -0,0 +1,2 @@
+$ python scripts/check_contract.py
+PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
diff --git a/evidence/lab6/02_openapi.txt b/evidence/lab6/02_openapi.txt
new file mode 100644
index 0000000..9194e40
--- /dev/null
+++ b/evidence/lab6/02_openapi.txt
@@ -0,0 +1,2 @@
+$ python scripts/validate_openapi.py
+PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
diff --git a/evidence/lab6/03_import_images.txt b/evidence/lab6/03_import_images.txt
new file mode 100644
index 0000000..7f0084f
--- /dev/null
+++ b/evidence/lab6/03_import_images.txt
@@ -0,0 +1,13 @@
+$ docker image load --input /tmp/images.tar
+Loaded image: messenger-check-30e5e3483b9c:api
+Loaded image: messenger-check-30e5e3483b9c:bootstrap
+Loaded image: grafana/grafana:13.3.1
+Loaded image: grafana/loki:3.7.10
+Loaded image: grafana/tempo:2.9.0
+Loaded image: nginx:1.28.0-alpine
+Loaded image: rabbitmq:4.4.0
+Loaded image: messenger-check-30e5e3483b9c:server
+Loaded image: otel/opentelemetry-collector-contrib:0.165.0
+Loaded image: postgres:16.10-bookworm
+Loaded image: prom/prometheus:v3.16.0
+Loaded image: redis:7.4.2-bookworm
diff --git a/evidence/lab6/03_readiness.txt b/evidence/lab6/03_readiness.txt
new file mode 100644
index 0000000..1b9c01c
--- /dev/null
+++ b/evidence/lab6/03_readiness.txt
@@ -0,0 +1,7 @@
+$ docker compose --env-file .env up -d --no-deps --wait --wait-timeout 240 nginx grafana
+ Container messenger-test-nginx-1 Running
+ Container messenger-test-grafana-1 Running
+ Container messenger-test-nginx-1 Waiting
+ Container messenger-test-grafana-1 Waiting
+ Container messenger-test-nginx-1 Healthy
+ Container messenger-test-grafana-1 Healthy
diff --git a/evidence/lab6/03_startup.txt b/evidence/lab6/03_startup.txt
new file mode 100644
index 0000000..5c211cd
--- /dev/null
+++ b/evidence/lab6/03_startup.txt
@@ -0,0 +1,127 @@
+$ docker compose --env-file .env up -d --no-build --pull never
+ Network messenger-test_default Creating
+ Network messenger-test_default Created
+ Network messenger-test_objects Creating
+ Network messenger-test_objects Created
+ Volume messenger-test_miniodata Creating
+ Volume messenger-test_miniodata Created
+ Volume messenger-test_lokidata Creating
+ Volume messenger-test_lokidata Created
+ Volume messenger-test_grafanadata Creating
+ Volume messenger-test_grafanadata Created
+ Volume messenger-test_prometheusdata Creating
+ Volume messenger-test_prometheusdata Created
+ Volume messenger-test_tempodata Creating
+ Volume messenger-test_tempodata Created
+ Volume messenger-test_rabbitmqdata Creating
+ Volume messenger-test_rabbitmqdata Created
+ Volume messenger-test_redisdata Creating
+ Volume messenger-test_redisdata Created
+ Volume messenger-test_pgdata Creating
+ Volume messenger-test_pgdata Created
+ Container messenger-test-grafana-1 Creating
+ Container messenger-test-tempo-1 Creating
+ Container messenger-test-broker-1 Creating
+ Container messenger-test-prometheus-1 Creating
+ Container messenger-test-postgres-1 Creating
+ Container messenger-test-minio-1 Creating
+ Container messenger-test-redis-1 Creating
+ Container messenger-test-otel-collector-1 Creating
+ Container messenger-test-loki-1 Creating
+ Container messenger-test-tempo-1 Created
+ Container messenger-test-postgres-1 Created
+ Container messenger-test-migrate-1 Creating
+ Container messenger-test-redis-1 Created
+ Container messenger-test-minio-1 Created
+ Container messenger-test-minio-bootstrap-1 Creating
+ Container messenger-test-otel-collector-1 Created
+ Container messenger-test-loki-1 Created
+ Container messenger-test-broker-1 Created
+ Container messenger-test-prometheus-1 Created
+ Container messenger-test-grafana-1 Created
+ Container messenger-test-migrate-1 Created
+ Container messenger-test-minio-bootstrap-1 Created
+ Container messenger-test-publisher-1 Creating
+ Container messenger-test-api-2 Creating
+ Container messenger-test-cleanup-1 Creating
+ Container messenger-test-worker-1 Creating
+ Container messenger-test-api-1 Creating
+ Container messenger-test-cleanup-1 Created
+ Container messenger-test-api-2 Created
+ Container messenger-test-publisher-1 Created
+ Container messenger-test-api-1 Created
+ Container messenger-test-nginx-1 Creating
+ Container messenger-test-worker-1 Created
+ Container messenger-test-nginx-1 Created
+ Container messenger-test-minio-1 Starting
+ Container messenger-test-prometheus-1 Starting
+ Container messenger-test-broker-1 Starting
+ Container messenger-test-tempo-1 Starting
+ Container messenger-test-loki-1 Starting
+ Container messenger-test-otel-collector-1 Starting
+ Container messenger-test-postgres-1 Starting
+ Container messenger-test-redis-1 Starting
+ Container messenger-test-grafana-1 Starting
+ Container messenger-test-minio-1 Started
+ Container messenger-test-minio-1 Waiting
+ Container messenger-test-tempo-1 Started
+ Container messenger-test-prometheus-1 Started
+ Container messenger-test-otel-collector-1 Started
+ Container messenger-test-grafana-1 Started
+ Container messenger-test-redis-1 Started
+ Container messenger-test-postgres-1 Started
+ Container messenger-test-postgres-1 Waiting
+ Container messenger-test-loki-1 Started
+ Container messenger-test-broker-1 Started
+ Container messenger-test-postgres-1 Healthy
+ Container messenger-test-migrate-1 Starting
+ Container messenger-test-migrate-1 Started
+ Container messenger-test-postgres-1 Waiting
+ Container messenger-test-broker-1 Waiting
+ Container messenger-test-migrate-1 Waiting
+ Container messenger-test-minio-1 Healthy
+ Container messenger-test-minio-bootstrap-1 Starting
+ Container messenger-test-postgres-1 Healthy
+ Container messenger-test-minio-bootstrap-1 Started
+ Container messenger-test-postgres-1 Waiting
+ Container messenger-test-broker-1 Waiting
+ Container messenger-test-minio-1 Waiting
+ Container messenger-test-minio-bootstrap-1 Waiting
+ Container messenger-test-migrate-1 Waiting
+ Container messenger-test-migrate-1 Waiting
+ Container messenger-test-minio-bootstrap-1 Waiting
+ Container messenger-test-postgres-1 Waiting
+ Container messenger-test-redis-1 Waiting
+ Container messenger-test-migrate-1 Waiting
+ Container messenger-test-minio-1 Waiting
+ Container messenger-test-minio-bootstrap-1 Waiting
+ Container messenger-test-minio-1 Healthy
+ Container messenger-test-postgres-1 Healthy
+ Container messenger-test-redis-1 Healthy
+ Container messenger-test-postgres-1 Healthy
+ Container messenger-test-minio-1 Healthy
+ Container messenger-test-broker-1 Healthy
+ Container messenger-test-broker-1 Healthy
+ Container messenger-test-migrate-1 Exited
+ Container messenger-test-migrate-1 Exited
+ Container messenger-test-migrate-1 Exited
+ Container messenger-test-publisher-1 Starting
+ Container messenger-test-migrate-1 Exited
+ Container messenger-test-publisher-1 Started
+ Container messenger-test-minio-bootstrap-1 Exited
+ Container messenger-test-worker-1 Starting
+ Container messenger-test-minio-bootstrap-1 Exited
+ Container messenger-test-api-1 Starting
+ Container messenger-test-minio-bootstrap-1 Exited
+ Container messenger-test-cleanup-1 Starting
+ Container messenger-test-api-1 Started
+ Container messenger-test-api-2 Starting
+ Container messenger-test-worker-1 Started
+ Container messenger-test-cleanup-1 Started
+ Container messenger-test-api-2 Started
+ Container messenger-test-api-1 Waiting
+ Container messenger-test-api-2 Waiting
+ Container messenger-test-api-1 Healthy
+ Container messenger-test-api-2 Healthy
+ Container messenger-test-nginx-1 Starting
+ Container messenger-test-nginx-1 Started
diff --git a/evidence/lab6/04_test_database.txt b/evidence/lab6/04_test_database.txt
new file mode 100644
index 0000000..e9aecb3
--- /dev/null
+++ b/evidence/lab6/04_test_database.txt
@@ -0,0 +1 @@
+$ docker compose --env-file .env exec -T postgres createdb -U messenger messenger_test
diff --git a/evidence/lab6/05_pytest.txt b/evidence/lab6/05_pytest.txt
new file mode 100644
index 0000000..3a500f5
--- /dev/null
+++ b/evidence/lab6/05_pytest.txt
@@ -0,0 +1,5 @@
+$ python -m pytest -q --junitxml=evidence/pytest.xml
+........................................................................ [ 34%]
+........................................................................ [ 69%]
+.............................................................. [100%]
+206 passed in 156.58s (0:02:36)
diff --git a/evidence/lab6/06_smoke.txt b/evidence/lab6/06_smoke.txt
new file mode 100644
index 0000000..80e646a
--- /dev/null
+++ b/evidence/lab6/06_smoke.txt
@@ -0,0 +1,2 @@
+$ python tests/smoke.py --base-url http://127.0.0.1:28080
+PASS: lab6 public API scenario; infrastructure and grading evidence remain separate.
diff --git a/evidence/lab6/07_observability.txt b/evidence/lab6/07_observability.txt
new file mode 100644
index 0000000..eccd43a
--- /dev/null
+++ b/evidence/lab6/07_observability.txt
@@ -0,0 +1,2 @@
+$ python -m scripts.check_observability --base-url http://127.0.0.1:28080 --grafana-url http://127.0.0.1:23000 --incidents --restart-telemetry
+PASS: telemetry, dashboard, privacy, correlations; alerts and outage recovery verified
diff --git a/evidence/lab6/08_persistence.txt b/evidence/lab6/08_persistence.txt
new file mode 100644
index 0000000..9d82836
--- /dev/null
+++ b/evidence/lab6/08_persistence.txt
@@ -0,0 +1,3 @@
+$ python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:28080
+Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
+PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
diff --git a/evidence/lab6/09_failover.txt b/evidence/lab6/09_failover.txt
new file mode 100644
index 0000000..618c399
--- /dev/null
+++ b/evidence/lab6/09_failover.txt
@@ -0,0 +1,4 @@
+$ python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:28080
+Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
+{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 4, "first_recovery_seconds": 2.813}
+PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
diff --git a/evidence/lab6/10_cleanup.txt b/evidence/lab6/10_cleanup.txt
new file mode 100644
index 0000000..8e16a74
--- /dev/null
+++ b/evidence/lab6/10_cleanup.txt
@@ -0,0 +1,89 @@
+$ docker compose --env-file .env down --volumes --remove-orphans
+ Container messenger-test-tempo-1 Stopping
+ Container messenger-test-publisher-1 Stopping
+ Container messenger-test-nginx-1 Stopping
+ Container messenger-test-prometheus-1 Stopping
+ Container messenger-test-grafana-1 Stopping
+ Container messenger-test-cleanup-1 Stopping
+ Container messenger-test-otel-collector-1 Stopping
+ Container messenger-test-worker-1 Stopping
+ Container messenger-test-loki-1 Stopping
+ Container messenger-test-prometheus-1 Stopped
+ Container messenger-test-prometheus-1 Removing
+ Container messenger-test-prometheus-1 Removed
+ Container messenger-test-otel-collector-1 Stopped
+ Container messenger-test-otel-collector-1 Removing
+ Container messenger-test-otel-collector-1 Removed
+ Container messenger-test-grafana-1 Stopped
+ Container messenger-test-grafana-1 Removing
+ Container messenger-test-grafana-1 Removed
+ Container messenger-test-tempo-1 Stopped
+ Container messenger-test-tempo-1 Removing
+ Container messenger-test-tempo-1 Removed
+ Container messenger-test-nginx-1 Stopped
+ Container messenger-test-nginx-1 Removing
+ Container messenger-test-nginx-1 Removed
+ Container messenger-test-api-2 Stopping
+ Container messenger-test-api-1 Stopping
+ Container messenger-test-api-2 Stopped
+ Container messenger-test-api-2 Removing
+ Container messenger-test-api-2 Removed
+ Container messenger-test-loki-1 Stopped
+ Container messenger-test-loki-1 Removing
+ Container messenger-test-loki-1 Removed
+ Container messenger-test-publisher-1 Stopped
+ Container messenger-test-publisher-1 Removing
+ Container messenger-test-publisher-1 Removed
+ Container messenger-test-worker-1 Stopped
+ Container messenger-test-worker-1 Removing
+ Container messenger-test-worker-1 Removed
+ Container messenger-test-broker-1 Stopping
+ Container messenger-test-broker-1 Stopped
+ Container messenger-test-broker-1 Removing
+ Container messenger-test-broker-1 Removed
+ Container messenger-test-api-1 Stopped
+ Container messenger-test-api-1 Removing
+ Container messenger-test-api-1 Removed
+ Container messenger-test-redis-1 Stopping
+ Container messenger-test-redis-1 Stopped
+ Container messenger-test-redis-1 Removing
+ Container messenger-test-redis-1 Removed
+ Container messenger-test-cleanup-1 Stopped
+ Container messenger-test-cleanup-1 Removing
+ Container messenger-test-cleanup-1 Removed
+ Container messenger-test-migrate-1 Stopping
+ Container messenger-test-minio-bootstrap-1 Stopping
+ Container messenger-test-minio-bootstrap-1 Stopped
+ Container messenger-test-minio-bootstrap-1 Removing
+ Container messenger-test-migrate-1 Stopped
+ Container messenger-test-migrate-1 Removing
+ Container messenger-test-minio-bootstrap-1 Removed
+ Container messenger-test-minio-1 Stopping
+ Container messenger-test-migrate-1 Removed
+ Container messenger-test-postgres-1 Stopping
+ Container messenger-test-minio-1 Stopped
+ Container messenger-test-minio-1 Removing
+ Container messenger-test-minio-1 Removed
+ Container messenger-test-postgres-1 Stopped
+ Container messenger-test-postgres-1 Removing
+ Container messenger-test-postgres-1 Removed
+ Volume messenger-test_lokidata Removing
+ Volume messenger-test_prometheusdata Removing
+ Network messenger-test_objects Removing
+ Network messenger-test_default Removing
+ Volume messenger-test_prometheusdata Removed
+ Volume messenger-test_tempodata Removing
+ Volume messenger-test_pgdata Removing
+ Volume messenger-test_rabbitmqdata Removing
+ Volume messenger-test_miniodata Removing
+ Volume messenger-test_redisdata Removing
+ Volume messenger-test_grafanadata Removing
+ Volume messenger-test_lokidata Removed
+ Volume messenger-test_pgdata Removed
+ Volume messenger-test_tempodata Removed
+ Network messenger-test_default Removed
+ Volume messenger-test_grafanadata Removed
+ Volume messenger-test_redisdata Removed
+ Volume messenger-test_miniodata Removed
+ Volume messenger-test_rabbitmqdata Removed
+ Network messenger-test_objects Removed
diff --git a/evidence/lab6/observability.json b/evidence/lab6/observability.json
new file mode 100644
index 0000000..04deadb
--- /dev/null
+++ b/evidence/lab6/observability.json
@@ -0,0 +1,888 @@
+{
+ "date_utc": "2026-11-22T15:17:36.007579+00:00",
+ "project": "messenger-test",
+ "load": {
+ "date_utc": "2026-11-22T15:17:29.101767+00:00",
+ "workload": "synthetic POST messages; fixed arrival rate; no retries",
+ "duration_seconds": 10,
+ "elapsed_seconds": 9.994,
+ "arrival_rate": 10,
+ "concurrency_limit": 2,
+ "successful_requests": 100,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "successful_rps": 10.006,
+ "successful_latency_ms": {
+ "p50": 26.145,
+ "p95": 51.412,
+ "p99": 80.911
+ },
+ "dataset": {
+ "users": 3,
+ "chats": 1,
+ "messages": 100
+ },
+ "resource_samples": [
+ {
+ "elapsed_seconds": 2.2,
+ "containers": [
+ {
+ "Name": "messenger-test-api-1",
+ "CPUPerc": "27.40%",
+ "MemUsage": "103.6MiB / 1.5GiB",
+ "MemPerc": "6.74%"
+ },
+ {
+ "Name": "messenger-test-api-2",
+ "CPUPerc": "10.94%",
+ "MemUsage": "87.22MiB / 1.5GiB",
+ "MemPerc": "5.68%"
+ },
+ {
+ "Name": "messenger-test-broker-1",
+ "CPUPerc": "0.54%",
+ "MemUsage": "114.3MiB / 768MiB",
+ "MemPerc": "14.88%"
+ },
+ {
+ "Name": "messenger-test-cleanup-1",
+ "CPUPerc": "0.00%",
+ "MemUsage": "74.72MiB / 256MiB",
+ "MemPerc": "29.19%"
+ },
+ {
+ "Name": "messenger-test-grafana-1",
+ "CPUPerc": "2.36%",
+ "MemUsage": "368.9MiB / 512MiB",
+ "MemPerc": "72.05%"
+ },
+ {
+ "Name": "messenger-test-loki-1",
+ "CPUPerc": "1.19%",
+ "MemUsage": "51.66MiB / 512MiB",
+ "MemPerc": "10.09%"
+ },
+ {
+ "Name": "messenger-test-minio-1",
+ "CPUPerc": "0.04%",
+ "MemUsage": "99.82MiB / 512MiB",
+ "MemPerc": "19.50%"
+ },
+ {
+ "Name": "messenger-test-nginx-1",
+ "CPUPerc": "0.68%",
+ "MemUsage": "10.36MiB / 128MiB",
+ "MemPerc": "8.09%"
+ },
+ {
+ "Name": "messenger-test-otel-collector-1",
+ "CPUPerc": "1.33%",
+ "MemUsage": "58.9MiB / 256MiB",
+ "MemPerc": "23.01%"
+ },
+ {
+ "Name": "messenger-test-postgres-1",
+ "CPUPerc": "10.20%",
+ "MemUsage": "81.98MiB / 512MiB",
+ "MemPerc": "16.01%"
+ },
+ {
+ "Name": "messenger-test-prometheus-1",
+ "CPUPerc": "1.12%",
+ "MemUsage": "27.15MiB / 512MiB",
+ "MemPerc": "5.30%"
+ },
+ {
+ "Name": "messenger-test-publisher-1",
+ "CPUPerc": "2.43%",
+ "MemUsage": "73.21MiB / 256MiB",
+ "MemPerc": "28.60%"
+ },
+ {
+ "Name": "messenger-test-redis-1",
+ "CPUPerc": "3.92%",
+ "MemUsage": "3.555MiB / 512MiB",
+ "MemPerc": "0.69%"
+ },
+ {
+ "Name": "messenger-test-tempo-1",
+ "CPUPerc": "0.54%",
+ "MemUsage": "170.6MiB / 512MiB",
+ "MemPerc": "33.32%"
+ },
+ {
+ "Name": "messenger-test-worker-1",
+ "CPUPerc": "0.02%",
+ "MemUsage": "70.18MiB / 2GiB",
+ "MemPerc": "3.43%"
+ }
+ ]
+ },
+ {
+ "elapsed_seconds": 7.39,
+ "containers": [
+ {
+ "Name": "messenger-test-api-1",
+ "CPUPerc": "66.64%",
+ "MemUsage": "106.6MiB / 1.5GiB",
+ "MemPerc": "6.94%"
+ },
+ {
+ "Name": "messenger-test-api-2",
+ "CPUPerc": "58.08%",
+ "MemUsage": "87.28MiB / 1.5GiB",
+ "MemPerc": "5.68%"
+ },
+ {
+ "Name": "messenger-test-broker-1",
+ "CPUPerc": "54.88%",
+ "MemUsage": "114.3MiB / 768MiB",
+ "MemPerc": "14.88%"
+ },
+ {
+ "Name": "messenger-test-cleanup-1",
+ "CPUPerc": "0.00%",
+ "MemUsage": "74.72MiB / 256MiB",
+ "MemPerc": "29.19%"
+ },
+ {
+ "Name": "messenger-test-grafana-1",
+ "CPUPerc": "1.64%",
+ "MemUsage": "368.9MiB / 512MiB",
+ "MemPerc": "72.05%"
+ },
+ {
+ "Name": "messenger-test-loki-1",
+ "CPUPerc": "1.11%",
+ "MemUsage": "51.69MiB / 512MiB",
+ "MemPerc": "10.10%"
+ },
+ {
+ "Name": "messenger-test-minio-1",
+ "CPUPerc": "48.68%",
+ "MemUsage": "99.89MiB / 512MiB",
+ "MemPerc": "19.51%"
+ },
+ {
+ "Name": "messenger-test-nginx-1",
+ "CPUPerc": "0.60%",
+ "MemUsage": "10.36MiB / 128MiB",
+ "MemPerc": "8.09%"
+ },
+ {
+ "Name": "messenger-test-otel-collector-1",
+ "CPUPerc": "1.51%",
+ "MemUsage": "59.55MiB / 256MiB",
+ "MemPerc": "23.26%"
+ },
+ {
+ "Name": "messenger-test-postgres-1",
+ "CPUPerc": "3.51%",
+ "MemUsage": "84.2MiB / 512MiB",
+ "MemPerc": "16.45%"
+ },
+ {
+ "Name": "messenger-test-prometheus-1",
+ "CPUPerc": "0.56%",
+ "MemUsage": "27.28MiB / 512MiB",
+ "MemPerc": "5.33%"
+ },
+ {
+ "Name": "messenger-test-publisher-1",
+ "CPUPerc": "0.98%",
+ "MemUsage": "73.21MiB / 256MiB",
+ "MemPerc": "28.60%"
+ },
+ {
+ "Name": "messenger-test-redis-1",
+ "CPUPerc": "0.48%",
+ "MemUsage": "3.438MiB / 512MiB",
+ "MemPerc": "0.67%"
+ },
+ {
+ "Name": "messenger-test-tempo-1",
+ "CPUPerc": "0.59%",
+ "MemUsage": "170.7MiB / 512MiB",
+ "MemPerc": "33.34%"
+ },
+ {
+ "Name": "messenger-test-worker-1",
+ "CPUPerc": "0.02%",
+ "MemUsage": "70.18MiB / 2GiB",
+ "MemPerc": "3.43%"
+ }
+ ]
+ }
+ ],
+ "limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark."
+ },
+ "api_replicas": 2,
+ "dashboard_uid": "messenger-overview",
+ "api_correlation": {
+ "trace_id": "9021bfee7e20d9e28cf9e11ee8239aa3",
+ "span_id": "792e18b5537f0825",
+ "log": {
+ "timestamp": "2026-11-22T15:17:31.262609+00:00",
+ "severity": "INFO",
+ "service_name": "messenger-api",
+ "event": "http.request",
+ "trace_id": "9021bfee7e20d9e28cf9e11ee8239aa3",
+ "span_id": "792e18b5537f0825",
+ "method": "POST",
+ "route": "/api/v1/chats/{chat_id}/messages",
+ "status_code": 201,
+ "request_id": "f21c27ad-b489-4c64-88e5-d7802fa2d822",
+ "duration_seconds": 0.020220299000357045
+ },
+ "spans": [
+ {
+ "name": "POST /api/v1/chats/{chat_id}/messages",
+ "kind": "SPAN_KIND_SERVER",
+ "spanId": "eS4YtVN/CCU=",
+ "startTimeUnixNano": "1795360651242347308",
+ "endTimeUnixNano": "1795360651262809878",
+ "status": {
+ "code": "STATUS_CODE_OK"
+ }
+ }
+ ]
+ },
+ "worker_correlation": {
+ "trace_id": "80b4251620cc7d99f7ea78672971f5d1",
+ "span_id": "0269328bf7350d5f",
+ "log": {
+ "timestamp": "2026-11-22T15:17:32.391619+00:00",
+ "severity": "INFO",
+ "service_name": "messenger-worker",
+ "event": "job.received",
+ "trace_id": "80b4251620cc7d99f7ea78672971f5d1",
+ "span_id": "0269328bf7350d5f",
+ "job_id": "8066d6a3-3c2f-45dd-9478-42c98a1c8247",
+ "attachment_id": "0cc8d21c-8fc7-4e76-9808-412a1bc5f4d2"
+ },
+ "spans": [
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "spanId": "Amkyi/c1DV8=",
+ "startTimeUnixNano": "1795360652391582582",
+ "endTimeUnixNano": "1795360653672341819",
+ "status": {
+ "code": "STATUS_CODE_OK"
+ }
+ }
+ ]
+ },
+ "metrics": {
+ "http_requests": [
+ {
+ "metric": {},
+ "value": [
+ 1795360655.251,
+ "267"
+ ]
+ }
+ ],
+ "http_duration": [
+ {
+ "metric": {},
+ "value": [
+ 1795360655.265,
+ "267"
+ ]
+ }
+ ],
+ "job_success": [
+ {
+ "metric": {},
+ "value": [
+ 1795360655.323,
+ "1"
+ ]
+ }
+ ],
+ "job_duration": [
+ {
+ "metric": {},
+ "value": [
+ 1795360655.389,
+ "1"
+ ]
+ }
+ ]
+ },
+ "upload_trace": {
+ "trace_id": "80b4251620cc7d99f7ea78672971f5d1",
+ "chain": [
+ {
+ "name": "POST /api/v1/chats/{chat_id}/attachments",
+ "kind": "SPAN_KIND_SERVER",
+ "span_id": "c58955b22c850737",
+ "parent_span_id": "",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "1b0ebdf48b6dfc58",
+ "parent_span_id": "c58955b22c850737",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "2b7b69652c0d4b56",
+ "parent_span_id": "c58955b22c850737",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "969f792e082e2df0",
+ "parent_span_id": "c58955b22c850737",
+ "job": {}
+ },
+ {
+ "name": "rabbitmq.publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "span_id": "5efc0e27535ad301",
+ "parent_span_id": "c58955b22c850737",
+ "job": {
+ "job.attempt": {
+ "intValue": "1"
+ }
+ }
+ },
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "span_id": "0269328bf7350d5f",
+ "parent_span_id": "5efc0e27535ad301",
+ "job": {
+ "job.attempt": {
+ "intValue": "1"
+ },
+ "outcome": {
+ "stringValue": "success"
+ }
+ }
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "5f50cdf285111d79",
+ "parent_span_id": "0269328bf7350d5f",
+ "job": {}
+ },
+ {
+ "name": "s3.write_result",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "b7a2da338b72c28d",
+ "parent_span_id": "0269328bf7350d5f",
+ "job": {}
+ },
+ {
+ "name": "postgres.commit_ready",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "8be3e2f2f3f422e5",
+ "parent_span_id": "0269328bf7350d5f",
+ "job": {}
+ },
+ {
+ "name": "postgres.claim_job",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "e22e601d7aa413b7",
+ "parent_span_id": "0269328bf7350d5f",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "c8247b2c4faa80a2",
+ "parent_span_id": "e22e601d7aa413b7",
+ "job": {}
+ },
+ {
+ "name": "s3.read_staging",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "9133f43b2a5b2286",
+ "parent_span_id": "0269328bf7350d5f",
+ "job": {}
+ }
+ ]
+ },
+ "incidents": [
+ {
+ "scenario": "postgres_unavailable",
+ "http_status": 503,
+ "correlation": {
+ "trace_id": "6c4b9559264492f9cef34f321e7883aa",
+ "span_id": "e0fb869b3be141af",
+ "log": {
+ "timestamp": "2026-11-22T15:17:39.566055+00:00",
+ "severity": "ERROR",
+ "service_name": "messenger-api",
+ "event": "http.request",
+ "trace_id": "6c4b9559264492f9cef34f321e7883aa",
+ "span_id": "e0fb869b3be141af",
+ "method": "POST",
+ "route": "/api/v1/users",
+ "status_code": 503,
+ "request_id": "3d3556ac-40a0-403b-815c-d0344a658cd2",
+ "duration_seconds": 0.008871385000020382
+ },
+ "spans": [
+ {
+ "name": "POST /api/v1/users",
+ "kind": "SPAN_KIND_SERVER",
+ "spanId": "4PuGmzvhQa8=",
+ "startTimeUnixNano": "1795360659557136044",
+ "endTimeUnixNano": "1795360659566360836",
+ "status": {
+ "code": "STATUS_CODE_ERROR"
+ }
+ }
+ ]
+ },
+ "diagnostic": {
+ "timestamp": "2026-11-22T15:17:39.564481+00:00",
+ "severity": "ERROR",
+ "service_name": "messenger-api",
+ "event": "dependency.unavailable",
+ "trace_id": "6c4b9559264492f9cef34f321e7883aa",
+ "span_id": "e0fb869b3be141af",
+ "request_id": "3d3556ac-40a0-403b-815c-d0344a658cd2",
+ "exception_type": "gaierror"
+ },
+ "db_span": "postgres.checkout",
+ "alert": "MessengerHighErrorRate",
+ "firing_after_seconds": 25.876,
+ "resolved_after_seconds": 66.29
+ },
+ {
+ "scenario": "worker_stopped_then_transient_retry",
+ "alert": "MessengerQueueLag",
+ "queue": {
+ "depth": 1.0,
+ "oldest_seconds": 27.203197,
+ "firing_after_seconds": 32.694,
+ "resolved": true,
+ "final_depth": 0
+ },
+ "recovered_status": "ready",
+ "trace_id": "2ee8f51dca356b60ce8e453bde093afc",
+ "chain": [
+ {
+ "name": "POST /api/v1/chats/{chat_id}/attachments",
+ "kind": "SPAN_KIND_SERVER",
+ "span_id": "71821fc771b19562",
+ "parent_span_id": "",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "b1f792c3bcc5fc94",
+ "parent_span_id": "71821fc771b19562",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "7fcf7dcf07bd18e6",
+ "parent_span_id": "71821fc771b19562",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "d3b3d23ac779207c",
+ "parent_span_id": "71821fc771b19562",
+ "job": {}
+ },
+ {
+ "name": "rabbitmq.publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "span_id": "07f4665665b14948",
+ "parent_span_id": "71821fc771b19562",
+ "job": {
+ "job.attempt": {
+ "intValue": "1"
+ }
+ }
+ },
+ {
+ "name": "rabbitmq.publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "span_id": "8086cee6d417141c",
+ "parent_span_id": "71821fc771b19562",
+ "job": {
+ "job.attempt": {
+ "intValue": "2"
+ }
+ }
+ },
+ {
+ "name": "rabbitmq.publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "span_id": "6452cf4a27cdff44",
+ "parent_span_id": "71821fc771b19562",
+ "job": {
+ "job.attempt": {
+ "intValue": "3"
+ }
+ }
+ },
+ {
+ "name": "rabbitmq.publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "span_id": "5ac004e8420e9a68",
+ "parent_span_id": "71821fc771b19562",
+ "job": {
+ "job.attempt": {
+ "intValue": "4"
+ }
+ }
+ },
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "span_id": "56de810ab11a66f5",
+ "parent_span_id": "07f4665665b14948",
+ "job": {
+ "job.attempt": {
+ "intValue": "1"
+ },
+ "outcome": {
+ "stringValue": "error"
+ }
+ }
+ },
+ {
+ "name": "postgres.claim_job",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "da8146c57677ee39",
+ "parent_span_id": "56de810ab11a66f5",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "615625bfe95f226f",
+ "parent_span_id": "da8146c57677ee39",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "b62561f9f5a29016",
+ "parent_span_id": "56de810ab11a66f5",
+ "job": {}
+ },
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "span_id": "e3540cedb5c50906",
+ "parent_span_id": "8086cee6d417141c",
+ "job": {
+ "job.attempt": {
+ "intValue": "2"
+ },
+ "outcome": {
+ "stringValue": "error"
+ }
+ }
+ },
+ {
+ "name": "postgres.claim_job",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "2291ce44145b0d7b",
+ "parent_span_id": "e3540cedb5c50906",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "d19831896662c7f9",
+ "parent_span_id": "2291ce44145b0d7b",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "3b1e2331f3264093",
+ "parent_span_id": "e3540cedb5c50906",
+ "job": {}
+ },
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "span_id": "a89e13520f615484",
+ "parent_span_id": "6452cf4a27cdff44",
+ "job": {
+ "job.attempt": {
+ "intValue": "3"
+ },
+ "outcome": {
+ "stringValue": "error"
+ }
+ }
+ },
+ {
+ "name": "postgres.claim_job",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "cc92a3cdf21e0ea2",
+ "parent_span_id": "a89e13520f615484",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "8c0b61ca848b45cc",
+ "parent_span_id": "cc92a3cdf21e0ea2",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "8785fe8fad33b4ad",
+ "parent_span_id": "a89e13520f615484",
+ "job": {}
+ },
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "span_id": "a6028a8ec3eb3e46",
+ "parent_span_id": "5ac004e8420e9a68",
+ "job": {
+ "job.attempt": {
+ "intValue": "4"
+ },
+ "outcome": {
+ "stringValue": "error"
+ }
+ }
+ },
+ {
+ "name": "postgres.claim_job",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "d5024c8fbc0fcd5c",
+ "parent_span_id": "a6028a8ec3eb3e46",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "d5fb43f65d678c70",
+ "parent_span_id": "d5024c8fbc0fcd5c",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "c583fcb1ba15ba93",
+ "parent_span_id": "a6028a8ec3eb3e46",
+ "job": {}
+ },
+ {
+ "name": "rabbitmq.publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "span_id": "34e7dbafc9a59e9d",
+ "parent_span_id": "71821fc771b19562",
+ "job": {
+ "job.attempt": {
+ "intValue": "5"
+ }
+ }
+ },
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "span_id": "73e4025d3620610d",
+ "parent_span_id": "34e7dbafc9a59e9d",
+ "job": {
+ "job.attempt": {
+ "intValue": "5"
+ },
+ "outcome": {
+ "stringValue": "error"
+ }
+ }
+ },
+ {
+ "name": "postgres.claim_job",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "7192541ca5b83140",
+ "parent_span_id": "73e4025d3620610d",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "4360b3b563e388a8",
+ "parent_span_id": "7192541ca5b83140",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "21cf4366c2d942ad",
+ "parent_span_id": "73e4025d3620610d",
+ "job": {}
+ },
+ {
+ "name": "rabbitmq.publish",
+ "kind": "SPAN_KIND_PRODUCER",
+ "span_id": "0e837636758d0fa6",
+ "parent_span_id": "71821fc771b19562",
+ "job": {
+ "job.attempt": {
+ "intValue": "6"
+ }
+ }
+ },
+ {
+ "name": "attachment.process.v1",
+ "kind": "SPAN_KIND_CONSUMER",
+ "span_id": "dec8c7f4bbbaedc8",
+ "parent_span_id": "0e837636758d0fa6",
+ "job": {
+ "job.attempt": {
+ "intValue": "6"
+ },
+ "outcome": {
+ "stringValue": "success"
+ }
+ }
+ },
+ {
+ "name": "postgres.claim_job",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "80d6e51214f68e05",
+ "parent_span_id": "dec8c7f4bbbaedc8",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "b0f968673ec0f4b1",
+ "parent_span_id": "80d6e51214f68e05",
+ "job": {}
+ },
+ {
+ "name": "s3.read_staging",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "415fed4f099dac87",
+ "parent_span_id": "dec8c7f4bbbaedc8",
+ "job": {}
+ },
+ {
+ "name": "postgres.checkout",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "41af4befc31e1ff5",
+ "parent_span_id": "dec8c7f4bbbaedc8",
+ "job": {}
+ },
+ {
+ "name": "s3.write_result",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "c6ebe655a1c26e9e",
+ "parent_span_id": "dec8c7f4bbbaedc8",
+ "job": {}
+ },
+ {
+ "name": "postgres.commit_ready",
+ "kind": "SPAN_KIND_INTERNAL",
+ "span_id": "24f071229ba36c24",
+ "parent_span_id": "dec8c7f4bbbaedc8",
+ "job": {}
+ }
+ ]
+ },
+ {
+ "scenario": "loki_unavailable",
+ "successful_messages": 204,
+ "dropped_after_export_failure": 6.0,
+ "dropped_at_enqueue": 0.0,
+ "samples": [
+ {
+ "seconds": 0.14,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 0.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 5.12,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 5.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 10.1,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 10.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 15.1,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 15.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 20.1,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 20.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 25.07,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 25.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 30.08,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 28.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 35.08,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 33.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 40.09,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 38.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 45.07,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 43.0,
+ "capacity_batches": 256.0
+ },
+ {
+ "seconds": 50.1,
+ "rss_bytes": 242114560.0,
+ "queued_batches": 48.0,
+ "capacity_batches": 256.0
+ }
+ ],
+ "log_export_recovered": true
+ }
+ ],
+ "privacy_checked_logs": 563,
+ "checks": {
+ "three_backends": true,
+ "dashboard_queries": true,
+ "server_consumer_traces": true,
+ "log_trace_correlation": true,
+ "low_cardinality": true,
+ "privacy": true,
+ "history_survives_restart": true
+ }
+}
diff --git a/evidence/lab6/pytest.xml b/evidence/lab6/pytest.xml
new file mode 100644
index 0000000..52da96a
--- /dev/null
+++ b/evidence/lab6/pytest.xml
@@ -0,0 +1 @@
+
\ No newline at end of file
diff --git a/evidence/lab6/run.json b/evidence/lab6/run.json
new file mode 100644
index 0000000..9796d39
--- /dev/null
+++ b/evidence/lab6/run.json
@@ -0,0 +1,60 @@
+{
+ "status": "passed",
+ "project": "messenger-check-30e5e3483b9c",
+ "profile": "full",
+ "started_at": "2026-11-22T13:55:48.616518+00:00",
+ "cleanup": [
+ {
+ "command": [
+ "docker",
+ "rm",
+ "--force",
+ "messenger-check-30e5e3483b9c-runner",
+ "messenger-check-30e5e3483b9c-engine"
+ ],
+ "returncode": 0,
+ "output": "messenger-check-30e5e3483b9c-runner\nmessenger-check-30e5e3483b9c-engine"
+ },
+ {
+ "command": [
+ "docker",
+ "volume",
+ "rm",
+ "messenger-check-30e5e3483b9c-workspace",
+ "messenger-check-30e5e3483b9c-data"
+ ],
+ "returncode": 0,
+ "output": "messenger-check-30e5e3483b9c-workspace\nmessenger-check-30e5e3483b9c-data"
+ },
+ {
+ "command": [
+ "docker",
+ "network",
+ "rm",
+ "messenger-check-30e5e3483b9c-network"
+ ],
+ "returncode": 0,
+ "output": "messenger-check-30e5e3483b9c-network"
+ },
+ {
+ "command": [
+ "docker",
+ "image",
+ "rm",
+ "messenger-check-30e5e3483b9c:tools",
+ "messenger-check-30e5e3483b9c:api",
+ "messenger-check-30e5e3483b9c:server",
+ "messenger-check-30e5e3483b9c:bootstrap"
+ ],
+ "returncode": 0,
+ "output": "Untagged: messenger-check-30e5e3483b9c:tools\nDeleted: sha256:ec48da89a014e0c8b3e2baced8c180ff3389a9a8b4f8298c10b6f418a604f52a\nUntagged: messenger-check-30e5e3483b9c:api\nUntagged: messenger-check-30e5e3483b9c:server\nUntagged: messenger-check-30e5e3483b9c:bootstrap"
+ }
+ ],
+ "remaining_resources": {
+ "container": [],
+ "volume": [],
+ "network": [],
+ "image": []
+ },
+ "finished_at": "2026-11-22T15:24:03.158690+00:00"
+}
diff --git a/evidence/lab6/verification.json b/evidence/lab6/verification.json
new file mode 100644
index 0000000..794bf87
--- /dev/null
+++ b/evidence/lab6/verification.json
@@ -0,0 +1,86 @@
+{
+ "status": "passed",
+ "profile": "full",
+ "started_at": "2026-11-22T15:09:15.220959+00:00",
+ "checks": [
+ {
+ "name": "00_init",
+ "exit_code": 0,
+ "duration_seconds": 0.165,
+ "log": "00_init.txt"
+ },
+ {
+ "name": "01_contract",
+ "exit_code": 0,
+ "duration_seconds": 0.215,
+ "log": "01_contract.txt"
+ },
+ {
+ "name": "02_openapi",
+ "exit_code": 0,
+ "duration_seconds": 2.277,
+ "log": "02_openapi.txt"
+ },
+ {
+ "name": "03_import_images",
+ "exit_code": 0,
+ "duration_seconds": 201.634,
+ "log": "03_import_images.txt"
+ },
+ {
+ "name": "03_startup",
+ "exit_code": 0,
+ "duration_seconds": 33.714,
+ "log": "03_startup.txt"
+ },
+ {
+ "name": "03_readiness",
+ "exit_code": 0,
+ "duration_seconds": 73.355,
+ "log": "03_readiness.txt"
+ },
+ {
+ "name": "04_test_database",
+ "exit_code": 0,
+ "duration_seconds": 0.473,
+ "log": "04_test_database.txt"
+ },
+ {
+ "name": "05_pytest",
+ "exit_code": 0,
+ "duration_seconds": 162.52,
+ "log": "05_pytest.txt"
+ },
+ {
+ "name": "06_smoke",
+ "exit_code": 0,
+ "duration_seconds": 6.452,
+ "log": "06_smoke.txt"
+ },
+ {
+ "name": "07_observability",
+ "exit_code": 0,
+ "duration_seconds": 309.151,
+ "log": "07_observability.txt"
+ },
+ {
+ "name": "08_persistence",
+ "exit_code": 0,
+ "duration_seconds": 57.668,
+ "log": "08_persistence.txt"
+ },
+ {
+ "name": "09_failover",
+ "exit_code": 0,
+ "duration_seconds": 20.13,
+ "log": "09_failover.txt"
+ },
+ {
+ "name": "10_cleanup",
+ "exit_code": 0,
+ "duration_seconds": 12.272,
+ "log": "10_cleanup.txt"
+ }
+ ],
+ "finished_at": "2026-11-22T15:23:55.136720+00:00"
+}
diff --git a/infra/README.md b/infra/README.md
index 5a39e86..e09e57d 100644
--- a/infra/README.md
+++ b/infra/README.md
@@ -1,7 +1,14 @@
-# Инфраструктура студента
+# Инфраструктура
-Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли.
+Две реплики API работают за Nginx. PostgreSQL хранит сообщения и задания,
+Redis — сессии, RabbitMQ доставляет задания, приватный MinIO хранит вложения.
+Publisher подтверждает доставку через брокер, worker фиксирует результат
+до подтверждения обработки. Настройки берутся из `.env`, секреты передаются
+файлами из `secrets/`. Миграции и настройка MinIO выполняются автоматически.
-В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд.
+Collector принимает телеметрию и отправляет её в Prometheus, Loki и Tempo.
+Дашборд и настройки источников Grafana лежат в [otel](otel/README.md).
-Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику.
+Проверка через `py -3.12 -X utf8 scripts/test.py` использует отдельный
+временный Docker-стенд. PostgreSQL и Redis доступны тестам только внутри
+временной среды. Сводка и журналы сохраняются в `evidence/lab6/`.
diff --git a/infra/broker/start-broker.sh b/infra/broker/start-broker.sh
new file mode 100644
index 0000000..a9e022b
--- /dev/null
+++ b/infra/broker/start-broker.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+export RABBITMQ_DEFAULT_PASS="$(cat "${BROKER_PASSWORD_FILE:?}")"
+export RABBITMQ_DEFAULT_USER="${BROKER_USER:?}"
+export RABBITMQ_DEFAULT_VHOST="${BROKER_VHOST:?}"
+export RABBITMQ_NODE_PORT="${BROKER_PORT:?}"
+exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server
diff --git a/infra/minio/Dockerfile b/infra/minio/Dockerfile
new file mode 100644
index 0000000..fd7cc06
--- /dev/null
+++ b/infra/minio/Dockerfile
@@ -0,0 +1,22 @@
+FROM golang:1.24.8-bookworm AS compiler
+ENV CGO_ENABLED=0 GOTOOLCHAIN=local GOBIN=/out GOMAXPROCS=4
+RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
+ go install -p 4 -ldflags='-s -w -X github.com/minio/minio/cmd.Version=RELEASE.2025-10-15T17-29-55Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-10-15T17-29-55Z' github.com/minio/minio@RELEASE.2025-10-15T17-29-55Z
+RUN --mount=type=cache,target=/go/pkg/mod --mount=type=cache,target=/root/.cache/go-build \
+ go install -p 4 -ldflags='-s -w' github.com/minio/mc@RELEASE.2025-08-13T08-35-41Z
+
+FROM python:3.12.15-slim-bookworm AS runtime
+RUN useradd --system --uid 10001 --no-create-home minio && mkdir /data && chown minio /data
+USER minio
+
+FROM runtime AS server
+COPY --from=compiler /out/minio /usr/local/bin/minio
+LABEL org.opencontainers.image.source="https://github.com/minio/minio" \
+ org.opencontainers.image.version="RELEASE.2025-10-15T17-29-55Z"
+VOLUME ["/data"]
+ENTRYPOINT ["/usr/local/bin/minio"]
+
+FROM runtime AS bootstrap
+COPY --from=compiler /out/mc /usr/local/bin/mc
+COPY infra/minio/bootstrap.py /bootstrap.py
+ENTRYPOINT ["python", "/bootstrap.py"]
diff --git a/infra/minio/bootstrap.py b/infra/minio/bootstrap.py
new file mode 100644
index 0000000..d4870f0
--- /dev/null
+++ b/infra/minio/bootstrap.py
@@ -0,0 +1,120 @@
+"""Идемпотентный private bucket и отдельная ограниченная учётная запись API."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+from tempfile import TemporaryDirectory
+import time
+from urllib.request import urlopen
+
+
+def secret(name):
+ value = Path(os.environ[name.upper() + "_FILE"]).read_text().strip()
+ if not value:
+ raise ValueError("Пустой файл секрета")
+ return value
+
+
+def main():
+ endpoint = os.environ["S3_ENDPOINT"]
+ bucket = os.environ["S3_BUCKET"]
+ root_user, root_password = secret("minio_root_user"), secret("minio_root_password")
+ access, password = secret("s3_access_key_id"), secret("s3_secret_access_key")
+ if access == root_user:
+ raise ValueError("API не должен использовать root credentials")
+ deadline = time.monotonic() + 60
+ while True:
+ try:
+ with urlopen(endpoint + "/minio/health/ready", timeout=2) as response:
+ if response.status == 200:
+ break
+ except OSError:
+ pass
+ if time.monotonic() >= deadline:
+ raise RuntimeError("MinIO не готов за 60 секунд")
+ time.sleep(1)
+ with TemporaryDirectory(prefix="minio-bootstrap-") as directory:
+ config = Path(directory)
+
+ config_file = config / "config.json"
+ config_file.write_text(
+ json.dumps(
+ {
+ "version": "10",
+ "aliases": {
+ "storage": {
+ "url": endpoint,
+ "accessKey": root_user,
+ "secretKey": root_password,
+ "api": "S3v4",
+ "path": "auto",
+ }
+ },
+ }
+ )
+ )
+ config_file.chmod(0o600)
+
+ def mc(*args):
+ result = subprocess.run(
+ ["mc", "--config-dir", directory, *args],
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ timeout=30,
+ )
+ if result.returncode:
+ raise RuntimeError(
+ "Ошибка bootstrap MinIO; значения credentials скрыты"
+ )
+
+ mc("mb", "--ignore-existing", "storage/" + bucket)
+ mc("anonymous", "set", "none", "storage/" + bucket)
+ policy = {
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+ "Resource": [f"arn:aws:s3:::{bucket}"],
+ },
+ {
+ "Effect": "Allow",
+ "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
+ "Resource": [
+ f"arn:aws:s3:::{bucket}/{prefix}/*"
+ for prefix in ("originals", "staging", "variants")
+ ],
+ },
+ ],
+ }
+ policy_file = config / "policy.json"
+ policy_file.write_text(json.dumps(policy))
+ mc(
+ "admin",
+ "policy",
+ "create",
+ "storage",
+ "messenger-attachments",
+ str(policy_file),
+ )
+
+ mc("admin", "user", "add", "storage", access, password)
+ mc(
+ "admin",
+ "policy",
+ "attach",
+ "storage",
+ "messenger-attachments",
+ "--user",
+ access,
+ )
+ print("MinIO private bucket and API policy configured", flush=True)
+
+
+if __name__ == "__main__":
+ try:
+ main()
+ except Exception as error:
+ print("MinIO bootstrap failed: " + type(error).__name__, flush=True)
+ raise SystemExit(1) from None
diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf
new file mode 100644
index 0000000..04571d4
--- /dev/null
+++ b/infra/nginx/nginx.conf
@@ -0,0 +1,124 @@
+
+worker_processes 2;
+
+events {
+ worker_connections 1024;
+}
+
+http {
+ map $upstream_http_x_request_id $response_request_id {
+ "" $request_id;
+ default $upstream_http_x_request_id;
+ }
+
+ log_format requests escape=json
+ '{"time":"$time_iso8601","request_id":"$response_request_id",'
+ '"method":"$request_method","uri":"$uri","status":$status,'
+ '"bytes":$body_bytes_sent,"request_time":$request_time,'
+ '"upstream":"$upstream_addr","upstream_status":"$upstream_status",'
+ '"upstream_time":"$upstream_response_time",'
+ '"instance_id":"$upstream_http_x_instance_id",'
+ '"limit_status":"$limit_req_status"}';
+
+ access_log /dev/stdout requests;
+ error_log /dev/stderr warn;
+ server_tokens off;
+
+ resolver 127.0.0.11 valid=5s ipv6=off;
+ resolver_timeout 1s;
+
+ limit_req_zone $binary_remote_addr zone=per_client:10m rate=50r/s;
+
+ gzip on;
+ gzip_vary on;
+ gzip_min_length 1024;
+ gzip_comp_level 3;
+ gzip_types application/json;
+
+ upstream api_backend {
+ zone api_backend 64k;
+ server api:8080 resolve max_fails=1 fail_timeout=5s;
+
+ keepalive 16;
+ keepalive_timeout 4s;
+ }
+
+ server {
+ listen 8080;
+
+ client_header_timeout 10s;
+ client_body_timeout 10s;
+ client_max_body_size 0;
+ client_body_buffer_size 64k;
+ send_timeout 10s;
+ keepalive_timeout 30s;
+
+ proxy_http_version 1.1;
+ proxy_set_header Connection "";
+ proxy_set_header Host $host;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+
+ proxy_connect_timeout 500ms;
+ proxy_send_timeout 10s;
+ proxy_read_timeout 10s;
+
+ proxy_next_upstream error timeout http_502 http_503 http_504;
+ proxy_next_upstream_tries 2;
+ proxy_next_upstream_timeout 2s;
+
+ proxy_intercept_errors off;
+ error_page 429 = @rate_limited;
+ error_page 502 503 504 = @unavailable;
+
+ location = /lb-health {
+ access_log off;
+ default_type text/plain;
+ return 200 "ok\n";
+ }
+
+ location /health/ {
+ proxy_pass http://api_backend;
+ }
+
+ location / {
+ limit_req zone=per_client burst=100 nodelay;
+ limit_req_status 429;
+ limit_req_log_level warn;
+ proxy_pass http://api_backend;
+ }
+
+ location ~ ^/api/v1/chats/[^/]+/attachments$ {
+ limit_req zone=per_client burst=100 nodelay;
+ limit_req_status 429;
+ proxy_request_buffering off;
+ proxy_buffering off;
+ proxy_read_timeout 65s;
+ proxy_send_timeout 15s;
+ proxy_next_upstream off;
+ proxy_pass http://api_backend;
+ }
+
+ location ~ ^/api/v1/attachments/[^/]+/content$ {
+ limit_req zone=per_client burst=100 nodelay;
+ limit_req_status 429;
+ proxy_buffering off;
+ proxy_read_timeout 65s;
+ proxy_pass http://api_backend;
+ }
+
+ location @rate_limited {
+ default_type application/json;
+ add_header X-Request-Id $request_id always;
+ add_header Retry-After 1 always;
+ return 429 '{"error":{"code":"rate_limited","message":"Request rate exceeded","request_id":"$request_id"}}';
+ }
+
+ location @unavailable {
+ default_type application/json;
+ add_header X-Request-Id $request_id always;
+ add_header Retry-After 1 always;
+ return 503 '{"error":{"code":"unavailable","message":"API temporarily unavailable","request_id":"$request_id"}}';
+ }
+ }
+}
diff --git a/infra/otel/README.md b/infra/otel/README.md
index c46fe89..67a9aee 100644
--- a/infra/otel/README.md
+++ b/infra/otel/README.md
@@ -1,7 +1,20 @@
-# Конфигурация наблюдаемости — реализовать
+# Наблюдаемость Messenger
-Добавьте `collector.yaml` с OTLP receivers, processors и exporters выбранной версии Collector. Укажите явные bind endpoints, внутренние сети, memory limits/batch и поведение при backpressure по уровню. Сопоставьте все три pipeline с реально развёрнутыми backend.
+API, publisher и worker экспортируют OTLP HTTP/protobuf в Collector (:4318).
+Метрики выдаются Prometheus на :8889, логи отправляются в native OTLP Loki,
+traces — в Tempo. Grafana читает три хранилища через provisioned datasources.
-Добавьте конфигурации storage, Grafana provisioning/datasources, provisioning/dashboards и импортируемые dashboard JSON. На 4 — alert rules и runbooks. Пустой dashboard или debug exporter, который только печатает данные в stdout, не заменяют хранилища/доказательства из `contracts/observability.md`.
+Конфиги backend, Collector, dashboard и provisioning находятся в этом каталоге.
+Dashboard UID: `messenger-overview`. Метрики — в секундах и счётчиках попыток;
+точные запросы/единицы находятся в JSON. Resource environment берётся из APP_ENV.
+В Loki индексируются service.name и окружение; instance ID — structured metadata.
+Строка лога содержит trace/span ID, ссылка TraceID открывает Tempo.
-Не публикуйте OTLP/metrics/административные ports в общую сеть без необходимости. В lab7 реализуйте TLS/mTLS по trust matrix; экспорт Authorization и message bodies запрещён.
+История хранится в prometheusdata/lokidata/tempodata, retention 3 дня;
+grafanadata содержит настройки Grafana. Внешний порт имеет только Grafana,
+по умолчанию 127.0.0.1:13000. Её healthcheck проверяет также Loki, Tempo и Collector.
+Бизнес-readiness API от этих сервисов не зависит.
+
+Полный прогон со сценариями отказов запускается командой
+`py -3.12 -X utf8 scripts/test.py`. Правила оповещений, запросы и шаги
+диагностики приведены в [отчёте](../../REPORT.md).
diff --git a/infra/otel/alerts.yaml b/infra/otel/alerts.yaml
new file mode 100644
index 0000000..95319e6
--- /dev/null
+++ b/infra/otel/alerts.yaml
@@ -0,0 +1,18 @@
+groups:
+ - name: messenger
+ interval: 5s
+ rules:
+ - alert: MessengerHighErrorRate
+ expr: |
+ sum(increase(messenger_http_requests_total{http_response_status_class="5xx",http_route!~"/health/.*"}[30s]))
+ / clamp_min(sum(increase(messenger_http_requests_total{http_route!~"/health/.*"}[30s])), 1) > 0.05
+ for: 5s
+ annotations:
+ summary: "Более 5% бизнес-запросов завершились HTTP 5xx за 30 секунд"
+ runbook: "REPORT.md#диагностика-инцидентов"
+ - alert: MessengerQueueLag
+ expr: max(messenger_queue_oldest_seconds) > 15 and max(messenger_queue_depth_ratio) > 0
+ for: 5s
+ annotations:
+ summary: "Старейшее ожидающее задание находится в очереди более 15 секунд"
+ runbook: "REPORT.md#диагностика-инцидентов"
diff --git a/infra/otel/collector.yaml b/infra/otel/collector.yaml
new file mode 100644
index 0000000..93a5e72
--- /dev/null
+++ b/infra/otel/collector.yaml
@@ -0,0 +1,61 @@
+receivers:
+ otlp:
+ protocols:
+ http:
+ endpoint: 0.0.0.0:4318
+processors:
+ memory_limiter:
+ check_interval: 1s
+ limit_mib: 192
+ spike_limit_mib: 32
+ batch:
+ timeout: 1s
+ send_batch_size: 128
+ send_batch_max_size: 256
+exporters:
+ prometheus:
+ endpoint: 0.0.0.0:8889
+ resource_to_telemetry_conversion:
+ enabled: true
+ otlphttp/loki:
+ endpoint: http://loki:3100/otlp
+ timeout: 3s
+ sending_queue:
+ queue_size: 256
+ num_consumers: 2
+ retry_on_failure:
+ max_elapsed_time: 30s
+ otlphttp/tempo:
+ endpoint: http://tempo:4318
+ timeout: 3s
+ sending_queue:
+ queue_size: 256
+ num_consumers: 2
+ retry_on_failure:
+ max_elapsed_time: 30s
+extensions:
+ health_check:
+ endpoint: 0.0.0.0:13133
+service:
+ telemetry:
+ metrics:
+ readers:
+ - pull:
+ exporter:
+ prometheus:
+ host: 0.0.0.0
+ port: 8888
+ extensions: [health_check]
+ pipelines:
+ metrics:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [prometheus]
+ logs:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [otlphttp/loki]
+ traces:
+ receivers: [otlp]
+ processors: [memory_limiter, batch]
+ exporters: [otlphttp/tempo]
diff --git a/infra/otel/grafana/dashboards/messenger.json b/infra/otel/grafana/dashboards/messenger.json
new file mode 100644
index 0000000..b21cf37
--- /dev/null
+++ b/infra/otel/grafana/dashboards/messenger.json
@@ -0,0 +1,617 @@
+{
+ "uid": "messenger-overview",
+ "title": "Messenger — наблюдаемость",
+ "schemaVersion": 39,
+ "version": 1,
+ "tags": [
+ "messenger",
+ "observability"
+ ],
+ "timezone": "browser",
+ "refresh": "5s",
+ "time": {
+ "from": "now-15m",
+ "to": "now"
+ },
+ "panels": [
+ {
+ "id": 1,
+ "title": "API: запросы в секунду",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 0,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by (http_request_method, http_route) (rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m]))",
+ "legendFormat": "{{http_request_method}} {{http_route}}"
+ }
+ ],
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "reqps"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 2,
+ "title": "API: доля 5xx",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 0,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "(sum(rate(messenger_http_requests_total{http_response_status_class=\"5xx\",http_route!~\"/health/.*\"}[1m])) or vector(0)) / clamp_min(sum(rate(messenger_http_requests_total{http_route!~\"/health/.*\"}[1m])), 0.000001)",
+ "legendFormat": "5xx / все бизнес-запросы"
+ }
+ ],
+ "description": "Ошибки сервера: 5xx / все запросы, кроме health. 4xx не считаются отказом сервера.",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 3,
+ "title": "API: время ответа p50/p95/p99",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 9,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.5, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
+ "legendFormat": "p50"
+ },
+ {
+ "refId": "B",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
+ "legendFormat": "p95"
+ },
+ {
+ "refId": "C",
+ "expr": "histogram_quantile(0.99, sum by (le) (rate(messenger_http_duration_seconds_bucket{http_route!~\"/health/.*\"}[1m])))",
+ "legendFormat": "p99"
+ }
+ ],
+ "description": "Histogram в секундах, buckets суммируются по репликам.",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 4,
+ "title": "Worker: попытки в секунду по исходу",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 9,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by (outcome) (rate(messenger_job_attempts_total[1m]))",
+ "legendFormat": "{{outcome}}"
+ }
+ ],
+ "description": "success — результат записан; error — попытка не удалась; deferred — истёк lease. Дубли без обработки не считаются.",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "ops"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 5,
+ "title": "Worker: время обработки p95",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 18,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.95, sum by (le, outcome) (rate(messenger_job_duration_seconds_bucket[1m])))",
+ "legendFormat": "{{outcome}}"
+ }
+ ],
+ "description": "Одна фактическая попытка, включая S3, преобразования и фиксацию metadata.",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 6,
+ "title": "Worker: обработанные попытки",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 18,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by (outcome) (messenger_job_attempts_total)",
+ "legendFormat": "{{outcome}}"
+ }
+ ],
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 7,
+ "title": "API: структурные логи",
+ "type": "logs",
+ "datasource": {
+ "type": "loki",
+ "uid": "loki"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 27,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "{service_name=\"messenger-api\"} | json",
+ "queryType": "range"
+ }
+ ],
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 8,
+ "title": "Worker: структурные логи",
+ "type": "logs",
+ "datasource": {
+ "type": "loki",
+ "uid": "loki"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 27,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "{service_name=\"messenger-worker\"} | json",
+ "queryType": "range"
+ }
+ ],
+ "description": "",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 9,
+ "title": "Реальные server/consumer traces",
+ "type": "table",
+ "datasource": {
+ "type": "tempo",
+ "uid": "tempo"
+ },
+ "gridPos": {
+ "x": 0,
+ "y": 36,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "queryType": "traceqlSearch",
+ "query": "{ resource.service.name =~ \"messenger-api|messenger-worker\" }",
+ "limit": 20
+ }
+ ],
+ "description": "Открыть trace в Explore; ссылка TraceID из строки Loki ведёт к той же трассе.",
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {}
+ },
+ {
+ "id": 10,
+ "title": "API: число 5xx бизнес-запросов",
+ "type": "stat",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "x": 12,
+ "y": 36,
+ "w": 12,
+ "h": 9
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(messenger_http_requests_total{http_response_status_class=\"5xx\",http_route!~\"/health/.*\"}) or vector(0)",
+ "instant": true
+ }
+ ],
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "options": {
+ "reduceOptions": {
+ "calcs": [
+ "lastNotNull"
+ ],
+ "fields": "",
+ "values": false
+ }
+ },
+ "description": "Накопленное число 5xx: показывает и первую ошибку нового ряда, когда rate ещё не имеет двух разных точек."
+ },
+ {
+ "id": 11,
+ "title": "Очередь: ожидающие задания",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 45
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "max by (queue_name) (messenger_queue_depth_ratio)",
+ "legendFormat": "Глубина {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ },
+ {
+ "id": 12,
+ "title": "Очередь: возраст старейшей задачи",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 45
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "max by (queue_name) (messenger_queue_oldest_seconds)",
+ "legendFormat": "Возраст {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ },
+ {
+ "id": 13,
+ "title": "PostgreSQL: соединения и ожидающие",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 53
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by (service_name) (messenger_db_pool_used_ratio)",
+ "legendFormat": "Занято {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ },
+ {
+ "refId": "B",
+ "expr": "sum by (service_name) (messenger_db_pool_waiting_ratio)",
+ "legendFormat": "Ожидают {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ },
+ {
+ "id": 14,
+ "title": "Worker: доля занятых слотов",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 53
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "percentunit"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum(messenger_worker_busy_ratio) / sum(messenger_worker_capacity_ratio)",
+ "legendFormat": "Загрузка {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ },
+ {
+ "id": 15,
+ "title": "Процессы: CPU в ядрах",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 61
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by (service_name) (rate(messenger_process_cpu_seconds_total[1m]))",
+ "legendFormat": "CPU {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ },
+ {
+ "id": 16,
+ "title": "Процессы: резидентная память",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 61
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "bytes"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "sum by (service_name) (messenger_process_rss_bytes)",
+ "legendFormat": "RSS {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ },
+ {
+ "id": 17,
+ "title": "Вложения: p95 времени до ready",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 0,
+ "y": 69
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "s"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "histogram_quantile(0.95, sum by (le) (rate(messenger_job_time_to_ready_seconds_bucket[5m])))",
+ "legendFormat": "p95 {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ },
+ {
+ "id": 18,
+ "title": "Collector: очереди экспорта",
+ "type": "timeseries",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ },
+ "gridPos": {
+ "h": 8,
+ "w": 12,
+ "x": 12,
+ "y": 69
+ },
+ "fieldConfig": {
+ "defaults": {
+ "unit": "short"
+ },
+ "overrides": []
+ },
+ "targets": [
+ {
+ "refId": "A",
+ "expr": "max by (exporter) (otelcol_exporter_queue_size)",
+ "legendFormat": "Размер {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ },
+ {
+ "refId": "B",
+ "expr": "max by (exporter) (otelcol_exporter_queue_capacity)",
+ "legendFormat": "Ёмкость {{service_name}}{{queue_name}}{{exporter}}",
+ "datasource": {
+ "type": "prometheus",
+ "uid": "prometheus"
+ }
+ }
+ ]
+ }
+ ],
+ "annotations": {
+ "list": []
+ },
+ "templating": {
+ "list": []
+ }
+}
diff --git a/infra/otel/grafana/provisioning/dashboards/provider.yaml b/infra/otel/grafana/provisioning/dashboards/provider.yaml
new file mode 100644
index 0000000..8af1de4
--- /dev/null
+++ b/infra/otel/grafana/provisioning/dashboards/provider.yaml
@@ -0,0 +1,10 @@
+apiVersion: 1
+providers:
+ - name: Messenger
+ folder: Messenger
+ type: file
+ disableDeletion: true
+ editable: false
+ updateIntervalSeconds: 10
+ options:
+ path: /var/lib/grafana/dashboards
diff --git a/infra/otel/grafana/provisioning/datasources/sources.yaml b/infra/otel/grafana/provisioning/datasources/sources.yaml
new file mode 100644
index 0000000..d95980b
--- /dev/null
+++ b/infra/otel/grafana/provisioning/datasources/sources.yaml
@@ -0,0 +1,36 @@
+apiVersion: 1
+datasources:
+ - name: Prometheus
+ uid: prometheus
+ type: prometheus
+ access: proxy
+ url: http://prometheus:9090
+ isDefault: true
+ editable: false
+ - name: Loki
+ uid: loki
+ type: loki
+ access: proxy
+ url: http://loki:3100
+ editable: false
+ jsonData:
+ derivedFields:
+ - name: TraceID
+ matcherRegex: '"trace_id":\s*"([0-9a-f]{32})"'
+ datasourceUid: tempo
+ url: '$${__value.raw}'
+ urlDisplayLabel: Open trace
+ - name: Tempo
+ uid: tempo
+ type: tempo
+ access: proxy
+ url: http://tempo:3200
+ editable: false
+ jsonData:
+ tracesToLogsV2:
+ datasourceUid: loki
+ spanStartTimeShift: '-1m'
+ spanEndTimeShift: '1m'
+ tags: [{key: service.name, value: service_name}]
+ filterByTraceID: true
+ filterBySpanID: false
diff --git a/infra/otel/loki.yaml b/infra/otel/loki.yaml
new file mode 100644
index 0000000..b022f72
--- /dev/null
+++ b/infra/otel/loki.yaml
@@ -0,0 +1,39 @@
+auth_enabled: false
+server:
+ http_listen_port: 3100
+common:
+ path_prefix: /loki
+ replication_factor: 1
+ ring:
+ kvstore:
+ store: inmemory
+ storage:
+ filesystem:
+ chunks_directory: /loki/chunks
+ rules_directory: /loki/rules
+schema_config:
+ configs:
+ - from: 2024-01-01
+ store: tsdb
+ object_store: filesystem
+ schema: v13
+ index:
+ prefix: index_
+ period: 24h
+limits_config:
+ allow_structured_metadata: true
+ retention_period: 72h
+ otlp_config:
+ resource_attributes:
+ ignore_defaults: true
+ attributes_config:
+ - action: index_label
+ attributes: [service.name, deployment.environment.name]
+ - action: structured_metadata
+ attributes: [service.version, service.instance.id]
+compactor:
+ working_directory: /loki/compactor
+ retention_enabled: true
+ delete_request_store: filesystem
+analytics:
+ reporting_enabled: false
diff --git a/infra/otel/prometheus.yaml b/infra/otel/prometheus.yaml
new file mode 100644
index 0000000..95b65b8
--- /dev/null
+++ b/infra/otel/prometheus.yaml
@@ -0,0 +1,11 @@
+global:
+ scrape_interval: 5s
+ evaluation_interval: 5s
+scrape_configs:
+ - job_name: messenger
+ static_configs:
+ - targets: [otel-collector:8889]
+ - job_name: collector
+ static_configs:
+ - targets: [otel-collector:8888]
+rule_files: [/etc/prometheus/alerts.yaml]
diff --git a/infra/otel/tempo.yaml b/infra/otel/tempo.yaml
new file mode 100644
index 0000000..85bfea8
--- /dev/null
+++ b/infra/otel/tempo.yaml
@@ -0,0 +1,22 @@
+server:
+ http_listen_port: 3200
+distributor:
+ receivers:
+ otlp:
+ protocols:
+ http:
+ endpoint: 0.0.0.0:4318
+ingester:
+ max_block_duration: 1m
+compactor:
+ compaction:
+ block_retention: 72h
+storage:
+ trace:
+ backend: local
+ wal:
+ path: /var/tempo/wal
+ local:
+ path: /var/tempo/blocks
+usage_report:
+ reporting_enabled: false
diff --git a/infra/redis/redis.conf b/infra/redis/redis.conf
new file mode 100644
index 0000000..f2ae982
--- /dev/null
+++ b/infra/redis/redis.conf
@@ -0,0 +1,5 @@
+dir /data
+appendonly yes
+appendfsync always
+maxmemory 256mb
+maxmemory-policy noeviction
diff --git a/infra/redis/start-redis.sh b/infra/redis/start-redis.sh
new file mode 100644
index 0000000..49ee706
--- /dev/null
+++ b/infra/redis/start-redis.sh
@@ -0,0 +1,10 @@
+#!/bin/sh
+set -eu
+
+password_hash=$(tr -d '\r\n' /run/redis/users.acl
+chown redis:redis /run/redis/users.acl
+chmod 600 /run/redis/users.acl
+exec /usr/local/bin/docker-entrypoint.sh redis-server /usr/local/etc/redis/redis.conf --aclfile /run/redis/users.acl
diff --git a/infra/testing/Dockerfile b/infra/testing/Dockerfile
new file mode 100644
index 0000000..59cddb5
--- /dev/null
+++ b/infra/testing/Dockerfile
@@ -0,0 +1,10 @@
+FROM docker:29.1.3-cli AS dockercli
+FROM python:3.12.15-slim-bookworm
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONUTF8=1 PIP_NO_CACHE_DIR=1
+COPY --from=dockercli /usr/local/bin/docker /usr/local/bin/docker
+COPY --from=dockercli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins
+WORKDIR /workspace
+COPY requirements.txt requirements-dev.txt ./
+RUN pip install -r requirements.txt -r requirements-dev.txt
+COPY scripts/container_tests.py /usr/local/bin/container_tests.py
+CMD ["python3", "/usr/local/bin/container_tests.py"]
diff --git a/migrations/env.py b/migrations/env.py
new file mode 100644
index 0000000..2b7f9d9
--- /dev/null
+++ b/migrations/env.py
@@ -0,0 +1,68 @@
+import asyncio
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import pool
+from sqlalchemy.engine import Connection
+from sqlalchemy.ext.asyncio import async_engine_from_config
+
+from app.core.config import Settings
+from app.models import Base
+
+config = context.config
+
+if config.config_file_name is not None:
+ fileConfig(config.config_file_name, disable_existing_loggers=False)
+
+target_metadata = Base.metadata
+
+
+def _database_url() -> str:
+ explicit = config.get_main_option("sqlalchemy.url")
+ if explicit:
+ return explicit
+ return Settings().get_database_url()
+
+
+def run_migrations_offline() -> None:
+ context.configure(
+ url=_database_url(),
+ target_metadata=target_metadata,
+ literal_binds=True,
+ compare_type=True,
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def do_run_migrations(connection: Connection) -> None:
+ context.configure(
+ connection=connection,
+ target_metadata=target_metadata,
+ compare_type=True,
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+async def run_async_migrations() -> None:
+ section = config.get_section(config.config_ini_section, {})
+ section["sqlalchemy.url"] = _database_url().replace("%", "%%")
+ connectable = async_engine_from_config(
+ section, prefix="sqlalchemy.", poolclass=pool.NullPool
+ )
+
+ async with connectable.connect() as connection:
+ await connection.run_sync(do_run_migrations)
+
+ await connectable.dispose()
+
+
+def run_migrations_online() -> None:
+ asyncio.run(run_async_migrations())
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/migrations/script.py.mako b/migrations/script.py.mako
new file mode 100644
index 0000000..17dcba0
--- /dev/null
+++ b/migrations/script.py.mako
@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+ ${downgrades if downgrades else "pass"}
diff --git a/migrations/versions/49d2c947a3f1_add_idempotency_keys.py b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py
new file mode 100644
index 0000000..153425e
--- /dev/null
+++ b/migrations/versions/49d2c947a3f1_add_idempotency_keys.py
@@ -0,0 +1,48 @@
+"""Добавляет результаты идемпотентных запросов."""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "49d2c947a3f1"
+down_revision = "abd273076068"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ """Создаёт таблицу ключей без изменения существующих данных."""
+ op.create_table(
+ "idempotency_keys",
+ sa.Column("user_id", sa.UUID(), nullable=False),
+ sa.Column("method", sa.String(8), nullable=False),
+ sa.Column("path", sa.String(255), nullable=False),
+ sa.Column("key", sa.String(128), nullable=False),
+ sa.Column("request_hash", sa.String(64), nullable=False),
+ sa.Column("message_id", sa.UUID(), nullable=False),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("clock_timestamp()"),
+ nullable=False,
+ ),
+ sa.ForeignKeyConstraint(
+ ["user_id"], ["users.id"], name=op.f("fk_idempotency_keys_user_id_users")
+ ),
+ sa.ForeignKeyConstraint(
+ ["message_id"],
+ ["messages.id"],
+ name=op.f("fk_idempotency_keys_message_id_messages"),
+ ),
+ sa.PrimaryKeyConstraint(
+ "user_id", "method", "path", "key", name=op.f("pk_idempotency_keys")
+ ),
+ )
+ op.create_index(
+ "ix_idempotency_keys_created_at", "idempotency_keys", ["created_at"]
+ )
+
+
+def downgrade() -> None:
+ """Удаляет таблицу результатов запросов."""
+ op.drop_index("ix_idempotency_keys_created_at", table_name="idempotency_keys")
+ op.drop_table("idempotency_keys")
diff --git a/migrations/versions/8f31b94a2d10_add_attachments.py b/migrations/versions/8f31b94a2d10_add_attachments.py
new file mode 100644
index 0000000..34d3a6d
--- /dev/null
+++ b/migrations/versions/8f31b94a2d10_add_attachments.py
@@ -0,0 +1,98 @@
+"""Attachment, связи сообщений и устойчивый журнал загрузок."""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "8f31b94a2d10"
+down_revision = "49d2c947a3f1"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.create_table(
+ "attachments",
+ sa.Column("id", sa.UUID(), nullable=False),
+ sa.Column("chat_id", sa.UUID(), nullable=False),
+ sa.Column("owner_id", sa.UUID(), nullable=False),
+ sa.Column("object_key", sa.String(128), nullable=False),
+ sa.Column("filename", sa.String(255), nullable=False),
+ sa.Column("content_type", sa.String(32), nullable=False),
+ sa.Column("size_bytes", sa.Integer(), nullable=False),
+ sa.Column("sha256", sa.String(64), nullable=False),
+ sa.Column("status", sa.String(16), server_default="ready", nullable=False),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("clock_timestamp()"),
+ nullable=False,
+ ),
+ sa.Column(
+ "updated_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("clock_timestamp()"),
+ nullable=False,
+ ),
+ sa.CheckConstraint(
+ "size_bytes BETWEEN 1 AND 10485760", name=op.f("ck_attachments_size_bytes")
+ ),
+ sa.CheckConstraint(
+ "content_type IN ('image/png', 'image/jpeg')",
+ name=op.f("ck_attachments_content_type"),
+ ),
+ sa.CheckConstraint(
+ "sha256 ~ '^[a-f0-9]{64}$'", name=op.f("ck_attachments_sha256")
+ ),
+ sa.CheckConstraint("status = 'ready'", name=op.f("ck_attachments_status")),
+ sa.ForeignKeyConstraint(
+ ["chat_id"], ["chats.id"], name=op.f("fk_attachments_chat_id_chats")
+ ),
+ sa.ForeignKeyConstraint(
+ ["owner_id"], ["users.id"], name=op.f("fk_attachments_owner_id_users")
+ ),
+ sa.PrimaryKeyConstraint("id", name=op.f("pk_attachments")),
+ sa.UniqueConstraint("object_key", name=op.f("uq_attachments_object_key")),
+ )
+ op.create_index("ix_attachments_chat_id", "attachments", ["chat_id"])
+ op.create_table(
+ "message_attachments",
+ sa.Column("message_id", sa.UUID(), nullable=False),
+ sa.Column("attachment_id", sa.UUID(), nullable=False),
+ sa.Column("position", sa.Integer(), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["message_id"],
+ ["messages.id"],
+ ondelete="CASCADE",
+ name=op.f("fk_message_attachments_message_id_messages"),
+ ),
+ sa.ForeignKeyConstraint(
+ ["attachment_id"],
+ ["attachments.id"],
+ name=op.f("fk_message_attachments_attachment_id_attachments"),
+ ),
+ sa.PrimaryKeyConstraint(
+ "message_id", "attachment_id", name=op.f("pk_message_attachments")
+ ),
+ )
+ op.create_table(
+ "upload_intents",
+ sa.Column("id", sa.UUID(), nullable=False),
+ sa.Column("object_key", sa.String(128), nullable=False),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("clock_timestamp()"),
+ nullable=False,
+ ),
+ sa.PrimaryKeyConstraint("id", name=op.f("pk_upload_intents")),
+ sa.UniqueConstraint("object_key", name=op.f("uq_upload_intents_object_key")),
+ )
+ op.create_index("ix_upload_intents_created_at", "upload_intents", ["created_at"])
+
+
+def downgrade() -> None:
+ op.drop_index("ix_upload_intents_created_at", table_name="upload_intents")
+ op.drop_table("upload_intents")
+ op.drop_table("message_attachments")
+ op.drop_index("ix_attachments_chat_id", table_name="attachments")
+ op.drop_table("attachments")
diff --git a/migrations/versions/a62c950e18b3_rabbitmq_cutover.py b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py
new file mode 100644
index 0000000..c343677
--- /dev/null
+++ b/migrations/versions/a62c950e18b3_rabbitmq_cutover.py
@@ -0,0 +1,23 @@
+"""Offline cutover: повторно опубликовать queued jobs в RabbitMQ."""
+
+from alembic import op
+
+revision = "a62c950e18b3"
+down_revision = "f9183d27a640"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+
+ op.execute(
+ """
+ UPDATE attachment_jobs j SET published_attempt = attempt
+ FROM attachments a WHERE a.id = j.attachment_id AND a.status = 'queued'
+ """
+ )
+
+
+def downgrade() -> None:
+
+ upgrade()
diff --git a/migrations/versions/abd273076068_add_counters.py b/migrations/versions/abd273076068_add_counters.py
new file mode 100644
index 0000000..57877dd
--- /dev/null
+++ b/migrations/versions/abd273076068_add_counters.py
@@ -0,0 +1,32 @@
+"""add counters
+
+Revision ID: abd273076068
+Revises: db06fd3126ff
+Create Date: 2026-10-26 22:01:01.910396
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "abd273076068"
+down_revision: Union[str, None] = "db06fd3126ff"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+
+ op.create_table(
+ "counters",
+ sa.Column("name", sa.String(length=64), nullable=False),
+ sa.Column("value", sa.BigInteger(), nullable=False),
+ sa.PrimaryKeyConstraint("name", name=op.f("pk_counters")),
+ )
+
+
+def downgrade() -> None:
+
+ op.drop_table("counters")
diff --git a/migrations/versions/b55d0c6819ab_job_trace_context.py b/migrations/versions/b55d0c6819ab_job_trace_context.py
new file mode 100644
index 0000000..ce20f75
--- /dev/null
+++ b/migrations/versions/b55d0c6819ab_job_trace_context.py
@@ -0,0 +1,19 @@
+"""Сохранить W3C context между HTTP-транзакцией и publisher."""
+
+import sqlalchemy as sa
+from alembic import op
+
+revision = "b55d0c6819ab"
+down_revision = "a62c950e18b3"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+ op.add_column(
+ "attachment_jobs", sa.Column("trace_context", sa.JSON(), nullable=True)
+ )
+
+
+def downgrade():
+ op.drop_column("attachment_jobs", "trace_context")
diff --git a/migrations/versions/c35e91a2f706_async_attachment_jobs.py b/migrations/versions/c35e91a2f706_async_attachment_jobs.py
new file mode 100644
index 0000000..3acb20c
--- /dev/null
+++ b/migrations/versions/c35e91a2f706_async_attachment_jobs.py
@@ -0,0 +1,113 @@
+"""Состояния, варианты и устойчивые jobs с намерением публикации."""
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+revision = "c35e91a2f706"
+down_revision = "8f31b94a2d10"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
+ op.create_check_constraint(
+ op.f("ck_attachments_status"),
+ "attachments",
+ "status IN ('queued', 'processing', 'ready', 'failed')",
+ )
+ op.alter_column("attachments", "status", server_default="queued")
+ op.add_column("attachments", sa.Column("error_code", sa.String(64), nullable=True))
+ op.add_column(
+ "attachments", sa.Column("thumbnail_key", sa.String(128), nullable=True)
+ )
+ op.add_column(
+ "attachments",
+ sa.Column(
+ "variants",
+ postgresql.JSONB(),
+ nullable=False,
+ server_default=sa.text("'[]'::jsonb"),
+ ),
+ )
+ op.create_check_constraint(
+ op.f("ck_attachments_error_code"),
+ "attachments",
+ "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)",
+ )
+ op.create_table(
+ "attachment_jobs",
+ sa.Column("id", sa.UUID(), nullable=False),
+ sa.Column("attachment_id", sa.UUID(), nullable=False),
+ sa.Column("staging_key", sa.String(128), nullable=True),
+ sa.Column("pipeline_version", sa.Integer(), server_default="1", nullable=False),
+ sa.Column("attempt", sa.Integer(), server_default="0", nullable=False),
+ sa.Column(
+ "published_attempt", sa.Integer(), server_default="0", nullable=False
+ ),
+ sa.Column(
+ "next_attempt_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("clock_timestamp()"),
+ nullable=False,
+ ),
+ sa.Column("lease_token", sa.UUID(), nullable=True),
+ sa.Column("lease_until", sa.DateTime(timezone=True), nullable=True),
+ sa.Column("last_error_code", sa.String(64), nullable=True),
+ sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("clock_timestamp()"),
+ nullable=False,
+ ),
+ sa.PrimaryKeyConstraint("id", name=op.f("pk_attachment_jobs")),
+ sa.ForeignKeyConstraint(
+ ["attachment_id"],
+ ["attachments.id"],
+ name=op.f("fk_attachment_jobs_attachment_id_attachments"),
+ ),
+ sa.UniqueConstraint(
+ "attachment_id", name=op.f("uq_attachment_jobs_attachment_id")
+ ),
+ sa.CheckConstraint(
+ "pipeline_version = 1", name=op.f("ck_attachment_jobs_pipeline_version")
+ ),
+ sa.CheckConstraint(
+ "attempt >= 0 AND published_attempt >= 0",
+ name=op.f("ck_attachment_jobs_attempt"),
+ ),
+ )
+ op.create_index("ix_attachment_jobs_due", "attachment_jobs", ["next_attempt_at"])
+ op.create_index("ix_attachment_jobs_lease", "attachment_jobs", ["lease_until"])
+
+ op.execute(
+ "INSERT INTO attachment_jobs (id, attachment_id, staging_key) SELECT id, id, object_key FROM attachments"
+ )
+ op.execute(
+ "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()"
+ )
+
+
+def downgrade() -> None:
+ connection = op.get_bind()
+ unfinished = connection.scalar(
+ sa.text("SELECT count(*) FROM attachments WHERE status <> 'ready'")
+ )
+ if unfinished:
+ raise RuntimeError(
+ "Перед downgrade нужно завершить или вручную разобрать все queued/processing/failed вложения"
+ )
+ op.drop_index("ix_attachment_jobs_lease", table_name="attachment_jobs")
+ op.drop_index("ix_attachment_jobs_due", table_name="attachment_jobs")
+ op.drop_table("attachment_jobs")
+ op.drop_constraint(op.f("ck_attachments_error_code"), "attachments", type_="check")
+ op.drop_column("attachments", "variants")
+ op.drop_column("attachments", "thumbnail_key")
+ op.drop_column("attachments", "error_code")
+ op.drop_constraint(op.f("ck_attachments_status"), "attachments", type_="check")
+ op.create_check_constraint(
+ op.f("ck_attachments_status"), "attachments", "status = 'ready'"
+ )
+ op.alter_column("attachments", "status", server_default="ready")
diff --git a/migrations/versions/db06fd3126ff_create_initial_schema.py b/migrations/versions/db06fd3126ff_create_initial_schema.py
new file mode 100644
index 0000000..77e552e
--- /dev/null
+++ b/migrations/versions/db06fd3126ff_create_initial_schema.py
@@ -0,0 +1,105 @@
+"""create initial schema
+
+Revision ID: db06fd3126ff
+Revises:
+Create Date: 2026-10-26 17:09:16.675339
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "db06fd3126ff"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+
+ op.create_table(
+ "chats",
+ sa.Column("id", sa.UUID(), nullable=False),
+ sa.Column("title", sa.String(length=100), nullable=False),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("now()"),
+ nullable=False,
+ ),
+ sa.PrimaryKeyConstraint("id", name=op.f("pk_chats")),
+ )
+ op.create_table(
+ "users",
+ sa.Column("id", sa.UUID(), nullable=False),
+ sa.Column("username", sa.String(length=32), nullable=False),
+ sa.Column("display_name", sa.String(length=100), nullable=False),
+ sa.Column("password_hash", sa.String(length=255), nullable=False),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("now()"),
+ nullable=False,
+ ),
+ sa.PrimaryKeyConstraint("id", name=op.f("pk_users")),
+ sa.UniqueConstraint("username", name=op.f("uq_users_username")),
+ )
+ op.create_table(
+ "chat_members",
+ sa.Column("chat_id", sa.UUID(), nullable=False),
+ sa.Column("user_id", sa.UUID(), nullable=False),
+ sa.ForeignKeyConstraint(
+ ["chat_id"],
+ ["chats.id"],
+ name=op.f("fk_chat_members_chat_id_chats"),
+ ondelete="CASCADE",
+ ),
+ sa.ForeignKeyConstraint(
+ ["user_id"], ["users.id"], name=op.f("fk_chat_members_user_id_users")
+ ),
+ sa.PrimaryKeyConstraint("chat_id", "user_id", name=op.f("pk_chat_members")),
+ )
+ op.create_index(
+ "ix_chat_members_user_id_chat_id",
+ "chat_members",
+ ["user_id", "chat_id"],
+ unique=False,
+ )
+ op.create_table(
+ "messages",
+ sa.Column("id", sa.UUID(), nullable=False),
+ sa.Column("chat_id", sa.UUID(), nullable=False),
+ sa.Column("sender_id", sa.UUID(), nullable=False),
+ sa.Column("text", sa.Text(), nullable=False),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ server_default=sa.text("clock_timestamp()"),
+ nullable=False,
+ ),
+ sa.ForeignKeyConstraint(
+ ["chat_id"], ["chats.id"], name=op.f("fk_messages_chat_id_chats")
+ ),
+ sa.ForeignKeyConstraint(
+ ["sender_id"], ["users.id"], name=op.f("fk_messages_sender_id_users")
+ ),
+ sa.PrimaryKeyConstraint("id", name=op.f("pk_messages")),
+ )
+ op.create_index(
+ "ix_messages_chat_id_created_at_id",
+ "messages",
+ ["chat_id", "created_at", "id"],
+ unique=False,
+ )
+
+
+def downgrade() -> None:
+
+ op.drop_index("ix_messages_chat_id_created_at_id", table_name="messages")
+ op.drop_table("messages")
+ op.drop_index("ix_chat_members_user_id_chat_id", table_name="chat_members")
+ op.drop_table("chat_members")
+ op.drop_table("users")
+ op.drop_table("chats")
diff --git a/migrations/versions/e7048b6da913_attachment_crop_webp.py b/migrations/versions/e7048b6da913_attachment_crop_webp.py
new file mode 100644
index 0000000..153c50b
--- /dev/null
+++ b/migrations/versions/e7048b6da913_attachment_crop_webp.py
@@ -0,0 +1,63 @@
+"""Ключи crop/WebP и дообработка старых ready-вложений."""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "e7048b6da913"
+down_revision = "c35e91a2f706"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.add_column("attachments", sa.Column("crop_key", sa.String(128), nullable=True))
+ op.add_column(
+ "attachments", sa.Column("optimized_key", sa.String(128), nullable=True)
+ )
+
+ op.execute(
+ """
+ INSERT INTO upload_intents (id, object_key)
+ SELECT gen_random_uuid(), j.staging_key
+ FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
+ WHERE a.status = 'ready' AND j.staging_key IS NOT NULL
+ AND j.staging_key <> a.object_key
+ ON CONFLICT (object_key) DO NOTHING
+ """
+ )
+ op.execute(
+ """
+ UPDATE attachment_jobs j
+ SET staging_key = a.object_key, attempt = 0, published_attempt = 0,
+ next_attempt_at = clock_timestamp(), lease_token = NULL, lease_until = NULL,
+ last_error_code = NULL, finished_at = NULL
+ FROM attachments a
+ WHERE a.id = j.attachment_id AND a.status = 'ready'
+ """
+ )
+ op.execute(
+ """
+ UPDATE attachments SET status = 'queued', variants = '[]'::jsonb,
+ updated_at = clock_timestamp()
+ WHERE status = 'ready'
+ """
+ )
+
+
+def downgrade() -> None:
+ active = op.get_bind().scalar(
+ sa.text(
+ "SELECT count(*) FROM attachments WHERE status IN ('queued', 'processing')"
+ )
+ )
+ if active:
+ raise RuntimeError("Перед downgrade необходимо завершить активные задания")
+ op.execute(
+ """
+ UPDATE attachments a SET variants = COALESCE(
+ (SELECT jsonb_agg(v) FROM jsonb_array_elements(a.variants) v
+ WHERE v->>'name' = 'thumbnail'), '[]'::jsonb)
+ """
+ )
+ op.drop_column("attachments", "optimized_key")
+ op.drop_column("attachments", "crop_key")
diff --git a/migrations/versions/f9183d27a640_job_replay.py b/migrations/versions/f9183d27a640_job_replay.py
new file mode 100644
index 0000000..0c495fb
--- /dev/null
+++ b/migrations/versions/f9183d27a640_job_replay.py
@@ -0,0 +1,51 @@
+"""Бюджет повторов и аудит операторского replay."""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "f9183d27a640"
+down_revision = "e7048b6da913"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.add_column(
+ "attachment_jobs",
+ sa.Column(
+ "cycle_start_attempt", sa.Integer(), nullable=False, server_default="0"
+ ),
+ )
+ op.add_column(
+ "attachment_jobs", sa.Column("attempt_limit", sa.Integer(), nullable=True)
+ )
+ op.add_column(
+ "attachment_jobs",
+ sa.Column("last_error_permanent", sa.Boolean(), nullable=True),
+ )
+ op.create_table(
+ "job_replays",
+ sa.Column("id", sa.Uuid(), primary_key=True),
+ sa.Column(
+ "job_id", sa.Uuid(), sa.ForeignKey("attachment_jobs.id"), nullable=False
+ ),
+ sa.Column("actor", sa.String(64), nullable=False),
+ sa.Column("reason", sa.String(32), nullable=False),
+ sa.Column("previous_attempt", sa.Integer(), nullable=False),
+ sa.Column("attempt_limit", sa.Integer(), nullable=False),
+ sa.Column("previous_error_code", sa.String(64), nullable=True),
+ sa.Column(
+ "created_at",
+ sa.DateTime(timezone=True),
+ nullable=False,
+ server_default=sa.func.clock_timestamp(),
+ ),
+ )
+ op.create_index("ix_job_replays_job_id", "job_replays", ["job_id"])
+
+
+def downgrade() -> None:
+ op.drop_table("job_replays")
+ op.drop_column("attachment_jobs", "last_error_permanent")
+ op.drop_column("attachment_jobs", "attempt_limit")
+ op.drop_column("attachment_jobs", "cycle_start_attempt")
diff --git a/pytest.ini b/pytest.ini
new file mode 100644
index 0000000..b7dd8fa
--- /dev/null
+++ b/pytest.ini
@@ -0,0 +1,6 @@
+[pytest]
+testpaths = student-tests
+python_files = test_*.py
+asyncio_mode = auto
+asyncio_default_fixture_loop_scope = session
+asyncio_default_test_loop_scope = session
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 6b103dd..823485e 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,2 +1,2 @@
-# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
openapi-spec-validator==0.7.2
+black==25.11.0
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..15ce048
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,56 @@
+alembic==1.20.0
+annotated-doc==0.0.5
+annotated-types==0.8.0
+anyio==4.15.1
+attrs==26.1.0
+certifi==2026.7.22
+charset-normalizer==3.5.2
+click==8.5.0
+fastapi==0.143.3
+h11==0.16.0
+httpcore2==2.13.1
+httpx2==2.13.1
+idna==3.20
+iniconfig==2.3.1
+jsonschema==4.26.0
+jsonschema-path==0.3.4
+jsonschema-specifications==2025.9.1
+lazy-object-proxy==1.12.0
+Mako==1.4.3
+MarkupSafe==3.0.4
+openapi-schema-validator==0.6.3
+openapi-spec-validator==0.7.2
+opentelemetry-api==1.45.1
+opentelemetry-sdk==1.45.1
+opentelemetry-exporter-otlp-proto-http==1.45.1
+opentelemetry-semantic-conventions==0.66b1
+packaging==26.3
+pathable==0.4.4
+pluggy==1.7.0
+pydantic==2.14.1
+pydantic_core==2.50.1
+Pygments==2.21.0
+pytest==9.1.1
+PyYAML==6.0.3
+referencing==0.36.2
+requests==2.34.2
+rfc3339-validator==0.1.4
+rpds-py==2026.9.1
+six==1.17.0
+SQLAlchemy==2.1.4
+starlette==1.8.0
+truststore==0.10.5
+typing-inspection==0.4.4
+typing_extensions==4.16.0
+urllib3==2.8.0
+uvicorn==0.54.0
+asyncpg==0.32.0
+argon2-cffi==25.1.0
+redis==8.1.0
+greenlet==3.5.6
+pydantic-settings==2.16.0
+pytest-asyncio==1.4.1
+httpx==0.28.1
+minio==7.2.20
+Pillow==13.0.0
+aio-pika==10.2.2
diff --git a/scripts/__init__.py b/scripts/__init__.py
new file mode 100644
index 0000000..592ea40
--- /dev/null
+++ b/scripts/__init__.py
@@ -0,0 +1 @@
+"""Команды управления учебным стендом."""
diff --git a/scripts/check_contract.py b/scripts/check_contract.py
index dcb34df..101f0d5 100644
--- a/scripts/check_contract.py
+++ b/scripts/check_contract.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Offline structural checks; full schema validation is a separate target."""
+
import ast
import json
from pathlib import Path
@@ -7,48 +8,71 @@ import re
ROOT = Path(__file__).resolve().parents[1]
+
def main():
- lab = json.loads((ROOT/'lab.json').read_text())['number']
- spec = json.loads((ROOT/'contracts/openapi.json').read_text())
- assert spec['openapi']=='3.1.0'
- assert spec['info']['version']=='1.%d.0'%lab
+ lab = json.loads((ROOT / "lab.json").read_text(encoding="utf-8"))["number"]
+ spec = json.loads((ROOT / "contracts/openapi.json").read_text(encoding="utf-8"))
+ assert spec["openapi"] == "3.1.0"
+ assert spec["info"]["version"] == "1.%d.0" % lab
operation_ids = set()
+
def walk(value):
- if isinstance(value,dict):
- if '$ref' in value:
- pointer = value['$ref']
- assert pointer.startswith('#/'), 'unexpected external reference'
+ if isinstance(value, dict):
+ if "$ref" in value:
+ pointer = value["$ref"]
+ assert pointer.startswith("#/"), "unexpected external reference"
target = spec
- for part in pointer[2:].split('/'):
- target = target[part.replace('~1','/').replace('~0','~')]
+ for part in pointer[2:].split("/"):
+ target = target[part.replace("~1", "/").replace("~0", "~")]
for v in value.values():
walk(v)
- elif isinstance(value,list):
+ elif isinstance(value, list):
for v in value:
walk(v)
- walk(spec)
- for route,methods in spec['paths'].items():
- placeholders = set(re.findall(r'\{([^}]+)\}',route))
- for method,op in methods.items():
- assert op['operationId'] not in operation_ids, 'duplicate operationId'
- operation_ids.add(op['operationId'])
- params = {p['name'] for p in op.get('parameters',[]) if p['in']=='path' and p.get('required')}
- assert placeholders==params, 'path parameter mismatch: '+route
- for response in op['responses'].values():
- assert 'X-Request-Id' in response['headers']
- if lab>=3:
- assert 'X-Instance-Id' in response['headers']
- assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2)
- assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4)
- for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')):
- ast.parse(file.read_text(),filename=str(file))
- for file in ROOT.rglob('*.json'):
- if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts):
- continue
- json.loads(file.read_text())
- for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'):
- assert (ROOT/required).is_file(), 'missing '+required
- print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.')
-if __name__=='__main__':
+ walk(spec)
+ for route, methods in spec["paths"].items():
+ placeholders = set(re.findall(r"\{([^}]+)\}", route))
+ for method, op in methods.items():
+ assert op["operationId"] not in operation_ids, "duplicate operationId"
+ operation_ids.add(op["operationId"])
+ params = {
+ p["name"]
+ for p in op.get("parameters", [])
+ if p["in"] == "path" and p.get("required")
+ }
+ assert placeholders == params, "path parameter mismatch: " + route
+ for response in op["responses"].values():
+ assert "X-Request-Id" in response["headers"]
+ if lab >= 3:
+ assert "X-Instance-Id" in response["headers"]
+ assert ("/api/v1/auth/sessions" in spec["paths"]) == (lab >= 2)
+ assert ("/api/v1/attachments/{attachment_id}" in spec["paths"]) == (lab >= 4)
+ for file in list((ROOT / "tests").glob("*.py")) + list(
+ (ROOT / "scripts").glob("*.py")
+ ):
+ ast.parse(file.read_text(encoding="utf-8"), filename=str(file))
+ for file in ROOT.rglob("*.json"):
+ if any(
+ part in {".git", ".venv", "node_modules", "vendor", "output"}
+ for part in file.parts
+ ):
+ continue
+ json.loads(file.read_text(encoding="utf-8"))
+ for required in (
+ "README.md",
+ "COURSE.md",
+ "CONTRIBUTING.md",
+ "REPORT.md",
+ "contracts/README.md",
+ "tests/smoke.py",
+ "tests/README.md",
+ ):
+ assert (ROOT / required).is_file(), "missing " + required
+ print(
+ "PASS: offline skeleton structure, local refs and Python syntax; application NOT tested."
+ )
+
+
+if __name__ == "__main__":
main()
diff --git a/scripts/check_jobs.py b/scripts/check_jobs.py
new file mode 100644
index 0000000..81c3e63
--- /dev/null
+++ b/scripts/check_jobs.py
@@ -0,0 +1,431 @@
+"""Fault/recovery очереди на явно выбранном Compose-стенде; только синтетические данные."""
+
+import argparse
+from datetime import datetime
+import hashlib
+from io import BytesIO
+import json
+from pathlib import Path
+import subprocess
+import time
+from uuid import UUID, uuid4
+
+from PIL import Image
+
+from scripts.check_outbox_recovery import COMPOSE, ROOT, compose, require_compose
+from scripts.deployment import ComposeProject
+from tests.client import Client, check
+
+PROBE = """
+import asyncio, hashlib, json
+from uuid import UUID
+from sqlalchemy import select
+from app.core.config import Settings
+from app.core.database import create_db_engine, create_session_factory
+from app.models.job import AttachmentJob
+from app.models.attachment import Attachment
+from app.storage.s3 import create_s3_storage
+async def main():
+ settings=Settings(); engine=create_db_engine(settings); storage=create_s3_storage(settings.s3)
+ try:
+ async with create_session_factory(engine)() as session:
+ job, att=(await session.execute(select(AttachmentJob,Attachment).join(Attachment,Attachment.id==AttachmentJob.attachment_id).where(Attachment.id==UUID(ATTACHMENT_ID)))).one()
+ result=dict(job_id=str(job.id),status=att.status,attempt=job.attempt,published_attempt=job.published_attempt,limit=job.attempt_limit,updated_at=att.updated_at,next_attempt_at=job.next_attempt_at,last_error_code=job.last_error_code,permanent=job.last_error_permanent)
+ objects=[]
+ for prefix in ('originals/'+UUID(ATTACHMENT_ID).hex+'/', 'variants/'+UUID(ATTACHMENT_ID).hex+'/'):
+ entries=await asyncio.to_thread(lambda: list(storage._client.list_objects(storage.settings.bucket,prefix=prefix,recursive=True)))
+ for obj in entries:
+ stream=await storage.open_object(obj.object_name); checksum=hashlib.sha256()
+ try:
+ async for chunk in stream: checksum.update(chunk)
+ finally: await stream.aclose()
+ objects.append(dict(name=obj.object_name.split('/')[-1],size=obj.size,sha256=checksum.hexdigest(),last_modified=obj.last_modified))
+ result['objects']=sorted(objects,key=lambda x:x['name']);print(json.dumps(result,default=str))
+ finally: await storage.aclose();await engine.dispose()
+asyncio.run(main())
+"""
+
+
+def probe(attachment_id):
+ code = "ATTACHMENT_ID = " + repr(str(UUID(attachment_id))) + "\n" + PROBE
+ result = subprocess.run(
+ COMPOSE + ["exec", "-T", "api", "python", "-"],
+ input=code,
+ encoding="utf-8",
+ capture_output=True,
+ cwd=ROOT,
+ timeout=30,
+ )
+ check(result.returncode == 0, "Probe failed; diagnostic output hidden")
+ return json.loads(result.stdout)
+
+
+def admin(*arguments, expected=0):
+ result = compose(
+ "run",
+ "--rm",
+ "--no-deps",
+ "worker",
+ "python",
+ "-m",
+ "app.core.job_admin",
+ *arguments,
+ )
+ check(result.returncode == expected, "Unexpected operator command exit status")
+ return json.loads(result.stdout)
+
+
+def verify_result(client, actor, attachment, original):
+ ready = client.wait_ready(actor, attachment, wait_seconds=180)
+ check(ready["id"] == attachment["id"], "Attachment identity changed")
+ path = "/api/v1/attachments/" + ready["id"] + "/content"
+ check(
+ client.call("GET", path, actor=actor)["bytes"] == original, "Original changed"
+ )
+ check(
+ {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"},
+ "Incomplete pipeline",
+ )
+ for variant in ready["variants"]:
+ data = client.call("GET", path + "?variant=" + variant["name"], actor=actor)[
+ "bytes"
+ ]
+ check(
+ hashlib.sha256(data).hexdigest() == variant["sha256"], "Checksum mismatch"
+ )
+ with Image.open(BytesIO(data)) as image:
+ check(
+ image.size == (variant["width"], variant["height"]),
+ "Metadata dimensions mismatch",
+ )
+ check(
+ image.format == ("WEBP" if variant["name"] == "optimized" else "JPEG"),
+ "Wrong format",
+ )
+ check(
+ not image.getexif()
+ and not image.info.get("icc_profile")
+ and not image.info.get("xmp"),
+ "Metadata leaked",
+ )
+ if variant["name"] == "crop":
+ check(image.size == (128, 128), "Wrong crop size")
+ snapshot = probe(ready["id"])
+ check(
+ len(snapshot["objects"]) == 4,
+ "Expected exactly one original and three variants",
+ )
+ return snapshot
+
+
+def crash(client, actor, chat, mode):
+ name = "messenger-worker-fault-" + uuid4().hex
+ try:
+ require_compose("stop", "worker")
+ attachment, original = client.attachment(actor, chat)
+ result = compose(
+ "run",
+ "--rm",
+ "--no-deps",
+ "--name",
+ name,
+ "worker",
+ "python",
+ "-m",
+ "app.core.job_worker",
+ "--" + mode,
+ attachment["id"],
+ )
+ phase = (
+ "after_object:optimized" if mode == "crash-after-object" else "before_ack"
+ )
+ marker = json.dumps({"fault": phase, "attachment_id": attachment["id"]})
+ check(
+ result.returncode == 137 and marker in result.stdout,
+ "Targeted crash not observed",
+ )
+ before = probe(attachment["id"])
+ check(
+ before["status"]
+ == ("processing" if mode == "crash-after-object" else "ready"),
+ "Unexpected crash state",
+ )
+ check(len(before["objects"]) == 4, "Crash must occur after all four PUTs")
+ finally:
+ subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
+ require_compose("start", "worker")
+ after = verify_result(client, actor, attachment, original)
+ if mode == "crash-before-ack":
+ # Wait for RabbitMQ to deliver and acknowledge the ready duplicate.
+ wait_queue_empty()
+ after = probe(attachment["id"])
+ check(after == before, "Ready duplicate rewrote result")
+ else:
+ check(
+ after["attempt"] == before["attempt"] + 1,
+ "Lease did not recover the interrupted attempt",
+ )
+ check(
+ [(x["name"], x["sha256"]) for x in before["objects"]]
+ == [(x["name"], x["sha256"]) for x in after["objects"]],
+ "Replay changed object content",
+ )
+ return dict(
+ scenario=mode,
+ attachment_id=attachment["id"],
+ exit_code=137,
+ before=before,
+ after=after,
+ )
+
+
+def retry_replay(client, actor, chat, permanent, on_queued=None):
+ name = "messenger-retry-fault-" + uuid4().hex
+ samples = []
+ configuration = ComposeProject().config["services"]["worker"]["environment"]
+ retry_base = float(configuration["JOB_RETRY_SECONDS"])
+ retry_max = float(configuration["JOB_RETRY_MAX_SECONDS"])
+ try:
+ require_compose("stop", "worker")
+ attachment, original = client.attachment(actor, chat)
+ if on_queued is not None:
+ on_queued(attachment)
+ mode = "fail-permanent" if permanent else "fail-transient"
+ require_compose(
+ "run",
+ "-d",
+ "--no-deps",
+ "--name",
+ name,
+ "worker",
+ "python",
+ "-m",
+ "app.core.job_worker",
+ "--" + mode,
+ attachment["id"],
+ )
+ deadline = time.monotonic() + 180
+ while time.monotonic() < deadline:
+ sample = probe(attachment["id"])
+ if (
+ sample["attempt"]
+ and sample["status"] in {"queued", "failed"}
+ and (not samples or samples[-1]["attempt"] != sample["attempt"])
+ ):
+ samples.append(sample)
+ if sample["status"] == "failed":
+ break
+ time.sleep(0.5)
+ check(sample["status"] == "failed", "Retry never reached failed")
+ check(
+ sample["attempt"] == (1 if permanent else sample["limit"]),
+ "Incorrect attempt budget",
+ )
+ check(sample["permanent"] == permanent, "Incorrect error classification")
+ if not permanent:
+ check(len(samples) == sample["limit"], "Missed retry samples")
+ for entry in samples:
+ delay = (
+ datetime.fromisoformat(entry["next_attempt_at"])
+ - datetime.fromisoformat(entry["updated_at"])
+ ).total_seconds()
+ ceiling = min(retry_max, retry_base * 2 ** (entry["attempt"] - 1))
+ check(
+ ceiling / 2 <= delay <= ceiling,
+ "Retry delay outside configured jitter bounds",
+ )
+ client.call(
+ "GET",
+ "/api/v1/attachments/" + attachment["id"] + "/content",
+ actor=actor,
+ expected=409,
+ )
+ job_id = sample["job_id"]
+ request_id = str(uuid4())
+ listing = admin("list-failed")
+ check(
+ any(row["job_id"] == job_id for row in listing),
+ "Job missing from failed registry",
+ )
+ subprocess.run(
+ ["docker", "rm", "-f", name], capture_output=True, timeout=30, check=True
+ )
+ first = admin(
+ "replay",
+ job_id,
+ "--request-id",
+ request_id,
+ "--actor",
+ "verification",
+ "--reason",
+ "dependency_restored",
+ )
+ check(first["status"] == "queued", "Replay did not queue job")
+ finally:
+ subprocess.run(["docker", "rm", "-f", name], capture_output=True, timeout=30)
+ require_compose("start", "worker")
+ after = verify_result(client, actor, attachment, original)
+ repeat = admin(
+ "replay",
+ job_id,
+ "--request-id",
+ request_id,
+ "--actor",
+ "verification",
+ "--reason",
+ "dependency_restored",
+ )
+ check(repeat["already_applied"], "Replay request id not idempotent")
+ refused = admin(
+ "replay",
+ job_id,
+ "--request-id",
+ str(uuid4()),
+ "--actor",
+ "verification",
+ "--reason",
+ "manual_review",
+ expected=2,
+ )
+ check(refused["error_code"] == "job_not_failed", "Successful job was replayed")
+ check(probe(attachment["id"]) == after, "Operator replay altered ready job")
+ check(len(admin("history", job_id)) == 1, "Incorrect audit count")
+ return dict(
+ scenario=mode,
+ attachment_id=attachment["id"],
+ samples=samples,
+ after=after,
+ replay_idempotent=True,
+ )
+
+
+def queue_state():
+ output = require_compose(
+ "exec",
+ "-T",
+ "broker",
+ "rabbitmqctl",
+ "list_queues",
+ "--formatter=json",
+ "name",
+ "messages_ready",
+ "messages_unacknowledged",
+ "consumers",
+ "durable",
+ )
+ return next(row for row in json.loads(output) if row["name"] == "attachment-jobs")
+
+
+def wait_queue_empty():
+ deadline = time.monotonic() + 20
+ while time.monotonic() < deadline:
+ state = queue_state()
+ if state["messages_ready"] == 0 and state["messages_unacknowledged"] == 0:
+ return state
+ time.sleep(0.5)
+ check(False, "RabbitMQ did not acknowledge all completed deliveries")
+
+
+def persistence(client, actor, chat):
+ try:
+ require_compose("stop", "worker")
+ attachment, original = client.attachment(actor, chat)
+ client.message(actor, chat, "", attachments=[attachment["id"]])
+ client.call(
+ "GET",
+ "/api/v1/attachments/" + attachment["id"] + "/content",
+ actor=actor,
+ expected=409,
+ )
+ time.sleep(2)
+ queued_before = queue_state()
+ check(
+ queued_before["messages_ready"] >= 1 and queued_before["durable"],
+ "No durable queued message before restart",
+ )
+ result = subprocess.run(
+ [str(ROOT / "scripts/restart.sh")],
+ cwd=ROOT,
+ capture_output=True,
+ timeout=180,
+ )
+ check(result.returncode == 0, "Restart hook failed")
+ check(
+ "worker"
+ not in require_compose(
+ "ps", "--status", "running", "--services"
+ ).splitlines(),
+ "Restart started deliberately stopped worker",
+ )
+ queued_after = queue_state()
+ check(
+ queued_after["messages_ready"] >= 1 and queued_after["durable"],
+ "Broker lost queued delivery across restart",
+ )
+ client.message(actor, chat, "session and chat survived broker/API restart")
+ finally:
+ require_compose("start", "worker")
+ after = verify_result(client, actor, attachment, original)
+ return dict(
+ scenario="worker_stopped_restart_all",
+ attachment_id=attachment["id"],
+ queue_before=queued_before,
+ queue_after=queued_after,
+ after=after,
+ )
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--base-url", required=True)
+ parser.add_argument("--ca-file")
+ parser.add_argument(
+ "--scenario",
+ choices=[
+ "all",
+ "persistence",
+ "crash-after-object",
+ "crash-before-ack",
+ "fail-transient",
+ "fail-permanent",
+ ],
+ default="all",
+ )
+ parser.add_argument("--output", default="evidence/jobs.json")
+ args = parser.parse_args()
+ client = Client(args.base_url, args.ca_file, timeout=40)
+ aliases = {r.instance_id for r in ComposeProject().replicas() if r.running}
+ check(
+ client.call("GET", "/health/live", schema="Health")["headers"].get(
+ "X-Instance-Id"
+ )
+ in aliases,
+ "Wrong Compose project for BASE_URL",
+ )
+ actor, _, _, chat = client.bootstrap()
+ results = []
+ operations = {
+ "persistence": persistence,
+ "crash-after-object": lambda *x: crash(*x, "crash-after-object"),
+ "crash-before-ack": lambda *x: crash(*x, "crash-before-ack"),
+ "fail-transient": lambda *x: retry_replay(*x, False),
+ "fail-permanent": lambda *x: retry_replay(*x, True),
+ }
+ selected = (
+ operations.values() if args.scenario == "all" else [operations[args.scenario]]
+ )
+ for operation in selected:
+ result = operation(client, actor, chat)
+ results.append(result)
+ Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+ Path(args.output).write_text(
+ json.dumps(dict(status="in_progress", results=results), indent=2) + "\n"
+ )
+ print("PASS: " + result["scenario"], flush=True)
+ Path(args.output).write_text(
+ json.dumps(dict(status="passed", results=results), indent=2) + "\n"
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/check_observability.py b/scripts/check_observability.py
new file mode 100644
index 0000000..dc74555
--- /dev/null
+++ b/scripts/check_observability.py
@@ -0,0 +1,728 @@
+"""Проверка наблюдаемости через реальные запросы Grafana к трём backend.
+
+--incidents явно разрешает ограниченные отказы на выбранном Compose-проекте.
+В evidence сохраняются только агрегаты и выбранные технические поля.
+"""
+
+import argparse
+import base64
+from concurrent.futures import ThreadPoolExecutor
+import math
+import subprocess
+import threading
+from datetime import datetime, timezone
+import json
+from pathlib import Path
+import time
+import urllib.parse
+import urllib.request
+from uuid import uuid4
+
+from scripts.deployment import ComposeProject
+from tests.client import Client, Failure, check
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+def eventually(operation, message, timeout=90):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ try:
+ value = operation()
+ if value:
+ return value
+ except (OSError, ValueError, KeyError, AssertionError):
+ pass
+ time.sleep(1)
+ raise AssertionError(message)
+
+
+class Grafana:
+ def __init__(self, base):
+ self.base = base.rstrip("/")
+ secret = (
+ (ROOT / "secrets/grafana_admin_password")
+ .read_text(encoding="utf-8")
+ .strip()
+ )
+ self.authorization = (
+ "Basic " + base64.b64encode(("admin:" + secret).encode()).decode()
+ )
+
+ def get(self, path, **params):
+ url = self.base + path
+ if params:
+ url += "?" + urllib.parse.urlencode(params)
+ request = urllib.request.Request(
+ url,
+ headers={"Authorization": self.authorization, "Accept": "application/json"},
+ )
+ with urllib.request.urlopen(request, timeout=10) as response:
+ return json.load(response)
+
+ def proxy(self, source, path, **params):
+ return self.get("/api/datasources/proxy/uid/" + source + path, **params)
+
+ def query(self, expression):
+ value = self.proxy("prometheus", "/api/v1/query", query=expression)
+ check(value["status"] == "success", "Prometheus query failed")
+ return value["data"]["result"]
+
+ def logs(self, selector, start):
+ value = self.proxy(
+ "loki",
+ "/loki/api/v1/query_range",
+ query=selector,
+ start=start,
+ end=time.time_ns(),
+ limit=1000,
+ direction="forward",
+ )
+ check(value["status"] == "success", "Loki query failed")
+ return [
+ json.loads(line)
+ for stream in value["data"]["result"]
+ for _, line, *_ in stream["values"]
+ ]
+
+ def trace(self, trace_id):
+ return self.proxy("tempo", "/api/traces/" + trace_id)
+
+
+def identifier_hex(value, byte_length):
+ if len(value) == byte_length * 2 and all(
+ c in "0123456789abcdefABCDEF" for c in value
+ ):
+ return value.lower()
+ return base64.b64decode(value, validate=True).hex()
+
+
+def spans_in(value):
+ found = []
+ if isinstance(value, dict):
+ for key, item in value.items():
+ if key == "spans":
+ found.extend(item)
+ else:
+ found.extend(spans_in(item))
+ elif isinstance(value, list):
+ for item in value:
+ found.extend(spans_in(item))
+ return found
+
+
+def correlation(
+ grafana, service, start, *, trace_id=None, attachment_id=None, request_id=None
+):
+ selector = '{service_name="' + service + '"} | json'
+ if service == "messenger-api":
+ selector += ' | event="http.request"'
+ if trace_id:
+ selector += ' | trace_id="' + trace_id + '"'
+ if attachment_id:
+ selector += ' | attachment_id="' + attachment_id + '"'
+ if request_id:
+ selector += ' | request_id="' + request_id + '"'
+ records = eventually(
+ lambda: grafana.logs(selector, start), "No correlated log for " + service
+ )
+ record = records[0]
+ trace_id = record["trace_id"]
+
+ def matching_trace():
+ candidate = grafana.trace(trace_id)
+ return (
+ candidate
+ if any(
+ identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
+ for span in spans_in(candidate)
+ )
+ else None
+ )
+
+ trace = eventually(matching_trace, "Matching span is missing in Tempo")
+ spans = [
+ span
+ for span in spans_in(trace)
+ if identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
+ ]
+ # Tempo 2.9 JSON использует protobuf bytes (Base64); Loki хранит hex.
+ check(
+ any(
+ identifier_hex(span.get("spanId", ""), 8) == record["span_id"]
+ for span in spans
+ ),
+ "Log span_id not found in real trace",
+ )
+ check(
+ all(
+ int(span["endTimeUnixNano"]) > int(span["startTimeUnixNano"])
+ for span in spans
+ ),
+ "Trace has no real duration",
+ )
+ return {
+ "trace_id": trace_id,
+ "span_id": record["span_id"],
+ "log": record,
+ "spans": [
+ {
+ key: span.get(key)
+ for key in (
+ "name",
+ "kind",
+ "spanId",
+ "startTimeUnixNano",
+ "endTimeUnixNano",
+ "status",
+ )
+ }
+ for span in spans
+ ],
+ }, trace
+
+
+def run_load(args):
+ check(
+ 1 <= args.duration <= 300
+ and 1 <= args.rate <= 40
+ and 1 <= args.concurrency <= 16,
+ "Load profile outside bounds",
+ )
+ client = Client(args.base_url)
+ actor, _, _, chat = client.bootstrap()
+ project = ComposeProject()
+ container_ids = project.run("ps", "--quiet").splitlines()
+ samples = []
+ stop = threading.Event()
+
+ def sample():
+ while not stop.is_set():
+ result = subprocess.run(
+ [
+ "docker",
+ "stats",
+ "--no-stream",
+ "--format",
+ "{{json .}}",
+ *container_ids,
+ ],
+ capture_output=True,
+ encoding="utf-8",
+ timeout=15,
+ )
+ if result.returncode == 0:
+ samples.append(
+ {
+ "elapsed_seconds": round(time.monotonic() - started, 3),
+ "containers": [
+ {
+ k: row[k]
+ for k in ("Name", "CPUPerc", "MemUsage", "MemPerc")
+ }
+ for row in (
+ json.loads(line) for line in result.stdout.splitlines()
+ )
+ ],
+ }
+ )
+ stop.wait(3)
+
+ def write(index):
+ local = Client(args.base_url)
+ before = time.monotonic()
+ try:
+ local.message(actor, chat, "synthetic-observability-" + str(index))
+ return time.monotonic() - before
+ except Failure:
+ return None
+
+ started = time.monotonic()
+ sampler = threading.Thread(target=sample)
+ sampler.start()
+ try:
+ with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
+ futures = []
+ for index in range(args.duration * args.rate):
+ time.sleep(max(0, started + index / args.rate - time.monotonic()))
+ futures.append(pool.submit(write, index))
+ results = [future.result() for future in futures]
+ elapsed = time.monotonic() - started
+ finally:
+ stop.set()
+ sampler.join()
+ latencies = sorted(value for value in results if value is not None)
+ failed = len(results) - len(latencies)
+
+ def percentile(q):
+ return (
+ round(latencies[max(0, math.ceil(q * len(latencies)) - 1)] * 1000, 3)
+ if latencies
+ else None
+ )
+
+ report = {
+ "date_utc": datetime.now(timezone.utc).isoformat(),
+ "workload": "synthetic POST messages; fixed arrival rate; no retries",
+ "duration_seconds": args.duration,
+ "elapsed_seconds": round(elapsed, 3),
+ "arrival_rate": args.rate,
+ "concurrency_limit": args.concurrency,
+ "successful_requests": len(latencies),
+ "failed_requests": failed,
+ "error_ratio": failed / len(results),
+ "successful_rps": round(len(latencies) / elapsed, 3),
+ "successful_latency_ms": {
+ name: percentile(q)
+ for name, q in [("p50", 0.5), ("p95", 0.95), ("p99", 0.99)]
+ },
+ "dataset": {"users": 3, "chats": 1, "messages": len(latencies)},
+ "resource_samples": samples,
+ "limits": "Latency covers successes from worker start; client scheduling is not included. Small controlled load, not a capacity benchmark.",
+ }
+ check(bool(latencies) and failed == 0, "Controlled load contains errors")
+
+ return report
+
+
+def metric(grafana, expression):
+ rows = grafana.query(expression)
+ return float(rows[0]["value"][1]) if rows else None
+
+
+def alert_firing(grafana, name):
+ return bool(grafana.query(f'ALERTS{{alertname="{name}",alertstate="firing"}}'))
+
+
+def trace_chain(grafana, trace_id, *, retry=False):
+ def inspect():
+ spans = spans_in(grafana.trace(trace_id))
+ by_id = {identifier_hex(span["spanId"], 8): span for span in spans}
+ workers = [
+ span
+ for span in spans
+ if span["name"] == "attachment.process.v1"
+ and span.get("kind") in ("SPAN_KIND_CONSUMER", 5)
+ ]
+ for worker in workers:
+ publisher = by_id.get(identifier_hex(worker.get("parentSpanId", ""), 8), {})
+ if publisher.get("kind") not in ("SPAN_KIND_PRODUCER", 4):
+ return None
+ upload = by_id.get(identifier_hex(publisher.get("parentSpanId", ""), 8), {})
+ if upload.get("name") != "POST /api/v1/chats/{chat_id}/attachments":
+ return None
+ names = {span["name"] for span in spans}
+ if (
+ not workers
+ or not {"s3.read_staging", "s3.write_result", "postgres.commit_ready"}
+ <= names
+ ):
+ return None
+ if retry:
+ outcomes = set()
+ for span in workers:
+ attrs = {row["key"]: row["value"] for row in span.get("attributes", [])}
+ outcomes.add(
+ (
+ int(attrs["job.attempt"]["intValue"]),
+ attrs.get("outcome", {}).get("stringValue"),
+ )
+ )
+ if not {(1, "error"), (2, "error")} <= outcomes or not any(
+ n > 2 and outcome == "success" for n, outcome in outcomes
+ ):
+ return None
+ return [
+ {
+ "name": span["name"],
+ "kind": span.get("kind"),
+ "span_id": identifier_hex(span["spanId"], 8),
+ "parent_span_id": identifier_hex(span.get("parentSpanId", ""), 8),
+ "job": {
+ entry["key"]: entry["value"]
+ for entry in span.get("attributes", [])
+ if entry["key"] in {"job.attempt", "outcome"}
+ },
+ }
+ for span in spans
+ ]
+
+ return eventually(inspect, "Missing upload -> producer -> consumer -> S3/DB chain")
+
+
+def telemetry_outage(project, client, grafana, actor, chat):
+ failed = 'sum({__name__=~"otelcol_exporter_send_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
+ refused = 'sum({__name__=~"otelcol_exporter_enqueue_failed_log_records(_total)?",exporter="otlphttp/loki"}) or vector(0)'
+ baseline = [metric(grafana, expr) for expr in (failed, refused)]
+ rss = 'sum({__name__=~"otelcol_process_memory_rss(_bytes)?"})'
+ samples = []
+ successful = 0
+ try:
+ project.run("stop", "loki")
+ start = time.monotonic()
+ for second in range(51):
+ for index in range(4):
+ client.message(actor, chat, f"outage-{second}-{index}")
+ successful += 1
+ if second % 5 == 0:
+ samples.append(
+ {
+ "seconds": round(time.monotonic() - start, 2),
+ "rss_bytes": metric(grafana, rss),
+ "queued_batches": metric(
+ grafana,
+ 'sum(otelcol_exporter_queue_size{exporter="otlphttp/loki"})',
+ ),
+ "capacity_batches": metric(
+ grafana,
+ 'sum(otelcol_exporter_queue_capacity{exporter="otlphttp/loki"})',
+ ),
+ }
+ )
+ time.sleep(max(0, start + second + 1 - time.monotonic()))
+ losses = [
+ metric(grafana, expr) - count
+ for expr, count in zip((failed, refused), baseline)
+ ]
+ check(sum(losses) > 0, "No measured Collector log loss during prolonged outage")
+ check(
+ all(
+ 0 <= row["queued_batches"] <= row["capacity_batches"] == 256
+ for row in samples
+ ),
+ "Unexpected queue bound",
+ )
+ check(
+ max(row["rss_bytes"] for row in samples) < 256 * 1024**2,
+ "Collector crossed container memory limit",
+ )
+ finally:
+ project.run("start", "loki")
+ eventually(
+ lambda: grafana.proxy("loki", "/loki/api/v1/labels").get("status") == "success",
+ "Loki did not recover",
+ )
+ restored = client.message(actor, chat, "telemetry-backend-restored")
+ correlation(
+ grafana,
+ "messenger-api",
+ time.time_ns() - 30 * 10**9,
+ trace_id=restored["headers"]["X-Trace-Id"],
+ )
+ return {
+ "scenario": "loki_unavailable",
+ "successful_messages": successful,
+ "dropped_after_export_failure": losses[0],
+ "dropped_at_enqueue": losses[1],
+ "samples": samples,
+ "log_export_recovered": True,
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--base-url", required=True)
+ parser.add_argument("--grafana-url", default="http://localhost:13000")
+ parser.add_argument("--incidents", action="store_true")
+ parser.add_argument("--restart-telemetry", action="store_true")
+ parser.add_argument("--output", default="evidence/observability.json")
+ parser.add_argument("--duration", type=int, default=10)
+ parser.add_argument("--rate", type=int, default=10)
+ parser.add_argument("--concurrency", type=int, default=2)
+ args = parser.parse_args()
+ load = run_load(args)
+ start = time.time_ns()
+ project = ComposeProject()
+ client = Client(args.base_url, timeout=40)
+ aliases = {r.instance_id for r in project.replicas() if r.running}
+ check(len(aliases) >= 2, "Need two real API containers")
+ check(
+ client.call("GET", "/health/live")["headers"].get("X-Instance-Id") in aliases,
+ "Wrong Compose project for BASE_URL",
+ )
+ grafana = Grafana(args.grafana_url)
+ eventually(
+ lambda: grafana.get("/api/health").get("database") == "ok",
+ "Grafana is not ready",
+ )
+ sources = grafana.get("/api/datasources")
+ check(
+ {"prometheus", "loki", "tempo"} <= {source["uid"] for source in sources},
+ "Missing provisioned datasource",
+ )
+ dashboard = grafana.get("/api/dashboards/uid/messenger-overview")["dashboard"]
+ check(len(dashboard["panels"]) >= 8, "Dashboard is missing panels")
+ actor, bob, eve, chat = client.bootstrap()
+ sentinel = "private-content-" + uuid4().hex
+ response = client.message(actor, chat, sentinel)
+ controlled_trace = response["headers"].get("X-Trace-Id")
+ check(bool(controlled_trace), "No API trace identifier")
+ attachment, _ = client.attachment(actor, chat)
+ client.wait_ready(actor, attachment)
+ api, api_trace = correlation(
+ grafana,
+ "messenger-api",
+ start,
+ trace_id=controlled_trace,
+ request_id=response["headers"]["X-Request-Id"],
+ )
+ worker, worker_trace = correlation(
+ grafana, "messenger-worker", start, attachment_id=attachment["id"]
+ )
+ accepted_chain = trace_chain(grafana, worker["trace_id"])
+ check(
+ api["log"]["request_id"] == response["headers"]["X-Request-Id"],
+ "Request id correlation lost",
+ )
+ check(
+ any(span["kind"] in ("SPAN_KIND_SERVER", 2) for span in api["spans"]),
+ "No server span",
+ )
+ check(
+ any(span["kind"] in ("SPAN_KIND_CONSUMER", 5) for span in worker["spans"]),
+ "No consumer span",
+ )
+ processed = eventually(
+ lambda: grafana.logs(
+ '{service_name="messenger-worker"} | json | event="job.processed" | trace_id="'
+ + worker["trace_id"]
+ + '"',
+ start,
+ ),
+ "No worker result log",
+ )
+ check(processed[0]["outcome"] == "success", "Controlled worker did not succeed")
+ metrics = {}
+ expressions = {
+ "http_requests": "sum(messenger_http_requests_total)",
+ "http_duration": "sum(messenger_http_duration_seconds_count)",
+ "job_success": 'sum(messenger_job_attempts_total{outcome="success"})',
+ "job_duration": "sum(messenger_job_duration_seconds_count)",
+ }
+ for name, expression in expressions.items():
+ metrics[name] = eventually(
+ lambda expression=expression: grafana.query(expression),
+ "Missing metric: " + expression,
+ )
+ check(float(metrics[name][0]["value"][1]) > 0, "Empty metric: " + name)
+ # Все PromQL из dashboard должны давать численные данные после traffic.
+ for panel in dashboard["panels"]:
+ if panel["datasource"]["uid"] == "prometheus":
+ eventually(
+ lambda panel=panel: all(
+ grafana.query(target["expr"]) for target in panel["targets"]
+ ),
+ "Empty dashboard panel: " + panel["title"],
+ )
+ forbidden_labels = {
+ "user_id",
+ "chat_id",
+ "message_id",
+ "attachment_id",
+ "request_id",
+ "filename",
+ "url",
+ "http_target",
+ "http_url",
+ }
+ series = grafana.query('{__name__=~"messenger_.*"}')
+ check(
+ not any(forbidden_labels & set(row["metric"]) for row in series),
+ "High cardinality metric label",
+ )
+ report = {
+ "date_utc": datetime.now(timezone.utc).isoformat(),
+ "project": project.name,
+ "load": load,
+ "api_replicas": len(aliases),
+ "dashboard_uid": dashboard["uid"],
+ "api_correlation": api,
+ "worker_correlation": worker,
+ "metrics": metrics,
+ "upload_trace": {"trace_id": worker["trace_id"], "chain": accepted_chain},
+ "incidents": [],
+ }
+ if args.incidents:
+
+ def fail_request():
+ return client.call(
+ "POST",
+ "/api/v1/users",
+ expected=503,
+ body={
+ "username": "fault_" + uuid4().hex[:12],
+ "display_name": "Synthetic",
+ "password": "not-exported-" + uuid4().hex,
+ },
+ )
+
+ before = time.monotonic()
+ try:
+ project.run("stop", "postgres")
+ failed = fail_request()
+ error_trace = failed["headers"].get("X-Trace-Id")
+ error, db_trace = correlation(
+ grafana,
+ "messenger-api",
+ start,
+ trace_id=error_trace,
+ request_id=failed["headers"]["X-Request-Id"],
+ )
+ diagnostics = eventually(
+ lambda: grafana.logs(
+ '{service_name="messenger-api"} | json | event="dependency.unavailable" | trace_id="'
+ + error_trace
+ + '"',
+ start,
+ ),
+ "No dependency diagnostic log",
+ )
+ checkout = next(
+ span
+ for span in spans_in(db_trace)
+ if span["name"] == "postgres.checkout"
+ and span.get("status", {}).get("code") in ("STATUS_CODE_ERROR", 2)
+ )
+
+ def db_alert():
+ for _ in range(4):
+ fail_request()
+ return alert_firing(grafana, "MessengerHighErrorRate")
+
+ eventually(db_alert, "Error rate alert did not fire", timeout=75)
+ report["incidents"].append(
+ {
+ "scenario": "postgres_unavailable",
+ "http_status": 503,
+ "correlation": error,
+ "diagnostic": diagnostics[0],
+ "db_span": checkout["name"],
+ "alert": "MessengerHighErrorRate",
+ "firing_after_seconds": round(time.monotonic() - before, 3),
+ }
+ )
+ finally:
+ project.run("start", "postgres")
+ eventually(lambda: _ready(client), "API did not recover after PostgreSQL")
+ eventually(
+ lambda: not alert_firing(grafana, "MessengerHighErrorRate"),
+ "Error rate alert not resolved",
+ )
+ report["incidents"][-1]["resolved_after_seconds"] = round(
+ time.monotonic() - before, 3
+ )
+
+ from scripts.check_jobs import retry_replay
+
+ queue = {}
+
+ def observe_queue(_):
+ observed = time.monotonic()
+ eventually(
+ lambda: alert_firing(grafana, "MessengerQueueLag"),
+ "Queue lag alert did not fire",
+ )
+ queue.update(
+ depth=metric(grafana, "max(messenger_queue_depth_ratio)"),
+ oldest_seconds=metric(grafana, "max(messenger_queue_oldest_seconds)"),
+ firing_after_seconds=round(time.monotonic() - observed, 3),
+ )
+
+ incident = retry_replay(
+ client, actor, chat, permanent=False, on_queued=observe_queue
+ )
+ retried_worker, retry_trace = correlation(
+ grafana, "messenger-worker", start, attachment_id=incident["attachment_id"]
+ )
+ chain = trace_chain(grafana, retried_worker["trace_id"], retry=True)
+ eventually(
+ lambda: not alert_firing(grafana, "MessengerQueueLag")
+ and metric(grafana, "max(messenger_queue_depth_ratio)") == 0,
+ "Backlog did not drain",
+ )
+ queue.update(resolved=True, final_depth=0)
+ report["incidents"].append(
+ {
+ "scenario": "worker_stopped_then_transient_retry",
+ "alert": "MessengerQueueLag",
+ "queue": queue,
+ "recovered_status": incident["after"]["status"],
+ "trace_id": retried_worker["trace_id"],
+ "chain": chain,
+ }
+ )
+ report["incidents"].append(
+ telemetry_outage(project, client, grafana, actor, chat)
+ )
+ else:
+ retry_trace = {}
+ # Не сохраняем API bodies или SDK exceptions; проверяем реальные выбранные сигналы.
+ logs = grafana.logs('{service_name=~"messenger-api|messenger-worker"}', start)
+ exported = json.dumps([logs, api_trace, worker_trace, retry_trace])
+ private = [
+ sentinel,
+ *(person["token"] for person in (actor, bob, eve)),
+ *(person["registration"]["password"] for person in (actor, bob, eve)),
+ ]
+ private.extend(
+ path.read_text(encoding="utf-8").strip()
+ for path in (ROOT / "secrets").iterdir()
+ if path.is_file()
+ )
+ check(
+ not any(secret and secret in exported for secret in private),
+ "Private value found in telemetry",
+ )
+ check(
+ all(
+ record.get("trace_id")
+ and record.get("span_id")
+ and record.get("timestamp")
+ and record.get("severity")
+ for record in logs
+ ),
+ "Incomplete structured logs",
+ )
+ report["privacy_checked_logs"] = len(logs)
+ report["checks"] = {
+ "three_backends": True,
+ "dashboard_queries": True,
+ "server_consumer_traces": True,
+ "log_trace_correlation": True,
+ "low_cardinality": True,
+ "privacy": True,
+ }
+ if args.restart_telemetry:
+ historical_time = time.time()
+ expression = "sum(messenger_http_requests_total)"
+ before = grafana.proxy(
+ "prometheus", "/api/v1/query", query=expression, time=historical_time
+ )["data"]["result"]
+ project.run("restart", "prometheus", "loki", "tempo", "grafana")
+ after = eventually(
+ lambda: grafana.proxy(
+ "prometheus", "/api/v1/query", query=expression, time=historical_time
+ )["data"]["result"],
+ "Prometheus history lost",
+ )
+ check(before == after, "Historical metric changed after storage restart")
+ correlation(grafana, "messenger-api", start, trace_id=api["trace_id"])
+ correlation(grafana, "messenger-worker", start, trace_id=worker["trace_id"])
+ report["checks"]["history_survives_restart"] = True
+ destination = ROOT / args.output
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_text(
+ json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8"
+ )
+ print(
+ "PASS: telemetry, dashboard, privacy, correlations"
+ + ("; alerts and outage recovery verified" if args.incidents else "")
+ )
+
+
+def _ready(client):
+ return client.call("GET", "/health/ready", expected=[200, 503])["status"] == 200
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/check_outbox_recovery.py b/scripts/check_outbox_recovery.py
new file mode 100644
index 0000000..949b44a
--- /dev/null
+++ b/scripts/check_outbox_recovery.py
@@ -0,0 +1,147 @@
+"""Сценарий COMMIT → SIGKILL до publish → ready без повторного upload.
+
+Требуется уже работающий отдельный стенд. Скрипт не выполняется при запуске API.
+Публичные tests не изменяет, secrets/tokens не печатает и evidence сам не сохраняет.
+"""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+from uuid import uuid4
+
+from scripts.deployment import ComposeProject
+from tests.client import Client, check
+
+ROOT = Path(__file__).resolve().parents[1]
+COMPOSE = ["docker", "compose", "--env-file", ".env"]
+
+
+def compose(*arguments: str, timeout: int = 90) -> subprocess.CompletedProcess:
+ return subprocess.run(
+ COMPOSE + list(arguments),
+ cwd=ROOT,
+ encoding="utf-8",
+ capture_output=True,
+ timeout=timeout,
+ )
+
+
+def require_compose(*arguments: str) -> str:
+ result = compose(*arguments)
+ check(result.returncode == 0, "Compose command failed; output hidden")
+ return result.stdout
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(
+ description="Отказ publisher после COMMIT перед RabbitMQ publish"
+ )
+ parser.add_argument("--base-url", required=True)
+ parser.add_argument("--ca-file")
+ args = parser.parse_args()
+ running = set(
+ require_compose("ps", "--status", "running", "--services").splitlines()
+ )
+ check(
+ {"api", "postgres", "minio", "broker", "publisher", "worker"} <= running,
+ "Start the application stack before this scenario",
+ )
+ client = Client(args.base_url, args.ca_file, timeout=40)
+ project = ComposeProject()
+ aliases = {replica.instance_id for replica in project.replicas() if replica.running}
+ live = client.call("GET", "/health/live", schema="Health")
+ check(
+ live["headers"].get("X-Instance-Id") in aliases,
+ "BASE_URL does not belong to the selected Compose project",
+ )
+ alice, _, _, chat = client.bootstrap()
+ attachment = None
+ fault_container = "messenger-outbox-fault-" + uuid4().hex
+ try:
+ require_compose("stop", "publisher")
+ attachment, original = client.attachment(alice, chat)
+ message = client.message(alice, chat, "", attachments=[attachment["id"]])[
+ "json"
+ ]
+ client.call(
+ "GET",
+ f"/api/v1/attachments/{attachment['id']}/content",
+ expected=409,
+ actor=alice,
+ )
+ killed = compose(
+ "run",
+ "--rm",
+ "--no-deps",
+ "--name",
+ fault_container,
+ "publisher",
+ "python",
+ "-m",
+ "app.core.outbox_publisher",
+ "--crash-before-publish",
+ attachment["id"],
+ )
+ marker = json.dumps(
+ {"fault": "before_publish", "attachment_id": attachment["id"]}
+ )
+ check(
+ killed.returncode == 137 and marker in killed.stdout,
+ "Expected targeted SIGKILL before publish was not observed",
+ )
+ pending = client.call(
+ "GET",
+ f"/api/v1/attachments/{attachment['id']}",
+ actor=alice,
+ schema="Attachment",
+ )["json"]
+ check(pending["status"] == "queued", "Crash must leave accepted job queued")
+ finally:
+
+ try:
+ subprocess.run(
+ ["docker", "rm", "-f", fault_container], capture_output=True, timeout=30
+ )
+ finally:
+ require_compose("start", "publisher")
+ ready = client.wait_ready(alice, attachment, wait_seconds=60)
+ check(ready["id"] == attachment["id"], "Recovered a different attachment")
+ check(
+ {variant["name"] for variant in ready["variants"]}
+ == {"thumbnail", "crop", "optimized"},
+ "Missing required variants",
+ )
+ path = f"/api/v1/attachments/{attachment['id']}/content"
+ check(
+ client.call("GET", path, actor=alice)["bytes"] == original,
+ "Original changed after recovery",
+ )
+ for variant in ready["variants"]:
+ downloaded = client.call(
+ "GET", path + "?variant=" + variant["name"], actor=alice
+ )["bytes"]
+ check(
+ len(downloaded) == variant["size_bytes"]
+ and hashlib.sha256(downloaded).hexdigest() == variant["sha256"],
+ "Recovered variant checksum mismatch",
+ )
+ print(
+ json.dumps(
+ {
+ "scenario": "commit_before_publish",
+ "attachment_id": ready["id"],
+ "message_id": message["id"],
+ "publisher_exit_code": killed.returncode,
+ "status_after_crash": pending["status"],
+ "status_after_restart": ready["status"],
+ "variants": [variant["name"] for variant in ready["variants"]],
+ "reuploaded": False,
+ }
+ )
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/check_s3.py b/scripts/check_s3.py
new file mode 100644
index 0000000..94c4000
--- /dev/null
+++ b/scripts/check_s3.py
@@ -0,0 +1,152 @@
+"""Реальный MinIO: повтор bootstrap, private bucket, ограниченные права и отказ."""
+
+import json
+import subprocess
+import time
+
+from scripts.deployment import ComposeProject
+from tests.client import Client, arguments, check, run
+
+PROBE = r"""
+import asyncio, json, sys
+from pathlib import Path
+from uuid import UUID
+from urllib.request import Request, urlopen
+from urllib.error import HTTPError, URLError
+from app.core.config import Settings
+from app.core.database import create_db_engine, create_session_factory
+from app.storage.s3 import create_s3_storage
+from app.models.attachment import Attachment
+from minio.error import S3Error
+async def main():
+ settings = Settings()
+ engine = create_db_engine(settings)
+ storage = create_s3_storage(settings.s3)
+ try:
+ async with create_session_factory(engine)() as session:
+ record = await session.get(Attachment, UUID(json.load(sys.stdin)['attachment_id']))
+ key = record.object_key
+ results = {}
+ for method in ('GET', 'PUT'):
+ request = Request(settings.s3.endpoint + '/' + settings.s3.bucket + '/' + key,
+ method=method, data=b'x' if method == 'PUT' else None)
+ try:
+ with urlopen(request, timeout=3) as response:
+ results[method] = response.status
+ except HTTPError as error:
+ results[method] = error.code
+ error.close()
+ try:
+ storage._client.get_bucket_policy(settings.s3.bucket)
+ results['admin_denied'] = False
+ except S3Error as error:
+ results['admin_denied'] = error.code == 'AccessDenied'
+ try:
+ buckets = storage._client.list_buckets()
+ results['other_buckets_denied'] = all(bucket.name == settings.s3.bucket for bucket in buckets)
+ except S3Error as error:
+ results['other_buckets_denied'] = error.code == 'AccessDenied'
+ try:
+ with urlopen(settings.s3.endpoint.replace(':9000', ':9001'), timeout=1):
+ results['console_unreachable'] = False
+ except HTTPError as error:
+ results['console_unreachable'] = False
+ error.close()
+ except URLError:
+ results['console_unreachable'] = True
+ results['temp_originals_absent'] = not any(Path('/tmp').glob('messenger-upload-*'))
+ results['root_files_absent'] = not any(Path('/run/secrets', name).exists()
+ for name in ('minio_root_user', 'minio_root_password'))
+ print(json.dumps(results))
+ finally:
+ await storage.aclose()
+ await engine.dispose()
+asyncio.run(main())
+"""
+
+
+def main():
+ def configure(parser):
+ parser.add_argument(
+ "--with-outage",
+ action="store_true",
+ help="временно остановить MinIO выбранного проекта",
+ )
+
+ args = arguments("MinIO private bucket and failure checks", configure)
+ project = ComposeProject()
+ client = Client(args.base_url, args.ca_file, timeout=40)
+ alice, _, _, chat = client.bootstrap()
+ aliases = {replica.instance_id for replica in project.replicas() if replica.running}
+ live = client.call("GET", "/health/live", schema="Health")
+ check(
+ live["headers"].get("X-Instance-Id") in aliases,
+ "BASE_URL does not belong to the selected Compose project",
+ )
+ attachment, original = client.attachment(alice, chat)
+ attachment = client.wait_ready(alice, attachment)
+
+ for _ in range(2):
+ project.run("run", "--rm", "--no-deps", "minio-bootstrap", timeout=100)
+ replicas = project.replicas()
+ check(bool(replicas), "API not running")
+ result = subprocess.run(
+ ["docker", "exec", "-i", replicas[0].id, "python", "-c", PROBE],
+ input=json.dumps({"attachment_id": attachment["id"]}),
+ encoding="utf-8",
+ capture_output=True,
+ timeout=20,
+ )
+ check(result.returncode == 0, "private bucket probe failed (details hidden)")
+ statuses = json.loads(result.stdout)
+ check(statuses["GET"] == statuses["PUT"] == 403, "anonymous S3 access allowed")
+ check(
+ statuses["admin_denied"]
+ and statuses["other_buckets_denied"]
+ and statuses["root_files_absent"],
+ "API has excessive access",
+ )
+ check(statuses["console_unreachable"], "console reachable from API network")
+ check(statuses["temp_originals_absent"], "API retained temporary originals")
+ check(
+ not project.config["services"]["minio"].get("ports"),
+ "MinIO publishes host ports",
+ )
+ env = project.config["services"]["minio"]["environment"]
+ check(env["MINIO_BROWSER"] == "off", "browser enabled")
+ check(
+ "127.0.0.1:9001" in project.config["services"]["minio"]["command"],
+ "console is not bound to loopback",
+ )
+ path = "/api/v1/attachments/" + attachment["id"] + "/content"
+ check(
+ client.call("GET", path, actor=alice)["bytes"] == original,
+ "bootstrap changed original",
+ )
+ if args.with_outage:
+ try:
+ project.run("stop", "minio")
+ client.call("GET", path, expected=503, actor=alice)
+ finally:
+ project.run("start", "minio")
+ deadline = time.monotonic() + 60
+ while True:
+ try:
+ client.call("GET", "/health/ready", schema="Health")
+ break
+ except Exception:
+ if time.monotonic() >= deadline:
+ raise
+ time.sleep(0.5)
+ check(
+ client.call("GET", path, actor=alice)["bytes"] == original,
+ "original lost after MinIO restart",
+ )
+ print(
+ "PASS: idempotent bootstrap, anonymous GET/PUT denied, restricted API credentials, private console"
+ + (", MinIO outage/recovery" if args.with_outage else "")
+ )
+
+
+if __name__ == "__main__":
+ run(main)
diff --git a/scripts/container_tests.py b/scripts/container_tests.py
new file mode 100644
index 0000000..49fe311
--- /dev/null
+++ b/scripts/container_tests.py
@@ -0,0 +1,201 @@
+"""Последовательная проверка лабораторной 6 в отдельном Docker-окружении."""
+
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import subprocess
+import tarfile
+import time
+
+ROOT = Path("/workspace")
+EVIDENCE = ROOT / "evidence"
+COMPOSE = ["docker", "compose", "--env-file", ".env"]
+BASE = ["--base-url", "http://127.0.0.1:28080"]
+
+
+def step(report, name, command, *, timeout=600, env=None, required=True):
+ print("Запуск " + name, flush=True)
+ started = time.monotonic()
+ log = name + ".txt"
+ with (EVIDENCE / log).open("w", encoding="utf-8", newline="\n") as output:
+ output.write("$ " + " ".join(command) + "\n")
+ output.flush()
+ result = subprocess.run(
+ command,
+ cwd=ROOT,
+ stdout=output,
+ stderr=subprocess.STDOUT,
+ timeout=timeout,
+ env={**os.environ, **(env or {})},
+ )
+ report["checks"].append(
+ {
+ "name": name,
+ "exit_code": result.returncode,
+ "duration_seconds": round(time.monotonic() - started, 3),
+ "log": log,
+ }
+ )
+ if result.returncode:
+ print("Ошибка " + name, flush=True)
+ if required:
+ result.check_returncode()
+ else:
+ print("Готово " + name, flush=True)
+
+
+def main():
+ with tarfile.open("/tmp/source.tar") as archive:
+ archive.extractall(ROOT, filter="data")
+ os.chdir(ROOT)
+ EVIDENCE.mkdir(exist_ok=True)
+ deadline = time.monotonic() + 120
+ while subprocess.run(["docker", "info"], capture_output=True).returncode:
+ if time.monotonic() > deadline:
+ raise RuntimeError("Проверочный Docker не запустился")
+ time.sleep(1)
+
+ os.environ.update(
+ COMPOSE_PROJECT_NAME="messenger-test",
+ COMPOSE_FILE="compose.yaml:compose.test.yaml",
+ LB_HOST_PORT="28080",
+ GRAFANA_HOST_PORT="23000",
+ DOCKER_TESTS="1",
+ TEST_USE_MIGRATIONS="1",
+ TEST_REDIS_HOST="127.0.0.1",
+ TEST_REDIS_PORT="26379",
+ TEST_REDIS_DB="15",
+ )
+ integration_only = os.environ.get("INTEGRATION_ONLY") == "1"
+ report = {
+ "status": "failed",
+ "profile": "integration" if integration_only else "full",
+ "started_at": datetime.now(timezone.utc).isoformat(),
+ "checks": [],
+ }
+
+ try:
+ step(report, "00_init", ["python", "scripts/init_local.py"])
+ step(report, "01_contract", ["python", "scripts/check_contract.py"])
+ step(report, "02_openapi", ["python", "scripts/validate_openapi.py"])
+ step(
+ report,
+ "03_import_images",
+ ["docker", "image", "load", "--input", "/tmp/images.tar"],
+ )
+ step(
+ report,
+ "03_startup",
+ COMPOSE + ["up", "-d", "--no-build", "--pull", "never"],
+ timeout=3600,
+ )
+ step(
+ report,
+ "03_readiness",
+ COMPOSE
+ + [
+ "up",
+ "-d",
+ "--no-deps",
+ "--wait",
+ "--wait-timeout",
+ "240",
+ "nginx",
+ "grafana",
+ ],
+ )
+ if not integration_only:
+ password = (ROOT / "secrets/db_password").read_text(encoding="utf-8")
+ test_env = {
+ "OTEL_ENABLED": "false",
+ "TEST_DATABASE_URL": (
+ f"postgresql+asyncpg://messenger:{password}@127.0.0.1:25432/messenger_test"
+ ),
+ "TEST_REDIS_PASSWORD": (ROOT / "secrets/redis_password").read_text(
+ encoding="utf-8"
+ ),
+ }
+ step(
+ report,
+ "04_test_database",
+ COMPOSE
+ + [
+ "exec",
+ "-T",
+ "postgres",
+ "createdb",
+ "-U",
+ "messenger",
+ "messenger_test",
+ ],
+ )
+ step(
+ report,
+ "05_pytest",
+ ["python", "-m", "pytest", "-q", "--junitxml=evidence/pytest.xml"],
+ timeout=1800,
+ env=test_env,
+ required=False,
+ )
+ step(report, "06_smoke", ["python", "tests/smoke.py", *BASE], required=False)
+ step(
+ report,
+ "07_observability",
+ [
+ "python",
+ "-m",
+ "scripts.check_observability",
+ *BASE,
+ "--grafana-url",
+ "http://127.0.0.1:23000",
+ "--incidents",
+ "--restart-telemetry",
+ ],
+ timeout=900,
+ required=False,
+ )
+ for number, mode, hook in (
+ ("08", "persistence", "restart"),
+ ("09", "failover", "stop-one"),
+ ):
+ step(
+ report,
+ f"{number}_{mode}",
+ [
+ "python",
+ "tests/resilience.py",
+ "--mode",
+ mode,
+ "--action-script",
+ f"scripts/{hook}.sh",
+ *BASE,
+ ],
+ required=False,
+ )
+ if all(check["exit_code"] == 0 for check in report["checks"]):
+ report["status"] = "passed"
+ finally:
+ try:
+ step(
+ report,
+ "10_cleanup",
+ COMPOSE + ["down", "--volumes", "--remove-orphans"],
+ timeout=180,
+ )
+ except (subprocess.SubprocessError, OSError):
+ report["status"] = "failed"
+ raise
+ finally:
+ report["finished_at"] = datetime.now(timezone.utc).isoformat()
+ (EVIDENCE / "verification.json").write_text(
+ json.dumps(report, indent=2, ensure_ascii=False) + "\n",
+ encoding="utf-8",
+ )
+
+ if report["status"] != "passed":
+ raise SystemExit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/deployment.py b/scripts/deployment.py
new file mode 100644
index 0000000..c07abd8
--- /dev/null
+++ b/scripts/deployment.py
@@ -0,0 +1,68 @@
+"""Общие операции с репликами текущего Compose-проекта."""
+
+from dataclasses import dataclass
+import json
+import os
+from pathlib import Path
+import subprocess
+
+ROOT = Path(__file__).resolve().parents[1]
+
+
+@dataclass(frozen=True)
+class Replica:
+
+ id: str
+ image: str
+ instance_id: str
+ running: bool
+ health: str | None
+
+
+class ComposeProject:
+
+ def __init__(self) -> None:
+ self.command = ["docker", "compose", "--env-file", ".env"]
+ self.config = json.loads(self.run("config", "--format", "json"))
+ self.name = self.config["name"]
+
+ def docker(self, *args: str, timeout: int = 90, env=None) -> str:
+ result = subprocess.run(
+ ["docker", *args],
+ cwd=ROOT,
+ encoding="utf-8",
+ capture_output=True,
+ timeout=timeout,
+ env={**os.environ, **(env or {})},
+ )
+ if result.returncode:
+ raise RuntimeError(result.stderr.strip() or result.stdout.strip())
+ return result.stdout.strip()
+
+ def run(self, *args: str, timeout: int = 90, env=None) -> str:
+ return self.docker(*self.command[1:], *args, timeout=timeout, env=env)
+
+ def replicas(self) -> list[Replica]:
+ ids = self.run("ps", "--all", "--quiet", "api").splitlines()
+ if not ids:
+ return []
+ template = (
+ '{"id":{{json .Id}},"image":{{json .Image}},'
+ '"hostname":{{json .Config.Hostname}},"running":{{json .State.Running}},'
+ '"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}null{{end}}}'
+ )
+ rows = self.docker("inspect", "--format", template, *ids).splitlines()
+ configured = str(
+ self.config["services"]["api"].get("environment", {}).get("INSTANCE_ID")
+ or ""
+ ).strip()
+ return [
+ Replica(
+ row["id"],
+ row["image"],
+ configured or row["hostname"],
+ row["running"],
+ row["health"],
+ )
+ for row in map(json.loads, rows)
+ ]
diff --git a/scripts/init_local.py b/scripts/init_local.py
new file mode 100644
index 0000000..0b149f3
--- /dev/null
+++ b/scripts/init_local.py
@@ -0,0 +1,39 @@
+"""Создаёт только отсутствующие локальные настройки и файловые секреты."""
+
+import os
+from pathlib import Path
+import secrets
+
+
+def main() -> None:
+ root = Path(__file__).resolve().parents[1]
+ destination = root / ".env"
+ if not destination.exists():
+ with destination.open("x", encoding="utf-8", newline="\n") as target:
+ target.write((root / ".env.example").read_text(encoding="utf-8"))
+ destination.chmod(0o600)
+ directory = root / "secrets"
+ directory.mkdir(mode=0o700, exist_ok=True)
+ for name in (
+ "db_password",
+ "redis_password",
+ "broker_password",
+ "minio_root_user",
+ "minio_root_password",
+ "s3_access_key_id",
+ "s3_secret_access_key",
+ "grafana_admin_password",
+ ):
+ path = directory / name
+ try:
+ descriptor = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o444)
+ except FileExistsError:
+ continue
+ with os.fdopen(descriptor, "w", encoding="utf-8") as target:
+ target.write(secrets.token_urlsafe(24))
+ print("Создан файл " + str(path.relative_to(root)))
+ print("Настройки готовы; существующие значения сохранены")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/restart.sh b/scripts/restart.sh
new file mode 100644
index 0000000..e64d577
--- /dev/null
+++ b/scripts/restart.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+
+compose() {
+ docker compose --env-file .env "$@"
+}
+
+wait_healthy() {
+ for service in "$@"; do
+ attempts=0
+ until states=$(compose ps --format '{{.Health}}' "$service") \
+ && [ -n "$states" ] \
+ && [ -z "$(printf '%s\n' "$states" | grep -v '^healthy$')" ]; do
+ attempts=$((attempts + 1))
+ if [ "$attempts" -ge 60 ]; then
+ echo "$service не стал healthy за 60 секунд" >&2
+ exit 1
+ fi
+ sleep 1
+ done
+ done
+}
+
+worker_running=$(compose ps --status running --quiet worker)
+compose stop worker publisher cleanup
+compose restart postgres redis minio broker
+wait_healthy postgres redis minio broker
+compose restart api
+wait_healthy api
+compose restart nginx
+wait_healthy nginx
+compose start publisher cleanup
+if [ -n "$worker_running" ]; then
+ compose start worker
+fi
diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh
new file mode 100644
index 0000000..0dcd4da
--- /dev/null
+++ b/scripts/stop-one.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
+
+case "$TARGET_INSTANCE_ID" in
+ *[!a-zA-Z0-9_.:-]*|'') exit 1 ;;
+esac
+
+target=""
+for container in $(docker compose --env-file .env ps -q api); do
+ hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container")
+ configured=$(docker inspect --format '{{range .Config.Env}}{{if eq (index (split . "=") 0) "INSTANCE_ID"}}{{println .}}{{end}}{{end}}' "$container" \
+ | sed -n 's/^INSTANCE_ID=//p' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')
+ if [ "$TARGET_INSTANCE_ID" = "$hostname" ] || [ "$TARGET_INSTANCE_ID" = "$configured" ]; then
+ if [ -n "$target" ]; then
+ echo 'Instance id matches multiple API containers' >&2
+ exit 1
+ fi
+ target="$container"
+ fi
+done
+
+if [ -z "$target" ]; then
+ echo 'Target API instance not found in this Compose project' >&2
+ exit 1
+fi
+
+echo "Stopping API container $target" >&2
+docker stop --timeout 8 "$target" >/dev/null
diff --git a/scripts/test.py b/scripts/test.py
new file mode 100644
index 0000000..08f2618
--- /dev/null
+++ b/scripts/test.py
@@ -0,0 +1,258 @@
+"""Запускает проверки в одноразовом Docker-окружении и сохраняет evidence."""
+
+import argparse
+from datetime import datetime, timezone
+import io
+import json
+import os
+from pathlib import Path
+import subprocess
+import tarfile
+import tempfile
+from uuid import uuid4
+
+ROOT = Path(__file__).resolve().parents[1]
+EXCLUDED = {
+ ".git",
+ ".agents",
+ ".aws",
+ ".codex",
+ ".env",
+ "secrets",
+ "evidence",
+ ".venv",
+ "venv",
+ "__pycache__",
+ ".pytest_cache",
+ ".github",
+ "docs",
+}
+TEXT_SUFFIXES = {".py", ".sh", ".conf", ".yaml", ".yml", ".json", ".txt", ".md", ".ini"}
+
+
+def docker(*args, **kwargs):
+ kwargs.setdefault("check", True)
+ return subprocess.run(["docker", *args], cwd=ROOT, **kwargs)
+
+
+def archive_source(destination):
+ with tarfile.open(destination, "w") as archive:
+ for path in sorted(ROOT.rglob("*")):
+ relative = path.relative_to(ROOT)
+ if (
+ not path.is_file()
+ or path.is_symlink()
+ or any(part in EXCLUDED for part in relative.parts)
+ or path.name.startswith(".env.")
+ and path.name != ".env.example"
+ or path.suffix in {".zip", ".log", ".pem", ".key"}
+ ):
+ continue
+ content = path.read_bytes()
+ if path.suffix in TEXT_SUFFIXES:
+ content = content.replace(b"\r\n", b"\n")
+ entry = tarfile.TarInfo(relative.as_posix())
+ entry.size = len(content)
+ entry.mode = 0o755 if path.suffix == ".sh" else 0o644
+ archive.addfile(entry, io.BytesIO(content))
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ mode = parser.add_mutually_exclusive_group()
+ mode.add_argument("--format", action="store_true", help="применить Black и выйти")
+ mode.add_argument(
+ "--integration-only",
+ action="store_true",
+ help="запустить проверки стенда и телеметрии без pytest",
+ )
+ args = parser.parse_args()
+ project = "messenger-check-" + uuid4().hex[:12]
+ image = project + ":tools"
+ image_env = {
+ "API_IMAGE": project + ":api",
+ "MINIO_IMAGE": project + ":server",
+ "MINIO_BOOTSTRAP_IMAGE": project + ":bootstrap",
+ }
+ engine = project + "-engine"
+ runner = project + "-runner"
+ workspace = project + "-workspace"
+ data = project + "-data"
+ network = project + "-network"
+ output = ROOT / "evidence" / "lab6"
+ if args.integration_only:
+ output /= "integration"
+ output.mkdir(parents=True, exist_ok=True)
+ summary = {
+ "status": "failed",
+ "project": project,
+ "profile": "integration" if args.integration_only else "full",
+ "started_at": datetime.now(timezone.utc).isoformat(),
+ }
+ try:
+ docker("build", "--tag", image, "--file", "infra/testing/Dockerfile", ".")
+ if args.format:
+ docker(
+ "run",
+ "--rm",
+ "--volume",
+ str(ROOT) + ":/workspace",
+ image,
+ "python3",
+ "-m",
+ "black",
+ "app",
+ "infra",
+ "migrations",
+ "scripts",
+ "student-tests",
+ "tests",
+ )
+ summary["status"] = "formatted"
+ return
+ compose = ("compose", "--env-file", ".env.example")
+ env = {**os.environ, **image_env}
+ config = json.loads(
+ docker(
+ *compose, "config", "--format", "json", env=env, capture_output=True
+ ).stdout
+ )
+ dependencies = sorted(
+ {
+ service["image"]
+ for service in config["services"].values()
+ if "build" not in service
+ }
+ )
+ docker(
+ *compose,
+ "pull",
+ "--ignore-buildable",
+ "--policy",
+ "missing",
+ env=env,
+ timeout=3600,
+ )
+ docker(
+ *compose, "build", "api", "minio", "minio-bootstrap", env=env, timeout=3600
+ )
+ docker("network", "create", network, stdout=subprocess.DEVNULL)
+ for volume in (workspace, data):
+ docker("volume", "create", volume, stdout=subprocess.DEVNULL)
+ docker(
+ "run",
+ "-d",
+ "--privileged",
+ "--name",
+ engine,
+ "--network",
+ network,
+ "--env",
+ "DOCKER_TLS_CERTDIR=",
+ "--volume",
+ workspace + ":/workspace",
+ "--volume",
+ data + ":/var/lib/docker",
+ "docker:29.1.3-dind",
+ "dockerd",
+ "--host=unix:///var/run/docker.sock",
+ "--host=tcp://127.0.0.1:2375",
+ "--tls=false",
+ stdout=subprocess.DEVNULL,
+ )
+ docker(
+ "create",
+ "--name",
+ runner,
+ "--network",
+ "container:" + engine,
+ "--env",
+ "DOCKER_HOST=tcp://127.0.0.1:2375",
+ *[
+ arg
+ for key, value in image_env.items()
+ for arg in ("--env", f"{key}={value}")
+ ],
+ "--env",
+ f"INTEGRATION_ONLY={int(args.integration_only)}",
+ "--volume",
+ workspace + ":/workspace",
+ image,
+ stdout=subprocess.DEVNULL,
+ )
+ with tempfile.TemporaryDirectory(prefix="messenger-check-") as directory:
+ archive = Path(directory) / "source.tar"
+ archive_source(archive)
+ docker("cp", str(archive), runner + ":/tmp/source.tar")
+ image_archive = Path(directory) / "images.tar"
+ docker(
+ "image",
+ "save",
+ "--output",
+ str(image_archive),
+ *image_env.values(),
+ *dependencies,
+ )
+ docker("cp", str(image_archive), runner + ":/tmp/images.tar")
+ docker("start", "--attach", runner, check=False, timeout=7200)
+ result = subprocess.run(
+ ["docker", "wait", runner],
+ cwd=ROOT,
+ text=True,
+ encoding="utf-8",
+ capture_output=True,
+ timeout=7200,
+ )
+ result.check_returncode()
+ docker("cp", runner + ":/workspace/evidence/.", str(output))
+ if int(result.stdout.strip()) != 0:
+ raise RuntimeError(
+ f"Проверки завершились ошибкой. См. {output.relative_to(ROOT)}."
+ )
+ summary["status"] = "passed"
+ finally:
+ cleanup = []
+ for command in (
+ ["rm", "--force", runner, engine],
+ ["volume", "rm", workspace, data],
+ ["network", "rm", network],
+ ["image", "rm", image, *image_env.values()],
+ ):
+ result = subprocess.run(
+ ["docker", *command], cwd=ROOT, encoding="utf-8", capture_output=True
+ )
+ cleanup.append(
+ {
+ "command": ["docker", *command],
+ "returncode": result.returncode,
+ "output": result.stdout.strip() + result.stderr.strip(),
+ }
+ )
+ remaining = {}
+ for kind in ("container", "volume", "network", "image"):
+ result = subprocess.run(
+ ["docker", kind, "ls", "--format", "{{json .}}"],
+ cwd=ROOT,
+ encoding="utf-8",
+ capture_output=True,
+ )
+ remaining[kind] = [
+ line for line in result.stdout.splitlines() if project in line
+ ]
+ if result.returncode:
+ remaining[kind].append("resource audit failed")
+ summary["cleanup"] = cleanup
+ summary["remaining_resources"] = remaining
+ summary["finished_at"] = datetime.now(timezone.utc).isoformat()
+ if any(remaining.values()):
+ summary["status"] = "failed"
+ (output / "run.json").write_text(
+ json.dumps(summary, indent=2, ensure_ascii=False) + "\n", encoding="utf-8"
+ )
+ if summary["status"] == "failed":
+ raise SystemExit(1)
+ print("PASS: проверки завершены, временное окружение удалено")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py
index 1dfe4a9..b357eb7 100644
--- a/scripts/validate_openapi.py
+++ b/scripts/validate_openapi.py
@@ -1,23 +1,28 @@
#!/usr/bin/env python3
"""Full OpenAPI and JSON Schema checks; install requirements-dev.txt first."""
+
import json
from pathlib import Path
from openapi_spec_validator import validate
from jsonschema import Draft202012Validator
root = Path(__file__).resolve().parents[1]
-spec = json.loads((root/'contracts/openapi.json').read_text())
+spec = json.loads((root / "contracts/openapi.json").read_text(encoding="utf-8"))
validate(spec)
from referencing import Registry, Resource
-# The base URI resolves local #/components references inside examples.
-resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'})
-registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource))
-for name, example in json.loads((root/'contracts/examples.json').read_text()).items():
- schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name}
+
+resource = dict(spec, **{"$schema": "https://json-schema.org/draft/2020-12/schema"})
+registry = Registry().with_resource(
+ "urn:messenger:openapi", Resource.from_contents(resource)
+)
+for name, example in json.loads(
+ (root / "contracts/examples.json").read_text(encoding="utf-8")
+).items():
+ schema = {"$ref": "urn:messenger:openapi#/components/schemas/" + name}
Draft202012Validator(schema, registry=registry).validate(example)
-for path in (root/'contracts').glob('*.schema.json'):
- schema = json.loads(path.read_text())
+for path in (root / "contracts").glob("*.schema.json"):
+ schema = json.loads(path.read_text(encoding="utf-8"))
Draft202012Validator.check_schema(schema)
- for example in schema.get('examples',[]):
+ for example in schema.get("examples", []):
Draft202012Validator(schema).validate(example)
-print('PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.')
+print("PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.")
diff --git a/student-tests/conftest.py b/student-tests/conftest.py
new file mode 100644
index 0000000..f11d9d1
--- /dev/null
+++ b/student-tests/conftest.py
@@ -0,0 +1,191 @@
+"""Фикстуры на отдельных PostgreSQL и Redis тестового Docker-стенда."""
+
+import asyncio
+import os
+from collections.abc import AsyncIterator, Callable
+
+import httpx
+import pytest
+from alembic import command
+from alembic.config import Config
+from fastapi import FastAPI
+from redis.asyncio import Redis
+from redis.exceptions import RedisError
+from sqlalchemy import text
+from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
+from sqlalchemy.pool import NullPool
+
+from app.core.config import Settings
+from app.core.redis import create_redis_client
+from app.main import create_app
+from app.models import Base
+
+DEFAULT_TEST_DATABASE_URL = (
+ "postgresql+asyncpg://messenger:messenger@localhost:5432/messenger_test"
+)
+
+
+def _database_url() -> str:
+ return os.environ.get("TEST_DATABASE_URL", DEFAULT_TEST_DATABASE_URL)
+
+
+def _redis_settings() -> dict[str, object]:
+ return {
+ "redis_host": os.environ.get("TEST_REDIS_HOST", "localhost"),
+ "redis_port": int(os.environ.get("TEST_REDIS_PORT", "6379")),
+ "redis_password": os.environ.get("TEST_REDIS_PASSWORD") or None,
+ "redis_db": int(os.environ.get("TEST_REDIS_DB", "15")),
+ }
+
+
+def _alembic_config(url: str) -> Config:
+ config = Config("alembic.ini")
+ config.set_main_option("sqlalchemy.url", url.replace("%", "%%"))
+ return config
+
+
+async def upgrade_head(url: str) -> None:
+ """alembic command.upgrade внутри вызывает asyncio.run, поэтому — в отдельном потоке."""
+ await asyncio.to_thread(command.upgrade, _alembic_config(url), "head")
+
+
+async def reset_schema(engine: AsyncEngine) -> None:
+ async with engine.begin() as connection:
+ await connection.run_sync(Base.metadata.drop_all)
+ await connection.execute(text("DROP TABLE IF EXISTS alembic_version"))
+
+
+async def close_app(app: FastAPI) -> None:
+ """ASGITransport не запускает lifespan, поэтому ресурсы приложения закрываются явно."""
+ await app.state.redis.aclose()
+ await app.state.engine.dispose()
+
+
+@pytest.fixture(scope="session")
+def database_url() -> str:
+ return _database_url()
+
+
+@pytest.fixture(scope="session")
+async def db_engine(database_url: str) -> AsyncIterator[AsyncEngine]:
+ engine = create_async_engine(database_url, poolclass=NullPool)
+ try:
+ async with engine.connect() as connection:
+ await connection.execute(text("SELECT 1"))
+ except (SQLAlchemyError, OSError) as error:
+ pytest.exit(
+ f"PostgreSQL недоступна по {engine.url.render_as_string(hide_password=True)}: "
+ f"{type(error).__name__}. Запускайте через `python -X utf8 scripts/test.py`.",
+ returncode=2,
+ )
+
+ await reset_schema(engine)
+
+ if os.environ.get("TEST_USE_MIGRATIONS") == "1":
+ await upgrade_head(database_url)
+ else:
+ async with engine.begin() as connection:
+ await connection.run_sync(Base.metadata.create_all)
+
+ yield engine
+ await engine.dispose()
+
+
+@pytest.fixture(scope="session")
+async def redis_client() -> AsyncIterator[Redis]:
+ settings = build_settings(_database_url())
+ if settings.redis_db == 0:
+ pytest.exit("TEST_REDIS_DB=0 запрещён: тесты очищают базу Redis.", returncode=2)
+
+ client = create_redis_client(settings)
+ try:
+ await client.ping()
+ except (RedisError, OSError) as error:
+ pytest.exit(
+ f"Redis недоступен: {type(error).__name__}. Запускайте через `python -X utf8 scripts/test.py`.",
+ returncode=2,
+ )
+
+ yield client
+ await client.flushdb()
+ await client.aclose()
+
+
+@pytest.fixture(autouse=True)
+async def clean_db(db_engine: AsyncEngine, redis_client: Redis) -> None:
+ names = ", ".join(f'"{table.name}"' for table in Base.metadata.sorted_tables)
+ if names:
+ async with db_engine.begin() as connection:
+ await connection.execute(text(f"TRUNCATE {names} RESTART IDENTITY CASCADE"))
+ await redis_client.flushdb()
+
+
+def build_settings(database_url: str, **overrides: object) -> Settings:
+ values: dict[str, object] = {
+ "s3_enabled": False,
+ "app_host": "127.0.0.1",
+ "app_port": 8080,
+ "app_env": "test",
+ "max_messages": 1000,
+ "max_in_flight": 32,
+ "log_level": "CRITICAL",
+ "database_url": database_url,
+ "session_ttl_seconds": 300,
+ "db_pool_size": 10,
+ "db_max_overflow": 10,
+ **_redis_settings(),
+ }
+ values.update(overrides)
+ return Settings(_env_file=None, **values)
+
+
+@pytest.fixture
+def settings(database_url: str) -> Settings:
+ return build_settings(database_url)
+
+
+@pytest.fixture
+def make_settings(database_url: str) -> Callable[..., Settings]:
+ def factory(**overrides: object) -> Settings:
+ url = str(overrides.pop("database_url", database_url))
+ return build_settings(url, **overrides)
+
+ return factory
+
+
+def make_client(
+ app, *, raise_app_exceptions: bool = True, timeout: float = 30
+) -> httpx.AsyncClient:
+ return httpx.AsyncClient(
+ transport=httpx.ASGITransport(
+ app=app, raise_app_exceptions=raise_app_exceptions
+ ),
+ base_url="http://testserver",
+ timeout=timeout,
+ )
+
+
+@pytest.fixture
+def app(settings: Settings):
+ application = create_app(settings)
+
+ @application.get("/__test__/explode", include_in_schema=False)
+ async def explode() -> None:
+ raise RuntimeError("test-secret")
+
+ return application
+
+
+@pytest.fixture
+async def client(app) -> AsyncIterator[httpx.AsyncClient]:
+ async with make_client(app) as test_client:
+ yield test_client
+ await close_app(app)
+
+
+@pytest.fixture
+async def client_no_raise(app) -> AsyncIterator[httpx.AsyncClient]:
+ async with make_client(app, raise_app_exceptions=False) as test_client:
+ yield test_client
+ await close_app(app)
diff --git a/student-tests/helpers.py b/student-tests/helpers.py
new file mode 100644
index 0000000..8272d52
--- /dev/null
+++ b/student-tests/helpers.py
@@ -0,0 +1,85 @@
+import base64
+from uuid import uuid4
+
+from httpx import AsyncClient
+
+PASSWORD = "Correct-horse-battery-1"
+
+
+def basic_header(username: str, password: str) -> dict[str, str]:
+ raw = f"{username}:{password}".encode()
+ return {"Authorization": "Basic " + base64.b64encode(raw).decode()}
+
+
+def bearer(actor: dict) -> dict[str, str]:
+ return {"Authorization": "Bearer " + actor["token"]}
+
+
+async def register(
+ client: AsyncClient,
+ username: str | None = None,
+ display_name: str = "Тестовый пользователь",
+ password: str = PASSWORD,
+) -> dict:
+ actual_username = username or "user_" + uuid4().hex[:12]
+ response = await client.post(
+ "/api/v1/users",
+ json={
+ "username": actual_username,
+ "display_name": display_name,
+ "password": password,
+ },
+ )
+ assert response.status_code == 201, response.text
+ return {**response.json(), "password": password}
+
+
+async def login(client: AsyncClient, user: dict) -> dict:
+ response = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], user["password"]),
+ )
+ assert response.status_code == 201, response.text
+ return {**user, "token": response.json()["token"]}
+
+
+async def create_user(client: AsyncClient, **kwargs) -> dict:
+ return await login(client, await register(client, **kwargs))
+
+
+async def create_chat(
+ client: AsyncClient,
+ user: dict,
+ *members: dict,
+ title: str = "Тестовый чат",
+) -> dict:
+ member_ids = [user["id"], *(member["id"] for member in members)]
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(user),
+ json={"title": title, "member_ids": member_ids},
+ )
+ assert response.status_code == 201, response.text
+ return response.json()
+
+
+def idempotency_header(key: str | None = None) -> dict[str, str]:
+ return {"Idempotency-Key": key or uuid4().hex}
+
+
+async def send_message(
+ client: AsyncClient, user: dict, chat: dict, text: str, key: str | None = None
+):
+ return await client.post(
+ f"/api/v1/chats/{chat['id']}/messages",
+ headers={**bearer(user), **idempotency_header(key)},
+ json={"text": text},
+ )
+
+
+def assert_request_id(response) -> str:
+ header_id = response.headers.get("X-Request-Id")
+ assert header_id
+ if response.status_code >= 400:
+ assert response.json()["error"]["request_id"] == header_id
+ return header_id
diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py
new file mode 100644
index 0000000..98c2eb5
--- /dev/null
+++ b/student-tests/test_attachments.py
@@ -0,0 +1,482 @@
+"""API и транзакции с реальной БД; байты S3 подменены управляемым адаптером."""
+
+import asyncio
+import hashlib
+from io import BytesIO
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+import struct
+from uuid import UUID, uuid4
+import zlib
+
+import pytest
+from PIL import Image
+from sqlalchemy import func, select
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.ext.asyncio import AsyncSession
+from starlette.requests import ClientDisconnect
+
+from app.core.s3 import S3Settings
+from app.core.jobs import JobSettings
+from app.domain.attachment_jobs import process_attachment_job
+from app.models.job import AttachmentJob
+from app.schemas.job import JobMessage
+from app.domain import attachments as domain
+from app.errors.exceptions import unavailable
+from app.models.attachment import Attachment, UploadIntent
+from app.storage import attachments as repository
+from app.storage.s3 import ObjectInfo
+from helpers import bearer, create_chat, create_user, idempotency_header
+
+PNG = Path("tests/fixtures/sample.png").read_bytes()
+
+
+class MemoryStorage:
+
+ def __init__(self):
+ self.settings = S3Settings(_env_file=None)
+ self.objects = {}
+ self.fail_put = self.fail_delete = self.down = False
+
+ async def put_file(self, key, path, **metadata):
+ self.objects[key] = (path.read_bytes(), metadata)
+ if self.fail_put:
+ raise unavailable()
+
+ async def delete_object(self, key):
+ if self.fail_delete:
+ raise unavailable()
+ self.objects.pop(key, None)
+
+ async def stat_object(self, key):
+ if self.down:
+ raise unavailable()
+ info = None
+ if key in self.objects:
+ data, metadata = self.objects[key]
+ info = ObjectInfo(
+ key, len(data), metadata["content_type"], datetime.now(timezone.utc)
+ )
+ return info
+
+ async def open_object(self, key):
+ async def chunks():
+ yield self.objects[key][0]
+
+ return chunks()
+
+ async def check_available(self):
+ return not self.down
+
+
+@pytest.fixture
+async def context(client, app):
+ storage = MemoryStorage()
+ app.state.s3 = storage
+ alice, bob, eve = [await create_user(client) for _ in range(3)]
+ chat = await create_chat(client, alice, bob)
+ return storage, alice, bob, eve, chat
+
+
+async def upload(
+ client, actor, chat, data=PNG, filename="фото.png", content_type="image/png"
+):
+ return await client.post(
+ f"/api/v1/chats/{chat['id']}/attachments",
+ params={"filename": filename},
+ headers={**bearer(actor), "Content-Type": content_type},
+ content=data,
+ )
+
+
+async def finish_attachment(client, storage, actor, attachment_id):
+ engine = client._transport.app.state.engine
+ async with AsyncSession(engine) as session:
+ job = await session.scalar(
+ select(AttachmentJob).where(
+ AttachmentJob.attachment_id == UUID(attachment_id)
+ )
+ )
+ message = JobMessage(
+ job_id=job.id,
+ attachment_id=job.attachment_id,
+ pipeline_version=job.pipeline_version,
+ attempt=job.attempt + 1,
+ created_at=job.created_at,
+ )
+ await session.rollback()
+ assert await process_attachment_job(
+ session, storage, message, JobSettings(_env_file=None)
+ )
+ result = await client.get(
+ f"/api/v1/attachments/{attachment_id}", headers=bearer(actor)
+ )
+ assert result.json()["status"] == "ready"
+ return result.json()
+
+
+async def test_upload_download_and_attachment_only_message(client, context):
+ storage, alice, bob, eve, chat = context
+ response = await upload(client, alice, chat)
+ assert response.status_code == 202, response.text
+ attachment = response.json()
+ assert attachment["status"] == "queued"
+ before = await client.get(
+ f"/api/v1/attachments/{attachment['id']}/content", headers=bearer(alice)
+ )
+ assert before.status_code == 409
+ attachment = await finish_attachment(client, storage, alice, attachment["id"])
+ assert attachment["sha256"] == hashlib.sha256(PNG).hexdigest()
+ assert {v["name"] for v in attachment["variants"]} == {
+ "thumbnail",
+ "crop",
+ "optimized",
+ }
+ assert attachment["error_code"] is None
+ assert "object_key" not in attachment
+ id = attachment["id"]
+ for actor in (alice, bob):
+ content = await client.get(
+ f"/api/v1/attachments/{id}/content", headers=bearer(actor)
+ )
+ assert content.status_code == 200 and content.content == PNG
+ assert content.headers["content-type"] == "image/png"
+ key = idempotency_header()
+ body = {"text": "", "attachment_ids": [id]}
+ path = f"/api/v1/chats/{chat['id']}/messages"
+ message = await client.post(path, json=body, headers={**bearer(bob), **key})
+ assert message.status_code == 201, message.text
+ repeat = await client.post(path, json=body, headers={**bearer(bob), **key})
+ assert repeat.json() == message.json()
+ history = await client.get(path, headers=bearer(alice))
+ assert history.json()["items"][0]["attachment_ids"] == [id]
+ for suffix in ("", "/content"):
+ denied = await client.get(
+ f"/api/v1/attachments/{id}{suffix}", headers=bearer(eve)
+ )
+ assert denied.status_code == 404
+
+
+@pytest.mark.parametrize(
+ "data,content_type",
+ [
+ (b"not an image", "image/png"),
+ (PNG[:30], "image/png"),
+ (PNG, "image/jpeg"),
+ (PNG, "application/octet-stream"),
+ ],
+ ids=["not-image", "truncated-png", "mime-spoofing", "unsupported-type"],
+)
+async def test_invalid_image_and_mime_are_415(client, context, data, content_type):
+ storage, alice, _, _, chat = context
+ response = await upload(client, alice, chat, data, content_type=content_type)
+ assert response.status_code == 415, response.text
+ assert not storage.objects
+
+
+async def test_jpeg_is_stored_without_reencoding(client, context):
+ storage, alice, _, _, chat = context
+ output = BytesIO()
+ Image.new("RGB", (32, 32), "blue").save(output, format="JPEG")
+ original = output.getvalue()
+ response = await upload(client, alice, chat, original, content_type="image/jpeg")
+ assert response.status_code == 202, response.text
+ id = response.json()["id"]
+ await finish_attachment(client, storage, alice, id)
+ downloaded = await client.get(
+ f"/api/v1/attachments/{id}/content", headers=bearer(alice)
+ )
+ assert downloaded.content == original
+
+
+@pytest.mark.parametrize(
+ "filename", ["../a.png", "a\\b.png", "a\x00.png", "a\n.png", "x" * 256]
+)
+async def test_unsafe_filename_is_400(client, context, filename):
+ storage, alice, _, _, chat = context
+ response = await upload(client, alice, chat, filename=filename)
+ assert response.status_code == 400
+ assert not storage.objects
+
+
+async def test_oversize_chunked_upload_is_413(client, context):
+ storage, alice, _, _, chat = context
+
+ async def chunks():
+ for _ in range(11):
+ yield b"x" * 1048576
+
+ response = await upload(client, alice, chat, chunks())
+ assert response.status_code == 413
+ assert not storage.objects
+
+
+async def test_oversize_content_length_is_413(client, context):
+ _, alice, _, _, chat = context
+ response = await client.post(
+ f"/api/v1/chats/{chat['id']}/attachments?filename=a.png",
+ headers={
+ **bearer(alice),
+ "Content-Type": "image/png",
+ "Content-Length": "10485761",
+ },
+ content=PNG,
+ )
+ assert response.status_code == 413
+
+
+async def test_exact_byte_limit_accepts_valid_png(client, context):
+ _, alice, _, _, chat = context
+ maximum = 10_485_760
+
+ payload = bytes(maximum - len(PNG) - 12)
+ kind = b"npAD"
+ padding = (
+ struct.pack("!I", len(payload))
+ + kind
+ + payload
+ + struct.pack("!I", zlib.crc32(kind + payload))
+ )
+ image = PNG[:-12] + padding + PNG[-12:]
+ response = await upload(client, alice, chat, image)
+ assert response.status_code == 202, response.text
+ assert response.json()["size_bytes"] == maximum
+ assert response.json()["sha256"] == hashlib.sha256(image).hexdigest()
+
+
+async def test_exact_pixel_limit_accepts_valid_png(client, context):
+ _, alice, _, _, chat = context
+ output = BytesIO()
+ with Image.new("RGB", (8000, 5000), "blue") as image:
+ image.save(output, format="PNG")
+ response = await upload(client, alice, chat, output.getvalue())
+ assert response.status_code == 202, response.text
+
+
+async def test_pixel_limit_before_decoding(client, context):
+ _, alice, _, _, chat = context
+
+ def chunk(kind, data):
+ return (
+ struct.pack("!I", len(data))
+ + kind
+ + data
+ + struct.pack("!I", zlib.crc32(kind + data))
+ )
+
+ header = struct.pack("!IIBBBBB", 8000, 5001, 8, 2, 0, 0, 0)
+ image = (
+ b"\x89PNG\r\n\x1a\n"
+ + chunk(b"IHDR", header)
+ + chunk(b"IDAT", zlib.compress(b"\0"))
+ + chunk(b"IEND", b"")
+ )
+ response = await upload(client, alice, chat, image)
+ assert response.status_code == 413, response.text
+
+
+async def test_unknown_and_other_chat_attachment_are_404(client, context):
+ _, alice, bob, _, chat = context
+ attachment = (await upload(client, alice, chat)).json()
+ other_chat = await create_chat(client, alice, bob)
+ for id in (attachment["id"], str(uuid4())):
+ response = await client.post(
+ f"/api/v1/chats/{other_chat['id']}/messages",
+ json={"text": "", "attachment_ids": [id]},
+ headers={**bearer(alice), **idempotency_header()},
+ )
+ assert response.status_code == 404
+
+
+async def test_duplicate_ids_are_400(client, context):
+ _, alice, _, _, chat = context
+ id = (await upload(client, alice, chat)).json()["id"]
+ response = await client.post(
+ f"/api/v1/chats/{chat['id']}/messages",
+ json={"text": "", "attachment_ids": [id, id]},
+ headers={**bearer(alice), **idempotency_header()},
+ )
+ assert response.status_code == 400
+
+
+async def test_empty_ids_and_absent_ids_have_same_idempotency(client, context):
+ _, alice, _, _, chat = context
+ headers = {**bearer(alice), **idempotency_header()}
+ path = f"/api/v1/chats/{chat['id']}/messages"
+ first = await client.post(path, json={"text": "same"}, headers=headers)
+ second = await client.post(
+ path, json={"text": "same", "attachment_ids": []}, headers=headers
+ )
+ assert first.status_code == second.status_code == 201
+ assert first.json() == second.json()
+
+
+async def test_s3_outage_is_503(client, context):
+ storage, alice, _, _, chat = context
+ id = (await upload(client, alice, chat)).json()["id"]
+ await finish_attachment(client, storage, alice, id)
+ storage.down = True
+ response = await client.get(
+ f"/api/v1/attachments/{id}/content", headers=bearer(alice)
+ )
+ assert response.status_code == 503
+ assert (await client.get("/health/ready")).status_code == 503
+
+
+async def test_uncertain_put_is_compensated(client, context, db_engine):
+ storage, alice, _, _, chat = context
+ storage.fail_put = True
+ response = await upload(client, alice, chat)
+ assert response.status_code == 503
+ assert not storage.objects
+ async with AsyncSession(db_engine) as session:
+ assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
+
+
+async def test_failure_after_s3_write_compensates(client, context, monkeypatch):
+ storage, alice, _, _, chat = context
+
+ async def fail(*args, **kwargs):
+ raise OperationalError("injected", {}, None)
+
+ monkeypatch.setattr(repository, "create_attachment", fail)
+ response = await upload(client, alice, chat)
+ assert response.status_code == 503
+ assert not storage.objects
+
+
+async def test_ambiguous_successful_commit_keeps_original(
+ client, context, db_engine, monkeypatch
+):
+ storage, alice, _, _, chat = context
+ original = AsyncSession.commit
+ count = 0
+
+ async def uncertain(session):
+ nonlocal count
+ count += 1
+ await original(session)
+ if count == 2:
+ raise OperationalError("lost commit acknowledgement", {}, None)
+
+ monkeypatch.setattr(AsyncSession, "commit", uncertain)
+ response = await upload(client, alice, chat)
+ assert response.status_code == 503
+ assert len(storage.objects) == 1
+ async with AsyncSession(db_engine) as session:
+ assert await session.scalar(select(func.count()).select_from(Attachment)) == 1
+ assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
+
+
+async def test_failed_compensation_then_dry_run_and_cleanup(client, context, db_engine):
+ storage, alice, _, _, chat = context
+ storage.fail_put = storage.fail_delete = True
+ assert (await upload(client, alice, chat)).status_code == 503
+ assert len(storage.objects) == 1
+ async with AsyncSession(db_engine) as session:
+ intent = await session.scalar(select(UploadIntent))
+ intent.created_at = datetime.now(timezone.utc) - timedelta(days=2)
+ await session.commit()
+ storage.fail_put = storage.fail_delete = False
+ async with AsyncSession(db_engine) as session:
+ dry_run = await domain.reconcile_attachments(session, storage)
+ assert dry_run.candidates == 1 and dry_run.deleted == 0
+ assert storage.objects
+ applied = await domain.reconcile_attachments(session, storage, dry_run=False)
+ assert applied.deleted == 1 and not storage.objects
+ repeated = await domain.reconcile_attachments(session, storage, dry_run=False)
+ assert repeated.deleted == 0
+
+
+async def test_cleanup_keeps_fresh_and_skips_locked_uploads(client, context, db_engine):
+ storage, _, _, _, _ = context
+ old_id, fresh_id = uuid4(), uuid4()
+ async with AsyncSession(db_engine) as session:
+ session.add_all(
+ [
+ UploadIntent(
+ id=old_id,
+ object_key="originals/old",
+ created_at=datetime.now(timezone.utc) - timedelta(days=2),
+ ),
+ UploadIntent(id=fresh_id, object_key="originals/fresh"),
+ ]
+ )
+ await session.commit()
+ async with AsyncSession(db_engine) as uploader, AsyncSession(db_engine) as cleaner:
+ await repository.lock_upload(uploader, old_id)
+ result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
+ assert result.examined == 0
+ await uploader.rollback()
+ result = await domain.reconcile_attachments(cleaner, storage, dry_run=False)
+ assert result.examined == 1
+ assert await cleaner.get(UploadIntent, fresh_id) is not None
+
+
+async def test_client_disconnect_before_put_has_no_side_effects(client, context):
+ storage, alice, _, _, chat = context
+
+ async def interrupted():
+ yield PNG[:16]
+ raise ClientDisconnect()
+
+ response = await upload(client, alice, chat, interrupted())
+ assert response.status_code == 400
+ assert not storage.objects
+
+
+async def test_upload_deadline_cancels_validation(client, context, monkeypatch):
+ storage, alice, _, _, chat = context
+ storage.settings = storage.settings.model_copy(
+ update={"upload_timeout_seconds": 0.02}
+ )
+
+ async def stalled(*args):
+ await asyncio.sleep(10)
+
+ monkeypatch.setattr(domain, "validate_image", stalled)
+ response = await upload(client, alice, chat)
+ assert response.status_code == 400
+ assert not storage.objects
+
+
+async def test_cancelled_put_is_compensated(client, context, db_engine, monkeypatch):
+ storage, alice, _, _, chat = context
+ entered = asyncio.Event()
+ original = storage.put_file
+
+ async def interrupted(*args, **kwargs):
+ await original(*args, **kwargs)
+ entered.set()
+ await asyncio.sleep(10)
+
+ monkeypatch.setattr(storage, "put_file", interrupted)
+ task = asyncio.create_task(upload(client, alice, chat))
+ await asyncio.wait_for(entered.wait(), timeout=10)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert not storage.objects
+ async with AsyncSession(db_engine) as session:
+ assert await session.scalar(select(func.count()).select_from(UploadIntent)) == 0
+
+
+async def test_cleanup_never_deletes_ready_attachment(client, context, db_engine):
+ storage, alice, _, _, chat = context
+ id = (await upload(client, alice, chat)).json()["id"]
+ await finish_attachment(client, storage, alice, id)
+ async with AsyncSession(db_engine) as session:
+ attachment = await session.get(Attachment, UUID(id))
+ key = attachment.object_key
+
+ session.add(
+ UploadIntent(
+ id=uuid4(),
+ object_key=key,
+ created_at=datetime.now(timezone.utc) - timedelta(days=2),
+ )
+ )
+ await session.commit()
+ result = await domain.reconcile_attachments(session, storage, dry_run=False)
+ assert result.examined == 1 and result.deleted == 0
+ assert key in storage.objects
diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py
new file mode 100644
index 0000000..6746320
--- /dev/null
+++ b/student-tests/test_auth.py
@@ -0,0 +1,298 @@
+import asyncio
+import hashlib
+from datetime import datetime
+
+from httpx import AsyncClient
+
+from app.main import create_app
+from conftest import close_app, make_client
+from helpers import (
+ PASSWORD,
+ assert_request_id,
+ basic_header,
+ bearer,
+ create_user,
+ login,
+ register,
+)
+
+
+async def test_login_returns_opaque_bearer_session(client: AsyncClient):
+ user = await register(client)
+
+ response = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], user["password"]),
+ )
+
+ assert response.status_code == 201
+ body = response.json()
+ assert set(body) == {"token", "token_type", "expires_at", "user"}
+ assert body["token_type"] == "Bearer"
+ assert len(body["token"]) >= 43
+ assert body["token"] not in (user["id"], user["username"])
+ assert body["user"]["id"] == user["id"]
+ assert body["expires_at"].endswith("Z")
+ assert datetime.fromisoformat(body["expires_at"])
+ assert_request_id(response)
+
+
+async def test_each_login_issues_a_different_token(client: AsyncClient):
+ user = await register(client)
+ first = await login(client, user)
+ second = await login(client, user)
+
+ assert first["token"] != second["token"]
+
+
+async def test_password_with_colon_works(client: AsyncClient):
+ user = await register(client, password="pass:with:colons-123")
+
+ assert (await login(client, user))["token"]
+
+
+async def test_unicode_password_works(client: AsyncClient):
+ user = await register(client, password="Пароль-с-юникодом-🔑-123")
+
+ assert (await login(client, user))["token"]
+
+
+async def test_wrong_password_and_unknown_user_look_identical(client: AsyncClient):
+ user = await register(client)
+
+ wrong_password = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], "wrong-password-123"),
+ )
+ unknown_user = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header("no_such_user", "wrong-password-123"),
+ )
+
+ for response in (wrong_password, unknown_user):
+ assert response.status_code == 401
+ assert response.json()["error"]["code"] == "unauthorized"
+ assert response.headers["WWW-Authenticate"].startswith(
+ 'Basic realm="messenger"'
+ )
+ assert_request_id(response)
+ assert (
+ wrong_password.json()["error"]["message"]
+ == unknown_user.json()["error"]["message"]
+ )
+
+
+async def test_login_requires_basic_scheme(client: AsyncClient):
+ user = await register(client)
+ session = await login(client, user)
+
+ for headers in ({}, bearer(session), {"Authorization": "Basic !!!not-base64"}):
+ response = await client.post("/api/v1/auth/sessions", headers=headers)
+ assert response.status_code == 401
+ assert response.headers["WWW-Authenticate"].startswith("Basic")
+
+
+async def test_business_endpoints_require_bearer(client: AsyncClient):
+ user = await register(client)
+
+ for headers in (
+ {},
+ basic_header(user["username"], user["password"]),
+ {"Authorization": "Bearer "},
+ {"Authorization": "Bearer unknown-token-value"},
+ {"X-User-Id": user["id"]},
+ ):
+ response = await client.get("/api/v1/users/me", headers=headers)
+ assert response.status_code == 401, headers
+ assert response.json()["error"]["code"] == "unauthorized"
+ assert response.headers["WWW-Authenticate"].startswith(
+ 'Bearer realm="messenger"'
+ )
+ assert_request_id(response)
+
+
+async def test_x_user_id_cannot_override_bearer_identity(client: AsyncClient):
+ alice = await create_user(client)
+ eve = await create_user(client)
+
+ response = await client.get(
+ "/api/v1/users/me",
+ headers={**bearer(alice), "X-User-Id": eve["id"]},
+ )
+
+ assert response.status_code == 200
+ assert response.json()["id"] == alice["id"]
+
+
+async def test_logout_revokes_only_current_session(client: AsyncClient):
+ user = await register(client)
+ first = await login(client, user)
+ second = await login(client, user)
+
+ assert (
+ await client.delete("/api/v1/auth/sessions/current", headers=bearer(first))
+ ).status_code == 204
+
+ assert (
+ await client.get("/api/v1/users/me", headers=bearer(first))
+ ).status_code == 401
+ assert (
+ await client.get("/api/v1/users/me", headers=bearer(second))
+ ).status_code == 200
+
+
+async def test_logout_twice_returns_401(client: AsyncClient):
+ session = await create_user(client)
+
+ first = await client.delete(
+ "/api/v1/auth/sessions/current", headers=bearer(session)
+ )
+ again = await client.delete(
+ "/api/v1/auth/sessions/current", headers=bearer(session)
+ )
+
+ assert first.status_code == 204
+ assert first.content == b""
+ assert again.status_code == 401
+
+
+async def test_session_expires_after_ttl(make_settings):
+ app = create_app(make_settings(session_ttl_seconds=2))
+ async with make_client(app) as client:
+ session = await create_user(client)
+ assert (
+ await client.get("/api/v1/users/me", headers=bearer(session))
+ ).status_code == 200
+
+ await asyncio.sleep(3)
+
+ response = await client.get("/api/v1/users/me", headers=bearer(session))
+ assert response.status_code == 401
+ assert response.headers["WWW-Authenticate"].startswith("Bearer")
+ await close_app(app)
+
+
+async def test_token_is_stored_only_as_sha256_key(client: AsyncClient, redis_client):
+ session = await create_user(client)
+
+ keys = [key async for key in redis_client.scan_iter("session:*")]
+ token_hash = hashlib.sha256(session["token"].encode()).hexdigest()
+
+ assert keys == [f"session:{token_hash}"]
+ assert await redis_client.get(keys[0]) == session["id"]
+ assert all(session["token"] not in key for key in keys)
+
+
+async def test_password_is_not_stored_in_plaintext(client: AsyncClient, db_engine):
+ from sqlalchemy import text
+
+ await register(client, username="hash_check")
+
+ async with db_engine.connect() as connection:
+ result = await connection.execute(
+ text("SELECT password_hash FROM users WHERE username = 'hash_check'")
+ )
+ stored = result.scalar_one()
+
+ assert PASSWORD not in stored
+ assert stored.startswith("$argon2id$")
+
+
+async def test_login_rejects_out_of_range_password_like_wrong_password(
+ client: AsyncClient,
+):
+ user = await register(client)
+ reference = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], "wrong-password-123"),
+ )
+
+ for password in ("x" * 11, "x" * 129, ""):
+ response = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], password),
+ )
+ assert response.status_code == 401
+ assert (
+ response.json()["error"]["message"] == reference.json()["error"]["message"]
+ )
+ assert (
+ response.headers["WWW-Authenticate"]
+ == reference.headers["WWW-Authenticate"]
+ )
+
+
+async def test_malformed_basic_credentials_return_401(client: AsyncClient):
+ import base64
+
+ def raw(value: bytes) -> dict[str, str]:
+ return {"Authorization": "Basic " + base64.b64encode(value).decode()}
+
+ for headers in (
+ raw(b"no-colon-here"),
+ raw(b":password-without-user"),
+ raw(b"\xff\xfe:invalid-utf8"),
+ {"Authorization": "Basic"},
+ {"Authorization": "Digest abc"},
+ ):
+ response = await client.post("/api/v1/auth/sessions", headers=headers)
+ assert response.status_code == 401, headers
+ assert response.headers["WWW-Authenticate"].startswith("Basic")
+
+
+async def test_unknown_user_takes_comparable_time_to_wrong_password(
+ client: AsyncClient,
+):
+ import statistics
+ import time
+
+ user = await register(client)
+
+ async def measure(username: str) -> float:
+ samples = []
+ for _ in range(5):
+ started = time.perf_counter()
+ await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(username, "wrong-password-123"),
+ )
+ samples.append(time.perf_counter() - started)
+ return statistics.median(samples)
+
+ wrong_password = await measure(user["username"])
+ unknown_user = await measure("no_such_user")
+
+ assert unknown_user > wrong_password * 0.5
+
+
+async def test_invalid_basic_username_returns_401_not_500(client: AsyncClient):
+ user = await register(client)
+ reference = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], "wrong-password-123"),
+ )
+
+ for username in ("bad\u0000name", "UPPER", "no", "x" * 33, "имя", "a b c"):
+ response = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(username, "wrong-password-123"),
+ )
+ assert response.status_code == 401, username
+ assert (
+ response.json()["error"]["message"] == reference.json()["error"]["message"]
+ )
+ assert (
+ response.headers["WWW-Authenticate"]
+ == reference.headers["WWW-Authenticate"]
+ )
+
+
+async def test_whitespace_only_password_at_login_returns_401(client: AsyncClient):
+ user = await register(client)
+
+ response = await client.post(
+ "/api/v1/auth/sessions", headers=basic_header(user["username"], " " * 12)
+ )
+
+ assert response.status_code == 401
+ assert response.headers["WWW-Authenticate"].startswith("Basic")
diff --git a/student-tests/test_chats.py b/student-tests/test_chats.py
new file mode 100644
index 0000000..b5f4fa9
--- /dev/null
+++ b/student-tests/test_chats.py
@@ -0,0 +1,208 @@
+from urllib.parse import quote
+from uuid import uuid4
+
+import pytest
+from httpx import AsyncClient
+
+from helpers import assert_request_id, bearer, create_chat, create_user
+
+
+async def test_create_and_get_chat(client: AsyncClient):
+ alice = await create_user(client)
+ bob = await create_user(client)
+ chat = await create_chat(client, alice, bob)
+
+ response = await client.get(
+ f"/api/v1/chats/{chat['id']}",
+ headers=bearer(bob),
+ )
+
+ assert response.status_code == 200
+ assert response.json() == chat
+ assert_request_id(response)
+
+
+async def test_chat_is_not_visible_to_outsider(client: AsyncClient):
+ alice = await create_user(client)
+ bob = await create_user(client)
+ eve = await create_user(client)
+ chat = await create_chat(client, alice, bob)
+
+ response = await client.get(
+ f"/api/v1/chats/{chat['id']}",
+ headers=bearer(eve),
+ )
+
+ assert response.status_code == 404
+ assert response.json()["error"]["code"] == "not_found"
+ assert_request_id(response)
+
+
+async def test_chat_creation_requires_creator_in_members(client: AsyncClient):
+ alice = await create_user(client)
+ bob = await create_user(client)
+
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "Чат", "member_ids": [bob["id"]]},
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+@pytest.mark.parametrize(
+ "payload",
+ [
+ {"title": "", "member_ids": []},
+ {"title": " ", "member_ids": []},
+ ],
+)
+async def test_chat_payload_boundaries(client: AsyncClient, payload: dict):
+ alice = await create_user(client)
+ payload["member_ids"] = [alice["id"]]
+
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json=payload,
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+async def test_duplicate_members_are_rejected(client: AsyncClient):
+ alice = await create_user(client)
+ bob = await create_user(client)
+
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "Чат", "member_ids": [alice["id"], bob["id"], bob["id"]]},
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+async def test_unknown_member_is_rejected(client: AsyncClient):
+ alice = await create_user(client)
+
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "Чат", "member_ids": [alice["id"], str(uuid4())]},
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+async def test_chat_exact_title_limit_is_accepted(client: AsyncClient):
+ alice = await create_user(client)
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "x" * 100, "member_ids": [alice["id"]]},
+ )
+
+ assert response.status_code == 201
+ assert len(response.json()["title"]) == 100
+
+
+async def test_chat_list_hides_chats_without_membership(client: AsyncClient):
+ alice = await create_user(client)
+ bob = await create_user(client)
+ chat_for_bob = await create_chat(client, bob, title="Only Bob")
+ chat_for_alice = await create_chat(client, alice, title="Only Alice")
+
+ response = await client.get(
+ "/api/v1/chats?limit=100",
+ headers=bearer(alice),
+ )
+
+ assert response.status_code == 200
+ ids = {item["id"] for item in response.json()["items"]}
+ assert chat_for_alice["id"] in ids
+ assert chat_for_bob["id"] not in ids
+
+
+async def test_nonexistent_chat_returns_404(client: AsyncClient):
+ alice = await create_user(client)
+
+ response = await client.get(
+ "/api/v1/chats/00000000-0000-4000-8000-000000000000",
+ headers=bearer(alice),
+ )
+
+ assert response.status_code == 404
+ assert response.json()["error"]["code"] == "not_found"
+ assert_request_id(response)
+
+
+async def test_failed_chat_creation_leaves_no_partial_data(
+ client: AsyncClient, db_engine
+):
+ from sqlalchemy import text
+
+ alice = await create_user(client)
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "Чат", "member_ids": [alice["id"], str(uuid4())]},
+ )
+ assert response.status_code == 400
+
+ async with db_engine.connect() as connection:
+ chats = (
+ await connection.execute(text("SELECT count(*) FROM chats"))
+ ).scalar_one()
+ members = (
+ await connection.execute(text("SELECT count(*) FROM chat_members"))
+ ).scalar_one()
+ assert chats == 0 and members == 0
+
+
+async def test_chat_list_pagination_and_order(client: AsyncClient):
+ alice = await create_user(client)
+ chats = [await create_chat(client, alice, title=f"chat-{i}") for i in range(5)]
+
+ page1 = (await client.get("/api/v1/chats?limit=2", headers=bearer(alice))).json()
+ page2 = (
+ await client.get(
+ f"/api/v1/chats?limit=2&cursor={quote(page1['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+ ).json()
+ page3 = (
+ await client.get(
+ f"/api/v1/chats?limit=2&cursor={quote(page2['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+ ).json()
+
+ ids = [c["id"] for page in (page1, page2, page3) for c in page["items"]]
+ assert ids == [c["id"] for c in chats]
+ assert page3["next_cursor"] is None
+
+
+async def test_chat_endpoints_require_bearer(client: AsyncClient):
+ assert (await client.get("/api/v1/chats")).status_code == 401
+ assert (
+ await client.post("/api/v1/chats", json={"title": "x", "member_ids": []})
+ ).status_code in (401, 400)
+
+
+async def test_nul_in_chat_title_is_rejected(client: AsyncClient):
+ alice = await create_user(client)
+
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "chat\u0000title", "member_ids": [alice["id"]]},
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py
new file mode 100644
index 0000000..bf07e6f
--- /dev/null
+++ b/student-tests/test_concurrency.py
@@ -0,0 +1,273 @@
+"""Конкурентность на реальной БД: гонка регистрации и параллельная запись."""
+
+import asyncio
+from collections import Counter
+from uuid import uuid4
+
+from sqlalchemy import text
+
+from app.main import create_app
+from conftest import close_app, make_client
+from helpers import (
+ PASSWORD,
+ basic_header,
+ bearer,
+ create_user,
+ idempotency_header,
+ register,
+)
+
+
+async def test_parallel_registration_of_same_username_creates_exactly_one_user(
+ make_settings,
+):
+ app = create_app(
+ make_settings(max_in_flight=64, db_pool_size=20, db_max_overflow=20)
+ )
+
+ async with make_client(app) as client:
+ responses = await asyncio.gather(
+ *(
+ client.post(
+ "/api/v1/users",
+ json={
+ "username": "race_user",
+ "display_name": "Race",
+ "password": PASSWORD,
+ },
+ )
+ for _ in range(16)
+ )
+ )
+ await close_app(app)
+
+ statuses = Counter(response.status_code for response in responses)
+ assert statuses == Counter({201: 1, 409: 15}), statuses
+
+
+async def test_100_messages_from_10_clients(make_settings):
+ app = create_app(
+ make_settings(
+ max_messages=1000, max_in_flight=100, db_pool_size=20, db_max_overflow=20
+ )
+ )
+
+ async with make_client(app) as client:
+ users = [
+ await create_user(client, username=f"load_{i}_{uuid4().hex[:6]}")
+ for i in range(10)
+ ]
+ created = await client.post(
+ "/api/v1/chats",
+ headers=bearer(users[0]),
+ json={"title": "Concurrency", "member_ids": [u["id"] for u in users]},
+ )
+ chat = created.json()
+
+ async def worker(index: int, user: dict) -> list[dict]:
+ responses = await asyncio.gather(
+ *(
+ client.post(
+ f"/api/v1/chats/{chat['id']}/messages",
+ headers={**bearer(user), **idempotency_header()},
+ json={"text": f"client-{index}-message-{n}"},
+ )
+ for n in range(10)
+ )
+ )
+ assert all(r.status_code == 201 for r in responses), [
+ r.text for r in responses if r.status_code != 201
+ ]
+ return [r.json() for r in responses]
+
+ results = await asyncio.gather(*(worker(i, u) for i, u in enumerate(users)))
+ messages = [m for batch in results for m in batch]
+
+ assert len(messages) == 100
+ assert len({m["id"] for m in messages}) == 100
+ assert Counter(m["sender_id"] for m in messages) == Counter(
+ {u["id"]: 10 for u in users}
+ )
+
+ history = await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=100", headers=bearer(users[0])
+ )
+ items = history.json()["items"]
+ assert len(items) == 100
+ assert len({m["id"] for m in items}) == 100
+ await close_app(app)
+
+
+async def test_chat_creation_in_parallel_is_atomic(make_settings, db_engine):
+ app = create_app(
+ make_settings(max_in_flight=32, db_pool_size=20, db_max_overflow=20)
+ )
+
+ async with make_client(app) as client:
+ alice = await create_user(client)
+ bob = await create_user(client)
+ responses = await asyncio.gather(
+ *(
+ client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": f"c{i}", "member_ids": [alice["id"], bob["id"]]},
+ )
+ for i in range(10)
+ )
+ )
+ assert all(r.status_code == 201 for r in responses)
+ await close_app(app)
+
+ async with db_engine.connect() as connection:
+ chats = (
+ await connection.execute(text("SELECT count(*) FROM chats"))
+ ).scalar_one()
+ members = (
+ await connection.execute(text("SELECT count(*) FROM chat_members"))
+ ).scalar_one()
+ assert chats == 10 and members == 20
+
+
+async def test_parallel_registration_of_distinct_usernames_all_succeed(make_settings):
+ app = create_app(make_settings(max_in_flight=64))
+ async with make_client(app) as client:
+ responses = await asyncio.gather(
+ *(
+ client.post(
+ "/api/v1/users",
+ json={
+ "username": f"distinct_{i}",
+ "display_name": "D",
+ "password": PASSWORD,
+ },
+ )
+ for i in range(12)
+ )
+ )
+ await close_app(app)
+
+ assert [r.status_code for r in responses] == [201] * 12
+ assert len({r.json()["id"] for r in responses}) == 12
+
+
+async def test_parallel_logins_issue_distinct_valid_tokens(make_settings):
+ app = create_app(make_settings(max_in_flight=64))
+ async with make_client(app) as client:
+ user = await register(client)
+ responses = await asyncio.gather(
+ *(
+ client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], user["password"]),
+ )
+ for _ in range(10)
+ )
+ )
+ tokens = [r.json()["token"] for r in responses]
+ checks = await asyncio.gather(
+ *(
+ client.get(
+ "/api/v1/users/me", headers={"Authorization": f"Bearer {token}"}
+ )
+ for token in tokens
+ )
+ )
+ await close_app(app)
+
+ assert all(r.status_code == 201 for r in responses)
+ assert len(set(tokens)) == 10
+ assert all(r.status_code == 200 for r in checks)
+
+
+async def test_parallel_logout_of_same_token_succeeds_once(make_settings):
+ app = create_app(make_settings(max_in_flight=64))
+ async with make_client(app) as client:
+ session = await create_user(client)
+ responses = await asyncio.gather(
+ *(
+ client.delete("/api/v1/auth/sessions/current", headers=bearer(session))
+ for _ in range(10)
+ )
+ )
+ await close_app(app)
+
+ assert Counter(r.status_code for r in responses) == Counter({204: 1, 401: 9})
+
+
+async def test_message_limit_is_exact_under_concurrency(make_settings, db_engine):
+ app = create_app(
+ make_settings(
+ max_messages=5, max_in_flight=64, db_pool_size=20, db_max_overflow=20
+ )
+ )
+ async with make_client(app) as client:
+ user = await create_user(client)
+ chat = (
+ await client.post(
+ "/api/v1/chats",
+ headers=bearer(user),
+ json={"title": "limit", "member_ids": [user["id"]]},
+ )
+ ).json()
+ responses = await asyncio.gather(
+ *(
+ client.post(
+ f"/api/v1/chats/{chat['id']}/messages",
+ headers={**bearer(user), **idempotency_header()},
+ json={"text": f"m{i}"},
+ )
+ for i in range(20)
+ )
+ )
+ await close_app(app)
+
+ assert Counter(r.status_code for r in responses) == Counter({201: 5, 503: 15})
+ async with db_engine.connect() as connection:
+ stored = (
+ await connection.execute(text("SELECT count(*) FROM messages"))
+ ).scalar_one()
+ counter = (
+ await connection.execute(
+ text("SELECT value FROM counters WHERE name = 'messages'")
+ )
+ ).scalar_one()
+ assert stored == counter == 5
+
+
+async def test_failed_chat_insert_rolls_back_whole_transaction(
+ make_settings, db_engine, monkeypatch
+):
+ import app.domain.chats as chats_domain
+
+ async def accept_everyone(_session, user_ids):
+ return set(user_ids)
+
+ app = create_app(make_settings())
+ async with make_client(app) as client:
+ alice = await create_user(client)
+ monkeypatch.setattr(
+ chats_domain.users_storage, "get_existing_user_ids", accept_everyone
+ )
+ response = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "broken", "member_ids": [alice["id"], str(uuid4())]},
+ )
+ monkeypatch.undo()
+ after = await client.post(
+ "/api/v1/chats",
+ headers=bearer(alice),
+ json={"title": "ok", "member_ids": [alice["id"]]},
+ )
+ await close_app(app)
+
+ assert response.status_code == 400
+ assert after.status_code == 201
+ async with db_engine.connect() as connection:
+ titles = (await connection.execute(text("SELECT title FROM chats"))).scalars()
+ members = (
+ await connection.execute(text("SELECT count(*) FROM chat_members"))
+ ).scalar_one()
+ assert list(titles) == ["ok"]
+ assert members == 1
diff --git a/student-tests/test_constraints.py b/student-tests/test_constraints.py
new file mode 100644
index 0000000..11aa1f3
--- /dev/null
+++ b/student-tests/test_constraints.py
@@ -0,0 +1,46 @@
+"""Ограничения на уровне БД: приложение — не единственный защитник данных."""
+
+import pytest
+from sqlalchemy import text
+from sqlalchemy.exc import IntegrityError
+
+from helpers import create_chat, create_user
+
+
+async def test_duplicate_username_is_rejected_by_database(client, db_engine):
+ await create_user(client, username="unique_name")
+
+ with pytest.raises(IntegrityError):
+ async with db_engine.begin() as connection:
+ await connection.execute(
+ text(
+ "INSERT INTO users (id, username, display_name, password_hash) "
+ "VALUES (gen_random_uuid(), 'unique_name', 'Other', 'hash')"
+ )
+ )
+
+
+async def test_message_requires_existing_chat(client, db_engine):
+ alice = await create_user(client)
+
+ with pytest.raises(IntegrityError):
+ async with db_engine.begin() as connection:
+ await connection.execute(
+ text(
+ "INSERT INTO messages (id, chat_id, sender_id, text) "
+ "VALUES (gen_random_uuid(), gen_random_uuid(), :sender, 'x')"
+ ),
+ {"sender": alice["id"]},
+ )
+
+
+async def test_duplicate_membership_is_rejected_by_database(client, db_engine):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+
+ with pytest.raises(IntegrityError):
+ async with db_engine.begin() as connection:
+ await connection.execute(
+ text("INSERT INTO chat_members (chat_id, user_id) VALUES (:c, :u)"),
+ {"c": chat["id"], "u": alice["id"]},
+ )
diff --git a/student-tests/test_errors.py b/student-tests/test_errors.py
new file mode 100644
index 0000000..f3bd5cd
--- /dev/null
+++ b/student-tests/test_errors.py
@@ -0,0 +1,77 @@
+import json
+import logging
+
+from httpx import AsyncClient
+
+from helpers import assert_request_id
+
+
+async def test_missing_token_returns_401(client: AsyncClient):
+ response = await client.get("/api/v1/users/me")
+
+ assert response.status_code == 401
+ assert response.json()["error"]["code"] == "unauthorized"
+ assert_request_id(response)
+
+
+async def test_garbage_token_returns_401(client: AsyncClient):
+ response = await client.get(
+ "/api/v1/users/me", headers={"Authorization": "Bearer garbage"}
+ )
+
+ assert response.status_code == 401
+ assert response.json()["error"]["code"] == "unauthorized"
+ assert_request_id(response)
+
+
+async def test_malformed_json_returns_400(client: AsyncClient):
+ response = await client.post(
+ "/api/v1/users",
+ content='{"username":"alice",',
+ headers={"Content-Type": "application/json"},
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+ assert_request_id(response)
+
+
+async def test_not_found_returns_404(client: AsyncClient):
+ response = await client.get("/no-such-path")
+
+ assert response.status_code == 404
+ assert response.json()["error"]["code"] == "not_found"
+ assert_request_id(response)
+
+
+async def test_method_not_allowed_preserves_405(client: AsyncClient):
+ response = await client.delete("/api/v1/users")
+
+ assert response.status_code == 405
+ assert response.json()["error"]["code"] == "invalid_request"
+ assert_request_id(response)
+
+
+async def test_unhandled_exception_is_hidden_from_client(
+ client_no_raise: AsyncClient, caplog
+):
+ with caplog.at_level(logging.ERROR, logger="app.errors.handlers"):
+ response = await client_no_raise.get("/__test__/explode")
+
+ assert response.status_code == 500
+ payload = response.json()
+ assert payload["error"]["code"] == "internal_error"
+ assert payload["error"]["message"] == "Внутренняя ошибка сервера"
+ assert "test-secret" not in json.dumps(payload, ensure_ascii=False)
+ assert "Traceback" not in json.dumps(payload, ensure_ascii=False)
+ assert "Unhandled error request_id=" in caplog.text
+ assert "type=RuntimeError" in caplog.text
+ assert "test-secret" not in caplog.text
+ assert_request_id(response)
+
+
+async def test_success_response_contains_request_id(client: AsyncClient):
+ response = await client.get("/health/live")
+
+ assert response.status_code == 200
+ assert response.headers.get("X-Request-Id")
diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py
new file mode 100644
index 0000000..9338c3b
--- /dev/null
+++ b/student-tests/test_failures.py
@@ -0,0 +1,138 @@
+"""Недоступность Redis и PostgreSQL: ограниченный 503, без анонимного доступа."""
+
+import time
+
+from app.main import create_app
+from conftest import close_app, make_client
+from helpers import PASSWORD, assert_request_id, basic_header, bearer, create_user
+
+ME = "/api/v1/users/me"
+DEAD_DATABASE_URL = "postgresql+asyncpg://nobody:secretpass@127.0.0.1:1/none"
+
+
+def assert_unavailable(response) -> None:
+ assert response.status_code == 503, response.text
+ assert response.json()["error"]["code"] == "unavailable"
+ assert "secretpass" not in response.text
+ assert "Traceback" not in response.text
+ assert_request_id(response)
+
+
+async def test_redis_down_gives_503_for_login_and_protected_endpoints(make_settings):
+ alive = create_app(make_settings())
+ async with make_client(alive) as client:
+ session = await create_user(client)
+ await close_app(alive)
+
+ app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5))
+ async with make_client(app, raise_app_exceptions=False) as client:
+ started = time.perf_counter()
+ protected = await client.get(ME, headers=bearer(session))
+ elapsed = time.perf_counter() - started
+
+ login = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(session["username"], session["password"]),
+ )
+ logout = await client.delete(
+ "/api/v1/auth/sessions/current", headers=bearer(session)
+ )
+ ready = await client.get("/health/ready")
+ live = await client.get("/health/live")
+
+ await close_app(app)
+
+ for response in (protected, login, logout, ready):
+ assert_unavailable(response)
+ assert live.status_code == 200
+ assert elapsed < 3
+
+
+async def test_redis_down_keeps_registration_working(make_settings):
+ app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5))
+ async with make_client(app, raise_app_exceptions=False) as client:
+ response = await client.post(
+ "/api/v1/users",
+ json={"username": "no_redis", "display_name": "N", "password": PASSWORD},
+ )
+ await close_app(app)
+
+ assert response.status_code == 201
+
+
+async def test_database_down_with_valid_session_gives_503(make_settings):
+ alive = create_app(make_settings())
+ async with make_client(alive) as client:
+ session = await create_user(client)
+ await close_app(alive)
+
+ app = create_app(
+ make_settings(database_url=DEAD_DATABASE_URL, db_connect_timeout_seconds=1)
+ )
+ async with make_client(app, raise_app_exceptions=False) as client:
+ protected = await client.get(ME, headers=bearer(session))
+ chats = await client.get("/api/v1/chats", headers=bearer(session))
+ login = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(session["username"], session["password"]),
+ )
+ await close_app(app)
+
+ for response in (protected, chats, login):
+ assert_unavailable(response)
+
+
+async def test_dependency_failure_never_authenticates_anonymous_requests(make_settings):
+ app = create_app(make_settings(redis_port=1, redis_connect_timeout_seconds=0.5))
+ async with make_client(app, raise_app_exceptions=False) as client:
+ no_token = await client.get(ME)
+ garbage = await client.get(ME, headers={"Authorization": "Bearer garbage"})
+ await close_app(app)
+
+ assert no_token.status_code == 401
+ assert_unavailable(garbage)
+
+
+async def test_statement_timeout_gives_503(make_settings, monkeypatch):
+ from sqlalchemy import text
+
+ import app.domain.users as users_domain
+
+ async def slow_lookup(session, _username):
+ await session.execute(text("SELECT pg_sleep(1)"))
+
+ app = create_app(make_settings(db_statement_timeout_ms=100))
+ monkeypatch.setattr(users_domain.users_storage, "get_user_by_username", slow_lookup)
+ async with make_client(app, raise_app_exceptions=False) as client:
+ response = await client.post(
+ "/api/v1/users",
+ json={"username": "slow_user", "display_name": "S", "password": PASSWORD},
+ )
+ await close_app(app)
+
+ assert_unavailable(response)
+ assert response.headers["Retry-After"] == "1"
+
+
+async def test_non_transient_database_error_stays_500(make_settings, monkeypatch):
+ from sqlalchemy import text
+
+ import app.domain.users as users_domain
+
+ async def broken_lookup(session, _username):
+ await session.execute(text("SELECT 1 / 0"))
+
+ app = create_app(make_settings())
+ monkeypatch.setattr(
+ users_domain.users_storage, "get_user_by_username", broken_lookup
+ )
+ async with make_client(app, raise_app_exceptions=False) as client:
+ response = await client.post(
+ "/api/v1/users",
+ json={"username": "broken_user", "display_name": "B", "password": PASSWORD},
+ )
+ await close_app(app)
+
+ assert response.status_code == 500
+ assert response.json()["error"]["code"] == "internal_error"
+ assert "division" not in response.text
diff --git a/student-tests/test_health.py b/student-tests/test_health.py
new file mode 100644
index 0000000..26d1589
--- /dev/null
+++ b/student-tests/test_health.py
@@ -0,0 +1,62 @@
+from httpx import AsyncClient
+
+from app.main import create_app
+from conftest import close_app, make_client
+from helpers import assert_request_id
+
+
+async def test_liveness(client: AsyncClient):
+ response = await client.get("/health/live")
+
+ assert response.status_code == 200
+ assert response.json() == {"status": "ok"}
+ assert_request_id(response)
+
+
+async def test_readiness(client: AsyncClient):
+ response = await client.get("/health/ready")
+
+ assert response.status_code == 200
+ assert response.json() == {"status": "ok"}
+ assert_request_id(response)
+
+
+async def test_readiness_is_503_when_database_is_down(make_settings):
+ app = create_app(
+ make_settings(
+ database_url="postgresql+asyncpg://nobody:nopass@127.0.0.1:1/none",
+ db_connect_timeout_seconds=1,
+ )
+ )
+ async with make_client(app) as client:
+ assert (await client.get("/health/live")).status_code == 200
+
+ ready = await client.get("/health/ready")
+ assert ready.status_code == 503
+ assert ready.json()["error"]["code"] == "unavailable"
+ assert_request_id(ready)
+ await close_app(app)
+
+
+async def test_business_request_is_503_when_database_is_down(make_settings):
+ app = create_app(
+ make_settings(
+ database_url="postgresql+asyncpg://nobody:secretpass@127.0.0.1:1/none",
+ db_connect_timeout_seconds=1,
+ )
+ )
+ async with make_client(app, raise_app_exceptions=False) as client:
+ response = await client.post(
+ "/api/v1/users",
+ json={
+ "username": "alice",
+ "display_name": "Alice",
+ "password": "Correct-horse-battery-1",
+ },
+ )
+ await close_app(app)
+
+ assert response.status_code == 503
+ assert response.json()["error"]["code"] == "unavailable"
+ assert "secretpass" not in response.text
+ assert_request_id(response)
diff --git a/student-tests/test_history_plan.py b/student-tests/test_history_plan.py
new file mode 100644
index 0000000..704c9cc
--- /dev/null
+++ b/student-tests/test_history_plan.py
@@ -0,0 +1,78 @@
+"""План запроса истории на >= 10 000 сообщений: индекс без Sort и полного скана."""
+
+import json
+from datetime import datetime
+from uuid import UUID
+
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncEngine
+
+from helpers import bearer, create_chat, create_user
+
+HISTORY_QUERY = """
+SELECT * FROM messages
+WHERE chat_id = :chat_id AND (created_at, id) > (:after_created_at, :after_id)
+ORDER BY created_at, id
+LIMIT 51
+"""
+
+
+def walk(node: dict) -> list[dict]:
+ nodes = [node]
+ for child in node.get("Plans", []):
+ nodes.extend(walk(child))
+ return nodes
+
+
+async def seed_messages(
+ engine: AsyncEngine, chat_id: str, sender_id: str, count: int
+) -> None:
+ async with engine.begin() as connection:
+ await connection.execute(
+ text(
+ """
+ INSERT INTO messages (id, chat_id, sender_id, text, created_at)
+ SELECT gen_random_uuid(), :chat_id, :sender_id, 'message ' || g,
+ now() - make_interval(secs => :count - g)
+ FROM generate_series(1, :count) AS g
+ """
+ ),
+ {"chat_id": chat_id, "sender_id": sender_id, "count": count},
+ )
+ await connection.execute(text("ANALYZE messages"))
+
+
+async def test_history_page_uses_composite_index(client, db_engine: AsyncEngine):
+ alice = await create_user(client)
+ target = await create_chat(client, alice, title="target")
+ noise = await create_chat(client, alice, title="noise")
+ await seed_messages(db_engine, target["id"], alice["id"], 10_000)
+ await seed_messages(db_engine, noise["id"], alice["id"], 10_000)
+
+ middle = (
+ await client.get(
+ f"/api/v1/chats/{target['id']}/messages?limit=100", headers=bearer(alice)
+ )
+ ).json()["items"][-1]
+
+ async with db_engine.connect() as connection:
+ raw_plan = (
+ await connection.execute(
+ text(f"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) {HISTORY_QUERY}"),
+ {
+ "chat_id": UUID(target["id"]),
+ "after_created_at": datetime.fromisoformat(middle["created_at"]),
+ "after_id": UUID(middle["id"]),
+ },
+ )
+ ).scalar_one()
+
+ plan = (raw_plan if isinstance(raw_plan, list) else json.loads(raw_plan))[0]["Plan"]
+ nodes = walk(plan)
+ node_types = {node["Node Type"] for node in nodes}
+ index_names = {node.get("Index Name") for node in nodes}
+
+ assert "ix_messages_chat_id_created_at_id" in index_names
+ assert "Sort" not in node_types
+ assert "Seq Scan" not in node_types
+ assert all(node.get("Actual Rows", 0) <= 51 for node in nodes)
diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py
new file mode 100644
index 0000000..7c3aa49
--- /dev/null
+++ b/student-tests/test_idempotency.py
@@ -0,0 +1,190 @@
+"""Повторы сообщений через приложения с общими хранилищами."""
+
+import asyncio
+from contextlib import AsyncExitStack
+
+import pytest
+from sqlalchemy import text
+
+from app.errors.exceptions import unavailable
+from app.main import create_app
+from app.storage import idempotency as key_storage
+from conftest import close_app, make_client
+from helpers import bearer, create_chat, create_user, idempotency_header, send_message
+
+
+@pytest.fixture
+async def replicas(request, make_settings):
+ overrides = getattr(request, "param", {})
+ apps = [
+ create_app(make_settings(instance_id=name, **overrides))
+ for name in ("api-a", "api-b")
+ ]
+ try:
+ async with AsyncExitStack() as stack:
+ clients = [
+ await stack.enter_async_context(make_client(app)) for app in apps
+ ]
+ yield clients
+ finally:
+ for app in apps:
+ await close_app(app)
+
+
+@pytest.mark.parametrize("key", [None, "", "a b", "a/b", "x" * 129])
+async def test_invalid_key_returns_400(client, key):
+ user = await create_user(client)
+ chat = await create_chat(client, user)
+ headers = bearer(user)
+ if key is not None:
+ headers["Idempotency-Key"] = key
+ response = await client.post(
+ f"/api/v1/chats/{chat['id']}/messages", headers=headers, json={"text": "hello"}
+ )
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+@pytest.mark.parametrize("key", ["a", "x" * 128, "key_1.2:3-4"])
+async def test_valid_key_boundaries(client, key):
+ user = await create_user(client)
+ chat = await create_chat(client, user)
+ assert (await send_message(client, user, chat, "hello", key)).status_code == 201
+
+
+async def test_replay_and_conflict_across_replicas(replicas, db_engine):
+ first, second = replicas
+ user = await create_user(first)
+ chat = await create_chat(first, user)
+ original = await send_message(first, user, chat, "Привет", "same-key")
+ repeated = await second.post(
+ f"/api/v1/chats/{chat['id']}/messages",
+ headers={
+ **bearer(user),
+ **idempotency_header("same-key"),
+ "Content-Type": "application/json",
+ },
+ content='{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442" }',
+ )
+ conflict = await send_message(second, user, chat, "другой текст", "same-key")
+ assert original.status_code == repeated.status_code == 201
+ assert original.json() == repeated.json()
+ assert original.headers["X-Instance-Id"] != repeated.headers["X-Instance-Id"]
+ assert conflict.status_code == 409
+ async with db_engine.connect() as connection:
+ assert (
+ await connection.execute(text("SELECT count(*) FROM messages"))
+ ).scalar_one() == 1
+ assert (
+ await connection.execute(
+ text("SELECT value FROM counters WHERE name='messages'")
+ )
+ ).scalar_one() == 1
+
+
+async def test_parallel_replays_create_one_message(replicas, db_engine):
+ user = await create_user(replicas[0])
+ chat = await create_chat(replicas[0], user)
+ responses = await asyncio.gather(
+ *(
+ send_message(replicas[index % 2], user, chat, "parallel", "one-key")
+ for index in range(24)
+ )
+ )
+ assert all(response.status_code == 201 for response in responses)
+ assert len({response.json()["id"] for response in responses}) == 1
+ assert {response.headers["X-Instance-Id"] for response in responses} == {
+ "api-a",
+ "api-b",
+ }
+ async with db_engine.connect() as connection:
+ assert (
+ await connection.execute(text("SELECT count(*) FROM messages"))
+ ).scalar_one() == 1
+ assert (
+ await connection.execute(text("SELECT count(*) FROM idempotency_keys"))
+ ).scalar_one() == 1
+
+
+async def test_key_scope_separates_users_and_chats(client):
+ alice = await create_user(client)
+ bob = await create_user(client)
+ shared = await create_chat(client, alice, bob)
+ other = await create_chat(client, alice)
+ responses = [
+ await send_message(client, alice, shared, "a", "shared-key"),
+ await send_message(client, bob, shared, "b", "shared-key"),
+ await send_message(client, alice, other, "c", "shared-key"),
+ ]
+ assert all(response.status_code == 201 for response in responses)
+ assert len({response.json()["id"] for response in responses}) == 3
+
+
+async def test_expired_key_can_be_reused_without_removing_message(client, db_engine):
+ user = await create_user(client)
+ chat = await create_chat(client, user)
+ first = await send_message(client, user, chat, "old", "expiring-key")
+ async with db_engine.begin() as connection:
+ await connection.execute(
+ text(
+ "UPDATE idempotency_keys SET created_at = clock_timestamp() - interval '25 hours'"
+ )
+ )
+ second = await send_message(client, user, chat, "new", "expiring-key")
+ assert first.status_code == second.status_code == 201
+ assert first.json()["id"] != second.json()["id"]
+ history = await client.get(
+ f"/api/v1/chats/{chat['id']}/messages", headers=bearer(user)
+ )
+ assert [item["text"] for item in history.json()["items"]] == ["old", "new"]
+
+
+async def test_failed_transaction_does_not_poison_key(client, db_engine, monkeypatch):
+ user = await create_user(client)
+ chat = await create_chat(client, user)
+ original = key_storage.save_result
+
+ async def fail_after_insert(*args):
+ await original(*args)
+ raise unavailable()
+
+ monkeypatch.setattr(key_storage, "save_result", fail_after_insert)
+ failed = await send_message(client, user, chat, "retry", "retry-key")
+ assert failed.status_code == 503
+ async with db_engine.connect() as connection:
+ for table in ("messages", "idempotency_keys", "counters"):
+ assert (
+ await connection.execute(text(f"SELECT count(*) FROM {table}"))
+ ).scalar_one() == 0
+ monkeypatch.setattr(key_storage, "save_result", original)
+ recovered = await send_message(client, user, chat, "retry", "retry-key")
+ assert recovered.status_code == 201
+
+
+@pytest.mark.parametrize("replicas", [{"max_messages": 1}], indirect=True)
+async def test_replay_works_when_message_limit_is_full(replicas):
+ user = await create_user(replicas[0])
+ chat = await create_chat(replicas[0], user)
+ first = await send_message(replicas[0], user, chat, "last", "last-key")
+ replay = await send_message(replicas[1], user, chat, "last", "last-key")
+ rejected = await send_message(replicas[1], user, chat, "extra", "other-key")
+ assert first.status_code == replay.status_code == 201
+ assert first.json() == replay.json()
+ assert rejected.status_code == 503
+
+
+async def test_replay_checks_session_and_membership(client, db_engine):
+ user = await create_user(client)
+ chat = await create_chat(client, user)
+ assert (
+ await send_message(client, user, chat, "private", "private-key")
+ ).status_code == 201
+ async with db_engine.begin() as connection:
+ await connection.execute(text("DELETE FROM chat_members"))
+ assert (
+ await send_message(client, user, chat, "private", "private-key")
+ ).status_code == 404
+ await client.delete("/api/v1/auth/sessions/current", headers=bearer(user))
+ assert (
+ await send_message(client, user, chat, "private", "private-key")
+ ).status_code == 401
diff --git a/student-tests/test_image_pipeline.py b/student-tests/test_image_pipeline.py
new file mode 100644
index 0000000..aaf0621
--- /dev/null
+++ b/student-tests/test_image_pipeline.py
@@ -0,0 +1,117 @@
+"""Реальные преобразования и OS-лимиты дочернего процесса worker."""
+
+import asyncio
+from io import BytesIO
+import os
+from pathlib import Path
+import subprocess
+import sys
+from uuid import uuid4
+
+from PIL import Image
+import pytest
+
+from app.core.image_processing import ImageSettings
+from app.domain.image_pipeline import create_variants
+from app.domain.job_files import render_variants
+from app.storage.jobs import JobWork
+from test_attachments import MemoryStorage
+
+
+@pytest.mark.parametrize("size,transparent", [((1600, 800), False), ((24, 12), True)])
+async def test_variant_geometry_metadata_and_alpha(tmp_path, size, transparent):
+ source = tmp_path / "source.png"
+ with Image.new("RGBA", size, (255, 0, 0, 0 if transparent else 255)) as image:
+ exif = Image.Exif()
+ exif[270] = "private description"
+ image.save(source, format="PNG", exif=exif)
+ result = create_variants(
+ source, tmp_path, "image/png", 40_000_000, ImageSettings(_env_file=None)
+ )
+ variants = {v.name: v for v in result.variants}
+ assert (variants["crop"].width, variants["crop"].height) == (128, 128)
+ assert (variants["thumbnail"].width, variants["thumbnail"].height) == (
+ (256, 128) if size[0] > 256 else size
+ )
+ assert (variants["optimized"].width, variants["optimized"].height) == (
+ (1280, 640) if size[0] > 1280 else size
+ )
+ for name in ("thumbnail.jpg", "crop.jpg", "optimized.webp"):
+ with Image.open(tmp_path / name) as image:
+ assert (
+ not image.getexif()
+ and not image.info.get("icc_profile")
+ and not image.info.get("xmp")
+ )
+ if transparent:
+ if name.endswith("webp"):
+ assert image.getpixel((0, 0))[3] == 0
+ else:
+ assert min(image.getpixel((0, 0))) >= 250
+
+
+async def test_exif_orientation_applied(tmp_path):
+ source = tmp_path / "source.jpg"
+ with Image.new("RGB", (400, 200), "blue") as image:
+ exif = Image.Exif()
+ exif[274] = 6
+ image.save(source, exif=exif)
+ result = create_variants(
+ source, tmp_path, "image/jpeg", 40_000_000, ImageSettings(_env_file=None)
+ )
+ assert (result.variants[0].width, result.variants[0].height) == (128, 256)
+
+
+@pytest.mark.parametrize("failure", ["memory", "cpu"])
+async def test_worker_os_limits(failure, tmp_path):
+ body = (
+ "return bytearray(2 * 1024 ** 3)" if failure == "memory" else "while True: pass"
+ )
+ code = (
+ "from app.core import thumbnail_worker as w\ndef fail(*args):\n "
+ + body
+ + "\nw.create_variants=fail\nraise SystemExit(w.main())"
+ )
+ result = await asyncio.to_thread(
+ subprocess.run,
+ [
+ sys.executable,
+ "-c",
+ code,
+ "source",
+ str(tmp_path),
+ str(tmp_path / "metadata"),
+ "image/png",
+ "40000000",
+ ],
+ env={
+ **os.environ,
+ "IMAGE_MEMORY_MIB": "256",
+ "IMAGE_CPU_SECONDS": "20" if failure == "memory" else "1",
+ },
+ capture_output=True,
+ timeout=15,
+ )
+ assert result.returncode == (3 if failure == "memory" else -9)
+ assert not (tmp_path / "metadata").exists()
+
+
+async def test_render_timeout_kills_child(monkeypatch, tmp_path):
+ create = asyncio.create_subprocess_exec
+ children = []
+
+ async def slow_child(*args, **kwargs):
+ child = await create(
+ sys.executable, "-c", "import time;time.sleep(60)", **kwargs
+ )
+ children.append(child)
+ return child
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", slow_child)
+ work = JobWork(
+ uuid4(), uuid4(), uuid4(), 1, "staging/test", "image/png", 1, "0" * 64
+ )
+ with pytest.raises(TimeoutError):
+ async with asyncio.timeout(0.2):
+ await render_variants(MemoryStorage(), work, tmp_path / "source", tmp_path)
+ assert len(children) == 1 and children[0].returncode is not None
diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py
new file mode 100644
index 0000000..1f26247
--- /dev/null
+++ b/student-tests/test_instance.py
@@ -0,0 +1,110 @@
+"""Идентификатор реплики и заголовки ответов без PostgreSQL, Redis и Docker."""
+
+from uuid import UUID
+
+import pytest
+from fastapi import Request
+
+from app.core.instance import resolve_instance_id
+from app.errors.exceptions import unavailable
+from app.main import create_app
+from conftest import close_app, make_client
+
+
+@pytest.fixture(autouse=True)
+def clean_db() -> None:
+ """Заменяет очистку БД из conftest: эти тесты не обращаются к хранилищам."""
+
+
+@pytest.mark.parametrize(
+ ("configured", "expected"),
+ [
+ ("api-explicit", "api-explicit"),
+ (" api-explicit ", "api-explicit"),
+ (None, "container-123"),
+ ("", "container-123"),
+ (" ", "container-123"),
+ ],
+)
+def test_resolve_prefers_configured_id_then_hostname(monkeypatch, configured, expected):
+ monkeypatch.setenv("HOSTNAME", " container-123 ")
+
+ assert resolve_instance_id(configured) == expected
+
+
+@pytest.mark.parametrize("hostname", [None, "", " "])
+def test_resolve_without_config_or_hostname_returns_uuid(monkeypatch, hostname):
+ if hostname is None:
+ monkeypatch.delenv("HOSTNAME", raising=False)
+ else:
+ monkeypatch.setenv("HOSTNAME", hostname)
+
+ assert UUID(resolve_instance_id(None)).version == 4
+
+
+@pytest.fixture
+async def instance_client(make_settings):
+ app = create_app(
+ make_settings(
+ instance_id="api-test",
+ database_url="postgresql+asyncpg://unused@127.0.0.1:1/unused",
+ redis_port=1,
+ )
+ )
+
+ @app.get("/__test__/request-id", include_in_schema=False)
+ async def request_id(request: Request) -> dict[str, str]:
+ return {"request_id": request.state.request_id}
+
+ @app.get("/__test__/explode", include_in_schema=False)
+ async def explode() -> None:
+ raise RuntimeError("test error")
+
+ @app.get("/__test__/unavailable", include_in_schema=False)
+ async def dependency_failure() -> None:
+ raise unavailable()
+
+ try:
+ async with make_client(app, raise_app_exceptions=False) as client:
+ yield client
+ finally:
+ await close_app(app)
+
+
+@pytest.mark.parametrize(
+ ("method", "path", "status"),
+ [
+ ("GET", "/health/live", 200),
+ ("POST", "/api/v1/users", 400),
+ ("GET", "/api/v1/users/me", 401),
+ ("GET", "/no-such-path", 404),
+ ("DELETE", "/health/live", 405),
+ ("GET", "/__test__/explode", 500),
+ ("GET", "/__test__/unavailable", 503),
+ ],
+)
+async def test_instance_header_is_present_on_success_and_errors(
+ instance_client, method, path, status
+):
+ response = await instance_client.request(method, path)
+
+ assert response.status_code == status
+ assert response.headers["X-Instance-Id"] == "api-test"
+ assert response.headers["X-Request-Id"]
+ if status >= 400:
+ assert (
+ response.json()["error"]["request_id"] == response.headers["X-Request-Id"]
+ )
+
+
+async def test_instance_id_is_stable_while_request_ids_are_distinct(instance_client):
+ first = await instance_client.get("/__test__/request-id")
+ second = await instance_client.get("/__test__/request-id")
+
+ assert first.status_code == second.status_code == 200
+ assert (
+ first.headers["X-Instance-Id"] == second.headers["X-Instance-Id"] == "api-test"
+ )
+ assert first.headers["X-Request-Id"] != second.headers["X-Request-Id"]
+ for response in (first, second):
+ assert response.json()["request_id"] == response.headers["X-Request-Id"]
diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py
new file mode 100644
index 0000000..3c6b5d6
--- /dev/null
+++ b/student-tests/test_jobs.py
@@ -0,0 +1,267 @@
+"""Retry/replay, fencing, cleanup, limits и транзакции реального PostgreSQL."""
+
+import asyncio
+from datetime import timedelta
+import errno
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from uuid import UUID, uuid4
+
+import pytest
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.core.jobs import JobSettings
+from app.domain import attachment_jobs
+from app.domain.attachment_jobs import (
+ claim_job,
+ process_attachment_job,
+ cleanup_staging,
+ build_result,
+)
+from app.domain.job_admin import replay_failed_job
+from app.domain.job_retry import classify_failure, retry_delay
+from app.errors.exceptions import JobOperationError, ProcessingError
+from app.models.attachment import Attachment
+from app.models.job import AttachmentJob
+from app.models.job_replay import JobReplay
+from app.schemas.job import JobMessage
+from app.storage import jobs
+from test_attachments import context, upload, finish_attachment
+
+
+async def envelope(session, attachment_id):
+ job = await session.scalar(
+ select(AttachmentJob).where(AttachmentJob.attachment_id == UUID(attachment_id))
+ )
+ return JobMessage(
+ job_id=job.id,
+ attachment_id=job.attachment_id,
+ pipeline_version=1,
+ attempt=job.attempt + 1,
+ created_at=job.created_at,
+ )
+
+
+async def test_retry_budget_backoff_replay_and_duplicate(client, context, db_engine):
+ storage, alice, _, _, chat = context
+ attachment = (await upload(client, alice, chat)).json()
+ settings = JobSettings(_env_file=None, job_max_attempts=3)
+
+ def fail(_):
+ raise TimeoutError
+
+ async with AsyncSession(db_engine, expire_on_commit=False) as session:
+ first = await envelope(session, attachment["id"])
+ for attempt in range(1, 4):
+ message = await envelope(session, attachment["id"])
+ assert await process_attachment_job(
+ session, storage, message, settings, before_processing=fail
+ )
+ job, item = await jobs.lock_job(session, first.job_id)
+ assert job.attempt == attempt
+ delay = (job.next_attempt_at - item.updated_at).total_seconds()
+ assert 5 * 2 ** (attempt - 1) / 2 <= delay <= 5 * 2 ** (attempt - 1)
+ assert item.status == ("failed" if attempt == 3 else "queued")
+ assert job.last_error_permanent is False
+ job.next_attempt_at = await jobs.database_time(session) - timedelta(
+ seconds=1
+ )
+ await session.commit()
+ assert item.error_code == "retry_exhausted"
+ request_id = uuid4()
+ result = await replay_failed_job(
+ session,
+ storage,
+ settings,
+ first.job_id,
+ request_id=request_id,
+ actor="test",
+ reason="dependency_restored",
+ )
+ assert result["attempt_limit"] == 6
+
+ assert await process_attachment_job(session, storage, first, settings)
+ job, item = await jobs.lock_job(session, first.job_id)
+ assert job.attempt == 3 and item.status == "queued"
+ await session.commit()
+ message = await envelope(session, attachment["id"])
+ assert await process_attachment_job(session, storage, message, settings)
+ before = dict(storage.objects)
+ assert await process_attachment_job(session, storage, message, settings)
+ assert storage.objects == before
+ repeat = await replay_failed_job(
+ session,
+ storage,
+ settings,
+ first.job_id,
+ request_id=request_id,
+ actor="test",
+ reason="dependency_restored",
+ )
+ assert repeat["already_applied"]
+ with pytest.raises(JobOperationError, match="job_not_failed"):
+ await replay_failed_job(
+ session,
+ storage,
+ settings,
+ first.job_id,
+ request_id=uuid4(),
+ actor="test",
+ reason="manual_review",
+ )
+ await session.rollback()
+ assert len((await session.scalars(select(JobReplay))).all()) == 1
+
+
+async def test_permanent_failure_and_missing_source_replay(client, context, db_engine):
+ storage, alice, _, _, chat = context
+ attachment = (await upload(client, alice, chat)).json()
+
+ def fail(_):
+ raise ProcessingError("image_limits")
+
+ settings = JobSettings(_env_file=None)
+ async with AsyncSession(db_engine, expire_on_commit=False) as session:
+ message = await envelope(session, attachment["id"])
+ await process_attachment_job(
+ session, storage, message, settings, before_processing=fail
+ )
+ job, item = await jobs.lock_job(session, message.job_id)
+ assert item.status == "failed" and job.attempt == 1 and job.last_error_permanent
+ job.staging_key = None
+ await session.commit()
+ with pytest.raises(JobOperationError, match="source_expired"):
+ await replay_failed_job(
+ session,
+ storage,
+ settings,
+ job.id,
+ request_id=uuid4(),
+ actor="test",
+ reason="manual_review",
+ )
+
+
+async def test_stale_worker_cannot_write_and_cleanup_keeps_active(
+ client, context, db_engine
+):
+ storage, alice, _, _, chat = context
+ attachment = (await upload(client, alice, chat)).json()
+ settings = JobSettings(_env_file=None)
+ async with AsyncSession(db_engine, expire_on_commit=False) as session:
+ message = await envelope(session, attachment["id"])
+ claim = await claim_job(session, message, settings)
+ job, item = await jobs.lock_job(session, message.job_id)
+ job.lease_token = uuid4()
+ job.finished_at = await jobs.database_time(session) - timedelta(days=10)
+ await session.commit()
+ assert await cleanup_staging(session, storage, settings, apply=True) == 0
+
+ before = dict(storage.objects)
+ assert await build_result(session, storage, claim.work)
+ assert storage.objects == before
+
+
+async def test_cleanup_ready_preserves_final_objects(client, context, db_engine):
+ storage, alice, _, _, chat = context
+ attachment = (await upload(client, alice, chat)).json()
+ await finish_attachment(client, storage, alice, attachment["id"])
+ async with AsyncSession(db_engine, expire_on_commit=False) as session:
+ message = await envelope(session, attachment["id"])
+ job, item = await jobs.lock_job(session, message.job_id)
+ staging = job.staging_key
+ job.finished_at = await jobs.database_time(session) - timedelta(days=2)
+ await session.commit()
+ assert (
+ await cleanup_staging(
+ session, storage, JobSettings(_env_file=None), apply=False
+ )
+ == 1
+ )
+ assert staging in storage.objects
+ assert (
+ await cleanup_staging(
+ session, storage, JobSettings(_env_file=None), apply=True
+ )
+ == 1
+ )
+ assert staging not in storage.objects and len(storage.objects) == 4
+ assert (
+ await cleanup_staging(
+ session, storage, JobSettings(_env_file=None), apply=True
+ )
+ == 0
+ )
+
+
+async def test_retry_classification_and_cap():
+ settings = JobSettings(_env_file=None)
+ for attempt in (1, 2, 6, 100):
+ cap = min(120, 5 * 2 ** min(30, attempt - 1))
+ assert all(cap / 2 <= retry_delay(settings, attempt) <= cap for _ in range(100))
+ assert classify_failure(TimeoutError()) == ("processing_timeout", False)
+ assert classify_failure(OSError(errno.EACCES, "denied"))[1]
+ assert not classify_failure(OSError(errno.ENOSPC, "full"))[1]
+
+
+async def test_timeout_records_retry_and_clears_lease(
+ client, context, db_engine, monkeypatch
+):
+ storage, alice, _, _, chat = context
+ attachment = (await upload(client, alice, chat)).json()
+
+ async def stalled(*args, **kwargs):
+ await asyncio.sleep(60)
+
+ monkeypatch.setattr(attachment_jobs, "build_result", stalled)
+ settings = JobSettings(_env_file=None).model_copy(
+ update={"job_timeout_seconds": 0.05}
+ )
+ async with AsyncSession(db_engine, expire_on_commit=False) as session:
+ message = await envelope(session, attachment["id"])
+ assert await process_attachment_job(session, storage, message, settings)
+ job, item = await jobs.lock_job(session, message.job_id)
+ assert item.status == "queued" and job.last_error_code == "processing_timeout"
+ assert job.lease_token is None and job.lease_until is None
+
+
+async def test_failed_retention_and_source_checksum_protect_replay(
+ client, context, db_engine
+):
+ storage, alice, _, _, chat = context
+ attachment = (await upload(client, alice, chat)).json()
+
+ def fail(_):
+ raise ProcessingError("invalid_image")
+
+ settings = JobSettings(_env_file=None)
+ async with AsyncSession(db_engine, expire_on_commit=False) as session:
+ message = await envelope(session, attachment["id"])
+ await process_attachment_job(
+ session, storage, message, settings, before_processing=fail
+ )
+ job, item = await jobs.lock_job(session, message.job_id)
+ key = job.staging_key
+ job.finished_at = await jobs.database_time(session) - timedelta(days=2)
+ await session.commit()
+ assert await cleanup_staging(session, storage, settings, apply=True) == 0
+ data, metadata = storage.objects[key]
+ storage.objects[key] = (b"x" * len(data), metadata)
+ with pytest.raises(JobOperationError, match="source_mismatch"):
+ await replay_failed_job(
+ session,
+ storage,
+ settings,
+ message.job_id,
+ request_id=uuid4(),
+ actor="test",
+ reason="manual_review",
+ )
+ await session.rollback()
+ assert not (await session.scalars(select(JobReplay))).all()
+ job, item = await jobs.lock_job(session, message.job_id)
+ job.finished_at = await jobs.database_time(session) - timedelta(days=8)
+ await session.commit()
+ assert await cleanup_staging(session, storage, settings, apply=True) == 1
+ assert key not in storage.objects
diff --git a/student-tests/test_lifecycle.py b/student-tests/test_lifecycle.py
new file mode 100644
index 0000000..11e4dcb
--- /dev/null
+++ b/student-tests/test_lifecycle.py
@@ -0,0 +1,50 @@
+"""Готовность при завершении и ограничение времени проверки зависимостей."""
+
+import asyncio
+import time
+
+import pytest
+
+from app.domain import health
+from app.main import create_app
+from conftest import close_app, make_client
+
+
+@pytest.fixture(autouse=True)
+def clean_db() -> None:
+ """Эти проверки не обращаются к хранилищам."""
+
+
+async def test_draining_keeps_liveness_but_rejects_new_work(make_settings):
+ app = create_app(make_settings())
+ app.state.lifecycle.start_draining()
+ try:
+ async with make_client(app) as client:
+ assert (await client.get("/health/live")).status_code == 200
+ assert (await client.get("/health/ready")).status_code == 503
+ assert (await client.get("/api/v1/users/me")).status_code == 503
+ finally:
+ await close_app(app)
+
+
+async def test_readiness_timeout_cancels_dependency_checks(make_settings, monkeypatch):
+ cancelled = []
+
+ async def stalled(_):
+ try:
+ await asyncio.sleep(30)
+ finally:
+ cancelled.append(True)
+
+ monkeypatch.setattr(health, "check_database", stalled)
+ monkeypatch.setattr(health, "check_redis", stalled)
+ app = create_app(make_settings(readiness_timeout_seconds=0.03))
+ try:
+ async with make_client(app) as client:
+ started = time.monotonic()
+ response = await client.get("/health/ready")
+ assert response.status_code == 503
+ assert time.monotonic() - started < 1
+ assert len(cancelled) == 2
+ finally:
+ await close_app(app)
diff --git a/student-tests/test_limits.py b/student-tests/test_limits.py
new file mode 100644
index 0000000..9cfca1d
--- /dev/null
+++ b/student-tests/test_limits.py
@@ -0,0 +1,122 @@
+"""Ограничения нагрузки: MAX_IN_FLIGHT -> 429, MAX_MESSAGES -> 503."""
+
+import asyncio
+
+import httpx
+
+from app.main import create_app
+from conftest import close_app, make_client
+from helpers import (
+ assert_request_id,
+ bearer,
+ create_chat,
+ create_user,
+ idempotency_header,
+)
+
+
+def _assert_error(response: httpx.Response, status: int, code: str) -> None:
+ assert response.status_code == status
+ body = response.json()
+ assert body["error"]["code"] == code
+ assert response.headers.get("X-Request-Id") == body["error"]["request_id"]
+
+
+async def test_message_limit_rejects_without_deleting_confirmed_messages(make_settings):
+ app = create_app(make_settings(max_messages=2))
+ async with make_client(app) as client:
+ user = await create_user(client)
+ chat = await create_chat(client, user)
+ url = f"/api/v1/chats/{chat['id']}/messages"
+
+ for index in range(2):
+ ok = await client.post(
+ url,
+ headers={**bearer(user), **idempotency_header()},
+ json={"text": f"confirmed-{index}"},
+ )
+ assert ok.status_code == 201
+
+ rejected = await client.post(
+ url,
+ headers={**bearer(user), **idempotency_header()},
+ json={"text": "rejected"},
+ )
+ _assert_error(rejected, 503, "unavailable")
+
+ history = await client.get(f"{url}?limit=100", headers=bearer(user))
+ assert history.status_code == 200
+ assert [m["text"] for m in history.json()["items"]] == [
+ "confirmed-0",
+ "confirmed-1",
+ ]
+ await close_app(app)
+
+
+async def test_in_flight_limit_returns_429_and_recovers(make_settings, monkeypatch):
+ import app.endpoints.messages as message_endpoint
+
+ app = create_app(make_settings(max_in_flight=2))
+
+ original_send_message = message_endpoint.send_message_idempotent
+ accepted_count = 0
+ all_slots_taken = asyncio.Event()
+ release = asyncio.Event()
+
+ async def blocking_send_message(*args, **kwargs):
+ nonlocal accepted_count
+ accepted_count += 1
+ if accepted_count == 2:
+ all_slots_taken.set()
+ await asyncio.wait_for(release.wait(), timeout=5)
+ return await original_send_message(*args, **kwargs)
+
+ async with make_client(app, timeout=10) as client:
+ user = await create_user(client)
+ chat = await create_chat(client, user)
+
+ monkeypatch.setattr(
+ message_endpoint, "send_message_idempotent", blocking_send_message
+ )
+ url = f"/api/v1/chats/{chat['id']}/messages"
+
+ held = [
+ asyncio.create_task(
+ client.post(
+ url,
+ headers={**bearer(user), **idempotency_header()},
+ json={"text": f"held-{i}"},
+ )
+ )
+ for i in range(2)
+ ]
+
+ await asyncio.wait_for(all_slots_taken.wait(), timeout=2)
+
+ rejected = await asyncio.gather(
+ *(
+ client.post(
+ url,
+ headers={**bearer(user), **idempotency_header()},
+ json={"text": f"rejected-{i}"},
+ )
+ for i in range(4)
+ )
+ )
+ for response in rejected:
+ _assert_error(response, 429, "rate_limited")
+ assert response.headers.get("Retry-After") == "1"
+
+ assert (await client.get("/health/live")).status_code == 200
+
+ release.set()
+ assert all(r.status_code == 201 for r in await asyncio.gather(*held))
+
+ recovered = await client.post(
+ url,
+ headers={**bearer(user), **idempotency_header()},
+ json={"text": "after"},
+ )
+ assert recovered.status_code == 201
+ assert_request_id(recovered)
+ await close_app(app)
diff --git a/student-tests/test_messages.py b/student-tests/test_messages.py
new file mode 100644
index 0000000..fecdac2
--- /dev/null
+++ b/student-tests/test_messages.py
@@ -0,0 +1,237 @@
+from urllib.parse import quote
+
+import pytest
+from httpx import AsyncClient
+
+from helpers import (
+ assert_request_id,
+ bearer,
+ create_chat,
+ create_user,
+ idempotency_header,
+ send_message,
+)
+
+
+async def _message(client: AsyncClient, user: dict, chat: dict, text: str):
+ return await send_message(client, user, chat, text)
+
+
+async def test_send_and_list_message(client: AsyncClient):
+ alice = await create_user(client)
+ bob = await create_user(client)
+ chat = await create_chat(client, alice, bob)
+
+ created = await _message(client, alice, chat, "Привет, мир 👋")
+
+ assert created.status_code == 201
+ message = created.json()
+ assert message["sender_id"] == alice["id"]
+ assert message["chat_id"] == chat["id"]
+ assert_request_id(created)
+
+ listed = await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=1",
+ headers=bearer(bob),
+ )
+
+ assert listed.status_code == 200
+ assert listed.json()["items"] == [message]
+ assert listed.json()["next_cursor"] is None
+ assert_request_id(listed)
+
+
+@pytest.mark.parametrize("text", ["", " ", "x" * 4001])
+async def test_invalid_message_text(client: AsyncClient, text: str):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+
+ response = await _message(client, alice, chat, text)
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+ assert_request_id(response)
+
+
+async def test_extra_message_field_is_rejected(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+
+ response = await client.post(
+ f"/api/v1/chats/{chat['id']}/messages",
+ headers={**bearer(alice), **idempotency_header()},
+ json={"text": "hello", "sender_id": alice["id"]},
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+ assert_request_id(response)
+
+
+async def test_messages_are_private_to_chat_members(client: AsyncClient):
+ alice = await create_user(client)
+ eve = await create_user(client)
+ chat = await create_chat(client, alice)
+ await _message(client, alice, chat, "секрет")
+
+ response = await client.get(
+ f"/api/v1/chats/{chat['id']}/messages",
+ headers=bearer(eve),
+ )
+
+ assert response.status_code == 404
+ assert response.json()["error"]["code"] == "not_found"
+
+
+async def test_pagination_has_no_duplicates(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+
+ created = [
+ (await _message(client, alice, chat, f"message-{index}")).json()
+ for index in range(5)
+ ]
+
+ page1 = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=2",
+ headers=bearer(alice),
+ )
+ ).json()
+ page2 = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=2&cursor={quote(page1['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+ ).json()
+ page3 = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=2&cursor={quote(page2['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+ ).json()
+
+ ids = [item["id"] for page in (page1, page2, page3) for item in page["items"]]
+ assert ids == [item["id"] for item in created]
+ assert len(ids) == len(set(ids)) == 5
+ assert len(page1["items"]) == 2
+ assert len(page2["items"]) == 2
+ assert len(page3["items"]) == 1
+ assert page3["next_cursor"] is None
+
+
+async def test_cursor_cannot_be_used_for_another_chat(client: AsyncClient):
+ alice = await create_user(client)
+ chat_a = await create_chat(client, alice, title="A")
+ chat_b = await create_chat(client, alice, title="B")
+
+ await _message(client, alice, chat_a, "a1")
+ await _message(client, alice, chat_a, "a2")
+ await _message(client, alice, chat_b, "b1")
+ await _message(client, alice, chat_b, "b2")
+
+ page = (
+ await client.get(
+ f"/api/v1/chats/{chat_a['id']}/messages?limit=1",
+ headers=bearer(alice),
+ )
+ ).json()
+
+ response = await client.get(
+ f"/api/v1/chats/{chat_b['id']}/messages?limit=1&cursor={quote(page['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+@pytest.mark.parametrize(
+ "limit, expected_status", [(0, 400), (1, 200), (100, 200), (101, 400)]
+)
+async def test_limit_boundaries(client: AsyncClient, limit: int, expected_status: int):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+ await _message(client, alice, chat, "one")
+
+ response = await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit={limit}",
+ headers=bearer(alice),
+ )
+
+ assert response.status_code == expected_status
+ assert_request_id(response)
+
+
+async def test_message_exact_max_length_is_accepted(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+
+ response = await _message(client, alice, chat, "x" * 4000)
+
+ assert response.status_code == 201
+ assert len(response.json()["text"]) == 4000
+
+
+async def test_nonexistent_chat_messages_return_404(client: AsyncClient):
+ alice = await create_user(client)
+
+ response = await client.get(
+ "/api/v1/chats/00000000-0000-4000-8000-000000000000/messages",
+ headers=bearer(alice),
+ )
+
+ assert response.status_code == 404
+ assert response.json()["error"]["code"] == "not_found"
+ assert_request_id(response)
+
+
+async def test_unauthenticated_requests_are_rejected(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+ url = f"/api/v1/chats/{chat['id']}/messages"
+
+ assert (await client.get(url)).status_code == 401
+ assert (await client.post(url, json={"text": "hi"})).status_code == 401
+
+
+async def test_messages_survive_new_login(client: AsyncClient):
+ from helpers import login
+
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+ message = (await _message(client, alice, chat, "до выхода")).json()
+ await client.delete("/api/v1/auth/sessions/current", headers=bearer(alice))
+
+ alice_again = await login(client, alice)
+ listed = await client.get(
+ f"/api/v1/chats/{chat['id']}/messages", headers=bearer(alice_again)
+ )
+
+ assert listed.json()["items"] == [message]
+
+
+async def test_history_order_is_created_at_then_id(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+ created = [(await _message(client, alice, chat, f"m{i}")).json() for i in range(10)]
+
+ listed = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=100", headers=bearer(alice)
+ )
+ ).json()["items"]
+
+ assert [m["id"] for m in listed] == [m["id"] for m in created]
+ keys = [(m["created_at"], m["id"]) for m in listed]
+ assert keys == sorted(keys)
+
+
+async def test_nul_in_message_text_is_rejected(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+
+ response = await _message(client, alice, chat, "hello\u0000world")
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
diff --git a/student-tests/test_migrations.py b/student-tests/test_migrations.py
new file mode 100644
index 0000000..c515ade
--- /dev/null
+++ b/student-tests/test_migrations.py
@@ -0,0 +1,79 @@
+"""Миграции: применение с нуля, идемпотентный повтор, совпадение со схемой моделей."""
+
+import pytest
+from alembic.autogenerate import compare_metadata
+from alembic.migration import MigrationContext
+from sqlalchemy import inspect, text
+
+from app.models import Base
+from conftest import reset_schema, upgrade_head
+
+EXPECTED_TABLES = {
+ "users",
+ "chats",
+ "chat_members",
+ "messages",
+ "counters",
+ "idempotency_keys",
+ "attachments",
+ "message_attachments",
+ "upload_intents",
+ "attachment_jobs",
+ "job_replays",
+}
+
+
+@pytest.fixture
+async def migrated_engine(database_url, db_engine):
+ await reset_schema(db_engine)
+ await upgrade_head(database_url)
+ yield db_engine
+
+ await reset_schema(db_engine)
+ async with db_engine.begin() as connection:
+ await connection.run_sync(Base.metadata.create_all)
+
+
+async def test_upgrade_creates_all_tables(migrated_engine):
+ async with migrated_engine.connect() as connection:
+ tables = await connection.run_sync(
+ lambda sync: set(inspect(sync).get_table_names())
+ )
+ assert EXPECTED_TABLES <= tables
+
+
+async def test_upgrade_twice_is_idempotent_and_keeps_data(
+ migrated_engine, database_url
+):
+ async with migrated_engine.begin() as connection:
+ await connection.execute(
+ text(
+ "INSERT INTO users (id, username, display_name, password_hash) "
+ "VALUES (gen_random_uuid(), 'keep_me', 'Keep', 'hash')"
+ )
+ )
+
+ await upgrade_head(database_url)
+
+ async with migrated_engine.connect() as connection:
+ count = (
+ await connection.execute(
+ text("SELECT count(*) FROM users WHERE username = 'keep_me'")
+ )
+ ).scalar_one()
+ assert count == 1
+
+
+async def test_models_and_migrations_are_in_sync(migrated_engine):
+ def compare(sync_connection):
+ context = MigrationContext.configure(
+ sync_connection, opts={"compare_type": True}
+ )
+ return compare_metadata(context, Base.metadata)
+
+ async with migrated_engine.connect() as connection:
+ diff = await connection.run_sync(compare)
+
+ assert (
+ diff == []
+ ), f"Создайте миграцию: alembic revision --autogenerate. Diff: {diff}"
diff --git a/student-tests/test_pagination.py b/student-tests/test_pagination.py
new file mode 100644
index 0000000..e45e5b1
--- /dev/null
+++ b/student-tests/test_pagination.py
@@ -0,0 +1,137 @@
+"""Курсоры: непрозрачность, привязка к ресурсу и устойчивость к подделке."""
+
+import base64
+import json
+from urllib.parse import quote
+
+import pytest
+from httpx import AsyncClient
+
+from helpers import bearer, create_chat, create_user, send_message
+
+
+def forged_cursor(**payload: str) -> str:
+ return base64.urlsafe_b64encode(json.dumps(payload).encode()).decode()
+
+
+@pytest.mark.parametrize(
+ "cursor",
+ [
+ "not-base64-%%%",
+ base64.urlsafe_b64encode(b"not json").decode(),
+ base64.urlsafe_b64encode(b"[1, 2, 3]").decode(),
+ forged_cursor(kind="messages"),
+ ],
+)
+async def test_garbage_cursor_returns_400(client: AsyncClient, cursor: str):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+
+ for url in (
+ f"/api/v1/chats?cursor={quote(cursor, safe='')}",
+ f"/api/v1/chats/{chat['id']}/messages?cursor={quote(cursor, safe='')}",
+ ):
+ response = await client.get(url, headers=bearer(alice))
+ assert response.status_code == 400, url
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+async def test_cursor_without_timezone_is_rejected(client: AsyncClient):
+ alice = await create_user(client)
+ cursor = forged_cursor(
+ kind="chats",
+ resource_id=alice["id"],
+ created_at="2026-01-01T00:00:00",
+ item_id="00000000-0000-4000-8000-000000000000",
+ )
+
+ response = await client.get(
+ f"/api/v1/chats?cursor={quote(cursor, safe='')}", headers=bearer(alice)
+ )
+
+ assert response.status_code == 400
+
+
+async def test_chat_list_cursor_is_bound_to_its_owner(client: AsyncClient):
+ alice = await create_user(client)
+ bob = await create_user(client)
+ for index in range(2):
+ await create_chat(client, alice, bob, title=f"shared-{index}")
+
+ page = (await client.get("/api/v1/chats?limit=1", headers=bearer(alice))).json()
+ response = await client.get(
+ f"/api/v1/chats?limit=1&cursor={quote(page['next_cursor'], safe='')}",
+ headers=bearer(bob),
+ )
+
+ assert response.status_code == 400
+
+
+async def test_messages_cursor_is_not_accepted_by_chat_list(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+ for index in range(2):
+ await send_message(client, alice, chat, f"m{index}")
+
+ page = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice)
+ )
+ ).json()
+ response = await client.get(
+ f"/api/v1/chats?cursor={quote(page['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+
+ assert response.status_code == 400
+
+
+async def test_limit_can_change_between_pages(client: AsyncClient):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+ sent = [
+ (await send_message(client, alice, chat, f"m{index}")).json()
+ for index in range(6)
+ ]
+
+ first = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice)
+ )
+ ).json()
+ rest = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=100"
+ f"&cursor={quote(first['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+ ).json()
+
+ assert [m["id"] for m in first["items"] + rest["items"]] == [m["id"] for m in sent]
+ assert rest["next_cursor"] is None
+
+
+async def test_message_sent_after_reading_last_page_is_seen_with_cursor(
+ client: AsyncClient,
+):
+ alice = await create_user(client)
+ chat = await create_chat(client, alice)
+ await send_message(client, alice, chat, "first")
+ await send_message(client, alice, chat, "second")
+
+ page = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=1", headers=bearer(alice)
+ )
+ ).json()
+ new = (await send_message(client, alice, chat, "third")).json()
+ rest = (
+ await client.get(
+ f"/api/v1/chats/{chat['id']}/messages?limit=100"
+ f"&cursor={quote(page['next_cursor'], safe='')}",
+ headers=bearer(alice),
+ )
+ ).json()
+
+ assert [m["text"] for m in rest["items"]] == ["second", "third"]
+ assert rest["items"][-1] == new
diff --git a/student-tests/test_s3_stream.py b/student-tests/test_s3_stream.py
new file mode 100644
index 0000000..67fc364
--- /dev/null
+++ b/student-tests/test_s3_stream.py
@@ -0,0 +1,63 @@
+"""Освобождение HTTP-соединения и порядок завершения SDK при отмене."""
+
+import asyncio
+from threading import Event
+from types import SimpleNamespace
+
+from app.core.s3 import S3Settings
+from app.storage.s3 import ObjectStream, run_sync
+
+
+async def test_sdk_thread_finishes_before_cancellation_returns():
+ entered, release, finished = Event(), Event(), Event()
+
+ def operation():
+ entered.set()
+ release.wait(timeout=3)
+ finished.set()
+
+ task = asyncio.create_task(run_sync(operation))
+ await asyncio.to_thread(entered.wait, 2)
+ task.cancel()
+ await asyncio.sleep(0)
+ assert not task.done()
+ release.set()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ assert finished.is_set()
+
+
+async def test_response_discarded_when_get_is_cancelled():
+ entered, release = Event(), Event()
+ discarded = []
+ response = object()
+
+ def operation():
+ entered.set()
+ release.wait(timeout=3)
+ return response
+
+ task = asyncio.create_task(run_sync(operation, discard_result=discarded.append))
+ await asyncio.to_thread(entered.wait, 2)
+ task.cancel()
+ release.set()
+ try:
+ await task
+ except asyncio.CancelledError:
+ pass
+ assert discarded == [response]
+
+
+async def test_stream_closes_even_before_first_iteration():
+ calls = []
+ response = SimpleNamespace(
+ close=lambda: calls.append("close"),
+ release_conn=lambda: calls.append("release"),
+ )
+ storage = SimpleNamespace(settings=S3Settings(_env_file=None))
+ stream = ObjectStream(storage, response)
+ await stream.aclose()
+ await stream.aclose()
+ assert calls == ["close", "release"]
diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py
new file mode 100644
index 0000000..e239e22
--- /dev/null
+++ b/student-tests/test_sessions.py
@@ -0,0 +1,173 @@
+"""Сессии в Redis: TTL, абсолютный срок, отзыв, потеря и перезапуск хранилища."""
+
+import asyncio
+import hashlib
+import os
+import time
+from datetime import datetime
+
+import pytest
+from httpx import AsyncClient
+from redis.asyncio import Redis
+
+from app.main import create_app
+from conftest import close_app, make_client
+from helpers import basic_header, bearer, create_user, register
+
+ME = "/api/v1/users/me"
+LOGOUT = "/api/v1/auth/sessions/current"
+
+
+def session_key(token: str) -> str:
+ return "session:" + hashlib.sha256(token.encode()).hexdigest()
+
+
+async def test_session_key_ttl_matches_expires_at(
+ client: AsyncClient, redis_client: Redis
+):
+ user = await register(client)
+ response = await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], user["password"]),
+ )
+ body = response.json()
+
+ expires_at = datetime.fromisoformat(body["expires_at"])
+ expire_time_ms = await redis_client.pexpiretime(session_key(body["token"]))
+
+ assert abs(expire_time_ms - expires_at.timestamp() * 1000) < 2
+ assert 299_000 < await redis_client.pttl(session_key(body["token"])) <= 300_000
+
+
+async def test_session_ttl_is_absolute_and_not_extended_by_activity(
+ client: AsyncClient, redis_client: Redis
+):
+ session = await create_user(client)
+ key = session_key(session["token"])
+ expire_before = await redis_client.pexpiretime(key)
+
+ for _ in range(5):
+ assert (await client.get(ME, headers=bearer(session))).status_code == 200
+
+ assert await redis_client.pexpiretime(key) == expire_before
+
+
+async def test_logout_removes_session_key(client: AsyncClient, redis_client: Redis):
+ session = await create_user(client)
+ key = session_key(session["token"])
+ assert await redis_client.exists(key) == 1
+
+ assert (await client.delete(LOGOUT, headers=bearer(session))).status_code == 204
+
+ assert await redis_client.exists(key) == 0
+ assert (await client.get(ME, headers=bearer(session))).status_code == 401
+
+
+async def test_expired_session_is_rejected_everywhere(make_settings):
+ app = create_app(make_settings(session_ttl_seconds=1))
+ async with make_client(app) as client:
+ session = await create_user(client)
+ assert (await client.get(ME, headers=bearer(session))).status_code == 200
+
+ await asyncio.sleep(1.5)
+
+ for method, url in (("GET", ME), ("GET", "/api/v1/chats"), ("DELETE", LOGOUT)):
+ response = await client.request(method, url, headers=bearer(session))
+ assert response.status_code == 401, (method, url)
+ assert response.headers["WWW-Authenticate"].startswith("Bearer")
+ await close_app(app)
+
+
+async def test_relogin_after_expiry_issues_new_working_token(make_settings):
+ app = create_app(make_settings(session_ttl_seconds=1))
+ async with make_client(app) as client:
+ user = await register(client)
+ first = (
+ await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], user["password"]),
+ )
+ ).json()
+ await asyncio.sleep(1.5)
+ second = (
+ await client.post(
+ "/api/v1/auth/sessions",
+ headers=basic_header(user["username"], user["password"]),
+ )
+ ).json()
+
+ assert first["token"] != second["token"]
+ assert (await client.get(ME, headers=bearer(first))).status_code == 401
+ assert (await client.get(ME, headers=bearer(second))).status_code == 200
+ await close_app(app)
+
+
+async def test_tampered_token_is_rejected(client: AsyncClient):
+ session = await create_user(client)
+ token = session["token"]
+ tampered = token[:-1] + ("A" if token[-1] != "A" else "B")
+
+ response = await client.get(ME, headers={"Authorization": f"Bearer {tampered}"})
+
+ assert response.status_code == 401
+
+
+async def test_session_of_deleted_user_is_rejected(client: AsyncClient, db_engine):
+ from sqlalchemy import text
+
+ session = await create_user(client)
+ async with db_engine.begin() as connection:
+ await connection.execute(
+ text("DELETE FROM users WHERE id = :id"), {"id": session["id"]}
+ )
+
+ response = await client.get(ME, headers=bearer(session))
+
+ assert response.status_code == 401
+
+
+async def test_redis_flush_invalidates_all_sessions(
+ client: AsyncClient, redis_client: Redis
+):
+ alice = await create_user(client)
+ bob = await create_user(client)
+
+ await redis_client.flushdb()
+
+ for session in (alice, bob):
+ assert (await client.get(ME, headers=bearer(session))).status_code == 401
+
+
+@pytest.mark.skipif(
+ os.environ.get("DOCKER_TESTS") != "1",
+ reason="перезапускает контейнер redis из compose; включается DOCKER_TESTS=1",
+)
+async def test_sessions_survive_redis_restart(client: AsyncClient, redis_client: Redis):
+ active = await create_user(client)
+ revoked = await create_user(client)
+ assert (await client.delete(LOGOUT, headers=bearer(revoked))).status_code == 204
+
+ process = await asyncio.create_subprocess_exec(
+ "docker",
+ "compose",
+ "restart",
+ "redis",
+ stdout=asyncio.subprocess.DEVNULL,
+ stderr=asyncio.subprocess.DEVNULL,
+ )
+ assert await process.wait() == 0
+
+ deadline = time.monotonic() + 30
+ while True:
+ try:
+ await redis_client.ping()
+ break
+ except Exception:
+ if time.monotonic() > deadline:
+ raise
+ await asyncio.sleep(0.5)
+
+ response = await client.get(ME, headers=bearer(active))
+ assert response.status_code == 200
+ assert response.json()["id"] == active["id"]
+ assert (await client.get(ME, headers=bearer(revoked))).status_code == 401
diff --git a/student-tests/test_telemetry.py b/student-tests/test_telemetry.py
new file mode 100644
index 0000000..495be65
--- /dev/null
+++ b/student-tests/test_telemetry.py
@@ -0,0 +1,176 @@
+"""Проверки приватности и исходов на реальных SDK exporters в памяти."""
+
+import json
+
+from fastapi import FastAPI
+import httpx
+from opentelemetry.sdk._logs.export import InMemoryLogRecordExporter
+from opentelemetry.sdk.metrics.export import InMemoryMetricReader
+from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+from opentelemetry.trace import SpanKind
+import pytest
+
+from app.core import telemetry as module
+
+
+@pytest.fixture(autouse=True)
+async def clean_db():
+ # Этот набор не использует БД, Redis и фикстуры приложения.
+ yield
+
+
+@pytest.fixture
+def telemetry(monkeypatch):
+ spans = InMemorySpanExporter()
+ logs = InMemoryLogRecordExporter()
+ metrics = InMemoryMetricReader()
+ monkeypatch.setattr(module, "OTLPSpanExporter", lambda **_: spans)
+ monkeypatch.setattr(module, "OTLPLogExporter", lambda **_: logs)
+ monkeypatch.setattr(
+ module, "PeriodicExportingMetricReader", lambda *_, **__: metrics
+ )
+ runtime = module.Telemetry("messenger-api", "unit", enabled=True)
+ yield runtime, spans, logs, metrics
+ runtime.shutdown()
+
+
+async def test_http_template_correlation_and_private_data(telemetry):
+ runtime, spans, logs, metrics = telemetry
+ app = FastAPI()
+
+ @app.post("/items/{item_id}")
+ async def item(item_id: str):
+ return {"ok": True}
+
+ app.add_middleware(module.HttpTelemetryMiddleware, telemetry=runtime)
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app), base_url="http://test"
+ ) as client:
+ # ASGI caller может иметь свой активный span, не относящийся к входящему HTTP.
+ with runtime.tracer.start_as_current_span("unrelated-task"):
+ response = await client.post(
+ "/items/private-id?token=private-query",
+ json={"text": "private-body"},
+ headers={"Authorization": "Bearer private-token"},
+ )
+ await client.get("/arbitrary-private-path")
+ runtime.traces.force_flush()
+ runtime.logs.force_flush()
+ finished = spans.get_finished_spans()
+ assert finished[0].kind == SpanKind.SERVER
+ assert finished[0].name == "POST /items/{item_id}"
+ assert finished[1].attributes["http.route"] == "unmatched"
+ assert finished[0].context.trace_id != finished[1].context.trace_id
+ assert all(span.parent is None for span in finished[:2])
+ body = json.loads(logs.get_finished_logs()[0].log_record.body)
+ assert body["trace_id"] == response.headers["X-Trace-Id"]
+ assert int(body["span_id"], 16) == finished[0].context.span_id
+ data = metrics.get_metrics_data()
+ all_attributes = [
+ dict(point.attributes)
+ for resource in data.resource_metrics
+ for scope in resource.scope_metrics
+ for metric in scope.metrics
+ for point in metric.data.data_points
+ ]
+ assert {
+ "http.request.method": "POST",
+ "http.route": "/items/{item_id}",
+ "http.response.status_class": "2xx",
+ } in all_attributes
+ exported = (
+ repr([(s.name, dict(s.attributes)) for s in finished])
+ + repr(all_attributes)
+ + repr([str(log.log_record.body) for log in logs.get_finished_logs()])
+ )
+ for private in (
+ "private-id",
+ "private-query",
+ "private-body",
+ "private-token",
+ "arbitrary-private-path",
+ ):
+ assert private not in exported
+
+
+async def test_unhandled_error_preserves_correlation_without_exception_payload(
+ telemetry,
+):
+ from app.core.request_id import RequestIdMiddleware
+ from app.errors.handlers import register_exception_handlers
+
+ runtime, spans, logs, _ = telemetry
+ app = FastAPI()
+
+ @app.get("/explode")
+ async def explode():
+ raise RuntimeError("private-error-with-credentials")
+
+ app.add_middleware(RequestIdMiddleware)
+ app.add_middleware(module.HttpTelemetryMiddleware, telemetry=runtime)
+ register_exception_handlers(app)
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app, raise_app_exceptions=False),
+ base_url="http://test",
+ ) as client:
+ response = await client.get("/explode")
+ runtime.traces.force_flush()
+ runtime.logs.force_flush()
+ record = json.loads(logs.get_finished_logs()[0].log_record.body)
+ assert response.status_code == record["status_code"] == 500
+ assert response.headers["X-Request-Id"] == record["request_id"]
+ assert spans.get_finished_spans()[0].status.status_code.name == "ERROR"
+ assert "private-error-with-credentials" not in repr(record)
+
+
+def test_job_success_and_error_record_real_outcomes(telemetry):
+ runtime, spans, logs, metrics = telemetry
+ for fail in (False, True):
+ try:
+ with runtime.tracer.start_as_current_span(
+ "attachment.process.v1",
+ kind=SpanKind.CONSUMER,
+ record_exception=False,
+ set_status_on_exception=False,
+ ):
+ with runtime.job_attempt():
+ if fail:
+ raise RuntimeError("private-exception-payload")
+ except RuntimeError:
+ pass
+ runtime.traces.force_flush()
+ runtime.logs.force_flush()
+ finished = spans.get_finished_spans()
+ assert [s.attributes["outcome"] for s in finished] == ["success", "error"]
+ assert finished[1].status.status_code.name == "ERROR"
+ assert not finished[1].events
+ records = [json.loads(item.log_record.body) for item in logs.get_finished_logs()]
+ assert [record["outcome"] for record in records] == ["success", "error"]
+ assert "private-exception-payload" not in repr(records)
+ data = metrics.get_metrics_data()
+ counters = [
+ point
+ for resource in data.resource_metrics
+ for scope in resource.scope_metrics
+ for metric in scope.metrics
+ if metric.name == "messenger.job.attempts"
+ for point in metric.data.data_points
+ ]
+ assert {p.attributes["outcome"]: p.value for p in counters} == {
+ "success": 1,
+ "error": 1,
+ }
+
+
+def test_disabled_telemetry_has_no_exporters():
+ runtime = module.Telemetry("messenger-api", enabled=False)
+ runtime.event("ignored")
+ runtime.shutdown()
+ assert not hasattr(runtime, "metrics")
+
+
+def test_tempo_base64_and_hex_identifiers_match():
+ from scripts.check_observability import identifier_hex
+
+ assert identifier_hex("3X/51BqmrXA=", 8) == "dd7ff9d41aa6ad70"
+ assert identifier_hex("dd7ff9d41aa6ad70", 8) == "dd7ff9d41aa6ad70"
diff --git a/student-tests/test_users.py b/student-tests/test_users.py
new file mode 100644
index 0000000..554cf30
--- /dev/null
+++ b/student-tests/test_users.py
@@ -0,0 +1,128 @@
+import pytest
+from httpx import AsyncClient
+
+from helpers import PASSWORD, assert_request_id, bearer, create_user, register
+
+
+async def test_register_does_not_leak_password(client: AsyncClient):
+ response = await client.post(
+ "/api/v1/users",
+ json={"username": "alice", "display_name": "Алиса 👋", "password": PASSWORD},
+ )
+
+ assert response.status_code == 201
+ body = response.json()
+ assert set(body) == {"id", "username", "display_name", "created_at"}
+ assert PASSWORD not in response.text
+ assert_request_id(response)
+
+
+async def test_get_current_user(client: AsyncClient):
+ user = await create_user(client, display_name="Алиса 👋")
+
+ response = await client.get("/api/v1/users/me", headers=bearer(user))
+
+ assert response.status_code == 200
+ assert response.json()["id"] == user["id"]
+ assert response.json()["username"] == user["username"]
+ assert_request_id(response)
+
+
+@pytest.mark.parametrize(
+ "body",
+ [
+ {"username": "alice", "display_name": "", "password": PASSWORD},
+ {"username": "alice", "display_name": " ", "password": PASSWORD},
+ {"username": "alice", "display_name": "a" * 101, "password": PASSWORD},
+ {"username": "ab", "display_name": "Alice", "password": PASSWORD},
+ {"username": "alice-123", "display_name": "Alice", "password": PASSWORD},
+ {"username": "AlicE", "display_name": "Alice", "password": PASSWORD},
+ {"username": "a" * 33, "display_name": "Alice", "password": PASSWORD},
+ {"username": "alice", "display_name": "Alice"},
+ {"username": "alice", "display_name": "Alice", "password": "x" * 11},
+ {"username": "alice", "display_name": "Alice", "password": "x" * 129},
+ ],
+)
+async def test_invalid_user_payloads(client: AsyncClient, body: dict):
+ response = await client.post("/api/v1/users", json=body)
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+ assert_request_id(response)
+
+
+async def test_password_length_boundaries_are_accepted(client: AsyncClient):
+ for index, password in enumerate(
+ ["x" * 12, "y" * 128, "п" * 12, " spaces inside "]
+ ):
+ response = await client.post(
+ "/api/v1/users",
+ json={
+ "username": f"pw_user_{index}",
+ "display_name": "Alice",
+ "password": password,
+ },
+ )
+ assert response.status_code == 201, (index, response.text)
+
+
+async def test_extra_user_field_is_rejected(client: AsyncClient):
+ response = await client.post(
+ "/api/v1/users",
+ json={
+ "username": "alice",
+ "display_name": "Alice",
+ "password": PASSWORD,
+ "extra": True,
+ },
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+
+
+async def test_duplicate_username_returns_conflict(client: AsyncClient):
+ user = await register(client, username="duplicate_user")
+
+ response = await client.post(
+ "/api/v1/users",
+ json={
+ "username": user["username"],
+ "display_name": "Другой",
+ "password": PASSWORD,
+ },
+ )
+
+ assert response.status_code == 409
+ assert response.json()["error"]["code"] == "conflict"
+ assert_request_id(response)
+
+
+async def test_user_exact_length_boundaries_are_accepted(client: AsyncClient):
+ for length in (3, 32):
+ user = await register(client, username=("a" * length), display_name="d" * 100)
+ assert len(user["username"]) == length
+ assert len(user["display_name"]) == 100
+
+
+@pytest.mark.parametrize("password", [" " * 12, "\t" * 12, " \n" * 6])
+async def test_whitespace_only_password_is_rejected(client: AsyncClient, password: str):
+ response = await client.post(
+ "/api/v1/users",
+ json={"username": "blank_pw", "display_name": "Blank", "password": password},
+ )
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
+ assert_request_id(response)
+
+
+@pytest.mark.parametrize("field", ["display_name", "password"])
+async def test_nul_in_user_fields_is_rejected(client: AsyncClient, field: str):
+ body = {"username": "nul_user", "display_name": "Nul", "password": PASSWORD}
+ body[field] = body[field] + "\u0000tail"
+
+ response = await client.post("/api/v1/users", json=body)
+
+ assert response.status_code == 400
+ assert response.json()["error"]["code"] == "invalid_request"
diff --git a/tests/README.md b/tests/README.md
index 0dfa3f8..c64c5aa 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -1,57 +1,59 @@
-# Публичные проверки
+# Проверки
-Python **3.9+**, стандартная библиотека. Сервер уже должен быть запущен; проверки не создают готовое приложение и сами не управляют Docker. `make check` можно выполнить на исходном skeleton, `make test` — только после реализации API. Полная OpenAPI-валидация отдельно: `make validate` после установки `requirements-dev.txt` в venv.
+## Запуск стенда на Windows 10
-```sh
-make check
-make test BASE_URL=http://localhost:8080
-# без make:
-python3 tests/smoke.py --base-url http://localhost:8080
+Откройте PowerShell в папке проекта. Нужны Python 3.12 и запущенный Docker
+Desktop с Compose. [Требования Docker Desktop к Windows](https://docs.docker.com/desktop/setup/install/windows-install/)
+
+```powershell
+python scripts/init_local.py
+docker compose --env-file .env up -d --build
+docker compose --env-file .env up -d --no-deps --wait --wait-timeout 240 nginx grafana
```
-`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite.
+API доступен на http://127.0.0.1:18081, дашборд Grafana — на
+http://127.0.0.1:13000/d/messenger-overview.
-Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются.
+Остановка с сохранением данных:
-## Проверка сохранности, lab2+
-
-Скопируйте `scripts/restart.example.sh` в `scripts/restart.sh`, реализуйте перезапуск **API и всех используемых хранилищ** с сохранением volumes и задайте executable bit. Скрипт может опираться на ваш compose.yaml или stack.yaml. Он не должен удалять volumes, повторно seed-ить БД или менять тестовые данные.
-
-```sh
-chmod +x scripts/restart.sh
-make test-persistence ACTION_SCRIPT=scripts/restart.sh
+```powershell
+docker compose --env-file .env down
```
-Тест создаёт данные и сессию, запускает указанный файл без shell interpolation, ожидает готовности и проверяет те же данные **с прежним токеном**. В lab4+ также проверяется исходный файл. Hook может работать до 90 секунд, готовность затем ожидается до 90 секунд. TTL сессии для этого теста — минимум 300 секунд. Успех не доказывает, что hook действительно перезапустил БД: приложите команды/состояния контейнеров до и после.
+Остановка с удалением учебных данных:
-## Проверка отказа, lab3+
-
-Реализуйте `scripts/stop-one.sh` на основе примера. `TARGET_INSTANCE_ID` в окружении содержит alias обслужившей тест API-реплики: сопоставьте его своему контейнеру/Swarm task и остановите именно его. Скрипт должен быстро вернуть 0 и **оставить эту реплику остановленной**; восстановление выполняйте отдельно. Swarm может создать новую task с другим ID — это допустимо.
-
-```sh
-chmod +x scripts/stop-one.sh
-make test-failover ACTION_SCRIPT=scripts/stop-one.sh
+```powershell
+docker compose --env-file .env down --volumes --remove-orphans
```
-Перед отказом тест наблюдает ≥2 API-реплики; 15 секунд отправляет сообщения параллельно выполнению hook, повторяет неопределённый POST с тем же ключом, проверяет восстановление записи за ≤10 секунд, отсутствие остановленной реплики в последних ответах, сохранность подтверждённых сообщений и отсутствие дублей. Ошибки переходного периода учитываются. Затем восстановите реплику и повторите для другой. Скрипт — проверка одного отказа API, не гарантия HA БД/LB/host. Вывод hook скрыт, чтобы не утекли секреты; отлаживайте его отдельно с безопасным выводом.
+## Запуск проверок
-## Нагрузка
+Полный прогон на отдельном стенде из PowerShell:
-```sh
-python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
+```powershell
+python -X utf8 scripts/test.py
```
-Это простой **closed-loop** генератор POST сообщений; он ограничен производительностью клиента и страдает coordinated omission. Он выдаёт successful RPS, ошибки и p50/p95/p99 **только успешных запросов**. Во время нагрузки нет прозрачных retry; timeout мог скрыть совершённую запись, поэтому это throughput HTTP-подтверждений, не точный счётчик COMMIT. Изменение размера истории входит в профиль. Для серьёзного исследования используйте k6/Locust/wrk либо свой обоснованный генератор, а не выводите максимальную пропускную способность из одного запуска этого скрипта. Для нагрузки pipeline изображений нужен отдельный сценарий студента.
+Для полного прогона Docker Desktop должен разрешать запуск привилегированных
+контейнеров. Команда создаёт настройки и файловые секреты временного стенда.
+Сводка прогона сохраняется в `evidence/lab6/verification.json`,
+результаты проверки телеметрии — в `evidence/lab6/observability.json`.
-## HTTPS, lab7
+Флаг `--integration-only` оставляет проверки API, телеметрии и отказов.
+Результаты этого режима сохраняются в `evidence/lab6/integration/`.
-```sh
-make test BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt
-make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca.crt
+Проверка структуры контракта и быстрый тест запущенного API:
+
+```powershell
+python -X utf8 scripts/check_contract.py
+python -X utf8 tests/smoke.py --base-url http://127.0.0.1:18081
```
-Все скрипты принимают `--ca-file`/`CA_FILE` и проверяют серверный сертификат и hostname. `--insecure` отсутствует. TLS-тест сначала проверяет рабочее доверенное соединение, затем намеренно пустой trust store и именно ошибку проверки сертификата; network timeout не засчитывается как правильный отказ. **Внутреннее mTLS, identity/authorization, OpenBao policy и hardening проверяют отдельные тесты студента** по trust matrix. Клиентский сертификат внешнему учебному REST-клиенту не требуется: mTLS находится на внутренних связях.
+`tests/load.py` создаёт синтетические сообщения. Пример запуска на учебном стенде:
-## Что ещё остаётся доказать
+```powershell
+python -X utf8 tests/load.py --base-url http://127.0.0.1:18081
+```
-Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production.
+Автоматические сценарии остановки worker, PostgreSQL и Loki входят в полный прогон.
+Назначение метрик и порядок диагностики описаны в [отчёте](../REPORT.md).
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/tests/client.py b/tests/client.py
index f0499a2..d266a66 100644
--- a/tests/client.py
+++ b/tests/client.py
@@ -1,4 +1,5 @@
"""HTTP helpers for public black-box tests; Python 3.9+, standard library only."""
+
import base64
import datetime as dt
import hashlib
@@ -13,205 +14,336 @@ import urllib.request
import uuid
ROOT = Path(__file__).resolve().parents[1]
-LAB = json.loads((ROOT / 'lab.json').read_text())['number']
-SPEC = json.loads((ROOT / 'contracts/openapi.json').read_text())
+LAB = json.loads((ROOT / "lab.json").read_text())["number"]
+SPEC = json.loads((ROOT / "contracts/openapi.json").read_text())
+
class Failure(AssertionError):
pass
+
def check(condition, message):
if not condition:
raise Failure(message)
-def timestamp(value):
- check(isinstance(value, str) and value.endswith('Z'), 'timestamp must be UTC ending in Z')
- try:
- return dt.datetime.fromisoformat(value[:-1] + '+00:00')
- except ValueError as exc:
- raise Failure('invalid RFC3339 timestamp') from exc
-def validate(value, schema, where='response'):
+def timestamp(value):
+ check(
+ isinstance(value, str) and value.endswith("Z"),
+ "timestamp must be UTC ending in Z",
+ )
+ try:
+ return dt.datetime.fromisoformat(value[:-1] + "+00:00")
+ except ValueError as exc:
+ raise Failure("invalid RFC3339 timestamp") from exc
+
+
+def validate(value, schema, where="response"):
"""Validate the JSON Schema subset used by the supplied contract, not arbitrary OpenAPI."""
- if '$ref' in schema:
+ if "$ref" in schema:
target = SPEC
- for part in schema['$ref'].split('/')[1:]:
+ for part in schema["$ref"].split("/")[1:]:
target = target[part]
return validate(value, target, where)
- kinds = schema.get('type', [])
+ kinds = schema.get("type", [])
if isinstance(kinds, str):
kinds = [kinds]
- matches = {'null':value is None, 'boolean':isinstance(value,bool), 'integer':isinstance(value,int) and not isinstance(value,bool), 'number':isinstance(value,(int,float)) and not isinstance(value,bool), 'string':isinstance(value,str), 'array':isinstance(value,list), 'object':isinstance(value,dict)}
+ matches = {
+ "null": value is None,
+ "boolean": isinstance(value, bool),
+ "integer": isinstance(value, int) and not isinstance(value, bool),
+ "number": isinstance(value, (int, float)) and not isinstance(value, bool),
+ "string": isinstance(value, str),
+ "array": isinstance(value, list),
+ "object": isinstance(value, dict),
+ }
if kinds:
- check(any(matches.get(k,False) for k in kinds), where + ': incorrect type')
- if 'enum' in schema:
- check(value in schema['enum'], where + ': invalid enum')
- if 'const' in schema:
- check(value == schema['const'], where + ': invalid const')
- if isinstance(value,dict):
- props = schema.get('properties',{})
- check(all(k in value for k in schema.get('required',[])), where + ': missing required fields')
- if schema.get('additionalProperties') is False:
- check(not set(value).difference(props), where + ': unexpected fields')
+ check(any(matches.get(k, False) for k in kinds), where + ": incorrect type")
+ if "enum" in schema:
+ check(value in schema["enum"], where + ": invalid enum")
+ if "const" in schema:
+ check(value == schema["const"], where + ": invalid const")
+ if isinstance(value, dict):
+ props = schema.get("properties", {})
+ check(
+ all(k in value for k in schema.get("required", [])),
+ where + ": missing required fields",
+ )
+ if schema.get("additionalProperties") is False:
+ check(not set(value).difference(props), where + ": unexpected fields")
for key, sub in props.items():
if key in value:
- validate(value[key], sub, where + '.' + key)
- if isinstance(value,list):
- check(len(value) >= schema.get('minItems',0), where + ': too few items')
- check(len(value) <= schema.get('maxItems',float('inf')), where + ': too many items')
- if schema.get('uniqueItems'):
- encoded = [json.dumps(x,sort_keys=True) for x in value]
- check(len(encoded)==len(set(encoded)), where + ': duplicate items')
- if 'items' in schema:
+ validate(value[key], sub, where + "." + key)
+ if isinstance(value, list):
+ check(len(value) >= schema.get("minItems", 0), where + ": too few items")
+ check(
+ len(value) <= schema.get("maxItems", float("inf")),
+ where + ": too many items",
+ )
+ if schema.get("uniqueItems"):
+ encoded = [json.dumps(x, sort_keys=True) for x in value]
+ check(len(encoded) == len(set(encoded)), where + ": duplicate items")
+ if "items" in schema:
for item in value:
- validate(item, schema['items'], where + '[]')
- if isinstance(value,str):
- check(len(value) >= schema.get('minLength',0), where + ': too short')
- check(len(value) <= schema.get('maxLength',float('inf')), where + ': too long')
- if 'pattern' in schema:
- check(re.search(schema['pattern'], value) is not None, where + ': pattern mismatch')
- if schema.get('format') == 'uuid':
+ validate(item, schema["items"], where + "[]")
+ if isinstance(value, str):
+ check(len(value) >= schema.get("minLength", 0), where + ": too short")
+ check(len(value) <= schema.get("maxLength", float("inf")), where + ": too long")
+ if "pattern" in schema:
+ check(
+ re.search(schema["pattern"], value) is not None,
+ where + ": pattern mismatch",
+ )
+ if schema.get("format") == "uuid":
try:
uuid.UUID(value)
except ValueError as exc:
- raise Failure(where + ': invalid UUID') from exc
- if schema.get('format') == 'date-time':
+ raise Failure(where + ": invalid UUID") from exc
+ if schema.get("format") == "date-time":
timestamp(value)
- if isinstance(value,(int,float)) and not isinstance(value,bool):
- check(value >= schema.get('minimum',-float('inf')), where + ': below minimum')
- check(value <= schema.get('maximum',float('inf')), where + ': above maximum')
- if 'anyOf' in schema:
- for alternative in schema['anyOf']:
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
+ check(value >= schema.get("minimum", -float("inf")), where + ": below minimum")
+ check(value <= schema.get("maximum", float("inf")), where + ": above maximum")
+ if "anyOf" in schema:
+ for alternative in schema["anyOf"]:
try:
validate(value, alternative, where)
break
except Failure:
pass
else:
- raise Failure(where + ': no anyOf branch matches')
+ raise Failure(where + ": no anyOf branch matches")
+
def shape(value, name):
- validate(value, SPEC['components']['schemas'][name])
+ validate(value, SPEC["components"]["schemas"][name])
return value
+
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
return None
+
class Client:
def __init__(self, base_url=None, ca_file=None, timeout=10):
- self.base = (base_url or os.environ.get('BASE_URL') or ('https://localhost:8443' if LAB==7 else 'http://localhost:8080')).rstrip('/')
- check(self.base.startswith(('http://','https://')), 'BASE_URL must be HTTP(S)')
- check(LAB != 7 or self.base.startswith('https://'), 'lab7 requires HTTPS')
+ self.base = (
+ base_url
+ or os.environ.get("BASE_URL")
+ or ("https://localhost:8443" if LAB == 7 else "http://localhost:8080")
+ ).rstrip("/")
+ check(self.base.startswith(("http://", "https://")), "BASE_URL must be HTTP(S)")
+ check(LAB != 7 or self.base.startswith("https://"), "lab7 requires HTTPS")
self.timeout = timeout
- context = ssl.create_default_context(cafile=ca_file or os.environ.get('CA_FILE') or None)
- self.opener = urllib.request.build_opener(NoRedirect(), urllib.request.HTTPSHandler(context=context))
+ context = ssl.create_default_context(
+ cafile=ca_file or os.environ.get("CA_FILE") or None
+ )
+ self.opener = urllib.request.build_opener(
+ NoRedirect(), urllib.request.HTTPSHandler(context=context)
+ )
- def call(self, method, path, expected=200, body=None, actor=None, headers=None, raw=None, schema=None):
- h = {'Accept':'application/json'}
+ def call(
+ self,
+ method,
+ path,
+ expected=200,
+ body=None,
+ actor=None,
+ headers=None,
+ raw=None,
+ schema=None,
+ ):
+ h = {"Accept": "application/json"}
if actor:
- h['X-User-Id' if LAB==1 else 'Authorization'] = actor['id'] if LAB==1 else 'Bearer ' + actor['token']
+ h["X-User-Id" if LAB == 1 else "Authorization"] = (
+ actor["id"] if LAB == 1 else "Bearer " + actor["token"]
+ )
if body is not None:
- raw = json.dumps(body,ensure_ascii=False).encode('utf-8')
- h['Content-Type'] = 'application/json'
+ raw = json.dumps(body, ensure_ascii=False).encode("utf-8")
+ h["Content-Type"] = "application/json"
h.update(headers or {})
- req = urllib.request.Request(self.base + path, data=raw, headers=h, method=method)
+ req = urllib.request.Request(
+ self.base + path, data=raw, headers=h, method=method
+ )
start = time.monotonic()
try:
response = self.opener.open(req, timeout=self.timeout)
except urllib.error.HTTPError as error:
response = error
- except (urllib.error.URLError,TimeoutError,OSError) as exc:
- # Do not print URL query, headers, request body, or credentials.
- raise Failure(method + ' ' + path.split('?')[0] + ': connection/TLS failure (' + type(exc).__name__ + ')') from None
+ except (urllib.error.URLError, TimeoutError, OSError) as exc:
+
+ raise Failure(
+ method
+ + " "
+ + path.split("?")[0]
+ + ": connection/TLS failure ("
+ + type(exc).__name__
+ + ")"
+ ) from None
with response:
status = response.code
rh = response.headers
- data = response.read(16*1024*1024 + 1)
- check(len(data) <= 16*1024*1024, 'response exceeds test safety limit')
- allowed = [expected] if isinstance(expected,int) else expected
- check(status in allowed, method + ' ' + path.split('?')[0] + ': expected ' + str(allowed) + ', received ' + str(status))
- check(bool(rh.get('X-Request-Id')), 'missing X-Request-Id')
- if LAB>=3:
- check(bool(rh.get('X-Instance-Id')), 'missing X-Instance-Id')
+ data = response.read(16 * 1024 * 1024 + 1)
+ check(len(data) <= 16 * 1024 * 1024, "response exceeds test safety limit")
+ allowed = [expected] if isinstance(expected, int) else expected
+ check(
+ status in allowed,
+ method
+ + " "
+ + path.split("?")[0]
+ + ": expected "
+ + str(allowed)
+ + ", received "
+ + str(status),
+ )
+ check(bool(rh.get("X-Request-Id")), "missing X-Request-Id")
+ if LAB >= 3:
+ check(bool(rh.get("X-Instance-Id")), "missing X-Instance-Id")
payload = None
- if schema or status>=400:
- check(rh.get_content_type()=='application/json', 'expected application/json')
+ if schema or status >= 400:
+ check(
+ rh.get_content_type() == "application/json", "expected application/json"
+ )
try:
payload = json.loads(data)
- except (ValueError,UnicodeError):
- raise Failure('invalid response JSON') from None
- shape(payload, 'Error' if status>=400 else schema)
- if status>=400:
- check(payload['error']['request_id']==rh.get('X-Request-Id'),'error request_id differs from header')
- if status==401 and LAB>=2:
- scheme = 'Basic' if path=='/api/v1/auth/sessions' else 'Bearer'
- check(rh.get('WWW-Authenticate','').lower().startswith(scheme.lower()), 'incorrect WWW-Authenticate challenge')
- if status==204:
- check(not data, '204 must not contain body')
- return {'status':status,'headers':rh,'json':payload,'bytes':data,'elapsed':time.monotonic()-start}
+ except (ValueError, UnicodeError):
+ raise Failure("invalid response JSON") from None
+ shape(payload, "Error" if status >= 400 else schema)
+ if status >= 400:
+ check(
+ payload["error"]["request_id"] == rh.get("X-Request-Id"),
+ "error request_id differs from header",
+ )
+ if status == 401 and LAB >= 2:
+ scheme = "Basic" if path == "/api/v1/auth/sessions" else "Bearer"
+ check(
+ rh.get("WWW-Authenticate", "").lower().startswith(scheme.lower()),
+ "incorrect WWW-Authenticate challenge",
+ )
+ if status == 204:
+ check(not data, "204 must not contain body")
+ return {
+ "status": status,
+ "headers": rh,
+ "json": payload,
+ "bytes": data,
+ "elapsed": time.monotonic() - start,
+ }
def user(self, label):
- username = 'u_' + label + '_' + uuid.uuid4().hex[:12]
- password = 'T3st-' + uuid.uuid4().hex
- body = {'username':username,'display_name':'Студент ' + label}
- if LAB>=2:
- body['password'] = password
- user = self.call('POST','/api/v1/users',201,body=body,schema='User')['json']
- check(user['username']==username,'username changed')
- actor = {'id':user['id'],'user':user,'registration':body}
- if LAB>=2:
- credential = base64.b64encode((username+':'+password).encode()).decode()
- session = self.call('POST','/api/v1/auth/sessions',201,headers={'Authorization':'Basic '+credential},schema='Session')['json']
- check(session['user']==user,'session user mismatch')
- check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),'session already expired')
- actor['token'] = session['token']
+ username = "u_" + label + "_" + uuid.uuid4().hex[:12]
+ password = "T3st-" + uuid.uuid4().hex
+ body = {"username": username, "display_name": "Студент " + label}
+ if LAB >= 2:
+ body["password"] = password
+ user = self.call("POST", "/api/v1/users", 201, body=body, schema="User")["json"]
+ check(user["username"] == username, "username changed")
+ actor = {"id": user["id"], "user": user, "registration": body}
+ if LAB >= 2:
+ credential = base64.b64encode((username + ":" + password).encode()).decode()
+ session = self.call(
+ "POST",
+ "/api/v1/auth/sessions",
+ 201,
+ headers={"Authorization": "Basic " + credential},
+ schema="Session",
+ )["json"]
+ check(session["user"] == user, "session user mismatch")
+ check(
+ timestamp(session["expires_at"]) > dt.datetime.now(dt.timezone.utc),
+ "session already expired",
+ )
+ actor["token"] = session["token"]
return actor
def bootstrap(self):
- self.call('GET','/health/live',schema='Health')
- self.call('GET','/health/ready',schema='Health')
- alice, bob, eve = (self.user(label) for label in ('alice','bob','eve'))
- chat = self.call('POST','/api/v1/chats',201,actor=alice,body={'title':'Контрактный тест','member_ids':[alice['id'],bob['id']]},schema='Chat')['json']
- check(set(chat['member_ids'])=={alice['id'],bob['id']},'chat member mismatch')
- return alice,bob,eve,chat
+ self.call("GET", "/health/live", schema="Health")
+ self.call("GET", "/health/ready", schema="Health")
+ alice, bob, eve = (self.user(label) for label in ("alice", "bob", "eve"))
+ chat = self.call(
+ "POST",
+ "/api/v1/chats",
+ 201,
+ actor=alice,
+ body={"title": "Контрактный тест", "member_ids": [alice["id"], bob["id"]]},
+ schema="Chat",
+ )["json"]
+ check(
+ set(chat["member_ids"]) == {alice["id"], bob["id"]}, "chat member mismatch"
+ )
+ return alice, bob, eve, chat
def message(self, actor, chat, text, key=None, attachments=None):
- body = {'text':text}
+ body = {"text": text}
if attachments is not None:
- body['attachment_ids'] = attachments
- headers = {'Idempotency-Key':key or uuid.uuid4().hex} if LAB>=3 else {}
- return self.call('POST','/api/v1/chats/'+chat['id']+'/messages',201,actor=actor,body=body,headers=headers,schema='Message')
+ body["attachment_ids"] = attachments
+ headers = {"Idempotency-Key": key or uuid.uuid4().hex} if LAB >= 3 else {}
+ return self.call(
+ "POST",
+ "/api/v1/chats/" + chat["id"] + "/messages",
+ 201,
+ actor=actor,
+ body=body,
+ headers=headers,
+ schema="Message",
+ )
def attachment(self, actor, chat):
- content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
- att = self.call('POST','/api/v1/chats/'+chat['id']+'/attachments?filename=sample.png',201 if LAB==4 else 202,actor=actor,raw=content,headers={'Content-Type':'image/png'},schema='Attachment')['json']
- check(att['status']==('ready' if LAB==4 else 'queued'),'incorrect upload status')
- check(att['chat_id']==chat['id'] and att['owner_id']==actor['id'],'attachment ownership mismatch')
- check(att['sha256']==hashlib.sha256(content).hexdigest() and att['size_bytes']==len(content),'original checksum/size mismatch')
- return att,content
+ content = (ROOT / "tests/fixtures/sample.png").read_bytes()
+ att = self.call(
+ "POST",
+ "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png",
+ 201 if LAB == 4 else 202,
+ actor=actor,
+ raw=content,
+ headers={"Content-Type": "image/png"},
+ schema="Attachment",
+ )["json"]
+ check(
+ att["status"] == ("ready" if LAB == 4 else "queued"),
+ "incorrect upload status",
+ )
+ check(
+ att["chat_id"] == chat["id"] and att["owner_id"] == actor["id"],
+ "attachment ownership mismatch",
+ )
+ check(
+ att["sha256"] == hashlib.sha256(content).hexdigest()
+ and att["size_bytes"] == len(content),
+ "original checksum/size mismatch",
+ )
+ return att, content
def wait_ready(self, actor, att, wait_seconds=60):
- deadline = time.monotonic()+wait_seconds
- while time.monotonic()=2 and i+length<=len(data),'invalid JPEG segment length')
+ check(i + 2 <= len(data), "truncated JPEG segment")
+ length = int.from_bytes(data[i : i + 2], "big")
+ check(length >= 2 and i + length <= len(data), "invalid JPEG segment length")
if marker in sof:
- check(length>=8,'invalid JPEG SOF')
- return (int.from_bytes(data[i+5:i+7],'big'),int.from_bytes(data[i+3:i+5],'big'))
+ check(length >= 8, "invalid JPEG SOF")
+ return (
+ int.from_bytes(data[i + 5 : i + 7], "big"),
+ int.from_bytes(data[i + 3 : i + 5], "big"),
+ )
i += length
- raise Failure('JPEG dimensions not found')
+ raise Failure("JPEG dimensions not found")
diff --git a/tests/load.py b/tests/load.py
index ad81d59..d1e7f01 100644
--- a/tests/load.py
+++ b/tests/load.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""Small closed-loop HTTP write benchmark; not an open-loop capacity proof."""
+
from concurrent.futures import ThreadPoolExecutor
import json
import math
@@ -11,48 +12,77 @@ from client import Client, Failure, LAB, arguments, check, run
def main():
def configure(p):
- p.add_argument('--duration',type=float,default=30)
- p.add_argument('--concurrency',type=int,default=4)
- p.add_argument('--output',default=None,help='Optional JSON metrics, no session tokens')
- args = arguments('Synthetic POST message load, closed loop; creates users/messages',configure)
- check(1<=args.duration<=3600,'duration must be 1..3600 seconds')
- check(1<=args.concurrency<=128,'concurrency must be 1..128')
- c = Client(args.base_url,args.ca_file)
- alice,bob,eve,chat = c.bootstrap()
+ p.add_argument("--duration", type=float, default=30)
+ p.add_argument("--concurrency", type=int, default=4)
+ p.add_argument(
+ "--output", default=None, help="Optional JSON metrics, no session tokens"
+ )
+
+ args = arguments(
+ "Synthetic POST message load, closed loop; creates users/messages", configure
+ )
+ check(1 <= args.duration <= 3600, "duration must be 1..3600 seconds")
+ check(1 <= args.concurrency <= 128, "concurrency must be 1..128")
+ c = Client(args.base_url, args.ca_file)
+ alice, bob, eve, chat = c.bootstrap()
barrier = threading.Barrier(args.concurrency)
+
def worker(worker_id):
- local = Client(args.base_url,args.ca_file,timeout=5)
+ local = Client(args.base_url, args.ca_file, timeout=5)
barrier.wait()
- deadline = time.monotonic()+args.duration
- latencies,errors = [],0
+ deadline = time.monotonic() + args.duration
+ latencies, errors = [], 0
index = 0
- while time.monotonic()=2,'persistence requires lab2+')
- check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
+ p.add_argument("--mode", choices=["persistence", "failover"], required=True)
+ p.add_argument(
+ "--action-script",
+ required=True,
+ help="Executable script authored by student; restarts stack or stops ONE API replica",
+ )
+
+ args = arguments(
+ "Explicit restart/failover scenario; modifies only synthetic API data",
+ configure,
+ )
+ check(LAB >= 2, "persistence requires lab2+")
+ check(args.mode != "failover" or LAB >= 3, "failover requires lab3+")
script = Path(args.action_script).expanduser().resolve()
- check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
- check('.example.' not in script.name,'copy and implement the example hook first')
- c = Client(args.base_url,args.ca_file,timeout=2)
- alice,bob,eve,chat = c.bootstrap()
- sent = c.message(alice,chat,'Сохранить при отказе')
- original = sent['json']
- target = sent['headers'].get('X-Instance-Id','')
+ check(
+ script.is_file() and os.access(script, os.X_OK),
+ "action script must exist and be executable (chmod +x)",
+ )
+ check(".example." not in script.name, "copy and implement the example hook first")
+ c = Client(args.base_url, args.ca_file, timeout=2)
+ alice, bob, eve, chat = c.bootstrap()
+ sent = c.message(alice, chat, "Сохранить при отказе")
+ original = sent["json"]
+ target = sent["headers"].get("X-Instance-Id", "")
attachment = content = None
- if LAB>=4:
- attachment,content = c.attachment(alice,chat)
- if LAB>=5:
- attachment = c.wait_ready(alice,attachment)
+ if LAB >= 4:
+ attachment, content = c.attachment(alice, chat)
+ if LAB >= 5:
+ attachment = c.wait_ready(alice, attachment)
env = dict(os.environ)
- env['TARGET_INSTANCE_ID'] = target
+ env["TARGET_INSTANCE_ID"] = target
observed = set()
- if args.mode=='failover':
+ if args.mode == "failover":
for _ in range(40):
- observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
- check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
- print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
- # No shell interpolation and no Docker commands in this test.
- proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
+ observed.add(
+ c.call("GET", "/api/v1/users/me", actor=alice, schema="User")[
+ "headers"
+ ]["X-Instance-Id"]
+ )
+ check(
+ len(observed) >= 2,
+ "less than two API instances observed before failure; verify LB/session affinity",
+ )
+ print(
+ "Executing explicit "
+ + args.mode
+ + " hook; hook output suppressed to avoid leaking secrets."
+ )
+
+ proc = subprocess.Popen(
+ [str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL
+ )
started = time.monotonic()
- successes,failures,after_instances = [],0,[]
+ successes, failures, after_instances = [], 0, []
try:
- if args.mode=='persistence':
+ if args.mode == "persistence":
try:
code = proc.wait(timeout=90)
except subprocess.TimeoutExpired:
- raise Failure('restart hook exceeded 90 seconds') from None
- check(code==0,'restart hook failed (inspect your script locally)')
- deadline = time.monotonic()+90
+ raise Failure("restart hook exceeded 90 seconds") from None
+ check(code == 0, "restart hook failed (inspect your script locally)")
+ deadline = time.monotonic() + 90
while True:
try:
- c.call('GET','/health/ready',schema='Health')
- c.call('GET','/api/v1/users/me',actor=alice,schema='User')
+ c.call("GET", "/health/ready", schema="Health")
+ c.call("GET", "/api/v1/users/me", actor=alice, schema="User")
break
except Failure:
- check(time.monotonic()=5,'too few successful writes after stop')
- check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
- print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
- # Existing token, no re-login. Read all confirmations and reject duplicates.
- messages = history(c,bob,chat)
- ids = [m['id'] for m in messages]
- check(len(ids)==len(set(ids)),'duplicated message IDs in history')
- check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
+ check(proc.poll() == 0, "stop-one hook did not finish successfully")
+ check(
+ first_recovery is not None and first_recovery <= 10,
+ "writes did not recover within 10 seconds from hook start",
+ )
+ check(len(after_instances) >= 5, "too few successful writes after stop")
+ check(
+ target not in after_instances[-5:],
+ "target replica still serves requests; hook must leave it stopped",
+ )
+ print(
+ json.dumps(
+ {
+ "observed_instances_before": len(observed),
+ "successful_writes": len(successes),
+ "failed_attempts": failures,
+ "first_recovery_seconds": round(first_recovery, 3),
+ },
+ ensure_ascii=False,
+ )
+ )
+
+ messages = history(c, bob, chat)
+ ids = [m["id"] for m in messages]
+ check(len(ids) == len(set(ids)), "duplicated message IDs in history")
+ check(
+ any(m == original for m in messages),
+ "confirmed pre-failure message lost or altered",
+ )
for msg in successes:
- check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
- # Bodies are unique in this scenario: an uncertain request may exist once, never twice.
- texts = [m['text'] for m in messages]
- check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
+ check(
+ any(m == msg for m in messages),
+ "confirmed in-flight message lost or altered",
+ )
+
+ texts = [m["text"] for m in messages]
+ check(
+ len(texts) == len(set(texts)), "uncertain retry created duplicate message"
+ )
if attachment:
- c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
- downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
- check(downloaded['bytes']==content,'attachment lost or changed across failure')
- print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
+ c.call(
+ "GET",
+ "/api/v1/attachments/" + attachment["id"],
+ actor=bob,
+ schema="Attachment",
+ )
+ downloaded = c.call(
+ "GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob
+ )
+ check(
+ downloaded["bytes"] == content,
+ "attachment lost or changed across failure",
+ )
+ print(
+ "PASS: "
+ + args.mode
+ + " preserved confirmed data and existing session. Hook scope needs operator evidence."
+ )
finally:
if proc.poll() is None:
proc.terminate()
@@ -125,5 +202,6 @@ def main():
proc.kill()
proc.wait()
-if __name__=='__main__':
+
+if __name__ == "__main__":
run(main)
diff --git a/tests/smoke.py b/tests/smoke.py
index 4cbfa55..97d20a0 100644
--- a/tests/smoke.py
+++ b/tests/smoke.py
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
"""One complete public acceptance scenario, cumulative by lab number."""
+
import base64
import hashlib
import uuid
@@ -7,76 +8,165 @@ from client import Client, LAB, arguments, check, run, timestamp
def main():
- args = arguments('Messenger public smoke; creates synthetic users and messages')
- c = Client(args.base_url,args.ca_file)
- alice,bob,eve,chat = c.bootstrap()
- path = '/api/v1/chats/'+chat['id']
- c.call('POST','/api/v1/users',409,body=alice['registration'])
- c.call('GET','/api/v1/users/me',401)
- c.call('GET',path,404,actor=eve)
- me = c.call('GET','/api/v1/users/me',actor=alice,schema='User')['json']
- check(me==alice['user'],'current user mismatch')
- listed = c.call('GET','/api/v1/chats?limit=100',actor=bob,schema='ChatPage')['json']
- check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat')
- hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json']
- check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider')
+ args = arguments("Messenger public smoke; creates synthetic users and messages")
+ c = Client(args.base_url, args.ca_file)
+ alice, bob, eve, chat = c.bootstrap()
+ path = "/api/v1/chats/" + chat["id"]
+ c.call("POST", "/api/v1/users", 409, body=alice["registration"])
+ c.call("GET", "/api/v1/users/me", 401)
+ c.call("GET", path, 404, actor=eve)
+ me = c.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"]
+ check(me == alice["user"], "current user mismatch")
+ listed = c.call("GET", "/api/v1/chats?limit=100", actor=bob, schema="ChatPage")[
+ "json"
+ ]
+ check(
+ any(x["id"] == chat["id"] for x in listed["items"]), "member cannot list chat"
+ )
+ hidden = c.call("GET", "/api/v1/chats?limit=100", actor=eve, schema="ChatPage")[
+ "json"
+ ]
+ check(all(x["id"] != chat["id"] for x in hidden["items"]), "chat leaks to outsider")
key = uuid.uuid4().hex
- first = c.message(alice,chat,'Привет, мир 👋',key=key)['json']
- second = c.message(bob,chat,'Сообщение Боба')['json']
- check(first['sender_id']==alice['id'] and second['sender_id']==bob['id'],'sender was not derived from actor')
- check(first['chat_id']==second['chat_id']==chat['id'],'wrong chat in message')
- check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed')
- page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json']
- check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor')
+ first = c.message(alice, chat, "Привет, мир 👋", key=key)["json"]
+ second = c.message(bob, chat, "Сообщение Боба")["json"]
+ check(
+ first["sender_id"] == alice["id"] and second["sender_id"] == bob["id"],
+ "sender was not derived from actor",
+ )
+ check(first["chat_id"] == second["chat_id"] == chat["id"], "wrong chat in message")
+ check(
+ first["text"] == "Привет, мир 👋" and second["text"] == "Сообщение Боба",
+ "message text changed",
+ )
+ page1 = c.call("GET", path + "/messages?limit=1", actor=bob, schema="MessagePage")[
+ "json"
+ ]
+ check(
+ len(page1["items"]) == 1 and page1["next_cursor"],
+ "first page must have one item and cursor",
+ )
from urllib.parse import quote
- page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json']
- check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history')
- ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int))
- check(page1['items']+page2['items']==ordered,'pagination lost, duplicated or reordered a message')
- h = {'Idempotency-Key':uuid.uuid4().hex} if LAB>=3 else {}
- c.call('POST',path+'/messages',400,actor=alice,body={'text':''},headers=h)
- c.call('POST',path+'/messages',404,actor=eve,body={'text':'чужое'},headers=h)
- c.call('GET',path+'/messages?limit=0',400,actor=alice)
- if LAB>=2:
- bad = base64.b64encode((alice['registration']['username']+':wrong-password').encode()).decode()
- c.call('POST','/api/v1/auth/sessions',401,headers={'Authorization':'Basic '+bad})
- good = base64.b64encode((alice['registration']['username']+':'+alice['registration']['password']).encode()).decode()
- c.call('GET','/api/v1/users/me',401,headers={'Authorization':'Basic '+good})
- c.call('GET','/api/v1/users/me',401,headers={'X-User-Id':alice['id']})
- me = c.call('GET','/api/v1/users/me',actor=alice,headers={'X-User-Id':eve['id']},schema='User')['json']
- check(me['id']==alice['id'],'X-User-Id overrode bearer identity')
- if LAB>=3:
- replay = c.message(alice,chat,first['text'],key=key)['json']
- check(replay==first,'idempotency replay differs')
- c.call('POST',path+'/messages',409,actor=alice,body={'text':'другой body'},headers={'Idempotency-Key':key})
- c.call('POST',path+'/messages',400,actor=alice,body={'text':'без ключа'})
- if LAB>=4:
- att,content = c.attachment(alice,chat)
- # A queued attachment must already be linkable to its own chat.
- attached = c.message(alice,chat,'',attachments=[att['id']])['json']
- check(attached['attachment_ids']==[att['id']],'message lost attachment')
- ready = c.wait_ready(bob,att) if LAB>=5 else att
- check(ready['error_code'] is None,'ready attachment has an error')
- download = c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)
- check(download['bytes']==content,'download differs from original')
- check(download['headers'].get_content_type()=='image/png','original content type mismatch')
- for suffix in ('','/content'):
- c.call('GET','/api/v1/attachments/'+att['id']+suffix,404,actor=eve)
- if LAB>=5:
- variants = [x for x in ready['variants'] if x['name']=='thumbnail']
- check(len(variants)==1,'expected exactly one thumbnail')
- v = variants[0]
- check((v['width'],v['height'])==(256,160),'thumbnail metadata has wrong dimensions')
- thumb = c.call('GET','/api/v1/attachments/'+att['id']+'/content?variant=thumbnail',actor=bob)
- check(thumb['headers'].get_content_type()=='image/jpeg','thumbnail content type mismatch')
- check(len(thumb['bytes'])==v['size_bytes'] and hashlib.sha256(thumb['bytes']).hexdigest()==v['sha256'],'thumbnail checksum mismatch')
- from image_probe import jpeg_size
- check(jpeg_size(thumb['bytes'])==(256,160),'actual JPEG dimensions mismatch')
- if LAB>=2:
- c.call('DELETE','/api/v1/auth/sessions/current',204,actor=alice)
- c.call('GET','/api/v1/users/me',401,actor=alice)
- c.call('DELETE','/api/v1/auth/sessions/current',401,actor=alice)
- print('PASS: lab%d public API scenario; infrastructure and grading evidence remain separate.' % LAB)
-if __name__=='__main__':
+ page2 = c.call(
+ "GET",
+ path + "/messages?limit=1&cursor=" + quote(page1["next_cursor"], safe=""),
+ actor=alice,
+ schema="MessagePage",
+ )["json"]
+ check(
+ len(page2["items"]) == 1 and page2["next_cursor"] is None,
+ "second page must finish history",
+ )
+ ordered = sorted(
+ [first, second],
+ key=lambda x: (timestamp(x["created_at"]), uuid.UUID(x["id"]).int),
+ )
+ check(
+ page1["items"] + page2["items"] == ordered,
+ "pagination lost, duplicated or reordered a message",
+ )
+ h = {"Idempotency-Key": uuid.uuid4().hex} if LAB >= 3 else {}
+ c.call("POST", path + "/messages", 400, actor=alice, body={"text": ""}, headers=h)
+ c.call(
+ "POST", path + "/messages", 404, actor=eve, body={"text": "чужое"}, headers=h
+ )
+ c.call("GET", path + "/messages?limit=0", 400, actor=alice)
+ if LAB >= 2:
+ bad = base64.b64encode(
+ (alice["registration"]["username"] + ":wrong-password").encode()
+ ).decode()
+ c.call(
+ "POST",
+ "/api/v1/auth/sessions",
+ 401,
+ headers={"Authorization": "Basic " + bad},
+ )
+ good = base64.b64encode(
+ (
+ alice["registration"]["username"]
+ + ":"
+ + alice["registration"]["password"]
+ ).encode()
+ ).decode()
+ c.call(
+ "GET", "/api/v1/users/me", 401, headers={"Authorization": "Basic " + good}
+ )
+ c.call("GET", "/api/v1/users/me", 401, headers={"X-User-Id": alice["id"]})
+ me = c.call(
+ "GET",
+ "/api/v1/users/me",
+ actor=alice,
+ headers={"X-User-Id": eve["id"]},
+ schema="User",
+ )["json"]
+ check(me["id"] == alice["id"], "X-User-Id overrode bearer identity")
+ if LAB >= 3:
+ replay = c.message(alice, chat, first["text"], key=key)["json"]
+ check(replay == first, "idempotency replay differs")
+ c.call(
+ "POST",
+ path + "/messages",
+ 409,
+ actor=alice,
+ body={"text": "другой body"},
+ headers={"Idempotency-Key": key},
+ )
+ c.call("POST", path + "/messages", 400, actor=alice, body={"text": "без ключа"})
+ if LAB >= 4:
+ att, content = c.attachment(alice, chat)
+
+ attached = c.message(alice, chat, "", attachments=[att["id"]])["json"]
+ check(attached["attachment_ids"] == [att["id"]], "message lost attachment")
+ ready = c.wait_ready(bob, att) if LAB >= 5 else att
+ check(ready["error_code"] is None, "ready attachment has an error")
+ download = c.call(
+ "GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob
+ )
+ check(download["bytes"] == content, "download differs from original")
+ check(
+ download["headers"].get_content_type() == "image/png",
+ "original content type mismatch",
+ )
+ for suffix in ("", "/content"):
+ c.call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, actor=eve)
+ if LAB >= 5:
+ variants = [x for x in ready["variants"] if x["name"] == "thumbnail"]
+ check(len(variants) == 1, "expected exactly one thumbnail")
+ v = variants[0]
+ check(
+ (v["width"], v["height"]) == (256, 160),
+ "thumbnail metadata has wrong dimensions",
+ )
+ thumb = c.call(
+ "GET",
+ "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail",
+ actor=bob,
+ )
+ check(
+ thumb["headers"].get_content_type() == "image/jpeg",
+ "thumbnail content type mismatch",
+ )
+ check(
+ len(thumb["bytes"]) == v["size_bytes"]
+ and hashlib.sha256(thumb["bytes"]).hexdigest() == v["sha256"],
+ "thumbnail checksum mismatch",
+ )
+ from image_probe import jpeg_size
+
+ check(
+ jpeg_size(thumb["bytes"]) == (256, 160),
+ "actual JPEG dimensions mismatch",
+ )
+ if LAB >= 2:
+ c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice)
+ c.call("GET", "/api/v1/users/me", 401, actor=alice)
+ c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=alice)
+ print(
+ "PASS: lab%d public API scenario; infrastructure and grading evidence remain separate."
+ % LAB
+ )
+
+
+if __name__ == "__main__":
run(main)