diff --git a/.env.example b/.env.example index 119384b5..2f8c6f24 100644 --- a/.env.example +++ b/.env.example @@ -11,7 +11,7 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id @@ -21,5 +21,11 @@ MAX_IMAGE_PIXELS=40000000 BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 -JOB_TIMEOUT_SECONDS=60 +JOB_TIMEOUT_SECONDS=45 +JOB_LEASE_SECONDS=90 +JOB_BACKOFF_BASE_SECONDS=0.4 +JOB_BACKOFF_CAP_SECONDS=8 STAGING_RETENTION_SECONDS=86400 +JPEG_QUALITY=85 +WEBP_QUALITY=80 +WEBP_METHOD=6 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..0669d639 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Щербацевич Владислав Сергеевич, БПИ2302 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявляется -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: из lab4 сохранены PostgreSQL, сессии Redis (Argon2id), две реплики API за HAProxy, приватный MinIO, идемпотентность сообщений. Загрузка отвечает 202 после staging и транзакции outbox. Отдельные RabbitMQ, relay и worker делают thumbnail, crop и optimized WebP. Есть backoff, DLQ, lease, явный replay и проверки разрыва commit/publish и crash до ack. +Команды запуска из чистого clone: `powershell -File scripts/bootstrap.ps1`, затем `docker compose --env-file .env up -d --build --scale api=2`. Подробности, версии образов и остановка — в REPORT.md. +Результаты проверок и ссылка на REPORT.md: smoke, grade4, persistence и failover прошли 2026-11-20. Идентификаторы и цифры — в REPORT.md и `evidence/`. +Ограничения / известные ошибки: один Docker-host, доставка at-least-once, upload не идемпотентен, replay только CLI оператора, pipeline_version = 1. После failover остановленную реплику возвращает `docker compose --env-file .env up -d --scale api=2`. +Источники и использование ИИ: контракт репозитория, документация RabbitMQ и Pillow. Реализацию в Cursor вёл ассистент Grok; защиту объясняет автор. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..251bca33 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab + +WORKDIR /app + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY src /app/src +COPY migrations /app/migrations + +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=40s --retries=12 \ + CMD ["python", "-c", "import os,urllib.request; p=os.environ.get('APP_PORT','8080'); urllib.request.urlopen('http://127.0.0.1:%s/health/ready'%p, timeout=2).read()"] + +CMD ["python", "-m", "app"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..ddec954c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,131 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Щербацевич Владислав Сергеевич, БПИ2302 +- Целевая оценка; выбранный трек 5, если нужен: 4. Трек 5 не заявляется. +- Commit/PR: локальная работа, публикация не выполнялась +- Использованные источники, библиотеки и помощь ИИ: контракт репозитория (`contracts/`, `tests/`, README лабораторной), документация RabbitMQ reliability/confirms, Pillow, aio-pika 9.5.5, SQLAlchemy 2, FastAPI. Реализацию очереди, worker, outbox и проверок уровня 4 в этой сессии вёл ассистент Cursor (Grok). Объяснение решения на защите остаётся за автором. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Стенд рассчитан на Docker Compose на одном хосте. Проверялось на Windows 10, Docker Desktop 29.6.1, Python 3.13 на хосте (в образе `python:3.13.16-slim-bookworm`). На Windows перед `scripts/check_contract.py` нужна переменная `PYTHONUTF8=1`: иначе чтение `openapi.json` падает на cp1251. На Linux-проверке преподавателя это не требуется. + +Образы закреплены тегом и digest: + +| Роль | Образ | +| --- | --- | +| API, worker, relay, gc, migrate | `python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641` | +| PostgreSQL | `postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74` | +| Redis | `redis:7.4.6-alpine@sha256:3b73847e72874be07e6657b129a94761662b79bc0f679273757d4218573b2a98` | +| MinIO | `pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372` | +| MinIO client | `pgsty/mc:RELEASE.2026-11-12T00-00-00Z@sha256:b532278b7a3747dbab6fe45eebb40a653efa704e6758e24e9d9ec54fc96c1800` | +| RabbitMQ | `rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc` | +| HAProxy | `haproxy:3.2-bookworm@sha256:baecd003ae5c2376e58d04a5b2ffc7d82d30e9bebe2869f76222b819f719dee3` | + +Официальные `minio/minio` и `minio/mc` с Docker Hub в этой среде недоступны (репозитории сняты, `quay.io/minio` отвечает 401). Взят тот же сервер MinIO из `pgsty/minio`. Его entrypoint не читает `MINIO_ROOT_*_FILE`, поэтому `infra/minio-entry.sh` экспортирует корневые учётные данные из Docker secrets и запускает `server /data`. Клиент `pgsty/mc` запускается с entrypoint на `infra/minio-bootstrap.sh`: бакет `messenger-attachments` остаётся приватным, приложению выдаётся отдельный пользователь и политика. + +Секреты создаёт bootstrap и кладёт их в `secrets/` (каталог в `.gitignore`). В репозиторий не входят `.env` и значения паролей. + +```powershell +powershell -File scripts/bootstrap.ps1 +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 +``` + +Дождаться `GET http://127.0.0.1:8080/health/ready` со статусом 200. Миграции `001`–`004` применяет одноразовый сервис `migrate` до старта API. Публичный порт только у HAProxy (`8080`). Порты API наружу не публикуются. + +Проверки: + +```powershell +$env:PYTHONUTF8 = "1" +python scripts/check_contract.py +python tests/smoke.py --base-url http://127.0.0.1:8080 +python student-tests/grade4.py +python student-tests/persistence_hook.py +python student-tests/failover.py +``` + +`make test` вызывает тот же smoke. На Windows хуки отказов — `scripts/restart.cmd` и `scripts/stop-one.cmd`; на POSIX — `scripts/restart.sh` и `scripts/stop-one.sh`. + +Безопасная остановка без удаления данных: `docker compose --env-file .env stop`. Полное удаление учебных томов (сообщения, сессии, объекты, очередь) — отдельное намеренное действие: `docker compose --env-file .env down -v`. Эту команду нельзя использовать как перезапуск. + +После failover остановленная реплика API остаётся остановленной (`restart: unless-stopped`). `docker compose restart` её не поднимает. Вернуть обе реплики: `docker compose --env-file .env up -d --scale api=2`. + +Повтор уже обработанной failed-задачи делает оператор внутри worker-контейнера: `python -m app.replay JOB_ID`. Это не публичный HTTP-метод. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab4 перенесены PostgreSQL как источник истины, сессии в Redis (AOF, Argon2id), две реплики API за HAProxy, приватный бакет, ключи идемпотентности сообщений, `X-Request-Id` и `X-Instance-Id` (hostname контейнера, короткий id). Загрузка больше не отвечает `201` и не делает resize в процессе API. + +`POST /api/v1/attachments` отвечает `202` только после двух устойчивых шагов: байты лежат в `staging/{attachment_id}`, и в одной транзакции PostgreSQL записаны вложение со статусом `queued`, строка `attachment_jobs` и строка `job_outbox`. API в брокер не публикует и не зависит от него для ответа `202`. `/health/ready` проверяет PostgreSQL, Redis и S3. + +Отдельный процесс `relay` читает неопубликованные строки outbox и публикует durable-сообщение в exchange `attachments` с publisher confirm. Только после confirm ставится `published_at`. Сообщение соответствует `contracts/jobs.schema.json`: `schema_version`, `type=attachment.process.v1`, `job_id`, `attachment_id`, `pipeline_version`, `attempt`, `created_at`. Байтов и учётных данных в нём нет. Ключи S3 worker читает из базы. + +Очередь `attachment.jobs` durable, prefetch равен `WORKER_CONCURRENCY` (2). Подтверждение consumer — ручное и только после эффекта: объекты `att/{id}/original`, `thumbnail`, `crop`, `optimized` записаны, варианты и статус `ready` зафиксированы одной транзакцией, staging удалён. Повторная доставка перезаписывает те же ключи. `ready` ставится, когда есть все три варианта. Более старый `pipeline_version` не затирает более новый. В этой работе версия всегда 1. + +Варианты (`src/app/pipeline.py`): ориентация EXIF применяется через `ImageOps.exif_transpose`, альфа накладывается на белый фон, метаданные снимаются копированием в новое RGB-изображение. Thumbnail вписывается в 256×256 без увеличения (для fixture 320×200 получается 256×160). Crop — наибольший центральный квадрат, затем 128×128, увеличение разрешено. Optimized — WebP, вписывание в 1280×1280 без увеличения. JPEG quality 85 и `optimize=True`, WebP quality 80 и method 6. Рендер идёт в отдельном процессе с таймаутом `JOB_TIMEOUT_SECONDS` (45), чтобы зависание декодера не блокировало цикл worker. + +Пределы: тело загрузки `MAX_UPLOAD_BYTES` = 10 МиБ, `MAX_IMAGE_PIXELS` = 40e6, оценка распакованного RGBA против `WORKER_IMAGE_MEMORY_BYTES` = 256 МиБ, память контейнера worker = 1024 МиБ. Постоянные ошибки (`invalid_image`, `payload_too_large`, пропавший staging, несовпадение checksum) сразу дают `failed`. Временные (сбой обработки, таймаут, S3 503) повторяются. Пауза: `0.4 * 2^(attempt-1)` секунд, потолок 8 с, плюс равномерный jitter `[0, 0.4]`, сумма тоже не выше 8 с. После 5 попыток статус `failed`, `error_code=retry_exhausted`, строка попадает в `job_failures` и копия — в `attachment.jobs.dlq`. + +Lease `JOB_LEASE_SECONDS` = 90 и обязан быть больше таймаута обработки. Просроченный `processing` sweeper возвращает в `queued` и кладёт новую строку outbox. Если попытки кончились — `failed`. Явный replay готовой задачи с тремя вариантами возвращает `already_ready` и новой работы не создаёт. Replay failed сбрасывает попытку, ставит `queued` и новую строку outbox. + +Staging GC не удаляет, пока задача `queued`/`processing` или failed моложе `STAGING_RETENTION_SECONDS` (86400). Осиротевшие объекты под `att/` по-прежнему удаляются после grace. + +Переменные `.env.example` (`JOB_*`, `JPEG_QUALITY`, `WEBP_*`, `PIPELINE_VERSION`, `STAGING_RETENTION_SECONDS`, `WORKER_IMAGE_MEMORY_BYTES`, `BROKER_URL_FILE`) читает `src/app/config.py`. `SESSION_STORE` в примере оставлен как подсказка lab2; сессии этой работы живут в Redis. `APP_ROLE` разделяет `api`, `migrate`, `gc`, `worker`, `relay`. + +Вложения `queued` и `processing` можно прикрепить к сообщению. `failed` даёт 409. Скачивание содержимого до `ready` даёт 409. Ответ `202` — снимок `queued` с пустым списком вариантов, даже если worker успел закончить раньше отправки ответа. ## Доказательства критериев +Дата прогона: 2026-11-20. Хост: 12th Gen Intel Core i7-12700H, 31,6 ГиБ RAM, Docker Desktop 29.6.1. Лимиты контейнеров заданы в `compose.yaml` (API 0,5 CPU / 512 МиБ, worker 1 CPU / 1024 МиБ, PostgreSQL 1 CPU / 512 МиБ, RabbitMQ 0,5 CPU / 384 МиБ, Redis 0,5 CPU / 128 МиБ, MinIO 0,5 CPU / 256 МиБ). + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт не ослаблен | `python scripts/check_contract.py` при `PYTHONUTF8=1` | PASS | вывод команды в сессии | +| Публичный smoke: 202, ready, JPEG thumbnail | `python tests/smoke.py --base-url http://127.0.0.1:8080` | `PASS: lab5 public API scenario...` за ~14 с | `evidence/smoke.txt` | +| Минимум lab4: 2 реплики, сессия, идемпотентность, приватный S3 | тот же smoke и failover ниже | два `X-Instance-Id` чередуются; бакет приватный, bootstrap создаёт отдельного пользователя | `evidence/failover.json` | +| 202 только после staging и записи задания; worker можно остановить | `python student-tests/grade4.py`, сценарий worker stopped | attachment `5886b435-004d-4aed-b8f7-9a288c7bc181`, job `c6d3c7ff-a181-41f9-a945-e194054b0637`: content 409, после рестарта API и RabbitMQ статус `ready` | `evidence/grade4.json` | +| Durable broker, at-least-once, ack после эффекта | тот же прогон, crash after S3 before ack | attachment `a5a24455-f783-4511-90f9-0b17dd43aaf5`, job `0fdaa8f2-054b-4aa0-a273-e63a425ce8c6`: в логе crash до ack, после рестарта `ready`, по одной строке crop/optimized/thumbnail | `evidence/grade4.json` | +| Состояния и три варианта | happy path того же прогона | attachment `a813b0a0-2bab-404f-aa5f-8850b46db20c` дошёл до `ready`; thumbnail/crop/optimized проверены кодом теста | `evidence/grade4.json` | +| Ограниченный retry и безопасный error_code | inject transient до исчерпания | job `d118acc1-f4cb-4a6e-8fd3-7f7f74d5cbd3`, `error_code=retry_exhausted`, 5 строк в реестре failed | `evidence/grade4.json` | +| Outbox: разрыв COMMIT и publish | relay не публикует, затем процесс поднимается снова | attachment `35000ce9-e4d5-4774-a7de-6b717a4cb208`: до рестарта 1 неопубликованная строка, затем `ready` | `evidence/grade4.json` | +| Crash relay после commit outbox до publish | fault `relay_exit_before_publish` | job `5104d370-c0d6-48f1-916a-6611afb8958f`, `gap_injected=true`, лог выхода есть, итог `ready` | `evidence/grade4.json` | +| Lease зависшего processing | lease истёк, sweeper вернул задачу | job `6ebf2c9f-090a-4637-94b2-2eb0a87f356b` дошёл до `ready` | `evidence/grade4.json` | +| Явный replay и запрет дубля ready | `python -m app.replay` из `student-tests/grade4.py` | первый replay `enqueued`, после `ready` второй replay `already_ready` | `evidence/grade4.json` | +| Таймаут обработки не роняет API | stall дольше `JOB_TIMEOUT_SECONDS` | job `ec75b92c-2534-4d96-adf5-4a92d252d389`: API оставался ready, затем replay довёл вложение до `ready` | `evidence/grade4.json` | +| Сохранность при рестарте томов | `python student-tests/persistence_hook.py` | `PASS: persistence preserved confirmed data and existing session.` Хук `scripts/restart.cmd` перезапускает postgres, redis, minio, rabbitmq, api, worker, relay, gc, lb и не удаляет тома | вывод команды в сессии 2026-11-20 | +| Остановка одной реплики API | `python student-tests/failover.py` | до отказа видны 2 instance id; 91 успешная запись, 1 сбой, первая успешная запись через 4,541 с; остановленная реплика перестала обслуживать. `PASS: failover preserved confirmed data and existing session.` | `evidence/failover.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +`student-tests/grade4.py` сам возвращает `infra/faults.json` к безопасному виду: все флаги выключены, `stall_seconds` и `timeout_seconds` равны 0. Так файл и лежит после прогона. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза для уровня 4: принятая задача доживает до одного и того же `ready`, если оборвать публикацию, процесс после записи объектов или зависший lease, потому что эффект идемпотентен по ключам S3, а намерение живёт в PostgreSQL. Это проверено сценариями выше, а не отдельным треком нагрузки. + +Короткий закрытый цикл отправки сообщений (не обработка изображений) нужен только как ориентир живого стенда. Прогрева отдельного не было, один прогон, клиентский overhead входит в задержку. + +Команда: `python tests/load.py --base-url http://127.0.0.1:8080 --duration 15 --concurrency 4 --output evidence/load.json` + +Результат: 15,604 с, 627 успешных POST, 0 ошибок, 40,183 успешных RPS, p50 80,3 мс, p95 198,0 мс, p99 302,7 мс. Один снимок `docker stats` в середине прогона (`evidence/docker-stats.txt`): API около 42% и 54% CPU при ~90 МиБ из 512 МиБ, HAProxy 17% CPU, PostgreSQL 14% CPU и 50 МиБ, Redis 8 МиБ. Worker и MinIO в этом профиле почти простаивали: сообщений с картинками цикл не шлёт. Снимок RabbitMQ (54% CPU, 186 МиБ из 384 МиБ) — одно мгновение, не профиль очереди картинок. Закрытый цикл занижает очередь ожидания и не доказывает предел throughput. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Worker остановлен, затем рестарт API и RabbitMQ с томами | 202 сохраняет задачу, content до ready даёт 409, после старта worker то же вложение ready | так для `5886b435-004d-4aed-b8f7-9a288c7bc181` | данные принятой загрузки сохранены | +| Relay не публикует строку outbox | после нового процесса relay та же задача доходит до ready | 1 неопубликованная строка, затем ready | ручное новое задание не создавалось | +| Процесс relay завершается после commit outbox и до publish | следующий процесс публикует ту же строку | `gap_injected=true`, ready | задача не потеряна | +| Процесс worker завершается после записи S3 и до ack | повторная доставка дописывает те же ключи и один набор вариантов | crash в логе, по одной строке каждого варианта, ready | дубля вариантов нет | +| Lease processing истёк | sweeper возвращает задачу в очередь | ready | подтверждённый результат не откатился | +| 5 временных ошибок | failed и `retry_exhausted`, replay снова ставит задачу, повторный replay ready ничего не делает | 5 строк реестра, `enqueued`, затем `already_ready` | дубль успешной задачи не исполнялся | +| Рестарт postgres, redis, minio, rabbitmq, api, worker, relay, gc, lb | сессия и подтверждённое сообщение живы | PASS persistence | тома не удалялись | +| `docker stop` одной реплики API | вторая продолжает писать, остановленная не отвечает | 2 instance id до отказа, восстановление записи за 4,541 с, 1 неуспешная попытка из цикла | подтверждённые сообщения и вложение целы | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд — один Docker-host. Потеря этой машины уносит PostgreSQL, Redis, MinIO и RabbitMQ вместе. Две реплики API переживут остановку одной из них и не переживают остановку HAProxy или базы. + +Доставка at-least-once. Идемпотентность держится на ключах объектов и условной записи `ready`, это не exactly-once. Повторная загрузка того же файла создаёт новое вложение: контракт не делает upload идемпотентным. + +`pipeline_version` в конфигурации всегда 1. Защита от перезаписи новым результатом старой версии в коде есть, автоматический прогон нескольких версий pipeline — материал трека 5 и здесь не измерялся. + +Replay — команда оператора в контейнере worker, не метод публичного API. Сравнение concurrency/prefetch и lag очереди на разных размерах изображений не проводилось: это трек 5. + +`/health/ready` не ждёт брокер. Пока relay жив, отставание outbox подхватывается само; если relay остановлен надолго, API продолжает принимать `202`, а обработка стоит до его старта. GC не слушает HTTP, поэтому healthcheck образа для него отключён в Compose: иначе контейнер помечался бы больным из-за проверки порта 8080. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..a3073faa --- /dev/null +++ b/compose.yaml @@ -0,0 +1,419 @@ +# Учебный стенд лабораторной 5. Секреты создаёт scripts/bootstrap, не этот файл. +# Две API-реплики: docker compose --env-file .env up -d --build --scale api=2 +name: mtusi-lab5 + +x-api-environment: &api-environment + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: lab + APP_ROLE: api + RUN_MIGRATIONS: "0" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SESSION_STORE: redis + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://minio:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + ATTACHMENT_GC_GRACE_SECONDS: ${ATTACHMENT_GC_GRACE_SECONDS:-900} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-45} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-90} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-0.4} + JOB_BACKOFF_CAP_SECONDS: ${JOB_BACKOFF_CAP_SECONDS:-8} + PIPELINE_VERSION: "1" + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + JPEG_QUALITY: "85" + WEBP_QUALITY: "80" + WEBP_METHOD: "6" + WORKER_IMAGE_MEMORY_BYTES: "268435456" + AWS_REQUEST_CHECKSUM_CALCULATION: when_required + AWS_RESPONSE_CHECKSUM_VALIDATION: when_required + DB_POOL_SIZE: ${DB_POOL_SIZE:-10} + DB_MAX_OVERFLOW: ${DB_MAX_OVERFLOW:-10} + DB_POOL_TIMEOUT_SECONDS: "5" + DB_CONNECT_TIMEOUT_SECONDS: "2" + DB_COMMAND_TIMEOUT_SECONDS: "5" + REDIS_SOCKET_TIMEOUT_SECONDS: "1" + REDIS_MAX_CONNECTIONS: "50" + +services: + postgres: + image: postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:5433:5432" + networks: + - lab + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + restart: unless-stopped + + redis: + image: redis:7.4.6-alpine@sha256:3b73847e72874be07e6657b129a94761662b79bc0f679273757d4218573b2a98 + command: + - redis-server + - --appendonly + - yes + - --appendfsync + - always + - --dir + - /data + volumes: + - redis_data:/data + ports: + - "127.0.0.1:6379:6379" + networks: + - lab + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 3s + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + restart: unless-stopped + + minio: + image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + entrypoint: ["/bin/sh", "/minio-entry.sh"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + MINIO_BROWSER: "off" + secrets: + - minio_root_user + - minio_root_password + volumes: + - minio_data:/data + - ./infra/minio-entry.sh:/minio-entry.sh:ro + ports: + - "127.0.0.1:9000:9000" + networks: + - lab + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"] + interval: 3s + timeout: 5s + retries: 20 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + restart: unless-stopped + + minio-init: + image: pgsty/mc:RELEASE.2026-11-12T00-00-00Z@sha256:b532278b7a3747dbab6fe45eebb40a653efa704e6758e24e9d9ec54fc96c1800 + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio-bootstrap.sh:/bootstrap.sh:ro + - ./infra/minio-policy.json:/policy.json:ro + networks: + - lab + depends_on: + minio: + condition: service_started + entrypoint: ["/bin/sh", "/bootstrap.sh"] + restart: "no" + + rabbitmq: + image: rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc + entrypoint: ["/bin/sh", "/entry.sh"] + secrets: + - rabbitmq_user + - rabbitmq_password + volumes: + - ./infra/rabbit-entry.sh:/entry.sh:ro + - rabbitmq_data:/var/lib/rabbitmq + networks: + - lab + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 10s + retries: 20 + start_period: 25s + deploy: + resources: + limits: + cpus: "0.50" + memory: 384M + restart: unless-stopped + + migrate: + build: . + environment: + APP_ROLE: migrate + RUN_MIGRATIONS: "1" + APP_ENV: lab + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + networks: + - lab + depends_on: + postgres: + condition: service_healthy + command: ["python", "-m", "app.migrate"] + restart: "no" + deploy: + replicas: 1 + resources: + limits: + cpus: "0.25" + memory: 128M + + api: + build: . + environment: *api-environment + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + networks: + - lab + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import os,urllib.request; p=os.environ.get('APP_PORT','8080'); urllib.request.urlopen('http://127.0.0.1:%s/health/ready'%p, timeout=2).read()", + ] + interval: 2s + timeout: 2s + retries: 16 + start_period: 20s + stop_grace_period: 8s + stop_signal: SIGTERM + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.50" + memory: 512M + restart: unless-stopped + + gc: + build: . + environment: + <<: *api-environment + APP_ROLE: gc + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + networks: + - lab + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + command: ["python", "-m", "app.gc"] + healthcheck: + disable: true + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + + worker: + build: . + environment: + <<: *api-environment + APP_ROLE: worker + BROKER_URL_FILE: /run/secrets/broker_url + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + FAULTS_FILE: /faults/faults.json + DB_POOL_SIZE: "4" + DB_MAX_OVERFLOW: "2" + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + volumes: + - ./infra/faults.json:/faults/faults.json:ro + networks: + - lab + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + command: ["python", "-m", "app.worker"] + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import os,time,sys; p='/tmp/worker-heartbeat'; sys.exit(0 if os.path.exists(p) and time.time()-os.path.getmtime(p)<30 else 1)", + ] + interval: 5s + timeout: 3s + retries: 12 + start_period: 25s + stop_grace_period: 10s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.00" + memory: 1024M + + relay: + build: . + environment: + APP_ROLE: relay + APP_ENV: lab + RUN_MIGRATIONS: "0" + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + FAULTS_FILE: /faults/faults.json + DB_POOL_SIZE: "2" + DB_MAX_OVERFLOW: "2" + DB_POOL_TIMEOUT_SECONDS: "5" + DB_CONNECT_TIMEOUT_SECONDS: "2" + DB_COMMAND_TIMEOUT_SECONDS: "5" + secrets: + - database_url + - broker_url + volumes: + - ./infra/faults.json:/faults/faults.json:ro + networks: + - lab + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + command: ["python", "-m", "app.relay"] + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import os,time,sys; p='/tmp/relay-heartbeat'; sys.exit(0 if os.path.exists(p) and time.time()-os.path.getmtime(p)<30 else 1)", + ] + interval: 5s + timeout: 3s + retries: 12 + start_period: 25s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 256M + + lb: + image: haproxy:3.2-bookworm@sha256:baecd003ae5c2376e58d04a5b2ffc7d82d30e9bebe2869f76222b819f719dee3 + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + - "[::1]:8080:8080" + networks: + - lab + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 5s + timeout: 2s + retries: 5 + start_period: 3s + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + restart: unless-stopped + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + rabbitmq_user: + file: ./secrets/rabbitmq_user + rabbitmq_password: + file: ./secrets/rabbitmq_password + broker_url: + file: ./secrets/broker_url + +volumes: + postgres_data: + redis_data: + minio_data: + rabbitmq_data: + +networks: + lab: diff --git a/evidence/docker-stats.txt b/evidence/docker-stats.txt new file mode 100644 index 00000000..a799c45a --- /dev/null +++ b/evidence/docker-stats.txt @@ -0,0 +1,10 @@ +mtusi-lab5-gc-1 cpu=0.00% mem=78.45MiB / 128MiB +mtusi-lab5-worker-1 cpu=0.33% mem=94.17MiB / 1GiB +mtusi-lab5-api-1 cpu=41.68% mem=87.46MiB / 512MiB +mtusi-lab5-api-2 cpu=53.60% mem=96.45MiB / 512MiB +mtusi-lab5-relay-1 cpu=0.94% mem=59.28MiB / 256MiB +mtusi-lab5-lb-1 cpu=17.22% mem=24.2MiB / 64MiB +mtusi-lab5-postgres-1 cpu=13.90% mem=49.89MiB / 512MiB +mtusi-lab5-rabbitmq-1 cpu=54.10% mem=185.6MiB / 384MiB +mtusi-lab5-minio-1 cpu=0.40% mem=76.19MiB / 256MiB +mtusi-lab5-redis-1 cpu=1.23% mem=8.191MiB / 128MiB diff --git a/evidence/failover.json b/evidence/failover.json new file mode 100644 index 00000000..7db24702 --- /dev/null +++ b/evidence/failover.json @@ -0,0 +1,10 @@ +{ + "date": "2026-11-20T16:50:00Z", + "command": "python student-tests/failover.py", + "hook": "scripts/stop-one.cmd", + "observed_instances_before": 2, + "successful_writes": 91, + "failed_attempts": 1, + "first_recovery_seconds": 4.541, + "result": "PASS: failover preserved confirmed data and existing session." +} diff --git a/evidence/grade4.json b/evidence/grade4.json new file mode 100644 index 00000000..a4212819 --- /dev/null +++ b/evidence/grade4.json @@ -0,0 +1,61 @@ +{ + "date": "2026-11-20T16:38:02Z", + "happy_path": { + "attachment_id": "a813b0a0-2bab-404f-aa5f-8850b46db20c", + "job_id": "8277933e-15c9-4347-af58-cfa21c02ee71", + "status": "ready" + }, + "worker_stopped": { + "attachment_id": "5886b435-004d-4aed-b8f7-9a288c7bc181", + "job_id": "c6d3c7ff-a181-41f9-a945-e194054b0637", + "content_before_worker": 409, + "status_after_recovery": "ready", + "rabbit_restarted": true, + "api_restarted": true + }, + "outbox_gap": { + "attachment_id": "35000ce9-e4d5-4774-a7de-6b717a4cb208", + "job_id": "13488d1a-9f1f-4c30-a573-8dcf36d1beb7", + "unpublished_before_restart": 1, + "status": "ready" + }, + "relay_crash": { + "attachment_id": "3f83494d-5756-42ce-b457-29af8bc3e788", + "job_id": "5104d370-c0d6-48f1-916a-6611afb8958f", + "gap_injected": true, + "log_seen": true, + "status": "ready" + }, + "crash_before_ack": { + "attachment_id": "a5a24455-f783-4511-90f9-0b17dd43aaf5", + "job_id": "0fdaa8f2-054b-4aa0-a273-e63a425ce8c6", + "log_seen": true, + "status": "ready", + "variant_rows": { + "crop": 1, + "optimized": 1, + "thumbnail": 1 + } + }, + "expired_lease": { + "attachment_id": "be34abc1-5932-468e-aa2a-d4ebcd6e41ac", + "job_id": "6ebf2c9f-090a-4637-94b2-2eb0a87f356b", + "status": "ready" + }, + "retry_and_replay": { + "attachment_id": "d1b86332-5eba-4e3b-a69c-d0c97400fc21", + "job_id": "d118acc1-f4cb-4a6e-8fd3-7f7f74d5cbd3", + "error_code": "retry_exhausted", + "failure_rows": 5, + "first_replay": "enqueued", + "second_replay": "already_ready", + "status": "ready" + }, + "timeout": { + "attachment_id": "2a4937cf-0108-40f3-9544-ec42fee9c135", + "job_id": "ec75b92c-2534-4d96-adf5-4a92d252d389", + "error_code": "retry_exhausted", + "status": "ready", + "api_ready_during_stall": true + } +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..8335b2f7 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.604, + "concurrency": 4, + "successful_requests": 627, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 40.183, + "successful_latency_ms": { + "p50": 80.292, + "p95": 197.958, + "p99": 302.695 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..6f19bcb4 Binary files /dev/null and b/evidence/smoke.txt differ diff --git a/infra/faults.json b/infra/faults.json new file mode 100644 index 00000000..1e00abc7 --- /dev/null +++ b/infra/faults.json @@ -0,0 +1 @@ +{"inject_transient": false, "crash_after_s3": false, "relay_exit_before_publish": false, "stall_seconds": 0, "timeout_seconds": 0} \ No newline at end of file diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..4f68a7e9 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,39 @@ +global + log stdout format raw local0 + maxconn 256 + +defaults + log global + mode http + option httplog + option dontlognull + option redispatch + timeout connect 3s + timeout client 60s + timeout server 60s + timeout http-request 30s + retries 2 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + hold other 5s + hold refused 5s + hold nx 5s + hold timeout 5s + hold obsolete 5s + +frontend public + bind :8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fall 2 rise 2 resolvers docker init-addr last,libc,none resolve-prefer ipv4 + server-template api 8 api:8080 check diff --git a/infra/minio-bootstrap.sh b/infra/minio-bootstrap.sh new file mode 100644 index 00000000..31c3f95f --- /dev/null +++ b/infra/minio-bootstrap.sh @@ -0,0 +1,36 @@ +#!/bin/sh +set -eu + +ROOT_USER=$(cat /run/secrets/minio_root_user) +ROOT_PASS=$(cat /run/secrets/minio_root_password) +API_KEY=$(cat /run/secrets/s3_access_key_id) +API_SECRET=$(cat /run/secrets/s3_secret_access_key) +BUCKET=${S3_BUCKET:-messenger-attachments} + +i=0 +while [ "$i" -lt 40 ]; do + if mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASS" >/dev/null 2>&1; then + break + fi + i=$((i + 1)) + sleep 1 +done +mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASS" + +mc mb --ignore-existing "local/${BUCKET}" +mc anonymous set none "local/${BUCKET}" + +if ! mc admin user info local "$API_KEY" >/dev/null 2>&1; then + mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null +else + mc admin user add local "$API_KEY" "$API_SECRET" >/dev/null 2>&1 || true +fi + +if ! mc admin policy info local messenger-objects >/dev/null 2>&1; then + mc admin policy create local messenger-objects /policy.json +else + mc admin policy create local messenger-objects /policy.json >/dev/null 2>&1 || true +fi +mc admin policy attach local messenger-objects --user "$API_KEY" >/dev/null 2>&1 || true + +echo "minio bootstrap complete bucket=${BUCKET}" diff --git a/infra/minio-entry.sh b/infra/minio-entry.sh new file mode 100644 index 00000000..bbfc6495 --- /dev/null +++ b/infra/minio-entry.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)" +export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +exec /usr/bin/docker-entrypoint.sh server /data --address :9000 --console-address :9001 diff --git a/infra/minio-policy.json b/infra/minio-policy.json new file mode 100644 index 00000000..2d45154e --- /dev/null +++ b/infra/minio-policy.json @@ -0,0 +1,21 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ApiObjectAccess", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject", + "s3:AbortMultipartUpload", + "s3:ListBucket", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::messenger-attachments", + "arn:aws:s3:::messenger-attachments/*" + ] + } + ] +} diff --git a/infra/rabbit-entry.sh b/infra/rabbit-entry.sh new file mode 100644 index 00000000..eb5087ce --- /dev/null +++ b/infra/rabbit-entry.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +export RABBITMQ_DEFAULT_USER="$(cat /run/secrets/rabbitmq_user)" +export RABBITMQ_DEFAULT_PASS="$(cat /run/secrets/rabbitmq_password)" +exec docker-entrypoint.sh rabbitmq-server diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..b9e03697 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,52 @@ +-- Пользователи, чаты, состав и сообщения. Пароли и тексты сообщений +-- не пишутся приложением в логи. password_hash — только Argon2id. +-- Сессии в этой схеме нет: их источник истины — Redis (см. compose.yaml). + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT uq_users_username UNIQUE (username), + CONSTRAINT ck_users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT ck_users_display_name_len CHECK ( + char_length(btrim(display_name)) >= 1 AND char_length(display_name) <= 100 + ), + CONSTRAINT ck_users_password_hash_len CHECK (char_length(password_hash) >= 20) +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT ck_chats_title CHECK (char_length(btrim(title)) >= 1 AND char_length(title) <= 100) +); + +-- Состав чата пишется в той же транзакции, что и строка chats. +-- Откат до COMMIT не оставляет чат без участников. +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id), + user_id UUID NOT NULL REFERENCES users (id), + ordinal SMALLINT NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT ck_chat_members_ordinal CHECK (ordinal >= 0 AND ordinal < 100) +); + +-- Список чатов пользователя: фильтр по user_id, затем порядок (created_at, id) чата. +CREATE INDEX ix_chat_members_user ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + sender_id UUID NOT NULL REFERENCES users (id), + body TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT ck_messages_text CHECK (char_length(btrim(body)) >= 1 AND char_length(body) <= 4000) +); + +-- История: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id LIMIT $4. +-- Ведущий chat_id отсекает чужие чаты, пара (created_at, id) продолжает +-- keyset с cursor и не читает предыдущие строки этой истории. +CREATE INDEX ix_messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..f7cbcf54 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,22 @@ +-- Общая идемпотентность POST сообщения для всех API-реплик. +-- Результат пишется в той же транзакции, что и messages: 5xx не сохраняется. + +CREATE TABLE idempotency_records ( + user_id UUID NOT NULL REFERENCES users (id), + method TEXT NOT NULL, + request_path TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status_code SMALLINT NOT NULL, + message_id UUID NOT NULL REFERENCES messages (id), + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, method, request_path, idempotency_key), + CONSTRAINT ck_idem_method CHECK (method = 'POST'), + CONSTRAINT ck_idem_status CHECK (status_code = 201), + CONSTRAINT ck_idem_key CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT ck_idem_path CHECK (char_length(request_path) BETWEEN 1 AND 300), + CONSTRAINT ck_idem_hash CHECK (char_length(request_hash) = 64) +); + +CREATE INDEX ix_idempotency_message ON idempotency_records (message_id); +CREATE INDEX ix_idempotency_created ON idempotency_records (created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..41ef5160 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,39 @@ +-- Вложения: метаданные в PostgreSQL, байты в S3. Пустой text сообщения +-- допускается только вместе с attachment_ids (проверяет приложение). + +ALTER TABLE messages DROP CONSTRAINT ck_messages_text; +ALTER TABLE messages ADD CONSTRAINT ck_messages_text CHECK (char_length(body) <= 4000); + +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + owner_id UUID NOT NULL REFERENCES users (id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL, + status TEXT NOT NULL, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + CONSTRAINT uq_attachments_object_key UNIQUE (object_key), + CONSTRAINT ck_attachments_filename CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT ck_attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT ck_attachments_size CHECK (size_bytes >= 1 AND size_bytes <= 10485760), + CONSTRAINT ck_attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT ck_attachments_status CHECK (status = 'ready'), + CONSTRAINT ck_attachments_error CHECK (error_code IS NULL) +); + +CREATE INDEX ix_attachments_chat ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages (id), + attachment_id UUID NOT NULL REFERENCES attachments (id), + ordinal SMALLINT NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT ck_message_attachments_ordinal CHECK (ordinal >= 0 AND ordinal < 10) +); + +CREATE INDEX ix_message_attachments_attachment ON message_attachments (attachment_id); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 00000000..53e4661b --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,94 @@ +-- Lab5: асинхронные состояния, варианты, transactional outbox и реестр отказов. +-- Сообщение брокера хранит только идентификаторы. Байты остаются в S3. + +ALTER TABLE attachments DROP CONSTRAINT ck_attachments_status; + +ALTER TABLE attachments ADD CONSTRAINT ck_attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')); + +ALTER TABLE attachments DROP CONSTRAINT ck_attachments_error; + +ALTER TABLE attachments ADD CONSTRAINT ck_attachments_error CHECK ((status = 'failed' AND error_code IS NOT NULL AND char_length(error_code) BETWEEN 1 AND 64) OR (status <> 'failed' AND error_code IS NULL)); + +ALTER TABLE attachments ADD COLUMN staging_key TEXT; + +ALTER TABLE attachments ADD CONSTRAINT ck_attachments_staging CHECK (staging_key IS NULL OR char_length(staging_key) BETWEEN 8 AND 300); + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments (id), + name TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + width INTEGER NOT NULL, + height INTEGER NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL, + pipeline_version INTEGER NOT NULL, + PRIMARY KEY (attachment_id, name), + CONSTRAINT uq_attachment_variants_object_key UNIQUE (object_key), + CONSTRAINT ck_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT ck_variants_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + CONSTRAINT ck_variants_size CHECK (size_bytes >= 1), + CONSTRAINT ck_variants_dim CHECK (width >= 1 AND height >= 1), + CONSTRAINT ck_variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT ck_variants_pipeline CHECK (pipeline_version >= 1) +); + +CREATE TABLE attachment_jobs ( + job_id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments (id), + pipeline_version INTEGER NOT NULL, + status TEXT NOT NULL, + attempt INTEGER NOT NULL, + max_attempts INTEGER NOT NULL, + lease_until TIMESTAMPTZ, + lease_owner TEXT, + next_attempt_at TIMESTAMPTZ, + error_code TEXT, + staging_key TEXT NOT NULL, + objects_written_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + CONSTRAINT uq_attachment_jobs_effect UNIQUE (attachment_id, pipeline_version), + CONSTRAINT ck_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT ck_jobs_attempt CHECK (attempt >= 0 AND attempt <= max_attempts), + CONSTRAINT ck_jobs_max CHECK (max_attempts BETWEEN 1 AND 20), + CONSTRAINT ck_jobs_pipeline CHECK (pipeline_version >= 1), + CONSTRAINT ck_jobs_staging CHECK (char_length(staging_key) BETWEEN 8 AND 300) +); + +CREATE INDEX ix_jobs_lease ON attachment_jobs (status, lease_until); + +CREATE TABLE job_outbox ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs (job_id), + payload TEXT NOT NULL, + available_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + published_at TIMESTAMPTZ, + publish_attempts INTEGER NOT NULL, + gap_injected BOOLEAN NOT NULL DEFAULT FALSE, + CONSTRAINT ck_outbox_attempts CHECK (publish_attempts >= 0) +); + +CREATE INDEX ix_outbox_pending ON job_outbox (available_at, created_at); + +CREATE TABLE job_failures ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs (job_id), + attachment_id UUID NOT NULL, + attempt INTEGER NOT NULL, + error_code TEXT NOT NULL, + permanent BOOLEAN NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT ck_job_failures_code CHECK (char_length(error_code) BETWEEN 1 AND 64) +); + +CREATE INDEX ix_job_failures_job ON job_failures (job_id, created_at); + +CREATE TABLE job_replays ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs (job_id), + outcome TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT ck_replays_outcome CHECK (outcome IN ('enqueued', 'already_ready', 'in_progress')) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..ee7ebdb9 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +sqlalchemy==2.0.40 +asyncpg==0.30.0 +redis==5.2.1 +pydantic==2.11.3 +boto3==1.38.13 +pillow==11.2.1 +argon2-cffi==23.1.0 +aio-pika==9.5.5 diff --git a/scripts/bootstrap.ps1 b/scripts/bootstrap.ps1 new file mode 100644 index 00000000..238dabee --- /dev/null +++ b/scripts/bootstrap.ps1 @@ -0,0 +1,52 @@ +$ErrorActionPreference = "Stop" +$Root = Split-Path -Parent $PSScriptRoot +$Secrets = Join-Path $Root "secrets" +New-Item -ItemType Directory -Force -Path $Secrets | Out-Null +$utf8 = New-Object System.Text.UTF8Encoding $false + +function Write-Secret([string]$Name, [string]$Value) { + $path = Join-Path $Secrets $Name + if (-not (Test-Path $path)) { + [System.IO.File]::WriteAllText($path, $Value, $utf8) + } +} + +function Read-Secret([string]$Name) { + $path = Join-Path $Secrets $Name + return ([System.IO.File]::ReadAllText($path, $utf8)).Trim() +} + +function New-Hex([int]$Bytes) { + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + try { + $buf = New-Object byte[] $Bytes + $rng.GetBytes($buf) + } finally { + $rng.Dispose() + } + return ([System.BitConverter]::ToString($buf) -replace "-", "").ToLower() +} + +Write-Secret "postgres_password" (New-Hex 18) +Write-Secret "minio_root_user" ("lab" + (New-Hex 4)) +Write-Secret "minio_root_password" (New-Hex 18) +Write-Secret "s3_access_key_id" ("s3" + (New-Hex 6)) +Write-Secret "s3_secret_access_key" (New-Hex 24) +Write-Secret "rabbitmq_user" "messenger" +Write-Secret "rabbitmq_password" (New-Hex 18) + +$pg = Read-Secret "postgres_password" +$rabbitUser = Read-Secret "rabbitmq_user" +$rabbitPassword = Read-Secret "rabbitmq_password" +Write-Secret "database_url" ("postgresql+asyncpg://messenger:{0}@postgres:5432/messenger" -f $pg) +Write-Secret "host_database_url" ("postgresql+asyncpg://messenger:{0}@127.0.0.1:5433/messenger" -f $pg) +Write-Secret "cache_url" "redis://redis:6379/0" +Write-Secret "host_cache_url" "redis://127.0.0.1:6379/0" +Write-Secret "host_s3_endpoint" "http://127.0.0.1:9000" +Write-Secret "broker_url" ("amqp://{0}:{1}@rabbitmq:5672/" -f $rabbitUser, $rabbitPassword) + +$envFile = Join-Path $Root ".env" +if (-not (Test-Path $envFile)) { + Copy-Item (Join-Path $Root ".env.example") $envFile +} +Write-Output "local secrets are ready" diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100644 index 00000000..00d69c0e --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,45 @@ +#!/bin/sh +set -eu +ROOT=$(CDPATH= cd -- "$(dirname "$0")/.." && pwd) +SECRETS="$ROOT/secrets" +mkdir -p "$SECRETS" +umask 077 + +write_secret() { + name=$1 + value=$2 + if [ ! -f "$SECRETS/$name" ]; then + printf '%s' "$value" > "$SECRETS/$name" + fi +} + +read_secret() { + tr -d '\r\n' < "$SECRETS/$1" +} + +hex_bytes() { + od -An -N "$1" -tx1 /dev/urandom | tr -d ' \n' +} + +write_secret postgres_password "$(hex_bytes 18)" +write_secret minio_root_user "lab$(hex_bytes 4)" +write_secret minio_root_password "$(hex_bytes 18)" +write_secret s3_access_key_id "s3$(hex_bytes 6)" +write_secret s3_secret_access_key "$(hex_bytes 24)" +write_secret rabbitmq_user messenger +write_secret rabbitmq_password "$(hex_bytes 18)" + +pg=$(read_secret postgres_password) +rabbit_user=$(read_secret rabbitmq_user) +rabbit_password=$(read_secret rabbitmq_password) +write_secret database_url "postgresql+asyncpg://messenger:${pg}@postgres:5432/messenger" +write_secret host_database_url "postgresql+asyncpg://messenger:${pg}@127.0.0.1:5433/messenger" +write_secret cache_url "redis://redis:6379/0" +write_secret host_cache_url "redis://127.0.0.1:6379/0" +write_secret host_s3_endpoint "http://127.0.0.1:9000" +write_secret broker_url "amqp://${rabbit_user}:${rabbit_password}@rabbitmq:5672/" + +if [ ! -f "$ROOT/.env" ]; then + cp "$ROOT/.env.example" "$ROOT/.env" +fi +echo "local secrets are ready" diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..221d34f6 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0\.." +docker compose --env-file .env restart postgres redis minio rabbitmq api worker relay gc lb +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..165925c3 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres redis minio rabbitmq api worker relay gc lb diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..5131d3aa --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,3 @@ +@echo off +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.ps1 b/scripts/stop-one.ps1 new file mode 100644 index 00000000..be833329 --- /dev/null +++ b/scripts/stop-one.ps1 @@ -0,0 +1,23 @@ +$ErrorActionPreference = "Stop" +$target = $env:TARGET_INSTANCE_ID +if (-not $target) { + Write-Error "TARGET_INSTANCE_ID is required" + exit 1 +} +$ids = docker ps -q --filter label=com.docker.compose.service=api --filter label=com.docker.compose.project=mtusi-lab5 +$found = "" +foreach ($id in $ids) { + if (-not $id) { continue } + $hostName = docker inspect -f "{{.Config.Hostname}}" $id + $short = $id.Substring(0, [Math]::Min(12, $id.Length)) + if ($hostName -eq $target -or $short -eq $target -or $id -eq $target) { + $found = $id + break + } +} +if (-not $found) { + Write-Error "api replica not found" + exit 1 +} +docker stop -t 5 $found | Out-Null +if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..9b3148d8 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu +target="${TARGET_INSTANCE_ID:?test provides target API instance alias}" +found="" +for id in $(docker ps -q --filter label=com.docker.compose.service=api --filter label=com.docker.compose.project=mtusi-lab5); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + short=$(printf '%s' "$id" | cut -c1-12) + if [ "$host" = "$target" ] || [ "$short" = "$target" ] || [ "$id" = "$target" ]; then + found=$id + break + fi +done +if [ -z "$found" ]; then + echo "api replica not found" >&2 + exit 1 +fi +docker stop -t 5 "$found" >/dev/null diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/__main__.py b/src/app/__main__.py new file mode 100644 index 00000000..9472a36b --- /dev/null +++ b/src/app/__main__.py @@ -0,0 +1,40 @@ +import logging +import sys + +import uvicorn + +from app.config import load_settings + + +def main() -> None: + settings = load_settings() + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + force=True, + ) + logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING) + logging.getLogger("sqlalchemy.pool").setLevel(logging.WARNING) + logging.getLogger("asyncpg").setLevel(logging.WARNING) + logging.getLogger("app").info( + "starting host=%s port=%s env=%s instance_id=%s", + settings.host, + settings.port, + settings.env, + settings.instance_id, + ) + uvicorn.run( + "app.main:app", + host=settings.host, + port=settings.port, + workers=1, + access_log=False, + timeout_keep_alive=5, + timeout_graceful_shutdown=5, + log_config=None, + ) + + +if __name__ == "__main__": + main() diff --git a/src/app/api/__init__.py b/src/app/api/__init__.py new file mode 100644 index 00000000..a2b23009 --- /dev/null +++ b/src/app/api/__init__.py @@ -0,0 +1,14 @@ +from fastapi import APIRouter + +from app.api.attachments import router as attachments_router +from app.api.auth import router as auth_router +from app.api.chats import router as chats_router +from app.api.messages import router as messages_router +from app.api.users import router as users_router + +api_router = APIRouter(prefix="/api/v1") +api_router.include_router(users_router) +api_router.include_router(auth_router) +api_router.include_router(chats_router) +api_router.include_router(messages_router) +api_router.include_router(attachments_router) diff --git a/src/app/api/attachments.py b/src/app/api/attachments.py new file mode 100644 index 00000000..8a0769ec --- /dev/null +++ b/src/app/api/attachments.py @@ -0,0 +1,71 @@ +from uuid import UUID + +from fastapi import APIRouter, Depends, Query, Request +from fastapi.responses import Response + +from app.crud.attachments import create_attachment, metadata_for_member, open_content +from app.deps import get_current_user +from app.errors import AppError +from app.present import PublicUser +from app.runtime import require_settings, require_store +from app.schemas.attachment import AttachmentOut + +router = APIRouter() + + +async def read_limited_body(request: Request) -> bytes: + limit = require_settings().max_upload_bytes + raw_length = request.headers.get("content-length") + if raw_length is not None: + try: + announced = int(raw_length) + except ValueError: + raise AppError(400, "invalid_request", "Некорректный запрос") from None + if announced > limit: + raise AppError(413, "payload_too_large", "Файл слишком большой") + chunks: list[bytes] = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > limit: + raise AppError(413, "payload_too_large", "Файл слишком большой") + if chunk: + chunks.append(chunk) + return b"".join(chunks) + + +@router.post("/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentOut) +async def upload_attachment( + chat_id: UUID, + request: Request, + filename: str = Query(min_length=1, max_length=255), + user: PublicUser = Depends(get_current_user), +) -> dict: + payload = await read_limited_body(request) + created = await create_attachment( + chat_id, + user.id, + filename, + request.headers.get("content-type"), + payload, + ) + return created.to_dict() + + +@router.get("/attachments/{attachment_id}", response_model=AttachmentOut) +async def get_attachment( + attachment_id: UUID, + user: PublicUser = Depends(get_current_user), +) -> dict: + return (await metadata_for_member(attachment_id, user.id)).to_dict() + + +@router.get("/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: UUID, + user: PublicUser = Depends(get_current_user), + variant: str = Query(default="original"), +) -> Response: + key, media_type = await open_content(attachment_id, user.id, variant) + payload = await require_store().get(key) + return Response(content=payload, media_type=media_type) diff --git a/src/app/api/auth.py b/src/app/api/auth.py new file mode 100644 index 00000000..b41dbb20 --- /dev/null +++ b/src/app/api/auth.py @@ -0,0 +1,37 @@ +from fastapi import APIRouter, Depends, Header +from starlette.responses import Response + +from app.crud.users import authenticate +from app.deps import BASIC_CHALLENGE, BEARER_CHALLENGE, require_token, unauthorized +from app.schemas.session import SessionOut +from app.security import parse_basic, verify_password +from app.sessions import create_session, revoke_session +from app.timeutil import format_time + +router = APIRouter() + + +@router.post("/auth/sessions", status_code=201, response_model=SessionOut) +async def create_session_route(authorization: str | None = Header(default=None)) -> dict: + parsed = parse_basic(authorization or "") + if parsed is None: + verify_password("invalid-basic-auth", None) + raise unauthorized(BASIC_CHALLENGE) + username, password = parsed + user = await authenticate(username, password) + if user is None: + raise unauthorized(BASIC_CHALLENGE) + token, expires_at = await create_session(user.id) + return { + "token": token, + "token_type": "Bearer", + "expires_at": format_time(expires_at), + "user": user.to_dict(), + } + + +@router.delete("/auth/sessions/current", status_code=204) +async def revoke_current_session(token: str = Depends(require_token)) -> Response: + if not await revoke_session(token): + raise unauthorized(BEARER_CHALLENGE) + return Response(status_code=204) diff --git a/src/app/api/chats.py b/src/app/api/chats.py new file mode 100644 index 00000000..9015540d --- /dev/null +++ b/src/app/api/chats.py @@ -0,0 +1,31 @@ +from uuid import UUID + +from fastapi import APIRouter, Depends, Query + +from app.crud.chats import chat_for_member, chats_for_user, create_chat as save_chat +from app.deps import get_current_user +from app.present import PublicUser +from app.schemas.chat import ChatOut, ChatPage, CreateChat + +router = APIRouter() + + +@router.post("/chats", status_code=201, response_model=ChatOut) +async def create_chat(body: CreateChat, user: PublicUser = Depends(get_current_user)) -> dict: + chat = await save_chat(user.id, body.title, tuple(body.member_ids)) + return chat.to_dict() + + +@router.get("/chats", response_model=ChatPage) +async def list_chats( + user: PublicUser = Depends(get_current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> dict: + return await chats_for_user(user.id, limit, cursor) + + +@router.get("/chats/{chat_id}", response_model=ChatOut) +async def get_chat(chat_id: UUID, user: PublicUser = Depends(get_current_user)) -> dict: + chat = await chat_for_member(chat_id, user.id) + return chat.to_dict() diff --git a/src/app/api/health.py b/src/app/api/health.py new file mode 100644 index 00000000..2eff6daf --- /dev/null +++ b/src/app/api/health.py @@ -0,0 +1,43 @@ +import logging + +from fastapi import APIRouter, Request +from pydantic import BaseModel +from sqlalchemy import text + +from app.db import is_dependency_error +from app.errors import AppError + +router = APIRouter() +logger = logging.getLogger("app.health") + + +class HealthOut(BaseModel): + status: str + + +@router.get("/health/live", response_model=HealthOut) +async def live() -> dict: + return {"status": "ok"} + + +@router.get("/health/ready", response_model=HealthOut) +async def ready(request: Request) -> dict: + if not getattr(request.app.state, "accepting", False): + raise AppError(503, "unavailable", "Сервер не готов") + engine = getattr(request.app.state, "engine", None) + redis = getattr(request.app.state, "redis", None) + store = getattr(request.app.state, "store", None) + if engine is None or redis is None or store is None: + raise AppError(503, "unavailable", "Сервис временно недоступен") + try: + async with engine.connect() as conn: + await conn.execute(text("SELECT 1")) + await redis.ping() + await store.ping() + except AppError: + raise + except Exception as exc: + if not is_dependency_error(exc): + logger.warning("readiness check failed type=%s", type(exc).__name__) + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + return {"status": "ok"} diff --git a/src/app/api/messages.py b/src/app/api/messages.py new file mode 100644 index 00000000..3c2ac431 --- /dev/null +++ b/src/app/api/messages.py @@ -0,0 +1,43 @@ +import re +from uuid import UUID + +from fastapi import APIRouter, Depends, Query, Request + +from app.crud.messages import create_message as save_message +from app.crud.messages import messages_for_member +from app.deps import get_current_user +from app.errors import AppError +from app.present import PublicUser +from app.schemas.message import CreateMessage, MessageOut, MessagePage + +router = APIRouter() +_IDEMPOTENCY_KEY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def require_idempotency_key(request: Request) -> str: + value = request.headers.get("idempotency-key") + if value is None or _IDEMPOTENCY_KEY.fullmatch(value) is None: + raise AppError(400, "invalid_request", "Некорректный Idempotency-Key") + return value + + +@router.post("/chats/{chat_id}/messages", status_code=201, response_model=MessageOut) +async def create_message( + chat_id: UUID, + body: CreateMessage, + request: Request, + user: PublicUser = Depends(get_current_user), +) -> dict: + key = require_idempotency_key(request) + message = await save_message(chat_id, user.id, body.text, key, body.attachment_ids) + return message.to_dict() + + +@router.get("/chats/{chat_id}/messages", response_model=MessagePage) +async def list_messages( + chat_id: UUID, + user: PublicUser = Depends(get_current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1), +) -> dict: + return await messages_for_member(chat_id, user.id, limit, cursor) diff --git a/src/app/api/users.py b/src/app/api/users.py new file mode 100644 index 00000000..72a90b27 --- /dev/null +++ b/src/app/api/users.py @@ -0,0 +1,19 @@ +from fastapi import APIRouter, Depends + +from app.crud.users import create_user +from app.deps import get_current_user +from app.present import PublicUser +from app.schemas.user import CreateUser, UserOut + +router = APIRouter() + + +@router.post("/users", status_code=201, response_model=UserOut) +async def register(body: CreateUser) -> dict: + user = await create_user(body.username, body.display_name, body.password) + return user.to_dict() + + +@router.get("/users/me", response_model=UserOut) +async def me(user: PublicUser = Depends(get_current_user)) -> dict: + return user.to_dict() diff --git a/src/app/broker.py b/src/app/broker.py new file mode 100644 index 00000000..4028ba18 --- /dev/null +++ b/src/app/broker.py @@ -0,0 +1,43 @@ +import logging + +import aio_pika + +logger = logging.getLogger("app.broker") + +EXCHANGE = "attachments" +WORK_QUEUE = "attachment.jobs" +DLQ = "attachment.jobs.dlq" +WORK_KEY = "attachment.process" +DLQ_KEY = "attachment.failed" + + +async def connect(url: str) -> aio_pika.abc.AbstractRobustConnection: + return await aio_pika.connect_robust(url, timeout=10) + + +async def declare(channel: aio_pika.abc.AbstractChannel) -> tuple[aio_pika.abc.AbstractExchange, aio_pika.abc.AbstractQueue]: + exchange = await channel.declare_exchange(EXCHANGE, aio_pika.ExchangeType.TOPIC, durable=True) + work = await channel.declare_queue( + WORK_QUEUE, + durable=True, + arguments={ + "x-dead-letter-exchange": EXCHANGE, + "x-dead-letter-routing-key": DLQ_KEY, + }, + ) + await work.bind(exchange, routing_key=WORK_KEY) + dead = await channel.declare_queue(DLQ, durable=True) + await dead.bind(exchange, routing_key=DLQ_KEY) + return exchange, work + + +def job_message(payload: str) -> aio_pika.Message: + return aio_pika.Message( + body=payload.encode("utf-8"), + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + content_type="application/json", + ) + + +async def publish(exchange: aio_pika.abc.AbstractExchange, payload: str, routing_key: str) -> None: + await exchange.publish(job_message(payload), routing_key=routing_key, mandatory=True) diff --git a/src/app/config.py b/src/app/config.py new file mode 100644 index 00000000..b3327454 --- /dev/null +++ b/src/app/config.py @@ -0,0 +1,194 @@ +import os +import socket +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + env: str + app_role: str + instance_id: str + run_migrations: bool + session_ttl_seconds: int + idempotency_ttl_seconds: int + database_url: str + cache_url: str + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + attachment_gc_grace_seconds: int + migrations_dir: Path + db_pool_size: int + db_max_overflow: int + db_pool_timeout_seconds: float + db_connect_timeout_seconds: float + db_command_timeout_seconds: float + redis_socket_timeout_seconds: float + redis_max_connections: int + broker_url: str + worker_concurrency: int + job_max_attempts: int + job_timeout_seconds: int + job_lease_seconds: int + job_backoff_base_seconds: float + job_backoff_cap_seconds: float + pipeline_version: int + staging_retention_seconds: int + worker_image_memory_bytes: int + jpeg_quality: int + webp_quality: int + webp_method: int + faults_file: str + + +def _int(name: str, default: int, low: int, high: int) -> int: + raw = os.environ.get(name, str(default)).strip() + try: + value = int(raw) + except ValueError as exc: + raise SystemExit(f"{name} must be an integer") from exc + if not low <= value <= high: + raise SystemExit(f"{name} is out of range") + return value + + +def _float(name: str, default: float, low: float, high: float) -> float: + raw = os.environ.get(name, str(default)).strip() + try: + value = float(raw) + except ValueError as exc: + raise SystemExit(f"{name} must be a number") from exc + if not low <= value <= high: + raise SystemExit(f"{name} is out of range") + return value + + +def _secret(direct_name: str, file_name: str) -> str: + direct = os.environ.get(direct_name, "").strip() + if direct: + return direct + path = os.environ.get(file_name, "").strip() + if not path: + raise SystemExit(f"set {direct_name} or {file_name}") + try: + value = Path(path).read_text(encoding="utf-8").strip() + except OSError as exc: + raise SystemExit(f"cannot read {file_name}") from exc + if not value: + raise SystemExit(f"{file_name} is empty") + return value + + +def _migrations_dir() -> Path: + configured = os.environ.get("MIGRATIONS_DIR", "").strip() + if configured: + return Path(configured) + return Path(__file__).resolve().parents[2] / "migrations" + + +def _flag(name: str, default: bool) -> bool: + raw = os.environ.get(name, "1" if default else "0").strip().lower() + if raw in {"1", "true", "yes", "on"}: + return True + if raw in {"0", "false", "no", "off"}: + return False + raise SystemExit(f"{name} must be a boolean") + + +def _instance_id() -> str: + configured = os.environ.get("INSTANCE_ID", "").strip() + if configured: + return configured + host = os.environ.get("HOSTNAME", "").strip() or socket.gethostname() + if not host: + raise SystemExit("INSTANCE_ID is empty") + return host + + +def load_settings() -> Settings: + host = os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0" + env_name = os.environ.get("APP_ENV", "lab").strip() or "lab" + role = os.environ.get("APP_ROLE", "api").strip() or "api" + if role not in {"api", "migrate", "gc", "worker", "relay", "replay"}: + raise SystemExit("APP_ROLE must be api, migrate, gc, worker, relay or replay") + port = _int("APP_PORT", 8080, 1, 65535) + database_url = _secret("DATABASE_URL", "DATABASE_URL_FILE") + if not database_url.startswith("postgresql+asyncpg://"): + raise SystemExit("DATABASE_URL must start with postgresql+asyncpg://") + cache_url = "" + s3_endpoint = "" + s3_region = os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1" + s3_bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() or "messenger-attachments" + s3_access_key_id = "" + s3_secret_access_key = "" + broker_url = "" + if role == "api": + cache_url = _secret("REDIS_URL", "CACHE_URL_FILE") + if not cache_url.startswith(("redis://", "rediss://")): + raise SystemExit("REDIS_URL must start with redis:// or rediss://") + if role in {"api", "gc", "worker"}: + s3_endpoint = os.environ.get("S3_ENDPOINT", "").strip() + if not s3_endpoint: + raise SystemExit("S3_ENDPOINT is required") + s3_access_key_id = _secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + s3_secret_access_key = _secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + if role in {"worker", "relay"}: + broker_url = _secret("BROKER_URL", "BROKER_URL_FILE") + if not broker_url.startswith(("amqp://", "amqps://")): + raise SystemExit("BROKER_URL must start with amqp:// or amqps://") + timeout_seconds = _int("JOB_TIMEOUT_SECONDS", 45, 1, 600) + lease_seconds = _int("JOB_LEASE_SECONDS", 90, 2, 3600) + if lease_seconds <= timeout_seconds: + raise SystemExit("JOB_LEASE_SECONDS must be greater than JOB_TIMEOUT_SECONDS") + migrations = _migrations_dir() + if not migrations.is_dir(): + raise SystemExit("migrations directory is missing") + return Settings( + host=host, + port=port, + env=env_name, + app_role=role, + instance_id=_instance_id(), + run_migrations=_flag("RUN_MIGRATIONS", True), + session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600), + idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 60, 30 * 24 * 3600), + database_url=database_url, + cache_url=cache_url, + s3_endpoint=s3_endpoint, + s3_region=s3_region, + s3_bucket=s3_bucket, + s3_access_key_id=s3_access_key_id, + s3_secret_access_key=s3_secret_access_key, + max_upload_bytes=_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024, 1024, 10 * 1024 * 1024), + max_image_pixels=_int("MAX_IMAGE_PIXELS", 40_000_000, 1, 40_000_000), + attachment_gc_grace_seconds=_int("ATTACHMENT_GC_GRACE_SECONDS", 900, 1, 24 * 3600), + migrations_dir=migrations, + db_pool_size=_int("DB_POOL_SIZE", 10, 1, 50), + db_max_overflow=_int("DB_MAX_OVERFLOW", 10, 0, 50), + db_pool_timeout_seconds=_float("DB_POOL_TIMEOUT_SECONDS", 5, 0.1, 30), + db_connect_timeout_seconds=_float("DB_CONNECT_TIMEOUT_SECONDS", 2, 0.1, 30), + db_command_timeout_seconds=_float("DB_COMMAND_TIMEOUT_SECONDS", 5, 0.1, 60), + redis_socket_timeout_seconds=_float("REDIS_SOCKET_TIMEOUT_SECONDS", 1, 0.1, 30), + redis_max_connections=_int("REDIS_MAX_CONNECTIONS", 50, 1, 200), + broker_url=broker_url, + worker_concurrency=_int("WORKER_CONCURRENCY", 2, 1, 8), + job_max_attempts=_int("JOB_MAX_ATTEMPTS", 5, 1, 20), + job_timeout_seconds=timeout_seconds, + job_lease_seconds=lease_seconds, + job_backoff_base_seconds=_float("JOB_BACKOFF_BASE_SECONDS", 0.4, 0.05, 60), + job_backoff_cap_seconds=_float("JOB_BACKOFF_CAP_SECONDS", 8, 0.1, 300), + pipeline_version=_int("PIPELINE_VERSION", 1, 1, 1000), + staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400, 30, 30 * 24 * 3600), + worker_image_memory_bytes=_int("WORKER_IMAGE_MEMORY_BYTES", 256 * 1024 * 1024, 8 * 1024 * 1024, 1024 * 1024 * 1024), + jpeg_quality=_int("JPEG_QUALITY", 85, 1, 95), + webp_quality=_int("WEBP_QUALITY", 80, 1, 100), + webp_method=_int("WEBP_METHOD", 6, 0, 6), + faults_file=os.environ.get("FAULTS_FILE", "").strip(), + ) diff --git a/src/app/crud/__init__.py b/src/app/crud/__init__.py new file mode 100644 index 00000000..cbee6a29 --- /dev/null +++ b/src/app/crud/__init__.py @@ -0,0 +1 @@ +"""Чтение и изменение PostgreSQL. Сессии живут в Redis, не в этих таблицах.""" diff --git a/src/app/crud/attachments.py b/src/app/crud/attachments.py new file mode 100644 index 00000000..7b5f4346 --- /dev/null +++ b/src/app/crud/attachments.py @@ -0,0 +1,243 @@ +import asyncio +import hashlib +import logging +from datetime import timedelta +from uuid import UUID, uuid4 + +from sqlalchemy import and_, or_, select +from sqlalchemy.exc import IntegrityError + +from app.db import map_integrity, session_scope +from app.errors import AppError +from app.images import declared_content_type, inspect_image +from app.jobs import job_document +from app.models import Attachment, AttachmentJob, AttachmentVariant, ChatMember, JobOutbox +from app.pipeline import variant_names +from app.present import PublicAttachment +from app.runtime import require_settings, require_store +from app.timeutil import utcnow + +logger = logging.getLogger("app.attachments") + +_VARIANT_ORDER = {name: index for index, name in enumerate(variant_names())} + + +def validate_filename(filename: str | None) -> str: + if filename is None or filename.strip() == "": + raise AppError(400, "invalid_request", "Некорректное имя файла") + if len(filename) > 255: + raise AppError(400, "invalid_request", "Некорректное имя файла") + for char in filename: + code = ord(char) + if char in {"/", "\\", "\x00"} or code < 32 or code == 127: + raise AppError(400, "invalid_request", "Некорректное имя файла") + return filename + + +def _variant_dict(row: AttachmentVariant) -> dict: + return { + "name": row.name, + "content_type": row.content_type, + "size_bytes": row.size_bytes, + "width": row.width, + "height": row.height, + "sha256": row.sha256, + } + + +def _public(row: Attachment, variants: list[dict] | None = None) -> PublicAttachment: + visible = tuple(variants or ()) if row.status == "ready" else () + return PublicAttachment( + id=row.id, + chat_id=row.chat_id, + owner_id=row.owner_id, + filename=row.filename, + content_type=row.content_type, + size_bytes=row.size_bytes, + sha256=row.sha256, + status=row.status, + variants=visible, + error_code=row.error_code, + created_at=row.created_at, + updated_at=row.updated_at, + ) + + +async def _require_member(session, chat_id: UUID, user_id: UUID) -> None: + member = await session.scalar( + select(ChatMember.user_id).where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id) + ) + if member is None: + raise AppError(404, "not_found", "Объект не найден") + + +async def _variants_for(session, attachment_id: UUID) -> list[dict]: + rows = ( + await session.scalars(select(AttachmentVariant).where(AttachmentVariant.attachment_id == attachment_id)) + ).all() + payload = [_variant_dict(row) for row in rows] + payload.sort(key=lambda item: _VARIANT_ORDER.get(item["name"], 9)) + return payload + + +async def create_attachment( + chat_id: UUID, + owner_id: UUID, + filename: str, + content_type_header: str | None, + payload: bytes, +) -> PublicAttachment: + settings = require_settings() + store = require_store() + filename = validate_filename(filename) + declared = declared_content_type(content_type_header) + if not payload: + raise AppError(415, "unsupported_media_type", "Неподдерживаемый тип изображения") + if len(payload) > settings.max_upload_bytes: + raise AppError(413, "payload_too_large", "Файл слишком большой") + async with session_scope() as session: + async with session.begin(): + await _require_member(session, chat_id, owner_id) + await asyncio.to_thread(inspect_image, payload, declared, settings.max_image_pixels) + digest = hashlib.sha256(payload).hexdigest() + attachment_id = uuid4() + job_id = uuid4() + staging_key = "staging/%s" % attachment_id + object_key = "att/%s/original" % attachment_id + created_at = utcnow() + await store.put(staging_key, payload, declared) + try: + async with session_scope() as session: + try: + async with session.begin(): + await _require_member(session, chat_id, owner_id) + session.add( + Attachment( + id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + content_type=declared, + size_bytes=len(payload), + sha256=digest, + object_key=object_key, + staging_key=staging_key, + status="queued", + error_code=None, + created_at=created_at, + updated_at=created_at, + ) + ) + await session.flush() + session.add( + AttachmentJob( + job_id=job_id, + attachment_id=attachment_id, + pipeline_version=settings.pipeline_version, + status="queued", + attempt=0, + max_attempts=settings.job_max_attempts, + staging_key=staging_key, + objects_written_at=None, + created_at=created_at, + updated_at=created_at, + ) + ) + await session.flush() + session.add( + JobOutbox( + id=uuid4(), + job_id=job_id, + payload=job_document(job_id, attachment_id, settings.pipeline_version, 1, created_at), + available_at=created_at, + created_at=created_at, + published_at=None, + publish_attempts=0, + gap_injected=False, + ) + ) + await session.flush() + except IntegrityError as exc: + raise map_integrity(exc, "Конфликт записи") from None + except Exception: + await store.delete(staging_key) + raise + return PublicAttachment( + id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + content_type=declared, + size_bytes=len(payload), + sha256=digest, + status="queued", + variants=(), + error_code=None, + created_at=created_at, + updated_at=created_at, + ) + + +async def metadata_for_member(attachment_id: UUID, user_id: UUID) -> PublicAttachment: + async with session_scope() as session: + async with session.begin(): + row = await session.get(Attachment, attachment_id) + if row is None: + raise AppError(404, "not_found", "Объект не найден") + await _require_member(session, row.chat_id, user_id) + variants = await _variants_for(session, row.id) if row.status == "ready" else [] + return _public(row, variants) + + +async def open_content(attachment_id: UUID, user_id: UUID, variant: str) -> tuple[str, str]: + if variant not in {"original", "thumbnail", "crop", "optimized"}: + raise AppError(400, "invalid_request", "Некорректный запрос") + async with session_scope() as session: + async with session.begin(): + row = await session.get(Attachment, attachment_id) + if row is None: + raise AppError(404, "not_found", "Объект не найден") + await _require_member(session, row.chat_id, user_id) + if row.status != "ready": + raise AppError(409, "conflict", "Вложение ещё не готово") + if variant == "original": + return row.object_key, row.content_type + found = await session.scalar( + select(AttachmentVariant).where( + AttachmentVariant.attachment_id == row.id, + AttachmentVariant.name == variant, + ) + ) + if found is None: + raise AppError(404, "not_found", "Объект не найден") + return found.object_key, found.content_type + + +async def known_object_keys() -> set[str]: + settings = require_settings() + retain_after = utcnow() - timedelta(seconds=settings.staging_retention_seconds) + async with session_scope() as session: + async with session.begin(): + keys = set((await session.scalars(select(Attachment.object_key))).all()) + keys.update((await session.scalars(select(AttachmentVariant.object_key))).all()) + staging_jobs = ( + await session.scalars( + select(AttachmentJob.staging_key).where( + or_( + AttachmentJob.status.in_(("queued", "processing")), + and_(AttachmentJob.status == "failed", AttachmentJob.updated_at > retain_after), + ) + ) + ) + ).all() + keys.update(staging_jobs) + staging_rows = ( + await session.scalars( + select(Attachment.staging_key).where( + Attachment.staging_key.is_not(None), + Attachment.status.in_(("queued", "processing")), + ) + ) + ).all() + keys.update(key for key in staging_rows if key) + return keys diff --git a/src/app/crud/chats.py b/src/app/crud/chats.py new file mode 100644 index 00000000..5bd42e9f --- /dev/null +++ b/src/app/crud/chats.py @@ -0,0 +1,101 @@ +from uuid import UUID, uuid4 + +from sqlalchemy import select, tuple_ +from sqlalchemy.exc import IntegrityError + +from app.db import map_integrity, session_scope +from app.errors import AppError +from app.models import Chat, ChatMember, User +from app.pagination import decode_position, slice_page +from app.present import PublicChat +from app.timeutil import utcnow + + +def _invalid_members() -> AppError: + return AppError(400, "invalid_request", "Некорректный список участников") + + +async def create_chat( + owner_id: UUID, + title: str, + member_ids: tuple[UUID, ...], + *, + fail_after_insert: bool = False, +) -> PublicChat: + if ( + not member_ids + or len(member_ids) > 100 + or len(set(member_ids)) != len(member_ids) + or owner_id not in member_ids + ): + raise _invalid_members() + chat_id = uuid4() + created_at = utcnow() + async with session_scope() as session: + try: + async with session.begin(): + found = set((await session.scalars(select(User.id).where(User.id.in_(member_ids)))).all()) + if found != set(member_ids): + raise _invalid_members() + session.add(Chat(id=chat_id, title=title, created_at=created_at)) + await session.flush() + # Точка для проверки отката: строка чата уже вставлена, COMMIT ещё нет. + if fail_after_insert: + raise RuntimeError("rollback-check") + for ordinal, member_id in enumerate(member_ids): + session.add(ChatMember(chat_id=chat_id, user_id=member_id, ordinal=ordinal)) + await session.flush() + except IntegrityError as exc: + raise map_integrity(exc, "Конфликт записи") from None + return PublicChat(id=chat_id, title=title, member_ids=tuple(member_ids), created_at=created_at) + + +async def _member_ids(session, chat_id: UUID) -> tuple[UUID, ...]: + rows = await session.execute( + select(ChatMember.user_id).where(ChatMember.chat_id == chat_id).order_by(ChatMember.ordinal) + ) + return tuple(row[0] for row in rows) + + +async def chat_for_member(chat_id: UUID, user_id: UUID) -> PublicChat: + async with session_scope() as session: + async with session.begin(): + member = await session.scalar( + select(ChatMember.user_id).where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id) + ) + chat = await session.get(Chat, chat_id) + if member is None or chat is None: + raise AppError(404, "not_found", "Объект не найден") + return PublicChat( + id=chat.id, + title=chat.title, + member_ids=await _member_ids(session, chat.id), + created_at=chat.created_at, + ) + + +async def chats_for_user(user_id: UUID, limit: int, cursor: str | None) -> dict: + position = decode_position(cursor, "c", user_id) if cursor else None + async with session_scope() as session: + async with session.begin(): + stmt = ( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(ChatMember.user_id == user_id) + ) + if position is not None: + created_at, cursor_id = position + stmt = stmt.where(tuple_(Chat.created_at, Chat.id) > tuple_(created_at, cursor_id)) + stmt = stmt.order_by(Chat.created_at.asc(), Chat.id.asc()).limit(limit + 1) + chats = (await session.scalars(stmt)).unique().all() + public = [] + for chat in chats: + public.append( + PublicChat( + id=chat.id, + title=chat.title, + member_ids=await _member_ids(session, chat.id), + created_at=chat.created_at, + ) + ) + return slice_page(public, limit, "c", user_id) diff --git a/src/app/crud/messages.py b/src/app/crud/messages.py new file mode 100644 index 00000000..b93563c1 --- /dev/null +++ b/src/app/crud/messages.py @@ -0,0 +1,196 @@ +import hashlib +import json +from datetime import timedelta +from uuid import UUID, uuid4 + +from sqlalchemy import delete, select, text, tuple_ +from sqlalchemy.exc import IntegrityError + +from app.db import map_integrity, session_scope +from app.errors import AppError +from app.models import Attachment, ChatMember, IdempotencyRecord, Message, MessageAttachment +from app.pagination import decode_position, slice_page +from app.present import PublicMessage +from app.runtime import require_settings +from app.timeutil import utcnow + + +def message_path(chat_id: UUID) -> str: + return "/api/v1/chats/%s/messages" % chat_id + + +def request_fingerprint(text_value: str, attachment_ids: list[UUID]) -> str: + payload = json.dumps( + { + "attachment_ids": [str(item) for item in attachment_ids], + "text": text_value, + }, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _lock_keys(scope: str) -> tuple[int, int]: + digest = hashlib.sha256(scope.encode("utf-8")).digest() + first = int.from_bytes(digest[:4], "big", signed=True) + second = int.from_bytes(digest[4:8], "big", signed=True) + return first, second + + +def _public(row: Message, attachment_ids: list[UUID]) -> PublicMessage: + return PublicMessage( + id=row.id, + chat_id=row.chat_id, + sender_id=row.sender_id, + text=row.text, + created_at=row.created_at, + attachment_ids=tuple(attachment_ids), + ) + + +async def _require_member(session, chat_id: UUID, user_id: UUID) -> None: + member = await session.scalar( + select(ChatMember.user_id).where(ChatMember.chat_id == chat_id, ChatMember.user_id == user_id) + ) + if member is None: + raise AppError(404, "not_found", "Объект не найден") + + +async def _attachment_map(session, message_ids: list[UUID]) -> dict[UUID, list[UUID]]: + grouped: dict[UUID, list[UUID]] = {message_id: [] for message_id in message_ids} + if not message_ids: + return grouped + rows = ( + await session.execute( + select(MessageAttachment.message_id, MessageAttachment.attachment_id) + .where(MessageAttachment.message_id.in_(message_ids)) + .order_by(MessageAttachment.message_id, MessageAttachment.ordinal) + ) + ).all() + for message_id, attachment_id in rows: + grouped.setdefault(message_id, []).append(attachment_id) + return grouped + + +async def create_message( + chat_id: UUID, + sender_id: UUID, + text_value: str, + idempotency_key: str, + attachment_ids: list[UUID] | None = None, +) -> PublicMessage: + settings = require_settings() + files = list(attachment_ids or []) + path = message_path(chat_id) + request_hash = request_fingerprint(text_value, files) + lock_a, lock_b = _lock_keys("POST:%s:%s:%s" % (sender_id, path, idempotency_key)) + ttl = timedelta(seconds=settings.idempotency_ttl_seconds) + async with session_scope() as session: + try: + async with session.begin(): + await session.execute( + text("SELECT pg_advisory_xact_lock(:a, :b)"), + {"a": lock_a, "b": lock_b}, + ) + await _require_member(session, chat_id, sender_id) + if files: + found = ( + await session.scalars(select(Attachment).where(Attachment.id.in_(files))) + ).all() + by_id = {row.id: row for row in found} + if len(by_id) != len(files): + raise AppError(404, "not_found", "Объект не найден") + for item in files: + row = by_id[item] + if row.chat_id != chat_id: + raise AppError(404, "not_found", "Объект не найден") + if row.status == "failed": + raise AppError(409, "conflict", "Вложение недоступно") + existing = await session.scalar( + select(IdempotencyRecord).where( + IdempotencyRecord.user_id == sender_id, + IdempotencyRecord.method == "POST", + IdempotencyRecord.request_path == path, + IdempotencyRecord.idempotency_key == idempotency_key, + ) + ) + now = utcnow() + if existing is not None and existing.created_at + ttl >= now: + if existing.request_hash != request_hash: + raise AppError(409, "conflict", "Повтор ключа с другим сообщением") + row = await session.get(Message, existing.message_id) + if row is None: + raise AppError(500, "internal_error", "Внутренняя ошибка") + mapping = await _attachment_map(session, [row.id]) + return _public(row, mapping.get(row.id, [])) + if existing is not None: + await session.execute( + delete(IdempotencyRecord).where( + IdempotencyRecord.user_id == existing.user_id, + IdempotencyRecord.method == existing.method, + IdempotencyRecord.request_path == existing.request_path, + IdempotencyRecord.idempotency_key == existing.idempotency_key, + ) + ) + await session.flush() + message_id = uuid4() + created_at = now + session.add( + Message( + id=message_id, + chat_id=chat_id, + sender_id=sender_id, + text=text_value, + created_at=created_at, + ) + ) + await session.flush() + for ordinal, attachment_id in enumerate(files): + session.add( + MessageAttachment( + message_id=message_id, + attachment_id=attachment_id, + ordinal=ordinal, + ) + ) + session.add( + IdempotencyRecord( + user_id=sender_id, + method="POST", + request_path=path, + idempotency_key=idempotency_key, + request_hash=request_hash, + status_code=201, + message_id=message_id, + created_at=created_at, + ) + ) + await session.flush() + except IntegrityError as exc: + raise map_integrity(exc, "Конфликт записи") from None + return PublicMessage( + id=message_id, + chat_id=chat_id, + sender_id=sender_id, + text=text_value, + created_at=created_at, + attachment_ids=tuple(files), + ) + + +async def messages_for_member(chat_id: UUID, user_id: UUID, limit: int, cursor: str | None) -> dict: + async with session_scope() as session: + async with session.begin(): + await _require_member(session, chat_id, user_id) + position = decode_position(cursor, "m", chat_id) if cursor else None + stmt = select(Message).where(Message.chat_id == chat_id) + if position is not None: + created_at, message_id = position + stmt = stmt.where(tuple_(Message.created_at, Message.id) > tuple_(created_at, message_id)) + stmt = stmt.order_by(Message.created_at.asc(), Message.id.asc()).limit(limit + 1) + rows = (await session.scalars(stmt)).all() + mapping = await _attachment_map(session, [row.id for row in rows]) + public = [_public(row, mapping.get(row.id, [])) for row in rows] + return slice_page(public, limit, "m", chat_id) diff --git a/src/app/crud/users.py b/src/app/crud/users.py new file mode 100644 index 00000000..884381fc --- /dev/null +++ b/src/app/crud/users.py @@ -0,0 +1,78 @@ +import re +from uuid import UUID, uuid4 + +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError + +from app.db import map_integrity, session_scope +from app.models import User +from app.present import PublicUser +from app.security import hash_password, verify_password +from app.timeutil import utcnow + +_USERNAME = re.compile(r"^[a-z0-9_]{3,32}$") + + +async def get_user(user_id: UUID) -> PublicUser | None: + async with session_scope() as session: + async with session.begin(): + row = ( + await session.execute( + select(User.id, User.username, User.display_name, User.created_at).where(User.id == user_id) + ) + ).one_or_none() + if row is None: + return None + return PublicUser(id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at) + + +async def create_user(username: str, display_name: str, password: str) -> PublicUser: + # Argon2 считается до открытия транзакции, чтобы не держать соединение пула. + password_hash = hash_password(password) + user_id = uuid4() + created_at = utcnow() + async with session_scope() as session: + try: + async with session.begin(): + session.add( + User( + id=user_id, + username=username, + display_name=display_name, + password_hash=password_hash, + created_at=created_at, + ) + ) + await session.flush() + except IntegrityError as exc: + raise map_integrity(exc, "Имя пользователя занято") from None + return PublicUser(id=user_id, username=username, display_name=display_name, created_at=created_at) + + +async def authenticate(username: str, password: str) -> PublicUser | None: + if _USERNAME.fullmatch(username) is None or not 12 <= len(password) <= 128: + verify_password("rejected-credential", None) + return None + async with session_scope() as session: + async with session.begin(): + row = ( + await session.execute( + select(User.id, User.username, User.display_name, User.created_at, User.password_hash).where( + User.username == username + ) + ) + ).one_or_none() + if row is None: + stored = None + snapshot = None + else: + stored = row.password_hash + snapshot = PublicUser( + id=row.id, + username=row.username, + display_name=row.display_name, + created_at=row.created_at, + ) + if not verify_password(password, stored): + return None + return snapshot diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 00000000..422ae050 --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,79 @@ +import logging +from contextlib import asynccontextmanager + +from redis.exceptions import RedisError +from sqlalchemy.exc import DBAPIError, IntegrityError, InterfaceError, OperationalError +from sqlalchemy.exc import TimeoutError as SATimeout +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine + +from app.config import Settings +from app.errors import AppError + +logger = logging.getLogger("app.db") + + +def build_engine(settings: Settings) -> AsyncEngine: + command_ms = str(int(settings.db_command_timeout_seconds * 1000)) + lock_ms = str(int(settings.db_pool_timeout_seconds * 1000)) + return create_async_engine( + settings.database_url, + pool_size=settings.db_pool_size, + max_overflow=settings.db_max_overflow, + pool_timeout=settings.db_pool_timeout_seconds, + pool_pre_ping=True, + pool_recycle=1800, + echo=False, + connect_args={ + "timeout": settings.db_connect_timeout_seconds, + "command_timeout": settings.db_command_timeout_seconds, + "server_settings": { + "statement_timeout": command_ms, + "lock_timeout": lock_ms, + "idle_in_transaction_session_timeout": "10000", + }, + }, + ) + + +def build_session_factory(engine: AsyncEngine) -> async_sessionmaker[AsyncSession]: + return async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + + +def is_dependency_error(exc: BaseException) -> bool: + if isinstance(exc, (OperationalError, InterfaceError, SATimeout, RedisError, ConnectionError, TimeoutError)): + return True + return isinstance(exc, DBAPIError) and bool(getattr(exc, "connection_invalidated", False)) + + +def map_integrity(exc: IntegrityError, duplicate_message: str) -> AppError: + orig = getattr(exc, "orig", None) + code = getattr(orig, "sqlstate", None) or getattr(orig, "pgcode", None) + if code == "23505": + return AppError(409, "conflict", duplicate_message) + if code in {"23503", "23514", "23502"}: + return AppError(400, "invalid_request", "Некорректный запрос") + return AppError(500, "internal_error", "Внутренняя ошибка") + + +@asynccontextmanager +async def session_scope(): + from app.runtime import require_factory + + factory = require_factory() + try: + session = factory() + except Exception as exc: + if is_dependency_error(exc): + logger.warning("database unavailable type=%s", type(exc).__name__) + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + raise + try: + async with session: + yield session + except AppError: + raise + except Exception as exc: + if is_dependency_error(exc): + logger.warning("database unavailable type=%s", type(exc).__name__) + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + raise diff --git a/src/app/deps.py b/src/app/deps.py new file mode 100644 index 00000000..7fdf11c8 --- /dev/null +++ b/src/app/deps.py @@ -0,0 +1,50 @@ +from fastapi import Depends, Header, Request +from app.crud.users import get_user +from app.errors import AppError +from app.present import PublicUser +from app.sessions import resolve_session + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' + + +def unauthorized(challenge: str) -> AppError: + return AppError( + 401, + "unauthorized", + "Требуется идентификация", + headers={"WWW-Authenticate": challenge}, + ) + + +async def get_current_user( + request: Request, + authorization: str | None = Header(default=None), +) -> PublicUser: + # Нет заголовка — клиент анонимный. Redis для этого не нужен и не делает его доверенным. + if authorization is None: + raise unauthorized(BEARER_CHALLENGE) + scheme, _, token = authorization.partition(" ") + token = token.strip() + if scheme.lower() != "bearer" or not token or len(token) > 512: + raise unauthorized(BEARER_CHALLENGE) + user_id = await resolve_session(token) + if user_id is None: + raise unauthorized(BEARER_CHALLENGE) + user = await get_user(user_id) + if user is None: + raise unauthorized(BEARER_CHALLENGE) + request.state.session_token = token + return user + + +def current_token(request: Request) -> str: + token = getattr(request.state, "session_token", "") + if not token: + raise unauthorized(BEARER_CHALLENGE) + return token + + +def require_token(request: Request, user: PublicUser = Depends(get_current_user)) -> str: + del user + return current_token(request) diff --git a/src/app/errors.py b/src/app/errors.py new file mode 100644 index 00000000..c0785a12 --- /dev/null +++ b/src/app/errors.py @@ -0,0 +1,54 @@ +import logging + +from fastapi import Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException + + +class AppError(Exception): + def __init__(self, status: int, code: str, message: str, headers: dict | None = None) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers + + +def request_id_of(request: Request) -> str: + return getattr(request.state, "request_id", "") or "missing" + + +def error_response( + status: int, + code: str, + message: str, + request_id: str, + headers: dict | None = None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + headers=headers, + ) + + +async def handle_app_error(request: Request, exc: AppError) -> JSONResponse: + return error_response(exc.status, exc.code, exc.message, request_id_of(request), exc.headers) + + +async def handle_dependency(request: Request, exc: Exception) -> JSONResponse: + # Текст исключения может содержать SQL или параметры. В ответ и лог его не кладём. + logging.getLogger("app").warning("dependency unavailable type=%s", type(exc).__name__) + return error_response(503, "unavailable", "Сервис временно недоступен", request_id_of(request)) + + +async def handle_validation_error(request: Request, _exc: RequestValidationError) -> JSONResponse: + return error_response(400, "invalid_request", "Некорректный запрос", request_id_of(request)) + + +async def handle_http_error(request: Request, exc: HTTPException) -> JSONResponse: + # Тело не в UTF-8 FastAPI превращает в HTTPException 400 с полем detail. + if exc.status_code == 400: + return error_response(400, "invalid_request", "Некорректный запрос", request_id_of(request)) + return JSONResponse(status_code=exc.status_code, content={"detail": exc.detail}, headers=exc.headers) diff --git a/src/app/faults.py b/src/app/faults.py new file mode 100644 index 00000000..3491117d --- /dev/null +++ b/src/app/faults.py @@ -0,0 +1,45 @@ +import json +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Faults: + inject_transient: bool = False + crash_after_s3: bool = False + relay_exit_before_publish: bool = False + stall_seconds: float = 0 + timeout_seconds: float = 0 + + +def read_faults(path: str) -> Faults: + if not path: + return Faults() + file = Path(path) + if not file.is_file(): + return Faults() + try: + raw = json.loads(file.read_text(encoding="utf-8")) + except (OSError, ValueError, UnicodeError): + return Faults() + if not isinstance(raw, dict): + return Faults() + + def flag(name: str) -> bool: + return bool(raw.get(name)) + + def number(name: str) -> float: + value = raw.get(name, 0) + if isinstance(value, bool) or not isinstance(value, (int, float)): + return 0 + if value < 0: + return 0 + return float(value) + + return Faults( + inject_transient=flag("inject_transient"), + crash_after_s3=flag("crash_after_s3"), + relay_exit_before_publish=flag("relay_exit_before_publish"), + stall_seconds=number("stall_seconds"), + timeout_seconds=number("timeout_seconds"), + ) diff --git a/src/app/gc.py b/src/app/gc.py new file mode 100644 index 00000000..970d162a --- /dev/null +++ b/src/app/gc.py @@ -0,0 +1,66 @@ +import asyncio +import logging +import os +import sys +from datetime import timedelta + +from app.config import load_settings +from app.crud.attachments import known_object_keys +from app.db import build_engine, build_session_factory +from app.migrate import wait_for_database, wait_for_schema +from app.runtime import configure +from app.s3 import build_store +from app.timeutil import utcnow + +logger = logging.getLogger("app.gc") + + +async def sweep_once() -> int: + settings = load_settings() + store = build_store(settings) + engine = build_engine(settings) + try: + await wait_for_database(engine) + await wait_for_schema(engine, settings.migrations_dir) + configure(build_session_factory(engine), None, settings, store) + cutoff = utcnow() - timedelta(seconds=settings.attachment_gc_grace_seconds) + known = await known_object_keys() + deleted = 0 + for key, modified in await store.list_objects(): + if key in known or modified > cutoff: + continue + await store.delete(key) + deleted += 1 + if deleted: + logger.info("orphan objects deleted count=%s", deleted) + return deleted + finally: + await engine.dispose() + + +async def run_loop() -> None: + settings = load_settings() + interval = min(60, max(15, settings.attachment_gc_grace_seconds // 6)) + while True: + try: + await sweep_once() + except Exception as exc: + logger.warning("gc sweep failed type=%s", type(exc).__name__) + await asyncio.sleep(interval) + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + force=True, + ) + if os.environ.get("GC_ONCE", "").strip() in {"1", "true", "yes"}: + asyncio.run(sweep_once()) + return + asyncio.run(run_loop()) + + +if __name__ == "__main__": + main() diff --git a/src/app/images.py b/src/app/images.py new file mode 100644 index 00000000..0a2f4b6a --- /dev/null +++ b/src/app/images.py @@ -0,0 +1,53 @@ +import io +from dataclasses import dataclass + +from PIL import Image, ImageFile, UnidentifiedImageError + +from app.errors import AppError + +ImageFile.LOAD_TRUNCATED_IMAGES = False + +_PNG = b"\x89PNG\r\n\x1a\n" +_JPEG = b"\xff\xd8\xff" + + +@dataclass(frozen=True) +class ImageMeta: + content_type: str + + +def declared_content_type(header: str | None) -> str: + value = (header or "").split(";", 1)[0].strip().lower() + if value in {"image/png", "image/jpeg"}: + return value + raise AppError(415, "unsupported_media_type", "Неподдерживаемый тип изображения") + + +def sniff_content_type(data: bytes) -> str | None: + if data.startswith(_PNG): + return "image/png" + if data.startswith(_JPEG): + return "image/jpeg" + return None + + +def inspect_image(data: bytes, declared: str, max_pixels: int) -> ImageMeta: + sniffed = sniff_content_type(data) + if sniffed is None or sniffed != declared: + raise AppError(415, "unsupported_media_type", "Неподдерживаемый тип изображения") + Image.MAX_IMAGE_PIXELS = max_pixels + try: + with Image.open(io.BytesIO(data)) as image: + width, height = image.size + if width < 1 or height < 1: + raise AppError(415, "unsupported_media_type", "Неподдерживаемый тип изображения") + if width * height > max_pixels: + raise AppError(413, "payload_too_large", "Слишком большое изображение") + image.verify() + except AppError: + raise + except Image.DecompressionBombError: + raise AppError(413, "payload_too_large", "Слишком большое изображение") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise AppError(415, "unsupported_media_type", "Неподдерживаемый тип изображения") from None + return ImageMeta(content_type=declared) diff --git a/src/app/jobs.py b/src/app/jobs.py new file mode 100644 index 00000000..b50d2a62 --- /dev/null +++ b/src/app/jobs.py @@ -0,0 +1,417 @@ +import json +import logging +import random +from dataclasses import dataclass +from datetime import datetime, timedelta +from uuid import UUID, uuid4 + +from sqlalchemy import select + +from app.db import session_scope +from app.models import Attachment, AttachmentJob, AttachmentVariant, JobFailure, JobOutbox, JobReplay +from app.pipeline import variant_names +from app.runtime import require_settings +from app.timeutil import format_time, utcnow + +logger = logging.getLogger("app.jobs") + +REQUIRED_VARIANTS = set(variant_names()) + + +@dataclass(frozen=True) +class AcquireResult: + action: str + job_id: UUID | None = None + attachment_id: UUID | None = None + pipeline_version: int = 1 + attempt: int = 0 + max_attempts: int = 1 + staging_key: str = "" + object_key: str = "" + content_type: str = "" + sha256: str = "" + size_bytes: int = 0 + objects_written_at: datetime | None = None + + +def backoff_seconds(attempt: int, base: float, cap: float) -> float: + delay = base * (2 ** min(max(attempt - 1, 0), 8)) + if delay > cap: + delay = cap + total = delay + random.uniform(0, base) + if total > cap: + return cap + return total + + +def job_document(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, created_at: datetime) -> str: + body = { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": max(1, attempt), + "created_at": format_time(created_at), + } + return json.dumps(body, separators=(",", ":"), sort_keys=True) + + +def _outbox(job: AttachmentJob, when: datetime, attempt: int) -> JobOutbox: + return JobOutbox( + id=uuid4(), + job_id=job.job_id, + payload=job_document(job.job_id, job.attachment_id, job.pipeline_version, attempt, job.created_at), + available_at=when, + created_at=when, + published_at=None, + publish_attempts=0, + gap_injected=False, + ) + + +def _failure(job: AttachmentJob, code: str, permanent: bool, when: datetime) -> JobFailure: + return JobFailure( + id=uuid4(), + job_id=job.job_id, + attachment_id=job.attachment_id, + attempt=job.attempt, + error_code=code, + permanent=permanent, + created_at=when, + ) + + +async def _variant_names(session, attachment_id: UUID) -> set[str]: + rows = ( + await session.scalars(select(AttachmentVariant.name).where(AttachmentVariant.attachment_id == attachment_id)) + ).all() + return set(rows) + + +async def acquire_job(job_id: UUID, owner: str, redelivered: bool) -> AcquireResult: + settings = require_settings() + now = utcnow() + until = now + timedelta(seconds=settings.job_lease_seconds) + async with session_scope() as session: + async with session.begin(): + job = await session.get(AttachmentJob, job_id, with_for_update=True) + if job is None: + return AcquireResult(action="missing") + attachment = await session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment is None: + return AcquireResult(action="missing") + if attachment.status == "ready" or job.status == "ready": + job.status = "ready" + job.lease_until = None + job.lease_owner = None + job.error_code = None + job.updated_at = now + if attachment.status != "ready": + attachment.status = "ready" + attachment.error_code = None + attachment.updated_at = now + return AcquireResult(action="ready", job_id=job.job_id, attachment_id=attachment.id) + if job.status == "failed" or attachment.status == "failed": + return AcquireResult(action="failed", job_id=job.job_id, attachment_id=attachment.id) + if job.status == "queued" and job.attempt >= job.max_attempts: + code = "retry_exhausted" + job.status = "failed" + job.error_code = code + job.lease_until = None + job.lease_owner = None + job.updated_at = now + attachment.status = "failed" + attachment.error_code = code + attachment.updated_at = now + session.add(_failure(job, code, True, now)) + return AcquireResult(action="failed", job_id=job.job_id, attachment_id=attachment.id) + held = ( + job.status == "processing" + and job.lease_owner + and job.lease_owner != owner + and job.lease_until is not None + and job.lease_until > now + and not redelivered + ) + if held: + return AcquireResult(action="busy", job_id=job.job_id, attachment_id=attachment.id) + if job.status == "queued": + job.attempt += 1 + job.status = "processing" + job.lease_until = until + job.lease_owner = owner + job.updated_at = now + attachment.status = "processing" + attachment.error_code = None + attachment.updated_at = now + return AcquireResult( + action="acquired", + job_id=job.job_id, + attachment_id=attachment.id, + pipeline_version=job.pipeline_version, + attempt=job.attempt, + max_attempts=job.max_attempts, + staging_key=job.staging_key, + object_key=attachment.object_key, + content_type=attachment.content_type, + sha256=attachment.sha256, + size_bytes=attachment.size_bytes, + objects_written_at=job.objects_written_at, + ) + + +async def mark_objects_written(job_id: UUID) -> None: + now = utcnow() + async with session_scope() as session: + async with session.begin(): + job = await session.get(AttachmentJob, job_id, with_for_update=True) + if job is None or job.objects_written_at is not None: + return + job.objects_written_at = now + job.updated_at = now + + +async def fail_or_retry(job_id: UUID, code: str, permanent: bool) -> str: + settings = require_settings() + now = utcnow() + async with session_scope() as session: + async with session.begin(): + job = await session.get(AttachmentJob, job_id, with_for_update=True) + if job is None or job.status == "ready": + return "ignored" + attachment = await session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment is None or attachment.status == "ready": + return "ignored" + terminal = permanent or job.attempt >= job.max_attempts + final_code = code if permanent else "retry_exhausted" + session.add(_failure(job, final_code if terminal else code, terminal, now)) + job.lease_until = None + job.lease_owner = None + job.updated_at = now + if terminal: + job.status = "failed" + job.error_code = final_code + attachment.status = "failed" + attachment.error_code = final_code + attachment.updated_at = now + payload = job_document(job.job_id, job.attachment_id, job.pipeline_version, max(1, job.attempt), job.created_at) + return "failed:" + payload + delay = backoff_seconds(job.attempt, settings.job_backoff_base_seconds, settings.job_backoff_cap_seconds) + job.status = "queued" + job.error_code = None + job.next_attempt_at = now + timedelta(seconds=delay) + attachment.status = "queued" + attachment.error_code = None + attachment.updated_at = now + session.add(_outbox(job, job.next_attempt_at, job.attempt + 1)) + logger.info( + "job retry scheduled job_id=%s attempt=%s backoff_seconds=%.3f", + job.job_id, + job.attempt, + delay, + ) + return "retry" + + +async def finalize_job(job_id: UUID, variants: list[dict]) -> str: + now = utcnow() + async with session_scope() as session: + async with session.begin(): + job = await session.get(AttachmentJob, job_id, with_for_update=True) + if job is None: + return "missing" + attachment = await session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment is None: + return "missing" + if job.status == "ready" and attachment.status == "ready": + return "already" + for item in variants: + existing = await session.scalar( + select(AttachmentVariant).where( + AttachmentVariant.attachment_id == attachment.id, + AttachmentVariant.name == item["name"], + ) + ) + if existing is not None and existing.pipeline_version > job.pipeline_version: + continue + if existing is None: + session.add( + AttachmentVariant( + attachment_id=attachment.id, + name=item["name"], + content_type=item["content_type"], + size_bytes=item["size_bytes"], + width=item["width"], + height=item["height"], + sha256=item["sha256"], + object_key=item["object_key"], + pipeline_version=job.pipeline_version, + ) + ) + else: + existing.content_type = item["content_type"] + existing.size_bytes = item["size_bytes"] + existing.width = item["width"] + existing.height = item["height"] + existing.sha256 = item["sha256"] + existing.object_key = item["object_key"] + existing.pipeline_version = job.pipeline_version + names = await _variant_names(session, attachment.id) + names.update(item["name"] for item in variants) + if not REQUIRED_VARIANTS <= names: + return "incomplete" + attachment.status = "ready" + attachment.error_code = None + attachment.updated_at = now + job.status = "ready" + job.error_code = None + job.lease_until = None + job.lease_owner = None + job.updated_at = now + return "ready" + + +async def sweep_expired_leases() -> int: + settings = require_settings() + now = utcnow() + changed = 0 + async with session_scope() as session: + async with session.begin(): + rows = ( + await session.scalars( + select(AttachmentJob) + .where( + AttachmentJob.status == "processing", + AttachmentJob.lease_until.is_not(None), + AttachmentJob.lease_until < now, + ) + .with_for_update(skip_locked=True) + ) + ).all() + for job in rows: + attachment = await session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment is None or attachment.status == "ready" or job.status == "ready": + job.status = "ready" + job.lease_until = None + job.lease_owner = None + job.updated_at = now + continue + job.lease_until = None + job.lease_owner = None + job.updated_at = now + if job.attempt >= job.max_attempts: + job.status = "failed" + job.error_code = "retry_exhausted" + attachment.status = "failed" + attachment.error_code = "retry_exhausted" + attachment.updated_at = now + session.add(_failure(job, "retry_exhausted", True, now)) + else: + delay = backoff_seconds(job.attempt, settings.job_backoff_base_seconds, settings.job_backoff_cap_seconds) + when = now + timedelta(seconds=delay) + job.status = "queued" + job.error_code = None + job.next_attempt_at = when + attachment.status = "queued" + attachment.error_code = None + attachment.updated_at = now + session.add(_outbox(job, when, job.attempt + 1)) + changed += 1 + if changed: + logger.info("expired leases recovered count=%s", changed) + return changed + + +async def mark_gap_and_list(limit: int, crash_enabled: bool) -> tuple[list[dict], UUID | None]: + now = utcnow() + async with session_scope() as session: + async with session.begin(): + rows = ( + await session.scalars( + select(JobOutbox) + .where(JobOutbox.published_at.is_(None), JobOutbox.available_at <= now) + .order_by(JobOutbox.created_at.asc(), JobOutbox.id.asc()) + .limit(limit) + .with_for_update(skip_locked=True) + ) + ).all() + crash_id = None + # Один аварийный выход на пачку: если разрыв уже отмечен, публикуем, а не падаем снова. + if crash_enabled and rows and not any(row.gap_injected for row in rows): + rows[0].gap_injected = True + crash_id = rows[0].id + batch = [ + { + "id": row.id, + "job_id": row.job_id, + "payload": row.payload, + "gap_injected": row.gap_injected, + } + for row in rows + ] + return batch, crash_id + + +async def mark_published(outbox_id: UUID) -> None: + now = utcnow() + async with session_scope() as session: + async with session.begin(): + row = await session.get(JobOutbox, outbox_id, with_for_update=True) + if row is None or row.published_at is not None: + return + row.published_at = now + row.publish_attempts += 1 + + +async def note_publish_attempt(outbox_id: UUID) -> None: + async with session_scope() as session: + async with session.begin(): + row = await session.get(JobOutbox, outbox_id) + if row is None: + return + row.publish_attempts += 1 + + +async def replay_job(job_id: UUID) -> dict: + now = utcnow() + async with session_scope() as session: + async with session.begin(): + job = await session.get(AttachmentJob, job_id, with_for_update=True) + if job is None: + return {"outcome": "not_found", "job_id": str(job_id)} + attachment = await session.get(Attachment, job.attachment_id, with_for_update=True) + if attachment is None: + return {"outcome": "not_found", "job_id": str(job_id)} + names = await _variant_names(session, attachment.id) + if attachment.status == "ready" and job.status == "ready" and REQUIRED_VARIANTS <= names: + session.add(JobReplay(id=uuid4(), job_id=job.job_id, outcome="already_ready", created_at=now)) + return { + "outcome": "already_ready", + "job_id": str(job.job_id), + "attachment_id": str(attachment.id), + } + if job.status in {"queued", "processing"} and attachment.status != "failed": + session.add(JobReplay(id=uuid4(), job_id=job.job_id, outcome="in_progress", created_at=now)) + return { + "outcome": "in_progress", + "job_id": str(job.job_id), + "attachment_id": str(attachment.id), + } + job.status = "queued" + job.attempt = 0 + job.error_code = None + job.lease_until = None + job.lease_owner = None + job.next_attempt_at = now + job.updated_at = now + attachment.status = "queued" + attachment.error_code = None + attachment.updated_at = now + session.add(_outbox(job, now, 1)) + session.add(JobReplay(id=uuid4(), job_id=job.job_id, outcome="enqueued", created_at=now)) + return { + "outcome": "enqueued", + "job_id": str(job.job_id), + "attachment_id": str(attachment.id), + } diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 00000000..9bccca38 --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,114 @@ +import asyncio +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from redis.exceptions import RedisError +from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError +from sqlalchemy.exc import TimeoutError as SATimeout +from starlette.exceptions import HTTPException + +from app.api import api_router +from app.api.health import router as health_router +from app.config import load_settings +from app.db import build_engine, build_session_factory, is_dependency_error +from app.errors import AppError, handle_app_error, handle_dependency, handle_http_error, handle_validation_error +from app.migrate import apply_migrations, wait_for_database, wait_for_schema +from app.middleware import request_context +from app.runtime import configure +from app.s3 import build_store +from app.security import init_security +from app.sessions import build_redis + +logger = logging.getLogger("app") + + +async def open_stores(engine, redis, store, settings) -> None: + await wait_for_database(engine) + for attempt in range(40): + try: + await redis.ping() + break + except Exception as exc: + if not is_dependency_error(exc): + logger.error("redis probe failed type=%s", type(exc).__name__) + raise SystemExit("redis probe failed") from None + logger.warning("redis not ready type=%s attempt=%s", type(exc).__name__, attempt + 1) + if attempt == 39: + raise SystemExit("redis is not ready") from None + await asyncio.sleep(0.5) + for attempt in range(40): + try: + await store.ping() + break + except AppError: + logger.warning("s3 not ready attempt=%s", attempt + 1) + if attempt == 39: + raise SystemExit("s3 is not ready") from None + await asyncio.sleep(0.5) + except Exception as exc: + logger.warning("s3 probe failed type=%s attempt=%s", type(exc).__name__, attempt + 1) + if attempt == 39: + raise SystemExit("s3 is not ready") from None + await asyncio.sleep(0.5) + if settings.run_migrations: + try: + applied = await apply_migrations(engine, settings.migrations_dir) + logger.info("migrations newly_applied=%s", applied) + except ProgrammingError as exc: + sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", "") + logger.error("migration rejected type=%s sqlstate=%s", type(exc).__name__, sqlstate) + raise SystemExit("migration failed") from None + else: + await wait_for_schema(engine, settings.migrations_dir) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings = load_settings() + init_security() + engine = build_engine(settings) + redis = build_redis(settings) + store = build_store(settings) + app.state.engine = engine + app.state.redis = redis + app.state.store = store + app.state.settings = settings + app.state.instance_id = settings.instance_id + app.state.accepting = False + try: + await open_stores(engine, redis, store, settings) + configure(build_session_factory(engine), redis, settings, store) + app.state.accepting = True + logger.info( + "ready env=%s instance_id=%s session_ttl_seconds=%s db_pool=%s", + settings.env, + settings.instance_id, + settings.session_ttl_seconds, + settings.db_pool_size, + ) + yield + finally: + app.state.accepting = False + logger.info("shutdown: new requests are no longer accepted") + await redis.aclose() + await engine.dispose() + + +def create_app() -> FastAPI: + app = FastAPI(title="MTUSI Messenger", version="1.5.0", lifespan=lifespan) + app.state.instance_id = "starting" + app.state.accepting = False + app.add_exception_handler(AppError, handle_app_error) + app.add_exception_handler(RequestValidationError, handle_validation_error) + app.add_exception_handler(HTTPException, handle_http_error) + for cls in (OperationalError, InterfaceError, SATimeout, RedisError): + app.add_exception_handler(cls, handle_dependency) + app.middleware("http")(request_context) + app.include_router(health_router) + app.include_router(api_router) + return app + + +app = create_app() diff --git a/src/app/middleware.py b/src/app/middleware.py new file mode 100644 index 00000000..9908a959 --- /dev/null +++ b/src/app/middleware.py @@ -0,0 +1,52 @@ +import logging +import time +from uuid import uuid4 + +from fastapi import Request + +from app.db import is_dependency_error +from app.errors import error_response + +logger = logging.getLogger("app.access") + + +def _instance_id(request: Request) -> str: + settings = getattr(request.app.state, "settings", None) + if settings is not None: + return settings.instance_id + return getattr(request.app.state, "instance_id", "starting") + + +async def request_context(request: Request, call_next): + request_id = uuid4().hex + request.state.request_id = request_id + started = time.perf_counter() + status = 500 + try: + accepting = getattr(request.app.state, "accepting", True) + if not accepting and request.url.path != "/health/live": + response = error_response(503, "unavailable", "Сервер останавливается", request_id) + else: + try: + response = await call_next(request) + except Exception as exc: + if is_dependency_error(exc): + logger.warning("dependency unavailable type=%s", type(exc).__name__) + response = error_response(503, "unavailable", "Сервис временно недоступен", request_id) + else: + logger.error("internal error request_id=%s type=%s", request_id, type(exc).__name__) + response = error_response(500, "internal_error", "Внутренняя ошибка", request_id) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = _instance_id(request) + status = response.status_code + return response + finally: + elapsed_ms = int((time.perf_counter() - started) * 1000) + logger.info( + "request method=%s path=%s status=%s request_id=%s duration_ms=%s", + request.method, + request.url.path, + status, + request_id, + elapsed_ms, + ) diff --git a/src/app/migrate.py b/src/app/migrate.py new file mode 100644 index 00000000..3d6770e5 --- /dev/null +++ b/src/app/migrate.py @@ -0,0 +1,141 @@ +import asyncio +import logging +import sys +from pathlib import Path + +from sqlalchemy import text +from sqlalchemy.exc import ProgrammingError +from sqlalchemy.ext.asyncio import AsyncEngine + +from app.config import load_settings +from app.db import build_engine, is_dependency_error + +logger = logging.getLogger("app.migrate") + +# Один номер на все процессы, чтобы две миграции не применяли DDL одновременно. +_ADVISORY_LOCK = 842021 + + +def sql_statements(script: str) -> list[str]: + lines = [] + for line in script.splitlines(): + if line.strip().startswith("--"): + continue + lines.append(line) + statements = [] + for part in "\n".join(lines).split(";"): + statement = part.strip() + if statement: + statements.append(statement) + return statements + + +def expected_versions(directory: Path) -> list[str]: + files = sorted(path for path in directory.glob("*.sql") if path.is_file()) + if not files: + raise SystemExit("no SQL migrations found") + return [path.name for path in files] + + +async def apply_migrations(engine: AsyncEngine, directory: Path) -> int: + files = sorted(path for path in directory.glob("*.sql") if path.is_file()) + if not files: + raise SystemExit("no SQL migrations found") + applied = 0 + async with engine.begin() as conn: + await conn.execute(text("SELECT pg_advisory_xact_lock(:lock_id)"), {"lock_id": _ADVISORY_LOCK}) + await conn.execute( + text( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + ) + for path in files: + version = path.name + exists = await conn.scalar( + text("SELECT version FROM schema_migrations WHERE version = :version"), + {"version": version}, + ) + if exists: + continue + for statement in sql_statements(path.read_text(encoding="utf-8")): + await conn.execute(text(statement)) + await conn.execute( + text("INSERT INTO schema_migrations (version) VALUES (:version)"), + {"version": version}, + ) + applied += 1 + logger.info("migration applied version=%s", version) + return applied + + +async def wait_for_database(engine: AsyncEngine) -> None: + for attempt in range(40): + try: + async with engine.connect() as conn: + await conn.execute(text("SELECT 1")) + return + except Exception as exc: + if not is_dependency_error(exc): + logger.error("database probe failed type=%s", type(exc).__name__) + raise SystemExit("database probe failed") from None + logger.warning("database not ready type=%s attempt=%s", type(exc).__name__, attempt + 1) + if attempt == 39: + raise SystemExit("database is not ready") from None + await asyncio.sleep(0.5) + + +async def wait_for_schema(engine: AsyncEngine, directory: Path) -> None: + needed = set(expected_versions(directory)) + for attempt in range(40): + try: + async with engine.connect() as conn: + rows = (await conn.execute(text("SELECT version FROM schema_migrations"))).all() + found = {row[0] for row in rows} + if needed <= found: + logger.info("schema ready versions=%s", ",".join(sorted(found))) + return + logger.warning("schema incomplete attempt=%s", attempt + 1) + except ProgrammingError: + logger.warning("schema_migrations missing attempt=%s", attempt + 1) + except Exception as exc: + if not is_dependency_error(exc): + logger.error("schema probe failed type=%s", type(exc).__name__) + raise SystemExit("schema probe failed") from None + logger.warning("schema probe unavailable type=%s attempt=%s", type(exc).__name__, attempt + 1) + if attempt == 39: + raise SystemExit("schema is not ready") + await asyncio.sleep(0.5) + + +async def run_job() -> None: + settings = load_settings() + engine = build_engine(settings) + try: + await wait_for_database(engine) + applied = await apply_migrations(engine, settings.migrations_dir) + logger.info("migrations newly_applied=%s", applied) + except ProgrammingError as exc: + sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", "") + logger.error("migration rejected type=%s sqlstate=%s", type(exc).__name__, sqlstate) + raise SystemExit("migration failed") from None + finally: + await engine.dispose() + + +def main() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + force=True, + ) + asyncio.run(run_job()) + + +if __name__ == "__main__": + main() diff --git a/src/app/models.py b/src/app/models.py new file mode 100644 index 00000000..cf60db28 --- /dev/null +++ b/src/app/models.py @@ -0,0 +1,300 @@ +import uuid +from datetime import datetime + +from sqlalchemy import ( + Boolean, + CheckConstraint, + DateTime, + ForeignKey, + Index, + Integer, + SmallInteger, + Text, + UniqueConstraint, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + __table_args__ = ( + UniqueConstraint("username", name="uq_users_username"), + CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="ck_users_username_format"), + CheckConstraint( + "char_length(btrim(display_name)) >= 1 AND char_length(display_name) <= 100", + name="ck_users_display_name_len", + ), + CheckConstraint("char_length(password_hash) >= 20", name="ck_users_password_hash_len"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + username: Mapped[str] = mapped_column(Text, nullable=False) + display_name: Mapped[str] = mapped_column(Text, nullable=False) + password_hash: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class Chat(Base): + __tablename__ = "chats" + __table_args__ = ( + CheckConstraint( + "char_length(btrim(title)) >= 1 AND char_length(title) <= 100", + name="ck_chats_title", + ), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + title: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class ChatMember(Base): + __tablename__ = "chat_members" + __table_args__ = ( + CheckConstraint("ordinal >= 0 AND ordinal < 100", name="ck_chat_members_ordinal"), + Index("ix_chat_members_user", "user_id", "chat_id"), + ) + + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("chats.id"), + primary_key=True, + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id"), + primary_key=True, + ) + ordinal: Mapped[int] = mapped_column(SmallInteger, nullable=False) + + +class Message(Base): + __tablename__ = "messages" + __table_args__ = ( + CheckConstraint( + "char_length(body) <= 4000", + name="ck_messages_text", + ), + # Keyset: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4 + Index("ix_messages_history", "chat_id", "created_at", "id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("chats.id"), nullable=False) + sender_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + text: Mapped[str] = mapped_column("body", Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class IdempotencyRecord(Base): + __tablename__ = "idempotency_records" + __table_args__ = ( + CheckConstraint("method = 'POST'", name="ck_idem_method"), + CheckConstraint("status_code = 201", name="ck_idem_status"), + CheckConstraint("idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="ck_idem_key"), + CheckConstraint("char_length(request_path) BETWEEN 1 AND 300", name="ck_idem_path"), + CheckConstraint("char_length(request_hash) = 64", name="ck_idem_hash"), + Index("ix_idempotency_message", "message_id"), + Index("ix_idempotency_created", "created_at"), + ) + + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("users.id"), + primary_key=True, + ) + method: Mapped[str] = mapped_column(Text, primary_key=True) + request_path: Mapped[str] = mapped_column(Text, primary_key=True) + idempotency_key: Mapped[str] = mapped_column(Text, primary_key=True) + request_hash: Mapped[str] = mapped_column(Text, nullable=False) + status_code: Mapped[int] = mapped_column(SmallInteger, nullable=False) + message_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("messages.id"), + nullable=False, + ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class Attachment(Base): + __tablename__ = "attachments" + __table_args__ = ( + UniqueConstraint("object_key", name="uq_attachments_object_key"), + CheckConstraint("char_length(filename) BETWEEN 1 AND 255", name="ck_attachments_filename"), + CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name="ck_attachments_content_type"), + CheckConstraint("size_bytes >= 1 AND size_bytes <= 10485760", name="ck_attachments_size"), + CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="ck_attachments_sha256"), + CheckConstraint( + "status IN ('queued', 'processing', 'ready', 'failed')", + name="ck_attachments_status", + ), + CheckConstraint( + "(status = 'failed' AND error_code IS NOT NULL) OR (status <> 'failed' AND error_code IS NULL)", + name="ck_attachments_error", + ), + CheckConstraint( + "staging_key IS NULL OR char_length(staging_key) BETWEEN 8 AND 300", + name="ck_attachments_staging", + ), + Index("ix_attachments_chat", "chat_id", "created_at", "id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("chats.id"), nullable=False) + owner_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + filename: Mapped[str] = mapped_column(Text, nullable=False) + content_type: Mapped[str] = mapped_column(Text, nullable=False) + size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) + sha256: Mapped[str] = mapped_column(Text, nullable=False) + object_key: Mapped[str] = mapped_column(Text, nullable=False) + staging_key: Mapped[str | None] = mapped_column(Text, nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False) + error_code: Mapped[str | None] = mapped_column(Text, nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class AttachmentVariant(Base): + __tablename__ = "attachment_variants" + __table_args__ = ( + UniqueConstraint("object_key", name="uq_attachment_variants_object_key"), + CheckConstraint("name IN ('thumbnail', 'crop', 'optimized')", name="ck_variants_name"), + CheckConstraint( + "content_type IN ('image/jpeg', 'image/png', 'image/webp')", + name="ck_variants_type", + ), + CheckConstraint("size_bytes >= 1", name="ck_variants_size"), + CheckConstraint("width >= 1 AND height >= 1", name="ck_variants_dim"), + CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="ck_variants_sha"), + CheckConstraint("pipeline_version >= 1", name="ck_variants_pipeline"), + ) + + attachment_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachments.id"), + primary_key=True, + ) + name: Mapped[str] = mapped_column(Text, primary_key=True) + content_type: Mapped[str] = mapped_column(Text, nullable=False) + size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) + width: Mapped[int] = mapped_column(Integer, nullable=False) + height: Mapped[int] = mapped_column(Integer, nullable=False) + sha256: Mapped[str] = mapped_column(Text, nullable=False) + object_key: Mapped[str] = mapped_column(Text, nullable=False) + pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False) + + +class AttachmentJob(Base): + __tablename__ = "attachment_jobs" + __table_args__ = ( + UniqueConstraint("attachment_id", "pipeline_version", name="uq_attachment_jobs_effect"), + CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="ck_jobs_status"), + CheckConstraint("attempt >= 0 AND attempt <= max_attempts", name="ck_jobs_attempt"), + CheckConstraint("max_attempts BETWEEN 1 AND 20", name="ck_jobs_max"), + CheckConstraint("pipeline_version >= 1", name="ck_jobs_pipeline"), + Index("ix_jobs_lease", "status", "lease_until"), + ) + + job_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + attachment_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachments.id"), + nullable=False, + ) + pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[str] = mapped_column(Text, nullable=False) + attempt: Mapped[int] = mapped_column(Integer, nullable=False) + max_attempts: Mapped[int] = mapped_column(Integer, nullable=False) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + lease_owner: Mapped[str | None] = mapped_column(Text, nullable=True) + next_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + error_code: Mapped[str | None] = mapped_column(Text, nullable=True) + staging_key: Mapped[str] = mapped_column(Text, nullable=False) + objects_written_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class JobOutbox(Base): + __tablename__ = "job_outbox" + __table_args__ = ( + CheckConstraint("publish_attempts >= 0", name="ck_outbox_attempts"), + Index("ix_outbox_pending", "available_at", "created_at"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + job_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachment_jobs.job_id"), + nullable=False, + ) + payload: Mapped[str] = mapped_column(Text, nullable=False) + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + publish_attempts: Mapped[int] = mapped_column(Integer, nullable=False) + gap_injected: Mapped[bool] = mapped_column(Boolean, nullable=False) + + +class JobFailure(Base): + __tablename__ = "job_failures" + __table_args__ = ( + CheckConstraint("char_length(error_code) BETWEEN 1 AND 64", name="ck_job_failures_code"), + Index("ix_job_failures_job", "job_id", "created_at"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + job_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachment_jobs.job_id"), + nullable=False, + ) + attachment_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False) + attempt: Mapped[int] = mapped_column(Integer, nullable=False) + error_code: Mapped[str] = mapped_column(Text, nullable=False) + permanent: Mapped[bool] = mapped_column(Boolean, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class JobReplay(Base): + __tablename__ = "job_replays" + __table_args__ = ( + CheckConstraint( + "outcome IN ('enqueued', 'already_ready', 'in_progress')", + name="ck_replays_outcome", + ), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + job_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachment_jobs.job_id"), + nullable=False, + ) + outcome: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + __table_args__ = ( + CheckConstraint("ordinal >= 0 AND ordinal < 10", name="ck_message_attachments_ordinal"), + Index("ix_message_attachments_attachment", "attachment_id"), + ) + + message_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("messages.id"), + primary_key=True, + ) + attachment_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), + ForeignKey("attachments.id"), + primary_key=True, + ) + ordinal: Mapped[int] = mapped_column(SmallInteger, nullable=False) diff --git a/src/app/pagination.py b/src/app/pagination.py new file mode 100644 index 00000000..f536d6c6 --- /dev/null +++ b/src/app/pagination.py @@ -0,0 +1,50 @@ +import base64 +import json +from datetime import datetime +from uuid import UUID + +from app.errors import AppError +from app.timeutil import format_time, parse_time + + +def encode_cursor(kind: str, resource_id: UUID, created_at: datetime, entity_id: UUID) -> str: + payload = { + "i": str(entity_id), + "k": kind, + "r": str(resource_id), + "t": format_time(created_at), + } + raw = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("ascii") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_position(token: str, kind: str, resource_id: UUID) -> tuple[datetime, UUID]: + if len(token) > 1024: + raise AppError(400, "invalid_request", "Некорректный cursor") + padding = "=" * (-len(token) % 4) + try: + data = json.loads(base64.urlsafe_b64decode(token + padding)) + if ( + not isinstance(data, dict) + or data.get("k") != kind + or data.get("r") != str(resource_id) + or not isinstance(data.get("t"), str) + or not isinstance(data.get("i"), str) + ): + raise ValueError("cursor") + created_at = parse_time(data["t"]) + entity_id = UUID(data["i"]) + except (KeyError, TypeError, ValueError, AttributeError, json.JSONDecodeError): + raise AppError(400, "invalid_request", "Некорректный cursor") from None + return created_at, entity_id + + +def slice_page(rows: list, limit: int, kind: str, resource_id: UUID) -> dict: + # SQL уже вернул не больше limit + 1 строк, отфильтрованных keyset-условием. + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + last = page[-1] + next_cursor = encode_cursor(kind, resource_id, last.created_at, last.id) + return {"items": [row.to_dict() for row in page], "next_cursor": next_cursor} diff --git a/src/app/pipeline.py b/src/app/pipeline.py new file mode 100644 index 00000000..aea1e99f --- /dev/null +++ b/src/app/pipeline.py @@ -0,0 +1,165 @@ +import io +import multiprocessing +import time + +from PIL import Image, ImageFile, ImageOps, UnidentifiedImageError + +ImageFile.LOAD_TRUNCATED_IMAGES = False + +THUMBNAIL_BOX = 256 +CROP_SIDE = 128 +OPTIMIZED_BOX = 1280 + +_VARIANT_ORDER = ("thumbnail", "crop", "optimized") + + +def _without_metadata(image: Image.Image) -> Image.Image: + bare = Image.new("RGB", image.size) + bare.paste(image) + return bare + + +def _to_rgb(image: Image.Image) -> Image.Image: + oriented = ImageOps.exif_transpose(image) + if oriented.mode == "RGB": + return _without_metadata(oriented) + if oriented.mode in {"RGBA", "LA"} or (oriented.mode == "P" and "transparency" in oriented.info): + rgba = oriented.convert("RGBA") + base = Image.new("RGB", rgba.size, (255, 255, 255)) + base.paste(rgba, mask=rgba.getchannel("A")) + return base + return _without_metadata(oriented.convert("RGB")) + + +def _fit_within(image: Image.Image, box: int) -> Image.Image: + width, height = image.size + scale = min(box / width, box / height, 1.0) + new_width = max(1, int(width * scale)) + new_height = max(1, int(height * scale)) + if new_width == width and new_height == height: + return image + return image.resize((new_width, new_height), Image.Resampling.LANCZOS) + + +def _center_square(image: Image.Image, side: int) -> Image.Image: + width, height = image.size + source = min(width, height) + left = (width - source) // 2 + top = (height - source) // 2 + cropped = image.crop((left, top, left + source, top + source)) + if cropped.size != (side, side): + cropped = cropped.resize((side, side), Image.Resampling.LANCZOS) + return cropped + + +def _jpeg_bytes(image: Image.Image, quality: int) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=quality, optimize=True) + return buffer.getvalue() + + +def _webp_bytes(image: Image.Image, quality: int, method: int) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="WEBP", quality=quality, method=method) + return buffer.getvalue() + + +def _fail(code: str, permanent: bool) -> dict: + return {"ok": False, "code": code, "permanent": permanent} + + +def render_variants( + data: bytes, + max_pixels: int, + memory_budget: int, + jpeg_quality: int, + webp_quality: int, + webp_method: int, +) -> dict: + Image.MAX_IMAGE_PIXELS = max_pixels + try: + with Image.open(io.BytesIO(data)) as source: + width, height = source.size + if width < 1 or height < 1: + return _fail("invalid_image", True) + if width * height > max_pixels or width * height * 4 > memory_budget: + return _fail("payload_too_large", True) + source.load() + rgb = _to_rgb(source) + thumbnail = _fit_within(rgb, THUMBNAIL_BOX) + crop = _center_square(rgb, CROP_SIDE) + optimized = _fit_within(rgb, OPTIMIZED_BOX) + variants = { + "thumbnail": { + "content_type": "image/jpeg", + "width": thumbnail.size[0], + "height": thumbnail.size[1], + "data": _jpeg_bytes(thumbnail, jpeg_quality), + }, + "crop": { + "content_type": "image/jpeg", + "width": crop.size[0], + "height": crop.size[1], + "data": _jpeg_bytes(crop, jpeg_quality), + }, + "optimized": { + "content_type": "image/webp", + "width": optimized.size[0], + "height": optimized.size[1], + "data": _webp_bytes(optimized, webp_quality, webp_method), + }, + } + return {"ok": True, "variants": variants} + except Image.DecompressionBombError: + return _fail("payload_too_large", True) + except MemoryError: + return _fail("payload_too_large", True) + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + return _fail("invalid_image", True) + + +def _spawn_render(queue, data, max_pixels, memory_budget, jpeg_quality, webp_quality, webp_method, stall_seconds) -> None: + if stall_seconds > 0: + time.sleep(stall_seconds) + queue.put(render_variants(data, max_pixels, memory_budget, jpeg_quality, webp_quality, webp_method)) + + +def render_in_process( + data: bytes, + max_pixels: int, + memory_budget: int, + jpeg_quality: int, + webp_quality: int, + webp_method: int, + timeout_seconds: float, + stall_seconds: float = 0, +) -> dict: + context = multiprocessing.get_context("spawn") + queue = context.Queue(maxsize=1) + process = context.Process( + target=_spawn_render, + args=(queue, data, max_pixels, memory_budget, jpeg_quality, webp_quality, webp_method, stall_seconds), + ) + process.start() + process.join(timeout_seconds) + try: + if process.is_alive(): + process.terminate() + process.join(5) + if process.is_alive(): + process.kill() + process.join(2) + return _fail("processing_failed", False) + if process.exitcode != 0: + return _fail("processing_failed", False) + try: + return queue.get_nowait() + except Exception: + return _fail("processing_failed", False) + finally: + queue.close() + queue.cancel_join_thread() + + +def variant_names() -> tuple[str, ...]: + return _VARIANT_ORDER diff --git a/src/app/present.py b/src/app/present.py new file mode 100644 index 00000000..213fb133 --- /dev/null +++ b/src/app/present.py @@ -0,0 +1,89 @@ +from dataclasses import dataclass +from datetime import datetime +from uuid import UUID + +from app.timeutil import format_time + + +@dataclass(frozen=True) +class PublicUser: + id: UUID + username: str + display_name: str + created_at: datetime + + def to_dict(self) -> dict: + return { + "id": self.id, + "username": self.username, + "display_name": self.display_name, + "created_at": format_time(self.created_at), + } + + +@dataclass(frozen=True) +class PublicChat: + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + def to_dict(self) -> dict: + return { + "id": self.id, + "title": self.title, + "member_ids": list(self.member_ids), + "created_at": format_time(self.created_at), + } + + +@dataclass(frozen=True) +class PublicMessage: + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: tuple[UUID, ...] = () + + def to_dict(self) -> dict: + return { + "id": self.id, + "chat_id": self.chat_id, + "sender_id": self.sender_id, + "text": self.text, + "created_at": format_time(self.created_at), + "attachment_ids": list(self.attachment_ids), + } + + +@dataclass(frozen=True) +class PublicAttachment: + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + error_code: str | None + created_at: datetime + updated_at: datetime + variants: tuple = () + + def to_dict(self) -> dict: + return { + "id": self.id, + "chat_id": self.chat_id, + "owner_id": self.owner_id, + "filename": self.filename, + "content_type": self.content_type, + "size_bytes": self.size_bytes, + "sha256": self.sha256, + "status": self.status, + "variants": list(self.variants), + "error_code": self.error_code, + "created_at": format_time(self.created_at), + "updated_at": format_time(self.updated_at), + } diff --git a/src/app/relay.py b/src/app/relay.py new file mode 100644 index 00000000..4748517f --- /dev/null +++ b/src/app/relay.py @@ -0,0 +1,140 @@ +import asyncio +import logging +import os +import signal +import sys +from pathlib import Path + +from app.broker import WORK_KEY, connect, declare, publish +from app.config import load_settings +from app.db import build_engine, build_session_factory +from app.faults import read_faults +from app.jobs import mark_gap_and_list, mark_published, note_publish_attempt +from app.migrate import wait_for_database, wait_for_schema +from app.runtime import configure + +logger = logging.getLogger("app.relay") + + +def configure_logging() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + force=True, + ) + logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING) + logging.getLogger("aio_pika").setLevel(logging.WARNING) + logging.getLogger("aiormq").setLevel(logging.WARNING) + + +def flush_logs() -> None: + for handler in logging.getLogger().handlers: + handler.flush() + sys.stdout.flush() + + +async def beat(path: str, stop: asyncio.Event) -> None: + file = Path(path) + while not stop.is_set(): + file.write_text("ok", encoding="utf-8") + try: + await asyncio.wait_for(stop.wait(), timeout=2) + except asyncio.TimeoutError: + continue + + +async def open_broker(url: str): + for attempt in range(40): + try: + return await connect(url) + except Exception as exc: + logger.warning("broker not ready type=%s attempt=%s", type(exc).__name__, attempt + 1) + await asyncio.sleep(0.5) + raise SystemExit("broker is not ready") + + +async def publish_loop(stop: asyncio.Event) -> None: + settings = require_settings_later() + while not stop.is_set(): + connection = None + try: + connection = await open_broker(settings.broker_url) + channel = await connection.channel(publisher_confirms=True) + exchange, _queue = await declare(channel) + logger.info("relay publishing with confirms") + while not stop.is_set(): + faults = read_faults(settings.faults_file) + batch, crash_id = await mark_gap_and_list(10, faults.relay_exit_before_publish) + if crash_id is not None: + logger.error( + "injected exit after outbox commit before broker publish outbox_id=%s", + crash_id, + ) + flush_logs() + os._exit(42) + if not batch: + try: + await asyncio.wait_for(stop.wait(), timeout=0.4) + except asyncio.TimeoutError: + continue + break + for item in batch: + try: + await publish(exchange, item["payload"], WORK_KEY) + except Exception as exc: + logger.warning("publish failed type=%s job_id=%s", type(exc).__name__, item["job_id"]) + await note_publish_attempt(item["id"]) + continue + await mark_published(item["id"]) + logger.info("outbox published job_id=%s", item["job_id"]) + except Exception as exc: + if stop.is_set(): + break + logger.warning("relay interrupted type=%s", type(exc).__name__) + await asyncio.sleep(1) + finally: + if connection is not None: + try: + await connection.close() + except Exception as exc: + logger.warning("broker close failed type=%s", type(exc).__name__) + + +def require_settings_later(): + from app.runtime import require_settings + + return require_settings() + + +async def serve() -> None: + settings = load_settings() + engine = build_engine(settings) + stop = asyncio.Event() + try: + await wait_for_database(engine) + await wait_for_schema(engine, settings.migrations_dir) + configure(build_session_factory(engine), None, settings, None) + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop.set) + except NotImplementedError: + pass + heart = asyncio.create_task(beat("/tmp/relay-heartbeat", stop)) + try: + await publish_loop(stop) + finally: + stop.set() + heart.cancel() + finally: + await engine.dispose() + + +def main() -> None: + configure_logging() + asyncio.run(serve()) + + +if __name__ == "__main__": + main() diff --git a/src/app/replay.py b/src/app/replay.py new file mode 100644 index 00000000..a345a1cf --- /dev/null +++ b/src/app/replay.py @@ -0,0 +1,43 @@ +import asyncio +import json +import logging +import sys +from uuid import UUID + +from app.config import load_settings +from app.db import build_engine, build_session_factory +from app.jobs import replay_job +from app.migrate import wait_for_database +from app.runtime import configure + + +async def run(job_id: UUID) -> int: + settings = load_settings() + engine = build_engine(settings) + try: + await wait_for_database(engine) + configure(build_session_factory(engine), None, settings, None) + result = await replay_job(job_id) + finally: + await engine.dispose() + print(json.dumps(result, ensure_ascii=False), flush=True) + if result.get("outcome") == "not_found": + return 2 + return 0 + + +def main() -> None: + logging.basicConfig(level=logging.WARNING, stream=sys.stderr, force=True) + if len(sys.argv) != 2: + print("usage: python -m app.replay JOB_ID", file=sys.stderr) + raise SystemExit(2) + try: + job_id = UUID(sys.argv[1]) + except ValueError: + print("job id must be a uuid", file=sys.stderr) + raise SystemExit(2) + raise SystemExit(asyncio.run(run(job_id))) + + +if __name__ == "__main__": + main() diff --git a/src/app/runtime.py b/src/app/runtime.py new file mode 100644 index 00000000..172061ce --- /dev/null +++ b/src/app/runtime.py @@ -0,0 +1,46 @@ +from app.config import Settings + +_factory = None +_redis = None +_store = None +_settings: Settings | None = None + + +def configure(factory, redis, settings: Settings, store=None) -> None: + global _factory, _redis, _store, _settings + _factory = factory + _redis = redis + _store = store + _settings = settings + + +def require_factory(): + if _factory is None: + from app.errors import AppError + + raise AppError(503, "unavailable", "Сервис временно недоступен") + return _factory + + +def require_redis(): + if _redis is None: + from app.errors import AppError + + raise AppError(503, "unavailable", "Сервис временно недоступен") + return _redis + + +def require_settings() -> Settings: + if _settings is None: + from app.errors import AppError + + raise AppError(503, "unavailable", "Сервис временно недоступен") + return _settings + + +def require_store(): + if _store is None: + from app.errors import AppError + + raise AppError(503, "unavailable", "Сервис временно недоступен") + return _store diff --git a/src/app/s3.py b/src/app/s3.py new file mode 100644 index 00000000..9811ae07 --- /dev/null +++ b/src/app/s3.py @@ -0,0 +1,102 @@ +import asyncio +import logging +from datetime import datetime, timezone + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError + +from app.config import Settings +from app.errors import AppError + +logger = logging.getLogger("app.s3") + + +class ObjectStore: + def __init__(self, settings: Settings) -> None: + self.bucket = settings.s3_bucket + self._client = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + aws_access_key_id=settings.s3_access_key_id, + aws_secret_access_key=settings.s3_secret_access_key, + region_name=settings.s3_region, + config=Config( + s3={"addressing_style": "path"}, + connect_timeout=2, + read_timeout=30, + retries={"max_attempts": 2, "mode": "standard"}, + ), + ) + + def _call(self, fn, *args, **kwargs): + try: + return fn(*args, **kwargs) + except EndpointConnectionError: + logger.warning("s3 endpoint unavailable type=%s", "EndpointConnectionError") + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code in {"NoSuchKey", "404"}: + raise AppError(404, "not_found", "Объект не найден") from None + logger.warning("s3 client error type=%s code=%s", type(exc).__name__, code or "unknown") + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + except BotoCoreError: + logger.warning("s3 unavailable type=%s", "BotoCoreError") + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + async def ping(self) -> None: + await asyncio.to_thread(self._call, self._client.head_bucket, Bucket=self.bucket) + + async def put(self, key: str, data: bytes, content_type: str) -> None: + await asyncio.to_thread( + self._call, + self._client.put_object, + Bucket=self.bucket, + Key=key, + Body=data, + ContentType=content_type, + ContentLength=len(data), + ) + + async def get(self, key: str) -> bytes: + def _read() -> bytes: + body = self._call(self._client.get_object, Bucket=self.bucket, Key=key)["Body"] + return body.read() + + return await asyncio.to_thread(_read) + + async def delete(self, key: str) -> None: + try: + await asyncio.to_thread(self._call, self._client.delete_object, Bucket=self.bucket, Key=key) + except AppError: + return + + async def list_objects(self) -> list[tuple[str, datetime]]: + def _list() -> list[tuple[str, datetime]]: + items: list[tuple[str, datetime]] = [] + for prefix in ("att/", "staging/"): + token = None + while True: + kwargs = {"Bucket": self.bucket, "Prefix": prefix} + if token: + kwargs["ContinuationToken"] = token + page = self._call(self._client.list_objects_v2, **kwargs) + for obj in page.get("Contents") or []: + key = str(obj["Key"]) + modified = obj["LastModified"] + if modified.tzinfo is None: + modified = modified.replace(tzinfo=timezone.utc) + items.append((key, modified.astimezone(timezone.utc))) + if not page.get("IsTruncated"): + break + token = page.get("NextContinuationToken") + if not token: + break + return items + + return await asyncio.to_thread(_list) + + +def build_store(settings: Settings) -> ObjectStore: + return ObjectStore(settings) diff --git a/src/app/schemas/__init__.py b/src/app/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app/schemas/attachment.py b/src/app/schemas/attachment.py new file mode 100644 index 00000000..5f11b022 --- /dev/null +++ b/src/app/schemas/attachment.py @@ -0,0 +1,27 @@ +from uuid import UUID + +from pydantic import BaseModel + + +class VariantOut(BaseModel): + name: str + content_type: str + size_bytes: int + width: int + height: int + sha256: str + + +class AttachmentOut(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + variants: list[VariantOut] + error_code: str | None + created_at: str + updated_at: str diff --git a/src/app/schemas/chat.py b/src/app/schemas/chat.py new file mode 100644 index 00000000..ea694c63 --- /dev/null +++ b/src/app/schemas/chat.py @@ -0,0 +1,35 @@ +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + title: str + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def check_title(cls, value: str) -> str: + if value.strip() == "" or len(value) > 100: + raise ValueError("title") + return value + + @field_validator("member_ids") + @classmethod + def unique_members(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError("duplicate members") + return value + + +class ChatOut(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: str + + +class ChatPage(BaseModel): + items: list[ChatOut] + next_cursor: str | None diff --git a/src/app/schemas/message.py b/src/app/schemas/message.py new file mode 100644 index 00000000..e0c3c610 --- /dev/null +++ b/src/app/schemas/message.py @@ -0,0 +1,43 @@ +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + text: str + attachment_ids: list[UUID] = [] + + @field_validator("text") + @classmethod + def check_text(cls, value: str) -> str: + if len(value) > 4000: + raise ValueError("text") + return value + + @field_validator("attachment_ids") + @classmethod + def check_attachments(cls, value: list[UUID]) -> list[UUID]: + if len(value) > 10 or len(set(value)) != len(value): + raise ValueError("attachment_ids") + return value + + @model_validator(mode="after") + def require_payload(self): + if self.text.strip() == "" and not self.attachment_ids: + raise ValueError("text") + return self + + +class MessageOut(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: str + attachment_ids: list[UUID] + + +class MessagePage(BaseModel): + items: list[MessageOut] + next_cursor: str | None diff --git a/src/app/schemas/session.py b/src/app/schemas/session.py new file mode 100644 index 00000000..beb07f13 --- /dev/null +++ b/src/app/schemas/session.py @@ -0,0 +1,10 @@ +from pydantic import BaseModel + +from app.schemas.user import UserOut + + +class SessionOut(BaseModel): + token: str + token_type: str + expires_at: str + user: UserOut diff --git a/src/app/schemas/user.py b/src/app/schemas/user.py new file mode 100644 index 00000000..2e313359 --- /dev/null +++ b/src/app/schemas/user.py @@ -0,0 +1,41 @@ +import re + +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, field_validator + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + username: str + display_name: str + password: str + + @field_validator("username") + @classmethod + def check_username(cls, value: str) -> str: + if re.fullmatch(r"[a-z0-9_]{3,32}", value) is None: + raise ValueError("username") + return value + + @field_validator("display_name") + @classmethod + def check_display_name(cls, value: str) -> str: + if value.strip() == "" or len(value) > 100: + raise ValueError("display_name") + return value + + @field_validator("password") + @classmethod + def check_password(cls, value: str) -> str: + # Длина в code points. Пароль не обрезается и не нормализуется. + if not 12 <= len(value) <= 128: + raise ValueError("password") + return value + + +class UserOut(BaseModel): + id: UUID + username: str + display_name: str + created_at: str diff --git a/src/app/security.py b/src/app/security.py new file mode 100644 index 00000000..40b9a3ca --- /dev/null +++ b/src/app/security.py @@ -0,0 +1,65 @@ +import base64 +import binascii + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError +from argon2.low_level import Type + +# Учебные параметры Argon2id: устойчивее SHA-256, но подъём на ноутбуке +_TIME_COST = 2 +_MEMORY_COST = 32768 +_PARALLELISM = 1 + +_hasher: PasswordHasher | None = None +_dummy_hash: str | None = None + + +def get_hasher() -> PasswordHasher: + global _hasher + if _hasher is None: + _hasher = PasswordHasher( + time_cost=_TIME_COST, + memory_cost=_MEMORY_COST, + parallelism=_PARALLELISM, + hash_len=32, + salt_len=16, + type=Type.ID, + ) + return _hasher + + +def init_security() -> None: + global _dummy_hash + if _dummy_hash is None: + _dummy_hash = get_hasher().hash("messenger-dummy-password-not-used-for-login") + + +def hash_password(password: str) -> str: + return get_hasher().hash(password) + + +def verify_password(password: str, password_hash: str | None) -> bool: + init_security() + try: + get_hasher().verify(password_hash or _dummy_hash, password) + matched = True + except (VerifyMismatchError, InvalidHashError): + matched = False + return password_hash is not None and matched + + +def parse_basic(header: str) -> tuple[str, str] | None: + if not header: + return None + scheme, _, rest = header.partition(" ") + if scheme.lower() != "basic" or not rest.strip(): + return None + try: + decoded = base64.b64decode(rest.strip(), validate=True) + text = decoded.decode("utf-8") + except (ValueError, UnicodeError, binascii.Error): + return None + username, separator, password = text.partition(":") + if separator != ":" or len(username) > 256 or len(password) > 1024: + return None + return username, password diff --git a/src/app/sessions.py b/src/app/sessions.py new file mode 100644 index 00000000..7c9b0274 --- /dev/null +++ b/src/app/sessions.py @@ -0,0 +1,90 @@ +import hashlib +import json +import logging +import secrets +from datetime import timedelta +from uuid import UUID + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from app.config import Settings +from app.errors import AppError +from app.runtime import require_redis, require_settings +from app.timeutil import format_time, parse_time, utcnow + +logger = logging.getLogger("app.sessions") + + +def build_redis(settings: Settings) -> Redis: + return Redis.from_url( + settings.cache_url, + decode_responses=True, + socket_timeout=settings.redis_socket_timeout_seconds, + socket_connect_timeout=settings.redis_socket_timeout_seconds, + health_check_interval=30, + max_connections=settings.redis_max_connections, + ) + + +def session_key(token: str) -> str: + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "sess:" + digest + + +async def _call(awaitable): + try: + return await awaitable + except RedisError: + logger.warning("redis unavailable type=%s", "RedisError") + raise AppError(503, "unavailable", "Сервис временно недоступен") from None + + +def _read_payload(raw: str) -> tuple[UUID, object] | None: + try: + data = json.loads(raw) + if not isinstance(data, dict): + raise ValueError("payload") + return UUID(str(data["user_id"])), parse_time(data["expires_at"]) + except (ValueError, KeyError, TypeError): + return None + + +async def create_session(user_id: UUID) -> tuple[str, object]: + settings = require_settings() + token = secrets.token_urlsafe(32) + expires_at = utcnow() + timedelta(seconds=settings.session_ttl_seconds) + payload = json.dumps( + {"user_id": str(user_id), "expires_at": format_time(expires_at)}, + separators=(",", ":"), + ) + # PXAT — абсолютное время. Повтор AOF после рестарта не продлевает TTL. + await _call(require_redis().set(session_key(token), payload, pxat=int(expires_at.timestamp() * 1000))) + return token, expires_at + + +async def resolve_session(token: str) -> UUID | None: + key = session_key(token) + raw = await _call(require_redis().get(key)) + if raw is None: + return None + parsed = _read_payload(raw) + if parsed is None: + await _call(require_redis().delete(key)) + return None + user_id, expires_at = parsed + if expires_at <= utcnow(): + await _call(require_redis().delete(key)) + return None + return user_id + + +async def revoke_session(token: str) -> bool: + raw = await _call(require_redis().getdel(session_key(token))) + if raw is None: + return False + parsed = _read_payload(raw) + if parsed is None: + return False + _, expires_at = parsed + return expires_at > utcnow() diff --git a/src/app/storage/__init__.py b/src/app/storage/__init__.py new file mode 100644 index 00000000..f1a2a517 --- /dev/null +++ b/src/app/storage/__init__.py @@ -0,0 +1 @@ +"""Состояние лабораторной 2 хранится в PostgreSQL и Redis, не в памяти процесса.""" diff --git a/src/app/storage/memory.py b/src/app/storage/memory.py new file mode 100644 index 00000000..e8b0b7f0 --- /dev/null +++ b/src/app/storage/memory.py @@ -0,0 +1,5 @@ +"""Память процесса больше не хранит пользователей, чаты и сообщения. + +Источник истины для этих данных — PostgreSQL. +Источник истины для сессий — Redis с AOF и абсолютным PXAT. +""" diff --git a/src/app/timeutil.py b/src/app/timeutil.py new file mode 100644 index 00000000..08421fb4 --- /dev/null +++ b/src/app/timeutil.py @@ -0,0 +1,20 @@ +from datetime import datetime, timezone + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def format_time(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def parse_time(value: str) -> datetime: + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("time") + parsed = datetime.fromisoformat(value[:-1] + "+00:00") + if parsed.tzinfo is None: + raise ValueError("time") + return parsed.astimezone(timezone.utc) diff --git a/src/app/worker.py b/src/app/worker.py new file mode 100644 index 00000000..3f3abe13 --- /dev/null +++ b/src/app/worker.py @@ -0,0 +1,273 @@ +import asyncio +import hashlib +import json +import logging +import os +import signal +import socket +import sys +from pathlib import Path +from uuid import UUID + +from app.broker import DLQ_KEY, connect, declare, publish +from app.config import load_settings +from app.db import build_engine, build_session_factory +from app.errors import AppError +from app.faults import read_faults +from app.jobs import acquire_job, fail_or_retry, finalize_job, mark_objects_written, sweep_expired_leases +from app.migrate import wait_for_database, wait_for_schema +from app.pipeline import render_in_process, variant_names +from app.runtime import configure, require_settings, require_store +from app.s3 import build_store + +logger = logging.getLogger("app.worker") + + +class Publisher: + exchange = None + + +def configure_logging() -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + force=True, + ) + logging.getLogger("sqlalchemy.engine").setLevel(logging.WARNING) + logging.getLogger("aio_pika").setLevel(logging.WARNING) + logging.getLogger("aiormq").setLevel(logging.WARNING) + + +def flush_logs() -> None: + for handler in logging.getLogger().handlers: + handler.flush() + sys.stdout.flush() + + +def parse_job_id(body: bytes) -> UUID | None: + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeError, ValueError): + return None + if not isinstance(payload, dict) or payload.get("type") != "attachment.process.v1": + return None + try: + return UUID(str(payload.get("job_id"))) + except (TypeError, ValueError): + return None + + +async def emit_terminal(result: str) -> None: + if not result.startswith("failed:"): + return + payload = result[len("failed:") :] + exchange = Publisher.exchange + if exchange is None: + logger.warning("dlq skipped because the broker exchange is not ready") + return + try: + await publish(exchange, payload, DLQ_KEY) + logger.info("terminal failure copied to dlq") + except Exception as exc: + logger.warning("dlq publish failed type=%s", type(exc).__name__) + + +async def handle(message, owner: str) -> None: + job_id = parse_job_id(message.body) + if job_id is None: + logger.warning("discarded job message type=invalid") + return + acquired = await acquire_job(job_id, owner, bool(getattr(message, "redelivered", False))) + if acquired.action != "acquired" or acquired.job_id is None or acquired.attachment_id is None: + return + settings = require_settings() + faults = read_faults(settings.faults_file) + if faults.inject_transient and acquired.objects_written_at is None: + logger.warning("injected transient failure job_id=%s attempt=%s", acquired.job_id, acquired.attempt) + await emit_terminal(await fail_or_retry(acquired.job_id, "processing_failed", False)) + return + store = require_store() + try: + data = await store.get(acquired.staging_key) + except AppError as exc: + permanent = exc.status == 404 + await emit_terminal(await fail_or_retry(acquired.job_id, "processing_failed", permanent)) + return + digest = hashlib.sha256(data).hexdigest() + if digest != acquired.sha256 or len(data) != acquired.size_bytes: + await emit_terminal(await fail_or_retry(acquired.job_id, "processing_failed", True)) + return + timeout = faults.timeout_seconds or settings.job_timeout_seconds + rendered = await asyncio.to_thread( + render_in_process, + data, + settings.max_image_pixels, + settings.worker_image_memory_bytes, + settings.jpeg_quality, + settings.webp_quality, + settings.webp_method, + timeout, + faults.stall_seconds, + ) + if not rendered.get("ok"): + await emit_terminal( + await fail_or_retry( + acquired.job_id, + str(rendered.get("code") or "processing_failed"), + bool(rendered.get("permanent")), + ) + ) + return + await store.put(acquired.object_key, data, acquired.content_type) + stored = [] + for name in variant_names(): + item = rendered["variants"][name] + blob = item["data"] + key = "att/%s/%s" % (acquired.attachment_id, name) + await store.put(key, blob, item["content_type"]) + stored.append( + { + "name": name, + "content_type": item["content_type"], + "size_bytes": len(blob), + "width": item["width"], + "height": item["height"], + "sha256": hashlib.sha256(blob).hexdigest(), + "object_key": key, + } + ) + await mark_objects_written(acquired.job_id) + if faults.crash_after_s3 and acquired.objects_written_at is None: + logger.error("injected crash after object write before ack job_id=%s", acquired.job_id) + flush_logs() + os._exit(86) + outcome = await finalize_job(acquired.job_id, stored) + if outcome == "incomplete": + await emit_terminal(await fail_or_retry(acquired.job_id, "processing_failed", False)) + return + try: + await store.delete(acquired.staging_key) + except AppError: + logger.warning("staging delete failed job_id=%s", acquired.job_id) + logger.info("job ready job_id=%s attachment_id=%s", acquired.job_id, acquired.attachment_id) + + +async def beat(path: str, stop: asyncio.Event) -> None: + file = Path(path) + while not stop.is_set(): + file.write_text("ok", encoding="utf-8") + try: + await asyncio.wait_for(stop.wait(), timeout=2) + except asyncio.TimeoutError: + continue + + +async def sweep_loop(stop: asyncio.Event) -> None: + while not stop.is_set(): + try: + await sweep_expired_leases() + except Exception as exc: + logger.warning("lease sweep failed type=%s", type(exc).__name__) + try: + await asyncio.wait_for(stop.wait(), timeout=2) + except asyncio.TimeoutError: + continue + + +async def open_broker(url: str): + for attempt in range(40): + try: + return await connect(url) + except Exception as exc: + logger.warning("broker not ready type=%s attempt=%s", type(exc).__name__, attempt + 1) + await asyncio.sleep(0.5) + raise SystemExit("broker is not ready") + + +async def consume(stop: asyncio.Event) -> None: + settings = require_settings() + owner = "%s:%s" % (socket.gethostname(), os.getpid()) + semaphore = asyncio.Semaphore(settings.worker_concurrency) + + async def on_message(message) -> None: + async with semaphore: + try: + await handle(message, owner) + await message.ack() + except Exception as exc: + logger.warning("worker message failed type=%s", type(exc).__name__) + try: + await message.nack(requeue=False) + except Exception as nack_error: + logger.warning("nack failed type=%s", type(nack_error).__name__) + + while not stop.is_set(): + connection = None + try: + connection = await open_broker(settings.broker_url) + channel = await connection.channel() + await channel.set_qos(prefetch_count=settings.worker_concurrency) + publisher = await connection.channel(publisher_confirms=True) + exchange, _queue = await declare(publisher) + _exchange, queue = await declare(channel) + Publisher.exchange = exchange + await queue.consume(on_message) + logger.info("worker consuming prefetch=%s", settings.worker_concurrency) + await stop.wait() + except Exception as exc: + if stop.is_set(): + break + logger.warning("worker consume interrupted type=%s", type(exc).__name__) + await asyncio.sleep(1) + finally: + if connection is not None: + try: + await connection.close() + except Exception as exc: + logger.warning("broker close failed type=%s", type(exc).__name__) + + +async def serve() -> None: + settings = load_settings() + engine = build_engine(settings) + store = build_store(settings) + stop = asyncio.Event() + try: + await wait_for_database(engine) + await wait_for_schema(engine, settings.migrations_dir) + for _attempt in range(40): + try: + await store.ping() + break + except Exception: + logger.warning("s3 not ready") + await asyncio.sleep(0.5) + else: + raise SystemExit("s3 is not ready") + configure(build_session_factory(engine), None, settings, store) + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop.set) + except NotImplementedError: + pass + heart = asyncio.create_task(beat("/tmp/worker-heartbeat", stop)) + sweeper = asyncio.create_task(sweep_loop(stop)) + try: + await consume(stop) + finally: + stop.set() + heart.cancel() + sweeper.cancel() + finally: + await engine.dispose() + + +def main() -> None: + configure_logging() + asyncio.run(serve()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 00000000..52d195f5 --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Негативные проверки вложений для оценки 4: лимиты, MIME, ACL, чужой чат.""" + +import json +import os +import struct +import sys +import time +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def png(width: int, height: int, extra: bytes = b"") -> bytes: + def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + extra + chunk(b"IEND", b"") + + +def upload(client, actor, chat, filename, body, content_type, expected): + return client.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], filename), + expected, + actor=actor, + raw=body, + headers={"Content-Type": content_type}, + schema="Attachment" if expected in (201, 202) else None, + ) + + +def main(): + client = Client(BASE, timeout=30) + alice, bob, eve, chat = client.bootstrap() + other = client.call( + "POST", + "/api/v1/chats", + 201, + actor=eve, + body={"title": "Чужой", "member_ids": [eve["id"]]}, + schema="Chat", + )["json"] + att, content = client.attachment(alice, chat) + att = client.wait_ready(alice, att) + check(content == SAMPLE, "fixture changed") + + too_big = upload(client, alice, chat, "big.png", SAMPLE + b"x" * (10 * 1024 * 1024), "image/png", 413) + check(too_big["json"]["error"]["code"] == "payload_too_large", "oversize code") + + spoof = upload(client, alice, chat, "spoof.png", SAMPLE, "image/jpeg", 415) + check(spoof["json"]["error"]["code"] == "unsupported_media_type", "spoofed MIME code") + + broken = upload(client, alice, chat, "broken.png", b"\x89PNG\r\n\x1a\nnot-an-image", "image/png", 415) + check(broken["json"]["error"]["code"] == "unsupported_media_type", "corrupt image code") + + huge = upload(client, alice, chat, "huge.png", png(10001, 4000), "image/png", 413) + check(huge["json"]["error"]["code"] == "payload_too_large", "pixel limit code") + + slash = upload(client, alice, chat, "a/b.png", SAMPLE, "image/png", 400) + check(slash["json"]["error"]["code"] == "invalid_request", "slash filename code") + + missing = client.call( + "GET", + "/api/v1/attachments/%s" % uuid.uuid4(), + 404, + actor=alice, + ) + check(missing["json"]["error"]["code"] == "not_found", "unknown attachment") + + stolen = client.call("GET", "/api/v1/attachments/%s" % att["id"], 404, actor=eve) + check(stolen["json"]["error"]["code"] == "not_found", "outsider metadata") + stolen_bytes = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], 404, actor=eve) + check(stolen_bytes["json"]["error"]["code"] == "not_found", "outsider bytes") + + foreign_file = upload(client, eve, other, "other.png", SAMPLE, "image/png", 202)["json"] + linked = client.call( + "POST", + "/api/v1/chats/%s/messages" % chat["id"], + 404, + actor=alice, + body={"text": "", "attachment_ids": [foreign_file["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + check(linked["json"]["error"]["code"] == "not_found", "cross-chat attachment") + + unknown_link = client.call( + "POST", + "/api/v1/chats/%s/messages" % chat["id"], + 404, + actor=alice, + body={"text": "x", "attachment_ids": [str(uuid.uuid4())]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + check(unknown_link["json"]["error"]["code"] == "not_found", "unknown attachment_id") + + ok = client.message(alice, chat, "", attachments=[att["id"]])["json"] + check(ok["attachment_ids"] == [att["id"]], "empty text with attachment") + download = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob) + check(download["bytes"] == SAMPLE, "member download mismatch") + + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "attachments.json").write_text( + json.dumps( + { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "oversize": too_big["status"], + "spoofed_mime": spoof["status"], + "corrupt": broken["status"], + "pixels": huge["status"], + "foreign_chat": linked["status"], + "outsider": stolen["status"], + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print("PASS: attachment validation and ACL") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/failover.py b/student-tests/failover.py new file mode 100644 index 00000000..a655e074 --- /dev/null +++ b/student-tests/failover.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Запуск публичного failover на Windows и POSIX с тем же scripts/stop-one.sh.""" + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def script(): + if os.name == "nt": + cmd = ROOT / "scripts" / "stop-one.cmd" + if cmd.is_file(): + return str(cmd) + sh = ROOT / "scripts" / "stop-one.sh" + return str(sh) + + +def main(): + env = os.environ.copy() + command = [ + sys.executable, + str(ROOT / "tests" / "resilience.py"), + "--mode", + "failover", + "--action-script", + script(), + "--base-url", + env.get("BASE_URL", "http://127.0.0.1:8080"), + ] + completed = subprocess.run(command, cwd=str(ROOT), env=env) + raise SystemExit(completed.returncode) + + +if __name__ == "__main__": + main() diff --git a/student-tests/gc_orphans.py b/student-tests/gc_orphans.py new file mode 100644 index 00000000..36227e21 --- /dev/null +++ b/student-tests/gc_orphans.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Orphan object after S3 PUT without DB row is removed after the grace period.""" + +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") + + +def compose(*args, timeout=120): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("docker compose failed: " + (proc.stderr or proc.stdout)[-400:]) + return proc.stdout + + +def main(): + client = Client(BASE, timeout=20) + alice, _bob, _eve, chat = client.bootstrap() + att, _content = client.attachment(alice, chat) + orphan = "att/orphan-%s" % uuid.uuid4().hex + script = """ +set -eu +ROOT_USER=$(cat /run/secrets/minio_root_user) +ROOT_PASS=$(cat /run/secrets/minio_root_password) +mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASS" >/dev/null +printf 'orphan' | mc pipe local/%s/%s +mc ls local/%s/%s >/dev/null +""" % (BUCKET, orphan, BUCKET, orphan) + compose("run", "--rm", "--no-deps", "--entrypoint", "/bin/sh", "minio-init", "-c", script) + time.sleep(2) + compose( + "run", + "--rm", + "--no-deps", + "-e", + "APP_ROLE=gc", + "-e", + "GC_ONCE=1", + "-e", + "ATTACHMENT_GC_GRACE_SECONDS=1", + "gc", + timeout=180, + ) + missing = compose( + "run", + "--rm", + "--no-deps", + "--entrypoint", + "/bin/sh", + "minio-init", + "-c", + """ +set -eu +ROOT_USER=$(cat /run/secrets/minio_root_user) +ROOT_PASS=$(cat /run/secrets/minio_root_password) +mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASS" >/dev/null +if mc ls local/%s/%s >/dev/null 2>&1; then exit 7; fi +""" + % (BUCKET, orphan), + ) + del missing + client.call("GET", "/api/v1/attachments/%s" % att["id"], actor=alice, schema="Attachment") + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "gc_orphans.json").write_text( + json.dumps( + { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "orphan_removed": True, + "live_attachment_kept": att["id"], + "grace_seconds": 1, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print("PASS: orphan object removed, live attachment kept") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/grade4.py b/student-tests/grade4.py new file mode 100644 index 00000000..3ff1e97a --- /dev/null +++ b/student-tests/grade4.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +"""Сценарии оценки 4: варианты, остановка worker, outbox, crash до ack, lease, replay. + +Стенд уже поднят. Скрипт останавливает worker, relay, rabbitmq и API и снова их запускает. +Токены и пароли не печатаются. +""" + +import hashlib +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run +from image_probe import jpeg_size + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +FAULTS = ROOT / "infra" / "faults.json" +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +SAFE = { + "inject_transient": False, + "crash_after_s3": False, + "relay_exit_before_publish": False, + "stall_seconds": 0, + "timeout_seconds": 0, +} + + +def compose(*args, timeout=180): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("docker compose %s failed" % args[0]) + return proc.stdout + + +def psql(sql): + proc = subprocess.run( + [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-v", + "ON_ERROR_STOP=1", + "-At", + "-c", + sql, + ], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=60, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("database query failed") + return proc.stdout.strip() + + +def logs(service): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", "logs", "--since", "3m", "--no-log-prefix", service], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=60, + encoding="utf-8", + errors="replace", + ) + return proc.stdout + + +def write_faults(data): + payload = dict(SAFE) + payload.update(data) + temporary = FAULTS.with_suffix(".tmp") + temporary.write_text(json.dumps(payload), encoding="utf-8") + temporary.replace(FAULTS) + + +def job_id_for(attachment_id): + uuid.UUID(attachment_id) + return psql("SELECT job_id FROM attachment_jobs WHERE attachment_id = '%s'" % attachment_id) + + +def variant_counts(attachment_id): + uuid.UUID(attachment_id) + text = psql( + "SELECT name || ':' || count(*) FROM attachment_variants WHERE attachment_id = '%s' GROUP BY name ORDER BY name" + % attachment_id + ) + counts = {} + if text: + for line in text.splitlines(): + name, amount = line.split(":", 1) + counts[name] = int(amount) + return counts + + +def assert_single_variants(attachment_id): + counts = variant_counts(attachment_id) + check(counts == {"crop": 1, "optimized": 1, "thumbnail": 1}, "variant rows are not unique") + return counts + + +def wait_status(client, actor, attachment, status, seconds=90): + deadline = time.monotonic() + seconds + last = attachment + while time.monotonic() < deadline: + last = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=actor, schema="Attachment")["json"] + if last["status"] == status: + return last + if status != "failed" and last["status"] == "failed": + raise Failure("attachment failed while waiting for " + status) + time.sleep(0.4) + raise Failure("attachment stayed " + str(last.get("status")) + " while waiting for " + status) + + +def replay(job): + uuid.UUID(job) + proc = subprocess.run( + [ + "docker", + "compose", + "--env-file", + ".env", + "run", + "--rm", + "--no-deps", + "--entrypoint", + "python", + "worker", + "-m", + "app.replay", + job, + ], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=90, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("replay command failed") + line = [item for item in proc.stdout.splitlines() if item.startswith("{")] + check(line, "replay did not print a result") + return json.loads(line[-1]) + + +def check_ready_image(client, actor, ready, content): + variants = {item["name"]: item for item in ready["variants"]} + check(set(variants) == {"thumbnail", "crop", "optimized"}, "ready variants mismatch") + thumb = variants["thumbnail"] + crop = variants["crop"] + optimized = variants["optimized"] + check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail metadata mismatch") + check(thumb["content_type"] == "image/jpeg", "thumbnail type mismatch") + check((crop["width"], crop["height"]) == (128, 128), "crop metadata mismatch") + check(crop["content_type"] == "image/jpeg", "crop type mismatch") + check((optimized["width"], optimized["height"]) == (320, 200), "optimized metadata mismatch") + check(optimized["content_type"] == "image/webp", "optimized type mismatch") + original = client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=actor) + check(original["bytes"] == content, "original bytes changed") + for name, meta, magic in ( + ("thumbnail", thumb, b"\xff\xd8"), + ("crop", crop, b"\xff\xd8"), + ("optimized", optimized, b"RIFF"), + ): + body = client.call( + "GET", + "/api/v1/attachments/" + ready["id"] + "/content?variant=" + name, + actor=actor, + ) + check(body["headers"].get_content_type() == meta["content_type"], name + " content type mismatch") + check(len(body["bytes"]) == meta["size_bytes"], name + " size mismatch") + check(hashlib.sha256(body["bytes"]).hexdigest() == meta["sha256"], name + " checksum mismatch") + check(body["bytes"][:4] == magic or body["bytes"][:2] == magic, name + " magic mismatch") + if name != "optimized": + check(jpeg_size(body["bytes"]) == (meta["width"], meta["height"]), name + " jpeg size mismatch") + check(optimized and body["bytes"][8:12] == b"WEBP", "optimized is not webp") + + +def main(): + write_faults({}) + evidence = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())} + client = Client(BASE, timeout=30) + alice, bob, _eve, chat = client.bootstrap() + try: + att, content = client.attachment(alice, chat) + check(att["status"] == "queued" and att["variants"] == [], "upload snapshot is not queued") + ready = client.wait_ready(bob, att, wait_seconds=90) + check_ready_image(client, bob, ready, content) + assert_single_variants(att["id"]) + evidence["happy_path"] = {"attachment_id": att["id"], "job_id": job_id_for(att["id"]), "status": "ready"} + print("PASS: thumbnail, crop and optimized") + + compose("stop", "worker") + queued, queued_bytes = client.attachment(alice, chat) + blocked = client.call("GET", "/api/v1/attachments/" + queued["id"] + "/content", 409, actor=bob) + check(blocked["json"]["error"]["code"] == "conflict", "content of a queued file is not 409") + linked = client.message(alice, chat, "", attachments=[queued["id"]])["json"] + check(linked["attachment_ids"] == [queued["id"]], "queued attachment was rejected") + client.message(bob, chat, "сообщение без вложения") + compose("restart", "api") + compose("restart", "rabbitmq") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + break + except Failure: + time.sleep(1) + else: + raise Failure("api did not recover after restart") + still = client.call("GET", "/api/v1/attachments/" + queued["id"], actor=alice, schema="Attachment")["json"] + check(still["status"] in {"queued", "processing", "ready"}, "accepted job disappeared") + compose("start", "worker") + recovered = wait_status(client, bob, queued, "ready", 90) + downloaded = client.call("GET", "/api/v1/attachments/" + queued["id"] + "/content", actor=bob) + check(downloaded["bytes"] == queued_bytes, "recovered original changed") + assert_single_variants(queued["id"]) + evidence["worker_stopped"] = { + "attachment_id": queued["id"], + "job_id": job_id_for(queued["id"]), + "content_before_worker": 409, + "status_after_recovery": recovered["status"], + "rabbit_restarted": True, + "api_restarted": True, + } + print("PASS: stopped worker, broker restart, same job reached ready") + + compose("stop", "relay") + held, _held_bytes = client.attachment(alice, chat) + held_job = job_id_for(held["id"]) + unpublished = psql( + "SELECT count(*) FROM job_outbox WHERE job_id = '%s' AND published_at IS NULL" % held_job + ) + check(unpublished == "1", "outbox row was published while relay was stopped") + compose("start", "relay") + held_ready = wait_status(client, alice, held, "ready", 90) + evidence["outbox_gap"] = { + "attachment_id": held["id"], + "job_id": held_job, + "unpublished_before_restart": int(unpublished), + "status": held_ready["status"], + } + print("PASS: commit without publish recovered from the outbox") + + write_faults({"relay_exit_before_publish": True}) + time.sleep(0.8) + crashed, _bytes = client.attachment(alice, chat) + crash_job = job_id_for(crashed["id"]) + seen = False + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + if "injected exit after outbox commit before broker publish" in logs("relay"): + seen = True + break + time.sleep(0.5) + check(seen, "relay did not exit between commit and publish") + crash_ready = wait_status(client, alice, crashed, "ready", 90) + gap = psql("SELECT gap_injected FROM job_outbox WHERE job_id = '%s' ORDER BY created_at LIMIT 1" % crash_job) + check(gap == "t", "gap latch was not stored") + jobs = psql("SELECT count(*) FROM attachment_jobs WHERE attachment_id = '%s'" % crashed["id"]) + check(jobs == "1", "crash recovery created another job") + evidence["relay_crash"] = { + "attachment_id": crashed["id"], + "job_id": crash_job, + "gap_injected": True, + "log_seen": True, + "status": crash_ready["status"], + } + write_faults({}) + print("PASS: relay crash before publish recovered the same job") + + write_faults({"crash_after_s3": True}) + time.sleep(0.8) + side, side_bytes = client.attachment(alice, chat) + side_job = job_id_for(side["id"]) + seen = False + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + if "injected crash after object write before ack job_id=%s" % side_job in logs("worker"): + seen = True + break + time.sleep(0.5) + check(seen, "worker did not exit after the object write") + side_ready = wait_status(client, bob, side, "ready", 90) + side_download = client.call("GET", "/api/v1/attachments/" + side["id"] + "/content", actor=bob) + check(side_download["bytes"] == side_bytes, "redelivery changed the original") + assert_single_variants(side["id"]) + evidence["crash_before_ack"] = { + "attachment_id": side["id"], + "job_id": side_job, + "log_seen": True, + "status": side_ready["status"], + "variant_rows": assert_single_variants(side["id"]), + } + write_faults({}) + print("PASS: crash after object write redelivered the same result") + + compose("stop", "worker", "relay") + leased, _leased_bytes = client.attachment(alice, chat) + leased_job = job_id_for(leased["id"]) + psql( + "UPDATE attachment_jobs SET status = 'processing', attempt = 1, lease_owner = 'dead', " + "lease_until = now() - interval '2 minutes' WHERE job_id = '%s'" % leased_job + ) + psql("UPDATE job_outbox SET published_at = now() WHERE job_id = '%s' AND published_at IS NULL" % leased_job) + psql("UPDATE attachments SET status = 'processing', error_code = NULL WHERE id = '%s'" % leased["id"]) + compose("start", "relay", "worker") + lease_ready = wait_status(client, alice, leased, "ready", 90) + evidence["expired_lease"] = { + "attachment_id": leased["id"], + "job_id": leased_job, + "status": lease_ready["status"], + } + print("PASS: expired processing lease was recovered") + + write_faults({"inject_transient": True}) + time.sleep(0.8) + failed, _failed_bytes = client.attachment(alice, chat) + failed_job = job_id_for(failed["id"]) + failed_meta = wait_status(client, alice, failed, "failed", 90) + check(failed_meta["error_code"] == "retry_exhausted", "exhausted job has no safe error code") + denied = client.call("GET", "/api/v1/attachments/" + failed["id"] + "/content", 409, actor=alice) + check(denied["json"]["error"]["code"] == "conflict", "failed content is not 409") + registry = psql("SELECT count(*) FROM job_failures WHERE job_id = '%s'" % failed_job) + check(int(registry) >= 1, "failure registry is empty") + dlq = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "list_queues", "name", "messages") + check("attachment.jobs.dlq" in dlq, "dlq is missing") + write_faults({}) + time.sleep(0.5) + first = replay(failed_job) + check(first["outcome"] == "enqueued", "failed job was not replayed") + replayed = wait_status(client, alice, failed, "ready", 90) + second = replay(failed_job) + check(second["outcome"] == "already_ready", "replay of a ready job was not a no-op") + assert_single_variants(failed["id"]) + evidence["retry_and_replay"] = { + "attachment_id": failed["id"], + "job_id": failed_job, + "error_code": failed_meta["error_code"], + "failure_rows": int(registry), + "first_replay": first["outcome"], + "second_replay": second["outcome"], + "status": replayed["status"], + } + print("PASS: backoff exhausted to failed, replay is safe") + + write_faults({"stall_seconds": 5, "timeout_seconds": 1}) + time.sleep(0.8) + stalled, _stalled_bytes = client.attachment(alice, chat) + client.message(alice, chat, "API отвечает, пока worker ограничен по времени") + client.call("GET", "/health/ready", schema="Health") + stalled_meta = wait_status(client, alice, stalled, "failed", 90) + check(stalled_meta["error_code"] == "retry_exhausted", "timed out job was not exhausted") + write_faults({}) + time.sleep(0.5) + stalled_job = job_id_for(stalled["id"]) + outcome = replay(stalled_job) + check(outcome["outcome"] == "enqueued", "timed out job was not replayed") + stalled_ready = wait_status(client, alice, stalled, "ready", 90) + evidence["timeout"] = { + "attachment_id": stalled["id"], + "job_id": stalled_job, + "error_code": stalled_meta["error_code"], + "status": stalled_ready["status"], + "api_ready_during_stall": True, + } + print("PASS: worker time limit does not block the API") + finally: + write_faults({}) + try: + compose("start", "worker", "relay", "api", "rabbitmq") + except Failure: + pass + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "grade4.json").write_text(json.dumps(evidence, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS: grade 4 failure scenarios") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/level3.py b/student-tests/level3.py new file mode 100644 index 00000000..057caabc --- /dev/null +++ b/student-tests/level3.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Проверки lab3: разные реплики, идемпотентность через LB, гонка одного ключа.""" + +import json +import os +import sys +import threading +import time +import uuid +from collections import Counter +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +def history(client, actor, chat): + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + result = [] + seen = set() + for _ in range(1000): + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + result.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return result + check(cursor not in seen, "cursor cycle") + seen.add(cursor) + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(cursor, safe="") + raise Failure("history pagination exceeded safety bound") + + +def test_instances(client, actor): + seen = [] + for _ in range(40): + seen.append(client.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"]["X-Instance-Id"]) + unique = sorted(set(seen)) + check(len(unique) >= 2, "less than two API instances observed: %s" % unique) + return {"unique_instance_ids": unique, "samples": len(seen), "counts": dict(Counter(seen))} + + +def test_idempotency(client, alice, bob, chat): + key = uuid.uuid4().hex + first = client.message(alice, chat, "Один ключ", key=key) + replay = client.message(alice, chat, "Один ключ", key=key) + check(first["json"] == replay["json"], "idempotent replay differs") + check(first["json"]["id"] == replay["json"]["id"], "idempotent replay allocated a new id") + conflict = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 409, + actor=alice, + body={"text": "другой body"}, + headers={"Idempotency-Key": key}, + ) + check(conflict["json"]["error"]["code"] == "conflict", "wrong body code") + missing = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=alice, + body={"text": "без ключа"}, + ) + check(missing["json"]["error"]["code"] == "invalid_request", "missing key code") + other_user = client.message(bob, chat, "Один ключ", key=key) + check(other_user["json"]["id"] != first["json"]["id"], "key leaked across users") + return first["json"]["id"] + + +def test_concurrent_key(client, alice, chat): + key = uuid.uuid4().hex + text = "Гонка ключа " + key + barrier = threading.Barrier(12) + results = [] + errors = [] + lock = threading.Lock() + + def worker(): + local = Client(BASE, timeout=20) + try: + barrier.wait(20) + response = local.message(alice, chat, text, key=key) + with lock: + results.append((response["status"], response["json"]["id"], response["headers"]["X-Instance-Id"])) + except Exception as exc: + with lock: + errors.append("%s: %s" % (type(exc).__name__, exc)) + + threads = [threading.Thread(target=worker) for _ in range(12)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + check(not errors, "concurrent idempotent POST failed: " + "; ".join(errors[:5])) + check(len(results) == 12, "expected 12 successes, got %s" % len(results)) + ids = {item[1] for item in results} + instances = sorted({item[2] for item in results}) + check(len(ids) == 1, "concurrent key created %s messages" % len(ids)) + listed = history(client, alice, chat) + matches = [item for item in listed if item["text"] == text] + check(len(matches) == 1, "history contains %s copies of the concurrent key" % len(matches)) + return {"message_id": next(iter(ids)), "instances": instances, "successes": len(results)} + + +def main(): + client = Client(BASE, timeout=10) + alice, bob, _eve, chat = client.bootstrap() + instances = test_instances(client, alice) + message_id = test_idempotency(client, alice, bob, chat) + race = test_concurrent_key(client, alice, chat) + EVIDENCE.mkdir(exist_ok=True) + report = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "base_url": BASE, + "instances": instances, + "idempotent_message_id_present": bool(message_id), + "concurrent_key": race, + } + (EVIDENCE / "level3.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS: level 3 replica routing and shared idempotency") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/level4.py b/student-tests/level4.py new file mode 100644 index 00000000..5633d010 --- /dev/null +++ b/student-tests/level4.py @@ -0,0 +1,642 @@ +#!/usr/bin/env python3 +"""Проверки уровня 4: гонка, откат, EXPLAIN, TTL, logout и отказ Redis/PostgreSQL. + +Стенд уже должен быть поднят: scripts/bootstrap и docker compose up. +Скрипт останавливает redis и postgres и снова их запускает. +Токены и пароли не печатаются и не пишутся в evidence. +""" + +import asyncio +import base64 +import json +import os +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +sys.path.insert(0, str(ROOT / "src")) + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +class Failure(AssertionError): + pass + + +def check(condition, message): + if not condition: + raise Failure(message) + + +def host_secret(name): + path = ROOT / "secrets" / name + if not path.is_file(): + raise Failure("missing secrets/%s; run scripts/bootstrap first" % name) + value = path.read_text(encoding="utf-8").strip() + if not value: + raise Failure("empty secrets/%s" % name) + return value + + +def call(method, path, expected, body=None, token=None, authorization=None, user_id=None, timeout=15, headers=None): + extra = headers or {} + headers = {"Accept": "application/json"} + headers.update(extra) + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + headers["Content-Type"] = "application/json" + if authorization: + headers["Authorization"] = authorization + elif token: + headers["Authorization"] = "Bearer " + token + if user_id: + headers["X-User-Id"] = user_id + request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + try: + response = urllib.request.urlopen(request, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise Failure(method + " " + path.split("?")[0] + " failed: " + type(exc).__name__) from None + with response: + status = response.status + payload = response.read() + header_id = response.headers.get("X-Request-Id") + challenge = response.headers.get("WWW-Authenticate", "") + allowed = [expected] if isinstance(expected, int) else list(expected) + check(status in allowed, "%s %s: expected %s, got %s" % (method, path.split("?")[0], allowed, status)) + check(header_id and header_id.strip(), "missing X-Request-Id") + parsed = None + if payload: + try: + parsed = json.loads(payload) + except json.JSONDecodeError as exc: + raise Failure("response is not JSON") from exc + if status >= 400: + check(isinstance(parsed, dict) and set(parsed) == {"error"}, "error envelope") + error = parsed["error"] + check(set(error) == {"code", "message", "request_id"}, "error fields") + check(error["request_id"] == header_id, "request_id differs from header") + lowered = error["message"].lower() + for leaked in ("traceback", "select ", "password", "syntax", "asyncpg", "sqlalchemy"): + check(leaked not in lowered, "error message leaked internals") + if status == 204: + check(not payload, "204 must not contain a body") + return {"status": status, "json": parsed, "www": challenge} + + +def basic(username, password): + raw = base64.b64encode(("%s:%s" % (username, password)).encode("utf-8")).decode("ascii") + return "Basic " + raw + + +def register(label, password=None): + username = ("u_%s_%s" % (label, uuid.uuid4().hex[:10])).lower() + password = password or ("T3st-" + uuid.uuid4().hex) + created = call( + "POST", + "/api/v1/users", + 201, + body={"username": username, "display_name": "Студент " + label, "password": password}, + )["json"] + check("password" not in created, "password leaked") + session = call("POST", "/api/v1/auth/sessions", 201, authorization=basic(username, password))["json"] + check(session["token_type"] == "Bearer" and session["user"]["id"] == created["id"], "session shape") + check(len(session["token"]) >= 32, "token is too short") + return {"id": created["id"], "username": username, "password": password, "token": session["token"]} + + +def compose(*args, timeout=120): + proc = subprocess.run( + ["docker", "compose", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("docker compose %s failed with code %s" % (args[0], proc.returncode)) + return proc.stdout + + +def psql(sql, timeout=60): + proc = subprocess.run( + [ + "docker", + "compose", + "exec", + "-T", + "postgres", + "psql", + "-U", + "messenger", + "-d", + "messenger", + "-v", + "ON_ERROR_STOP=1", + "-At", + "-c", + sql, + ], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("psql failed") + return proc.stdout.strip() + + +def redis_cli(*args): + proc = subprocess.run( + ["docker", "compose", "exec", "-T", "redis", "redis-cli", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=20, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("redis-cli failed") + return proc.stdout.strip() + + +def sql_quote(value): + text = str(value) + if "'" in text or ";" in text or "\n" in text: + raise Failure("refusing unsafe SQL literal") + return "'" + text + "'" + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def wait_ready(timeout=90): + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + try: + call("GET", "/health/ready", 200, timeout=5) + return + except Failure as exc: + last = str(exc) + time.sleep(0.5) + raise Failure("API did not become ready: " + last) + + +def test_credentials(): + user = register("cred") + wrong = call( + "POST", + "/api/v1/auth/sessions", + 401, + authorization=basic(user["username"], "wrong-password-value"), + ) + missing = call( + "POST", + "/api/v1/auth/sessions", + 401, + authorization=basic("no_such_user_zz", "wrong-password-value"), + ) + check(wrong["json"]["error"]["code"] == "unauthorized", "wrong password code") + check(wrong["json"]["error"]["message"] == missing["json"]["error"]["message"], "credential errors differ") + check(wrong["www"].lower().startswith("basic "), "basic challenge") + bare = call("GET", "/api/v1/users/me", 401) + check(bare["www"].lower().startswith("bearer "), "bearer challenge") + only_header = call("GET", "/api/v1/users/me", 401, user_id=user["id"]) + check(only_header["json"]["error"]["code"] == "unauthorized", "X-User-Id alone was accepted") + other = register("other") + me = call("GET", "/api/v1/users/me", 200, token=user["token"], user_id=other["id"])["json"] + check(me["id"] == user["id"], "X-User-Id overrode bearer") + basic_on_api = call("GET", "/api/v1/users/me", 401, authorization=basic(user["username"], user["password"])) + check(basic_on_api["www"].lower().startswith("bearer "), "basic accepted on business API") + print("PASS: credentials, same 401, X-User-Id ignored") + + +def test_unicode_password(): + password = "Пароль:пример-12" + check(12 <= len(password) <= 128 and ":" in password, "fixture password") + user = register("uni", password=password) + again = call("POST", "/api/v1/auth/sessions", 201, authorization=basic(user["username"], password))["json"] + check(again["user"]["id"] == user["id"], "unicode password login failed") + print("PASS: unicode password with colon") + + +def test_logout(): + user = register("bye") + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + rejected = call("GET", "/api/v1/users/me", 401, token=user["token"]) + check(rejected["www"].lower().startswith("bearer "), "revoked token challenge") + again = call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + check(again["json"]["error"]["code"] == "unauthorized", "second logout") + print("PASS: logout revokes the session") + + +def test_injection_and_rollback(): + user = register("sql") + chat = call( + "POST", + "/api/v1/chats", + 201, + token=user["token"], + body={"title": "SQL", "member_ids": [user["id"]]}, + )["json"] + payload = "'; DROP TABLE messages; --" + sent = call( + "POST", + "/api/v1/chats/%s/messages" % chat["id"], + 201, + token=user["token"], + body={"text": payload}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + )["json"] + check(sent["text"] == payload, "text was rewritten") + check(psql("SELECT to_regclass('public.messages')") == "messages", "messages table disappeared") + listed = call("GET", "/api/v1/chats/%s/messages?limit=10" % chat["id"], 200, token=user["token"])["json"] + check(listed["items"][0]["text"] == payload, "stored text differs") + + before = int(psql("SELECT count(*) FROM chats")) + title = "partial-" + uuid.uuid4().hex + rejected = call( + "POST", + "/api/v1/chats", + 400, + token=user["token"], + body={"title": title, "member_ids": [user["id"], str(uuid.uuid4())]}, + ) + check(rejected["json"]["error"]["code"] == "invalid_request", "unknown member code") + after = int(psql("SELECT count(*) FROM chats")) + check(before == after, "unknown member committed a chat") + check(psql("SELECT count(*) FROM chats WHERE title = %s" % sql_quote(title)) == "0", "partial chat row exists") + print("PASS: parameterized text and unknown-member chat is not stored") + return user, chat + + +async def failpoint(owner_id, other_id, title): + os.environ["DATABASE_URL"] = host_secret("host_database_url") + os.environ["REDIS_URL"] = host_secret("host_cache_url") + from app.config import load_settings + from app.crud.chats import create_chat + from app.db import build_engine, build_session_factory + from app.runtime import configure + from app.sessions import build_redis + + settings = load_settings() + engine = build_engine(settings) + redis = build_redis(settings) + configure(build_session_factory(engine), redis, settings) + try: + try: + await create_chat( + uuid.UUID(owner_id), + title, + (uuid.UUID(owner_id), uuid.UUID(other_id)), + fail_after_insert=True, + ) + except RuntimeError as exc: + check(str(exc) == "rollback-check", "failpoint raised a different error") + else: + raise Failure("failpoint committed a chat") + finally: + await redis.aclose() + await engine.dispose() + + +def test_failpoint(owner_id, other_id): + title = "rollback-" + uuid.uuid4().hex + before = int(psql("SELECT count(*) FROM chats")) + try: + asyncio.run(failpoint(owner_id, other_id, title)) + except Failure: + raise + except Exception as exc: + raise Failure("rollback check failed: " + type(exc).__name__) from None + after = int(psql("SELECT count(*) FROM chats")) + left = psql("SELECT count(*) FROM chats WHERE title = %s" % sql_quote(title)) + check(before == after and left == "0", "chat insert survived the rollback") + print("PASS: composite chat insert rolls back before commit") + return {"chats_before": before, "chats_after": after, "title_rows": int(left)} + + +def test_race(): + username = "race_" + uuid.uuid4().hex[:8] + password = "Race-password-" + uuid.uuid4().hex[:8] + body = {"username": username, "display_name": "Гонка", "password": password} + barrier = threading.Barrier(16) + statuses = [] + lock = threading.Lock() + + def worker(): + try: + barrier.wait(20) + result = call("POST", "/api/v1/users", [201, 409], body=body, timeout=30) + with lock: + statuses.append(result["status"]) + except Exception as exc: + with lock: + statuses.append(str(exc)[:120]) + + threads = [threading.Thread(target=worker) for _ in range(16)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + created = statuses.count(201) + conflict = statuses.count(409) + check(created == 1 and conflict == 15 and len(statuses) == 16, "race statuses %s" % statuses) + print("PASS: 16 parallel registrations, one 201 and fifteen 409") + return {"created": created, "conflict": conflict} + + +def assert_plan(name, plan): + check("ix_messages_history" in plan, name + " plan ignores ix_messages_history") + check("Seq Scan" not in plan, name + " plan seq-scans messages") + check("Index Scan" in plan or "Index Only Scan" in plan, name + " plan is not an index scan") + check("Buffers:" in plan, name + " plan has no buffer stats") + + +def test_explain(user): + chat = call( + "POST", + "/api/v1/chats", + 201, + token=user["token"], + body={"title": "План", "member_ids": [user["id"]]}, + )["json"] + chat_sql = sql_quote(chat["id"]) + user_sql = sql_quote(user["id"]) + psql( + "INSERT INTO messages (id, chat_id, sender_id, body, created_at) " + "SELECT gen_random_uuid(), %s, %s, 'row-' || g, " + "TIMESTAMPTZ '2026-01-01 00:00:00+00' + (g || ' microseconds')::interval " + "FROM generate_series(1, 10000) AS g" % (chat_sql, user_sql), + timeout=60, + ) + count = int(psql("SELECT count(*) FROM messages WHERE chat_id = %s" % chat_sql)) + check(count >= 10000, "expected at least 10000 messages, got %s" % count) + psql("ANALYZE messages") + midpoint = psql( + "SELECT to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD HH24:MI:SS.US'), id::text " + "FROM messages WHERE chat_id = %s ORDER BY created_at, id OFFSET 5000 LIMIT 1" % chat_sql + ) + created_at, message_id = midpoint.split("|") + late = psql( + "EXPLAIN (ANALYZE, BUFFERS) SELECT id, body FROM messages " + "WHERE chat_id = %s AND (created_at, id) > (%s::timestamptz, %s::uuid) " + "ORDER BY created_at, id LIMIT 50" + % (chat_sql, sql_quote(created_at + "+00"), sql_quote(message_id)), + timeout=60, + ) + first = psql( + "EXPLAIN (ANALYZE, BUFFERS) SELECT id, body FROM messages " + "WHERE chat_id = %s ORDER BY created_at, id LIMIT 50" % chat_sql, + timeout=60, + ) + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "explain_messages.txt").write_text( + "messages=%s\n\n-- first page\n%s\n\n-- page after row 5000\n%s\n" % (count, first, late), + encoding="utf-8", + ) + assert_plan("first", first) + assert_plan("late", late) + print("PASS: EXPLAIN uses ix_messages_history on %s messages" % count) + return {"messages": count, "chat_id": chat["id"]} + + +def test_expiry_and_migrations(): + from app.timeutil import parse_time + + users_before = int(psql("SELECT count(*) FROM users")) + port = free_port() + env = os.environ.copy() + env["PYTHONPATH"] = str(ROOT / "src") + env["PYTHONUNBUFFERED"] = "1" + env["APP_HOST"] = "127.0.0.1" + env["APP_PORT"] = str(port) + env["APP_ENV"] = "lab" + env["DATABASE_URL"] = host_secret("host_database_url") + env["REDIS_URL"] = host_secret("host_cache_url") + env["S3_ENDPOINT"] = host_secret("host_s3_endpoint") + env["S3_ACCESS_KEY_ID"] = host_secret("s3_access_key_id") + env["S3_SECRET_ACCESS_KEY"] = host_secret("s3_secret_access_key") + env["S3_BUCKET"] = "messenger-attachments" + env["SESSION_TTL_SECONDS"] = "2" + env.pop("DATABASE_URL_FILE", None) + env.pop("CACHE_URL_FILE", None) + proc = subprocess.Popen( + [sys.executable, "-m", "app"], + cwd=str(ROOT), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + ) + logs = [] + + def read_logs(): + if proc.stdout is None: + return + for line in proc.stdout: + if "://" not in line and "password" not in line.lower(): + logs.append(line) + + reader = threading.Thread(target=read_logs, daemon=True) + reader.start() + base = "http://127.0.0.1:%s" % port + try: + deadline = time.monotonic() + 40 + ready = False + while time.monotonic() < deadline: + if proc.poll() is not None: + raise Failure("short-ttl server exited: " + "".join(logs[-8:])) + try: + urllib.request.urlopen(base + "/health/ready", timeout=2).read() + ready = True + break + except (urllib.error.URLError, TimeoutError, OSError): + time.sleep(0.2) + check(ready, "short-ttl server did not become ready: " + "".join(logs[-8:])) + username = "ttl_" + uuid.uuid4().hex[:8] + password = "Ttl-password-" + uuid.uuid4().hex[:8] + body = json.dumps( + {"username": username, "display_name": "TTL", "password": password}, + ensure_ascii=False, + ).encode("utf-8") + request = urllib.request.Request( + base + "/api/v1/users", + data=body, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=20) as response: + created = json.loads(response.read()) + login = urllib.request.Request( + base + "/api/v1/auth/sessions", + data=b"", + headers={"Authorization": basic(username, password), "Accept": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(login, timeout=20) as response: + session = json.loads(response.read()) + expires = parse_time(session["expires_at"]) + delta = (expires - datetime.now(timezone.utc)).total_seconds() + check(0 < delta <= 3, "SESSION_TTL_SECONDS=2 was not applied") + time.sleep(delta + 1.0) + probe = urllib.request.Request( + base + "/api/v1/users/me", + headers={"Authorization": "Bearer " + session["token"], "Accept": "application/json"}, + ) + try: + with urllib.request.urlopen(probe, timeout=10) as response: + status = response.status + except urllib.error.HTTPError as error: + status = error.status + check(status == 401, "expired session status %s" % status) + finally: + if proc.poll() is None: + proc.terminate() + try: + proc.wait(timeout=8) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + reader.join(timeout=2) + users_after = int(psql("SELECT count(*) FROM users")) + versions = psql("SELECT count(*) FROM schema_migrations") + version_name = psql("SELECT version FROM schema_migrations ORDER BY version") + check(users_after == users_before + 1, "migrations duplicated or dropped users") + names = psql("SELECT version FROM schema_migrations ORDER BY version") + check(versions == "3", "expected three applied migrations, got %s" % versions) + check( + "001_initial.sql" in names and "002_idempotency.sql" in names and "003_attachments.sql" in names, + "migration set changed: %s" % names, + ) + print("PASS: short TTL expires and migrations stay idempotent") + return {"users_before": users_before, "users_after": users_after, "migration": version_name} + + +def test_redis_persistence(user): + from app.sessions import session_key + + before = int(redis_cli("PTTL", session_key(user["token"]))) + check(before > 300000, "session ttl is unexpectedly short") + compose("restart", "redis") + wait_ready() + after = int(redis_cli("PTTL", session_key(user["token"]))) + check(after > 0, "session disappeared after redis restart") + check(after <= before + 1000, "redis restart extended the session ttl") + me = call("GET", "/api/v1/users/me", 200, token=user["token"])["json"] + check(me["id"] == user["id"], "token did not survive redis restart") + print("PASS: Redis AOF kept the session and did not refresh TTL") + return {"pttl_before_ms": before, "pttl_after_ms": after} + + +def test_dependency_loss(user, chat_id): + observed = {} + compose("stop", "redis") + try: + live = call("GET", "/health/live", 200) + ready = call("GET", "/health/ready", 503) + anonymous = call("GET", "/api/v1/users/me", 401) + trusted = call("GET", "/api/v1/users/me", 503, token=user["token"]) + check(live["json"]["status"] == "ok", "live failed while redis is down") + check(ready["json"]["error"]["code"] == "unavailable", "ready code") + check(anonymous["json"]["error"]["code"] == "unauthorized", "anonymous became trusted") + check(trusted["json"]["error"]["code"] == "unavailable", "unverified token was accepted") + observed["redis_down"] = { + "live": live["status"], + "ready": ready["status"], + "anonymous": anonymous["status"], + "bearer": trusted["status"], + } + finally: + compose("start", "redis") + wait_ready() + me = call("GET", "/api/v1/users/me", 200, token=user["token"])["json"] + check(me["id"] == user["id"], "token lost after redis came back") + + compose("stop", "postgres") + try: + live = call("GET", "/health/live", 200) + ready = call("GET", "/health/ready", 503, timeout=20) + anonymous = call("GET", "/api/v1/users/me", 401, timeout=20) + denied = call("GET", "/api/v1/chats/" + chat_id, 503, token=user["token"], timeout=20) + check(live["json"]["status"] == "ok", "live depends on postgres") + check(ready["json"]["error"]["code"] == "unavailable", "ready while postgres is down") + check(anonymous["json"]["error"]["code"] == "unauthorized", "anonymous trusted without postgres") + check(denied["json"]["error"]["code"] == "unavailable", "request served without postgres") + observed["postgres_down"] = { + "live": live["status"], + "ready": ready["status"], + "anonymous": anonymous["status"], + "bearer": denied["status"], + } + finally: + compose("start", "postgres") + wait_ready(timeout=90) + history = call("GET", "/api/v1/chats/%s/messages?limit=5" % chat_id, 200, token=user["token"])["json"] + check(history["items"], "messages lost after postgres restart") + print("PASS: redis and postgres outages return 503 and do not trust anonymous clients") + return observed + + +def main(): + os.environ["DATABASE_URL"] = host_secret("host_database_url") + os.environ["REDIS_URL"] = host_secret("host_cache_url") + wait_ready(timeout=60) + test_credentials() + test_unicode_password() + test_logout() + user, chat = test_injection_and_rollback() + other = register("pair") + rollback = test_failpoint(user["id"], other["id"]) + race = test_race() + explain = test_explain(user) + expiry = test_expiry_and_migrations() + pttl = test_redis_persistence(user) + faults = test_dependency_loss(user, chat["id"]) + remaining = int(psql("SELECT count(*) FROM messages WHERE chat_id = %s" % sql_quote(explain["chat_id"]))) + check(remaining == explain["messages"], "bulk history lost after postgres restart") + EVIDENCE.mkdir(exist_ok=True) + report = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "base_url": BASE, + "race": race, + "rollback": rollback, + "explain": {"messages": explain["messages"]}, + "expiry_and_migrations": expiry, + "redis_restart": pttl, + "dependency_loss": faults, + "messages_after_postgres_restart": remaining, + } + (EVIDENCE / "level4.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS: level 4 checks") + + +if __name__ == "__main__": + try: + main() + except Failure as exc: + print("FAIL:", exc) + sys.exit(1) diff --git a/student-tests/persistence.py b/student-tests/persistence.py new file mode 100644 index 00000000..d2edcf6c --- /dev/null +++ b/student-tests/persistence.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Тот же сценарий, что tests/resilience.py, но hook запускается так, как умеет ОС. + +Linux выполняет scripts/restart.sh напрямую. +Windows не умеет CreateProcess для shell-скрипта, поэтому здесь используется Git Bash. +Скрипт тот же: перезапуск api, postgres и redis без удаления томов. +""" + +import os +import subprocess +import sys +import time +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check, run + + +def hook_command(): + script = ROOT / "scripts" / "restart.sh" + if os.name != "nt": + return [str(script)] + bash = Path(os.environ.get("BASH", r"C:\Program Files\Git\bin\bash.exe")) + check(bash.is_file(), "Git Bash is required to launch scripts/restart.sh on Windows") + return [str(bash), "-c", "sh scripts/restart.sh"] + + +def history(client, actor, chat): + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + result = [] + seen = set() + for _ in range(1000): + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + result.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return result + check(cursor not in seen, "cursor cycle") + seen.add(cursor) + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(cursor, safe="") + raise Failure("history pagination exceeded safety bound") + + +def main(): + client = Client("http://127.0.0.1:8080", timeout=5) + alice, bob, _eve, chat = client.bootstrap() + original = client.message(alice, chat, "Сохранить при отказе")["json"] + print("Executing scripts/restart.sh") + completed = subprocess.run(hook_command(), cwd=str(ROOT)) + check(completed.returncode == 0, "restart hook failed") + deadline = time.monotonic() + 90 + while True: + try: + client.call("GET", "/health/ready", schema="Health") + client.call("GET", "/api/v1/users/me", actor=alice, schema="User") + break + except Failure: + check(time.monotonic() < deadline, "stack/session did not recover within 90 seconds") + time.sleep(0.5) + messages = history(client, bob, chat) + check(any(item == original for item in messages), "confirmed message lost or altered") + print("PASS: persistence preserved confirmed data and existing session") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/persistence_hook.py b/student-tests/persistence_hook.py new file mode 100644 index 00000000..db14fd87 --- /dev/null +++ b/student-tests/persistence_hook.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Публичный persistence-сценарий с restart.cmd на Windows и restart.sh на POSIX.""" + +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def script(): + if os.name == "nt": + cmd = ROOT / "scripts" / "restart.cmd" + if cmd.is_file(): + return str(cmd) + return str(ROOT / "scripts" / "restart.sh") + + +def main(): + env = os.environ.copy() + command = [ + sys.executable, + str(ROOT / "tests" / "resilience.py"), + "--mode", + "persistence", + "--action-script", + script(), + "--base-url", + env.get("BASE_URL", "http://127.0.0.1:8080"), + ] + completed = subprocess.run(command, cwd=str(ROOT), env=env) + raise SystemExit(completed.returncode) + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling.py b/student-tests/rolling.py new file mode 100644 index 00000000..e697ee2a --- /dev/null +++ b/student-tests/rolling.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Поочерёдная замена API-реплик под нагрузкой POST сообщений.""" + +import json +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +def compose(*args, timeout=180): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("docker compose %s failed" % args[0]) + return proc.stdout + + +def api_ids(): + return [line.strip() for line in compose("ps", "-q", "api").splitlines() if line.strip()] + + +def inspect_health(container_id): + proc = subprocess.run( + ["docker", "inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}", container_id], + capture_output=True, + text=True, + timeout=20, + encoding="utf-8", + errors="replace", + ) + return proc.stdout.strip() + + +def wait_healthy(container_id, timeout=90): + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + last = inspect_health(container_id) + if last == "healthy": + return + time.sleep(0.5) + raise Failure("replica %s did not become healthy: %s" % (container_id[:12], last)) + + +def wait_ready(timeout=90): + client = Client(BASE, timeout=3) + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = str(exc) + time.sleep(0.4) + raise Failure("LB ready wait failed: " + last) + + +def main(): + client = Client(BASE, timeout=8) + alice, _bob, _eve, chat = client.bootstrap() + halt = threading.Event() + latencies = [] + errors = [] + lock = threading.Lock() + + def worker(index): + local = Client(BASE, timeout=5) + n = 0 + while not halt.is_set(): + started = time.monotonic() + try: + local.message(alice, chat, "rolling-%s-%s" % (index, n)) + with lock: + latencies.append(time.monotonic() - started) + except Failure: + with lock: + errors.append(time.monotonic()) + n += 1 + + threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(4)] + for thread in threads: + thread.start() + time.sleep(5) + before = api_ids() + check(len(before) >= 2, "rolling update needs >=2 replicas") + compose("build", "api") + replaced = [] + for old in list(before): + current = api_ids() + compose("up", "-d", "--no-deps", "--scale", "api=%d" % (len(current) + 1), "--no-recreate", "api") + deadline = time.monotonic() + 60 + new_ids = api_ids() + while len(new_ids) <= len(current) and time.monotonic() < deadline: + time.sleep(0.5) + new_ids = api_ids() + added = [item for item in new_ids if item not in current] + check(added, "new replica was not created") + wait_healthy(added[-1]) + wait_ready() + subprocess.run(["docker", "stop", "-t", "5", old], capture_output=True) + subprocess.run(["docker", "rm", "-f", old], capture_output=True) + remaining = api_ids() + check(old not in remaining, "old replica %s still in service" % old[:12]) + compose("up", "-d", "--no-deps", "--scale", "api=%d" % len(before), "api") + wait_ready() + replaced.append(old[:12]) + time.sleep(5) + halt.set() + for thread in threads: + thread.join(timeout=10) + after = api_ids() + check(len(after) >= 2, "lost replicas after rolling update") + check(len(latencies) > 0, "no successful writes during rolling update") + report = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "replaced": replaced, + "replicas_before": [item[:12] for item in before], + "replicas_after": [item[:12] for item in after], + "successful_writes": len(latencies), + "failed_writes": len(errors), + "error_ratio": round(len(errors) / max(1, len(latencies) + len(errors)), 6), + } + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "rolling.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print("PASS: rolling update under load") + print(json.dumps(report, ensure_ascii=False)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 00000000..dd78f315 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 + +import base64 +import json +import os +import socket +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +SECRET = "SECRET_MESSAGE_DO_NOT_LOG_7f3a" +CLIENTS = 10 +PER_CLIENT = 10 + + +class Failure(AssertionError): + pass + + +def check(condition, message): + if not condition: + raise Failure(message) + + +def free_port(): + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +class Server: + def __init__(self, port): + self.base = "http://127.0.0.1:%d" % port + env = os.environ.copy() + env["PYTHONPATH"] = str(ROOT / "src") + env["PYTHONUNBUFFERED"] = "1" + env["APP_HOST"] = "127.0.0.1" + env["APP_PORT"] = str(port) + env["APP_ENV"] = "lab" + env["DATABASE_URL"] = host_secret("host_database_url") + env["REDIS_URL"] = host_secret("host_cache_url") + env["S3_ENDPOINT"] = host_secret("host_s3_endpoint") + env["S3_ACCESS_KEY_ID"] = host_secret("s3_access_key_id") + env["S3_SECRET_ACCESS_KEY"] = host_secret("s3_secret_access_key") + env["S3_BUCKET"] = "messenger-attachments" + env["SESSION_TTL_SECONDS"] = "3600" + env.pop("DATABASE_URL_FILE", None) + env.pop("CACHE_URL_FILE", None) + self.logs = [] + self.proc = subprocess.Popen( + [sys.executable, "-m", "app"], + cwd=str(ROOT), + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + ) + self._reader = threading.Thread(target=self._read, daemon=True) + self._reader.start() + + def _read(self): + assert self.proc.stdout is not None + for line in self.proc.stdout: + self.logs.append(line) + + def wait_ready(self): + deadline = time.monotonic() + 40 + last = "" + while time.monotonic() < deadline: + if self.proc.poll() is not None: + raise Failure("server exited early:\n" + "".join(self.logs[-40:])) + try: + call(self.base, "GET", "/health/ready", 200) + return + except Failure as exc: + last = str(exc) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + last = type(exc).__name__ + time.sleep(0.1) + raise Failure("server did not become ready: " + last + "\n" + "".join(self.logs[-40:])) + + def stop(self): + if self.proc.poll() is None: + self.proc.terminate() + try: + self.proc.wait(timeout=8) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait(timeout=5) + self._reader.join(timeout=2) + + def output(self): + return "".join(self.logs) + + +def host_secret(name): + path = ROOT / "secrets" / name + if not path.is_file(): + raise Failure("missing secrets/%s; run scripts/bootstrap.ps1 or scripts/bootstrap.sh" % name) + value = path.read_text(encoding="utf-8").strip() + if not value: + raise Failure("empty secrets/%s" % name) + return value + + +def call( + base, + method, + path, + expected, + body=None, + token=None, + user_id=None, + authorization=None, + raw=None, + content_type="application/json", + timeout=20, + headers=None, +): + extra_headers = headers or {} + headers = {"Accept": "application/json"} + headers.update(extra_headers) + data = None + if raw is not None: + data = raw + headers["Content-Type"] = content_type + elif body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + headers["Content-Type"] = content_type + if authorization: + headers["Authorization"] = authorization + elif token: + headers["Authorization"] = "Bearer " + token + if user_id: + headers["X-User-Id"] = user_id + request = urllib.request.Request(base + path, data=data, headers=headers, method=method) + try: + response = urllib.request.urlopen(request, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise Failure(method + " " + path.split("?")[0] + " failed: " + type(exc).__name__) from None + with response: + status = response.status + payload = response.read() + header_id = response.headers.get("X-Request-Id") + instance_id = response.headers.get("X-Instance-Id") + content_type_value = response.headers.get_content_type() + allowed = [expected] if isinstance(expected, int) else list(expected) + check(status in allowed, "%s %s: expected %s, got %s" % (method, path.split("?")[0], allowed, status)) + check(header_id and header_id.strip() and "," not in header_id, "missing or duplicated X-Request-Id") + check(instance_id and instance_id.strip() and "," not in instance_id, "missing or duplicated X-Instance-Id") + parsed = None + if payload: + check(content_type_value == "application/json", "expected application/json") + try: + parsed = json.loads(payload) + except json.JSONDecodeError as exc: + raise Failure("response is not JSON") from exc + if status >= 400: + check(isinstance(parsed, dict) and set(parsed) == {"error"}, "error envelope") + error = parsed["error"] + check(set(error) == {"code", "message", "request_id"}, "error fields") + check(error["request_id"] == header_id, "request_id differs from header") + check(error["message"] and "Traceback" not in error["message"], "error message leaked internals") + return {"status": status, "json": parsed, "request_id": header_id, "instance_id": instance_id} + + +def send_message(base, chat_id, token, text, expected=201, key=None): + headers = {"Idempotency-Key": key or uuid.uuid4().hex} + return call( + base, + "POST", + "/api/v1/chats/%s/messages" % chat_id, + expected, + token=token, + body={"text": text}, + headers=headers, + ) + + +def create_user(base, label): + username = ("u_%s_%s" % (label, uuid.uuid4().hex[:10])).lower() + password = "T3st-" + uuid.uuid4().hex + body = {"username": username, "display_name": "Студент " + label, "password": password} + created = call(base, "POST", "/api/v1/users", 201, body=body)["json"] + check(created["username"] == username, "username changed") + check("password" not in created, "password leaked in user response") + raw = base64.b64encode(("%s:%s" % (username, password)).encode("utf-8")).decode("ascii") + session = call(base, "POST", "/api/v1/auth/sessions", 201, authorization="Basic " + raw)["json"] + check(session["token_type"] == "Bearer", "token type") + check(session["user"]["id"] == created["id"], "session user") + check("password" not in session["user"], "password leaked in session") + created["token"] = session["token"] + created["instance_id"] = session.get("instance_id") + return created + + +def replace_cursor(token, fields): + padding = "=" * (-len(token) % 4) + data = json.loads(base64.urlsafe_b64decode(token + padding)) + data.update(fields) + raw = json.dumps(data, separators=(",", ":"), sort_keys=True).encode("ascii") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def test_invalid_json(base): + broken = call(base, "POST", "/api/v1/users", 400, raw=b"{") + check(broken["json"]["error"]["code"] == "invalid_request", "malformed JSON code") + extra = call( + base, + "POST", + "/api/v1/users", + 400, + body={"username": "abc_user", "display_name": "Имя", "extra": True}, + ) + check(extra["json"]["error"]["code"] == "invalid_request", "extra field code") + bad_utf = call(base, "POST", "/api/v1/users", 400, raw=b"\xff") + check(bad_utf["json"]["error"]["code"] == "invalid_request", "invalid utf-8 code") + alice = create_user(base, "utf") + chat = call( + base, + "POST", + "/api/v1/chats", + 201, + token=alice["token"], + body={"title": "UTF", "member_ids": [alice["id"]]}, + )["json"] + bad_message = call( + base, + "POST", + "/api/v1/chats/%s/messages" % chat["id"], + 400, + token=alice["token"], + raw=b"\xff", + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + check(bad_message["json"]["error"]["code"] == "invalid_request", "invalid utf-8 message code") + print("PASS: invalid JSON") + + +def test_duplicate_username(base): + first = create_user(base, "dup") + again = call( + base, + "POST", + "/api/v1/users", + 409, + body={ + "username": first["username"], + "display_name": "Другой", + "password": "T3st-" + uuid.uuid4().hex, + }, + ) + check(again["json"]["error"]["code"] == "conflict", "duplicate username code") + me = call(base, "GET", "/api/v1/users/me", 200, token=first["token"])["json"] + check(me["id"] == first["id"], "first user disappeared after duplicate") + print("PASS: duplicate username") + + +def test_empty_and_foreign(base): + alice = create_user(base, "alice") + bob = create_user(base, "bob") + chat = call( + base, + "POST", + "/api/v1/chats", + 201, + token=alice["token"], + body={"title": "Только Алиса", "member_ids": [alice["id"]]}, + )["json"] + path = "/api/v1/chats/" + chat["id"] + for text in ("", " ", "\n\t"): + denied = call( + base, + "POST", + path + "/messages", + 400, + token=alice["token"], + body={"text": text}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + check(denied["json"]["error"]["code"] == "invalid_request", "empty message code") + for method, target, payload in ( + ("GET", path, None), + ("GET", path + "/messages", None), + ("POST", path + "/messages", {"text": "чужое"}), + ): + extra = {"Idempotency-Key": uuid.uuid4().hex} if method == "POST" else None + hidden = call(base, method, target, 404, token=bob["token"], body=payload, headers=extra) + check(hidden["json"]["error"]["code"] == "not_found", "foreign chat code") + check(chat["title"] not in hidden["json"]["error"]["message"], "foreign chat leaked title") + listed = call(base, "GET", "/api/v1/chats?limit=100", 200, token=bob["token"])["json"] + check(all(item["id"] != chat["id"] for item in listed["items"]), "foreign chat listed") + missing = call(base, "GET", "/api/v1/chats/" + str(uuid.uuid4()), 404, token=bob["token"]) + check(missing["json"]["error"]["code"] == "not_found", "missing chat code") + bad_limit = call(base, "GET", path + "/messages?limit=0", 400, token=alice["token"]) + check(bad_limit["json"]["error"]["code"] == "invalid_request", "limit=0 code") + print("PASS: empty message and foreign chat") + + +def test_pagination(base): + alice = create_user(base, "page") + other = create_user(base, "other") + chat = call( + base, + "POST", + "/api/v1/chats", + 201, + token=alice["token"], + body={"title": "Страницы", "member_ids": [alice["id"]]}, + )["json"] + other_chat = call( + base, + "POST", + "/api/v1/chats", + 201, + token=other["token"], + body={"title": "Другой", "member_ids": [other["id"]]}, + )["json"] + texts = ["p0", "Привет, мир 👋", "p2", "p3", "p4"] + created = [] + for text in texts: + created.append(send_message(base, chat["id"], alice["token"], text)["json"]) + send_message(base, other_chat["id"], other["token"], "x") + send_message(base, other_chat["id"], other["token"], "y") + collected = [] + cursor = None + pages = 0 + while True: + target = "/api/v1/chats/%s/messages?limit=2" % chat["id"] + if cursor: + target += "&cursor=" + quote(cursor, safe="") + page = call(base, "GET", target, 200, token=alice["token"])["json"] + pages += 1 + collected.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + check(pages < 10, "pagination did not finish") + check(pages == 3, "expected 3 pages, got %d" % pages) + check(collected == created, "pagination lost, duplicated or reordered messages") + stolen = call( + base, + "GET", + "/api/v1/chats/%s/messages?limit=1" % other_chat["id"], + 200, + token=other["token"], + )["json"] + check(stolen["next_cursor"], "foreign page should expose a cursor") + rejected = call( + base, + "GET", + "/api/v1/chats/%s/messages?limit=2&cursor=%s" % (chat["id"], quote(stolen["next_cursor"], safe="")), + 400, + token=alice["token"], + ) + check(rejected["json"]["error"]["code"] == "invalid_request", "foreign cursor code") + first_page = call( + base, + "GET", + "/api/v1/chats/%s/messages?limit=1" % chat["id"], + 200, + token=alice["token"], + )["json"] + for fields in ({"t": None}, {"t": 123}, {"i": 123}): + broken = call( + base, + "GET", + "/api/v1/chats/%s/messages?limit=1&cursor=%s" + % (chat["id"], quote(replace_cursor(first_page["next_cursor"], fields), safe="")), + 400, + token=alice["token"], + ) + check(broken["json"]["error"]["code"] == "invalid_request", "tampered cursor code") + print("PASS: pagination") + + +def test_concurrency(base): + users = [create_user(base, "c%d" % index) for index in range(CLIENTS)] + chat = call( + base, + "POST", + "/api/v1/chats", + 201, + token=users[0]["token"], + body={"title": "Гонка", "member_ids": [user["id"] for user in users]}, + )["json"] + barrier = threading.Barrier(CLIENTS) + found = [] + errors = [] + lock = threading.Lock() + + def worker(user): + try: + barrier.wait(10) + local = [] + for index in range(PER_CLIENT): + message = send_message( + base, + chat["id"], + user["token"], + "m-%s-%d" % (user["id"][:8], index), + )["json"] + local.append(message["id"]) + with lock: + found.extend(local) + except Exception as exc: + with lock: + errors.append("%s: %s" % (type(exc).__name__, exc)) + + threads = [threading.Thread(target=worker, args=(user,)) for user in users] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + check(not errors, "concurrent sends failed: " + "; ".join(errors[:5])) + check(len(found) == CLIENTS * PER_CLIENT, "submitted count") + check(len(set(found)) == CLIENTS * PER_CLIENT, "duplicate message ids") + + listed = [] + cursor = None + while True: + target = "/api/v1/chats/%s/messages?limit=40" % chat["id"] + if cursor: + target += "&cursor=" + quote(cursor, safe="") + page = call(base, "GET", target, 200, token=users[0]["token"])["json"] + listed.extend(item["id"] for item in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + check(len(listed) == 100 and len(set(listed)) == 100, "history does not contain 100 unique messages") + check(set(listed) == set(found), "history ids differ from accepted sends") + EVIDENCE.mkdir(exist_ok=True) + report = { + "date": "2026-10-25", + "command": "python student-tests/run.py", + "clients": CLIENTS, + "messages_per_client": PER_CLIENT, + "submitted": len(found), + "unique_ids": len(set(found)), + "listed": len(listed), + "failed_requests": len(errors), + "pass": True, + } + (EVIDENCE / "concurrency.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + print("PASS: concurrency 100 unique messages from 10 clients") + + +def test_logs(server): + alice = create_user(server.base, "logs") + chat = call( + server.base, + "POST", + "/api/v1/chats", + 201, + token=alice["token"], + body={"title": "Логи", "member_ids": [alice["id"]]}, + )["json"] + send_message(server.base, chat["id"], alice["token"], SECRET) + time.sleep(0.2) + output = server.output() + check("duration_ms=" in output and "method=POST" in output, "request log line missing") + check(SECRET not in output, "message text leaked into logs") + check(alice["token"] not in output, "session token leaked into logs") + check("lab-only-password" not in output and "postgresql+asyncpg://" not in output, "database url leaked into logs") + print("PASS: logs omit message text") + + +def main(): + port = free_port() + server = Server(port) + try: + server.wait_ready() + call(server.base, "GET", "/health/live", 200) + call(server.base, "GET", "/api/v1/users/me", 401) + header_only = create_user(server.base, "header") + rejected = call(server.base, "GET", "/api/v1/users/me", 401, user_id=header_only["id"]) + check(rejected["json"]["error"]["code"] == "unauthorized", "X-User-Id was accepted") + test_invalid_json(server.base) + test_duplicate_username(server.base) + test_empty_and_foreign(server.base) + test_pagination(server.base) + test_concurrency(server.base) + test_logs(server) + finally: + server.stop() + print("PASS: student tests") + + +if __name__ == "__main__": + try: + main() + except Failure as exc: + print("FAIL:", exc) + sys.exit(1) diff --git a/student-tests/s3_private.py b/student-tests/s3_private.py new file mode 100644 index 00000000..042e7a7c --- /dev/null +++ b/student-tests/s3_private.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Прямой анонимный доступ к bucket должен быть закрыт.""" + +import json +import os +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +S3 = os.environ.get("S3_PUBLIC_URL", "http://127.0.0.1:9000").rstrip("/") +BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") + + +def anonymous(path: str) -> int: + request = urllib.request.Request(S3 + path, method="GET") + try: + with urllib.request.urlopen(request, timeout=10) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.status + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise Failure("direct s3 probe failed: " + type(exc).__name__) from None + + +def main(): + client = Client(BASE, timeout=20) + alice, _bob, _eve, chat = client.bootstrap() + att, content = client.attachment(alice, chat) + att = client.wait_ready(alice, att) + listed = anonymous("/" + BUCKET) + obj = anonymous("/" + BUCKET + "/att/" + att["id"]) + check(listed in {403, 404}, "anonymous list status %s" % listed) + check(obj in {403, 404}, "anonymous object status %s" % obj) + download = client.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=alice) + check(download["bytes"] == content, "authorized download after anonymous probe") + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "s3_private.json").write_text( + json.dumps( + { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "anonymous_list": listed, + "anonymous_object": obj, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print("PASS: private bucket rejects anonymous GET") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/track1.py b/student-tests/track1.py new file mode 100644 index 00000000..6cb94f5b --- /dev/null +++ b/student-tests/track1.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Трек 5.1: цена масштабирования API при фиксированном и растущем бюджете CPU/RAM.""" + +import json +import math +import os +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +DURATION = float(os.environ.get("TRACK1_DURATION", "30")) +CONCURRENCY = int(os.environ.get("TRACK1_CONCURRENCY", "8")) +WARMUP = float(os.environ.get("TRACK1_WARMUP", "8")) + + +def compose(*args, timeout=180): + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + encoding="utf-8", + errors="replace", + ) + if proc.returncode != 0: + raise Failure("docker compose %s failed: %s" % (args[0], proc.stderr[-400:])) + return proc.stdout + + +def api_ids(): + return [line.strip() for line in compose("ps", "-q", "api").splitlines() if line.strip()] + + +def wait_ready(timeout=90): + client = Client(BASE, timeout=3) + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + return + except Failure as exc: + last = str(exc) + time.sleep(0.4) + raise Failure("ready wait failed: " + last) + + +def scale_api(n): + compose("up", "-d", "--no-deps", "--scale", "api=%d" % n, "api") + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + if len(api_ids()) >= n: + wait_ready() + return + time.sleep(0.5) + raise Failure("failed to scale api to %s" % n) + + +def apply_limits(cpus_each, memory_mb_each): + for container_id in api_ids(): + args = ["docker", "update", "--cpus", "%.3f" % cpus_each, "--memory", "%dm" % memory_mb_each, container_id] + proc = subprocess.run(args, capture_output=True, text=True, timeout=30) + if proc.returncode != 0: + raise Failure("docker update failed") + + +def docker_stats(): + proc = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"], + capture_output=True, + text=True, + timeout=30, + encoding="utf-8", + errors="replace", + ) + rows = [] + for line in proc.stdout.splitlines(): + parts = line.split("\t") + if len(parts) == 3: + rows.append({"name": parts[0], "cpu": parts[1], "mem": parts[2]}) + return rows + + +def run_load(duration, concurrency, warmup): + client = Client(BASE, timeout=8) + alice, _bob, _eve, chat = client.bootstrap() + if warmup > 0: + deadline = time.monotonic() + warmup + index = 0 + while time.monotonic() < deadline: + client.message(alice, chat, "warmup-%s" % index) + index += 1 + barrier = threading.Barrier(concurrency) + + def worker(worker_id): + local = Client(BASE, timeout=5) + barrier.wait() + end = time.monotonic() + duration + latencies, errors = [], 0 + index = 0 + while time.monotonic() < end: + started = time.monotonic() + try: + local.message(alice, chat, "track1-%s-%s" % (worker_id, index)) + latencies.append(time.monotonic() - started) + except Failure: + errors += 1 + index += 1 + return latencies, errors + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + results = list(pool.map(worker, range(concurrency))) + elapsed = time.monotonic() - started + latencies = sorted(item for values, _ in results for item in values) + errors = sum(n for _, n in results) + total = len(latencies) + errors + + def percentile(p): + if not latencies: + return None + return round(1000 * latencies[max(0, math.ceil(p * len(latencies)) - 1)], 3) + + return { + "elapsed_seconds": round(elapsed, 3), + "concurrency": concurrency, + "successful_requests": len(latencies), + "failed_requests": errors, + "error_ratio": round(errors / total, 6) if total else None, + "successful_rps": round(len(latencies) / elapsed, 3) if elapsed else None, + "successful_latency_ms": {"p50": percentile(0.50), "p95": percentile(0.95), "p99": percentile(0.99)}, + "docker_stats": docker_stats(), + } + + +def measure(label, replicas, cpus_each, memory_mb_each): + scale_api(replicas) + apply_limits(cpus_each, memory_mb_each) + wait_ready() + time.sleep(2) + result = run_load(DURATION, CONCURRENCY, WARMUP) + result.update( + { + "label": label, + "replicas": replicas, + "cpus_each": cpus_each, + "memory_mb_each": memory_mb_each, + "total_cpu": round(replicas * cpus_each, 3), + "total_memory_mb": replicas * memory_mb_each, + } + ) + print(json.dumps({k: result[k] for k in ("label", "replicas", "total_cpu", "successful_rps", "error_ratio")}, ensure_ascii=False)) + return result + + +def main(): + wait_ready() + fixed_budget = [ + measure("fixed-1", 1, 1.00, 512), + measure("fixed-2", 2, 0.50, 256), + measure("fixed-4", 4, 0.25, 128), + ] + growing = [ + measure("grow-2x0.25", 2, 0.25, 128), + measure("grow-2x0.50", 2, 0.50, 256), + measure("grow-2x1.00", 2, 1.00, 512), + ] + scale_api(2) + apply_limits(0.50, 256) + report = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "hypothesis": "При фиксированном суммарном CPU/RAM добавление реплик перестаёт помогать, когда узким местом становится PostgreSQL/HAProxy, а не CPU процесса API.", + "profile": { + "duration_seconds": DURATION, + "concurrency": CONCURRENCY, + "warmup_seconds": WARMUP, + "workload": "closed-loop POST /messages through HAProxy", + }, + "fixed_total_budget": fixed_budget, + "growing_resources": growing, + } + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "track1.json").write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + check(all(item["successful_requests"] > 0 for item in fixed_budget + growing), "a track1 series had no successes") + print("PASS: track1 series written to evidence/track1.json") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/upload_mem.py b/student-tests/upload_mem.py new file mode 100644 index 00000000..fe546aa1 --- /dev/null +++ b/student-tests/upload_mem.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""10 параллельных загрузок и пик RSS API-реплик по docker stats.""" + +import json +import os +import subprocess +import sys +import threading +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run + +EVIDENCE = ROOT / "evidence" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +WORKERS = 10 +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def api_mem_mib() -> float: + proc = subprocess.run( + ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=20, + ) + ids = [line.strip() for line in proc.stdout.splitlines() if line.strip()] + if not ids: + raise Failure("no api containers") + stats = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *ids], + capture_output=True, + text=True, + timeout=20, + ) + if stats.returncode != 0: + raise Failure("docker stats failed") + peak = 0.0 + for line in stats.stdout.splitlines(): + used = line.split("/", 1)[0].strip() + if used.endswith("GiB"): + peak = max(peak, float(used[:-3].strip()) * 1024) + elif used.endswith("MiB"): + peak = max(peak, float(used[:-3].strip())) + elif used.endswith("KiB"): + peak = max(peak, float(used[:-3].strip()) / 1024) + return peak + + +def main(): + client = Client(BASE, timeout=30) + alice, _bob, _eve, chat = client.bootstrap() + before = api_mem_mib() + barrier = threading.Barrier(WORKERS) + errors = [] + lock = threading.Lock() + + def worker(index: int) -> None: + local = Client(BASE, timeout=30) + try: + barrier.wait(20) + local.call( + "POST", + "/api/v1/chats/%s/attachments?filename=p%d.png" % (chat["id"], index), + 202, + actor=alice, + raw=SAMPLE, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + except Exception as exc: + with lock: + errors.append("%s: %s" % (type(exc).__name__, exc)) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(WORKERS)] + for thread in threads: + thread.start() + peak = before + deadline = time.monotonic() + 30 + while any(thread.is_alive() for thread in threads) and time.monotonic() < deadline: + peak = max(peak, api_mem_mib()) + time.sleep(0.4) + for thread in threads: + thread.join() + peak = max(peak, api_mem_mib()) + check(not errors, "concurrent uploads failed: " + "; ".join(errors[:3])) + EVIDENCE.mkdir(exist_ok=True) + (EVIDENCE / "upload_mem.json").write_text( + json.dumps( + { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "workers": WORKERS, + "rss_before_mib": round(before, 2), + "rss_peak_mib": round(peak, 2), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + print("PASS: %s concurrent uploads, peak RSS %.1f MiB" % (WORKERS, peak)) + + +if __name__ == "__main__": + run(main)