From a1b972f7d9f663c3ee385ab4634c2da86cff2be6 Mon Sep 17 00:00:00 2001 From: Wladislava1 <146841829+Wladislava1@users.noreply.github.com> Date: Thu, 12 Nov 2026 13:05:02 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A1=D0=BE=D0=B1=D1=80=D0=B0=D1=82=D1=8C=20?= =?UTF-8?q?=D1=81=D1=82=D0=B5=D0=BD=D0=B4=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80?= =?UTF-8?q?=D0=B0=D1=82=D0=BE=D1=80=D0=BD=D0=BE=D0=B9=205:=20=D0=BE=D1=87?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B4=D1=8C,=20worker=20=D0=B8=20=D0=B8?= =?UTF-8?q?=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5=D0=BD=D1=82=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D1=8C=20pipeline.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitattributes | 4 + .github/pull_request_template.md | 28 +- Dockerfile | 27 + REPORT.md | 128 ++++- compose.yaml | 290 +++++++++++ evidence/exif.json | 20 + evidence/failover.txt | 24 + evidence/negative.json | 23 + evidence/persistence.txt | 20 + evidence/pipeline.json | 318 +++++++++++ infra/haproxy.cfg | 47 ++ infra/minio-api-policy.json | 19 + migrations/001_init.sql | 48 ++ migrations/002_idempotency.sql | 19 + migrations/003_attachments.sql | 42 ++ migrations/004_jobs.sql | 108 ++++ requirements.txt | 10 + scripts/bootstrap.py | 77 +++ scripts/minio-init.sh | 30 ++ scripts/rabbit-entry.sh | 10 + scripts/restart.sh | 8 + scripts/stop-one.sh | 23 + scripts/wait_ready.py | 30 ++ src/blobs.py | 228 ++++++++ src/broker.py | 132 +++++ src/errors.py | 32 ++ src/faults.py | 49 ++ src/images.py | 129 +++++ src/jobs.py | 831 +++++++++++++++++++++++++++++ src/main.py | 475 +++++++++++++++++ src/migrate.py | 112 ++++ src/passwords.py | 26 + src/reconcile.py | 174 +++++++ src/replay.py | 24 + src/schemas.py | 128 +++++ src/store.py | 868 +++++++++++++++++++++++++++++++ src/worker.py | 182 +++++++ student-tests/run_negative.py | 130 +++++ student-tests/run_pipeline.py | 653 +++++++++++++++++++++++ 39 files changed, 5492 insertions(+), 34 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/exif.json create mode 100644 evidence/failover.txt create mode 100644 evidence/negative.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/pipeline.json create mode 100644 infra/haproxy.cfg create mode 100644 infra/minio-api-policy.json create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_attachments.sql create mode 100644 migrations/004_jobs.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100755 scripts/minio-init.sh create mode 100755 scripts/rabbit-entry.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 scripts/wait_ready.py create mode 100644 src/blobs.py create mode 100644 src/broker.py create mode 100644 src/errors.py create mode 100644 src/faults.py create mode 100644 src/images.py create mode 100644 src/jobs.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/passwords.py create mode 100644 src/reconcile.py create mode 100644 src/replay.py create mode 100644 src/schemas.py create mode 100644 src/store.py create mode 100644 src/worker.py create mode 100644 student-tests/run_negative.py create mode 100644 student-tests/run_pipeline.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..fe560927 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +*.sh text eol=lf +*.cfg text eol=lf +Dockerfile text eol=lf +*.sql text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..5f2b58c1 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Орлова Владислава, БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: Трек 2, идемпотентность pipeline -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: Стенд лабораторной 4 сохранён: FastAPI, PostgreSQL, Redis, HAProxy, две реплики API, MinIO, миграции отдельным контейнером. Добавлены RabbitMQ, отдельный worker, outbox, состояния queued/processing/ready/failed, thumbnail, crop и optimized WebP. Трек 2 проверяет дубль, чужой порядок, падение до и после эффекта и старую версию pipeline. +Команды запуска из чистого clone: `python -X utf8 scripts/bootstrap.py`, затем `docker compose --env-file .env up -d --build --scale api=2`. Программы make нет, команды Makefile запущены через `python -X utf8`. Публичные `.sh` на Windows запущены через WSL. Подробности в REPORT.md. +Результаты проверок и ссылка на REPORT.md: smoke, persistence, failover, негативные проверки и сценарии трека 2 прошли. Числа и команды лежат в REPORT.md и evidence/. +Ограничения / известные ошибки: по одному экземпляру Postgres, Redis, MinIO, RabbitMQ и HAProxy. При отказе обеих реплик клиент через HAProxy видит его HTML 503. Exactly-once нет. Прямой загрузки по временной ссылке нет. Нагрузку трека 1 я не измеряла. +Источники и использование ИИ: задание, контракт, документация RabbitMQ, стенд лабораторной 4. Код и отчёт я собирала в Cursor. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [ ] make check и make test выполнены; приложен реальный вывод. Программы make нет. Те же команды выполнены через `python -X utf8`, публичные `.sh` через WSL. Вывод записан в REPORT.md. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..8e50b3af --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +RUN useradd --uid 10001 --create-home --shell /usr/sbin/nologin app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --upgrade pip==25.0.1 \ + && pip install --no-cache-dir -r /app/requirements.txt + +COPY migrations /app/migrations +COPY src /app/src + +USER 10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=25s --retries=8 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + +CMD ["python", "/app/src/main.py"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..99ae776b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,125 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Орлова Владислава, БПИ2301 +- Целевая оценка: 5. Трек 2, идемпотентность pipeline +- Commit/PR: ещё не публиковала +- Источники и помощь: задание и контракт этой лабораторной, рабочий стенд лабораторной 4, документация RabbitMQ про publisher confirm и consumer ack. Код и отчёт я собирала в Cursor. Числа ниже сняты с моего запуска 12 ноября 2026, я их не округляла ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Engine и Python 3.9+. У меня Docker Engine 26.1.4 и Python на Windows. Программы `make` в системе нет, поэтому команды из Makefile я запускала через `python -X utf8`. Флаг нужен, чтобы Windows не читала JSON в кодировке cp1251. Публичные `.sh` Windows не запускает как программу, их я запускала через WSL. + +```sh +python -X utf8 scripts/bootstrap.py +docker compose --env-file .env up -d --build --scale api=2 +python -X utf8 scripts/wait_ready.py +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +``` + +Проверки сохранности и остановки реплики: + +```sh +wsl -e bash -lc "cd '/mnt/c/Git/высоконагруженные приложения/Lab5' && python3 -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080" +wsl -e bash -lc "cd '/mnt/c/Git/высоконагруженные приложения/Lab5' && python3 -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080" +``` + +`bootstrap.py` один раз создаёт `secrets/` и `.env`. Пароли в консоль не печатаются. Повторный запуск уже созданные файлы не перезаписывает. В git эти файлы не входят. + +Остановить стенд, не удаляя данные: `docker compose --env-file .env stop`. Удалить учебные данные вместе с volume: `docker compose --env-file .env down -v`. Вторую команду я использую только когда хочу чистую базу. + +Имя проекта Compose — `mtusi-lab5`. Соседняя папка лабораторной 4 из-за этого не пересекается со стендом. Наружу открыт только `127.0.0.1:8080` у балансировщика. + +Образы, которые реально поднимаются: + +| Роль | Образ | +| --- | --- | +| API и worker | свой Dockerfile, база `python:3.12.10-slim-bookworm@sha256:fd95fa221297a88e1cf49c55ec1828edd7c5a428187e67b5d1805692d11588db` | +| Postgres | `postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74` | +| Redis | `redis:7.4.2-bookworm@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9` | +| RabbitMQ | `rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc` | +| Балансировщик | `haproxy:2.8.14@sha256:2344d1efbf7a932c7a3d49b60748c28af81c70fd11fcadd983e2be1f0376a69d` | +| MinIO | `pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372` | + +Сервер внутри образа MinIO отвечает `minio version RELEASE.2026-08-04T00-00-00Z`. Официальный `minio/minio` с Docker Hub больше не скачивается, а `quay.io/minio/minio` без логина отвечает `unauthorized`. Поэтому стенд закреплён на публичном образе `pgsty/minio` с digest. Это сервер MinIO с тем же протоколом S3. + +У образа RabbitMQ 3.13.7 входной скрипт отвергает переменные `RABBITMQ_DEFAULT_USER_FILE` и `RABBITMQ_DEFAULT_PASS_FILE`. `scripts/rabbit-entry.sh` читает секреты и пишет `rabbitmq.conf` с `default_user` и `default_pass`. Имя узла зафиксировано как `rabbit@rabbitmq`: каталог mnesia переживает пересоздание контейнера. При первой загрузке имя узла было случайным, брокер не стал healthy, и я один раз удалила volume RabbitMQ. Это было до измеренного прогона сохранности. Сам прогон volume не удалял. + +Лимиты: две реплики API по 0,50 CPU и 512 МиБ, worker 1,00 CPU и 768 МиБ, Postgres 1,00 CPU и 512 МиБ, Redis 0,25 CPU и 128 МиБ, RabbitMQ 1,00 CPU и 1024 МиБ, MinIO 0,50 CPU и 512 МиБ, HAProxy 0,25 CPU и 128 МиБ. Память API я подняла с 256 МиБ лабораторной 4, потому что приём файла всё ещё проверяет картинку до ответа 202. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент ходит в HAProxy. HAProxy по кругу отправляет запрос на одну из двух реплик API. `X-Instance-Id` — hostname контейнера, поэтому `scripts/stop-one.sh` останавливает именно ту реплику, которую назвал тест. `restart: unless-stopped`: явный `docker stop` оставляет контейнер остановленным. + +Сессии лежат в Redis с AOF. Сообщения, пользователи, вложения и задания лежат в PostgreSQL. Это источник истины. Сами байты лежат в приватном bucket `messenger-attachments`. Миграции применяет одноразовый контейнер `migrate`. Реплики API схему не накатывают. Схема ждёт версию `004_jobs`. + +Из лабораторной 4 я перенесла регистрацию, сессии, чаты, ленту, идемпотентность POST сообщения, две реплики, приватный bucket и сверку объектов. Чужой объект по-прежнему даёт 404. UUID в разном регистре — один идентификатор, в базе он в нижнем регистре. Пропущенное поле `attachment_ids` и пустой список дают одно и то же тело идемпотентности. Ключ идемпотентности — это пользователь, метод, путь и заголовок `Idempotency-Key`. Это не id сообщения. Тот же ключ с другим телом даёт 409, вторая строка не создаётся. Тот же ключ в другом чате создаёт новое сообщение, потому что путь другой. Ошибка пользователя остаётся 4xx. + +Загрузка теперь отвечает 202 и `status=queued`. Ответ уходит после двух устойчивых записей: байты уже лежат в `staging//original`, а в одной транзакции записаны строка вложения, задание и строка outbox. Байтов и паролей в сообщении брокера нет. В сообщении только поля `contracts/jobs.schema.json`: `job_id`, `attachment_id`, `pipeline_version`, `attempt` и время. `job_id` не меняется при повторной доставке. Повторная доставка сама по себе не увеличивает `attempt`. + +Публикация в RabbitMQ идёт после COMMIT. У канала включён publisher confirm, сообщение persistent. Если брокер в этот момент недоступен, API всё равно возвращает 202: строка outbox остаётся с пустым `published_at`, worker позже её публикует. Непрерывного публикатора внутри API нет. Цикл чтения outbox крутится в worker раз в секунду, и API один раз пытается опустошить outbox при своём старте. + +Worker — отдельный процесс. Пока он остановлен, API не уменьшает картинку. Он пишет оригинал байт в байт и три производных файла: JPEG thumbnail вписывается в 256×256 без увеличения, JPEG crop — центр 128×128, для маленького кадра увеличение разрешено, WebP optimized вписывается в 1280×1280 без увеличения. JPEG пишется с quality 85 и `optimize=True`. WebP пишется с quality 80 и method 4, без EXIF. Перед декодированием проверяются сигнатура, размер заголовка и предел 40 миллионов пикселей. Ориентация EXIF применяется к кадру до уменьшения. + +Попыток по умолчанию 5. Временная ошибка ставит следующую попытку с паузой: база `min(30, 0.5 * 2^(attempt-1))` секунд плюс случайная добавка до 25 процентов. Постоянная ошибка, например битый файл, сразу даёт `failed` и `invalid_image`, без серии повторов. Зависший `processing` с истёкшим `lease_until` sweeper возвращает в `queued` и не увеличивает `attempt`. Живой worker продлевает аренду. Чужой владелец аренды получает отложенную доставку на 2 секунды и не начинает вторую копию. Подтверждение потребителя уходит после COMMIT эффекта. Для уже готового или уже проваленного задания подтверждение тоже уходит, но файлы заново не пишутся. Это at-least-once. Exactly-once доставки здесь нет. Ключ эффекта — `attachment_id`, `pipeline_version` и имя варианта. + +`/health/ready` проверяет миграцию `004_jobs`, Redis и bucket. Остановленный брокер или worker сам по себе готовность API не роняет: иначе демонстрация «загрузить файл при остановленном worker» была бы невозможна. Текст 503 у API фиксированный: «Сервис временно недоступен». + +Переменные из `.env.example` попали в стенд так. `SESSION_STORE` в примере равен `postgres`, в `compose.yaml` у API стоит `redis`. `BROKER_URL_FILE`, `WORKER_CONCURRENCY=2`, `JOB_MAX_ATTEMPTS=5`, `JOB_TIMEOUT_SECONDS=60`, `STAGING_RETENTION_SECONDS=86400`, `MAX_UPLOAD_BYTES=10485760`, `MAX_IMAGE_PIXELS=40000000` читаются так, как названы. Имена файлов ключей S3 в примере длиннее, контейнер читает `S3_ACCESS_KEY_FILE` и `S3_SECRET_KEY_FILE`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Машина: Intel Core i5-10300H, 8 логических процессоров, 17008717824 байт памяти. Дата прогона 12 ноября 2026. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда и что я увидела | Файл | +| --- | --- | --- | +| Контракт на диске | `python -X utf8 scripts/check_contract.py` напечатал `PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.` | вывод в этом разделе | +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` напечатал `PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.` Ответ загрузки 202, затем ready, оригинал совпал, thumbnail 256×160 | — | +| Сохранность после рестарта хранилищ | `tests/resilience.py --mode persistence`, hook `scripts/restart.sh`, через WSL, код выхода 0. PASS. StartedAt изменился у postgres, redis, minio, rabbitmq и обеих API. У балансировщика и worker не изменился. Volume не удалялись | `evidence/persistence.txt` | +| Остановка одной реплики | Два прогона `tests/resilience.py --mode failover` через WSL. До остановки были видны 2 реплики. Ошибок за прогон: 1 и 1. Успешных записей: 97 и 98. Восстановление: 1,469 с и 0,975 с. Оба раза остановился api-2 с кодом 143. Отдельно `TARGET_INSTANCE_ID=2e03f20dd3b5` остановил api-1, тоже код 143, api-2 остался работать | `evidence/failover.txt` | +| Старые текстовые правила и границы файла | `python -X utf8 student-tests/run_negative.py`. Пустой текст, битый JSON и короткий пароль — 400. Пропущенный список и `[]` — одно сообщение. Другое тело при том же ключе — 409, второй строки нет. Тот же ключ в другом чате — новое сообщение. Чужой чат и неизвестный UUID вложения — 404. Без токена — 401. Имя с `/` — 400. Заголовок 8000×8000 и тело больше 10 МиБ — 413. Битый PNG и PNG с типом image/jpeg — 415 | `evidence/negative.json` | +| Файл при остановленном worker | `python -X utf8 student-tests/run_pipeline.py`. Загрузка 202, скачивание 409, текстовое сообщение 201, вложение можно прикрепить к сообщению. В очереди `attachments.process` было 1 сообщение до рестарта API, 1 после рестарта API и 1 после рестарта RabbitMQ. Затем worker довёл файл до ready. SHA-256 оригинала `99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75`. Thumbnail 256×160, crop 128×128, optimized 320×200, строк вариантов 4 | `evidence/pipeline.json`, сценарий `durability` | +| Разрыв COMMIT и publish | Тот же скрипт, сценарий `outbox_gap`. Worker остановлен, публикация из API пропущена специально. У outbox `published_at` пустой, глубина очереди 0. После старта worker файл стал ready, строк вариантов 4 | `evidence/pipeline.json` | +| Повтор, чужой порядок, старая версия | Дубль того же задания оставил 4 строки вариантов. Сообщение с `attempt=2` при задании на попытке 1 не сменило попытку, итог ready. Новая `pipeline_version` 2 дошла до ready, старое задание стало `superseded`, повтор старого задания байты thumbnail не изменил | `evidence/pipeline.json` | +| Падение до ack | `crash_before_effect`: RestartCount worker с 0 стал 1, файл ready. `crash_after_objects`: StartedAt сменился с `2026-11-12T09:24:05.644199607Z` на `2026-11-12T09:24:09.042866995Z`. `crash_after_commit`: с `2026-11-12T09:24:21.448659497Z` на `2026-11-12T09:24:24.489665631Z`. Во всех трёх случаях строк вариантов 4 | `evidence/pipeline.json` | +| Временная ошибка и битый файл | `transient_once`: события `leased`, `retry`, `leased`, `ready`, попытка стала 2, строк вариантов 4. Битые байты в staging дали `invalid_image` на попытке 1. После возврата байтов replay вернул `queued`, повторный replay вернул `already_ready`, строк вариантов 4 | `evidence/pipeline.json` | +| Истёкшая аренда | Заданию вручную поставлены `lease_owner=dead` и `lease_until` на две минуты в прошлом, неопубликованный outbox удалён. Worker сам вернул его в работу. События: `lease_expired`, `leased`, `ready`. Попытка осталась 1. До ready прошло 17,078 с от старта worker | `evidence/pipeline.json`, сценарий `lease` | +| Текст 503 | После остановки Redis балансировщик ответил 503 страницей HAProxy: `503 Service Unavailable` и `No server is available to handle this request.` Прямой запрос в контейнер API ответил 503 и «Сервис временно недоступен». В обоих текстах не было слов redis, postgres, minio, password, select, rabbit, amqp, secret, database, bucket, messenger-attachments. После healthy Redis первый запрос через балансировщик получил 200 за 0,063 с | `evidence/pipeline.json`, сценарий `ready_503` | +| EXIF | JPEG 200×100 с Orientation 6 загружен через API. Ответ 202, затем ready. Thumbnail 100×200, crop 128×128, optimized 100×200. Отдельный вызов `render` в контейнере worker сохранил исходные байты | `evidence/exif.json` | +| Очередь failed в конце | `rabbitmqctl list_queues`: `attachments.failed` — 1, `attachments.process` — 0, `attachments.retry` — 0. Единица осталась от намеренного `invalid_image`. Replay пишет новую рабочую доставку и старое сообщение DLQ не удаляет | вывод этой команды | -## Эксперименты и отказы +Публичные файлы в `contracts/` и `tests/` я не меняла. После всех сценариев smoke я запустила ещё раз, он снова напечатал PASS. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +## Трек 2 -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Я выбрала трек 2, а не трек про очередь под перегрузкой. Трек 2 стоит на тех же механизмах, которые и так нужны для оценки 4: outbox, повтор с паузой, падение после записи объекта и до ack, явный replay. Их можно показать на одном файле и конкретных идентификаторах. Трек 1 требовал бы отдельный нагрузочный опыт. Таких чисел у меня нет, и я их не придумывала. -## Ограничения и следующие шаги +Гипотеза: повторная доставка, чужой порядок и старое задание после новой версии не создают второй набор файлов и не затирают более новый результат. Доставка при этом остаётся at-least-once. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Как я ломала worker. В таблице `fault_injections` есть строка с видом сбоя. Пока строки нет, код её не включает. Виды: пропуск публикации из API, выход процесса до эффекта, после записи объектов и после COMMIT, одна временная ошибка. Выход — `os._exit(75)`. Политика `unless-stopped` поднимает контейнер снова. + +Что наблюдали. Все семь сценариев трека в `evidence/pipeline.json` закончились `ok: true`. Новый `pipeline_version` остался 2. Старое задание получило `error_code=superseded` только на строке задания, у вложения `error_code` пустой. Повтор успешного задания вернул `already_ready` и не создал пятый вариант. + +Это не exactly-once. Брокер может принести сообщение дважды. Второй раз worker видит уже записанный эффект или уже закрытое задание и только подтверждает доставку. + +## Ограничения + +Стенд переживает остановку одной реплики API, рестарт Postgres, Redis, MinIO и RabbitMQ, если volume на месте, и остановку worker на время приёма файла. Выключение компьютера и поломка диска данные не сохраняют. Postgres, Redis, MinIO, RabbitMQ и HAProxy в стенде по одному экземпляру. Их отказ останавливает соответствующую часть сервиса. + +Прямую загрузку по временной ссылке я не добавляла. + +Если обе реплики API не проходят проверку готовности, клиент через HAProxy видит английскую HTML-страницу 503, а не JSON API. Сам API при этом отвечает фиксированной русской фразой. Оба текста я проверила: имён хранилищ, SQL и секретов там нет. + +Непрерывная публикация outbox живёт в worker. Если worker остановлен и разовая публикация из API пропущена, задание ждёт старта worker. Это я показала. Отдельного второго публикатора нет. + +Аренду я проверила записью в SQL уже истёкшего срока, а не убийством процесса на середине Pillow. Сработал путь sweeper: `lease_expired`, затем обычная обработка, попытка не выросла. + +Если декодирование упирается в срок аренды, рабочий поток Pillow остаётся демоном и может досчитать кадр уже после того, как попытка брошена. Такой случай я не запускала. + +Строки `fault_injections` — учебный переключатель. В обычной работе таблица пустая, и эти ветки не выполняются. + +Повтор успешного задания не исполняет его заново, но старое сообщение в `attachments.failed` не стирает. После прогона там осталось 1 сообщение. + +Байты JPEG и WebP зависят от этой сборки Pillow. Другая версия библиотеки может дать другой файл при тех же правилах размера. + +Нагрузку, throughput и p95 я не измеряла. Трек 1 я не заявляю. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..263b2ad4 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,290 @@ +name: mtusi-lab5 + +services: + postgres: + image: postgres:16.10-bookworm@sha256:38471f330eb885e04de130b768d6db4e10469e2311879c7e5c699f6d2d8a1c74 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: + - postgres + - -c + - fsync=on + - -c + - synchronous_commit=on + - -c + - full_page_writes=on + - -c + - max_connections=80 + - -c + - shared_buffers=64MB + volumes: + - postgres-data:/var/lib/postgresql/data + secrets: + - postgres_password + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + cpus: "1.00" + mem_limit: 512m + restart: unless-stopped + networks: + - lab + + redis: + image: redis:7.4.2-bookworm@sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9 + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --dir + - /data + - --save + - "60" + - "1" + - --bind + - 0.0.0.0 + - --protected-mode + - "no" + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + cpus: "0.25" + mem_limit: 128m + restart: unless-stopped + networks: + - lab + + rabbitmq: + image: rabbitmq:3.13.7-alpine@sha256:d7af1c87c5f1eda13fcfca06db452bf3aeab6619fc3358b68535c0c02c4e52bc + hostname: rabbitmq + entrypoint: ["/bin/sh", "/init/rabbit-entry.sh"] + volumes: + - rabbitmq-data:/var/lib/rabbitmq + - ./scripts/rabbit-entry.sh:/init/rabbit-entry.sh:ro + secrets: + - rabbit_user + - rabbit_password + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 10s + retries: 12 + start_period: 180s + cpus: "1.00" + mem_limit: 1024m + restart: unless-stopped + networks: + - lab + + minio: + image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + command: ["server", "/data", "--console-address", "127.0.0.1:9001"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + volumes: + - minio-data:/data + secrets: + - minio_root_user + - minio_root_password + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 8s + cpus: "0.50" + mem_limit: 512m + restart: unless-stopped + networks: + - lab + + minio-init: + image: pgsty/minio@sha256:b6bfe7239bfc83fb90d31612d9704d86039dd714f7904b3f1ad68f211e602372 + entrypoint: ["/bin/sh", "/init/minio-init.sh"] + volumes: + - ./scripts/minio-init.sh:/init/minio-init.sh:ro + - ./infra/minio-api-policy.json:/init/policy.json:ro + secrets: + - minio_root_user + - minio_root_password + - s3_access_key + - s3_secret_key + depends_on: + minio: + condition: service_healthy + restart: "no" + networks: + - lab + + migrate: + build: . + image: mtusi-lab5-api:local + command: ["python", "/app/src/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + cpus: "0.50" + mem_limit: 256m + restart: "no" + networks: + - lab + + api: + build: . + image: mtusi-lab5-api:local + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: redis + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + POOL_MAX: ${POOL_MAX:-10} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: minio:9000 + S3_BUCKET: messenger-attachments + S3_SECURE: "false" + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-120} + secrets: + - database_url + - cache_url + - broker_url + - s3_access_key + - s3_secret_key + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + rabbitmq: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 25s + init: true + stop_grace_period: 10s + cpus: "0.50" + mem_limit: 512m + deploy: + replicas: 2 + restart: unless-stopped + networks: + - lab + + worker: + build: . + image: mtusi-lab5-api:local + command: ["python", "/app/src/worker.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: minio:9000 + S3_BUCKET: messenger-attachments + S3_SECURE: "false" + S3_ACCESS_KEY_FILE: /run/secrets/s3_access_key + S3_SECRET_KEY_FILE: /run/secrets/s3_secret_key + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: + - database_url + - broker_url + - s3_access_key + - s3_secret_key + depends_on: + migrate: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + healthcheck: + disable: true + init: true + stop_grace_period: 20s + cpus: "1.00" + mem_limit: 768m + restart: unless-stopped + networks: + - lab + + lb: + image: haproxy:2.8.14@sha256:2344d1efbf7a932c7a3d49b60748c28af81c70fd11fcadd983e2be1f0376a69d + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + api: + condition: service_healthy + cpus: "0.25" + mem_limit: 128m + restart: unless-stopped + networks: + - lab + +networks: + lab: + name: mtusi-lab5-net + +volumes: + postgres-data: + redis-data: + minio-data: + rabbitmq-data: + +secrets: + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + postgres_password: + file: ./secrets/postgres_password + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + s3_access_key: + file: ./secrets/s3_access_key + s3_secret_key: + file: ./secrets/s3_secret_key + rabbit_user: + file: ./secrets/rabbit_user + rabbit_password: + file: ./secrets/rabbit_password + broker_url: + file: ./secrets/broker_url diff --git a/evidence/exif.json b/evidence/exif.json new file mode 100644 index 00000000..4c2d34bb --- /dev/null +++ b/evidence/exif.json @@ -0,0 +1,20 @@ +{ + "file": "JPEG 200x100, EXIF Orientation 6, red pixels", + "in_process_render": { + "where": "docker exec mtusi-lab5-worker-1 python, function images.render", + "original_unchanged": true, + "thumbnail": [100, 200], + "crop": [128, 128], + "optimized": [100, 200] + }, + "http_upload": { + "command": "POST /api/v1/chats/{id}/attachments?filename=orient6.jpg", + "upload_status": "queued", + "final_status": "ready", + "variants": { + "thumbnail": [100, 200], + "crop": [128, 128], + "optimized": [100, 200] + } + } +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..d14e7af4 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,24 @@ +Команда (через WSL): +python3 -X utf8 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 + +Прогон 1, код выхода 0: +{"observed_instances_before": 2, "successful_writes": 97, "failed_attempts": 1, "first_recovery_seconds": 1.469} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +Остановлен mtusi-lab5-api-2, exit 143, FinishedAt 2026-11-12T09:00:09.569371803Z. +mtusi-lab5-api-1 остался running. + +Между прогонами реплика поднята командой: +docker compose --env-file .env up -d --no-deps --scale api=2 api + +Прогон 2, код выхода 0: +{"observed_instances_before": 2, "successful_writes": 98, "failed_attempts": 1, "first_recovery_seconds": 0.975} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +Снова остановлен mtusi-lab5-api-2, exit 143. api-1 остался running. +Публичный тест оба раза выбрал ту реплику, которая последней ответила. Это был api-2. + +Отдельная проверка второй реплики, тот же scripts/stop-one.sh: +TARGET_INSTANCE_ID=2e03f20dd3b5 +Остановлен mtusi-lab5-api-1, hostname 2e03f20dd3b5, exit 143. +mtusi-lab5-api-2 остался running. + +После проверки реплика снова запущена. Дата: 12 ноября 2026. diff --git a/evidence/negative.json b/evidence/negative.json new file mode 100644 index 00000000..720da0cb --- /dev/null +++ b/evidence/negative.json @@ -0,0 +1,23 @@ +{ + "command": "python -X utf8 student-tests/run_negative.py http://127.0.0.1:8080", + "result": "PASS negative", + "results": [ + {"name": "chat_uuid_case", "status": 200}, + {"name": "blank_text", "status": 400}, + {"name": "bad_json", "status": 400}, + {"name": "short_password", "status": 400}, + {"name": "omitted_equals_empty", "status": 201}, + {"name": "same_key_other_body", "status": 409}, + {"name": "no_second_row", "status": 200}, + {"name": "same_key_other_chat", "status": 201}, + {"name": "foreign_chat", "status": 404}, + {"name": "bad_cursor", "status": 400}, + {"name": "slash_name", "status": 400}, + {"name": "huge_pixels", "status": 413}, + {"name": "bad_png", "status": 415}, + {"name": "type_mismatch", "status": 415}, + {"name": "oversize", "status": 413}, + {"name": "unknown_attachment", "status": 404}, + {"name": "no_token", "status": 401} + ] +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..95246884 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,20 @@ +Команда (через WSL, потому что Windows не запускает .sh): +python3 -X utf8 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 + +Вывод: +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +Код выхода: 0. Дата: 12 ноября 2026. + +StartedAt после hook: +postgres 2026-11-12T08:58:51.311769003Z +redis 2026-11-12T08:58:50.947096648Z +minio 2026-11-12T08:58:51.30945974Z +rabbitmq 2026-11-12T08:59:03.356512147Z +api-1 2026-11-12T08:59:15.032550589Z +api-2 2026-11-12T08:59:14.961115981Z +lb 2026-11-12T08:56:49.466505107Z (не перезапускался) +worker 2026-11-12T08:56:32.031493932Z (скрипт его не трогает) + +Hook перезапускает postgres, redis, minio, rabbitmq и api. Volume не удалялись. diff --git a/evidence/pipeline.json b/evidence/pipeline.json new file mode 100644 index 00000000..f5b7a8dd --- /dev/null +++ b/evidence/pipeline.json @@ -0,0 +1,318 @@ +{ + "results": [ + { + "name": "durability", + "ok": true, + "seconds": 122.813, + "detail": { + "upload_status": "queued", + "content_while_queued": 409, + "text_status": 201, + "text_id": "0cb05d10-70d9-46a2-9fad-3272a92af1b3", + "linked_id": "5ffaee48-8e7c-4742-a5f1-b2991fed1a4b", + "queue_before_api_restart": 1, + "queue_after_api_restart": 1, + "queue_after_rabbit_restart": 1, + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "outbox_gap", + "ok": true, + "seconds": 26.781, + "detail": { + "unpublished_before_worker": 1, + "queue_depth": 0, + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "duplicate", + "ok": true, + "seconds": 19.203, + "detail": { + "republish": true, + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "reorder", + "ok": true, + "seconds": 19.688, + "detail": { + "attempt": 1, + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "stale_version", + "ok": true, + "seconds": 26.906, + "detail": { + "bump": { + "job_id": "a7af6898-08db-4a00-aff6-a2b654e3e0da", + "pipeline_version": 2 + }, + "old_job_status": "superseded", + "old_error_code": "superseded", + "attachment_error_code": null, + "picture": { + "pipeline_version": 2, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + }, + "thumbnail_unchanged": true + } + }, + { + "name": "crash_before_effect", + "ok": true, + "seconds": 19.281, + "detail": { + "restart_count_before": 0, + "restart_count_after": 1, + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "crash_after_objects", + "ok": true, + "seconds": 14.968, + "detail": { + "started_at": "2026-11-12T09:24:05.644199607Z", + "restarted_at": "2026-11-12T09:24:09.042866995Z", + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "crash_after_commit", + "ok": true, + "seconds": 11.563, + "detail": { + "started_at": "2026-11-12T09:24:21.448659497Z", + "restarted_at": "2026-11-12T09:24:24.489665631Z", + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "transient_once", + "ok": true, + "seconds": 13.187, + "detail": { + "attempt": 2, + "events": [ + "leased", + "retry", + "leased", + "ready" + ], + "variant_rows": 4, + "status": "ready" + } + }, + { + "name": "permanent_replay", + "ok": true, + "seconds": 28.407, + "detail": { + "corrupt_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "failed_code": "invalid_image", + "attempt_at_fail": 1, + "restored_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "replay": { + "result": "queued" + }, + "second_replay": { + "result": "already_ready" + }, + "variant_rows": 4, + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "lease", + "ok": true, + "seconds": 41.14, + "detail": { + "seconds_until_ready": 17.078, + "events": [ + "lease_expired", + "leased", + "ready" + ], + "attempt": 1, + "picture": { + "pipeline_version": 1, + "variant_rows": 4, + "thumbnail": [ + 256, + 160 + ], + "crop": [ + 128, + 128 + ], + "optimized": [ + 320, + 200 + ], + "original_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75" + } + } + }, + { + "name": "ready_503", + "ok": true, + "seconds": 13.625, + "detail": { + "lb_status": 503, + "lb_body": "

503 Service Unavailable

\nNo server is available to handle this request.\n\n", + "api_status": 503, + "api_message": "Сервис временно недоступен", + "forbidden": [], + "recovered": 200, + "recovery_seconds": 0.063 + } + } + ] +} \ No newline at end of file diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..94f7d800 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,47 @@ +global + maxconn 256 + tune.bufsize 65536 + tune.maxrewrite 8192 + +defaults + mode http + timeout connect 500ms + timeout client 60s + timeout server 60s + timeout http-request 60s + timeout http-keep-alive 2s + timeout queue 2s + timeout check 800ms + option redispatch + option dontlognull + retries 2 + retry-on conn-failure empty-response 502 503 504 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + accepted_payload_size 8192 + +frontend web + bind *:8080 + default_backend api + +backend api + balance roundrobin + # Короткий JSON сообщения можно повторить. Загрузку файла не повторяем: + # повтор создал бы второй объект. Тело загрузки не буферизуется целиком. + acl upload method POST path_reg -i ^/api/v1/chats/[0-9a-fA-F-]+/attachments$ + http-request wait-for-body time 2s if METH_POST !upload + http-request disable-l7-retry if upload + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template api 8 api:8080 check inter 500ms fall 2 rise 2 resolvers docker init-addr none resolve-prefer ipv4 on-marked-down shutdown-sessions diff --git a/infra/minio-api-policy.json b/infra/minio-api-policy.json new file mode 100644 index 00000000..cfef8ef8 --- /dev/null +++ b/infra/minio-api-policy.json @@ -0,0 +1,19 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject", + "s3:DeleteObject" + ], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:GetBucketLocation", "s3:ListBucket"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..994fa1d1 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,48 @@ +-- Пользователи, чаты, участники и сообщения. +-- Создание чата в приложении идёт одной транзакцией: строка chats и строки chat_members. +-- Индекс messages_history_idx совпадает с keyset-запросом истории: +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id +-- LIMIT $4 + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_nonblank CHECK (btrim(display_name) <> ''), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_nonblank CHECK (btrim(title) <> ''), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + ordinal smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_ordinal_nonneg CHECK (ordinal >= 0) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + body text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_body_nonblank CHECK (btrim(body) <> ''), + CONSTRAINT messages_body_len CHECK (char_length(body) BETWEEN 1 AND 4000) +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..d4e557b5 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,19 @@ +-- Результат успешного POST сообщения. +-- Область ключа: пользователь + метод + путь чата + Idempotency-Key. +-- Строка пишется в той же транзакции, что и messages, поэтому 5xx +-- не оставляет «занятый» ключ. Другое тело при живом ключе — отказ +-- на уровне приложения (хеш не совпал), без второй строки messages. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + http_method text NOT NULL, + request_path text NOT NULL, + idem_key text NOT NULL, + request_hash text NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id), + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, http_method, request_path, idem_key), + CONSTRAINT idempotency_method_post CHECK (http_method = 'POST'), + CONSTRAINT idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_hash_len CHECK (char_length(request_hash) = 64) +); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..ebb52e38 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,42 @@ +-- Метаданные вложения. Байты лежат в приватном bucket, не в этой таблице. +-- Пустой текст допустим, если к сообщению прикреплено вложение. Пробелы по-прежнему нельзя. + +ALTER TABLE messages DROP CONSTRAINT messages_body_nonblank; +ALTER TABLE messages DROP CONSTRAINT messages_body_len; +ALTER TABLE messages + ADD CONSTRAINT messages_body_len CHECK (char_length(body) <= 4000); +ALTER TABLE messages + ADD CONSTRAINT messages_body_nonblank CHECK (body = '' OR btrim(body) <> ''); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL UNIQUE, + status text NOT NULL DEFAULT 'ready', + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type_chk CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_chk CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha_chk CHECK (char_length(sha256) = 64), + CONSTRAINT attachments_status_chk CHECK (status = 'ready'), + CONSTRAINT attachments_key_chk CHECK (char_length(object_key) BETWEEN 1 AND 512) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + ordinal integer NOT NULL, + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, ordinal), + CONSTRAINT message_attachments_ordinal_chk CHECK (ordinal >= 0 AND ordinal < 10) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/migrations/004_jobs.sql b/migrations/004_jobs.sql new file mode 100644 index 00000000..f735350c --- /dev/null +++ b/migrations/004_jobs.sql @@ -0,0 +1,108 @@ +-- Задание на обработку, варианты и outbox. +-- Сообщение брокера не содержит байты и пароли: в outbox лежит только JSON задания. +-- Ключ эффекта — (attachment_id, pipeline_version, name). Повтор не создаёт вторую строку. + +ALTER TABLE attachments DROP CONSTRAINT attachments_status_chk; + +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; + +ALTER TABLE attachments ADD COLUMN error_code text; + +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; + +ALTER TABLE attachments ADD COLUMN staging_key text; + +ALTER TABLE attachments ADD COLUMN staging_deleted_at timestamptz; + +ALTER TABLE attachments ADD CONSTRAINT attachments_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed')); + +ALTER TABLE attachments ADD CONSTRAINT attachments_pipeline_chk CHECK (pipeline_version >= 1); + +ALTER TABLE attachments ADD CONSTRAINT attachments_error_chk CHECK ( + (status = 'failed' AND error_code IN ('invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout')) + OR (status <> 'failed' AND error_code IS NULL) +); + +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + status text NOT NULL, + attempt integer NOT NULL, + max_attempts integer NOT NULL, + lease_owner text, + lease_until timestamptz, + next_attempt_at timestamptz, + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachment_jobs_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded')), + CONSTRAINT attachment_jobs_attempt_chk CHECK (attempt >= 1 AND attempt <= max_attempts), + CONSTRAINT attachment_jobs_max_chk CHECK (max_attempts BETWEEN 1 AND 20), + CONSTRAINT attachment_jobs_version_chk CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_error_chk CHECK ( + error_code IS NULL OR error_code IN ( + 'invalid_image', 'processing_failed', 'retry_exhausted', 'processing_timeout', 'superseded' + ) + ), + CONSTRAINT attachment_jobs_one_version UNIQUE (attachment_id, pipeline_version) +); + +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs (status, next_attempt_at); + +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL UNIQUE, + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_name_chk CHECK (name IN ('original', 'thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_type_chk CHECK (content_type IN ('image/png', 'image/jpeg', 'image/webp')), + CONSTRAINT attachment_variants_size_chk CHECK (size_bytes >= 1), + CONSTRAINT attachment_variants_dim_chk CHECK (width >= 1 AND height >= 1), + CONSTRAINT attachment_variants_sha_chk CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +CREATE TABLE outbox ( + id uuid PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id), + attempt integer NOT NULL, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + published_at timestamptz, + CONSTRAINT outbox_attempt_chk CHECK (attempt >= 1) +); + +CREATE INDEX outbox_pending_idx ON outbox (created_at) WHERE published_at IS NULL; + +CREATE TABLE job_events ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + kind text NOT NULL, + detail text, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT job_events_kind_chk CHECK (char_length(kind) BETWEEN 1 AND 40), + CONSTRAINT job_events_detail_chk CHECK (detail IS NULL OR char_length(detail) <= 64) +); + +CREATE TABLE fault_injections ( + id uuid PRIMARY KEY, + kind text NOT NULL, + attachment_id uuid, + armed boolean NOT NULL DEFAULT true, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT fault_injections_kind_chk CHECK (kind IN ( + 'skip_inline_publish', 'crash_before_effect', 'crash_after_objects', + 'crash_after_commit', 'transient_once' + )) +); + +CREATE INDEX fault_injections_armed_idx ON fault_injections (kind, created_at) WHERE armed; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..24ee2fef --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.12 +uvicorn[standard]==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +minio==7.2.15 +pillow==11.1.0 +pika==1.3.2 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..e1e4ba1d --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +"""Создать локальные .env и secrets для стенда. Пароль в консоль не печатается.""" +import secrets +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRET_DIR = ROOT / "secrets" +PASSWORD_FILE = SECRET_DIR / "postgres_password" +DATABASE_FILE = SECRET_DIR / "database_url" +CACHE_FILE = SECRET_DIR / "cache_url" +MINIO_USER_FILE = SECRET_DIR / "minio_root_user" +MINIO_PASSWORD_FILE = SECRET_DIR / "minio_root_password" +S3_ACCESS_FILE = SECRET_DIR / "s3_access_key" +S3_SECRET_FILE = SECRET_DIR / "s3_secret_key" +RABBIT_USER_FILE = SECRET_DIR / "rabbit_user" +RABBIT_PASSWORD_FILE = SECRET_DIR / "rabbit_password" +BROKER_FILE = SECRET_DIR / "broker_url" +ENV_FILE = ROOT / ".env" +ENV_EXAMPLE = ROOT / ".env.example" + + +def main(): + files = (PASSWORD_FILE, DATABASE_FILE, CACHE_FILE) + if any(path.exists() for path in files) and not all(path.exists() for path in files): + print("secrets/ заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + SECRET_DIR.mkdir(exist_ok=True) + if all(path.exists() for path in files): + print("secrets already exist") + else: + password = secrets.token_hex(24) + PASSWORD_FILE.write_text(password + "\n", encoding="utf-8") + DATABASE_FILE.write_text( + "postgresql://messenger:{}@postgres:5432/messenger\n".format(password), + encoding="utf-8", + ) + CACHE_FILE.write_text("redis://redis:6379/0\n", encoding="utf-8") + print("secrets created") + s3_files = (MINIO_USER_FILE, MINIO_PASSWORD_FILE, S3_ACCESS_FILE, S3_SECRET_FILE) + if any(path.exists() for path in s3_files) and not all(path.exists() for path in s3_files): + print("secrets/ для MinIO заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + if all(path.exists() for path in s3_files): + print("s3 secrets already exist") + else: + MINIO_USER_FILE.write_text("labroot", encoding="utf-8") + MINIO_PASSWORD_FILE.write_text(secrets.token_hex(24), encoding="utf-8") + S3_ACCESS_FILE.write_text("api" + secrets.token_hex(8), encoding="utf-8") + S3_SECRET_FILE.write_text(secrets.token_hex(24), encoding="utf-8") + print("s3 secrets created") + broker_files = (RABBIT_USER_FILE, RABBIT_PASSWORD_FILE, BROKER_FILE) + if any(path.exists() for path in broker_files) and not all(path.exists() for path in broker_files): + print("secrets/ для брокера заполнены не полностью, ничего не перезаписываю", file=sys.stderr) + return 1 + if all(path.exists() for path in broker_files): + print("broker secrets already exist") + else: + user = "lab" + password = secrets.token_hex(24) + RABBIT_USER_FILE.write_text(user, encoding="utf-8") + RABBIT_PASSWORD_FILE.write_text(password, encoding="utf-8") + BROKER_FILE.write_text( + "amqp://{}:{}@rabbitmq:5672/%2F\n".format(user, password), + encoding="utf-8", + ) + print("broker secrets created") + if ENV_FILE.exists(): + print(".env already exists") + else: + ENV_FILE.write_text(ENV_EXAMPLE.read_text(encoding="utf-8"), encoding="utf-8") + print(".env created") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/minio-init.sh b/scripts/minio-init.sh new file mode 100755 index 00000000..5685ff3b --- /dev/null +++ b/scripts/minio-init.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Идемпотентно создаёт приватный bucket и отдельного пользователя API. +# Корневой пароль остаётся только у этого контейнера и у MinIO. +set -eu +ROOT_USER=$(tr -d '\r\n' < /run/secrets/minio_root_user) +ROOT_PW=$(tr -d '\r\n' < /run/secrets/minio_root_password) +ACCESS=$(tr -d '\r\n' < /run/secrets/s3_access_key) +SECRET=$(tr -d '\r\n' < /run/secrets/s3_secret_key) + +attempt=0 +until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PW" >/dev/null; do + attempt=$((attempt + 1)) + if [ "$attempt" -gt 30 ]; then + echo "minio alias failed" >&2 + exit 1 + fi + sleep 1 +done + +mc mb --ignore-existing local/messenger-attachments >/dev/null +mc anonymous set none local/messenger-attachments >/dev/null + +if ! mc admin user info local "$ACCESS" >/dev/null 2>&1; then + mc admin user add local "$ACCESS" "$SECRET" >/dev/null +fi +mc admin policy detach local messenger-api --user "$ACCESS" >/dev/null 2>&1 || true +mc admin policy rm local messenger-api >/dev/null 2>&1 || true +mc admin policy create local messenger-api /init/policy.json >/dev/null +mc admin policy attach local messenger-api --user "$ACCESS" >/dev/null +echo "minio bucket ready" diff --git a/scripts/rabbit-entry.sh b/scripts/rabbit-entry.sh new file mode 100755 index 00000000..a81a07e7 --- /dev/null +++ b/scripts/rabbit-entry.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Пароль читается из secret и попадает только в файл конфигурации внутри контейнера. +set -eu +user=$(tr -d '\r\n' < /run/secrets/rabbit_user) +pass=$(tr -d '\r\n' < /run/secrets/rabbit_password) +mkdir -p /tmp/rabbit-conf +printf 'default_user = %s\ndefault_pass = %s\nvm_memory_high_watermark.absolute = 768MB\n' "$user" "$pass" > /tmp/rabbit-conf/rabbitmq.conf +chmod 644 /tmp/rabbit-conf/rabbitmq.conf +export RABBITMQ_CONFIG_FILE=/tmp/rabbit-conf/rabbitmq +exec /usr/local/bin/docker-entrypoint.sh rabbitmq-server diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..da61b1ec --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Рестарт API и всех хранилищ со состоянием. Volume не удаляем, миграции заново не заливаем. +# Балансировщик без своего состояния не трогаем: он продолжает слушать тот же порт. +docker compose --env-file .env restart postgres redis minio rabbitmq +docker compose --env-file .env up -d --wait --no-deps postgres redis minio rabbitmq +docker compose --env-file .env restart api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..14c6254d --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,23 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?}" + +found="" +for id in $(docker ps -q \ + --filter label=com.docker.compose.project=mtusi-lab5 \ + --filter label=com.docker.compose.service=api); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + short=$(printf %s "$id" | cut -c1-12) + if [ "$host" = "$TARGET_INSTANCE_ID" ] || [ "$short" = "$TARGET_INSTANCE_ID" ]; then + found=$id + break + fi +done + +if [ -z "$found" ]; then + echo "api replica not found" >&2 + exit 1 +fi + +# SIGTERM, затем процесс остаётся остановленным (restart: unless-stopped). +docker stop --time 5 "$found" >/dev/null diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..dd556dca --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Ждать GET /health/ready через балансировщик. Без токенов и тел ошибок в выводе.""" +import os +import sys +import time +import urllib.error +import urllib.request + +URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + "/health/ready" + + +def main(): + deadline = time.monotonic() + 90 + last = "not-started" + while time.monotonic() < deadline: + try: + response = urllib.request.urlopen(URL, timeout=2) + if response.status == 200 and response.headers.get("X-Instance-Id"): + print("ready") + return 0 + last = str(response.status) + except (urllib.error.URLError, TimeoutError, OSError) as exc: + last = type(exc).__name__ + time.sleep(1) + print("not ready: " + last, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/blobs.py b/src/blobs.py new file mode 100644 index 00000000..1da6f090 --- /dev/null +++ b/src/blobs.py @@ -0,0 +1,228 @@ +"""Проверка PNG/JPEG и доступ к приватному bucket. Ключ объекта назначает сервер.""" +import io +import os +from datetime import datetime, timezone + +from minio import Minio +from minio.error import S3Error +from PIL import Image, UnidentifiedImageError +from PIL.Image import DecompressionBombError + +from errors import ApiError, DependencyDown + +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + +_client = None + + +def read_secret(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def client(): + global _client + if _client is None: + endpoint = os.environ.get("S3_ENDPOINT", "minio:9000").strip() + access = read_secret(os.environ["S3_ACCESS_KEY_FILE"]) + secret = read_secret(os.environ["S3_SECRET_KEY_FILE"]) + secure = os.environ.get("S3_SECURE", "false").strip().lower() == "true" + _client = Minio(endpoint, access_key=access, secret_key=secret, secure=secure) + return _client + + +def bucket_ready(): + try: + return client().bucket_exists(BUCKET) + except Exception: + return False + + +def put_bytes(key, data, content_type, created_at): + stamp = created_at.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + try: + client().put_object( + BUCKET, + key, + io.BytesIO(data), + length=len(data), + content_type=content_type, + metadata={"created-at": stamp}, + ) + except S3Error as exc: + raise DependencyDown() from exc + + +def delete_quiet(key): + try: + client().remove_object(BUCKET, key) + return True + except Exception: + return False + + +def open_object(key): + try: + return client().get_object(BUCKET, key) + except S3Error as exc: + code = getattr(exc, "code", "") + if code in ("NoSuchKey", "NoSuchBucket"): + return None + raise DependencyDown() from exc + + +def object_key(chat_id, attachment_id): + return "chats/{}/attachments/{}/original".format(chat_id, attachment_id) + + +def staging_object_key(attachment_id): + return "staging/{}/original".format(attachment_id) + + +def variant_object_key(attachment_id, pipeline_version, name): + return "ready/{}/v{}/{}".format(attachment_id, int(pipeline_version), name) + + +def header_size(data, declared): + if declared == "image/png": + return _png_size(data) + if declared == "image/jpeg": + return _jpeg_size(data) + return None + + +def read_bytes(key): + body = open_object(key) + if body is None: + return None + try: + return body.read() + finally: + body.close() + body.release_conn() + + +def declared_type(header): + if not header: + return None + media = header.split(";", 1)[0].strip().lower() + if media in ("image/png", "image/jpeg"): + return media + return None + + +def _png_size(data): + if len(data) < 24 or data[:8] != b"\x89PNG\r\n\x1a\n" or data[12:16] != b"IHDR": + return None + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + return width, height + + +def _jpeg_size(data): + if len(data) < 4 or data[:2] != b"\xff\xd8": + return None + index = 2 + limit = len(data) + while index + 8 < limit: + if data[index] != 0xFF: + return None + while index < limit and data[index] == 0xFF: + index += 1 + if index >= limit: + return None + marker = data[index] + index += 1 + if marker in (0xD8, 0xD9): + continue + if index + 2 > limit: + return None + segment = int.from_bytes(data[index:index + 2], "big") + if segment < 2 or index + segment > limit: + return None + if marker in (0xC0, 0xC1, 0xC2): + if segment < 7: + return None + height = int.from_bytes(data[index + 3:index + 5], "big") + width = int.from_bytes(data[index + 5:index + 7], "big") + return width, height + index += segment + return None + + +def _too_many_pixels(width, height): + if width <= 0 or height <= 0: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") + if width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS: + raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей") + + +def inspect_image(data, declared): + if declared == "image/png": + size = _png_size(data) + elif declared == "image/jpeg": + size = _jpeg_size(data) + else: + size = None + detected = None + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + if size is None or detected != declared: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") + _too_many_pixels(size[0], size[1]) + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + except DecompressionBombError as exc: + raise ApiError(413, "payload_too_large", "В изображении больше 40 миллионов пикселей") from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG") from exc + return declared + + +def list_objects(prefix=""): + found = [] + prefixes = (prefix,) if prefix else ("chats/", "staging/", "ready/") + try: + for one in prefixes: + found.extend(_list_prefix(one)) + except S3Error as exc: + raise DependencyDown() from exc + return found + + +def _list_prefix(prefix): + found = [] + try: + for item in client().list_objects(BUCKET, prefix=prefix, recursive=True): + if item.is_dir: + continue + stat = client().stat_object(BUCKET, item.object_name) + raw = None + if stat.metadata: + meta = {str(key).lower(): value for key, value in stat.metadata.items()} + raw = meta.get("x-amz-meta-created-at") or meta.get("created-at") + created = _parse_stamp(raw) if raw else None + if created is None and item.last_modified is not None: + created = item.last_modified + found.append({"key": item.object_name, "created_at": created}) + except S3Error as exc: + raise DependencyDown() from exc + return found + + +def _parse_stamp(value): + text = value.strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(text) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..70922d38 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,132 @@ +"""RabbitMQ: publisher confirm при публикации, consumer ack отдельно в worker.""" +import json +import os +from contextlib import contextmanager + +import pika +from psycopg.types.json import Json + +from blobs import read_secret + +PROCESS_QUEUE = "attachments.process" +RETRY_QUEUE = "attachments.retry" +FAILED_QUEUE = "attachments.failed" + + +def broker_url(): + return read_secret(os.environ["BROKER_URL_FILE"]) + + +def _parameters(): + params = pika.URLParameters(broker_url()) + params.heartbeat = 60 + params.blocked_connection_timeout = 10 + params.socket_timeout = 10 + params.connection_attempts = 1 + return params + + +def ensure_topology(channel): + channel.queue_declare(queue=PROCESS_QUEUE, durable=True) + channel.queue_declare( + queue=RETRY_QUEUE, + durable=True, + arguments={ + "x-dead-letter-exchange": "", + "x-dead-letter-routing-key": PROCESS_QUEUE, + }, + ) + channel.queue_declare(queue=FAILED_QUEUE, durable=True) + + +@contextmanager +def broker_channel(): + connection = pika.BlockingConnection(_parameters()) + channel = connection.channel() + channel.confirm_delivery() + ensure_topology(channel) + try: + yield channel + finally: + try: + connection.close() + except Exception: + pass + + +def publish_raw(channel, queue, body, expiration_ms=None): + fields = { + "delivery_mode": 2, + "content_type": "application/json", + } + if expiration_ms is not None: + fields["expiration"] = str(int(expiration_ms)) + channel.basic_publish( + exchange="", + routing_key=queue, + body=body, + properties=pika.BasicProperties(**fields), + mandatory=True, + ) + + +def publish_pending(conn, only_job_id=None): + published = 0 + while published < 20: + with conn.transaction(): + conn.execute("SET LOCAL statement_timeout = '20s'") + params = [] + sql = """ + SELECT o.id, o.payload + FROM outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE o.published_at IS NULL + AND (j.next_attempt_at IS NULL OR j.next_attempt_at <= now()) + """ + if only_job_id is not None: + sql += " AND o.job_id = %s" + params.append(only_job_id) + sql += """ + ORDER BY o.created_at + FOR UPDATE OF o SKIP LOCKED + LIMIT 1 + """ + row = conn.execute(sql, params).fetchone() + if row is None: + return published + body = json.dumps( + row["payload"], + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + with broker_channel() as channel: + publish_raw(channel, PROCESS_QUEUE, body) + conn.execute( + "UPDATE outbox SET published_at = now() WHERE id = %s AND published_at IS NULL", + (row["id"],), + ) + published += 1 + if only_job_id is not None: + return published + return published + + +def publish_delay(body, delay_ms): + with broker_channel() as channel: + publish_raw(channel, RETRY_QUEUE, body, expiration_ms=max(1000, int(delay_ms))) + + +def publish_failed(body): + with broker_channel() as channel: + publish_raw(channel, FAILED_QUEUE, body) + + +def queue_depth(queue): + with broker_channel() as channel: + declared = channel.queue_declare(queue=queue, passive=True) + return int(declared.method.message_count) + + +def dump_payload(payload): + return Json(payload) diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..341036d8 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,32 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class DependencyDown(Exception): + """Postgres или Redis не ответили. Наружу это 503 без имени хранилища.""" + + +class IdempotencyConflict(Exception): + """Тот же Idempotency-Key уже зафиксирован с другим телом.""" + + +class AttachmentRejected(Exception): + """Вложение есть, но прикрепить его к сообщению нельзя.""" + + +class AttachmentNotReady(Exception): + """Байты ещё нельзя отдать: queued, processing или failed.""" + + +def error_body(code, message, request_id): + return { + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + } diff --git a/src/faults.py b/src/faults.py new file mode 100644 index 00000000..b18de66b --- /dev/null +++ b/src/faults.py @@ -0,0 +1,49 @@ +"""Точечные сбои для проверки. Без строки в fault_injections код их не включает.""" +import uuid + +from psycopg.pq import TransactionStatus + + +def _idle(conn): + # Чтение выше могло открыть транзакцию. Её откат не теряет запись: + # сама запись начинается следующим transaction() и делает COMMIT. + if conn.info.transaction_status != TransactionStatus.IDLE: + conn.rollback() + + +def take_fault(conn, kind, attachment_id=None): + _idle(conn) + with conn.transaction(): + row = conn.execute( + """ + UPDATE fault_injections AS target + SET armed = false + WHERE target.id = ( + SELECT id + FROM fault_injections + WHERE armed + AND kind = %s + AND (attachment_id IS NULL OR attachment_id = %s) + ORDER BY created_at + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + RETURNING id + """, + (kind, attachment_id), + ).fetchone() + return row is not None + + +def arm_fault(conn, kind, attachment_id=None): + fault_id = uuid.uuid4() + _idle(conn) + with conn.transaction(): + conn.execute( + """ + INSERT INTO fault_injections (id, kind, attachment_id, armed) + VALUES (%s, %s, %s, true) + """, + (fault_id, kind, attachment_id), + ) + return str(fault_id) diff --git a/src/images.py b/src/images.py new file mode 100644 index 00000000..83a705e9 --- /dev/null +++ b/src/images.py @@ -0,0 +1,129 @@ +"""Варианты изображения. Оригинал не перекодируется.""" +import io + +from PIL import Image, ImageOps, UnidentifiedImageError +from PIL.Image import DecompressionBombError + +from blobs import MAX_PIXELS, header_size + +THUMB_BOX = 256 +CROP_SIZE = 128 +OPT_BOX = 1280 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +class PermanentJobError(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +class TransientJobError(Exception): + def __init__(self, code): + super().__init__(code) + self.code = code + + +def _rgb(image): + if image.mode == "RGBA" or image.mode == "LA" or (image.mode == "P" and "transparency" in image.info): + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + if image.mode == "RGB": + return image + return image.convert("RGB") + + +def load_rgb(data, declared): + size = header_size(data, declared) + detected = None + if data.startswith(b"\x89PNG\r\n\x1a\n"): + detected = "image/png" + elif data.startswith(b"\xff\xd8\xff"): + detected = "image/jpeg" + if size is None or detected != declared: + raise PermanentJobError("invalid_image") + width, height = size + if width <= 0 or height <= 0 or width > MAX_PIXELS or height > MAX_PIXELS or width * height > MAX_PIXELS: + raise PermanentJobError("invalid_image") + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + oriented = ImageOps.exif_transpose(image) + rgb = _rgb(oriented) + rgb.load() + return rgb + except DecompressionBombError as exc: + raise PermanentJobError("invalid_image") from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise PermanentJobError("invalid_image") from exc + + +def fit_inside(image, box): + width, height = image.size + if width <= box and height <= box: + return image.copy() + if width >= height: + new_w = box + new_h = max(1, (height * box) // width) + else: + new_h = box + new_w = max(1, (width * box) // height) + return image.resize((new_w, new_h), Image.Resampling.LANCZOS) + + +def center_square(image, size): + width, height = image.size + side = min(width, height) + left = (width - side) // 2 + top = (height - side) // 2 + cropped = image.crop((left, top, left + side, top + side)) + if cropped.size != (size, size): + cropped = cropped.resize((size, size), Image.Resampling.LANCZOS) + return cropped + + +def _jpeg(image): + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True) + return { + "bytes": buffer.getvalue(), + "content_type": "image/jpeg", + "width": image.size[0], + "height": image.size[1], + } + + +def _webp(image): + buffer = io.BytesIO() + image.save(buffer, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD) + return { + "bytes": buffer.getvalue(), + "content_type": "image/webp", + "width": image.size[0], + "height": image.size[1], + } + + +def render(data, declared): + rgb = load_rgb(data, declared) + size = header_size(data, declared) + thumb = fit_inside(rgb, THUMB_BOX) + crop = center_square(rgb, CROP_SIZE) + optimized = fit_inside(rgb, OPT_BOX) + return { + "original": { + "bytes": data, + "content_type": declared, + "width": size[0], + "height": size[1], + }, + "thumbnail": _jpeg(thumb), + "crop": _jpeg(crop), + "optimized": _webp(optimized), + } diff --git a/src/jobs.py b/src/jobs.py new file mode 100644 index 00000000..6342c26c --- /dev/null +++ b/src/jobs.py @@ -0,0 +1,831 @@ +"""Состояния задания. Эффект один на (attachment_id, pipeline_version, вариант).""" +import hashlib +import json +import os +import random +import threading +import uuid +from datetime import datetime, timezone + +from psycopg.pq import TransactionStatus +from psycopg.types.json import Json + +from blobs import delete_quiet, put_bytes, read_bytes, variant_object_key +from errors import DependencyDown +from broker import publish_delay, publish_failed, publish_raw, broker_channel, PROCESS_QUEUE +from faults import take_fault +from images import PermanentJobError, TransientJobError, render + +REQUIRED = ("original", "thumbnail", "crop", "optimized") +PUBLIC_CODES = ("invalid_image", "processing_failed", "retry_exhausted", "processing_timeout") + + +class LostLease(Exception): + pass + + +def _idle(conn): + if conn.info.transaction_status != TransactionStatus.IDLE: + conn.rollback() + + +def max_attempts(): + raw = os.environ.get("JOB_MAX_ATTEMPTS", "5") + value = int(raw) + if not 1 <= value <= 20: + raise RuntimeError("JOB_MAX_ATTEMPTS") + return value + + +def lease_seconds(): + raw = os.environ.get("JOB_TIMEOUT_SECONDS", "60") + value = int(raw) + if value < 5: + raise RuntimeError("JOB_TIMEOUT_SECONDS") + return value + + +def retention_seconds(): + raw = os.environ.get("STAGING_RETENTION_SECONDS", "86400") + value = int(raw) + if value < 60: + raise RuntimeError("STAGING_RETENTION_SECONDS") + return value + + +def utc_now(): + return datetime.now(timezone.utc) + + +def dump_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + text = value.astimezone(timezone.utc).isoformat(timespec="microseconds") + if text.endswith("+00:00"): + text = text[:-6] + "Z" + return text + + +def backoff_seconds(failed_attempt): + base = min(30.0, 0.5 * (2 ** (failed_attempt - 1))) + return base + random.uniform(0, base * 0.25) + + +def job_message(job_id, attachment_id, pipeline_version, attempt, created_at): + if not isinstance(created_at, str): + created_at = dump_time(created_at) + return { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": created_at, + } + + +def _uuid(value): + try: + return str(uuid.UUID(str(value))) + except (ValueError, AttributeError, TypeError): + return None + + +def _stamp(value): + if not isinstance(value, str) or not value.endswith("Z"): + return None + try: + datetime.fromisoformat(value[:-1] + "+00:00") + except ValueError: + return None + return value + + +def parse_job(body): + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeError, ValueError, AttributeError): + return None + if not isinstance(payload, dict): + return None + allowed = { + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", + "traceparent", + "tracestate", + } + if set(payload) - allowed: + return None + required = allowed - {"traceparent", "tracestate"} + if not required <= set(payload): + return None + if payload.get("schema_version") != 1 or payload.get("type") != "attachment.process.v1": + return None + job_id = _uuid(payload.get("job_id")) + attachment_id = _uuid(payload.get("attachment_id")) + created_at = _stamp(payload.get("created_at")) + version = payload.get("pipeline_version") + attempt = payload.get("attempt") + if job_id is None or attachment_id is None or created_at is None: + return None + if type(version) is not int or type(attempt) is not int: + return None + if version < 1 or attempt < 1: + return None + if "traceparent" in payload: + traceparent = payload["traceparent"] + if not isinstance(traceparent, str) or len(traceparent) != 55: + return None + if "tracestate" in payload: + tracestate = payload["tracestate"] + if not isinstance(tracestate, str) or len(tracestate) > 512: + return None + payload["job_id"] = job_id + payload["attachment_id"] = attachment_id + return payload + + +def _event(conn, job, kind, detail=None): + conn.execute( + """ + INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail) + VALUES (%s, %s, %s, %s, %s) + """, + (job["id"], job["attachment_id"], job["pipeline_version"], kind, detail), + ) + + +def _message_body(job, attempt, created_at): + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], attempt, created_at) + return json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + + +def fail_job(conn, job, code): + if code not in PUBLIC_CODES and code != "superseded": + code = "processing_failed" + _idle(conn) + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', error_code = %s, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status IN ('queued', 'processing') + """, + (code, job["id"]), + ) + if code in PUBLIC_CODES: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND pipeline_version = %s + AND status IN ('queued', 'processing') + """, + (code, job["attachment_id"], job["pipeline_version"]), + ) + _event(conn, job, "failed", code) + try: + publish_failed(_message_body(job, job["attempt"], job["created_at"])) + except Exception: + # Реестр failed уже в базе. Повторная доставка увидит failed и подтвердит сообщение. + return + + +def schedule_retry(conn, job, code, owner): + attempt = int(job["attempt"]) + if attempt >= int(job["max_attempts"]): + fail_job(conn, job, "retry_exhausted") + return + delay = backoff_seconds(attempt) + new_attempt = attempt + 1 + created = utc_now() + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], new_attempt, created) + _idle(conn) + with conn.transaction(): + updated = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = %s, lease_owner = NULL, lease_until = NULL, + next_attempt_at = now() + make_interval(secs => %s), + error_code = NULL, updated_at = now() + WHERE id = %s AND status = 'processing' AND lease_owner = %s + RETURNING id + """, + (new_attempt, delay, job["id"], owner), + ).fetchone() + if updated is None: + return + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status = 'processing' + """, + (job["attachment_id"], job["pipeline_version"]), + ) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (uuid.uuid4(), job["id"], new_attempt, Json(payload), created), + ) + _event(conn, job, "retry", code) + + +def release_expired(conn): + released = 0 + _idle(conn) + with conn.transaction(): + rows = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', lease_owner = NULL, lease_until = NULL, updated_at = now() + WHERE status = 'processing' AND lease_until < now() + RETURNING id, attachment_id, pipeline_version, attempt, created_at + """ + ).fetchall() + for row in rows: + payload = job_message(row["id"], row["attachment_id"], row["pipeline_version"], row["attempt"], utc_now()) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + SELECT %s, %s, %s, %s, now() + WHERE NOT EXISTS ( + SELECT 1 FROM outbox WHERE job_id = %s AND published_at IS NULL + ) + """, + (uuid.uuid4(), row["id"], row["attempt"], Json(payload), row["id"]), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status = 'processing' + """, + (row["attachment_id"], row["pipeline_version"]), + ) + _event(conn, row, "lease_expired", None) + released += 1 + return released + + +def cleanup_staging(conn): + removed = 0 + _idle(conn) + with conn.transaction(): + rows = conn.execute( + """ + SELECT a.id, a.staging_key + FROM attachments a + WHERE a.staging_key IS NOT NULL + AND a.staging_deleted_at IS NULL + AND a.status IN ('ready', 'failed') + AND NOT EXISTS ( + SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id = a.id + AND j.status IN ('queued', 'processing') + ) + """ + ).fetchall() + for row in rows: + if delete_quiet(row["staging_key"]): + with conn.transaction(): + conn.execute( + """ + UPDATE attachments + SET staging_deleted_at = now() + WHERE id = %s AND staging_deleted_at IS NULL + """, + (row["id"],), + ) + removed += 1 + return removed + + +def _load_job(conn, job_id): + return conn.execute( + """ + SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts, + lease_owner, lease_until, next_attempt_at, error_code, created_at + FROM attachment_jobs + WHERE id = %s + """, + (job_id,), + ).fetchone() + + +def _load_attachment(conn, attachment_id): + return conn.execute( + """ + SELECT id, content_type, sha256, object_key, staging_key, staging_deleted_at, + status, pipeline_version + FROM attachments + WHERE id = %s + """, + (attachment_id,), + ).fetchone() + + +def _acquire(conn, job, owner, seconds): + _idle(conn) + with conn.transaction(): + locked = conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', lease_owner = %s, + lease_until = now() + make_interval(secs => %s), + updated_at = now() + WHERE id = %s + AND status IN ('queued', 'processing') + AND attempt = %s + AND pipeline_version = %s + AND (lease_owner IS NULL OR lease_owner = %s OR lease_until < now()) + RETURNING id + """, + (owner, seconds, job["id"], job["attempt"], job["pipeline_version"], owner), + ).fetchone() + if locked is None: + return False + conn.execute( + """ + UPDATE attachments + SET status = 'processing', updated_at = now() + WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing') + """, + (job["attachment_id"], job["pipeline_version"]), + ) + _event(conn, job, "leased", None) + return True + + +def _variants(conn, attachment_id, version): + rows = conn.execute( + """ + SELECT name, object_key, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + """, + (attachment_id, version), + ).fetchall() + return {row["name"]: row for row in rows} + + +def _source(conn, attachment): + if attachment["staging_key"] and attachment["staging_deleted_at"] is None: + data = read_bytes(attachment["staging_key"]) + if data: + return data + data = read_bytes(attachment["object_key"]) + if data: + return data + row = conn.execute( + """ + SELECT object_key + FROM attachment_variants + WHERE attachment_id = %s AND name = 'original' + ORDER BY pipeline_version DESC + LIMIT 1 + """, + (attachment["id"],), + ).fetchone() + if row is None: + raise TransientJobError("processing_failed") + data = read_bytes(row["object_key"]) + if not data: + raise TransientJobError("processing_failed") + return data + + +def _render_limited(data, declared, seconds): + budget = max(5, seconds - 5) + box = [] + errors = [] + + def run(): + try: + box.append(render(data, declared)) + except Exception as exc: + errors.append(exc) + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(budget) + if thread.is_alive(): + raise TransientJobError("processing_timeout") + if errors: + exc = errors[0] + if isinstance(exc, (PermanentJobError, TransientJobError)): + raise exc + raise PermanentJobError("invalid_image") + return box[0] + + +def _beat(pool, job_id, owner, seconds, stop): + pause = max(2, seconds // 3) + while not stop.wait(pause): + try: + with pool.connection() as conn: + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE id = %s AND lease_owner = %s AND status = 'processing' + """, + (seconds, job_id, owner), + ) + except Exception: + return + + +def _supersede(conn, job): + _idle(conn) + with conn.transaction(): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'superseded', error_code = 'superseded', lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status IN ('queued', 'processing') + """, + (job["id"],), + ) + _event(conn, job, "superseded", None) + + +def _finalize(conn, job, owner, rendered): + _idle(conn) + with conn.transaction(): + current = conn.execute( + """ + SELECT lease_owner, status, pipeline_version + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (job["id"],), + ).fetchone() + if current is None or current["lease_owner"] != owner or current["status"] != "processing": + raise LostLease() + attachment = conn.execute( + """ + SELECT pipeline_version, status + FROM attachments + WHERE id = %s + FOR UPDATE + """, + (job["attachment_id"],), + ).fetchone() + if attachment is None or int(attachment["pipeline_version"]) > int(job["pipeline_version"]): + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'superseded', error_code = 'superseded', lease_owner = NULL, + lease_until = NULL, updated_at = now() + WHERE id = %s + """, + (job["id"],), + ) + _event(conn, job, "superseded", None) + return "superseded" + now = utc_now() + for name in REQUIRED: + item = rendered[name] + conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, object_key, created_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + ( + job["attachment_id"], + job["pipeline_version"], + name, + item["content_type"], + len(item["bytes"]), + item["width"], + item["height"], + item["sha256"], + item["object_key"], + now, + ), + ) + stored = conn.execute( + """ + SELECT count(*) AS n + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + AND name IN ('original', 'thumbnail', 'crop', 'optimized') + """, + (job["attachment_id"], job["pipeline_version"]), + ).fetchone() + if int(stored["n"]) < len(REQUIRED): + raise TransientJobError("processing_failed") + original = rendered["original"] + conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, pipeline_version = %s, + object_key = %s, updated_at = %s + WHERE id = %s AND pipeline_version <= %s + """, + ( + job["pipeline_version"], + original["object_key"], + now, + job["attachment_id"], + job["pipeline_version"], + ), + ) + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = %s + WHERE id = %s AND lease_owner = %s + """, + (now, job["id"], owner), + ) + _event(conn, job, "ready", None) + return "ready" + + +def _drop_staging(conn, attachment): + _idle(conn) + with conn.transaction(): + active = conn.execute( + """ + SELECT count(*) AS n + FROM attachment_jobs + WHERE attachment_id = %s AND status IN ('queued', 'processing') + """, + (attachment["id"],), + ).fetchone() + if int(active["n"]) > 0 or not attachment["staging_key"]: + return + if delete_quiet(attachment["staging_key"]): + with conn.transaction(): + conn.execute( + """ + UPDATE attachments + SET staging_deleted_at = now() + WHERE id = %s AND staging_deleted_at IS NULL + """, + (attachment["id"],), + ) + + +def process_delivery(pool, body, owner): + payload = parse_job(body) + if payload is None: + return "ack" + with pool.connection() as conn: + job = _load_job(conn, payload["job_id"]) + if job is None: + return "ack" + if job["status"] in ("ready", "failed", "superseded"): + return "ack" + if int(payload["attempt"]) != int(job["attempt"]) or int(payload["pipeline_version"]) != int(job["pipeline_version"]): + return "ack" + if job["next_attempt_at"] is not None and job["next_attempt_at"] > utc_now(): + delay = (job["next_attempt_at"] - utc_now()).total_seconds() + publish_delay(body, int(delay * 1000)) + return "ack" + with pool.connection() as conn: + attachment = _load_attachment(conn, job["attachment_id"]) + if attachment is None: + fail_job(conn, job, "invalid_image") + return "ack" + if int(attachment["pipeline_version"]) > int(job["pipeline_version"]): + _supersede(conn, job) + return "ack" + if not _acquire(conn, job, owner, lease_seconds()): + publish_delay(body, 2000) + return "ack" + stop = threading.Event() + beater = threading.Thread( + target=_beat, + args=(pool, job["id"], owner, lease_seconds(), stop), + daemon=True, + ) + beater.start() + try: + with pool.connection() as conn: + if take_fault(conn, "crash_before_effect", job["attachment_id"]): + os_exit() + if take_fault(conn, "transient_once", job["attachment_id"]): + raise TransientJobError("processing_failed") + present = _variants(conn, job["attachment_id"], job["pipeline_version"]) + source = None if set(REQUIRED) <= set(present) else _source(conn, attachment) + if source is None: + rendered = _rendered_from_rows(conn, job, present) + if source is not None: + if hashlib.sha256(source).hexdigest() != attachment["sha256"]: + raise PermanentJobError("invalid_image") + rendered = _render_limited(source, attachment["content_type"], lease_seconds()) + for name in REQUIRED: + item = rendered[name] + item["sha256"] = hashlib.sha256(item["bytes"]).hexdigest() + item["object_key"] = variant_object_key(job["attachment_id"], job["pipeline_version"], name) + if name in present: + continue + put_bytes(item["object_key"], item["bytes"], item["content_type"], utc_now()) + with pool.connection() as conn: + if take_fault(conn, "crash_after_objects", job["attachment_id"]): + os_exit() + with pool.connection() as conn: + _finalize(conn, job, owner, rendered) + with pool.connection() as conn: + if take_fault(conn, "crash_after_commit", job["attachment_id"]): + os_exit() + with pool.connection() as conn: + _drop_staging(conn, attachment) + return "ack" + except PermanentJobError as exc: + with pool.connection() as conn: + fail_job(conn, job, exc.code) + return "ack" + except LostLease: + return "ack" + except (TransientJobError, DependencyDown) as exc: + code = exc.code if isinstance(exc, TransientJobError) else "processing_failed" + with pool.connection() as conn: + schedule_retry(conn, job, code, owner) + return "ack" + finally: + stop.set() + + +def os_exit(): + os._exit(75) + + +def _rendered_from_rows(conn, job, present): + # Повтор после уже записанного эффекта не перекодирует файлы. + rendered = {} + for name in REQUIRED: + row = present[name] + data = read_bytes(row["object_key"]) + if data is None: + raise TransientJobError("processing_failed") + meta = conn.execute( + """ + SELECT content_type, width, height, sha256, object_key + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (job["attachment_id"], job["pipeline_version"], name), + ).fetchone() + rendered[name] = { + "bytes": data, + "content_type": meta["content_type"], + "width": int(meta["width"]), + "height": int(meta["height"]), + "sha256": meta["sha256"], + "object_key": meta["object_key"], + } + return rendered + + +def bump_pipeline(conn, attachment_id): + _idle(conn) + with conn.transaction(): + row = conn.execute( + "SELECT id, pipeline_version FROM attachments WHERE id = %s FOR UPDATE", + (attachment_id,), + ).fetchone() + if row is None: + return None + version = int(row["pipeline_version"]) + 1 + job_id = uuid.uuid4() + created = utc_now() + final_key = variant_object_key(attachment_id, version, "original") + conn.execute( + """ + UPDATE attachments + SET pipeline_version = %s, status = 'queued', error_code = NULL, + object_key = %s, updated_at = %s + WHERE id = %s + """, + (version, final_key, created, attachment_id), + ) + attempts = max_attempts() + conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at, updated_at + ) VALUES (%s, %s, %s, 'queued', 1, %s, %s, %s) + """, + (job_id, attachment_id, version, attempts, created, created), + ) + payload = job_message(job_id, attachment_id, version, 1, created) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job_id, Json(payload), created), + ) + conn.execute( + """ + INSERT INTO job_events (job_id, attachment_id, pipeline_version, kind, detail) + VALUES (%s, %s, %s, 'version', %s) + """, + (job_id, attachment_id, version, str(version)), + ) + return {"job_id": str(job_id), "pipeline_version": version} + + +def replay_job(conn, job_id): + _idle(conn) + with conn.transaction(): + job = conn.execute( + """ + SELECT id, attachment_id, pipeline_version, status, attempt, max_attempts, created_at + FROM attachment_jobs + WHERE id = %s + FOR UPDATE + """, + (job_id,), + ).fetchone() + if job is None: + return {"result": "missing"} + if job["status"] == "ready": + return {"result": "already_ready"} + names = conn.execute( + """ + SELECT name FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + """, + (job["attachment_id"], job["pipeline_version"]), + ).fetchall() + have = {row["name"] for row in names} + if set(REQUIRED) <= have: + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = now() + WHERE id = %s AND status <> 'ready' + """, + (job["id"],), + ) + return {"result": "already_done"} + if job["status"] != "failed": + return {"result": "not_failed"} + created = utc_now() + conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', attempt = 1, error_code = NULL, lease_owner = NULL, + lease_until = NULL, next_attempt_at = NULL, updated_at = %s + WHERE id = %s + """, + (created, job["id"]), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = %s + WHERE id = %s AND pipeline_version = %s AND status = 'failed' + """, + (created, job["attachment_id"], job["pipeline_version"]), + ) + payload = job_message(job["id"], job["attachment_id"], job["pipeline_version"], 1, created) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job["id"], Json(payload), created), + ) + _event(conn, job, "replay", None) + return {"result": "queued"} + + +def republish(conn, job_id, attempt=None): + job = conn.execute( + """ + SELECT id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE id = %s + """, + (job_id,), + ).fetchone() + if job is None: + return False + chosen = int(job["attempt"] if attempt is None else attempt) + body = _message_body(job, chosen, utc_now()) + with broker_channel() as channel: + publish_raw(channel, PROCESS_QUEUE, body) + return True diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..6295e0e0 --- /dev/null +++ b/src/main.py @@ -0,0 +1,475 @@ +import json +import logging +import os +import re +import uuid +from contextlib import asynccontextmanager + +import anyio +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from pydantic import ValidationError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from blobs import MAX_BYTES, declared_type, inspect_image, open_object +from errors import ApiError, AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict, error_body +from schemas import CreateChat, CreateMessage, CreateUser +from store import Store, instance_id + +LOGGER = logging.getLogger("messenger") +store = Store() +INSTANCE_ID = instance_id() + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +MAX_JSON_BYTES = 65536 + + +@asynccontextmanager +async def lifespan(_app): + await anyio.to_thread.run_sync(store.startup) + yield + store.shutting_down = True + await anyio.to_thread.run_sync(store.shutdown) + + +app = FastAPI( + title="mtusi-messenger", + version="lab5", + lifespan=lifespan, + redirect_slashes=False, + docs_url="/docs", + redoc_url=None, + openapi_url="/openapi.json", +) + + +def request_id_of(request: Request): + rid = getattr(request.state, "request_id", None) + if rid: + return rid + return str(uuid.uuid4()) + + +def stamp(request: Request, response): + response.headers["X-Request-Id"] = request_id_of(request) + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +def json_ok(request: Request, payload, status_code=200): + return stamp(request, JSONResponse(payload, status_code=status_code)) + + +def json_error(request: Request, status, code, message, extra_headers=None): + headers = {"X-Request-Id": request_id_of(request), "X-Instance-Id": INSTANCE_ID} + if extra_headers: + headers.update(extra_headers) + return JSONResponse(error_body(code, message, request_id_of(request)), status_code=status, headers=headers) + + +@app.middleware("http") +async def wrap_request(request: Request, call_next): + incoming = request.headers.get("X-Request-Id", "") + if incoming and len(incoming) <= 200 and incoming.isprintable() and incoming.strip(): + request.state.request_id = incoming.strip() + else: + request.state.request_id = str(uuid.uuid4()) + if store.shutting_down and request.url.path != "/health/live": + return json_error(request, 503, "unavailable", "Сервис временно недоступен") + response = await call_next(request) + return stamp(request, response) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def request_validation_handler(request: Request, _exc: RequestValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(ValidationError) +async def pydantic_handler(request: Request, _exc: ValidationError): + return json_error(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return json_error(request, 404, "not_found", "Объект не найден") + return json_error(request, exc.status_code if exc.status_code < 500 else 500, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(Exception) +async def unexpected_handler(request: Request, exc: Exception): + if isinstance(exc, ApiError): + return json_error(request, exc.status, exc.code, exc.message, exc.headers) + LOGGER.error("unhandled %s", type(exc).__name__) + return json_error(request, 500, "internal_error", "Внутренняя ошибка") + + +def unavailable(): + raise ApiError(503, "unavailable", "Сервис временно недоступен") + + +def bearer_unauthorized(): + raise ApiError(401, "unauthorized", "Требуется авторизация", {"WWW-Authenticate": BEARER_CHALLENGE}) + + +def basic_unauthorized(): + raise ApiError(401, "unauthorized", "Неверные учётные данные", {"WWW-Authenticate": BASIC_CHALLENGE}) + + +def invalid(): + raise ApiError(400, "invalid_request", "Некорректный запрос") + + +async def call_store(func, *args): + try: + return await anyio.to_thread.run_sync(func, *args) + except DependencyDown: + unavailable() + except IdempotencyConflict: + raise ApiError(409, "conflict", "Ключ уже использован с другим телом") + except AttachmentRejected: + raise ApiError(409, "conflict", "Вложение нельзя прикрепить") + except AttachmentNotReady: + raise ApiError(409, "conflict", "Вложение недоступно") + + +def parse_basic(request: Request): + header = request.headers.get("Authorization", "") + if not header.lower().startswith("basic "): + basic_unauthorized() + raw = header.split(" ", 1)[1].strip() + try: + import base64 + + decoded = base64.b64decode(raw, validate=True).decode("utf-8") + except (ValueError, UnicodeError): + basic_unauthorized() + if ":" not in decoded: + basic_unauthorized() + username, password = decoded.split(":", 1) + return username, password + + +def parse_bearer(request: Request): + header = request.headers.get("Authorization", "") + # Basic на бизнес-методе не открывает сессию. X-User-Id не смотрим. + if not header.lower().startswith("bearer "): + bearer_unauthorized() + token = header.split(" ", 1)[1].strip() + if len(token) < 32 or len(token) > 512 or any(ch.isspace() for ch in token): + bearer_unauthorized() + return token + + +async def current_user(request: Request): + token = parse_bearer(request) + user = await call_store(store.user_for_token, token) + if user is None: + bearer_unauthorized() + request.state.user_id = user["id"] + request.state.token = token + return user + + +def parse_chat_id(chat_id): + try: + return str(uuid.UUID(chat_id)) + except ValueError: + invalid() + + +def parse_limit(raw): + if raw is None: + return 50 + if not isinstance(raw, str) or len(raw) > 3 or not raw.isdigit(): + invalid() + value = int(raw) + if value < 1 or value > 100: + invalid() + return value + + +async def read_json(request: Request): + ctype = request.headers.get("content-type", "") + if "application/json" not in ctype.lower(): + raise ApiError(415, "unsupported_media_type", "Нужен application/json") + length = request.headers.get("content-length") + if length is not None and (not length.isdigit() or int(length) > MAX_JSON_BYTES): + raise ApiError(413, "payload_too_large", "Слишком большое тело") + raw = await request.body() + if len(raw) > MAX_JSON_BYTES: + raise ApiError(413, "payload_too_large", "Слишком большое тело") + try: + payload = json.loads(raw) + except (ValueError, UnicodeError): + invalid() + if not isinstance(payload, dict): + invalid() + return payload + + +def parse_idempotency_key(request: Request): + key = request.headers.get("Idempotency-Key") + if key is None or not IDEMPOTENCY_RE.fullmatch(key): + invalid() + return key + + +@app.get("/health/live") +async def liveness(request: Request): + return json_ok(request, {"status": "ok"}) + + +@app.get("/health/ready") +async def readiness(request: Request): + if store.shutting_down: + raise ApiError(503, "unavailable", "Сервис временно недоступен") + ok = await call_store(store.ready) + if not ok: + unavailable() + return json_ok(request, {"status": "ok"}) + + +@app.post("/api/v1/users", status_code=201) +async def register_user(request: Request, body: CreateUser): + user = await call_store(store.create_user, body.username, body.display_name, body.password) + if user is None: + raise ApiError(409, "conflict", "Имя пользователя занято") + return json_ok(request, user, 201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + username, password = parse_basic(request) + session = await call_store(store.login, username, password) + if session is None: + basic_unauthorized() + request.state.user_id = session["user"]["id"] + return json_ok(request, session, 201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session(request: Request): + user = await current_user(request) + request.state.user_id = user["id"] + removed = await call_store(store.revoke, request.state.token) + if not removed: + bearer_unauthorized() + return stamp(request, Response(status_code=204)) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + user = await current_user(request) + return json_ok(request, user) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await current_user(request) + payload = await read_json(request) + try: + body = CreateChat.model_validate(payload) + except ValidationError: + invalid() + try: + chat = await call_store(store.create_chat, body.title, body.member_ids, user["id"]) + except ValueError: + invalid() + return json_ok(request, chat, 201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await current_user(request) + limit = parse_limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + if cursor is not None and (not cursor or len(cursor) > 512): + invalid() + try: + items, next_cursor = await call_store(store.list_chats, user["id"], limit, cursor) + except ValueError: + invalid() + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(request: Request, chat_id: str): + user = await current_user(request) + chat = await call_store(store.get_chat, parse_chat_id(chat_id), user["id"]) + if chat is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + payload = await read_json(request) + key = parse_idempotency_key(request) + try: + body = CreateMessage.model_validate(payload) + except ValidationError: + invalid() + try: + message = await call_store( + store.add_message, canonical, user["id"], body.text, key, body.attachment_ids + ) + except ValueError: + invalid() + if message is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, message, 201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + limit = parse_limit(request.query_params.get("limit")) + cursor = request.query_params.get("cursor") + if cursor is not None and (not cursor or len(cursor) > 512): + invalid() + try: + result = await call_store(store.list_messages, canonical, user["id"], limit, cursor) + except ValueError: + invalid() + if result is None: + raise ApiError(404, "not_found", "Объект не найден") + items, next_cursor = result + return json_ok(request, {"items": items, "next_cursor": next_cursor}) + + +def check_filename(value): + if not isinstance(value, str) or not value or len(value) > 255: + invalid() + if "/" in value or "\\" in value or "\x00" in value: + invalid() + if any(ord(char) < 32 or ord(char) == 127 for char in value): + invalid() + return value + + +def parse_attachment_id(value): + try: + return str(uuid.UUID(value)) + except (TypeError, ValueError): + invalid() + + +async def read_upload(request: Request): + header = request.headers.get("content-length") + if header is not None: + try: + announced = int(header) + except ValueError: + invalid() + if announced > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ") + if announced < 1: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + chunks = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > MAX_BYTES: + raise ApiError(413, "payload_too_large", "Файл больше 10 МиБ") + chunks.append(chunk) + if total < 1: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + return b"".join(chunks) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(request: Request, chat_id: str): + user = await current_user(request) + canonical = parse_chat_id(chat_id) + member = await call_store(store.is_member, canonical, user["id"]) + if not member: + raise ApiError(404, "not_found", "Объект не найден") + filename = check_filename(request.query_params.get("filename")) + media = declared_type(request.headers.get("content-type")) + if media is None: + raise ApiError(415, "unsupported_media_type", "Нужен PNG или JPEG") + data = await read_upload(request) + content_type = await anyio.to_thread.run_sync(inspect_image, data, media) + saved = await call_store( + store.save_attachment, canonical, user["id"], filename, content_type, data + ) + if saved is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, saved, 202) + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(request: Request, attachment_id: str): + user = await current_user(request) + canonical = parse_attachment_id(attachment_id) + found = await call_store(store.get_attachment, canonical, user["id"]) + if found is None: + raise ApiError(404, "not_found", "Объект не найден") + return json_ok(request, found) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(request: Request, attachment_id: str): + user = await current_user(request) + canonical = parse_attachment_id(attachment_id) + variant = request.query_params.get("variant") + if variant is None: + variant = "original" + if variant not in ("original", "thumbnail", "crop", "optimized"): + invalid() + opened = await call_store(store.open_attachment, canonical, user["id"], variant) + if opened is None: + raise ApiError(404, "not_found", "Объект не найден") + + def chunks(): + body = open_object(opened["object_key"]) + if body is None: + return + try: + for piece in body.stream(64 * 1024): + yield piece + finally: + body.close() + body.release_conn() + + probe = await anyio.to_thread.run_sync(open_object, opened["object_key"]) + if probe is None: + raise ApiError(404, "not_found", "Объект не найден") + probe.close() + probe.release_conn() + response = StreamingResponse(chunks(), media_type=opened["content_type"]) + response.headers["Content-Length"] = str(opened["size_bytes"]) + return stamp(request, response) + + +def setup_logging(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + + +if __name__ == "__main__": + import uvicorn + + setup_logging() + host = os.environ.get("APP_HOST", "0.0.0.0") + port = int(os.environ.get("APP_PORT", "8080")) + uvicorn.run( + app, + host=host, + port=port, + timeout_graceful_shutdown=5, + timeout_keep_alive=2, + access_log=False, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..cc35b87c --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,112 @@ +"""Однократные миграции. Запускается отдельным контейнером, не каждой репликой API.""" +import logging +import os +import sys +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +LOGGER = logging.getLogger("migrate") +MIGRATION_LOCK = 421003 +CONNECT_TIMEOUT = 3 + + +def read_secret_file(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def database_url(): + path = os.environ.get("DATABASE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise RuntimeError("DATABASE_URL_FILE") + return url + + +def split_sql(script): + parts = [] + buf = [] + for line in script.splitlines(): + if line.strip().startswith("--"): + continue + buf.append(line) + if line.strip().endswith(";"): + statement = "\n".join(buf).strip() + if statement.endswith(";"): + statement = statement[:-1].strip() + if statement: + parts.append(statement) + buf = [] + tail = "\n".join(buf).strip().rstrip(";").strip() + if tail: + parts.append(tail) + return parts + + +def wait_postgres(conninfo): + last = None + for _ in range(60): + try: + with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT) as conn: + conn.execute("SELECT 1") + return + except psycopg.OperationalError as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("postgres not ready: %s", last) + raise SystemExit(1) + + +def apply(conninfo): + directory = Path(__file__).resolve().parents[1] / "migrations" + with psycopg.connect(conninfo, connect_timeout=CONNECT_TIMEOUT, row_factory=dict_row) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK,)) + conn.commit() + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + conn.commit() + for path in sorted(directory.glob("*.sql")): + version = path.stem + found = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (version,), + ).fetchone() + conn.rollback() + if found: + LOGGER.info("migration %s already applied", version) + continue + with conn.transaction(): + for statement in split_sql(path.read_text(encoding="utf-8")): + conn.execute(statement) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + LOGGER.info("migration %s applied", version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK,)) + conn.commit() + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + conninfo = database_url() + wait_postgres(conninfo) + apply(conninfo) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/passwords.py b/src/passwords.py new file mode 100644 index 00000000..67c8daa2 --- /dev/null +++ b/src/passwords.py @@ -0,0 +1,26 @@ +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHash, VerificationError, VerifyMismatchError + +# Параметры ближе к учебному стенду, чем к максимальным: time=2, 19 MiB, Argon2id. +# Хеш всё равно медленный password KDF, не SHA-256 и не открытый текст. +hasher = PasswordHasher( + time_cost=2, + memory_cost=19456, + parallelism=1, + hash_len=32, + salt_len=16, +) + +# Чтобы неизвестный username не отвечал заметно быстрее известного. +DUMMY_HASH = hasher.hash("dummy-password-not-a-user") + + +def hash_password(password): + return hasher.hash(password) + + +def verify_password(password, encoded): + try: + return hasher.verify(encoded, password) + except (VerifyMismatchError, VerificationError, InvalidHash): + return False diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..acb822d0 --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,174 @@ +"""Сверка строк attachments и объектов bucket. + +Свежий объект моложе grace не удаляется: загрузка могла записать файл +раньше, чем строка в базе. Строка, на которую ссылается сообщение, тоже +не удаляется, даже если файла уже нет. По умолчанию только план, без удалений. +""" +import argparse +import json +import os +import sys +import time +from datetime import datetime, timezone + +import psycopg +from psycopg.rows import dict_row + +from blobs import BUCKET, delete_quiet, list_objects + + +def grace_seconds(): + raw = os.environ.get("RECONCILE_GRACE_SECONDS", "120") + value = int(raw) + if value < 1: + raise RuntimeError("RECONCILE_GRACE_SECONDS") + return value + + +def database_url(): + path = os.environ.get("DATABASE_URL_FILE") + if not path: + raise RuntimeError("DATABASE_URL_FILE") + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def aware(value): + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def plan(grace): + started = time.perf_counter() + objects = list_objects() + listed = time.perf_counter() + now = datetime.now(timezone.utc) + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + rows = conn.execute( + """ + SELECT a.id, a.object_key, a.staging_key, a.created_at, a.status, + EXISTS ( + SELECT 1 FROM message_attachments m + WHERE m.attachment_id = a.id + ) AS in_use + FROM attachments a + """ + ).fetchall() + variant_rows = conn.execute("SELECT object_key FROM attachment_variants").fetchall() + by_key = {} + for row in rows: + by_key[row["object_key"]] = row + if row["staging_key"]: + by_key[row["staging_key"]] = row + for item in variant_rows: + by_key.setdefault(item["object_key"], {"status": "variant"}) + object_keys = {item["key"] for item in objects} + actions = [] + preserved = [] + for item in objects: + row = by_key.get(item["key"]) + if row is not None: + continue + created = aware(item["created_at"]) or now + age = (now - created).total_seconds() + entry = {"key": item["key"], "age_seconds": round(age, 3)} + if age < grace: + entry["reason"] = "fresh_object" + preserved.append(entry) + continue + entry["action"] = "delete_orphan_object" + actions.append(entry) + for row in rows: + if row["object_key"] in object_keys: + continue + created = aware(row["created_at"]) or now + age = (now - created).total_seconds() + entry = { + "attachment_id": str(row["id"]), + "key": row["object_key"], + "age_seconds": round(age, 3), + } + if row["status"] != "ready": + entry["reason"] = "not_ready" + preserved.append(entry) + continue + if row["in_use"]: + entry["reason"] = "missing_but_in_use" + preserved.append(entry) + continue + if age < grace: + entry["reason"] = "fresh_row" + preserved.append(entry) + continue + entry["action"] = "delete_orphan_row" + actions.append(entry) + return { + "bucket": BUCKET, + "grace_seconds": grace, + "objects": len(objects), + "rows": len(rows), + "list_seconds": round(listed - started, 6), + "plan_seconds": round(time.perf_counter() - started, 6), + "actions": actions, + "preserved": preserved, + } + + +def apply_plan(report): + deleted_objects = 0 + deleted_rows = 0 + with psycopg.connect(database_url(), row_factory=dict_row) as conn: + for action in report["actions"]: + if action["action"] == "delete_orphan_object": + if delete_quiet(action["key"]): + deleted_objects += 1 + elif action["action"] == "delete_orphan_row": + with conn.transaction(): + result = conn.execute( + """ + DELETE FROM attachments + WHERE id = %s + AND status = 'ready' + AND NOT EXISTS ( + SELECT 1 FROM message_attachments m + WHERE m.attachment_id = attachments.id + ) + AND NOT EXISTS ( + SELECT 1 FROM attachment_jobs j + WHERE j.attachment_id = attachments.id + ) + AND NOT EXISTS ( + SELECT 1 FROM attachment_variants v + WHERE v.attachment_id = attachments.id + ) + """, + (action["attachment_id"],), + ) + deleted_rows += result.rowcount + report["deleted_objects"] = deleted_objects + report["deleted_rows"] = deleted_rows + return report + + +def main(): + parser = argparse.ArgumentParser(description="Сверить вложения в базе и в bucket") + parser.add_argument("--apply", action="store_true", help="выполнить удаления; без флага только план") + parser.add_argument("--grace", type=int, default=None) + args = parser.parse_args() + grace = args.grace if args.grace is not None else grace_seconds() + if grace < 1: + print("grace должен быть не меньше 1 секунды", file=sys.stderr) + return 2 + report = plan(grace) + report["apply"] = bool(args.apply) + if args.apply: + apply_plan(report) + print(json.dumps(report, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 00000000..009f577b --- /dev/null +++ b/src/replay.py @@ -0,0 +1,24 @@ +"""Явный повтор failed-задания. Уже готовый эффект второй раз не исполняется.""" +import json +import sys + +import psycopg +from psycopg.rows import dict_row + +from blobs import read_secret +from jobs import replay_job + + +def main(): + if len(sys.argv) != 2: + print("usage: replay.py JOB_ID", file=sys.stderr) + return 2 + url = read_secret(__import__("os").environ["DATABASE_URL_FILE"]) + with psycopg.connect(url, row_factory=dict_row) as conn: + result = replay_job(conn, sys.argv[1]) + print(json.dumps(result, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/src/schemas.py b/src/schemas.py new file mode 100644 index 00000000..e11752d8 --- /dev/null +++ b/src/schemas.py @@ -0,0 +1,128 @@ +import re +import uuid + +from pydantic import BaseModel, ConfigDict, field_validator, model_validator + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +def _visible(value, limit): + if not isinstance(value, str) or not value.strip() or len(value) > limit: + raise ValueError("text") + return value + + +class CreateUser(StrictModel): + username: str + display_name: str + password: str + + @field_validator("username") + @classmethod + def username_ok(cls, value): + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + raise ValueError("username") + return value + + @field_validator("display_name") + @classmethod + def display_name_ok(cls, value): + return _visible(value, 100) + + @field_validator("password") + @classmethod + def password_ok(cls, value): + if not isinstance(value, str) or not value.strip(): + raise ValueError("password") + # Длина в code points, не в байтах: bcrypt тут не подходит из-за усечения UTF-8. + if len(value) < 12 or len(value) > 128: + raise ValueError("password") + return value + + +class CreateChat(StrictModel): + title: str + member_ids: list + + @field_validator("title") + @classmethod + def title_ok(cls, value): + return _visible(value, 100) + + @field_validator("member_ids") + @classmethod + def member_ids_ok(cls, value): + if not isinstance(value, list) or not (1 <= len(value) <= 100): + raise ValueError("member_ids") + # Канонический UUID (нижний регистр). Иначе один и тот же id + # в разном регистре выглядит как два участника и упирается в PK. + normalized = [] + seen = set() + for item in value: + if not isinstance(item, str): + raise ValueError("member_ids") + try: + canonical = str(uuid.UUID(item)) + except ValueError as exc: + raise ValueError("member_ids") from exc + if canonical in seen: + raise ValueError("member_ids") + seen.add(canonical) + normalized.append(canonical) + return normalized + + +class CreateMessage(StrictModel): + text: str + attachment_ids: list = [] + + @model_validator(mode="before") + @classmethod + def fill_attachments(cls, value): + # Нет поля и явный [] — одно и то же тело для Idempotency-Key. + if not isinstance(value, dict): + return value + data = dict(value) + data.setdefault("attachment_ids", []) + return data + + @field_validator("text") + @classmethod + def text_ok(cls, value): + if not isinstance(value, str) or len(value) > 4000: + raise ValueError("text") + if value != "" and not value.strip(): + raise ValueError("text") + return value + + @field_validator("attachment_ids") + @classmethod + def attachments_ok(cls, value): + if not isinstance(value, list) or len(value) > 10: + raise ValueError("attachment_ids") + normalized = [] + seen = set() + for item in value: + if not isinstance(item, str): + raise ValueError("attachment_ids") + try: + canonical = str(uuid.UUID(item)) + except ValueError as exc: + raise ValueError("attachment_ids") from exc + if canonical in seen: + raise ValueError("attachment_ids") + seen.add(canonical) + normalized.append(canonical) + return normalized + + @model_validator(mode="after") + def text_or_file(self): + if self.text == "" and not self.attachment_ids: + raise ValueError("text") + if self.text and not self.text.strip(): + raise ValueError("text") + return self diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..baed1c03 --- /dev/null +++ b/src/store.py @@ -0,0 +1,868 @@ +import base64 +import hashlib +import json +import logging +import os +import secrets +import socket +import time +import uuid +from datetime import datetime, timedelta, timezone + +import psycopg +import redis +from psycopg.errors import CheckViolation, ForeignKeyViolation, UniqueViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from errors import AttachmentNotReady, AttachmentRejected, DependencyDown, IdempotencyConflict +from passwords import DUMMY_HASH, hash_password, verify_password + +LOGGER = logging.getLogger("messenger") + +POOL_TIMEOUT_SECONDS = 3 +STATEMENT_TIMEOUT_MS = 5000 +LOCK_TIMEOUT_MS = 3000 +REDIS_MAX_CONNECTIONS = 16 +REDIS_SOCKET_TIMEOUT = 2 +SCHEMA_VERSION = "004_jobs" + +SQL_MESSAGES_FIRST = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s +ORDER BY created_at, id +LIMIT %s +""" + +SQL_MESSAGES_AFTER = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE chat_id = %s + AND (created_at, id) > (%s, %s) +ORDER BY created_at, id +LIMIT %s +""" + +SQL_MESSAGE_BY_ID = """ +SELECT id, chat_id, sender_id, body AS text, created_at +FROM messages +WHERE id = %s +""" + + +def instance_id(): + # Docker без hostname: в контейнере это id контейнера, он не меняется, + # пока живёт этот контейнер. stop-one.sh находит контейнер по этому значению. + return socket.gethostname() + + +def pool_max(): + raw = os.environ.get("POOL_MAX", "10") + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError("POOL_MAX") from exc + if not 1 <= value <= 32: + raise RuntimeError("POOL_MAX") + return value + + +def utc_now(): + return datetime.now(timezone.utc) + + +def dump_time(value): + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + text = value.astimezone(timezone.utc).isoformat(timespec="microseconds") + if text.endswith("+00:00"): + text = text[:-6] + "Z" + return text + + +def parse_time(value): + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("time") + return datetime.fromisoformat(value[:-1] + "+00:00") + + +def encode_cursor(scope, created_at, item_id): + raw = "{}|{}|{}".format(scope, created_at, item_id).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(cursor, expected_scope): + if not isinstance(cursor, str) or not cursor or len(cursor) > 512: + raise ValueError("cursor") + pad = "=" * (-len(cursor) % 4) + try: + decoded = base64.urlsafe_b64decode(cursor + pad).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise ValueError("cursor") from exc + parts = decoded.split("|") + if len(parts) != 3: + raise ValueError("cursor") + scope, created_at, item_id = parts + if scope != expected_scope: + raise ValueError("cursor") + parse_time(created_at) + try: + uuid.UUID(item_id) + except ValueError as exc: + raise ValueError("cursor") from exc + return created_at, item_id + + +def session_key(token): + digest = hashlib.sha256(token.encode("utf-8")).hexdigest() + return "sess:" + digest + + +def message_fingerprint(text, attachment_ids): + # Канонический JSON: порядок ключей не важен, пропущенный attachment_ids равен []. + canonical = json.dumps( + {"attachment_ids": list(attachment_ids), "text": text}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def advisory_key(user_id, path, idem_key): + raw = "\0".join((user_id, "POST", path, idem_key)).encode("utf-8") + digest = hashlib.sha256(raw).digest() + return int.from_bytes(digest[:8], "big", signed=True) + + +def chat_path(chat_id): + return "/api/v1/chats/" + chat_id + "/messages" + + +def public_user(row): + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": dump_time(row["created_at"]), + } + + +def public_message(row, attachment_ids=None): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "attachment_ids": list(attachment_ids or []), + "created_at": dump_time(row["created_at"]), + } + + +def public_variant(row): + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "width": int(row["width"]), + "height": int(row["height"]), + "sha256": row["sha256"], + } + + +def public_attachment(row, variants=None): + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "sha256": row["sha256"], + "status": row["status"], + "variants": [public_variant(item) for item in (variants or [])], + "error_code": row["error_code"], + "created_at": dump_time(row["created_at"]), + "updated_at": dump_time(row["updated_at"]), + } + + +def public_chat(row, member_ids=None): + members = member_ids if member_ids is not None else row["member_ids"] + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(item) for item in members], + "created_at": dump_time(row["created_at"]), + } + + +def read_secret_file(path): + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + + +def _db_down(exc): + raise DependencyDown() from exc + + +class Store: + def __init__(self): + self.pool = None + self.redis = None + self.ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + self.idem_ttl = int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")) + self.pool_size = pool_max() + self.shutting_down = False + if self.ttl < 1 or self.idem_ttl < 1: + raise RuntimeError("TTL") + kind = os.environ.get("SESSION_STORE", "redis").strip().lower() + if kind != "redis": + raise RuntimeError("SESSION_STORE") + + def database_url(self): + path = os.environ.get("DATABASE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise RuntimeError("DATABASE_URL_FILE") + return url + + def redis_url(self): + path = os.environ.get("CACHE_URL_FILE") + if path: + return read_secret_file(path) + url = os.environ.get("CACHE_URL", "").strip() + if not url: + raise RuntimeError("CACHE_URL_FILE") + return url + + def startup(self): + conninfo = self.database_url() + self._wait_schema(conninfo) + self.pool = ConnectionPool( + conninfo, + min_size=1, + max_size=self.pool_size, + timeout=POOL_TIMEOUT_SECONDS, + max_waiting=64, + max_lifetime=300, + kwargs={ + "connect_timeout": POOL_TIMEOUT_SECONDS, + "row_factory": dict_row, + "application_name": "mtusi-lab5-" + instance_id(), + "options": "-c statement_timeout={} -c lock_timeout={} -c idle_in_transaction_session_timeout=15000".format( + STATEMENT_TIMEOUT_MS, + LOCK_TIMEOUT_MS, + ), + }, + open=True, + ) + pool = redis.ConnectionPool.from_url( + self.redis_url(), + max_connections=REDIS_MAX_CONNECTIONS, + socket_timeout=REDIS_SOCKET_TIMEOUT, + socket_connect_timeout=REDIS_SOCKET_TIMEOUT, + health_check_interval=30, + decode_responses=True, + ) + self.redis = redis.Redis(connection_pool=pool) + self._wait_redis() + self._wait_bucket() + try: + from broker import broker_channel, publish_pending + + with broker_channel(): + pass + with self._connect() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("outbox startup deferred %s", type(exc).__name__) + LOGGER.info("ready instance=%s ttl=%s pool_max=%s", instance_id(), self.ttl, self.pool_size) + + def shutdown(self): + self.shutting_down = True + if self.pool is not None: + self.pool.close() + if self.redis is not None: + self.redis.close() + + def ready(self): + if self.shutting_down or self.pool is None or self.redis is None: + return False + try: + with self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is None: + return False + self.redis.ping() + from blobs import bucket_ready + + if not bucket_ready(): + return False + except Exception: + LOGGER.warning("ready check failed") + return False + return True + + def _wait_redis(self): + last = None + for _ in range(60): + try: + self.redis.ping() + return + except redis.RedisError as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("redis not ready: %s", last) + raise SystemExit(1) + + def _wait_schema(self, conninfo): + # Схему накатывает сервис migrate. Реплики только ждут её, не применяют сами. + last = None + for _ in range(90): + try: + with psycopg.connect(conninfo, connect_timeout=POOL_TIMEOUT_SECONDS, row_factory=dict_row) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is not None: + return + last = "migration missing" + except psycopg.Error as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("schema not ready: %s", last) + raise SystemExit(1) + + def _wait_bucket(self): + from blobs import bucket_ready + + for _ in range(60): + if bucket_ready(): + return + time.sleep(1) + LOGGER.error("object storage not ready") + raise SystemExit(1) + + def _connect(self): + try: + return self.pool.connection(timeout=POOL_TIMEOUT_SECONDS) + except PoolTimeout as exc: + _db_down(exc) + + def create_user(self, username, display_name, password): + password_hash = hash_password(password) + user_id = uuid.uuid4() + created = utc_now() + try: + with self._connect() as conn: + with conn.transaction(): + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (username) DO NOTHING + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash, created), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def login(self, username, password): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + verify_password(password, DUMMY_HASH) + return None + if not verify_password(password, row["password_hash"]): + return None + return self._issue(public_user(row)) + + def _issue(self, user): + token = secrets.token_urlsafe(32) + expires = utc_now() + timedelta(seconds=self.ttl) + payload = json.dumps( + {"user_id": user["id"], "expires_at": dump_time(expires)}, + separators=(",", ":"), + ) + # pxat — абсолютное время. Рестарт Redis не сдвигает срок. + # В значении тоже лежит expires_at: даже если TTL ключа разъедется, сессия кончится вовремя. + try: + self.redis.set(session_key(token), payload, pxat=int(expires.timestamp() * 1000)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return { + "token": token, + "token_type": "Bearer", + "expires_at": dump_time(expires), + "user": user, + } + + def user_for_token(self, token): + key = session_key(token) + try: + raw = self.redis.get(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + if not raw: + return None + try: + payload = json.loads(raw) + expires = parse_time(payload["expires_at"]) + user_id = payload["user_id"] + uuid.UUID(user_id) + except (ValueError, KeyError, TypeError): + return None + if utc_now() >= expires: + try: + self.redis.delete(key) + except redis.RedisError as exc: + raise DependencyDown() from exc + return None + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (user_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_user(row) + + def revoke(self, token): + try: + deleted = self.redis.delete(session_key(token)) + except redis.RedisError as exc: + raise DependencyDown() from exc + return deleted == 1 + + def create_chat(self, title, member_ids, caller_id): + try: + caller_uuid = uuid.UUID(str(caller_id)) + member_uuids = [] + seen = set() + for item in member_ids: + member_uuid = uuid.UUID(str(item)) + if member_uuid in seen: + raise ValueError("members") + seen.add(member_uuid) + member_uuids.append(member_uuid) + except ValueError as exc: + raise ValueError("members") from exc + if caller_uuid not in seen: + raise ValueError("members") + chat_id = uuid.uuid4() + created = utc_now() + canonical_members = [str(item) for item in member_uuids] + try: + with self._connect() as conn: + try: + with conn.transaction(): + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created), + ) + for ordinal, member_id in enumerate(member_uuids): + conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, ordinal) + VALUES (%s, %s, %s) + """, + (chat_id, member_id, ordinal), + ) + row = conn.execute( + "SELECT id, title, created_at FROM chats WHERE id = %s", + (chat_id,), + ).fetchone() + except ForeignKeyViolation as exc: + raise ValueError("members") from exc + except UniqueViolation as exc: + raise ValueError("members") from exc + except psycopg.OperationalError as exc: + _db_down(exc) + return public_chat(row, canonical_members) + + def _member(self, conn, chat_id, user_id): + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + + def get_chat(self, chat_id, user_id): + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE c.id = %s + GROUP BY c.id + """, + (chat_id,), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_chat(row) + + def list_chats(self, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, "chats:" + user_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + sql = """ + SELECT c.id, c.title, c.created_at, + array_agg(m.user_id ORDER BY m.ordinal) AS member_ids + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = %s + JOIN chat_members m ON m.chat_id = c.id + """ + params = [user_id] + if after is not None: + sql += " WHERE (c.created_at, c.id) > (%s, %s)" + params.extend(after) + sql += " GROUP BY c.id ORDER BY c.created_at, c.id LIMIT %s" + params.append(limit + 1) + try: + with self._connect() as conn: + rows = conn.execute(sql, params).fetchall() + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_chat(row) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor("chats:" + user_id, last["created_at"], last["id"]) + return page, None + + def _attachment_ids(self, conn, message_ids): + if not message_ids: + return {} + rows = conn.execute( + """ + SELECT message_id, attachment_id + FROM message_attachments + WHERE message_id = ANY(%s::uuid[]) + ORDER BY ordinal + """, + (list(message_ids),), + ).fetchall() + found = {} + for row in rows: + found.setdefault(str(row["message_id"]), []).append(str(row["attachment_id"])) + return found + + def is_member(self, chat_id, user_id): + try: + with self._connect() as conn: + return self._member(conn, chat_id, user_id) is not None + except psycopg.OperationalError as exc: + _db_down(exc) + + def _variants(self, conn, attachment_id, pipeline_version): + return conn.execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name <> 'original' + ORDER BY CASE name + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END + """, + (attachment_id, pipeline_version), + ).fetchall() + + def save_attachment(self, chat_id, owner_id, filename, content_type, data): + from psycopg.types.json import Json + + from blobs import delete_quiet, put_bytes, staging_object_key, variant_object_key + from broker import publish_pending + from faults import take_fault + from jobs import job_message, max_attempts + + attachment_id = uuid.uuid4() + job_id = uuid.uuid4() + created = utc_now() + digest = hashlib.sha256(data).hexdigest() + staging = staging_object_key(attachment_id) + final_key = variant_object_key(attachment_id, 1, "original") + payload = job_message(job_id, attachment_id, 1, 1, created) + attempts = max_attempts() + put_bytes(staging, data, content_type, created) + try: + with self._connect() as conn: + with conn.transaction(): + if self._member(conn, chat_id, owner_id) is None: + raise LookupError("chat") + row = conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key, staging_key, status, + pipeline_version, error_code, created_at, updated_at + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'queued', 1, NULL, %s, %s) + RETURNING id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, status, error_code, created_at, updated_at + """, + ( + attachment_id, + chat_id, + owner_id, + filename, + content_type, + len(data), + digest, + final_key, + staging, + created, + created, + ), + ).fetchone() + conn.execute( + """ + INSERT INTO attachment_jobs ( + id, attachment_id, pipeline_version, status, attempt, + max_attempts, created_at, updated_at + ) + VALUES (%s, %s, 1, 'queued', 1, %s, %s, %s) + """, + (job_id, attachment_id, attempts, created, created), + ) + conn.execute( + """ + INSERT INTO outbox (id, job_id, attempt, payload, created_at) + VALUES (%s, %s, 1, %s, %s) + """, + (uuid.uuid4(), job_id, Json(payload), created), + ) + except (LookupError, ForeignKeyViolation): + delete_quiet(staging) + return None + except psycopg.OperationalError as exc: + delete_quiet(staging) + _db_down(exc) + except Exception: + delete_quiet(staging) + raise + try: + with self._connect() as conn: + skipped = take_fault(conn, "skip_inline_publish", attachment_id) + if not skipped: + with self._connect() as conn: + publish_pending(conn, only_job_id=job_id) + except Exception as exc: + LOGGER.warning("publish deferred %s", type(exc).__name__) + return public_attachment(row, []) + + def get_attachment(self, attachment_id, user_id): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.status, a.error_code, a.pipeline_version, + a.created_at, a.updated_at + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + variants = [] + if row is not None and row["status"] == "ready": + variants = self._variants(conn, row["id"], row["pipeline_version"]) + except psycopg.OperationalError as exc: + _db_down(exc) + if row is None: + return None + return public_attachment(row, variants) + + def open_attachment(self, attachment_id, user_id, variant): + try: + with self._connect() as conn: + row = conn.execute( + """ + SELECT a.object_key, a.content_type, a.size_bytes, a.status, a.pipeline_version + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + if row is None: + return None + if row["status"] != "ready": + raise AttachmentNotReady() + if variant == "original": + return { + "object_key": row["object_key"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + } + found = conn.execute( + """ + SELECT object_key, content_type, size_bytes + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (attachment_id, row["pipeline_version"], variant), + ).fetchone() + except psycopg.OperationalError as exc: + _db_down(exc) + if found is None: + return None + return { + "object_key": found["object_key"], + "content_type": found["content_type"], + "size_bytes": int(found["size_bytes"]), + } + + def add_message(self, chat_id, sender_id, text, idem_key, attachment_ids): + # Один ключ сериализуется между репликами через advisory lock в той же + # транзакции, где INSERT сообщения. Откат не оставляет ключ занятым. + path = chat_path(chat_id) + digest = message_fingerprint(text, attachment_ids) + lock_key = advisory_key(sender_id, path, idem_key) + message_id = uuid.uuid4() + try: + with self._connect() as conn: + with conn.transaction(): + # Иначе lock_timeout сессии (3 с) оборвёт очередь одинаковых ключей. + conn.execute("SET LOCAL lock_timeout = '15s'") + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + if self._member(conn, chat_id, sender_id) is None: + return None + existing = conn.execute( + """ + SELECT request_hash, message_id, + created_at >= now() - make_interval(secs => %s) AS fresh + FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' + AND request_path = %s AND idem_key = %s + """, + (self.idem_ttl, sender_id, path, idem_key), + ).fetchone() + if existing is not None and existing["fresh"]: + if existing["request_hash"] != digest: + raise IdempotencyConflict() + row = conn.execute(SQL_MESSAGE_BY_ID, (existing["message_id"],)).fetchone() + if row is None: + raise RuntimeError("idempotency target missing") + linked = self._attachment_ids(conn, [row["id"]]).get(str(row["id"]), []) + return public_message(row, linked) + if existing is not None: + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' + AND request_path = %s AND idem_key = %s + """, + (sender_id, path, idem_key), + ) + if attachment_ids: + found = conn.execute( + """ + SELECT id, chat_id, status + FROM attachments + WHERE id = ANY(%s::uuid[]) + """, + (list(attachment_ids),), + ).fetchall() + by_id = {str(item["id"]): item for item in found} + for attachment_id in attachment_ids: + item = by_id.get(attachment_id) + if item is None or str(item["chat_id"]) != chat_id: + return None + if item["status"] == "failed" or item["status"] not in ("queued", "processing", "ready"): + raise AttachmentRejected() + row = conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, body, created_at) + VALUES (%s, %s, %s, %s, clock_timestamp()) + RETURNING id, chat_id, sender_id, body AS text, created_at + """, + (message_id, chat_id, sender_id, text), + ).fetchone() + for ordinal, attachment_id in enumerate(attachment_ids): + conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, ordinal) + VALUES (%s, %s, %s) + """, + (row["id"], attachment_id, ordinal), + ) + conn.execute( + """ + INSERT INTO idempotency_keys ( + user_id, http_method, request_path, idem_key, + request_hash, message_id, created_at + ) + VALUES (%s, 'POST', %s, %s, %s, %s, clock_timestamp()) + """, + (sender_id, path, idem_key, digest, row["id"]), + ) + return public_message(row, attachment_ids) + except CheckViolation as exc: + raise ValueError("message") from exc + except psycopg.OperationalError as exc: + _db_down(exc) + + def list_messages(self, chat_id, user_id, limit, cursor): + after = None + if cursor is not None: + created_at, item_id = decode_cursor(cursor, chat_id) + after = (parse_time(created_at), uuid.UUID(item_id)) + try: + with self._connect() as conn: + if self._member(conn, chat_id, user_id) is None: + return None + if after is None: + rows = conn.execute(SQL_MESSAGES_FIRST, (chat_id, limit + 1)).fetchall() + else: + rows = conn.execute( + SQL_MESSAGES_AFTER, + (chat_id, after[0], after[1], limit + 1), + ).fetchall() + linked = self._attachment_ids(conn, [row["id"] for row in rows[:limit]]) + except psycopg.OperationalError as exc: + _db_down(exc) + page = [public_message(row, linked.get(str(row["id"]), [])) for row in rows[:limit]] + if len(rows) > limit and page: + last = page[-1] + return page, encode_cursor(chat_id, last["created_at"], last["id"]) + return page, None diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..544a3722 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,182 @@ +"""Отдельный процесс. API не уменьшает картинку, даже если worker остановлен.""" +import logging +import os +import socket +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +import pika +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from blobs import bucket_ready, read_secret +from broker import PROCESS_QUEUE, _parameters, broker_channel, ensure_topology, publish_pending +from jobs import cleanup_staging, lease_seconds, process_delivery, release_expired + +LOGGER = logging.getLogger("worker") +SCHEMA_VERSION = "004_jobs" + + +def database_url(): + return read_secret(os.environ["DATABASE_URL_FILE"]) + + +def concurrency(): + value = int(os.environ.get("WORKER_CONCURRENCY", "2")) + if not 1 <= value <= 8: + raise RuntimeError("WORKER_CONCURRENCY") + return value + + +def wait_schema(conninfo): + last = "error" + for _ in range(90): + try: + with psycopg.connect(conninfo, connect_timeout=3, row_factory=dict_row) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (SCHEMA_VERSION,), + ).fetchone() + if row is not None: + return + last = "migration missing" + except psycopg.Error as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("schema not ready: %s", last) + raise SystemExit(1) + + +def wait_bucket(): + for _ in range(60): + if bucket_ready(): + return + time.sleep(1) + LOGGER.error("object storage not ready") + raise SystemExit(1) + + +def wait_broker(): + last = "error" + for _ in range(60): + try: + with broker_channel(): + return + except Exception as exc: + last = type(exc).__name__ + time.sleep(1) + LOGGER.error("broker not ready: %s", last) + raise SystemExit(1) + + +def make_pool(conninfo, slots): + return ConnectionPool( + conninfo, + min_size=1, + max_size=slots * 2 + 2, + timeout=3, + kwargs={ + "connect_timeout": 3, + "row_factory": dict_row, + "application_name": "mtusi-lab5-worker", + "options": "-c statement_timeout=5000 -c lock_timeout=3000 -c idle_in_transaction_session_timeout=30000", + }, + open=True, + ) + + +def consume(pool, owner, slots): + executor = ThreadPoolExecutor(max_workers=slots, thread_name_prefix="job") + while True: + connection = None + try: + connection = pika.BlockingConnection(_parameters()) + channel = connection.channel() + ensure_topology(channel) + channel.basic_qos(prefetch_count=slots) + + def on_message(ch, method, _props, body): + executor.submit(run_one, pool, owner, ch, method.delivery_tag, body) + + channel.basic_consume(PROCESS_QUEUE, on_message, auto_ack=False) + channel.start_consuming() + except Exception as exc: + LOGGER.warning("consumer %s", type(exc).__name__) + time.sleep(1) + finally: + if connection is not None: + try: + connection.close() + except Exception: + pass + + +def run_one(pool, owner, channel, tag, body): + try: + process_delivery(pool, body, owner) + action = "ack" + except Exception as exc: + LOGGER.warning("delivery %s", type(exc).__name__) + action = "nack" + + def finish(): + try: + if action == "ack": + channel.basic_ack(tag) + else: + channel.basic_nack(tag, requeue=True) + except Exception: + pass + + try: + channel.connection.add_callback_threadsafe(finish) + except Exception: + pass + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + conninfo = database_url() + wait_schema(conninfo) + wait_bucket() + wait_broker() + slots = concurrency() + pool = make_pool(conninfo, slots) + owner = socket.gethostname() + stop = threading.Event() + + def drain(): + while not stop.wait(1): + try: + with pool.connection() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("drain %s", type(exc).__name__) + + def sweep(): + ticks = 0 + while not stop.wait(5): + ticks += 1 + try: + with pool.connection() as conn: + release_expired(conn) + if ticks % 12 == 0: + cleanup_staging(conn) + except Exception as exc: + LOGGER.warning("sweep %s", type(exc).__name__) + + threading.Thread(target=drain, name="drain", daemon=True).start() + threading.Thread(target=sweep, name="sweep", daemon=True).start() + try: + with pool.connection() as conn: + publish_pending(conn) + except Exception as exc: + LOGGER.warning("startup drain %s", type(exc).__name__) + LOGGER.info("worker ready owner=%s concurrency=%s lease=%s", owner, slots, lease_seconds()) + consume(pool, owner, slots) + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_negative.py b/student-tests/run_negative.py new file mode 100644 index 00000000..7040c17a --- /dev/null +++ b/student-tests/run_negative.py @@ -0,0 +1,130 @@ +"""Негативные проверки API. Токены и пароли не печатаются.""" +import json +import struct +import sys +import uuid +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def png(width, height, payload=b""): + def chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + payload + chunk(b"IEND", b"") + + +def main(): + base = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8080" + client = Client(base) + alice = client.user("n1") + bob = client.user("n2") + eve = client.user("n3") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Негативные", "member_ids": [alice["id"].upper(), bob["id"]]}, + schema="Chat", + )["json"] + other = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Другой", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + upper = client.call("GET", "/api/v1/chats/" + chat["id"].upper(), actor=bob, schema="Chat")["json"] + results = [] + + def record(name, status): + results.append({"name": name, "status": status}) + + record("chat_uuid_case", 200 if upper["id"] == chat["id"] else 0) + path = "/api/v1/chats/" + chat["id"] + "/messages" + record("blank_text", client.call("POST", path, 400, actor=alice, body={"text": " "}, headers={"Idempotency-Key": "blank-1"})["status"]) + record("bad_json", client.call("POST", path, 400, actor=alice, raw=b"{", headers={"Content-Type": "application/json", "Idempotency-Key": "bad-json"})["status"]) + record("short_password", client.call("POST", "/api/v1/users", 400, body={"username": "abc", "display_name": "Имя", "password": "short"})["status"]) + key = uuid.uuid4().hex + first = client.call("POST", path, 201, actor=alice, body={"text": "одно"}, headers={"Idempotency-Key": key}, schema="Message")["json"] + second = client.call( + "POST", + path, + 201, + actor=alice, + body={"text": "одно", "attachment_ids": []}, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + record("omitted_equals_empty", 201 if first == second else 0) + record("same_key_other_body", client.call("POST", path, 409, actor=alice, body={"text": "другое"}, headers={"Idempotency-Key": key})["status"]) + history = client.call("GET", path.replace("/messages", "/messages?limit=100"), actor=bob, schema="MessagePage")["json"] + record("no_second_row", 200 if sum(1 for item in history["items"] if item["text"] in ("одно", "другое")) == 1 else 0) + other_path = "/api/v1/chats/" + other["id"] + "/messages" + other_message = client.call( + "POST", + other_path, + 201, + actor=alice, + body={"text": "одно"}, + headers={"Idempotency-Key": key}, + schema="Message", + )["json"] + record("same_key_other_chat", 201 if other_message["id"] != first["id"] else 0) + record("foreign_chat", client.call("GET", "/api/v1/chats/" + chat["id"], 404, actor=eve)["status"]) + record("bad_cursor", client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?cursor=not-a-cursor", 400, actor=alice)["status"]) + upload = "/api/v1/chats/" + chat["id"] + "/attachments" + record("slash_name", client.call("POST", upload + "?filename=a/b.png", 400, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/png"})["status"]) + record("huge_pixels", client.call("POST", upload + "?filename=big.png", 413, actor=alice, raw=png(8000, 8000), headers={"Content-Type": "image/png"})["status"]) + record("bad_png", client.call("POST", upload + "?filename=bad.png", 415, actor=alice, raw=b"\x89PNG\r\n\x1a\nbroken", headers={"Content-Type": "image/png"})["status"]) + record("type_mismatch", client.call("POST", upload + "?filename=no.jpg", 415, actor=alice, raw=png(1, 1), headers={"Content-Type": "image/jpeg"})["status"]) + record("oversize", client.call("POST", upload + "?filename=over.png", 413, actor=alice, raw=b"x" * (10 * 1024 * 1024 + 1), headers={"Content-Type": "image/png"})["status"]) + missing = str(uuid.uuid4()) + record("unknown_attachment", client.call("GET", "/api/v1/attachments/" + missing.upper(), 404, actor=alice)["status"]) + record("no_token", client.call("GET", "/api/v1/users/me", 401)["status"]) + body = json.dumps({"results": results}, ensure_ascii=False, indent=2) + print(body) + failed = [item for item in results if item["status"] not in (200, 201, 400, 401, 404, 409, 413, 415)] + if failed: + return 1 + expected = { + "chat_uuid_case": 200, + "blank_text": 400, + "bad_json": 400, + "short_password": 400, + "omitted_equals_empty": 201, + "same_key_other_body": 409, + "no_second_row": 200, + "same_key_other_chat": 201, + "foreign_chat": 404, + "bad_cursor": 400, + "slash_name": 400, + "huge_pixels": 413, + "bad_png": 415, + "type_mismatch": 415, + "oversize": 413, + "unknown_attachment": 404, + "no_token": 401, + } + for name, status in expected.items(): + got = next(item["status"] for item in results if item["name"] == name) + if got != status: + print("mismatch " + name, file=sys.stderr) + return 1 + print("PASS negative") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as exc: + print("FAIL " + str(exc), file=sys.stderr) + raise SystemExit(1) diff --git a/student-tests/run_pipeline.py b/student-tests/run_pipeline.py new file mode 100644 index 00000000..aab7688a --- /dev/null +++ b/student-tests/run_pipeline.py @@ -0,0 +1,653 @@ +"""Проверки очереди и трека 2. Токены и пароли не печатаются.""" +import hashlib +import json +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 +from image_probe import jpeg_size # noqa: E402 + +BASE = "http://127.0.0.1:8080" +EVIDENCE = ROOT / "evidence" / "pipeline.json" +INSIDE = r""" +import json, logging, os, hashlib +from datetime import datetime, timezone +logging.disable(logging.CRITICAL) +req = json.loads(REQUEST_JSON) +from blobs import read_secret, read_bytes, put_bytes, delete_quiet +from jobs import bump_pipeline, replay_job, republish +import psycopg +from psycopg.rows import dict_row +url = read_secret(os.environ["DATABASE_URL_FILE"]) +action = req["action"] +out = {"action": action} +with psycopg.connect(url, row_factory=dict_row) as conn: + if action == "republish": + out["ok"] = bool(republish(conn, req["job_id"], req.get("attempt"))) + elif action == "bump": + out["result"] = bump_pipeline(conn, req["attachment_id"]) + elif action == "replay": + out["result"] = replay_job(conn, req["job_id"]) + elif action == "corrupt": + row = conn.execute( + "SELECT staging_key FROM attachments WHERE id = %s", + (req["attachment_id"],), + ).fetchone() + key = row["staging_key"] + data = read_bytes(key) + stamp = datetime.now(timezone.utc) + put_bytes(key + ".held", data, "application/octet-stream", stamp) + put_bytes(key, b"not-an-image", "image/png", stamp) + out["original_sha256"] = hashlib.sha256(data).hexdigest() + out["bytes"] = len(data) + elif action == "restore": + row = conn.execute( + "SELECT staging_key FROM attachments WHERE id = %s", + (req["attachment_id"],), + ).fetchone() + key = row["staging_key"] + data = read_bytes(key + ".held") + put_bytes(key, data, "image/png", datetime.now(timezone.utc)) + delete_quiet(key + ".held") + out["restored_sha256"] = hashlib.sha256(data).hexdigest() + out["bytes"] = len(data) + else: + raise SystemExit("unknown") +print(json.dumps(out)) +""" + + +def run(args, timeout=180, check=True): + proc = subprocess.run( + args, + cwd=str(ROOT), + capture_output=True, + timeout=timeout, + ) + if check and proc.returncode != 0: + err = proc.stderr.decode("utf-8", "replace")[-1200:] + raise RuntimeError(args[0] + " failed: " + err) + return proc + + +def compose(*args, timeout=180): + run(["docker", "compose", "--env-file", ".env", *args], timeout=timeout) + + +def inspect(name, template): + proc = run(["docker", "inspect", "-f", template, name], timeout=30) + return proc.stdout.decode().strip() + + +def wait_health(name, seconds=180): + deadline = time.monotonic() + seconds + last = "" + while time.monotonic() < deadline: + last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + if last.startswith("running healthy"): + return last + time.sleep(1) + raise RuntimeError(name + " not healthy: " + last) + + +def restart_service(service, containers, seconds=180): + if isinstance(containers, str): + containers = [containers] + before = {name: inspect(name, "{{.State.StartedAt}}") for name in containers} + compose("restart", service) + deadline = time.monotonic() + seconds + last = "" + while time.monotonic() < deadline: + ready = True + for name in containers: + started = inspect(name, "{{.State.StartedAt}}") + last = inspect(name, "{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{end}}") + if started == before[name] or not last.startswith("running healthy"): + ready = False + if ready: + return + time.sleep(1) + raise RuntimeError(service + " did not restart: " + last) + + +def psql(sql): + proc = run( + [ + "docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-v", "ON_ERROR_STOP=1", "-q", "-t", "-A", "-c", sql, + ], + timeout=30, + ) + return proc.stdout.decode("utf-8").strip() + + +def state(attachment_id): + sql = """ + SELECT json_build_object( + 'status', a.status, + 'error_code', a.error_code, + 'pipeline_version', a.pipeline_version, + 'jobs', ( + SELECT coalesce(json_agg(json_build_object( + 'id', j.id, 'status', j.status, 'attempt', j.attempt, + 'pipeline_version', j.pipeline_version, 'error_code', j.error_code + ) ORDER BY j.pipeline_version), '[]'::json) + FROM attachment_jobs j WHERE j.attachment_id = a.id + ), + 'variants', ( + SELECT coalesce(json_agg(json_build_object( + 'name', v.name, 'pipeline_version', v.pipeline_version, + 'width', v.width, 'height', v.height, 'content_type', v.content_type + ) ORDER BY v.pipeline_version, v.name), '[]'::json) + FROM attachment_variants v WHERE v.attachment_id = a.id + ), + 'unpublished', ( + SELECT count(*) FROM outbox o + JOIN attachment_jobs j ON j.id = o.job_id + WHERE j.attachment_id = a.id AND o.published_at IS NULL + ), + 'events', ( + SELECT coalesce(json_agg(e.kind ORDER BY e.created_at), '[]'::json) + FROM job_events e WHERE e.attachment_id = a.id + ) + )::text + FROM attachments a WHERE a.id = '{id}'::uuid + """.format(id=attachment_id) + return json.loads(psql(sql)) + + +def arm(kind, attachment_id): + target = "NULL" if attachment_id is None else "'" + attachment_id + "'::uuid" + psql( + "INSERT INTO fault_injections (id, kind, attachment_id, armed) " + "VALUES (gen_random_uuid(), '{kind}', {target}, true)".format(kind=kind, target=target) + ) + + +def queues(): + proc = run( + [ + "docker", "compose", "--env-file", ".env", "exec", "-T", "rabbitmq", + "rabbitmqctl", "list_queues", "name", "messages", "--quiet", + ], + timeout=30, + ) + found = {} + for line in proc.stdout.decode().splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].isdigit(): + found[parts[0]] = int(parts[1]) + return found + + +def depth(): + last = "" + for _ in range(20): + try: + found = queues() + return found.get("attachments.process", -1) + except RuntimeError as exc: + last = str(exc) + time.sleep(1) + raise RuntimeError(last) + + +def inside(req, timeout=90): + code = INSIDE.replace("REQUEST_JSON", json.dumps(json.dumps(req))) + proc = run( + [ + "docker", "compose", "--env-file", ".env", "run", "--rm", "--no-deps", "-T", + "--entrypoint", "python", "worker", "-c", code, + ], + timeout=timeout, + ) + lines = [line for line in proc.stdout.decode("utf-8").splitlines() if line.startswith("{")] + if not lines: + raise RuntimeError("no json from worker helper") + return json.loads(lines[-1]) + + +def stop_worker(): + compose("stop", "worker") + + +def start_worker(): + compose("start", "worker") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if inspect("mtusi-lab5-worker-1", "{{.State.Status}}") == "running": + return + time.sleep(0.5) + raise RuntimeError("worker did not start") + + +def restart_count(): + return int(inspect("mtusi-lab5-worker-1", "{{.RestartCount}}")) + + +def wait_status(actor, attachment_id, want, seconds=90): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + body = client.call( + "GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment" + )["json"] + last = body["status"] + if last == want: + return body + if want != "failed" and last == "failed": + raise Failure("attachment failed " + attachment_id) + time.sleep(0.4) + raise Failure("timeout status=" + str(last) + " want=" + want) + + +def check_picture(actor, ready, original): + download = client.call("GET", "/api/v1/attachments/" + ready["id"] + "/content", actor=actor) + if download["bytes"] != original: + raise Failure("original bytes differ") + names = {item["name"]: item for item in ready["variants"]} + thumb = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=thumbnail", actor=actor + ) + crop = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=crop", actor=actor + ) + optimized = client.call( + "GET", "/api/v1/attachments/" + ready["id"] + "/content?variant=optimized", actor=actor + ) + if jpeg_size(thumb["bytes"]) != (256, 160): + raise Failure("thumbnail size") + if jpeg_size(crop["bytes"]) != (128, 128): + raise Failure("crop size") + if optimized["bytes"][:4] != b"RIFF" or optimized["bytes"][8:12] != b"WEBP": + raise Failure("optimized is not webp") + if names["thumbnail"]["width"] != 256 or names["thumbnail"]["height"] != 160: + raise Failure("thumbnail metadata") + if names["crop"]["width"] != 128 or names["crop"]["height"] != 128: + raise Failure("crop metadata") + if names["optimized"]["width"] != 320 or names["optimized"]["height"] != 200: + raise Failure("optimized metadata") + if hashlib.sha256(thumb["bytes"]).hexdigest() != names["thumbnail"]["sha256"]: + raise Failure("thumbnail sha") + rows = state(ready["id"]) + version_rows = [row for row in rows["variants"] if row["pipeline_version"] == rows["pipeline_version"]] + if len(version_rows) != 4: + raise Failure("variant rows " + str(len(version_rows))) + return { + "pipeline_version": rows["pipeline_version"], + "variant_rows": len(version_rows), + "thumbnail": [names["thumbnail"]["width"], names["thumbnail"]["height"]], + "crop": [names["crop"]["width"], names["crop"]["height"]], + "optimized": [names["optimized"]["width"], names["optimized"]["height"]], + "original_sha256": hashlib.sha256(original).hexdigest(), + } + + +def upload(actor, chat): + att, content = client.attachment(actor, chat) + return att, content + + +results = [] + + +def scenario(name, fn): + started = time.monotonic() + try: + detail = fn() + results.append({"name": name, "ok": True, "seconds": round(time.monotonic() - started, 3), "detail": detail}) + except Exception as exc: + results.append({"name": name, "ok": False, "seconds": round(time.monotonic() - started, 3), "error": type(exc).__name__ + ": " + str(exc)}) + raise + + +client = Client(BASE, timeout=20) + + +def durability(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + blocked = client.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob) + text = client.message(alice, chat, "текст пока файл в очереди")["json"] + linked = client.message(alice, chat, "", attachments=[att["id"]])["json"] + before_api = depth() + if before_api < 1: + raise Failure("job was not in the durable queue") + restart_service("api", ["mtusi-lab5-api-1", "mtusi-lab5-api-2"]) + after_api = depth() + restart_service("rabbitmq", "mtusi-lab5-rabbitmq-1", 180) + after_rabbit = depth() + if after_api < 1 or after_rabbit < 1: + raise Failure("queue lost the job") + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return { + "upload_status": att["status"], + "content_while_queued": blocked["status"], + "text_status": 201, + "text_id": text["id"], + "linked_id": linked["id"], + "queue_before_api_restart": before_api, + "queue_after_api_restart": after_api, + "queue_after_rabbit_restart": after_rabbit, + "picture": picture, + } + + +def outbox_gap(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + arm("skip_inline_publish", None) + before = depth() + att, content = upload(alice, chat) + info = state(att["id"]) + queued = depth() + if info["unpublished"] != 1 or queued != before: + raise Failure("outbox was published early") + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return {"unpublished_before_worker": info["unpublished"], "queue_depth": queued, "picture": picture} + + +def duplicate(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + job_id = info["jobs"][0]["id"] + published = inside({"action": "republish", "job_id": job_id}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + return {"republish": published["ok"], "picture": picture} + + +def reorder(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + job_id = info["jobs"][0]["id"] + inside({"action": "republish", "job_id": job_id, "attempt": 2}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + after = state(att["id"]) + if after["jobs"][0]["attempt"] != 1 or after["status"] != "ready": + raise Failure("reorder changed the job") + return {"attempt": after["jobs"][0]["attempt"], "picture": picture} + + +def stale_version(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + info = state(att["id"]) + old_job = info["jobs"][0]["id"] + bumped = inside({"action": "bump", "attachment_id": att["id"]}) + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + before = check_picture(bob, ready, content) + thumb_before = client.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob + )["bytes"] + inside({"action": "republish", "job_id": old_job, "attempt": 1}) + time.sleep(2) + ready2 = client.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"] + thumb_after = client.call( + "GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob + )["bytes"] + after = state(att["id"]) + old = [job for job in after["jobs"] if job["id"] == old_job][0] + if ready2["status"] != "ready" or after["pipeline_version"] != 2: + raise Failure("new version was overwritten") + if old["status"] != "superseded": + raise Failure("old job was not superseded") + if thumb_before != thumb_after: + raise Failure("old job changed thumbnail") + return { + "bump": bumped["result"], + "old_job_status": old["status"], + "old_error_code": old["error_code"], + "attachment_error_code": after["error_code"], + "picture": before, + "thumbnail_unchanged": True, + } + + +def crash(kind): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + arm(kind, att["id"]) + start_worker() + born = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}") + ready = wait_status(bob, att["id"], "ready", 120) + picture = check_picture(bob, ready, content) + restarted = inspect("mtusi-lab5-worker-1", "{{.State.StartedAt}}") + if born == restarted: + raise Failure("worker did not exit") + return {"started_at": born, "restarted_at": restarted, "picture": picture} + + +def transient(): + alice, bob, _eve, chat = client.bootstrap() + stop_worker() + att, _content = upload(alice, chat) + arm("transient_once", att["id"]) + start_worker() + wait_status(bob, att["id"], "ready", 90) + info = state(att["id"]) + version_rows = [row for row in info["variants"] if row["pipeline_version"] == info["pipeline_version"]] + if info["jobs"][0]["attempt"] != 2 or "retry" not in info["events"] or len(version_rows) != 4: + raise Failure("transient retry did not keep one result") + return { + "attempt": info["jobs"][0]["attempt"], + "events": info["events"], + "variant_rows": len(version_rows), + "status": info["status"], + } + + +def permanent_and_replay(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + att, content = upload(alice, chat) + corrupted = inside({"action": "corrupt", "attachment_id": att["id"]}) + start_worker() + failed = wait_status(bob, att["id"], "failed", 90) + info = state(att["id"]) + restored = inside({"action": "restore", "attachment_id": att["id"]}) + replayed = inside({"action": "replay", "job_id": info["jobs"][0]["id"]}) + ready = wait_status(bob, att["id"], "ready", 90) + picture = check_picture(bob, ready, content) + again = inside({"action": "replay", "job_id": info["jobs"][0]["id"]}) + after = state(att["id"]) + version_rows = [row for row in after["variants"] if row["pipeline_version"] == after["pipeline_version"]] + if failed["error_code"] != "invalid_image" or info["jobs"][0]["attempt"] != 1: + raise Failure("permanent error was retried") + if replayed["result"].get("result") != "queued" or again["result"].get("result") != "already_ready": + raise Failure("replay result") + if len(version_rows) != 4 or restored["restored_sha256"] != corrupted["original_sha256"]: + raise Failure("replay did not keep one result") + return { + "corrupt_sha256": corrupted["original_sha256"], + "failed_code": failed["error_code"], + "attempt_at_fail": info["jobs"][0]["attempt"], + "restored_sha256": restored["restored_sha256"], + "replay": replayed["result"], + "second_replay": again["result"], + "variant_rows": len(version_rows), + "picture": picture, + } + + +def lease(): + stop_worker() + alice, bob, _eve, chat = client.bootstrap() + arm("skip_inline_publish", None) + before = depth() + att, content = upload(alice, chat) + info = state(att["id"]) + if info["unpublished"] != 1 or depth() != before: + raise Failure("lease setup published the job") + job_id = info["jobs"][0]["id"] + psql( + "UPDATE attachment_jobs SET status = 'processing', lease_owner = 'dead', " + "lease_until = now() - interval '2 minutes', updated_at = now() " + "WHERE id = '{id}'::uuid".format(id=job_id) + ) + psql( + "UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = '{id}'::uuid".format( + id=att["id"] + ) + ) + psql("DELETE FROM outbox WHERE job_id = '{id}'::uuid AND published_at IS NULL".format(id=job_id)) + started = time.monotonic() + start_worker() + ready = wait_status(bob, att["id"], "ready", 90) + elapsed = round(time.monotonic() - started, 3) + picture = check_picture(bob, ready, content) + info = state(att["id"]) + if "lease_expired" not in info["events"] or info["jobs"][0]["attempt"] != 1: + raise Failure("expired lease was not resumed") + return { + "seconds_until_ready": elapsed, + "events": info["events"], + "attempt": info["jobs"][0]["attempt"], + "picture": picture, + } + + +def http_status(url): + req = urllib.request.Request(url, headers={"Accept": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=10) as response: + return response.status, response.read(4096).decode("utf-8", "replace") + except urllib.error.HTTPError as error: + return error.code, error.read(4096).decode("utf-8", "replace") + + +def leaked(text): + lowered = text.lower() + return [ + word for word in ( + "redis", "postgres", "minio", "password", "select", "rabbit", "amqp", + "secret", "database", "bucket", "messenger-attachments", + ) + if word in lowered + ] + + +def ready_503(): + compose("stop", "redis") + time.sleep(2) + lb_status, lb_body = http_status(BASE + "/health/ready") + proc = run( + [ + "docker", "exec", "mtusi-lab5-api-1", "python", "-c", + "import urllib.request,urllib.error\n" + "req=urllib.request.Request('http://127.0.0.1:8080/health/ready')\n" + "try:\n" + " r=urllib.request.urlopen(req, timeout=5)\n" + " print(r.status)\n" + " print(r.read(4096).decode())\n" + "except urllib.error.HTTPError as e:\n" + " print(e.code)\n" + " print(e.read(4096).decode())\n", + ], + timeout=20, + ) + lines = proc.stdout.decode("utf-8", "replace").splitlines() + api_status = int(lines[0]) + api_body = "\n".join(lines[1:]) + compose("start", "redis") + wait_health("mtusi-lab5-redis-1") + started = time.monotonic() + deadline = started + 40 + recovered = None + while time.monotonic() < deadline: + try: + recovered, _body = http_status(BASE + "/health/ready") + except (urllib.error.URLError, TimeoutError, OSError): + recovered = 0 + if recovered == 200: + break + time.sleep(0.5) + recovery_seconds = round(time.monotonic() - started, 3) + if lb_status != 503 or api_status != 503 or leaked(lb_body) or leaked(api_body): + raise Failure("ready 503 text") + if "Сервис временно недоступен" not in api_body or recovered != 200: + raise Failure("ready 503 text") + return { + "lb_status": lb_status, + "lb_body": lb_body, + "api_status": api_status, + "api_message": "Сервис временно недоступен", + "forbidden": [], + "recovered": recovered, + "recovery_seconds": recovery_seconds, + } + + +def main(): + wait_health("mtusi-lab5-api-1") + wait_health("mtusi-lab5-api-2") + psql("UPDATE fault_injections SET armed = false WHERE armed") + start_worker() + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if depth() == 0 and queues().get("attachments.retry", 0) == 0: + break + time.sleep(0.5) + steps = [ + ("durability", durability), + ("outbox_gap", outbox_gap), + ("duplicate", duplicate), + ("reorder", reorder), + ("stale_version", stale_version), + ("crash_before_effect", lambda: crash("crash_before_effect")), + ("crash_after_objects", lambda: crash("crash_after_objects")), + ("crash_after_commit", lambda: crash("crash_after_commit")), + ("transient_once", transient), + ("permanent_replay", permanent_and_replay), + ("lease", lease), + ("ready_503", ready_503), + ] + if len(sys.argv) > 1 and sys.argv[1] == "resume" and EVIDENCE.is_file(): + done = [ + item for item in json.loads(EVIDENCE.read_text(encoding="utf-8"))["results"] if item.get("ok") + ] + results.extend(done) + steps = [item for item in steps if item[0] not in {row["name"] for row in done}] + failed = None + for name, fn in steps: + try: + scenario(name, fn) + except Exception as exc: + failed = exc + break + EVIDENCE.write_text(json.dumps({"results": results}, ensure_ascii=False, indent=2), encoding="utf-8") + if failed is not None: + print("FAIL " + type(failed).__name__ + ": " + str(failed), file=sys.stderr) + return 1 + print("PASS pipeline") + return 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except Failure as exc: + print("FAIL " + str(exc), file=sys.stderr) + raise SystemExit(1) -- 2.54.0