diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..dfdb8b77 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..fee0cecf 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Казанцева Виктория / БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: трек 2, идемпотентный versioned pipeline -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: API лабораторий 1–4 сохранён, upload переведён с 201 на 202 `queued`. Добавлены durable staging, transactional outbox, RabbitMQ с publisher confirm, отдельный worker, thumbnail/crop/optimized, lease, ограниченный retry с backoff и jitter, DLQ/`failed_jobs`, явный replay и версионированный pipeline v1/v2. +Команды запуска из чистого clone: `python scripts/bootstrap.py`, затем `docker compose --env-file .env up -d --build`. Подробности, образы и проверки — в REPORT.md. +Результаты проверок и ссылка на REPORT.md: `check_contract`, геометрия изображений, `tests/smoke.py`, `student_tests/run_grade.py` и `student_tests/run_hooks.py` прошли 6 ноября 2026. Сводка без токенов — `evidence/grade5-track2.json` и REPORT.md. Публичный `tests/resilience.py` на Windows не запускает POSIX-скрипт (`WinError 193`); те же команды Docker выполнены из `student_tests/run_hooks.py`. +Ограничения / известные ошибки: объектное хранилище — `pgsty/silo` вместо недоступного публично образа MinIO. Lease не продлевается и равен timeout обработки. Трек 1 не делался. Это at-least-once с идемпотентным эффектом, не exactly-once. +Источники: контракт репозитория, документация RabbitMQ; FastAPI, uvicorn, psycopg, argon2-cffi, boto3, pika, Pillow. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..345df6ae --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin app +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY migrations ./migrations + +USER app +EXPOSE 8080 +CMD ["python", "-m", "app.main"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..80d068bc 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,126 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Казанцева Виктория / БПИ2301 +- Целевая оценка: 5. Трек 2 — доказанная идемпотентность versioned pipeline +- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/31 +- Дата прогона: 6 ноября 2026, хост Windows 10, Docker 29.2.1, Compose v5.1.0, Python 3.11 (клиент) и Python 3.12.8 в образе +- Источники и библиотеки: контракт репозитория, документация RabbitMQ; FastAPI, uvicorn, psycopg, argon2-cffi, boto3, pika, Pillow. Секреты в отчёт не копировались ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose и Python 3.9+. На Windows публичные тесты читают JSON без явной кодировки, поэтому перед ними нужен `PYTHONUTF8=1`. -## Архитектура и решения +```sh +python scripts/bootstrap.py +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +`bootstrap.py` один раз создаёт локальный `.env` со случайными паролями и не печатает их. Повторный запуск существующий файл не перезаписывает. Миграции применяет сам API и worker при старте (`migrations/001_init.sql`, advisory lock). Отдельной ручной миграции нет. Готовность: `GET http://127.0.0.1:8080/health/ready` возвращает 200. Брокер и worker для ready не нужны: намерение лежит в outbox. + +```sh +python scripts/check_contract.py +python tests/smoke.py --base-url http://localhost:8080 +python student_tests/test_images.py +python student_tests/run_grade.py +python student_tests/run_hooks.py +``` + +На Linux те же команды через `make check`, `make test`, и после `chmod +x scripts/restart.sh scripts/stop-one.sh`: + +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +sh scripts/start-apis.sh +``` + +Остановка без удаления данных: `docker compose --env-file .env stop`. Удаление учебных volumes: `docker compose --env-file .env down -v`. + +## Архитектура + +Публичный вход только у nginx (`127.0.0.1:8080` в этом прогоне был опубликован как `8080:8080`). За ним две реплики API, `api-1` и `api-2`. Тяжёлую обработку делает отдельный процесс `python -m app.worker`. Состояние: PostgreSQL 16.8, объектное хранилище, RabbitMQ 3.13.7. Volumes: `pgdata`, `miniodata`, `rabbitdata`. + +Официальные образы `minio/minio` и `quay.io/minio/minio` на момент сборки анонимно не скачиваются. Объектное хранилище — совместимый образ `pgsty/silo:RELEASE.2026-10-26T00-00-00Z`. Клиент говорит с ним по S3 API (`boto3`, path-style). Бакет создаёт одноразовый `minio-init`. + +Приём файла: + +1. Проверка участника чата, лимита очереди, `Content-Type`, magic bytes, размера и числа пикселей по заголовку, до полного decode. +2. Оригинал целиком пишется в `staging/{attachment_id}`. +3. В одной транзакции создаются строка attachment со статусом `queued`, job и строка outbox. +4. Ответ 202 — это снимок, зафиксированный до публикации. Даже если worker успел раньше, клиент получает `queued`. +5. Relay API и worker публикует outbox после COMMIT. Подтверждение брокера (publisher confirm, `delivery_mode=2`, `mandatory`) не заменяет consumer ack. + +Worker подтверждает сообщение только после устойчивого эффекта: объект в `originals/{id}`, варианты `variants/{id}/v{version}/{name}`, commit в PostgreSQL. Prefetch каждого потребителя равен 1, потребителей `WORKER_CONCURRENCY` (в `.env` — 2). Память контейнера worker ограничена 512 MiB, `MALLOC_ARENA_MAX=2`. Это не жёсткий `RLIMIT_AS`. + +Ключ эффекта: `(attachment_id, pipeline_version, variant)`. Повтор той же версии не создаёт второй набор. Более старая версия не затирает больший `ready_pipeline_version`. Если новая версия упала, публичный статус возвращается к последней успешной версии. + +Соответствие `.env.example`: `DATABASE_URL` и `BROKER_URL` заданы прямо в локальном `.env`, а не через `*_FILE`; код принимает оба способа. `SESSION_TTL_SECONDS=3600`, `IDEMPOTENCY_TTL_SECONDS=86400`, `MAX_UPLOAD_BYTES=10485760`, `MAX_IMAGE_PIXELS=40000000`, `JOB_MAX_ATTEMPTS=5`, `JOB_TIMEOUT_SECONDS=60`, `JOB_BACKOFF_BASE_SECONDS=0.4`, `JOB_BACKOFF_CAP_SECONDS=8`, `STAGING_RETENTION_SECONDS=86400`, `MAX_QUEUED_JOBS=500`. Короткий backoff нужен, чтобы исчерпание пяти попыток укладывалось в окно теста. Сессии и идемпотентность хранятся в PostgreSQL, отдельный Redis не используется. + +Pipeline v1: JPEG thumbnail и crop quality 85, WebP quality 80 method 6. Pipeline v2 меняет только качество (JPEG 60/70, WebP 50 method 4), геометрия та же. EXIF orientation применяется до кодирования, альфа кладётся на белый, метаданные в варианты не копируются. Thumbnail вписывается в 256×256 без увеличения, crop — центр 128×128 с увеличением, optimized вписывается в 1280×1280 без увеличения. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда | Наблюдение | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт skeleton | `python scripts/check_contract.py` | `PASS: offline skeleton structure...` | stdout | +| Геометрия вариантов | `python student_tests/test_images.py` | `PASS: image contract geometry` | stdout | +| Публичный smoke, upload 202 и JPEG thumbnail | `python tests/smoke.py --base-url http://localhost:8080` | `PASS: lab5 public API scenario...` Два прогона: до сценариев отказов и после рестарта | stdout | +| Compose | `docker compose --env-file .env config --quiet` | код 0 | — | +| Уровень 3: worker остановлен, 202, content 409, чат жив, рестарт API и RabbitMQ, тот же attachment ready | `python student_tests/run_grade.py` | очередь до рестарта 1 и после 1; attachment `7e2aaf32-e234-4568-8018-0aec26ac0308` | `evidence/grade5-track2.json` | +| Уровень 4: crop, WebP, лимиты, outbox, retry, DLQ, replay, crash до ack | тот же прогон | все сценарии `pass`, см. идентификаторы ниже | `evidence/grade5-track2.json` | +| Трек 2 | тот же прогон | versioned pipeline, reorder, stale job | `evidence/grade5-track2.json` | +| Hook рестарта всех хранилищ | `python student_tests/run_hooks.py` | StartedAt всех семи контейнеров сменился; сообщение `c7f0d384-570b-451d-aff3-99fe85f74841` и объект `90c60051-aa82-4dea-930a-785505bf7770` читаются прежним токеном | stdout | +| Отказ одной реплики | тот же прогон | `api-1` восстановился за 1.563 с, 23 записи, 0 ошибок; `api-2` за 2.578 с, 25 записей, 2 ошибки переходного периода. Остановленная реплика не была в хвосте ответов | stdout | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Идентификаторы трека, без токенов: -## Эксперименты и отказы +- outbox при паузе публикации: attachment `d29ac3f0-963c-4579-b7ca-9470e42d90b3`, тот же job `a457806d-b295-43ee-a671-2fa36f24388a`, затем ready +- постоянная ошибка и безопасный replay: attachment `f184e530-2798-4d1a-9a0d-fbf0ceff4d0a`, job `f52005ae-a113-47bf-879a-f7b820f74591`; второй replay — `already_ready` +- одна временная ошибка: attachment `dd1faa48-e8cf-4560-a537-f2f170717b28`, attempt 2 +- пять временных ошибок: attachment `7c532988-a75e-4967-9eec-0213bd37fdeb`, `retry_exhausted`, затем явный replay до ready +- дубль доставки: attachment `76144ebf-bc05-41a9-bd54-e94459b2e79a`, три варианта v1, повторный SHA тот же +- crash до side effect: `18e6a58f-4bbd-47d9-b9ac-d0b6402cb46f` +- crash после записи объекта: `67349346-ef4b-43ce-8388-2dfe0a8df36d` +- crash после commit до ack: `7ce2d67e-c042-4c49-9747-4dc0c1117250` +- устаревшая v1 после готовой v2: attachment `b7eee35e-0d2d-4cb1-9143-7af26c55a9a0`, новый job `bc8bfec7-1460-4f0f-a9aa-d62e2cef1bcf` +- reorder: attachment `18c8cf9d-1a82-452d-b0ee-9028873859fc`, старый job стал `superseded`, публичный результат — v2 -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Рестарт hook, StartedAt до и после: postgres `18:56:06Z` → `18:57:32Z`, minio `18:56:05Z` → `18:57:31Z`, rabbitmq `18:56:06Z` → `18:57:32Z`, api-1/api-2, worker и nginx тоже сменили время старта. Volumes не удалялись. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Публичный `tests/resilience.py` на этом Windows-хосте не смог запустить `.sh`: `WinError 193`. Скрипты оставлены POSIX и с LF. `student_tests/run_hooks.py` выполняет те же `docker compose restart` и `docker stop` по метке `lab.instance_id`. На Linux оценщик запускает сами `scripts/restart.sh` и `scripts/stop-one.sh`. + +## Трек 2 + +Гипотеза: доставки at-least-once достаточно, если эффект именуется `(attachment_id, pipeline_version, variant)`, а публичный результат двигается только вперёд по `ready_pipeline_version`. Подтверждённый результат не теряется, старая версия не затирает новую. + +Методика: одноразовые режимы в `runtime_control` — пауза outbox, `duplicate`, `before_effect`, `after_s3`, `after_commit`, повторная доставка того же `job_id`, новая версия pipeline и публикация старого сообщения после неё. Проверялись статус, число строк вариантов, SHA-256, audit и `ready_pipeline_version`. Новое задание руками не создавалось. Объекты одной версии пишутся только если ключа ещё нет; вставка варианта — `ON CONFLICT DO NOTHING`. + +Результат: все перечисленные сценарии завершились `pass` за один прогон `run_grade.py` (около 67 с). После crash worker сам поднялся (`restart: on-failure`, выход 75). `docker stop` выходит 0 и оставляет worker остановленным. + +Почему это не exactly-once. Брокер может доставить сообщение дважды, relay может опубликовать строку повторно, если процесс умер после confirm и до отметки `published_at`. Идемпотентность находится в ключе эффекта и в статусе job, а не в транспорте. Повтор успешной job — no-op и ack, а не второе исполнение. Exactly-once доставки нет: есть повторная доставка и однократный видимый эффект. + +Ограничения измерения. Инъекция кооперативная: worker читает флаг и вызывает `os._exit` в названной точке, это не случайный обрыв на произвольной инструкции. Жёсткий kill посреди записи объекта опирается на lease 60 с и на сравнение уже записанных байтов. Lease не продлевается во время работы; нормальная работа обязана уложиться в `JOB_TIMEOUT_SECONDS`, тем же пределом ограничен decode. Два перекрывающихся исполнителя одной версии не создают второй набор ключей, но лишний decode возможен, если lease уже истёк. Трек 1 (сравнение prefetch под нагрузкой) не выполнялся. + +## Отказы + +| Воздействие | Ожидание | Наблюдение | Восстановление | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Worker остановлен, затем рестарт API и RabbitMQ | 202, content 409, сообщение чата 201, очередь жива | очередь 1 → 1, тот же attachment | ready после старта worker, оригинал тот же | +| Пауза outbox между COMMIT и publish | job сохранён, брокер пуст | unpublished ≥ 1, глубина очереди 0, тот же job_id | ready без нового задания | +| Постоянная ошибка | `invalid_image`, content и attach 409 | так и есть | replay того же job до ready; повторный replay не исполняет дубль | +| Пять временных ошибок | `retry_exhausted`, запись в реестре failed | так и есть | явный replay до ready | +| Crash после S3 до commit и после commit до ack | тот же результат, три варианта v1 | audit `fault_after_s3` и `fault_after_commit` | ready без второго набора | +| Старая версия после новой | SHA v2 и `ready_pipeline_version=2` сохраняются | 6 строк вариантов, публичный набор не откатился | старый job superseded или no-op | +| Рестарт Postgres, Silo, RabbitMQ, API, worker, nginx | сессия, сообщение и байты на месте | hook и grade-прогон | без повторного логина | +| Остановка одной API-реплики | пишет вторая, остановленная не отвечает | оба идентификатора проверены отдельно | 1.6–2.6 с, реплика остаётся остановленной до `compose start` | -## Ограничения и следующие шаги +Повторы: задержка `min(cap, base * 2^(attempt-1)) * U(0.5, 1.5)`. Временная ошибка возвращает job в `queued` и пишет новую строку outbox. Постоянная и исчерпание попыток пишут `failed_jobs` и копию сообщения в `attachment.dlq`. Staging готового объекта удаляется после commit; staging failed хранится `STAGING_RETENTION_SECONDS`, чтобы replay мог его прочитать. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Ограничения + +- Готовность не проверяет RabbitMQ и worker. Это намеренно: принятая работа лежит в PostgreSQL и staging. +- `max_fails=0` у nginx. Иначе один отказ на старте на 3 с выключает оба upstream и даёт 502 «no live upstreams». Мёртвая реплика стоит одной попытки соединения (1 с), затем запрос уходит на живую. Официальный failover-клиент с timeout 2 с это выдержал. +- Сильный kill `-9` внутри lease не мгновенно подхватывается другим worker. Подхват — после `JOB_TIMEOUT_SECONDS`. +- Память worker ограничена cgroup, не адресным лимитом процесса. Decode сверх таймаута считается временной ошибкой; поток decode при этом может ещё завершиться в фоне. +- На этом хосте порт 8080 был занят контейнером `mtusi-lb` прежнего стенда. Остановлен только он, его volumes не удалялись. Чтобы вернуть старый вход, нужно остановить nginx этой лабораторной и снова запустить `mtusi-lb`. +- Операторские маршруты описаны в `contracts/extensions.md`. `contracts/openapi.json` и публичные тесты не менялись. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..9cb8e330 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger lab5 application package.""" diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 00000000..07172ca3 --- /dev/null +++ b/app/broker.py @@ -0,0 +1,91 @@ +"""RabbitMQ publisher confirms. Payloads are job ids, never file bytes or credentials.""" +from __future__ import annotations + +import json +import logging +import threading + +import pika +from pika.exceptions import AMQPError, NackError, UnroutableError + +log = logging.getLogger("messenger.broker") + +EXCHANGE = "attachment.jobs" +QUEUE = "attachment.process" +DLQ = "attachment.dlq" +ROUTING_PROCESS = "process" +ROUTING_DLQ = "dlq" + + +def _declare(channel) -> None: + channel.exchange_declare(exchange=EXCHANGE, exchange_type="direct", durable=True) + channel.queue_declare(queue=DLQ, durable=True) + channel.queue_bind(queue=DLQ, exchange=EXCHANGE, routing_key=ROUTING_DLQ) + channel.queue_declare(queue=QUEUE, durable=True) + channel.queue_bind(queue=QUEUE, exchange=EXCHANGE, routing_key=ROUTING_PROCESS) + + +class Publisher: + def __init__(self, url: str): + self._url = url + self._lock = threading.Lock() + self._connection = None + self._channel = None + + def _reset(self) -> None: + try: + if self._connection is not None and self._connection.is_open: + self._connection.close() + except Exception: + pass + self._connection = None + self._channel = None + + def _channel_locked(self): + if self._channel is not None and self._channel.is_open: + return self._channel + self._reset() + params = pika.URLParameters(self._url) + params.heartbeat = 30 + params.blocked_connection_timeout = 30 + params.socket_timeout = 10 + self._connection = pika.BlockingConnection(params) + channel = self._connection.channel() + channel.confirm_delivery() + _declare(channel) + self._channel = channel + return channel + + def publish(self, payload: dict, routing_key: str = ROUTING_PROCESS) -> bool: + body = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + with self._lock: + try: + channel = self._channel_locked() + channel.basic_publish( + exchange=EXCHANGE, + routing_key=routing_key, + body=body, + properties=pika.BasicProperties( + content_type="application/json", + delivery_mode=2, + type="attachment.process.v1", + ), + mandatory=True, + ) + return True + except (AMQPError, NackError, UnroutableError, OSError): + log.warning("broker publish was not confirmed", exc_info=True) + self._reset() + return False + + +def open_consumer(url: str): + params = pika.URLParameters(url) + params.heartbeat = 30 + params.blocked_connection_timeout = 30 + params.socket_timeout = 10 + connection = pika.BlockingConnection(params) + channel = connection.channel() + _declare(channel) + channel.basic_qos(prefetch_count=1) + return connection, channel diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..4aaafb25 --- /dev/null +++ b/app/db.py @@ -0,0 +1,99 @@ +"""PostgreSQL pool and migrations. The database is the source of truth.""" +from __future__ import annotations + +import logging +import time +from contextlib import contextmanager +from pathlib import Path + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +log = logging.getLogger("messenger.db") + +_pool: ConnectionPool | None = None +_MIGRATION = Path(__file__).resolve().parents[1] / "migrations" / "001_init.sql" +_LOCK = 8152026 + + +def init_pool(database_url: str, min_size: int = 1, max_size: int = 8) -> ConnectionPool: + global _pool + if _pool is not None: + return _pool + deadline = time.monotonic() + 60 + last = None + while True: + try: + pool = ConnectionPool( + database_url, + min_size=min_size, + max_size=max_size, + kwargs={"row_factory": dict_row, "autocommit": False}, + open=True, + check=ConnectionPool.check_connection, + ) + with pool.connection() as conn: + conn.execute("SELECT 1") + _pool = pool + return pool + except Exception as exc: # noqa: BLE001 - startup must wait for Postgres + last = exc + if time.monotonic() > deadline: + raise RuntimeError("database did not become reachable") from last + time.sleep(1) + + +def pool() -> ConnectionPool: + if _pool is None: + raise RuntimeError("database pool is not initialized") + return _pool + + +@contextmanager +def connection(): + with pool().connection() as conn: + try: + yield conn + conn.commit() + except Exception: + conn.rollback() + raise + + +def _statements(sql: str) -> list[str]: + statements = [] + current: list[str] = [] + for line in sql.splitlines(): + stripped = line.strip() + if stripped.startswith("--"): + continue + current.append(line) + if stripped.endswith(";"): + statement = "\n".join(current).strip() + if statement: + statements.append(statement) + current = [] + trailing = "\n".join(current).strip() + if trailing: + statements.append(trailing) + return statements + + +def migrate(database_url: str) -> None: + init_pool(database_url) + statements = _statements(_MIGRATION.read_text(encoding="utf-8")) + with connection() as conn: + conn.execute("SELECT pg_advisory_xact_lock(%s)", (_LOCK,)) + for statement in statements: + conn.execute(statement) + log.info("schema ready") + + +def ping() -> bool: + try: + with connection() as conn: + conn.execute("SELECT 1") + return True + except Exception: + log.warning("database ping failed", exc_info=True) + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..8d611a87 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,19 @@ +"""API errors that map onto the contract body. Details never include SQL or secrets.""" +from __future__ import annotations + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def unauthorized(kind: str) -> ApiError: + if kind == "basic": + header = 'Basic realm="messenger", charset="UTF-8"' + else: + header = 'Bearer realm="messenger"' + return ApiError(401, "unauthorized", "Требуется действительная аутентификация", {"WWW-Authenticate": header}) diff --git a/app/httpapi.py b/app/httpapi.py new file mode 100644 index 00000000..12ad7427 --- /dev/null +++ b/app/httpapi.py @@ -0,0 +1,1070 @@ +"""HTTP API for the messenger contract. Heavy image work is never done here.""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +import logging +import re +import secrets +import uuid +from datetime import timedelta + +from fastapi import FastAPI, Request +from fastapi.responses import Response +from psycopg.errors import UniqueViolation +from psycopg.types.json import Jsonb +from starlette.exceptions import HTTPException as StarletteHTTPException + +from app import runtime +from app.db import connection, ping +from app.errors import ApiError, unauthorized +from app.images import ImageRejected, inspect_upload +from app.outbox import add_outbox, audit, relay_tick +from app.timeutil import iso, utcnow + +log = logging.getLogger("messenger.api") + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +VARIANTS = {"original", "thumbnail", "crop", "optimized"} + + +class RequestIdMiddleware: + def __init__(self, app, instance_id: str): + self.app = app + self.instance_id = instance_id + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + request_id = uuid.uuid4().hex + scope.setdefault("state", {})["request_id"] = request_id + + async def send_with_ids(message): + if message["type"] == "http.response.start": + headers = list(message.get("headers") or []) + headers.append((b"x-request-id", request_id.encode("ascii"))) + headers.append((b"x-instance-id", self.instance_id.encode("utf-8"))) + message = {**message, "headers": headers} + await send(message) + + await self.app(scope, receive, send_with_ids) + + +def respond(status: int, payload, headers: dict | None = None) -> Response: + body = json.dumps(payload, ensure_ascii=False).encode("utf-8") + return Response(content=body, status_code=status, media_type="application/json", headers=headers) + + +def error_payload(request: Request, code: str, message: str) -> dict: + request_id = getattr(request.state, "request_id", None) or uuid.uuid4().hex + return {"error": {"code": code, "message": message, "request_id": request_id}} + + +def create_app() -> FastAPI: + settings, _store, _publisher = runtime.require() + app = FastAPI(docs_url=None, redoc_url=None, openapi_url=None, redirect_slashes=False) + app.add_middleware(RequestIdMiddleware, instance_id=settings.instance_id) + + @app.exception_handler(ApiError) + async def on_api_error(request: Request, exc: ApiError): + response = respond(exc.status, error_payload(request, exc.code, exc.message)) + for key, value in exc.headers.items(): + response.headers[key] = value + return response + + @app.exception_handler(StarletteHTTPException) + async def on_http(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return respond(404, error_payload(request, "not_found", "Объект не найден")) + return respond(400, error_payload(request, "invalid_request", "Некорректный запрос")) + + @app.exception_handler(Exception) + async def on_unexpected(request: Request, exc: Exception): + log.exception("unhandled API error") + return respond(500, error_payload(request, "internal_error", "Внутренняя ошибка")) + + @app.get("/health/live") + async def live(): + return respond(200, {"status": "ok"}) + + @app.get("/health/ready") + async def ready(request: Request): + store = runtime.store + if not ping() or store is None or not store.healthy(): + raise ApiError(503, "unavailable", "Зависимость недоступна") + return respond(200, {"status": "ok"}) + + @app.post("/api/v1/users") + async def register_user(request: Request): + data = await _json_object(request) + _reject_unknown(data, {"username", "display_name", "password"}) + username = _username(data.get("username")) + display_name = _bounded_text(data.get("display_name"), 1, 100, "display_name") + password = _password(data.get("password")) + user_id = uuid.uuid4() + password_hash = runtime.hasher.hash(password) + try: + with connection() as conn: + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (user_id, username, display_name, password_hash), + ).fetchone() + except UniqueViolation as exc: + raise ApiError(409, "conflict", "Имя пользователя занято") from exc + return respond(201, _user_json(row)) + + @app.post("/api/v1/auth/sessions") + async def create_session(request: Request): + username, password = _basic_credentials(request) + with connection() as conn: + row = conn.execute("SELECT * FROM users WHERE username = %s", (username,)).fetchone() + stored = row["password_hash"] if row else runtime.dummy_hash + try: + runtime.hasher.verify(stored, password) + valid = row is not None + except Exception: + valid = False + if not valid or row is None: + raise unauthorized("basic") + if len(password) < 12 or len(password) > 128: + raise unauthorized("basic") + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + expires = utcnow() + timedelta(seconds=runtime.settings.session_ttl_seconds) + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, row["id"], expires), + ) + payload = { + "token": token, + "token_type": "Bearer", + "expires_at": iso(expires), + "user": _user_json(row), + } + return respond(201, payload) + + @app.delete("/api/v1/auth/sessions/current") + async def revoke_session(request: Request): + with connection() as conn: + _user, token_hash = _bearer(request, conn) + conn.execute( + "UPDATE sessions SET revoked_at = clock_timestamp() WHERE token_hash = %s AND revoked_at IS NULL", + (token_hash,), + ) + return Response(status_code=204) + + @app.get("/api/v1/users/me") + async def current_user(request: Request): + with connection() as conn: + user, _token = _bearer(request, conn) + return respond(200, _user_json(user)) + + @app.post("/api/v1/chats") + async def create_chat(request: Request): + data = await _json_object(request) + _reject_unknown(data, {"title", "member_ids"}) + title = _bounded_text(data.get("title"), 1, 100, "title") + member_ids = _uuid_list(data.get("member_ids"), minimum=1, maximum=100) + with connection() as conn: + user, _token = _bearer(request, conn) + if user["id"] not in member_ids: + raise ApiError(400, "invalid_request", "Создатель должен входить в список участников") + found = conn.execute("SELECT id FROM users WHERE id = ANY(%s)", (member_ids,)).fetchall() + if len(found) != len(member_ids): + raise ApiError(400, "invalid_request", "Список участников содержит неизвестных пользователей") + chat_id = uuid.uuid4() + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, title), + ).fetchone() + for position, member_id in enumerate(member_ids): + conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + (chat_id, member_id, position), + ) + payload = _chat_json(conn, row) + return respond(201, payload) + + @app.get("/api/v1/chats") + async def list_chats(request: Request): + limit = _limit(request) + with connection() as conn: + user, _token = _bearer(request, conn) + cursor = _cursor(request, "chats", str(user["id"])) + rows = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats AS c + JOIN chat_members AS m ON m.chat_id = c.id AND m.user_id = %s + WHERE (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid)) + ORDER BY c.created_at, c.id + LIMIT %s + """, + (user["id"], cursor[0], cursor[0], cursor[1], limit + 1), + ).fetchall() + page, next_cursor = _page(rows, limit, lambda row: {"scope": "chats", "owner": str(user["id"]), "created_at": iso(row["created_at"]), "id": str(row["id"])}) + items = [_chat_json(conn, row) for row in page] + return respond(200, {"items": items, "next_cursor": next_cursor}) + + @app.get("/api/v1/chats/{chat_id}") + async def get_chat(chat_id: str, request: Request): + parsed = _parse_uuid(chat_id) + with connection() as conn: + user, _token = _bearer(request, conn) + row = _require_chat(conn, parsed, user["id"]) + payload = _chat_json(conn, row) + return respond(200, payload) + + @app.post("/api/v1/chats/{chat_id}/messages") + async def create_message(chat_id: str, request: Request): + parsed = _parse_uuid(chat_id) + data = await _json_object(request) + _reject_unknown(data, {"text", "attachment_ids"}) + if "text" not in data: + raise ApiError(400, "invalid_request", "Текст сообщения обязателен") + text = data["text"] + if not isinstance(text, str) or len(text) > 4000 or "\x00" in text: + raise ApiError(400, "invalid_request", "Некорректный текст сообщения") + if "attachment_ids" in data: + attachment_ids = _uuid_list(data["attachment_ids"], minimum=0, maximum=10) + else: + attachment_ids = [] + if text.strip() == "" and not attachment_ids: + raise ApiError(400, "invalid_request", "Пустое сообщение без вложений недопустимо") + key = _idempotency_key(request) + path = "/api/v1/chats/" + str(parsed) + "/messages" + digest = hashlib.sha256( + json.dumps({"attachment_ids": [str(item) for item in attachment_ids], "text": text}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + with connection() as conn: + user, _token = _bearer(request, conn) + _require_chat(conn, parsed, user["id"]) + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' AND path = %s AND idem_key = %s + AND created_at < clock_timestamp() - make_interval(secs => %s) + """, + (user["id"], path, key, runtime.settings.idempotency_ttl_seconds), + ) + inserted = conn.execute( + """ + INSERT INTO idempotency_keys (user_id, http_method, path, idem_key, request_hash) + VALUES (%s, 'POST', %s, %s, %s) + ON CONFLICT DO NOTHING + RETURNING idem_key + """, + (user["id"], path, key, digest), + ).fetchone() + if inserted is None: + existing = conn.execute( + """ + SELECT request_hash, response_json FROM idempotency_keys + WHERE user_id = %s AND http_method = 'POST' AND path = %s AND idem_key = %s + """, + (user["id"], path, key), + ).fetchone() + if existing is None or existing["response_json"] is None: + raise ApiError(503, "unavailable", "Повтор запроса ещё выполняется") + if existing["request_hash"] != digest: + raise ApiError(409, "conflict", "Ключ идемпотентности уже использован с другим сообщением") + return respond(201, existing["response_json"]) + _claim_attachments(conn, parsed, attachment_ids) + message = conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, body) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, body, created_at + """, + (uuid.uuid4(), parsed, user["id"], text), + ).fetchone() + for position, attachment_id in enumerate(attachment_ids): + conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message["id"], attachment_id, position), + ) + payload = _message_json(message, attachment_ids) + conn.execute( + """ + UPDATE idempotency_keys SET response_json = %s + WHERE user_id = %s AND http_method = 'POST' AND path = %s AND idem_key = %s + """, + (Jsonb(payload), user["id"], path, key), + ) + return respond(201, payload) + + @app.get("/api/v1/chats/{chat_id}/messages") + async def list_messages(chat_id: str, request: Request): + parsed = _parse_uuid(chat_id) + limit = _limit(request) + with connection() as conn: + user, _token = _bearer(request, conn) + _require_chat(conn, parsed, user["id"]) + cursor = _cursor(request, "messages", str(parsed)) + rows = conn.execute( + """ + SELECT id, chat_id, sender_id, body, created_at + FROM messages + WHERE chat_id = %s + AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid)) + ORDER BY created_at, id + LIMIT %s + """, + (parsed, cursor[0], cursor[0], cursor[1], limit + 1), + ).fetchall() + page, next_cursor = _page( + rows, + limit, + lambda row: {"scope": "messages", "owner": str(parsed), "created_at": iso(row["created_at"]), "id": str(row["id"])}, + ) + items = [] + for row in page: + ids = _attachment_ids(conn, row["id"]) + items.append(_message_json(row, ids)) + return respond(200, {"items": items, "next_cursor": next_cursor}) + + @app.post("/api/v1/chats/{chat_id}/attachments") + async def upload_attachment(chat_id: str, request: Request): + parsed = _parse_uuid(chat_id) + filename = request.query_params.get("filename") + _filename(filename) + declared = _media_type(request) + with connection() as conn: + user, _token = _bearer(request, conn) + _require_chat(conn, parsed, user["id"]) + if _queue_limit(conn) <= conn.execute( + "SELECT count(*) AS n FROM attachments WHERE status IN ('queued', 'processing')" + ).fetchone()["n"]: + raise ApiError(503, "unavailable", "Очередь обработки заполнена") + owner_id = user["id"] + data = await _read_body(request, runtime.settings.max_upload_bytes) + try: + content_type, _width, _height = inspect_upload( + data, declared, runtime.settings.max_upload_bytes, runtime.settings.max_image_pixels + ) + except ImageRejected as exc: + raise ApiError(exc.status, exc.code, exc.message) from exc + digest = hashlib.sha256(data).hexdigest() + attachment_id = uuid.uuid4() + staging_key = f"staging/{attachment_id}" + original_key = f"originals/{attachment_id}" + store = runtime.store + try: + store.put(staging_key, data, content_type) + except Exception as exc: + log.warning("staging upload failed") + raise ApiError(503, "unavailable", "Хранилище временно недоступно") from exc + try: + with connection() as conn: + _require_chat(conn, parsed, owner_id) + if _queue_limit(conn) <= conn.execute( + "SELECT count(*) AS n FROM attachments WHERE status IN ('queued', 'processing')" + ).fetchone()["n"]: + raise ApiError(503, "unavailable", "Очередь обработки заполнена") + row = conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, + pipeline_version, staging_key, original_key + ) VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', 1, %s, %s) + RETURNING * + """, + (attachment_id, parsed, owner_id, filename, content_type, len(data), digest, staging_key, original_key), + ).fetchone() + job_id = uuid.uuid4() + job = conn.execute( + """ + INSERT INTO jobs (job_id, attachment_id, pipeline_version, attempt, status) + VALUES (%s, %s, 1, 1, 'queued') + RETURNING job_id, attachment_id, pipeline_version, attempt, created_at + """, + (job_id, attachment_id), + ).fetchone() + add_outbox(conn, job["job_id"], attachment_id, 1, 1, job["created_at"]) + audit(conn, job_id, attachment_id, "accepted", {"pipeline_version": 1}) + snapshot = _attachment_json(row, []) + except Exception: + try: + store.delete(staging_key) + except Exception: + log.warning("failed to delete uncommitted staging object") + raise + try: + relay_tick() + except Exception: + log.warning("publish deferred to the outbox relay") + return respond(202, snapshot) + + @app.get("/api/v1/attachments/{attachment_id}") + async def get_attachment(attachment_id: str, request: Request): + parsed = _parse_uuid(attachment_id) + with connection() as conn: + user, _token = _bearer(request, conn) + row = _visible_attachment(conn, parsed, user["id"]) + payload = _attachment_json(row, _visible_variants(conn, row)) + return respond(200, payload) + + @app.get("/api/v1/attachments/{attachment_id}/content") + async def download_attachment(attachment_id: str, request: Request): + parsed = _parse_uuid(attachment_id) + variant = request.query_params.get("variant") or "original" + if variant not in VARIANTS: + raise ApiError(400, "invalid_request", "Неизвестный вариант") + with connection() as conn: + user, _token = _bearer(request, conn) + row = _visible_attachment(conn, parsed, user["id"]) + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово") + if variant == "original": + key = row["original_key"] + content_type = row["content_type"] + else: + found = conn.execute( + """ + SELECT object_key, content_type FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s AND name = %s + """, + (row["id"], row["ready_pipeline_version"], variant), + ).fetchone() + if found is None: + raise ApiError(404, "not_found", "Вариант не найден") + key, content_type = found["object_key"], found["content_type"] + try: + blob = runtime.store.get(key) + except Exception as exc: + log.warning("object download failed") + raise ApiError(503, "unavailable", "Хранилище временно недоступно") from exc + if blob is None: + raise ApiError(404, "not_found", "Вариант не найден") + return Response(content=blob, media_type=content_type) + + @app.post("/internal/control/outbox") + async def set_outbox(request: Request): + _operator(request) + data = await _json_object(request) + _reject_unknown(data, {"paused"}) + if not isinstance(data.get("paused"), bool): + raise ApiError(400, "invalid_request", "paused должен быть boolean") + with connection() as conn: + conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'outbox_paused'", (Jsonb(data["paused"]),)) + return respond(200, {"paused": data["paused"]}) + + @app.post("/internal/control/fault") + async def set_fault(request: Request): + _operator(request) + data = await _json_object(request) + _reject_unknown(data, {"mode", "remaining"}) + mode = data.get("mode") + allowed = {None, "transient", "permanent", "duplicate", "before_effect", "after_s3", "after_commit"} + if mode not in allowed: + raise ApiError(400, "invalid_request", "Неизвестный режим fault injection") + if mode is None: + value = {"mode": None, "armed": False} + else: + remaining = data.get("remaining", 1) + if not isinstance(remaining, int) or isinstance(remaining, bool) or remaining < 1: + raise ApiError(400, "invalid_request", "remaining должен быть положительным целым") + value = {"mode": mode, "armed": True, "remaining": remaining} + with connection() as conn: + conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'worker_fault'", (Jsonb(value),)) + return respond(200, value) + + @app.post("/internal/control/backpressure") + async def set_backpressure(request: Request): + _operator(request) + data = await _json_object(request) + _reject_unknown(data, {"max_queued_jobs"}) + value = data.get("max_queued_jobs") + if value is not None and (not isinstance(value, int) or isinstance(value, bool) or value < 0): + raise ApiError(400, "invalid_request", "max_queued_jobs должен быть неотрицательным целым или null") + with connection() as conn: + if value is None: + conn.execute("UPDATE runtime_control SET value = 'null'::jsonb WHERE key = 'max_queued_jobs'") + else: + conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'max_queued_jobs'", (Jsonb(value),)) + return respond(200, {"max_queued_jobs": value}) + + @app.get("/internal/metrics") + async def metrics(request: Request): + _operator(request) + with connection() as conn: + attachments = _counts(conn, "attachments") + jobs = _counts(conn, "jobs") + unpublished = conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL").fetchone()["n"] + failed = conn.execute("SELECT count(*) AS n FROM failed_jobs").fetchone()["n"] + paused = conn.execute("SELECT value FROM runtime_control WHERE key = 'outbox_paused'").fetchone()["value"] + return respond(200, { + "attachments": attachments, + "jobs": jobs, + "outbox_unpublished": unpublished, + "failed_registry": failed, + "outbox_paused": paused is True, + }) + + @app.get("/internal/attachments/{attachment_id}") + async def internal_attachment(attachment_id: str, request: Request): + _operator(request) + parsed = _parse_uuid(attachment_id) + with connection() as conn: + row = conn.execute("SELECT * FROM attachments WHERE id = %s", (parsed,)).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + jobs = conn.execute("SELECT * FROM jobs WHERE attachment_id = %s ORDER BY pipeline_version", (parsed,)).fetchall() + variants = conn.execute( + "SELECT * FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name", + (parsed,), + ).fetchall() + audits = conn.execute( + "SELECT id, action, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id", + (parsed,), + ).fetchall() + job_ids = [job["job_id"] for job in jobs] + messages = [] + if job_ids: + messages = conn.execute( + "SELECT payload FROM outbox WHERE job_id = ANY(%s) ORDER BY id", + (job_ids,), + ).fetchall() + return respond(200, { + "attachment": _internal_attachment(row), + "jobs": [_internal_job(job) for job in jobs], + "variants": [_internal_variant(item) for item in variants], + "messages": [item["payload"] for item in messages], + "audits": [{"id": item["id"], "action": item["action"], "detail": item["detail"], "created_at": iso(item["created_at"])} for item in audits], + }) + + @app.post("/internal/attachments/{attachment_id}/replay") + async def replay_attachment(attachment_id: str, request: Request): + _operator(request) + parsed = _parse_uuid(attachment_id) + with connection() as conn: + row = _lock_attachment(conn, parsed) + job = conn.execute( + "SELECT * FROM jobs WHERE attachment_id = %s AND pipeline_version = %s FOR UPDATE", + (parsed, row["pipeline_version"]), + ).fetchone() + if job is None: + raise ApiError(404, "not_found", "Задание не найдено") + if row["status"] == "ready" and row["ready_pipeline_version"] == row["pipeline_version"]: + audit(conn, job["job_id"], parsed, "replay_rejected", {"reason": "already_ready"}) + payload = {"replayed": False, "reason": "already_ready", "job_id": str(job["job_id"]), "pipeline_version": row["pipeline_version"]} + elif job["status"] == "processing" and job["lease_until"] is not None and conn.execute("SELECT %s > clock_timestamp() AS active", (job["lease_until"],)).fetchone()["active"]: + payload = {"replayed": False, "reason": "in_progress", "job_id": str(job["job_id"]), "pipeline_version": row["pipeline_version"]} + else: + conn.execute( + """ + UPDATE jobs + SET status = 'queued', attempt = 1, error_code = NULL, error_class = NULL, last_error = NULL, + next_attempt_at = NULL, lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp() + WHERE job_id = %s + """, + (job["job_id"],), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = %s AND pipeline_version = %s + """, + (parsed, row["pipeline_version"]), + ) + conn.execute( + "UPDATE failed_jobs SET replayed_at = clock_timestamp() WHERE job_id = %s AND replayed_at IS NULL", + (job["job_id"],), + ) + add_outbox(conn, job["job_id"], parsed, row["pipeline_version"], 1, job["created_at"]) + audit(conn, job["job_id"], parsed, "replay", {"pipeline_version": row["pipeline_version"]}) + payload = {"replayed": True, "reason": "queued", "job_id": str(job["job_id"]), "pipeline_version": row["pipeline_version"]} + try: + relay_tick() + except Exception: + log.warning("replay publish deferred") + return respond(200, payload) + + @app.post("/internal/attachments/{attachment_id}/reprocess") + async def reprocess_attachment(attachment_id: str, request: Request): + _operator(request) + parsed = _parse_uuid(attachment_id) + data = await _json_object(request) + _reject_unknown(data, {"pipeline_version"}) + version = data.get("pipeline_version") + if version not in {1, 2}: + raise ApiError(400, "invalid_request", "Поддерживаются pipeline_version 1 и 2") + with connection() as conn: + row = _lock_attachment(conn, parsed) + if version < int(row["pipeline_version"]): + raise ApiError(409, "conflict", "Нельзя заменить результат более старой версией pipeline") + if version == int(row["pipeline_version"]) and row["status"] == "ready" and row["ready_pipeline_version"] == version: + job = conn.execute( + "SELECT job_id FROM jobs WHERE attachment_id = %s AND pipeline_version = %s", + (parsed, version), + ).fetchone() + audit(conn, job["job_id"] if job else None, parsed, "reprocess_rejected", {"reason": "already_ready", "pipeline_version": version}) + payload = {"replayed": False, "reason": "already_ready", "job_id": str(job["job_id"]) if job else None, "pipeline_version": version} + else: + job = conn.execute( + "SELECT * FROM jobs WHERE attachment_id = %s AND pipeline_version = %s FOR UPDATE", + (parsed, version), + ).fetchone() + if job is None: + job_id = uuid.uuid4() + job = conn.execute( + """ + INSERT INTO jobs (job_id, attachment_id, pipeline_version, attempt, status) + VALUES (%s, %s, %s, 1, 'queued') + RETURNING * + """, + (job_id, parsed, version), + ).fetchone() + else: + conn.execute( + """ + UPDATE jobs + SET status = 'queued', attempt = 1, error_code = NULL, error_class = NULL, last_error = NULL, + next_attempt_at = NULL, lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp() + WHERE job_id = %s + """, + (job["job_id"],), + ) + conn.execute( + """ + UPDATE attachments + SET pipeline_version = %s, status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = %s + """, + (version, parsed), + ) + add_outbox(conn, job["job_id"], parsed, version, 1, job["created_at"]) + audit(conn, job["job_id"], parsed, "reprocess", {"pipeline_version": version}) + payload = {"replayed": True, "reason": "queued", "job_id": str(job["job_id"]), "pipeline_version": version} + try: + relay_tick() + except Exception: + log.warning("reprocess publish deferred") + return respond(200, payload) + + @app.post("/internal/jobs/{job_id}/redeliver") + async def redeliver_job(job_id: str, request: Request): + """Inject one extra broker delivery of the same logical job without creating a new job.""" + _operator(request) + parsed = _parse_uuid(job_id) + with connection() as conn: + job = conn.execute("SELECT * FROM jobs WHERE job_id = %s", (parsed,)).fetchone() + if job is None: + raise ApiError(404, "not_found", "Задание не найдено") + body = { + "schema_version": 1, + "job_id": str(job["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(job["attachment_id"]), + "pipeline_version": int(job["pipeline_version"]), + "attempt": int(job["attempt"]), + "created_at": iso(job["created_at"]), + } + audit(conn, job["job_id"], job["attachment_id"], "redeliver_requested", {"attempt": int(job["attempt"])}) + if not runtime.publisher.publish(body): + raise ApiError(503, "unavailable", "Брокер не подтвердил повторную доставку") + return respond(200, {"enqueued": True, "job_id": str(parsed), "attempt": body["attempt"], "pipeline_version": body["pipeline_version"]}) + + @app.get("/internal/failed") + async def list_failed(request: Request): + _operator(request) + with connection() as conn: + rows = conn.execute( + """ + SELECT id, job_id, attachment_id, pipeline_version, attempt, error_code, error_class, created_at, replayed_at + FROM failed_jobs ORDER BY id DESC LIMIT 100 + """ + ).fetchall() + return respond(200, {"items": [ + { + "id": row["id"], + "job_id": str(row["job_id"]), + "attachment_id": str(row["attachment_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "error_code": row["error_code"], + "error_class": row["error_class"], + "created_at": iso(row["created_at"]), + "replayed_at": iso(row["replayed_at"]) if row["replayed_at"] else None, + } + for row in rows + ]}) + + return app + + +async def _json_object(request: Request) -> dict: + raw = await request.body() + content_type = request.headers.get("content-type", "") + if content_type.split(";", 1)[0].strip().lower() != "application/json": + raise ApiError(400, "invalid_request", "Ожидался application/json") + try: + value = json.loads(raw.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный JSON") from exc + if not isinstance(value, dict): + raise ApiError(400, "invalid_request", "Ожидался JSON-объект") + return value + + +def _reject_unknown(data: dict, allowed: set[str]) -> None: + if set(data) - allowed: + raise ApiError(400, "invalid_request", "Запрос содержит неизвестные поля") + + +def _bounded_text(value, minimum: int, maximum: int, field: str) -> str: + if not isinstance(value, str) or "\x00" in value or len(value) < minimum or len(value) > maximum or value.strip() == "": + raise ApiError(400, "invalid_request", "Некорректное поле " + field) + return value + + +def _username(value) -> str: + if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None: + raise ApiError(400, "invalid_request", "Некорректное имя пользователя") + return value + + +def _password(value) -> str: + if not isinstance(value, str) or "\x00" in value or len(value) < 12 or len(value) > 128: + raise ApiError(400, "invalid_request", "Некорректный пароль") + return value + + +def _parse_uuid(value: str) -> uuid.UUID: + try: + parsed = uuid.UUID(str(value)) + except (ValueError, AttributeError, TypeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный UUID") from exc + return parsed + + +def _uuid_list(value, minimum: int, maximum: int) -> list[uuid.UUID]: + if not isinstance(value, list) or len(value) < minimum or len(value) > maximum: + raise ApiError(400, "invalid_request", "Некорректный список идентификаторов") + parsed = [] + for item in value: + if not isinstance(item, str): + raise ApiError(400, "invalid_request", "Некорректный список идентификаторов") + parsed.append(_parse_uuid(item)) + if len(set(parsed)) != len(parsed): + raise ApiError(400, "invalid_request", "Список идентификаторов содержит дубликаты") + return parsed + + +def _idempotency_key(request: Request) -> str: + key = request.headers.get("idempotency-key", "") + if not IDEMPOTENCY_RE.fullmatch(key): + raise ApiError(400, "invalid_request", "Некорректный Idempotency-Key") + return key + + +def _basic_credentials(request: Request) -> tuple[str, str]: + header = request.headers.get("authorization", "") + if not header.lower().startswith("basic "): + raise unauthorized("basic") + try: + decoded = base64.b64decode(header.split(" ", 1)[1].strip(), validate=True).decode("utf-8") + except Exception as exc: + raise unauthorized("basic") from exc + if ":" not in decoded: + raise unauthorized("basic") + username, password = decoded.split(":", 1) + return username, password + + +def _bearer(request: Request, conn): + header = request.headers.get("authorization", "") + if not header.startswith("Bearer ") and not header.lower().startswith("bearer "): + raise unauthorized("bearer") + token = header.split(" ", 1)[1].strip() + if token == "": + raise unauthorized("bearer") + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + row = conn.execute( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions AS s + JOIN users AS u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp() + """, + (token_hash,), + ).fetchone() + if row is None: + raise unauthorized("bearer") + return row, token_hash + + +def _user_json(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso(row["created_at"]), + } + + +def _chat_json(conn, row) -> dict: + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY position, user_id", + (row["id"],), + ).fetchall() + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(item["user_id"]) for item in members], + "created_at": iso(row["created_at"]), + } + + +def _require_chat(conn, chat_id: uuid.UUID, user_id): + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats AS c + JOIN chat_members AS m ON m.chat_id = c.id AND m.user_id = %s + WHERE c.id = %s + """, + (user_id, chat_id), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return row + + +def _message_json(row, attachment_ids) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["body"], + "created_at": iso(row["created_at"]), + "attachment_ids": [str(item) for item in attachment_ids], + } + + +def _attachment_ids(conn, message_id) -> list: + rows = conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position", + (message_id,), + ).fetchall() + return [row["attachment_id"] for row in rows] + + +def _claim_attachments(conn, chat_id, attachment_ids: list) -> None: + for attachment_id in attachment_ids: + row = conn.execute("SELECT chat_id, status FROM attachments WHERE id = %s", (attachment_id,)).fetchone() + if row is None or row["chat_id"] != chat_id: + raise ApiError(404, "not_found", "Объект не найден") + if row["status"] == "failed": + raise ApiError(409, "conflict", "Нельзя прикрепить вложение с ошибкой обработки") + + +def _limit(request: Request) -> int: + raw = request.query_params.get("limit") + if raw is None: + return 50 + if not re.fullmatch(r"[0-9]+", raw): + raise ApiError(400, "invalid_request", "Некорректный limit") + value = int(raw) + if value < 1 or value > 100: + raise ApiError(400, "invalid_request", "Некорректный limit") + return value + + +def _cursor(request: Request, scope: str, owner: str): + raw = request.query_params.get("cursor") + if raw is None: + return None, None + if raw == "": + raise ApiError(400, "invalid_request", "Некорректный cursor") + try: + padding = "=" * (-len(raw) % 4) + payload = json.loads(base64.urlsafe_b64decode(raw + padding).decode("utf-8")) + except Exception as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + if not isinstance(payload, dict) or payload.get("scope") != scope or payload.get("owner") != owner: + raise ApiError(400, "invalid_request", "Курсор не относится к этому запросу") + try: + created_at = str(payload["created_at"]) + if not created_at.endswith("Z"): + raise ValueError("timestamp") + from datetime import datetime + + created = datetime.fromisoformat(created_at[:-1] + "+00:00") + identifier = str(uuid.UUID(str(payload["id"]))) + except (KeyError, ValueError, TypeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + return created, identifier + + +def _page(rows, limit: int, cursor_of): + if len(rows) <= limit: + return rows, None + visible = rows[:limit] + payload = cursor_of(visible[-1]) + token = base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8")).decode("ascii").rstrip("=") + return visible, token + + +def _filename(value: str | None) -> None: + if not isinstance(value, str) or value == "" or len(value) > 255: + raise ApiError(400, "invalid_request", "Некорректное имя файла") + if any(character in value for character in "/\\") or any(ord(character) < 32 for character in value): + raise ApiError(400, "invalid_request", "Некорректное имя файла") + + +def _media_type(request: Request) -> str: + media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if media not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Допустимы только PNG и JPEG") + return media + + +async def _read_body(request: Request, limit: int) -> bytes: + declared = request.headers.get("content-length") + if declared is not None: + if not declared.isdigit() or int(declared) > limit: + raise ApiError(413, "payload_too_large", "Превышен размер файла") + chunks = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > limit: + raise ApiError(413, "payload_too_large", "Превышен размер файла") + chunks.append(chunk) + return b"".join(chunks) + + +def _queue_limit(conn) -> int: + row = conn.execute("SELECT value FROM runtime_control WHERE key = 'max_queued_jobs'").fetchone() + value = row["value"] if row else None + if isinstance(value, int) and not isinstance(value, bool): + return value + return runtime.settings.max_queued_jobs + + +def _visible_attachment(conn, attachment_id, user_id): + row = conn.execute( + """ + SELECT a.* + FROM attachments AS a + JOIN chat_members AS m ON m.chat_id = a.chat_id AND m.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return row + + +def _visible_variants(conn, attachment) -> list[dict]: + if attachment["status"] != "ready" or attachment["ready_pipeline_version"] is None: + return [] + rows = conn.execute( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY name + """, + (attachment["id"], attachment["ready_pipeline_version"]), + ).fetchall() + return [ + { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + for row in rows + ] + + +def _attachment_json(row, variants: list[dict]) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": variants, + "error_code": row["error_code"] if row["status"] == "failed" else None, + "created_at": iso(row["created_at"]), + "updated_at": iso(row["updated_at"]), + } + + +def _operator(request: Request) -> None: + expected = runtime.settings.operator_token + presented = request.headers.get("x-operator-token", "") + if expected == "" or not hmac.compare_digest( + hashlib.sha256(presented.encode("utf-8")).digest(), + hashlib.sha256(expected.encode("utf-8")).digest(), + ): + raise unauthorized("bearer") + + +def _lock_attachment(conn, attachment_id): + row = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,)).fetchone() + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return row + + +def _counts(conn, table: str) -> dict: + rows = conn.execute(f"SELECT status, count(*) AS n FROM {table} GROUP BY status").fetchall() + return {row["status"]: row["n"] for row in rows} + + +def _internal_attachment(row) -> dict: + return { + "id": str(row["id"]), + "status": row["status"], + "error_code": row["error_code"], + "pipeline_version": row["pipeline_version"], + "ready_pipeline_version": row["ready_pipeline_version"], + "sha256": row["sha256"], + "size_bytes": row["size_bytes"], + "staging_deleted": row["staging_deleted_at"] is not None, + "created_at": iso(row["created_at"]), + "updated_at": iso(row["updated_at"]), + } + + +def _internal_job(row) -> dict: + return { + "job_id": str(row["job_id"]), + "pipeline_version": row["pipeline_version"], + "attempt": row["attempt"], + "status": row["status"], + "error_code": row["error_code"], + "error_class": row["error_class"], + "created_at": iso(row["created_at"]), + } + + +def _internal_variant(row) -> dict: + return { + "pipeline_version": row["pipeline_version"], + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..732bd51e --- /dev/null +++ b/app/images.py @@ -0,0 +1,182 @@ +"""Image inspection and variant rendering. + +Thumbnail: JPEG, fit inside 256×256, no upscale, EXIF orientation applied, alpha on white. +Crop: JPEG, centered square resized to 128×128; upscale is allowed. +Optimized: WebP, fit inside 1280×1280, no upscale. Quality and metadata policy live in +pipeline_params and are recorded in the report. +""" +from __future__ import annotations + +import io +import math +from dataclasses import dataclass + +from PIL import Image, ImageOps, UnidentifiedImageError + +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} + + +class ImageRejected(Exception): + def __init__(self, status: int, code: str, message: str): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +@dataclass(frozen=True) +class PipelineParams: + version: int + jpeg_quality: int + crop_quality: int + webp_quality: int + webp_method: int + + +def pipeline_params(version: int) -> PipelineParams: + """Version 1 is the course pipeline. Version 2 changes only quality, so a stale + writer can be detected by checksum without changing geometry.""" + if version == 1: + return PipelineParams(1, jpeg_quality=85, crop_quality=85, webp_quality=80, webp_method=6) + if version == 2: + return PipelineParams(2, jpeg_quality=60, crop_quality=70, webp_quality=50, webp_method=4) + raise ImageRejected(400, "invalid_request", "Неизвестная версия pipeline") + + +def _png_size(data: bytes) -> tuple[int, int]: + if len(data) < 24 or data[:8] != PNG_MAGIC or data[12:16] != b"IHDR": + raise ImageRejected(415, "unsupported_media_type", "Файл не является корректным PNG") + if b"IEND" not in data: + raise ImageRejected(415, "unsupported_media_type", "PNG повреждён") + width = int.from_bytes(data[16:20], "big") + height = int.from_bytes(data[20:24], "big") + return width, height + + +def _jpeg_size(data: bytes) -> tuple[int, int]: + if len(data) < 4 or data[:2] != b"\xff\xd8": + raise ImageRejected(415, "unsupported_media_type", "Файл не является корректным JPEG") + index = 2 + while index < len(data): + if data[index] != 0xFF: + raise ImageRejected(415, "unsupported_media_type", "JPEG повреждён") + while index < len(data) and data[index] == 0xFF: + index += 1 + if index >= len(data): + break + marker = data[index] + index += 1 + if marker in {0xD8, 0xD9, 0xDA} or marker == 0x01 or 0xD0 <= marker <= 0xD7: + if marker in {0xD9, 0xDA}: + break + continue + if index + 2 > len(data): + break + length = int.from_bytes(data[index:index + 2], "big") + if length < 2 or index + length > len(data): + raise ImageRejected(415, "unsupported_media_type", "JPEG повреждён") + if marker in _SOF: + if length < 8: + raise ImageRejected(415, "unsupported_media_type", "JPEG повреждён") + height = int.from_bytes(data[index + 3:index + 5], "big") + width = int.from_bytes(data[index + 5:index + 7], "big") + return width, height + index += length + raise ImageRejected(415, "unsupported_media_type", "Не удалось прочитать размеры JPEG") + + +def inspect_upload(data: bytes, declared: str, max_bytes: int, max_pixels: int) -> tuple[str, int, int]: + """Validate a client upload from headers and the container, before a full decode.""" + if len(data) > max_bytes: + raise ImageRejected(413, "payload_too_large", "Превышен размер файла") + if not data: + raise ImageRejected(415, "unsupported_media_type", "Пустое изображение") + if declared == "image/png": + if not data.startswith(PNG_MAGIC): + raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует PNG") + width, height = _png_size(data) + content_type = "image/png" + elif declared == "image/jpeg": + if not data.startswith(b"\xff\xd8\xff"): + raise ImageRejected(415, "unsupported_media_type", "Содержимое не соответствует JPEG") + width, height = _jpeg_size(data) + content_type = "image/jpeg" + else: + raise ImageRejected(415, "unsupported_media_type", "Допустимы только PNG и JPEG") + if width < 1 or height < 1: + raise ImageRejected(415, "unsupported_media_type", "Некорректные размеры изображения") + if width > max_pixels or height > max_pixels or width * height > max_pixels: + raise ImageRejected(413, "payload_too_large", "Превышено число пикселей") + return content_type, width, height + + +def fit_inside(width: int, height: int, box: int, allow_upscale: bool) -> tuple[int, int]: + scale = min(box / width, box / height) + if not allow_upscale: + scale = min(scale, 1.0) + return max(1, math.floor(width * scale)), max(1, math.floor(height * scale)) + + +def _to_rgb_white(image: Image.Image) -> Image.Image: + if image.mode == "RGB": + return image + if image.mode == "P" and "transparency" in image.info: + image = image.convert("RGBA") + if image.mode in {"RGBA", "LA"}: + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + + +def decode_image(data: bytes, max_pixels: int) -> Image.Image: + Image.MAX_IMAGE_PIXELS = max_pixels + try: + image = Image.open(io.BytesIO(data)) + width, height = image.size + if width < 1 or height < 1 or width * height > max_pixels: + raise ImageRejected(413, "payload_too_large", "Превышено число пикселей") + image = ImageOps.exif_transpose(image) + width, height = image.size + if width * height > max_pixels: + raise ImageRejected(413, "payload_too_large", "Превышено число пикселей") + image.load() + except ImageRejected: + raise + except (UnidentifiedImageError, Image.DecompressionBombError, OSError, ValueError) as exc: + raise ImageRejected(415, "unsupported_media_type", "Изображение повреждено") from exc + return _to_rgb_white(image) + + +def _jpeg_bytes(image: Image.Image, quality: int) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=quality, optimize=True, progressive=False) + return buffer.getvalue() + + +def _webp_bytes(image: Image.Image, quality: int, method: int) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="WEBP", quality=quality, method=method, lossless=False, exact=False) + return buffer.getvalue() + + +def render_variants(source: Image.Image, params: PipelineParams) -> dict[str, dict]: + width, height = source.size + thumb_w, thumb_h = fit_inside(width, height, 256, allow_upscale=False) + thumbnail = source if (thumb_w, thumb_h) == (width, height) else source.resize((thumb_w, thumb_h), Image.Resampling.LANCZOS) + side = min(width, height) + left = (width - side) // 2 + top = (height - side) // 2 + crop = source.crop((left, top, left + side, top + side)).resize((128, 128), Image.Resampling.LANCZOS) + opt_w, opt_h = fit_inside(width, height, 1280, allow_upscale=False) + optimized = source if (opt_w, opt_h) == (width, height) else source.resize((opt_w, opt_h), Image.Resampling.LANCZOS) + thumb_bytes = _jpeg_bytes(thumbnail, params.jpeg_quality) + crop_bytes = _jpeg_bytes(crop, params.crop_quality) + webp_bytes = _webp_bytes(optimized, params.webp_quality, params.webp_method) + return { + "thumbnail": {"bytes": thumb_bytes, "content_type": "image/jpeg", "width": thumb_w, "height": thumb_h}, + "crop": {"bytes": crop_bytes, "content_type": "image/jpeg", "width": 128, "height": 128}, + "optimized": {"bytes": webp_bytes, "content_type": "image/webp", "width": opt_w, "height": opt_h}, + } diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..c355401e --- /dev/null +++ b/app/main.py @@ -0,0 +1,35 @@ +"""API process entrypoint.""" +from __future__ import annotations + +import logging + +import uvicorn + +from app.broker import Publisher +from app.db import migrate +from app.httpapi import create_app +from app.outbox import start_relay +from app.runtime import init +from app.s3store import ObjectStore +from app.settings import load + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + settings = load() + migrate(settings.database_url) + store = ObjectStore( + settings.s3_endpoint, + settings.s3_region, + settings.s3_bucket, + settings.s3_access_key_id, + settings.s3_secret_access_key, + ) + store.ensure_bucket() + init(settings, store, Publisher(settings.broker_url)) + start_relay() + uvicorn.run(create_app(), host=settings.host, port=settings.port, log_level="info", access_log=False) + + +if __name__ == "__main__": + main() diff --git a/app/outbox.py b/app/outbox.py new file mode 100644 index 00000000..a176d448 --- /dev/null +++ b/app/outbox.py @@ -0,0 +1,114 @@ +"""Transactional outbox and operator controls shared by the API and the worker.""" +from __future__ import annotations + +import logging +import uuid + +from psycopg.types.json import Jsonb + +from app import runtime +from app.broker import ROUTING_DLQ +from app.db import connection +from app.timeutil import iso + +log = logging.getLogger("messenger.outbox") + + +def payload_for(job_id, attachment_id, pipeline_version: int, attempt: int, created_at) -> dict: + created = created_at if isinstance(created_at, str) else iso(created_at) + return { + "schema_version": 1, + "job_id": str(job_id), + "type": "attachment.process.v1", + "attachment_id": str(attachment_id), + "pipeline_version": int(pipeline_version), + "attempt": int(attempt), + "created_at": created, + } + + +def add_outbox(conn, job_id, attachment_id, pipeline_version: int, attempt: int, created_at) -> dict: + body = payload_for(job_id, attachment_id, pipeline_version, attempt, created_at) + conn.execute( + "INSERT INTO outbox (job_id, payload) VALUES (%s, %s)", + (job_id, Jsonb(body)), + ) + return body + + +def audit(conn, job_id, attachment_id, action: str, detail: dict) -> None: + conn.execute( + "INSERT INTO job_audit (job_id, attachment_id, action, detail) VALUES (%s, %s, %s, %s)", + (job_id, attachment_id, action, Jsonb(detail)), + ) + + +def outbox_paused(conn) -> bool: + row = conn.execute("SELECT value FROM runtime_control WHERE key='outbox_paused'").fetchone() + return bool(row and row["value"] is True) + + +def relay_tick(limit: int = 10) -> int: + """Publish committed outbox rows. A crash after COMMIT and before confirm leaves the row unpublished.""" + settings, _store, publisher = runtime.require() + published = 0 + with connection() as conn: + if outbox_paused(conn): + return 0 + conn.execute( + """ + UPDATE outbox AS o + SET published_at = clock_timestamp() + FROM jobs AS j + WHERE o.job_id = j.job_id + AND o.published_at IS NULL + AND j.status IN ('ready', 'failed', 'superseded') + """ + ) + rows = conn.execute( + """ + SELECT o.id, o.payload + FROM outbox AS o + JOIN jobs AS j ON j.job_id = o.job_id + WHERE o.published_at IS NULL + AND j.status = 'queued' + AND (j.next_attempt_at IS NULL OR j.next_attempt_at <= clock_timestamp()) + ORDER BY o.id + FOR UPDATE OF o SKIP LOCKED + LIMIT %s + """, + (limit,), + ).fetchall() + for row in rows: + if publisher.publish(row["payload"]): + conn.execute("UPDATE outbox SET published_at = clock_timestamp() WHERE id = %s", (row["id"],)) + published += 1 + else: + break + if published: + log.info("published %s job message(s)", published) + return published + + +def publish_dlq(body: dict) -> None: + _settings, _store, publisher = runtime.require() + publisher.publish(body, routing_key=ROUTING_DLQ) + + +def start_relay() -> None: + import threading + import time + + def loop() -> None: + while True: + try: + relay_tick() + except Exception: + log.warning("outbox relay iteration failed", exc_info=True) + time.sleep(0.4) + + threading.Thread(target=loop, name="outbox-relay", daemon=True).start() + + +def new_job_id() -> uuid.UUID: + return uuid.uuid4() diff --git a/app/pipeline.py b/app/pipeline.py new file mode 100644 index 00000000..9b859eda --- /dev/null +++ b/app/pipeline.py @@ -0,0 +1,592 @@ +"""Idempotent attachment pipeline. + +The effect key is (attachment_id, pipeline_version, variant). Object keys include that +identity, database rows use it as a primary key, and a completed job is a no-op. +Delivery is at least once; this module does not claim exactly-once delivery. +""" +from __future__ import annotations + +import hashlib +import logging +import os +import threading +import time +import uuid + +from psycopg.types.json import Jsonb + +from app import runtime +from app.db import connection +from app.images import ImageRejected, decode_image, pipeline_params, render_variants +from app.outbox import add_outbox, audit, publish_dlq + +log = logging.getLogger("messenger.pipeline") + +VARIANT_NAMES = ("thumbnail", "crop", "optimized") + + +class TransientJobError(Exception): + def __init__(self, message: str): + super().__init__(message) + self.message = message + + +class PermanentJobError(Exception): + def __init__(self, code: str, message: str): + super().__init__(message) + self.code = code + self.message = message + + +def _safe(text: str) -> str: + cleaned = " ".join(text.split()) + return cleaned[:180] or "processing failed" + + +def _parse_message(body: dict) -> dict: + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if not isinstance(body, dict) or set(body) - required - {"traceparent", "tracestate"}: + raise PermanentJobError("processing_failed", "invalid job message") + if any(key not in body for key in required): + raise PermanentJobError("processing_failed", "invalid job message") + if body["schema_version"] != 1 or body["type"] != "attachment.process.v1": + raise PermanentJobError("processing_failed", "invalid job message") + try: + return { + "job_id": uuid.UUID(str(body["job_id"])), + "attachment_id": uuid.UUID(str(body["attachment_id"])), + "pipeline_version": int(body["pipeline_version"]), + "attempt": int(body["attempt"]), + "created_at": str(body["created_at"]), + } + except (ValueError, TypeError) as exc: + raise PermanentJobError("processing_failed", "invalid job message") from exc + + +def _read_fault(conn) -> dict: + row = conn.execute("SELECT value FROM runtime_control WHERE key = 'worker_fault' FOR UPDATE").fetchone() + value = row["value"] if row else {} + return dict(value) if isinstance(value, dict) else {} + + +def _write_fault(conn, value: dict) -> None: + conn.execute("UPDATE runtime_control SET value = %s WHERE key = 'worker_fault'", (Jsonb(value),)) + + +def _consume_counted_fault() -> str | None: + """Disarm one-shot and counted faults. Crash faults stay armed until their injection point.""" + with connection() as conn: + value = _read_fault(conn) + if not value.get("armed"): + return None + mode = value.get("mode") + if mode == "transient": + remaining = int(value.get("remaining") or 1) - 1 + if remaining <= 0: + _write_fault(conn, {"mode": None, "armed": False}) + else: + _write_fault(conn, {"mode": "transient", "armed": True, "remaining": remaining}) + return "transient" + if mode in {"permanent", "duplicate"}: + _write_fault(conn, {"mode": None, "armed": False}) + return mode + if mode in {"before_effect", "after_s3", "after_commit"}: + return mode + return None + + +def _crash(mode: str, job_id, attachment_id) -> None: + with connection() as conn: + value = _read_fault(conn) + if value.get("armed") and value.get("mode") == mode: + _write_fault(conn, {"mode": None, "armed": False}) + audit(conn, job_id, attachment_id, "fault_" + mode, {"mode": mode}) + if mode in {"before_effect", "after_s3"}: + conn.execute( + """ + UPDATE jobs + SET status = 'queued', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp() + WHERE job_id = %s AND status = 'processing' + """, + (job_id,), + ) + conn.execute( + """ + UPDATE attachments AS a + SET status = 'queued', updated_at = clock_timestamp() + FROM jobs AS j + WHERE j.job_id = %s AND a.id = j.attachment_id + AND a.pipeline_version = j.pipeline_version AND a.status = 'processing' + """, + (job_id,), + ) + log.warning("fault injection exit: %s", mode) + os._exit(75) + + +def _load(conn, message: dict): + job = conn.execute("SELECT * FROM jobs WHERE job_id = %s FOR UPDATE", (message["job_id"],)).fetchone() + attachment = None + if job is not None: + attachment = conn.execute("SELECT * FROM attachments WHERE id = %s FOR UPDATE", (job["attachment_id"],)).fetchone() + return job, attachment + + +def _finish_noop(conn, job, attachment, action: str, redelivered: bool) -> None: + audit(conn, job["job_id"] if job else None, attachment["id"] if attachment else None, action, {"redelivered": redelivered}) + + +def _supersede(conn, job, attachment, redelivered: bool) -> None: + conn.execute( + """ + UPDATE jobs + SET status = 'superseded', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp() + WHERE job_id = %s AND status IN ('queued', 'processing') + """, + (job["job_id"],), + ) + _finish_noop(conn, job, attachment, "superseded", redelivered) + + +def _acquire(conn, job, worker_id: str, timeout_seconds: int): + return conn.execute( + """ + UPDATE jobs + SET status = 'processing', + lease_owner = %s, + lease_until = clock_timestamp() + make_interval(secs => %s), + updated_at = clock_timestamp() + WHERE job_id = %s + AND status IN ('queued', 'processing') + AND (lease_until IS NULL OR lease_until < clock_timestamp() OR lease_owner = %s) + RETURNING job_id + """, + (worker_id, timeout_seconds, job["job_id"], worker_id), + ).fetchone() + + +def _schedule_retry(conn, job, attachment, message: str) -> None: + settings, _store, _publisher = runtime.require() + attempt = int(job["attempt"]) + if attempt >= settings.job_max_attempts: + _fail(conn, job, attachment, "retry_exhausted", "transient", "retry budget exhausted") + return + import random + + delay = min(settings.job_backoff_cap_seconds, settings.job_backoff_base_seconds * (2 ** (attempt - 1))) + delay = delay * (0.5 + random.random()) + next_attempt = attempt + 1 + conn.execute( + """ + UPDATE jobs + SET status = 'queued', attempt = %s, next_attempt_at = clock_timestamp() + make_interval(secs => %s), + lease_until = NULL, lease_owner = NULL, error_class = 'transient', error_code = 'processing_failed', + last_error = %s, updated_at = clock_timestamp() + WHERE job_id = %s + """, + (next_attempt, delay, _safe(message), job["job_id"]), + ) + conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() + WHERE id = %s AND pipeline_version = %s AND status <> 'ready' + """, + (attachment["id"], job["pipeline_version"]), + ) + add_outbox(conn, job["job_id"], attachment["id"], job["pipeline_version"], next_attempt, job["created_at"]) + audit(conn, job["job_id"], attachment["id"], "retry", {"attempt": next_attempt, "delay_seconds": round(delay, 3)}) + + +def _fail(conn, job, attachment, code: str, error_class: str, message: str) -> None: + conn.execute( + """ + UPDATE jobs + SET status = 'failed', error_code = %s, error_class = %s, last_error = %s, + lease_until = NULL, lease_owner = NULL, next_attempt_at = NULL, updated_at = clock_timestamp() + WHERE job_id = %s + """, + (code, error_class, _safe(message), job["job_id"]), + ) + if attachment["ready_pipeline_version"] is not None: + # A failed newer attempt must not discard an already published result. + conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, pipeline_version = ready_pipeline_version, + updated_at = clock_timestamp() + WHERE id = %s AND ready_pipeline_version IS NOT NULL + """, + (attachment["id"],), + ) + else: + conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = %s, updated_at = clock_timestamp() + WHERE id = %s AND pipeline_version = %s AND status <> 'ready' + """, + (code, attachment["id"], job["pipeline_version"]), + ) + conn.execute( + """ + INSERT INTO failed_jobs (job_id, attachment_id, pipeline_version, attempt, error_code, error_class, safe_message) + VALUES (%s, %s, %s, %s, %s, %s, %s) + """, + (job["job_id"], attachment["id"], job["pipeline_version"], job["attempt"], code, error_class, _safe(message)), + ) + audit(conn, job["job_id"], attachment["id"], "failed", {"error_code": code, "error_class": error_class}) + publish_dlq( + { + "schema_version": 1, + "job_id": str(job["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(attachment["id"]), + "pipeline_version": int(job["pipeline_version"]), + "attempt": int(job["attempt"]), + "created_at": message_created(job), + } + ) + + +def message_created(job) -> str: + from app.timeutil import iso as format_iso + + value = job["created_at"] + return value if isinstance(value, str) else format_iso(value) + + +def _put_original(store, attachment, data: bytes) -> None: + existing = store.get(attachment["original_key"]) + digest = hashlib.sha256(data).hexdigest() + if digest != attachment["sha256"]: + raise PermanentJobError("processing_failed", "checksum mismatch") + if existing is None: + store.put(attachment["original_key"], data, attachment["content_type"]) + return + if hashlib.sha256(existing).hexdigest() != attachment["sha256"]: + raise TransientJobError("stored original does not match the accepted checksum") + + +def _store_variants(store, attachment, version: int, rendered: dict, max_pixels: int) -> list[dict]: + stored = [] + for name in VARIANT_NAMES: + item = rendered[name] + key = f"variants/{attachment['id']}/v{version}/{name}" + existing = store.get(key) + if existing is None: + store.put(key, item["bytes"], item["content_type"]) + blob = item["bytes"] + width, height = item["width"], item["height"] + else: + blob = existing + if hashlib.sha256(blob).hexdigest() == hashlib.sha256(item["bytes"]).hexdigest(): + width, height = item["width"], item["height"] + else: + probed = decode_image(blob, max_pixels) + width, height = probed.size + stored.append( + { + "name": name, + "content_type": item["content_type"] if existing is None else _content_type_for(name, blob, item["content_type"]), + "size_bytes": len(blob), + "width": width, + "height": height, + "sha256": hashlib.sha256(blob).hexdigest(), + "object_key": key, + } + ) + return stored + + +def _content_type_for(name: str, blob: bytes, encoded_type: str) -> str: + if blob.startswith(b"\xff\xd8"): + return "image/jpeg" + if len(blob) >= 12 and blob[:4] == b"RIFF" and blob[8:12] == b"WEBP": + return "image/webp" + return encoded_type + + +def _source_bytes(store, attachment) -> bytes: + data = store.get(attachment["staging_key"]) + if data is None: + data = store.get(attachment["original_key"]) + if data is None: + raise TransientJobError("source object is not available yet") + return data + + +def _commit_success(job, attachment, variants: list[dict], worker_id: str, crash_after_commit: bool) -> bool: + """Returns True when this version became the published result.""" + settings, store, _publisher = runtime.require() + with connection() as conn: + current = conn.execute("SELECT pipeline_version, ready_pipeline_version FROM attachments WHERE id = %s FOR UPDATE", (attachment["id"],)).fetchone() + for item in variants: + conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, width, height, sha256, object_key + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + ( + attachment["id"], + job["pipeline_version"], + item["name"], + item["content_type"], + item["size_bytes"], + item["width"], + item["height"], + item["sha256"], + item["object_key"], + ), + ) + updated = conn.execute( + """ + UPDATE attachments + SET status = 'ready', ready_pipeline_version = %s, error_code = NULL, updated_at = clock_timestamp() + WHERE id = %s + AND pipeline_version = %s + AND (ready_pipeline_version IS NULL OR ready_pipeline_version <= %s) + RETURNING id + """, + (job["pipeline_version"], attachment["id"], job["pipeline_version"], job["pipeline_version"]), + ).fetchone() + conn.execute( + """ + UPDATE jobs + SET status = 'ready', error_code = NULL, error_class = NULL, last_error = NULL, + lease_until = NULL, lease_owner = NULL, next_attempt_at = NULL, updated_at = clock_timestamp() + WHERE job_id = %s + """, + (job["job_id"],), + ) + if crash_after_commit: + _write_fault(conn, {"mode": None, "armed": False}) + audit(conn, job["job_id"], attachment["id"], "fault_after_commit", {"worker": worker_id}) + audit( + conn, + job["job_id"], + attachment["id"], + "committed", + {"pipeline_version": job["pipeline_version"], "published": bool(updated), "variants": len(variants)}, + ) + published = updated is not None and current is not None + if crash_after_commit: + os._exit(75) + if published: + try: + store.delete(attachment["staging_key"]) + with connection() as conn: + conn.execute( + "UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = %s AND status = 'ready'", + (attachment["id"],), + ) + except Exception: + log.warning("staging cleanup deferred for %s", attachment["id"]) + return published + + +def _execute(message: dict, attachment: dict, worker_id: str, fault: str | None) -> None: + settings, store, _publisher = runtime.require() + if fault == "before_effect": + _crash("before_effect", message["job_id"], attachment["id"]) + deadline = time.monotonic() + settings.job_timeout_seconds + data = _source_bytes(store, attachment) + if len(data) > settings.max_upload_bytes: + raise PermanentJobError("invalid_image", "object exceeds the byte limit") + if time.monotonic() > deadline: + raise TransientJobError("processing deadline exceeded") + try: + image = _decode_limited(data, settings.max_image_pixels, max(1, deadline - time.monotonic())) + params = pipeline_params(int(attachment_version(message))) + rendered = render_variants(image, params) + except ImageRejected as exc: + raise PermanentJobError("invalid_image", exc.message) from exc + if time.monotonic() > deadline: + raise TransientJobError("processing deadline exceeded") + _put_original(store, attachment, data) + variants = _store_variants(store, attachment, message["pipeline_version"], rendered, settings.max_image_pixels) + if fault == "after_s3": + _crash("after_s3", message["job_id"], attachment["id"]) + _commit_success(message_as_job(message, attachment), attachment, variants, worker_id, fault == "after_commit") + + +def attachment_version(message: dict) -> int: + return int(message["pipeline_version"]) + + +def message_as_job(message: dict, attachment: dict) -> dict: + return { + "job_id": message["job_id"], + "pipeline_version": message["pipeline_version"], + "attempt": message["attempt"], + "created_at": message["created_at"], + "attachment_id": attachment["id"], + } + + +def _decode_limited(data: bytes, max_pixels: int, timeout: float): + box: dict = {} + + def run() -> None: + try: + box["image"] = decode_image(data, max_pixels) + except Exception as exc: # noqa: BLE001 - transferred to the caller + box["error"] = exc + + thread = threading.Thread(target=run, daemon=True) + thread.start() + thread.join(timeout) + if thread.is_alive(): + raise TransientJobError("decode timed out") + if "error" in box: + raise box["error"] + return box["image"] + + +def _handle_known_error(message: dict, exc: Exception) -> None: + with connection() as conn: + job, attachment = _load(conn, message) + if job is None or attachment is None or job["status"] in {"ready", "failed", "superseded"}: + return + if isinstance(exc, PermanentJobError): + _fail(conn, job, attachment, exc.code, "permanent", exc.message) + elif isinstance(exc, TransientJobError): + _schedule_retry(conn, job, attachment, exc.message) + else: + _schedule_retry(conn, job, attachment, "temporary processing failure") + + +def process_delivery(raw: bytes, redelivered: bool, worker_id: str) -> str: + """Returns 'ack' or 'nack'. Ack happens only after the durable decision is committed.""" + import json + + try: + body = json.loads(raw.decode("utf-8")) + message = _parse_message(body) + except (UnicodeError, json.JSONDecodeError, PermanentJobError): + log.warning("discarding a job message that does not match the contract") + return "ack" + try: + fault = _consume_counted_fault() + except Exception: + log.warning("fault control unreadable", exc_info=True) + return "nack" + if fault == "transient": + _handle_known_error(message, TransientJobError("injected transient failure")) + return "ack" + if fault == "permanent": + _handle_known_error(message, PermanentJobError("invalid_image", "injected permanent failure")) + return "ack" + runs = 2 if fault == "duplicate" else 1 + for _ in range(runs): + outcome = _process_once(message, redelivered, worker_id, fault if runs == 1 else None) + if outcome == "nack": + return "nack" + fault = None + return "ack" + + +def _process_once(message: dict, redelivered: bool, worker_id: str, fault: str | None) -> str: + settings, _store, _publisher = runtime.require() + with connection() as conn: + job, attachment = _load(conn, message) + if job is None or attachment is None: + audit(conn, message["job_id"], message["attachment_id"], "missing", {"redelivered": redelivered}) + return "ack" + if job["status"] in {"ready", "superseded", "failed"}: + _finish_noop(conn, job, attachment, "noop_" + job["status"], redelivered) + return "ack" + if message["attempt"] != int(job["attempt"]): + _finish_noop(conn, job, attachment, "noop_attempt", redelivered) + return "ack" + newer_ready = ( + attachment["ready_pipeline_version"] is not None + and int(attachment["ready_pipeline_version"]) > int(job["pipeline_version"]) + ) + newer_desired = int(attachment["pipeline_version"]) > int(job["pipeline_version"]) + if newer_ready or newer_desired: + _supersede(conn, job, attachment, redelivered) + return "ack" + if _acquire(conn, job, worker_id, settings.job_timeout_seconds) is None: + return "nack" + conn.execute( + """ + UPDATE attachments + SET status = 'processing', updated_at = clock_timestamp() + WHERE id = %s AND pipeline_version = %s AND status IN ('queued', 'processing') + """, + (attachment["id"], job["pipeline_version"]), + ) + audit(conn, job["job_id"], attachment["id"], "leased", {"worker": worker_id, "redelivered": redelivered}) + try: + _execute(message, attachment, worker_id, fault) + except (PermanentJobError, TransientJobError, ImageRejected) as exc: + if isinstance(exc, ImageRejected): + exc = PermanentJobError("invalid_image", exc.message) + _handle_known_error(message, exc) + except Exception: + log.warning("unexpected pipeline error", exc_info=True) + _handle_known_error(message, TransientJobError("temporary processing failure")) + return "ack" + + +def recover_expired_leases() -> int: + recovered = 0 + with connection() as conn: + rows = conn.execute( + """ + UPDATE jobs + SET status = 'queued', lease_until = NULL, lease_owner = NULL, updated_at = clock_timestamp() + WHERE status = 'processing' AND lease_until IS NOT NULL AND lease_until < clock_timestamp() + RETURNING job_id, attachment_id, pipeline_version, attempt, created_at + """ + ).fetchall() + for job in rows: + conn.execute( + """ + UPDATE attachments + SET status = 'queued', updated_at = clock_timestamp() + WHERE id = %s AND pipeline_version = %s AND status = 'processing' + """, + (job["attachment_id"], job["pipeline_version"]), + ) + add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], job["created_at"]) + audit(conn, job["job_id"], job["attachment_id"], "lease_expired", {}) + recovered += 1 + if recovered: + log.info("recovered %s expired lease(s)", recovered) + return recovered + + +def sweep_staging() -> None: + settings, store, _publisher = runtime.require() + with connection() as conn: + ready = conn.execute( + """ + SELECT id, staging_key FROM attachments + WHERE staging_deleted_at IS NULL AND status = 'ready' + LIMIT 25 + """ + ).fetchall() + failed = conn.execute( + """ + SELECT id, staging_key FROM attachments + WHERE staging_deleted_at IS NULL AND status = 'failed' + AND updated_at < clock_timestamp() - make_interval(secs => %s) + LIMIT 25 + """, + (settings.staging_retention_seconds,), + ).fetchall() + for row in list(ready) + list(failed): + try: + store.delete(row["staging_key"]) + except Exception: + log.warning("staging delete failed") + continue + with connection() as conn: + conn.execute( + "UPDATE attachments SET staging_deleted_at = clock_timestamp() WHERE id = %s", + (row["id"],), + ) diff --git a/app/runtime.py b/app/runtime.py new file mode 100644 index 00000000..edd877ec --- /dev/null +++ b/app/runtime.py @@ -0,0 +1,37 @@ +"""Shared runtime objects created at process startup.""" +from __future__ import annotations + +from argon2 import PasswordHasher +from argon2.low_level import Type + +from app.broker import Publisher +from app.settings import Settings +from app.s3store import ObjectStore + +settings: Settings | None = None +store: ObjectStore | None = None +publisher: Publisher | None = None +hasher = PasswordHasher( + time_cost=2, + memory_cost=19456, + parallelism=1, + hash_len=32, + salt_len=16, + type=Type.ID, +) +dummy_hash: str | None = None + + +def init(app_settings: Settings, object_store: ObjectStore, message_publisher: Publisher) -> None: + global settings, store, publisher, dummy_hash + settings = app_settings + store = object_store + publisher = message_publisher + if dummy_hash is None: + dummy_hash = hasher.hash("dummy-password-for-equal-time") + + +def require() -> tuple[Settings, ObjectStore, Publisher]: + if settings is None or store is None or publisher is None: + raise RuntimeError("runtime is not initialized") + return settings, store, publisher diff --git a/app/s3store.py b/app/s3store.py new file mode 100644 index 00000000..dce2aa93 --- /dev/null +++ b/app/s3store.py @@ -0,0 +1,60 @@ +"""Private S3-compatible store. Object keys are server-generated.""" +from __future__ import annotations + +import logging + +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError + +log = logging.getLogger("messenger.s3") + + +class ObjectStore: + def __init__(self, endpoint: str, region: str, bucket: str, access_key: str, secret_key: str): + self.bucket = bucket + self.client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=Config( + s3={"addressing_style": "path"}, + signature_version="s3v4", + retries={"max_attempts": 3, "mode": "standard"}, + connect_timeout=3, + read_timeout=20, + ), + ) + + def ensure_bucket(self) -> None: + try: + self.client.head_bucket(Bucket=self.bucket) + except ClientError: + self.client.create_bucket(Bucket=self.bucket) + log.info("bucket ready") + + def healthy(self) -> bool: + try: + self.client.head_bucket(Bucket=self.bucket) + return True + except Exception: + log.warning("object store ping failed", exc_info=True) + return False + + def put(self, key: str, data: bytes, content_type: str) -> None: + self.client.put_object(Bucket=self.bucket, Key=key, Body=data, ContentType=content_type) + + def get(self, key: str) -> bytes | None: + try: + response = self.client.get_object(Bucket=self.bucket, Key=key) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code in {"404", "NoSuchKey", "NotFound"}: + return None + raise + return response["Body"].read() + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=self.bucket, Key=key) diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 00000000..00e73711 --- /dev/null +++ b/app/settings.py @@ -0,0 +1,74 @@ +"""Process configuration. Secrets come from the environment or *_FILE, never from the image.""" +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _raw(name: str, default: str | None = None) -> str: + file_name = os.environ.get(name + "_FILE") + if file_name: + return Path(file_name).read_text(encoding="utf-8").strip() + if name in os.environ and os.environ[name] != "": + return os.environ[name] + if default is not None: + return default + raise RuntimeError("missing configuration " + name) + + +def _int(name: str, default: int) -> int: + return int(_raw(name, str(default))) + + +@dataclass(frozen=True) +class Settings: + instance_id: str + database_url: str + session_ttl_seconds: int + idempotency_ttl_seconds: int + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + broker_url: str + worker_concurrency: int + job_max_attempts: int + job_timeout_seconds: int + job_backoff_base_seconds: float + job_backoff_cap_seconds: float + staging_retention_seconds: int + operator_token: str + max_queued_jobs: int + host: str + port: int + + +def load() -> Settings: + return Settings( + instance_id=_raw("INSTANCE_ID", "api-local"), + database_url=_raw("DATABASE_URL"), + session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600), + idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400), + s3_endpoint=_raw("S3_ENDPOINT", "http://minio:9000"), + s3_region=_raw("S3_REGION", "us-east-1"), + s3_bucket=_raw("S3_BUCKET", "messenger-attachments"), + s3_access_key_id=_raw("S3_ACCESS_KEY_ID"), + s3_secret_access_key=_raw("S3_SECRET_ACCESS_KEY"), + max_upload_bytes=_int("MAX_UPLOAD_BYTES", 10485760), + max_image_pixels=_int("MAX_IMAGE_PIXELS", 40000000), + broker_url=_raw("BROKER_URL"), + worker_concurrency=max(1, _int("WORKER_CONCURRENCY", 2)), + job_max_attempts=max(1, _int("JOB_MAX_ATTEMPTS", 5)), + job_timeout_seconds=max(1, _int("JOB_TIMEOUT_SECONDS", 60)), + job_backoff_base_seconds=float(_raw("JOB_BACKOFF_BASE_SECONDS", "0.5")), + job_backoff_cap_seconds=float(_raw("JOB_BACKOFF_CAP_SECONDS", "30")), + staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400), + operator_token=_raw("OPERATOR_TOKEN", ""), + max_queued_jobs=_int("MAX_QUEUED_JOBS", 500), + host=_raw("APP_HOST", "0.0.0.0"), + port=_int("APP_PORT", 8080), + ) diff --git a/app/timeutil.py b/app/timeutil.py new file mode 100644 index 00000000..d91f7b4e --- /dev/null +++ b/app/timeutil.py @@ -0,0 +1,15 @@ +"""UTC timestamps rendered as RFC 3339 with a Z suffix.""" +from __future__ import annotations + +from datetime import datetime, timezone + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.") + f"{value.microsecond:06d}Z" diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..a1a832e2 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,105 @@ +"""Separate worker process. It never serves the public HTTP API.""" +from __future__ import annotations + +import logging +import os +import signal +import threading +import time +import uuid + +from app import runtime +from app.broker import QUEUE, open_consumer +from app.db import migrate +from app.outbox import start_relay +from app.pipeline import process_delivery, recover_expired_leases, sweep_staging +from app.runtime import init +from app.s3store import ObjectStore +from app.settings import load + +log = logging.getLogger("messenger.worker") + + +def _on_term(_signum, _frame) -> None: + os._exit(0) + + +def _consume(index: int, broker_url: str) -> None: + worker_id = f"worker-{index}-{uuid.uuid4()}" + while True: + connection = None + try: + connection, channel = open_consumer(broker_url) + + def on_message(channel, method, _properties, body, connection=connection, worker_id=worker_id): + def work() -> None: + try: + result = process_delivery(body, bool(method.redelivered), worker_id) + except Exception: + log.exception("delivery failed") + result = "nack" + if result != "ack": + time.sleep(1) + + def finish() -> None: + if result == "ack": + channel.basic_ack(method.delivery_tag) + else: + channel.basic_nack(method.delivery_tag, requeue=True) + + connection.add_callback_threadsafe(finish) + + threading.Thread(target=work, name=f"delivery-{index}", daemon=True).start() + + channel.basic_consume(queue=QUEUE, on_message_callback=on_message, auto_ack=False) + log.info("consumer %s waiting; prefetch=1", worker_id) + channel.start_consuming() + except Exception: + log.warning("consumer connection dropped", exc_info=True) + time.sleep(1) + finally: + try: + if connection is not None and connection.is_open: + connection.close() + except Exception: + pass + + +def _maintain() -> None: + while True: + try: + recover_expired_leases() + sweep_staging() + except Exception: + log.warning("maintenance iteration failed", exc_info=True) + time.sleep(2) + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + signal.signal(signal.SIGTERM, _on_term) + signal.signal(signal.SIGINT, _on_term) + settings = load() + migrate(settings.database_url) + store = ObjectStore( + settings.s3_endpoint, + settings.s3_region, + settings.s3_bucket, + settings.s3_access_key_id, + settings.s3_secret_access_key, + ) + store.ensure_bucket() + from app.broker import Publisher + + init(settings, store, Publisher(settings.broker_url)) + start_relay() + threading.Thread(target=_maintain, name="maintenance", daemon=True).start() + for index in range(settings.worker_concurrency): + threading.Thread(target=_consume, args=(index, settings.broker_url), name=f"consumer-{index}", daemon=True).start() + log.info("worker started with concurrency %s", settings.worker_concurrency) + while True: + time.sleep(3600) + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..78ef737c --- /dev/null +++ b/compose.yaml @@ -0,0 +1,155 @@ +name: mtusi-lab5 + +x-app: &app + image: mtusi-lab5-app:local + build: + context: . + env_file: + - .env + init: true + restart: unless-stopped + depends_on: + postgres: + condition: service_healthy + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + +services: + postgres: + image: postgres:16.8-alpine + container_name: mtusi-lab5-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 3s + retries: 30 + restart: unless-stopped + + minio: + image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z + container_name: mtusi-lab5-minio + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + volumes: + - miniodata:/data + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + restart: unless-stopped + + minio-init: + image: pgsty/silo:RELEASE.2026-10-26T00-00-00Z + container_name: mtusi-lab5-minio-init + depends_on: + minio: + condition: service_healthy + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: ${S3_BUCKET} + entrypoint: ["/bin/sh", "-c"] + command: + - | + set -eu + mc alias set local http://minio:9000 "$$MINIO_ROOT_USER" "$$MINIO_ROOT_PASSWORD" + mc mb -p "local/$$S3_BUCKET" || true + mc anonymous set none "local/$$S3_BUCKET" + restart: "no" + + rabbitmq: + image: rabbitmq:3.13.7-management-alpine + container_name: mtusi-lab5-rabbitmq + environment: + RABBITMQ_DEFAULT_USER: ${RABBITMQ_DEFAULT_USER} + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_DEFAULT_PASS} + volumes: + - rabbitdata:/var/lib/rabbitmq + - ./infra/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 10s + retries: 20 + start_period: 20s + restart: unless-stopped + + api-1: + <<: *app + container_name: mtusi-lab5-api-1 + environment: + INSTANCE_ID: api-1 + labels: + lab.role: api + lab.instance_id: api-1 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 3s + timeout: 5s + retries: 30 + start_period: 20s + + api-2: + <<: *app + container_name: mtusi-lab5-api-2 + environment: + INSTANCE_ID: api-2 + labels: + lab.role: api + lab.instance_id: api-2 + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 3s + timeout: 5s + retries: 30 + start_period: 20s + + worker: + <<: *app + container_name: mtusi-lab5-worker + command: ["python", "-m", "app.worker"] + environment: + INSTANCE_ID: worker-1 + MALLOC_ARENA_MAX: "2" + mem_limit: 512m + restart: on-failure + labels: + lab.role: worker + + nginx: + image: nginx:1.27.5-alpine + container_name: mtusi-lab5-nginx + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + ports: + - "8080:8080" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/live"] + interval: 3s + timeout: 3s + retries: 20 + restart: unless-stopped + +volumes: + pgdata: + miniodata: + rabbitdata: diff --git a/contracts/extensions.md b/contracts/extensions.md new file mode 100644 index 00000000..17610586 --- /dev/null +++ b/contracts/extensions.md @@ -0,0 +1,30 @@ +# Расширения лабораторной 5 + +Обязательный контракт `contracts/openapi.json` не изменён. Ниже — операторский интерфейс и параметры pipeline, нужные для уровня 4 и трека 5. Эти маршруты не входят в публичный API и требуют заголовок `X-Operator-Token`. + +## Pipeline + +Ключ эффекта: `(attachment_id, pipeline_version, variant)`. + +| Версия | Назначение | JPEG thumbnail/crop | WebP optimized | +| --- | --- | --- | --- | +| 1 | Основной курс | quality 85, optimize, без progressive и без EXIF | quality 80, method 6, lossy | +| 2 | Эксперимент идемпотентности | quality 60 / 70 | quality 50, method 4 | + +Геометрия одинаковая: thumbnail вписывается в 256×256 без увеличения, crop — центр 128×128, optimized вписывается в 1280×1280 без увеличения. EXIF orientation применяется до кодирования, альфа-канал кладётся на белый. Метаданные исходника в варианты не копируются. Версия 2 нужна только чтобы отличить новый результат от старого по SHA-256. + +Повторная доставка, смена порядка и устаревшая версия не создают второй набор объектов одной и той же версии и не заменяют уже опубликованный более новый `ready_pipeline_version`. + +## Операторские маршруты + +- `POST /internal/control/outbox` `{"paused": true}` — остановить публикацию outbox после COMMIT. +- `POST /internal/control/fault` `{"mode": "before_effect"|"after_s3"|"after_commit"|"duplicate"|"transient"|"permanent", "remaining": 1}` — одноразовая инъекция. `transient` уменьшает `remaining`. +- `POST /internal/control/backpressure` `{"max_queued_jobs": 0}` или `null` — вернуть лимит окружения. +- `GET /internal/metrics`, `GET /internal/attachments/{id}`, `GET /internal/failed`. +- `POST /internal/attachments/{id}/replay` — новая попытка того же `job_id`. Уже `ready` задание не исполняется повторно. +- `POST /internal/attachments/{id}/reprocess` `{"pipeline_version": 2}` — новая версия pipeline. +- `POST /internal/jobs/{id}/redeliver` — ещё одна доставка того же сообщения брокером. + +## Повторы + +`JOB_MAX_ATTEMPTS` по умолчанию 5. Временная ошибка возвращает задание в `queued` с задержкой `base * 2^(attempt-1)`, умноженной на случайный коэффициент 0.5–1.5, но не дольше `JOB_BACKOFF_CAP_SECONDS`. Постоянная ошибка и исчерпание попыток пишут `failed_jobs` и копию сообщения в DLQ. Коды: `invalid_image`, `processing_failed`, `retry_exhausted`. diff --git a/evidence/grade5-track2.json b/evidence/grade5-track2.json new file mode 100644 index 00000000..1ef23364 --- /dev/null +++ b/evidence/grade5-track2.json @@ -0,0 +1,144 @@ +{ + "track": 2, + "results": [ + { + "scenario": "grade4 variants and broker payload", + "result": "pass", + "detail": { + "attachment_id": "9fa448b7-09b8-4913-8b7f-2c739b5755b3", + "job_id": "590aa490-01f5-407d-91ea-495ebf067b4e" + } + }, + { + "scenario": "idempotency key order", + "result": "pass", + "detail": { + "message_id": "b0f70dd7-777f-40d4-82e5-9b0ed7a49f83" + } + }, + { + "scenario": "validation", + "result": "pass", + "detail": { + "rejected": [ + "extra_field", + "invalid_image", + "pixel_limit" + ] + } + }, + { + "scenario": "backpressure", + "result": "pass", + "detail": { + "max_queued_jobs": 0, + "status": 503 + } + }, + { + "scenario": "outbox commit/publish gap", + "result": "pass", + "detail": { + "attachment_id": "2a18cbd8-25bf-4d7f-86df-df2eeba65217", + "job_id": "4f6cb35f-26b0-4ac3-ba82-f4df1a531876" + } + }, + { + "scenario": "worker stop, broker restart, resume", + "result": "pass", + "detail": { + "attachment_id": "54a139ee-963c-48c5-8bae-13853c4630bb", + "queue_before": 1, + "queue_after_restart": 1 + } + }, + { + "scenario": "permanent failure, DLQ registry, safe replay", + "result": "pass", + "detail": { + "attachment_id": "27931bef-3404-4b63-a7d7-1c757f43263f", + "job_id": "b2c59c26-07ac-4419-9b85-a0fb6fab79e2" + } + }, + { + "scenario": "transient retry with backoff", + "result": "pass", + "detail": { + "attachment_id": "27158119-313a-4582-bd19-73a700ce5cbc", + "attempt": 2 + } + }, + { + "scenario": "retry budget then explicit replay", + "result": "pass", + "detail": { + "attachment_id": "ea24b78e-fd31-4a47-ab82-05b0583964b6" + } + }, + { + "scenario": "duplicate delivery is a no-op", + "result": "pass", + "detail": { + "attachment_id": "3d7f00ae-089b-474b-9f6c-e92becc45a3f", + "job_id": "ffc5490a-e210-4652-a3a8-63210a1a6675" + } + }, + { + "scenario": "crash before side effect", + "result": "pass", + "detail": { + "attachment_id": "e7367911-2e2b-4a36-8001-b777fc1a0352", + "audit": "fault_before_effect" + } + }, + { + "scenario": "crash after object write", + "result": "pass", + "detail": { + "attachment_id": "946b250d-8ead-4a2b-9dc4-797db8a27fed", + "audit": "fault_after_s3" + } + }, + { + "scenario": "crash after commit before ack", + "result": "pass", + "detail": { + "attachment_id": "d3bd57dd-2ee9-48ba-8ded-3d94a798db6a", + "audit": "fault_after_commit" + } + }, + { + "scenario": "versioned pipeline rejects stale result", + "result": "pass", + "detail": { + "attachment_id": "1dc4f3f5-74bd-4f6a-820a-3a2fa00c5f80", + "new_job_id": "613fa584-6bbb-441c-b280-2354b9d4837c" + } + }, + { + "scenario": "reordered older job cannot win", + "result": "pass", + "detail": { + "attachment_id": "e83b88ea-54ea-4f20-bcb1-60bbe3ba41b7" + } + }, + { + "scenario": "volume restart preserves session message and object", + "result": "pass", + "detail": { + "message_id": "f7f9c162-7ac1-4f61-9bba-a6a02e0bf805", + "attachment_id": "1dc4f3f5-74bd-4f6a-820a-3a2fa00c5f80" + } + }, + { + "scenario": "two API replicas", + "result": "pass", + "detail": { + "instances": [ + "api-1", + "api-2" + ] + } + } + ] +} \ No newline at end of file diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..17b1abe3 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,33 @@ +worker_processes auto; +error_log /dev/stderr warn; +pid /tmp/nginx.pid; + +events { + worker_connections 1024; +} + +http { + access_log off; + client_max_body_size 12m; + proxy_http_version 1.1; + proxy_set_header Connection ""; + gzip off; + + upstream messenger_api { + server api-1:8080 max_fails=0; + server api-2:8080 max_fails=0; + } + + server { + listen 8080; + location / { + proxy_pass http://messenger_api; + proxy_connect_timeout 1s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 3s; + } + } +} diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 00000000..2ed5a302 --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,2 @@ +disk_free_limit.absolute = 50MB +consumer_timeout = 3600000 diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..e84d70c6 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,154 @@ +-- Idempotent schema for lab5. Applied under a session advisory lock at startup. + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + body text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX IF NOT EXISTS messages_chat_order_idx ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + sha256 text NOT NULL, + status text NOT NULL, + error_code text, + pipeline_version integer NOT NULL, + ready_pipeline_version integer, + staging_key text NOT NULL, + original_key text NOT NULL, + staging_deleted_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed')) +); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position integer NOT NULL, + PRIMARY KEY (message_id, position) +); + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + http_method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + request_hash text NOT NULL, + response_json jsonb, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, http_method, path, idem_key) +); + +-- One logical job per (attachment, pipeline version). job_id is stable across redelivery and replay. +CREATE TABLE IF NOT EXISTS jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + attempt integer NOT NULL, + status text NOT NULL, + error_code text, + error_class text, + last_error text, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + next_attempt_at timestamptz, + lease_until timestamptz, + lease_owner text, + CONSTRAINT jobs_version_uniq UNIQUE (attachment_id, pipeline_version), + CONSTRAINT jobs_status_chk CHECK (status IN ('queued', 'processing', 'ready', 'failed', 'superseded')) +); +CREATE INDEX IF NOT EXISTS jobs_status_idx ON jobs (status, next_attempt_at); + +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +CREATE TABLE IF NOT EXISTS outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES jobs (job_id), + payload jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz +); +CREATE INDEX IF NOT EXISTS outbox_unpublished_idx ON outbox (id) WHERE published_at IS NULL; + +CREATE TABLE IF NOT EXISTS failed_jobs ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL, + attachment_id uuid NOT NULL, + pipeline_version integer NOT NULL, + attempt integer NOT NULL, + error_code text NOT NULL, + error_class text NOT NULL, + safe_message text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + replayed_at timestamptz +); + +CREATE TABLE IF NOT EXISTS job_audit ( + id bigserial PRIMARY KEY, + job_id uuid, + attachment_id uuid, + action text NOT NULL, + detail jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE IF NOT EXISTS runtime_control ( + key text PRIMARY KEY, + value jsonb NOT NULL +); + +INSERT INTO runtime_control (key, value) VALUES + ('outbox_paused', 'false'::jsonb), + ('worker_fault', '{"mode": null, "armed": false}'::jsonb), + ('max_queued_jobs', 'null'::jsonb) +ON CONFLICT (key) DO NOTHING; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..dd916ad3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +psycopg[binary,pool]==3.2.4 +argon2-cffi==23.1.0 +boto3==1.35.99 +pika==1.3.2 +Pillow==11.1.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..20318e05 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Create a local .env for the synthetic stand. Existing files are left untouched.""" +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TARGET = ROOT / ".env" + + +def main() -> None: + if TARGET.exists(): + print("local env already exists") + return + database_password = secrets.token_hex(16) + object_password = secrets.token_hex(16) + broker_password = secrets.token_hex(16) + operator = secrets.token_hex(16) + lines = [ + "POSTGRES_USER=messenger", + "POSTGRES_PASSWORD=" + database_password, + "POSTGRES_DB=messenger", + "MINIO_ROOT_USER=labminio", + "MINIO_ROOT_PASSWORD=" + object_password, + "S3_ACCESS_KEY_ID=labminio", + "S3_SECRET_ACCESS_KEY=" + object_password, + "S3_ENDPOINT=http://minio:9000", + "S3_REGION=us-east-1", + "S3_BUCKET=messenger-attachments", + "RABBITMQ_DEFAULT_USER=messenger", + "RABBITMQ_DEFAULT_PASS=" + broker_password, + "DATABASE_URL=postgresql://messenger:" + database_password + "@postgres:5432/messenger", + "BROKER_URL=amqp://messenger:" + broker_password + "@rabbitmq:5672/", + "SESSION_TTL_SECONDS=3600", + "IDEMPOTENCY_TTL_SECONDS=86400", + "MAX_UPLOAD_BYTES=10485760", + "MAX_IMAGE_PIXELS=40000000", + "WORKER_CONCURRENCY=2", + "JOB_MAX_ATTEMPTS=5", + "JOB_TIMEOUT_SECONDS=60", + "JOB_BACKOFF_BASE_SECONDS=0.4", + "JOB_BACKOFF_CAP_SECONDS=8", + "STAGING_RETENTION_SECONDS=86400", + "MAX_QUEUED_JOBS=500", + "OPERATOR_TOKEN=" + operator, + "APP_HOST=0.0.0.0", + "APP_PORT=8080", + "", + ] + TARGET.write_text("\n".join(lines), encoding="utf-8", newline="\n") + print("wrote local env") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..80fd534a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Restart API and every stateful dependency. Volumes are not removed. +docker compose --env-file .env restart postgres minio rabbitmq api-1 api-2 worker nginx +exit 0 diff --git a/scripts/start-apis.sh b/scripts/start-apis.sh new file mode 100755 index 00000000..eebc0504 --- /dev/null +++ b/scripts/start-apis.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env start api-1 api-2 +exit 0 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..59af4af8 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cid=$(docker ps -q --filter "label=lab.instance_id=${TARGET_INSTANCE_ID}") +if [ -z "$cid" ]; then + echo "no running API replica for the requested instance" >&2 + exit 1 +fi +# Leave this replica stopped. Do not stop the database, broker, or load balancer. +docker stop -t 5 $cid >/dev/null +exit 0 diff --git a/student_tests/run_grade.py b/student_tests/run_grade.py new file mode 100644 index 00000000..f0ea03df --- /dev/null +++ b/student_tests/run_grade.py @@ -0,0 +1,359 @@ +"""Grade 4/5 scenarios for the local stand. Prints no session tokens.""" +import hashlib +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib import error, request + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure + +BASE = "http://localhost:8080" +EVIDENCE = ROOT / "evidence" / "grade5-track2.json" +ALLOWED_MESSAGE_KEYS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at", "traceparent", "tracestate"} + + +def env_value(name): + for line in (ROOT / ".env").read_text(encoding="utf-8").splitlines(): + if line.startswith(name + "="): + return line.split("=", 1)[1] + raise SystemExit("missing local setting " + name) + + +TOKEN = env_value("OPERATOR_TOKEN") +RESULTS = [] + + +def record(name, detail): + RESULTS.append({"scenario": name, "result": "pass", "detail": detail}) + print("PASS:", name) + + +def fail(name, message): + RESULTS.append({"scenario": name, "result": "fail", "detail": message}) + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text(json.dumps(RESULTS, ensure_ascii=False, indent=2), encoding="utf-8") + raise SystemExit(name + ": " + message) + + +def compose(*args): + completed = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True, capture_output=True) + if completed.returncode != 0: + raise SystemExit("compose " + " ".join(args) + " failed: " + completed.stderr[-400:]) + return completed.stdout + + +def queue_depth(): + completed = subprocess.run( + ["docker", "exec", "mtusi-lab5-rabbitmq", "rabbitmqctl", "list_queues", "name", "messages"], + text=True, capture_output=True, + ) + if completed.returncode != 0: + return None + for line in completed.stdout.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[0] == "attachment.process" and parts[1].isdigit(): + return int(parts[1]) + return 0 + + +def operator(method, path, body=None, expected=200): + data = None if body is None else json.dumps(body).encode("utf-8") + headers = {"X-Operator-Token": TOKEN, "Accept": "application/json"} + if body is not None: + headers["Content-Type"] = "application/json" + call = request.Request(BASE + path, data=data, headers=headers, method=method) + try: + response = request.urlopen(call, timeout=10) + except error.HTTPError as exc: + response = exc + payload = json.loads(response.read().decode("utf-8")) + if response.status != expected: + raise Failure(method + " " + path + " -> " + str(response.status) + " " + payload.get("error", {}).get("code", "")) + return payload + + +def reset_controls(): + operator("POST", "/internal/control/fault", {"mode": None}) + operator("POST", "/internal/control/outbox", {"paused": False}) + operator("POST", "/internal/control/backpressure", {"max_queued_jobs": None}) + compose("start", "worker") + + +def wait_health(timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + Client(BASE, timeout=3).call("GET", "/health/ready", schema="Health") + return + except Failure: + time.sleep(0.5) + raise SystemExit("API did not become ready") + + +def wait_attachment(client, actor, attachment_id, wanted, timeout=90): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = client.call("GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment")["json"] + if last["status"] == wanted: + return last + if wanted != "failed" and last["status"] == "failed": + raise Failure("attachment failed with " + str(last["error_code"])) + time.sleep(0.4) + raise Failure("timed out waiting for " + wanted + ", last status " + str(last and last["status"])) + + +def upload_sample(client, actor, chat): + content = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + attachment = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=sample.png", + 202, + actor=actor, + raw=content, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + if attachment["status"] != "queued": + raise Failure("upload snapshot is not queued") + return attachment, content + + +def main(): + reset_controls() + wait_health() + client = Client(BASE, timeout=10) + alice, bob, _eve, chat = client.bootstrap() + + attachment, content = upload_sample(client, alice, chat) + ready = wait_attachment(client, bob, attachment["id"], "ready") + names = {item["name"]: item for item in ready["variants"]} + if set(names) != {"thumbnail", "crop", "optimized"}: + fail("variants", "unexpected variant set") + if (names["thumbnail"]["width"], names["thumbnail"]["height"]) != (256, 160): + fail("variants", "thumbnail metadata") + if (names["crop"]["width"], names["crop"]["height"]) != (128, 128) or names["crop"]["content_type"] != "image/jpeg": + fail("variants", "crop contract") + if names["optimized"]["content_type"] != "image/webp" or (names["optimized"]["width"], names["optimized"]["height"]) != (320, 200): + fail("variants", "optimized contract") + original = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + thumb = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", actor=bob) + optimized = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=optimized", actor=bob) + if original["bytes"] != content or thumb["headers"].get_content_type() != "image/jpeg": + fail("variants", "bytes or thumbnail type") + if hashlib.sha256(thumb["bytes"]).hexdigest() != names["thumbnail"]["sha256"] or optimized["bytes"][8:12] != b"WEBP": + fail("variants", "checksum or webp header") + state = operator("GET", "/internal/attachments/" + attachment["id"]) + if any(set(message) - ALLOWED_MESSAGE_KEYS or any(isinstance(value, str) and len(value) > 512 for value in message.values()) for message in state["messages"]): + fail("variants", "broker payload contains unexpected data") + record("grade4 variants and broker payload", {"attachment_id": attachment["id"], "job_id": state["jobs"][0]["job_id"]}) + + key = "order-" + uuid.uuid4().hex + path = "/api/v1/chats/" + chat["id"] + "/messages" + first = client.call("POST", path, 201, actor=alice, raw=b'{"text":"same-body","attachment_ids":[]}', headers={"Content-Type": "application/json", "Idempotency-Key": key}, schema="Message")["json"] + second = client.call("POST", path, 201, actor=alice, raw=b'{"attachment_ids":[],"text":"same-body"}', headers={"Content-Type": "application/json", "Idempotency-Key": key}, schema="Message")["json"] + if first != second: + fail("idempotency-order", "canonical replay differs") + record("idempotency key order", {"message_id": first["id"]}) + + client.call("POST", "/api/v1/users", 400, body={"username": "abc", "display_name": "Имя", "password": "short", "extra": 1}) + client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bad.png", 415, actor=alice, raw=b"not-an-image", headers={"Content-Type": "image/png"}) + import struct + import zlib + ihdr = struct.pack(">IIBBBBB", 8000, 8000, 8, 2, 0, 0, 0) + chunk = struct.pack(">I", len(ihdr)) + b"IHDR" + ihdr + struct.pack(">I", zlib.crc32(b"IHDR" + ihdr) & 0xFFFFFFFF) + bomb = b"\x89PNG\r\n\x1a\n" + chunk + struct.pack(">I", 0) + b"IEND" + struct.pack(">I", zlib.crc32(b"IEND") & 0xFFFFFFFF) + client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=bomb.png", 413, actor=alice, raw=bomb, headers={"Content-Type": "image/png"}) + record("validation", {"rejected": ["extra_field", "invalid_image", "pixel_limit"]}) + + operator("POST", "/internal/control/backpressure", {"max_queued_jobs": 0}) + try: + client.call("POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=blocked.png", 503, actor=alice, raw=content, headers={"Content-Type": "image/png"}) + finally: + operator("POST", "/internal/control/backpressure", {"max_queued_jobs": None}) + record("backpressure", {"max_queued_jobs": 0, "status": 503}) + + operator("POST", "/internal/control/outbox", {"paused": True}) + try: + paused, _content = upload_sample(client, alice, chat) + time.sleep(1.2) + metrics = operator("GET", "/internal/metrics") + hidden = operator("GET", "/internal/attachments/" + paused["id"]) + if metrics["outbox_unpublished"] < 1 or hidden["attachment"]["status"] != "queued" or hidden["jobs"][0]["job_id"] is None: + fail("outbox-gap", "intent was not retained while publish was paused") + if queue_depth() not in {0, None}: + fail("outbox-gap", "broker received a message before publish resumed") + job_id = hidden["jobs"][0]["job_id"] + finally: + operator("POST", "/internal/control/outbox", {"paused": False}) + resumed = wait_attachment(client, alice, paused["id"], "ready") + after = operator("GET", "/internal/attachments/" + paused["id"]) + if after["jobs"][0]["job_id"] != job_id or resumed["status"] != "ready": + fail("outbox-gap", "recovery created a different job") + record("outbox commit/publish gap", {"attachment_id": paused["id"], "job_id": job_id}) + + compose("stop", "worker") + try: + held, held_bytes = upload_sample(client, alice, chat) + deadline = time.monotonic() + 20 + while time.monotonic() < deadline and operator("GET", "/internal/metrics")["outbox_unpublished"] != 0: + time.sleep(0.3) + if operator("GET", "/internal/attachments/" + held["id"])["attachment"]["status"] != "queued": + fail("worker-stop", "API processed the image synchronously") + client.call("GET", "/api/v1/attachments/" + held["id"] + "/content", 409, actor=bob) + client.message(bob, chat, "чат жив без worker") + depth_before = queue_depth() + compose("restart", "api-1", "api-2", "rabbitmq") + wait_health() + depth_after = None + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + depth_after = queue_depth() + if depth_after is not None: + break + time.sleep(1) + if not depth_before or not depth_after: + fail("worker-stop", "durable queue did not retain the message across broker restart") + finally: + compose("start", "worker") + held_ready = wait_attachment(client, bob, held["id"], "ready") + downloaded = client.call("GET", "/api/v1/attachments/" + held["id"] + "/content?variant=thumbnail", actor=alice) + if held_ready["id"] != held["id"] or downloaded["headers"].get_content_type() != "image/jpeg": + fail("worker-stop", "restarted worker did not finish the accepted job") + if client.call("GET", "/api/v1/attachments/" + held["id"] + "/content", actor=bob)["bytes"] != held_bytes: + fail("worker-stop", "original changed") + record("worker stop, broker restart, resume", {"attachment_id": held["id"], "queue_before": depth_before, "queue_after_restart": depth_after}) + + operator("POST", "/internal/control/fault", {"mode": "permanent"}) + rejected, _bytes = upload_sample(client, alice, chat) + failed = wait_attachment(client, alice, rejected["id"], "failed") + if failed["error_code"] != "invalid_image": + fail("permanent-replay", "unexpected error code") + client.call("GET", "/api/v1/attachments/" + rejected["id"] + "/content", 409, actor=alice) + client.call("POST", "/api/v1/chats/" + chat["id"] + "/messages", 409, actor=alice, body={"text": "", "attachment_ids": [rejected["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex}) + replay = operator("POST", "/internal/attachments/" + rejected["id"] + "/replay") + if not replay["replayed"] or replay["job_id"] != operator("GET", "/internal/attachments/" + rejected["id"])["jobs"][0]["job_id"]: + fail("permanent-replay", "replay changed the logical job") + restored = wait_attachment(client, alice, rejected["id"], "ready") + again = operator("POST", "/internal/attachments/" + rejected["id"] + "/replay") + if again["reason"] != "already_ready" or len(restored["variants"]) != 3: + fail("permanent-replay", "successful job was replayed as new work") + record("permanent failure, DLQ registry, safe replay", {"attachment_id": rejected["id"], "job_id": replay["job_id"]}) + + operator("POST", "/internal/control/fault", {"mode": "transient", "remaining": 1}) + retried, _bytes = upload_sample(client, alice, chat) + retried_ready = wait_attachment(client, alice, retried["id"], "ready", timeout=60) + retried_state = operator("GET", "/internal/attachments/" + retried["id"]) + if retried_state["jobs"][0]["attempt"] != 2 or retried_ready["status"] != "ready": + fail("backoff", "single transient failure did not retry exactly once") + record("transient retry with backoff", {"attachment_id": retried["id"], "attempt": 2}) + + operator("POST", "/internal/control/fault", {"mode": "transient", "remaining": 5}) + exhausted, _bytes = upload_sample(client, alice, chat) + exhausted_state = wait_attachment(client, alice, exhausted["id"], "failed", timeout=60) + if exhausted_state["error_code"] != "retry_exhausted": + fail("retry-budget", "attempts were not capped") + if operator("GET", "/internal/metrics")["failed_registry"] < 1: + fail("retry-budget", "failed registry is empty") + operator("POST", "/internal/attachments/" + exhausted["id"] + "/replay") + wait_attachment(client, alice, exhausted["id"], "ready", timeout=60) + record("retry budget then explicit replay", {"attachment_id": exhausted["id"]}) + + operator("POST", "/internal/control/fault", {"mode": "duplicate"}) + duplicated, _bytes = upload_sample(client, alice, chat) + wait_attachment(client, alice, duplicated["id"], "ready") + duplicate_state = operator("GET", "/internal/attachments/" + duplicated["id"]) + visible = [item for item in duplicate_state["variants"] if item["pipeline_version"] == 1] + if len(visible) != 3 or "noop_ready" not in {item["action"] for item in duplicate_state["audits"]}: + fail("duplicate", "inline duplicate created another variant set or skipped the no-op") + before = {item["sha256"] for item in visible} + operator("POST", "/internal/jobs/" + duplicate_state["jobs"][0]["job_id"] + "/redeliver") + deadline = time.monotonic() + 20 + while time.monotonic() < deadline and sum(1 for item in operator("GET", "/internal/attachments/" + duplicated["id"])["audits"] if item["action"] == "noop_ready") < 2: + time.sleep(0.4) + after_state = operator("GET", "/internal/attachments/" + duplicated["id"]) + after = {item["sha256"] for item in after_state["variants"] if item["pipeline_version"] == 1} + if before != after or len(after_state["variants"]) != 3: + fail("duplicate", "broker redelivery rewrote the result") + record("duplicate delivery is a no-op", {"attachment_id": duplicated["id"], "job_id": duplicate_state["jobs"][0]["job_id"]}) + + for mode, label in (("before_effect", "crash before side effect"), ("after_s3", "crash after object write"), ("after_commit", "crash after commit before ack")): + operator("POST", "/internal/control/fault", {"mode": mode}) + crashed, _bytes = upload_sample(client, alice, chat) + wait_attachment(client, alice, crashed["id"], "ready", timeout=90) + audits = {item["action"] for item in operator("GET", "/internal/attachments/" + crashed["id"])["audits"]} + expected = {"before_effect": "fault_before_effect", "after_s3": "fault_after_s3", "after_commit": "fault_after_commit"}[mode] + if expected not in audits or len([item for item in operator("GET", "/internal/attachments/" + crashed["id"])["variants"] if item["pipeline_version"] == 1]) != 3: + fail(label, "fault was not recovered idempotently") + record(label, {"attachment_id": crashed["id"], "audit": expected}) + + baseline, _bytes = upload_sample(client, alice, chat) + baseline_ready = wait_attachment(client, alice, baseline["id"], "ready") + first_sha = {item["name"]: item["sha256"] for item in baseline_ready["variants"]} + reprocess = operator("POST", "/internal/attachments/" + baseline["id"] + "/reprocess", {"pipeline_version": 2}) + version_two = wait_attachment(client, alice, baseline["id"], "ready") + second_sha = {item["name"]: item["sha256"] for item in version_two["variants"]} + version_state = operator("GET", "/internal/attachments/" + baseline["id"]) + if version_state["attachment"]["ready_pipeline_version"] != 2 or first_sha["thumbnail"] == second_sha["thumbnail"]: + fail("pipeline-version", "version 2 did not publish a distinct result") + if len(version_state["variants"]) != 6: + fail("pipeline-version", "versioned variants were overwritten in place") + operator("POST", "/internal/jobs/" + version_state["jobs"][0]["job_id"] + "/redeliver") + time.sleep(2) + stale = operator("GET", "/internal/attachments/" + baseline["id"]) + public = client.call("GET", "/api/v1/attachments/" + baseline["id"], actor=alice, schema="Attachment")["json"] + if stale["attachment"]["ready_pipeline_version"] != 2 or {item["sha256"] for item in public["variants"]} != set(second_sha.values()): + fail("pipeline-version", "stale pipeline overwrote the newer result") + record("versioned pipeline rejects stale result", {"attachment_id": baseline["id"], "new_job_id": reprocess["job_id"]}) + + operator("POST", "/internal/control/outbox", {"paused": True}) + try: + ordered, _bytes = upload_sample(client, alice, chat) + operator("POST", "/internal/attachments/" + ordered["id"] + "/reprocess", {"pipeline_version": 2}) + finally: + operator("POST", "/internal/control/outbox", {"paused": False}) + ordered_ready = wait_attachment(client, alice, ordered["id"], "ready") + ordered_state = operator("GET", "/internal/attachments/" + ordered["id"]) + if ordered_state["attachment"]["ready_pipeline_version"] != 2 or "superseded" not in {job["status"] for job in ordered_state["jobs"]}: + fail("reorder", "older queued job was not superseded by the newer pipeline") + if {item["name"] for item in ordered_ready["variants"]} != {"thumbnail", "crop", "optimized"}: + fail("reorder", "public result is not the complete newer pipeline") + record("reordered older job cannot win", {"attachment_id": ordered["id"]}) + + persisted = client.message(alice, chat, "пережить рестарт")["json"] + compose("restart", "postgres", "minio", "rabbitmq", "api-1", "api-2", "worker", "nginx") + wait_health(120) + me = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"] + history = client.call("GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage")["json"] + if me["id"] != alice["id"] or not any(item == persisted for item in history["items"]): + fail("persistence", "session or message did not survive restart") + if client.call("GET", "/api/v1/attachments/" + baseline["id"] + "/content", actor=bob)["bytes"] != content: + fail("persistence", "object bytes did not survive restart") + record("volume restart preserves session message and object", {"message_id": persisted["id"], "attachment_id": baseline["id"]}) + + observed = set() + for _ in range(20): + observed.add(client.call("GET", "/api/v1/users/me", actor=bob, schema="User")["headers"]["X-Instance-Id"]) + if observed != {"api-1", "api-2"}: + fail("replicas", "load balancer did not expose both API replicas") + record("two API replicas", {"instances": sorted(observed)}) + EVIDENCE.parent.mkdir(exist_ok=True) + EVIDENCE.write_text(json.dumps({"track": 2, "results": RESULTS}, ensure_ascii=False, indent=2), encoding="utf-8") + print("PASS: grade 4 and track 2 scenarios") + + +if __name__ == "__main__": + try: + main() + except Failure as exc: + fail("client", str(exc)) + finally: + try: + reset_controls() + except Exception: + pass diff --git a/student_tests/run_hooks.py b/student_tests/run_hooks.py new file mode 100644 index 00000000..c896d9e6 --- /dev/null +++ b/student_tests/run_hooks.py @@ -0,0 +1,172 @@ +"""Run the same Docker actions as scripts/restart.sh and scripts/stop-one.sh. + +The public harness executes those POSIX scripts directly. This Windows host +cannot spawn a .sh file (WinError 193), so this runner calls the same +docker compose / docker stop commands and checks the contract around them. +It prints no tokens. +""" +import subprocess +import sys +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure + +CONTAINERS = [ + "mtusi-lab5-postgres", + "mtusi-lab5-minio", + "mtusi-lab5-rabbitmq", + "mtusi-lab5-api-1", + "mtusi-lab5-api-2", + "mtusi-lab5-worker", + "mtusi-lab5-nginx", +] + + +def started(): + completed = subprocess.run( + ["docker", "inspect", "--format", "{{.Name}} {{.State.StartedAt}}", *CONTAINERS], + text=True, + capture_output=True, + check=True, + ) + return completed.stdout.strip() + + +def compose(*args): + completed = subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, text=True, capture_output=True) + if completed.returncode != 0: + raise SystemExit("compose failed: " + completed.stderr[-400:]) + + +def stop_instance(instance): + found = subprocess.run( + ["docker", "ps", "-q", "--filter", "label=lab.instance_id=" + instance], + text=True, + capture_output=True, + check=True, + ) + cid = found.stdout.strip() + if not cid: + raise SystemExit("no running replica " + instance) + completed = subprocess.run(["docker", "stop", "-t", "5", cid], text=True, capture_output=True) + if completed.returncode != 0: + raise SystemExit("stop failed: " + completed.stderr[-200:]) + + +def wait_ready(client, actor=None, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", schema="Health") + if actor is not None: + client.call("GET", "/api/v1/users/me", actor=actor, schema="User") + return + except Failure: + time.sleep(0.5) + raise SystemExit("stack did not become ready") + + +def failover(client, alice, chat, target): + client.timeout = 2 + started_at = time.monotonic() + outcome = {} + + def stopper(): + try: + stop_instance(target) + outcome["ok"] = True + except SystemExit as exc: + outcome["error"] = str(exc) + + import threading + + thread = threading.Thread(target=stopper) + thread.start() + successes = [] + after = [] + failures = 0 + first = None + key = uuid.uuid4().hex + text = "Запись во время отказа " + key + while time.monotonic() - started_at < 15: + stopped = not thread.is_alive() and outcome.get("ok") + try: + response = client.message(alice, chat, text, key=key) + if stopped: + if first is None: + first = time.monotonic() - started_at + after.append(response["headers"]["X-Instance-Id"]) + successes.append(response["json"]["id"]) + key = uuid.uuid4().hex + text = "Запись во время отказа " + key + except Failure: + failures += 1 + time.sleep(0.1) + thread.join(timeout=10) + if outcome.get("error"): + raise SystemExit(outcome["error"]) + if first is None or first > 10 or len(after) < 5 or target in after[-5:]: + raise SystemExit( + "failover " + target + " failed: recovery=" + str(first) + " tail=" + ",".join(after[-5:]) + ) + print( + "PASS failover", + target, + "recovery_s", + round(first, 3), + "writes", + len(successes), + "failed_attempts", + failures, + ) + + +def main(): + print("BEFORE") + print(started()) + client = Client("http://localhost:8080", timeout=10) + alice, bob, _eve, chat = client.bootstrap() + original = client.message(alice, chat, "Сохранить при отказе")["json"] + attachment, content = client.attachment(alice, chat) + attachment = client.wait_ready(alice, attachment) + compose("restart", "postgres", "minio", "rabbitmq", "api-1", "api-2", "worker", "nginx") + print("AFTER") + print(started()) + client.timeout = 10 + wait_ready(client, alice) + me = client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["json"] + if me["id"] != alice["id"]: + raise SystemExit("session did not survive restart") + history = client.call( + "GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", actor=bob, schema="MessagePage" + )["json"] + if not any(item == original for item in history["items"]): + raise SystemExit("message did not survive restart") + downloaded = client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", actor=bob) + if downloaded["bytes"] != content: + raise SystemExit("original bytes changed across restart") + print("PASS persistence", original["id"], attachment["id"]) + + client.timeout = 10 + observed = set() + for _ in range(40): + observed.add(client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"]["X-Instance-Id"]) + if observed != {"api-1", "api-2"}: + raise SystemExit("expected both replicas, saw " + ",".join(sorted(observed))) + for target in ("api-1", "api-2"): + failover(client, alice, chat, target) + compose("start", target) + client.timeout = 10 + wait_ready(client) + print("PASS hooks") + + +if __name__ == "__main__": + try: + main() + finally: + compose("start", "api-1", "api-2") diff --git a/student_tests/test_images.py b/student_tests/test_images.py new file mode 100644 index 00000000..f9fc3fc7 --- /dev/null +++ b/student_tests/test_images.py @@ -0,0 +1,65 @@ +"""Geometry checks for the image contract. Runs without the stand.""" +import io +import struct +import sys +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "tests")) + +from app.images import decode_image, inspect_upload, pipeline_params, render_variants +from image_probe import jpeg_size +from PIL import Image + + +def check(condition, message): + if not condition: + raise SystemExit(message) + + +def png_chunk(tag, data): + crc = zlib.crc32(tag + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) + + +def main(): + sample = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + content_type, width, height = inspect_upload(sample, "image/png", 10485760, 40000000) + check(content_type == "image/png" and (width, height) == (320, 200), "sample header mismatch") + rendered = render_variants(decode_image(sample, 40000000), pipeline_params(1)) + check(jpeg_size(rendered["thumbnail"]["bytes"]) == (256, 160), "thumbnail geometry") + check((rendered["thumbnail"]["width"], rendered["thumbnail"]["height"]) == (256, 160), "thumbnail metadata") + check(jpeg_size(rendered["crop"]["bytes"]) == (128, 128), "crop geometry") + check(rendered["optimized"]["bytes"][8:12] == b"WEBP", "optimized is not WebP") + check((rendered["optimized"]["width"], rendered["optimized"]["height"]) == (320, 200), "optimized geometry") + tiny = io.BytesIO() + Image.new("RGB", (20, 10), (1, 2, 3)).save(tiny, format="PNG") + tiny_rendered = render_variants(decode_image(tiny.getvalue(), 40000000), pipeline_params(1)) + check((tiny_rendered["thumbnail"]["width"], tiny_rendered["thumbnail"]["height"]) == (20, 10), "small thumbnail was upscaled") + check((tiny_rendered["crop"]["width"], tiny_rendered["crop"]["height"]) == (128, 128), "small crop was not upscaled") + oriented = io.BytesIO() + source = Image.new("RGB", (320, 200), (255, 0, 0)) + exif = source.getexif() + exif[274] = 6 + source.save(oriented, format="JPEG", exif=exif) + turned = decode_image(oriented.getvalue(), 40000000) + check(turned.size == (200, 320), "EXIF orientation was ignored") + turned_variants = render_variants(turned, pipeline_params(1)) + check((turned_variants["thumbnail"]["width"], turned_variants["thumbnail"]["height"]) == (160, 256), "oriented thumbnail geometry") + version_one = rendered["thumbnail"]["bytes"] + version_two = render_variants(decode_image(sample, 40000000), pipeline_params(2))["thumbnail"]["bytes"] + check(version_one != version_two, "pipeline versions are indistinguishable") + bomb = b"\x89PNG\r\n\x1a\n" + png_chunk(b"IHDR", struct.pack(">IIBBBBB", 8000, 8000, 8, 2, 0, 0, 0)) + png_chunk(b"IEND", b"") + try: + inspect_upload(bomb, "image/png", 10485760, 40000000) + except Exception as exc: + check(getattr(exc, "status", None) == 413, "pixel bomb was not rejected as too large") + else: + raise SystemExit("pixel bomb was accepted") + print("PASS: image contract geometry") + + +if __name__ == "__main__": + main()