diff --git a/.env.example b/.env.example index 119384b..646a832 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,44 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Copy to .env for a LOCAL synthetic stand. Never commit .env. +# Compose interpolates the variables below. The API also accepts *_FILE secrets. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +LOG_LEVEL=info +SHUTDOWN_GRACE_SECONDS=10 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +CACHE_TTL_SECONDS=60 +# Sessions stay in Redis, carried from lab3/lab4. +SESSION_STORE=redis API_REPLICAS=2 +API_CPUS=0.50 +API_MEM=256m +POSTGRES_CPUS=1.0 +POSTGRES_MEM=512m +DB_POOL_MAX=4 +LB_DRAIN_SECONDS=2 IDEMPOTENCY_TTL_SECONDS=86400 + +POSTGRES_PASSWORD=lab5-dev-only-change-me +DATABASE_URL=postgresql://messenger:lab5-dev-only-change-me@postgres:5432/messenger +REDIS_URL=redis://redis:6379/0 + S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=lab5appkey00000001 +S3_SECRET_ACCESS_KEY=lab5-dev-only-change-me MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +ORPHAN_GRACE_SECONDS=120 +S3_PUBLIC_ENDPOINT=http://127.0.0.1:9000 +PRESIGN_TTL_SECONDS=60 + +# RabbitMQ is the durable broker. The URL is built in compose.yaml from this password. +RABBITMQ_PASSWORD=lab5-dev-only-change-me +BROKER_URL=amqp://messenger:lab5-dev-only-change-me@rabbitmq:5672/ WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_BACKOFF_BASE_SECONDS=0.5 +JOB_BACKOFF_CAP_SECONDS=30 STAGING_RETENTION_SECONDS=86400 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a..486cf77 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Булацев, БПИ2301 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: нет -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: база lab4 (API ×2, Postgres, Redis, SeaweedFS, вложения). Upload переведён на 202: staging в S3, задание в Postgres, RabbitMQ, отдельный worker, thumbnail/crop/optimized, outbox между COMMIT и publish. +Команды запуска из чистого clone: см. REPORT.md (`cp .env.example .env`, `docker compose --env-file .env up -d --build --scale api=2`). +Результаты проверок и ссылка на REPORT.md: smoke PASS, `scripts/lab5_checks.py` PASS, `scripts/grade4_checks.py` PASS, разрыв с остановленным RabbitMQ доходит до ready. Подробности в REPORT.md. +Ограничения / известные ошибки: трек 5 не делался. Crash до ack и повтор persistence/failover на этом стенде ещё не прогонялись. Пока lab4 занимает 8080, этот стенд поднимался на 8081. +Источники и использование ИИ: контракт репозитория, документация RabbitMQ и Pillow. Код и прогон проверок — с помощью Cursor. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. - [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/REPORT.md b/REPORT.md index e98f75e..7c51cf7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,84 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Булацев, БПИ2301 +- Целевая оценка; выбранный трек 5, если нужен: 4. Трек 5 не заявляется. +- Commit/PR: локальный commit после переноса lab4 +- Использованные источники, библиотеки и помощь ИИ: контракт репозитория, RabbitMQ reliability / publisher confirms, aio-pika, Pillow. Реализация и прогон проверок сделаны с помощью Cursor. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose v2 и Python 3.9+ на хосте. Образы закреплены: `python:3.12.8-slim-bookworm`, `postgres:16.6-alpine`, `redis:7.4.2-alpine`, `haproxy:2.8.14-alpine`, `chrislusf/seaweedfs:3.85`, `rabbitmq:3.13.7-alpine`. + +```sh +cp .env.example .env +# заменить синтетические пароли в локальном .env; файл не коммитить +docker compose --env-file .env up -d --build --scale api=2 +``` + +`migrate` применяет SQL один раз, включая `005_jobs.sql`. `s3-init` создаёт приватный bucket. Брокер — RabbitMQ с томом `rabbitmq_data`. Очереди `attachment.jobs`, `attachment.jobs.retry` и `attachment.jobs.dlq` объявляет приложение при первом соединении. Снаружи опубликован только `127.0.0.1:8080` у HAProxy. Реплики API, worker, Postgres, Redis, RabbitMQ и SeaweedFS наружу не публикуются. Порт S3 `127.0.0.1:9000` оставлен для presigned PUT с хоста, как в lab4. + +Готовность: `GET http://127.0.0.1:8080/health/ready` отвечает 200, когда доступны Postgres, Redis и bucket. Недоступный брокер не переводит API в 503 и не заставляет API делать resize: задание остаётся в таблице `attachment_jobs` с `published_at IS NULL`. + +```sh +python -X utf8 scripts/check_contract.py +python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080 +python -X utf8 scripts/lab5_checks.py +python -X utf8 scripts/grade4_checks.py +``` + +Остановка без удаления данных: `docker compose --env-file .env stop`. Остановить только worker: `sh scripts/stop-worker.sh`. Полное удаление учебных volumes: `docker compose --env-file .env down -v`. + +На этой машине lab4 уже занимает `127.0.0.1:8080` и `127.0.0.1:9000`, поэтому прогон ниже шёл через дополнительный файл `compose.altports.yaml` (не часть сдачи): API на `127.0.0.1:8081`, S3 на `127.0.0.1:9001`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Из lab4 перенесены две реплики API за HAProxy, Postgres, Redis с AOF, SeaweedFS, сессии, идемпотентность сообщений и presigned upload. В этой работе приём файла отделён от обработки. + +`POST /attachments` по-прежнему читает все байты клиента. Ответ `202` со `status=queued` возвращается только после PUT в приватный prefix `staging/` и COMMIT строки вложения и задания. В сообщение брокера входят `job_id`, `attachment_id`, `pipeline_version`, `attempt` и `created_at`. Байтов, ключей доступа и клиентского пути там нет. + +Публикация идёт из outbox: `published_at` ставится после publisher confirm. Повтор той же строки даёт то же `job_id`. Worker — отдельный процесс `python -m app.worker`. Он копирует оригинал в `originals/`, строит варианты и только потом делает ack. Пока статус не `ready`, `GET .../content` отвечает 409. `queued` и `processing` можно прикрепить к сообщению, `failed` — нельзя. + +Обязательный thumbnail: JPEG quality 90, вписан в 256×256 без увеличения, размеры округляются вниз, EXIF orientation применяется, альфа кладётся на белый. Fixture 320×200 даёт 256×160. На уровне 4 вместе с ним пишутся crop (центрированный JPEG 128×128, маленькие кадры увеличиваются) и optimized (WebP quality 80, method 6, вписан в 1280×1280 без увеличения, EXIF не копируется). `ready` ставится одним COMMIT вместе со всеми тремя вариантами. Ключ эффекта — `(attachment_id, pipeline_version, name)`, pipeline_version=1. + +Повтор: до 5 попыток (`JOB_MAX_ATTEMPTS`). Временный сбой возвращает job в `queued` с backoff `0.5 * 2^(attempt-1)` секунд, не больше 30, и множителем 0.5–1.5. Постоянная ошибка картинки — `invalid_image` без повторов. Исчерпание попыток — `retry_exhausted`. Оба финальных кода пишутся в Postgres и лучшим усилием в DLQ. Повторная доставка уже `ready` job только проверяет строку и делает ack. Явный replay: `python -m app.replay JOB_ID` или `POST /api/v1/_lab/attachments/{id}/replay` при `APP_ENV=lab`. Для `ready` это no-op, `job_id` не меняется. + +Lease равен `JOB_TIMEOUT_SECONDS + 15`. Просроченный `processing` забирает reaper и публикует ту же job снова. Prefetch равен `WORKER_CONCURRENCY` (по умолчанию 2). Staging удаляется после COMMIT результата. Байты `failed` в prefix `staging/` снимаются не раньше `STAGING_RETENTION_SECONDS` (86400); метаданные остаются. + +| Роль | Сервис | Образ | Проверка | Volume | Порт | +| --- | --- | --- | --- | --- | --- | +| api | api ×2 | сборка `mtusi-messenger:lab5` | `/health/ready` | нет | только внутри сети | +| worker | worker | тот же образ, другая команда | процесс жив, healthcheck выключен | нет | не опубликован | +| postgres | postgres | `postgres:16.6-alpine` | `pg_isready` | `pg_data` | не опубликован | +| redis | redis | `redis:7.4.2-alpine` | `redis-cli ping` | `redis_data` | не опубликован | +| load_balancer | lb | `haproxy:2.8.14-alpine` | backend `/health/ready` | нет | `127.0.0.1:8080` | +| s3 | s3 | `chrislusf/seaweedfs:3.85` | master status и порт 9000 | `s3_data` | `127.0.0.1:9000` для presign | +| broker | rabbitmq | `rabbitmq:3.13.7-alpine` | `rabbitmq-diagnostics ping` | `rabbitmq_data` | не опубликован | ## Доказательства критериев +Стенд: та же машина, что и lab4 (12th Gen Intel Core i5-12450H, 16 ГБ RAM). Лимиты: API 0.50 CPU / 256 МиБ, worker 1 CPU / 768 МиБ. Прогон 2026-11-09, base URL `http://127.0.0.1:8081`. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8081` | PASS: 202, затем ready, оригинал совпал, thumbnail 256×160 | вывод команды | +| Upload при остановленном worker | `python -X utf8 scripts/lab5_checks.py` | worker остановлен, upload 202, content 409, текстовое сообщение с этим id — 201; после start worker вложение `aae0c472-a799-4d51-97af-f88e7dbb0b5f` стало ready, варианты thumbnail/crop/optimized | вывод команды | +| Негативные файлы lab4 на новом коде | `python -X utf8 scripts/grade4_checks.py` | PASS: 202 queued, затем скачивание оригинала; 400/413/415/404 без 500 | вывод команды | +| Разрыв COMMIT и publish | `docker compose stop rabbitmq`, upload, SQL, `start rabbitmq` | upload 202, `published_at` NULL, status queued; после старта брокера job `1c92a0a4-f823-4f94-b3d9-c10c30bf2e4c` стал ready, вариантов 3 | вывод `psql` | +| Ограниченный retry, lease, DLQ, replay, crash до ack | код в `app/jobs.py`, `app/worker.py`, `WORKER_CRASH_POINT=after_s3` | в этом прогоне отдельно не воспроизводились | — | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +`scripts/restart.sh` перезапускает postgres, redis, s3, rabbitmq, api и worker и не удаляет volumes. `scripts/stop-one.sh` по-прежнему останавливает одну API-реплику. Повторный прогон persistence/failover на стенде lab5 в этом заходе не делался. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановлен worker, затем upload | 202 и сохранённое задание, content 409, чат жив | так и было; после start worker тот же id стал ready | около 1 с после start, файл не потерян | +| Остановлен RabbitMQ после COMMIT | 202, `published_at` пустой; после старта брокера задача доходит до ready | SQL показал unpublished=t, затем ready и 3 варианта | около 19 с после `start rabbitmq`, новый upload не создавался | + +Трек 5 не выполнялся. Отдельного замера throughput/RSS очереди нет. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд выдерживает остановку worker и разрыв между COMMIT и подтверждением брокера, пока живы Postgres и том RabbitMQ. Он не выдерживает потерю тома Postgres, тома S3 или единственного Docker-host. Потеря тома RabbitMQ до confirm компенсируется outbox в Postgres; потеря staging-объекта до копирования в `originals/` делает job невыполнимой. + +Не сделано в этом прогоне: crash worker после PUT и до ack, явный replay failed, persistence/failover скрипты lab3 на новом стенде, трек 5. `WORKER_CRASH_POINT=after_s3` завершает процесс после записи объектов и до COMMIT; повторная доставка должна сойтись в те же ключи. Это ещё нужно показать на защите. diff --git a/app/attachments.py b/app/attachments.py index ca1e726..499bbe4 100644 --- a/app/attachments.py +++ b/app/attachments.py @@ -8,7 +8,7 @@ import logging from datetime import datetime, timezone from uuid import UUID, uuid4 -from app import config, db, s3 +from app import config, db, jobs, s3 from app.errors import AppError from app.store import format_ts @@ -16,7 +16,8 @@ logger = logging.getLogger("messenger.attachments") _sweep_task: asyncio.Task | None = None -def _public(row) -> dict: +def _public(row, variants=None) -> dict: + error_code = row["error_code"] if "error_code" in row.keys() else None return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), @@ -26,13 +27,47 @@ def _public(row) -> dict: "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], - "variants": [], - "error_code": None, + "variants": variants or [], + "error_code": error_code if row["status"] == "failed" else None, "created_at": format_ts(row["created_at"]), "updated_at": format_ts(row["updated_at"]), } +async def _variants(attachment_id) -> list: + async with db.require().acquire() as conn: + rows = await conn.fetch( + """ + SELECT name, content_type, size_bytes, width, height, sha256 + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = 1 + ORDER BY CASE name + WHEN 'thumbnail' THEN 1 + WHEN 'crop' THEN 2 + WHEN 'optimized' THEN 3 + ELSE 4 + END + """, + attachment_id, + ) + return [ + { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + for row in rows + ] + + +async def present(row) -> dict: + variants = await _variants(row["id"]) if row["status"] == "ready" else [] + return _public(row, variants) + + async def _drop_intent(key: str) -> None: async with db.require().acquire() as conn: await conn.execute( @@ -61,7 +96,7 @@ async def create_attachment( data: bytes, ) -> dict: attachment_id = uuid4() - key = f"originals/{attachment_id}" + key = f"staging/{attachment_id}" created = datetime.now(timezone.utc) digest = hashlib.sha256(data).hexdigest() async with db.require().acquire() as conn: @@ -93,7 +128,7 @@ async def create_attachment( id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, error_code, created_at, updated_at ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ready', NULL, $9, $9) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', NULL, $9, $9) """, attachment_id, UUID(chat_id), @@ -105,6 +140,7 @@ async def create_attachment( key, created, ) + await jobs.insert_job(conn, attachment_id, created) await conn.execute( "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", key, @@ -115,6 +151,7 @@ async def create_attachment( except Exception: await _compensate(key) raise AppError(503, "service unavailable") from None + jobs.kick_publish() return _public( { "id": attachment_id, @@ -124,7 +161,8 @@ async def create_attachment( "content_type": content_type, "size_bytes": len(data), "sha256": digest, - "status": "ready", + "status": "queued", + "error_code": None, "created_at": created, "updated_at": created, } @@ -136,7 +174,8 @@ async def get_for_member(attachment_id: str, user_id: str): row = await conn.fetchrow( """ SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, - a.size_bytes, a.sha256, a.object_key, a.status, a.created_at, a.updated_at + a.size_bytes, a.sha256, a.object_key, a.status, a.error_code, + a.created_at, a.updated_at FROM attachments a JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 WHERE a.id = $1 @@ -167,9 +206,31 @@ async def sweep_orphans() -> int: for row in rows: key = row["object_key"] linked = await conn.fetchval( - "SELECT 1 FROM attachments WHERE object_key = $1", + """ + SELECT 1 FROM attachments WHERE object_key = $1 + UNION ALL + SELECT 1 FROM attachment_variants WHERE object_key = $1 + LIMIT 1 + """, key, ) + busy = False + parts = key.split("/") + if len(parts) >= 2: + try: + owner = UUID(parts[1]) + except ValueError: + owner = None + if owner is not None: + busy = await conn.fetchval( + """ + SELECT 1 FROM attachment_jobs + WHERE attachment_id = $1 AND status IN ('queued', 'processing') + """, + owner, + ) + if busy: + continue if linked: await conn.execute( "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", @@ -192,10 +253,35 @@ async def sweep_orphans() -> int: return deleted +async def sweep_failed_staging() -> int: + """Drop staging bytes of failed jobs after the retention window. Metadata stays.""" + deleted = 0 + async with db.require().acquire() as conn: + rows = await conn.fetch( + """ + SELECT id, object_key + FROM attachments + WHERE status = 'failed' + AND object_key LIKE 'staging/%' + AND updated_at < now() - ($1::int * interval '1 second') + ORDER BY updated_at + LIMIT 20 + """, + config.STAGING_RETENTION_SECONDS, + ) + for row in rows: + await s3.delete_key(row["object_key"]) + async with db.require().acquire() as conn: + await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", row["object_key"]) + deleted += 1 + return deleted + + async def _sweep_loop() -> None: while True: try: await sweep_orphans() + await sweep_failed_staging() except asyncio.CancelledError: raise except Exception: diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 0000000..812e2dc --- /dev/null +++ b/app/broker.py @@ -0,0 +1,124 @@ +"""Durable RabbitMQ topology. Messages contain job ids, never file bytes.""" + +from __future__ import annotations + +import asyncio +import json +import logging + +import aio_pika + +from app import config + +logger = logging.getLogger("messenger.broker") + +EXCHANGE = "attachment.jobs" +QUEUE = "attachment.jobs" +RETRY_QUEUE = "attachment.jobs.retry" +DLQ = "attachment.jobs.dlq" +ROUTING_KEY = "process" + +_lock = asyncio.Lock() +_connection: aio_pika.RobustConnection | None = None +_channel: aio_pika.RobustChannel | None = None +_exchange: aio_pika.Exchange | None = None + + +async def _ensure() -> tuple[aio_pika.RobustChannel, aio_pika.Exchange]: + global _connection, _channel, _exchange + async with _lock: + if _channel is not None and not _channel.is_closed: + assert _exchange is not None + return _channel, _exchange + if not config.BROKER_URL: + raise RuntimeError("BROKER_URL is not set") + _connection = await aio_pika.connect_robust(config.BROKER_URL) + _channel = await _connection.channel(publisher_confirms=True) + _exchange = await _channel.declare_exchange(EXCHANGE, aio_pika.ExchangeType.DIRECT, durable=True) + await _channel.declare_queue( + RETRY_QUEUE, + durable=True, + arguments={ + "x-dead-letter-exchange": EXCHANGE, + "x-dead-letter-routing-key": ROUTING_KEY, + }, + ) + await _channel.declare_queue(DLQ, durable=True) + queue = await _channel.declare_queue(QUEUE, durable=True) + await queue.bind(_exchange, routing_key=ROUTING_KEY) + return _channel, _exchange + + +async def publish_job(payload: dict) -> None: + channel, exchange = await _ensure() + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + await exchange.publish( + aio_pika.Message( + body=body, + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=payload["job_id"], + ), + routing_key=ROUTING_KEY, + mandatory=True, + ) + + +async def publish_delay(payload: dict, delay_seconds: float) -> None: + channel, _exchange = await _ensure() + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + milliseconds = max(1, int(delay_seconds * 1000)) + await channel.default_exchange.publish( + aio_pika.Message( + body=body, + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + expiration=str(milliseconds), + message_id=payload["job_id"], + ), + routing_key=RETRY_QUEUE, + mandatory=True, + ) + + +async def publish_dlq(payload: dict) -> None: + channel, _exchange = await _ensure() + body = json.dumps(payload, separators=(",", ":")).encode("utf-8") + await channel.default_exchange.publish( + aio_pika.Message( + body=body, + content_type="application/json", + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=payload["job_id"], + ), + routing_key=DLQ, + mandatory=True, + ) + + +async def consume(handler, prefetch: int): + channel, _exchange = await _ensure() + await channel.set_qos(prefetch_count=max(1, prefetch)) + queue = await channel.declare_queue(QUEUE, durable=True) + tag = await queue.consume(handler) + return queue, tag + + +async def close() -> None: + global _connection, _channel, _exchange + async with _lock: + if _connection is not None: + await _connection.close() + _connection = None + _channel = None + _exchange = None + + +def payload_from(raw: bytes) -> dict: + parsed = json.loads(raw.decode("utf-8")) + required = ("schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at") + if not isinstance(parsed, dict) or any(key not in parsed for key in required): + raise ValueError("job payload is incomplete") + if parsed["schema_version"] != 1 or parsed["type"] != "attachment.process.v1": + raise ValueError("job payload has the wrong type") + return parsed diff --git a/app/config.py b/app/config.py index ee6d6b0..b682a0d 100644 --- a/app/config.py +++ b/app/config.py @@ -47,6 +47,15 @@ MAX_UPLOAD_BYTES = min(int(_env("MAX_UPLOAD_BYTES", "10485760")), 10_485_760) MAX_IMAGE_PIXELS = min(int(_env("MAX_IMAGE_PIXELS", "40000000")), 40_000_000) ORPHAN_GRACE_SECONDS = int(_env("ORPHAN_GRACE_SECONDS", "120")) S3_SWEEP_INTERVAL_SECONDS = int(_env("S3_SWEEP_INTERVAL_SECONDS", "30")) +BROKER_URL = _file_or_env("BROKER_URL") +WORKER_CONCURRENCY = max(1, int(_env("WORKER_CONCURRENCY", "2"))) +JOB_MAX_ATTEMPTS = max(1, int(_env("JOB_MAX_ATTEMPTS", "5"))) +JOB_TIMEOUT_SECONDS = max(1, int(_env("JOB_TIMEOUT_SECONDS", "60"))) +JOB_LEASE_SECONDS = JOB_TIMEOUT_SECONDS + 15 +JOB_BACKOFF_BASE_SECONDS = float(_env("JOB_BACKOFF_BASE_SECONDS", "0.5")) +JOB_BACKOFF_CAP_SECONDS = float(_env("JOB_BACKOFF_CAP_SECONDS", "30")) +STAGING_RETENTION_SECONDS = int(_env("STAGING_RETENTION_SECONDS", "86400")) +WORKER_CRASH_POINT = _env("WORKER_CRASH_POINT", "") def lab_mode() -> bool: diff --git a/app/direct.py b/app/direct.py index 78caa96..5428dd7 100644 --- a/app/direct.py +++ b/app/direct.py @@ -9,6 +9,7 @@ from uuid import UUID, uuid4 from app import config, db, images, s3 from app.attachments import _compensate, _public +from app import jobs from app.errors import AppError from app.store import format_ts @@ -159,7 +160,7 @@ async def complete_upload(upload_id: str, owner_id: str, claimed_sha: str) -> di id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, error_code, created_at, updated_at ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'ready', NULL, $9, $9) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, 'queued', NULL, $9, $9) """, row["id"], row["chat_id"], @@ -171,6 +172,7 @@ async def complete_upload(upload_id: str, owner_id: str, claimed_sha: str) -> di row["object_key"], created, ) + await jobs.insert_job(conn, row["id"], created) await conn.execute( "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", row["object_key"], @@ -183,6 +185,7 @@ async def complete_upload(upload_id: str, owner_id: str, claimed_sha: str) -> di raise except Exception: raise AppError(503, "service unavailable") from None + jobs.kick_publish() return _public( { "id": row["id"], @@ -192,7 +195,8 @@ async def complete_upload(upload_id: str, owner_id: str, claimed_sha: str) -> di "content_type": row["content_type"], "size_bytes": len(data), "sha256": digest, - "status": "ready", + "status": "queued", + "error_code": None, "created_at": created, "updated_at": created, } diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 0000000..5224013 --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,619 @@ +"""Job rows are the transactional outbox. The broker never stores file bytes.""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import random +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from app import broker, config, db, pipeline, s3 +from app.errors import AppError +from app.pipeline import PermanentJobError, TransientJobError +from app.store import format_ts + +logger = logging.getLogger("messenger.jobs") +_outbox_task: asyncio.Task | None = None +_publish_lock = asyncio.Lock() + + +def message_of(row) -> dict: + return { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": int(row["pipeline_version"]), + "attempt": int(row["attempt"]), + "created_at": format_ts(row["created_at"]), + } + + +def backoff_seconds(attempt: int) -> float: + base = min(config.JOB_BACKOFF_CAP_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))) + return base * random.uniform(0.5, 1.5) + + +async def insert_job(conn, attachment_id: UUID, created: datetime) -> UUID: + job_id = uuid4() + await conn.execute( + """ + INSERT INTO attachment_jobs ( + job_id, attachment_id, pipeline_version, attempt, status, + next_attempt_at, created_at, updated_at + ) + VALUES ($1, $2, 1, 1, 'queued', $3, $3, $3) + """, + job_id, + attachment_id, + created, + ) + await conn.execute( + """ + INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) + VALUES ($1, $2, 'queued', 'upload committed') + """, + job_id, + attachment_id, + ) + return job_id + + +def kick_publish() -> None: + try: + asyncio.get_running_loop().create_task(publish_pending()) + except RuntimeError: + return + + +async def publish_pending(limit: int = 20) -> int: + if not config.BROKER_URL: + return 0 + async with _publish_lock: + try: + async with db.require().acquire() as conn: + async with conn.transaction(): + rows = await conn.fetch( + """ + SELECT job_id, attachment_id, pipeline_version, attempt, created_at + FROM attachment_jobs + WHERE published_at IS NULL + AND status = 'queued' + AND (next_attempt_at IS NULL OR next_attempt_at <= now()) + ORDER BY created_at + LIMIT $1 + FOR UPDATE SKIP LOCKED + """, + limit, + ) + for row in rows: + await broker.publish_job(message_of(row)) + await conn.execute( + "UPDATE attachment_jobs SET published_at = now() WHERE job_id = $1", + row["job_id"], + ) + return len(rows) + except Exception: + logger.warning("outbox publish failed; the job row stays unpublished") + return 0 + + +async def _outbox_loop() -> None: + while True: + try: + await publish_pending() + except asyncio.CancelledError: + raise + except Exception: + logger.warning("outbox loop failed") + await asyncio.sleep(0.25) + + +def start_outbox() -> None: + global _outbox_task + if _outbox_task is None or _outbox_task.done(): + _outbox_task = asyncio.create_task(_outbox_loop()) + + +async def stop_outbox() -> None: + global _outbox_task + task = _outbox_task + _outbox_task = None + if task is None: + return + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + await broker.close() + + +def _aware(value: datetime | None) -> datetime | None: + if value is None: + return None + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value + + +async def claim(job_id: UUID, pipeline_version: int): + token = uuid4() + async with db.require().acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + "SELECT * FROM attachment_jobs WHERE job_id = $1 FOR UPDATE", + job_id, + ) + if row is None: + return "missing", None + if int(row["pipeline_version"]) != pipeline_version: + return "stale", row + if row["status"] == "ready": + return "ready", row + attachment = await conn.fetchrow( + "SELECT status FROM attachments WHERE id = $1", + row["attachment_id"], + ) + if attachment is not None and attachment["status"] == "ready": + await conn.execute( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_token = NULL, + lease_until = NULL, updated_at = now() + WHERE job_id = $1 AND status <> 'ready' + """, + job_id, + ) + return "ready", row + if row["status"] == "failed": + return "failed", row + lease = _aware(row["lease_until"]) + if row["status"] == "processing" and lease is not None and lease > datetime.now(timezone.utc): + return "busy", row + if int(row["attempt"]) > config.JOB_MAX_ATTEMPTS: + await _fail(conn, row, "retry_exhausted") + return "failed", row + await conn.execute( + """ + UPDATE attachment_jobs + SET status = 'processing', + lease_token = $2, + lease_until = now() + ($3::int * interval '1 second'), + error_code = NULL, + updated_at = now() + WHERE job_id = $1 + """, + job_id, + token, + config.JOB_LEASE_SECONDS, + ) + await conn.execute( + """ + UPDATE attachments + SET status = 'processing', error_code = NULL, updated_at = now() + WHERE id = $1 AND status <> 'ready' + """, + row["attachment_id"], + ) + claimed = dict(row) + claimed["lease_token"] = token + claimed["status"] = "processing" + return "claimed", claimed + + +async def _fail(conn, row, code: str) -> None: + await conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', error_code = $2, lease_token = NULL, lease_until = NULL, updated_at = now() + WHERE job_id = $1 AND status <> 'ready' + """, + row["job_id"], + code, + ) + await conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = $2, updated_at = now() + WHERE id = $1 AND status <> 'ready' + """, + row["attachment_id"], + code, + ) + await conn.execute( + """ + INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) + VALUES ($1, $2, 'failed', $3) + """, + row["job_id"], + row["attachment_id"], + code, + ) + + +async def fail_job(job_id: UUID, lease_token: UUID, code: str) -> bool: + async with db.require().acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + """ + SELECT * FROM attachment_jobs + WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' + FOR UPDATE + """, + job_id, + lease_token, + ) + if row is None: + return False + await _fail(conn, row, code) + failed = dict(row) + try: + await broker.publish_dlq(message_of(failed)) + except Exception: + logger.warning("failed job is stored; DLQ publish will be retried by an operator replay") + return True + + +async def schedule_retry(job_id: UUID, lease_token: UUID, code: str) -> str: + async with db.require().acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + """ + SELECT * FROM attachment_jobs + WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' + FOR UPDATE + """, + job_id, + lease_token, + ) + if row is None: + return "lost" + nxt = int(row["attempt"]) + 1 + if nxt > config.JOB_MAX_ATTEMPTS: + await _fail(conn, row, "retry_exhausted") + failed = True + else: + delay = backoff_seconds(int(row["attempt"])) + await conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', + attempt = $2, + error_code = $3, + lease_token = NULL, + lease_until = NULL, + published_at = NULL, + next_attempt_at = now() + ($4::double precision * interval '1 second'), + updated_at = now() + WHERE job_id = $1 + """, + job_id, + nxt, + code, + delay, + ) + await conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = $1 AND status <> 'ready' + """, + row["attachment_id"], + ) + await conn.execute( + """ + INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) + VALUES ($1, $2, 'retry', $3) + """, + job_id, + row["attachment_id"], + code, + ) + failed = False + failed_row = None + if failed: + failed_row = row + if failed: + try: + await broker.publish_dlq(message_of(failed_row)) + except Exception: + logger.warning("retry exhausted; DLQ publish failed") + return "failed" + return "retry" + + +async def _intent(key: str) -> None: + async with db.require().acquire() as conn: + await conn.execute( + """ + INSERT INTO blob_writes (object_key, committed) + VALUES ($1, FALSE) + ON CONFLICT (object_key) DO NOTHING + """, + key, + ) + + +async def complete_job(job_id: UUID, lease_token: UUID, attachment_id: UUID, final_key: str, variants: list) -> bool: + async with db.require().acquire() as conn: + async with conn.transaction(): + owned = await conn.fetchval( + """ + UPDATE attachment_jobs + SET status = 'ready', error_code = NULL, lease_token = NULL, lease_until = NULL, updated_at = now() + WHERE job_id = $1 AND lease_token = $2 AND status = 'processing' + RETURNING job_id + """, + job_id, + lease_token, + ) + if owned is None: + return False + current = await conn.fetchrow( + "SELECT status FROM attachments WHERE id = $1 FOR UPDATE", + attachment_id, + ) + if current is None: + raise PermanentJobError("invalid_image") + if current["status"] != "ready": + await conn.execute( + """ + UPDATE attachments + SET status = 'ready', object_key = $2, error_code = NULL, updated_at = now() + WHERE id = $1 AND status <> 'ready' + """, + attachment_id, + final_key, + ) + for variant in variants: + await conn.execute( + """ + INSERT INTO attachment_variants ( + attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, object_key + ) + VALUES ($1, 1, $2, $3, $4, $5, $6, $7, $8) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING + """, + attachment_id, + variant["name"], + variant["content_type"], + variant["size_bytes"], + variant["width"], + variant["height"], + variant["sha256"], + variant["object_key"], + ) + await conn.execute( + "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", + variant["object_key"], + ) + await conn.execute( + "UPDATE blob_writes SET committed = TRUE WHERE object_key = $1", + final_key, + ) + await conn.execute( + """ + INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) + VALUES ($1, $2, 'ready', 'variants committed') + """, + job_id, + attachment_id, + ) + return True + + +async def load_attachment(attachment_id: UUID): + async with db.require().acquire() as conn: + return await conn.fetchrow( + """ + SELECT id, filename, content_type, size_bytes, sha256, object_key, status + FROM attachments + WHERE id = $1 + """, + attachment_id, + ) + + +async def process(job_id: UUID, pipeline_version: int) -> str: + state, row = await claim(job_id, pipeline_version) + if state != "claimed": + return state + assert row is not None + lease = row["lease_token"] + try: + await asyncio.wait_for(_effect(row, lease), config.JOB_TIMEOUT_SECONDS) + except PermanentJobError as exc: + await fail_job(job_id, lease, exc.code) + return "failed" + except (TransientJobError, asyncio.TimeoutError, AppError): + return await schedule_retry(job_id, lease, "processing_failed") + except Exception: + logger.warning("job processing failed job_id=%s", job_id) + return await schedule_retry(job_id, lease, "processing_failed") + return "ready" + + +async def _effect(row, lease: UUID) -> None: + attachment = await load_attachment(row["attachment_id"]) + if attachment is None: + raise PermanentJobError("invalid_image") + if attachment["status"] == "ready": + committed = await complete_job(row["job_id"], lease, attachment["id"], attachment["object_key"], []) + if not committed: + raise TransientJobError("processing_failed") + return + data = await s3.get_bytes(attachment["object_key"]) + if len(data) != attachment["size_bytes"] or hashlib.sha256(data).hexdigest() != attachment["sha256"]: + raise PermanentJobError("invalid_image") + final_key = f"originals/{attachment['id']}" + if attachment["object_key"] != final_key: + await _intent(final_key) + await s3.put_bytes(final_key, data, attachment["content_type"]) + rendered = await asyncio.to_thread(pipeline.render, data, attachment["content_type"]) + stored = [] + for name, (payload, content_type, width, height) in rendered.items(): + suffix = "webp" if content_type == "image/webp" else "jpg" + key = f"variants/{attachment['id']}/{name}.{suffix}" + await _intent(key) + await s3.put_bytes(key, payload, content_type) + stored.append( + { + "name": name, + "content_type": content_type, + "size_bytes": len(payload), + "width": width, + "height": height, + "sha256": hashlib.sha256(payload).hexdigest(), + "object_key": key, + } + ) + if config.WORKER_CRASH_POINT == "after_s3": + logger.warning("injected crash after object write before result commit job_id=%s", row["job_id"]) + os._exit(1) + committed = await complete_job(row["job_id"], lease, attachment["id"], final_key, stored) + if not committed: + raise TransientJobError("processing_failed") + if attachment["object_key"] != final_key: + try: + await s3.delete_key(attachment["object_key"]) + async with db.require().acquire() as conn: + await conn.execute("DELETE FROM blob_writes WHERE object_key = $1", attachment["object_key"]) + except Exception: + logger.warning("staging object left for retention attachment_id=%s", attachment["id"]) + + +async def replay_attachment(attachment_id: str) -> dict: + async with db.require().acquire() as conn: + job_id = await conn.fetchval( + """ + SELECT job_id FROM attachment_jobs + WHERE attachment_id = $1 AND pipeline_version = 1 + """, + UUID(attachment_id), + ) + if job_id is None: + return {"result": "missing"} + return await replay(job_id) + + +async def replay(job_id: UUID) -> dict: + async with db.require().acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + "SELECT * FROM attachment_jobs WHERE job_id = $1 FOR UPDATE", + job_id, + ) + if row is None: + return {"result": "missing"} + if row["status"] == "ready": + await conn.execute( + """ + INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) + VALUES ($1, $2, 'replay', 'noop ready') + """, + row["job_id"], + row["attachment_id"], + ) + return {"result": "noop", "job_id": str(row["job_id"])} + lease = _aware(row["lease_until"]) + if row["status"] == "processing" and lease is not None and lease > datetime.now(timezone.utc): + return {"result": "busy", "job_id": str(row["job_id"])} + await conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', + attempt = 1, + error_code = NULL, + lease_token = NULL, + lease_until = NULL, + published_at = NULL, + next_attempt_at = now(), + updated_at = now() + WHERE job_id = $1 AND status <> 'ready' + """, + job_id, + ) + await conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = $1 AND status <> 'ready' + """, + row["attachment_id"], + ) + await conn.execute( + """ + INSERT INTO attachment_job_events (job_id, attachment_id, kind, detail) + VALUES ($1, $2, 'replay', 'operator replay') + """, + row["job_id"], + row["attachment_id"], + ) + await publish_pending() + return {"result": "queued", "job_id": str(job_id)} + + +async def reap_expired() -> int: + async with db.require().acquire() as conn: + async with conn.transaction(): + rows = await conn.fetch( + """ + SELECT job_id, attachment_id, attempt + FROM attachment_jobs + WHERE status = 'processing' AND lease_until < now() + FOR UPDATE SKIP LOCKED + """ + ) + changed = 0 + for row in rows: + nxt = int(row["attempt"]) + 1 + if nxt > config.JOB_MAX_ATTEMPTS: + await conn.execute( + """ + UPDATE attachment_jobs + SET status = 'failed', error_code = 'retry_exhausted', + lease_token = NULL, lease_until = NULL, updated_at = now() + WHERE job_id = $1 AND status = 'processing' + """, + row["job_id"], + ) + await conn.execute( + """ + UPDATE attachments + SET status = 'failed', error_code = 'retry_exhausted', updated_at = now() + WHERE id = $1 AND status <> 'ready' + """, + row["attachment_id"], + ) + else: + await conn.execute( + """ + UPDATE attachment_jobs + SET status = 'queued', + attempt = $2, + error_code = 'lease_expired', + lease_token = NULL, + lease_until = NULL, + published_at = NULL, + next_attempt_at = now(), + updated_at = now() + WHERE job_id = $1 AND status = 'processing' + """, + row["job_id"], + nxt, + ) + await conn.execute( + """ + UPDATE attachments + SET status = 'queued', error_code = NULL, updated_at = now() + WHERE id = $1 AND status <> 'ready' + """, + row["attachment_id"], + ) + changed += 1 + return changed diff --git a/app/main.py b/app/main.py index 2c56af8..9a450e2 100644 --- a/app/main.py +++ b/app/main.py @@ -12,7 +12,7 @@ from starlette.requests import Request from starlette.responses import Response from starlette.routing import Route -from app import attachments, config, db, rediscli +from app import attachments, config, db, jobs, rediscli from app.docs import openapi_json, swagger_ui from app.errors import AppError, error_response, from_app_error from app.middleware import RequestContextMiddleware @@ -30,6 +30,7 @@ from app.routes import ( health_ready, lab_flags, lab_metrics, + lab_replay, lab_sweep, list_chats, list_messages, @@ -70,10 +71,12 @@ async def startup() -> None: await db.apply_migrations() await rediscli.connect() attachments.start_sweeper() + jobs.start_outbox() async def shutdown() -> None: await attachments.stop_sweeper() + await jobs.stop_outbox() await rediscli.close() await db.close() @@ -102,6 +105,7 @@ def build_app() -> Starlette: Route("/api/v1/_lab/metrics", lab_metrics, methods=["GET"]), Route("/api/v1/_lab/flags", lab_flags, methods=["POST"]), Route("/api/v1/_lab/sweep", lab_sweep, methods=["POST"]), + Route("/api/v1/_lab/attachments/{attachment_id}/replay", lab_replay, methods=["POST"]), ], exception_handlers={404: not_found_handler, AppError: app_error_handler}, on_startup=[startup], diff --git a/app/pipeline.py b/app/pipeline.py new file mode 100644 index 0000000..e05c01d --- /dev/null +++ b/app/pipeline.py @@ -0,0 +1,95 @@ +"""Image variants. The worker calls this; the API process does not resize.""" + +from __future__ import annotations + +import io +import math + +from PIL import Image, ImageOps, UnidentifiedImageError + +from app import config +from app.images import ImageRejected, inspect_image + + +class PermanentJobError(Exception): + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +class TransientJobError(Exception): + def __init__(self, code: str = "processing_failed") -> None: + super().__init__(code) + self.code = code + + +def _flatten(image: Image.Image) -> Image.Image: + if image.mode == "RGB": + return image + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + + +def _open_rgb(data: bytes, content_type: str) -> Image.Image: + try: + inspect_image(data, content_type) + except ImageRejected as exc: + raise PermanentJobError("invalid_image") from exc + Image.MAX_IMAGE_PIXELS = config.MAX_IMAGE_PIXELS + try: + with Image.open(io.BytesIO(data)) as image: + image.load() + oriented = ImageOps.exif_transpose(image) or image + return _flatten(oriented).copy() + except Image.DecompressionBombError as exc: + raise PermanentJobError("invalid_image") from exc + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc: + raise PermanentJobError("invalid_image") from exc + + +def _fit_down(image: Image.Image, box: int) -> Image.Image: + width, height = image.size + scale = min(box / width, box / height, 1.0) + target = (max(1, math.floor(width * scale)), max(1, math.floor(height * scale))) + if target == (width, height): + return image + return image.resize(target, Image.Resampling.LANCZOS) + + +def _crop_square(image: Image.Image, side: int) -> Image.Image: + width, height = image.size + edge = min(width, height) + left = (width - edge) // 2 + top = (height - edge) // 2 + cropped = image.crop((left, top, left + edge, top + edge)) + if cropped.size != (side, side): + cropped = cropped.resize((side, side), Image.Resampling.LANCZOS) + return cropped + + +def _jpeg(image: Image.Image) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=90, optimize=True) + return buffer.getvalue() + + +def _webp(image: Image.Image) -> bytes: + # quality=80, method=6. EXIF is omitted because the saved image is a new RGB buffer. + buffer = io.BytesIO() + image.save(buffer, format="WEBP", quality=80, method=6) + return buffer.getvalue() + + +def render(data: bytes, content_type: str) -> dict: + """Return thumbnail, crop and optimized payloads. Does not touch storage.""" + image = _open_rgb(data, content_type) + thumb = _fit_down(image, 256) + crop = _crop_square(image, 128) + optimized = _fit_down(image, 1280) + return { + "thumbnail": ( _jpeg(thumb), "image/jpeg", thumb.size[0], thumb.size[1]), + "crop": (_jpeg(crop), "image/jpeg", crop.size[0], crop.size[1]), + "optimized": (_webp(optimized), "image/webp", optimized.size[0], optimized.size[1]), + } diff --git a/app/replay.py b/app/replay.py new file mode 100644 index 0000000..bf0a962 --- /dev/null +++ b/app/replay.py @@ -0,0 +1,30 @@ +"""Operator replay of one failed job. A ready job is left unchanged.""" + +from __future__ import annotations + +import asyncio +import sys +from uuid import UUID + +from app import db, jobs + + +async def _run(job_id: UUID) -> int: + await db.connect() + try: + result = await jobs.replay(job_id) + finally: + await db.close() + print(result["result"]) + return 0 if result["result"] in {"noop", "queued"} else 1 + + +def main() -> None: + if len(sys.argv) != 2: + print("usage: python -m app.replay JOB_ID", file=sys.stderr) + raise SystemExit(2) + raise SystemExit(asyncio.run(_run(UUID(sys.argv[1])))) + + +if __name__ == "__main__": + main() diff --git a/app/routes.py b/app/routes.py index a536aeb..76d2343 100644 --- a/app/routes.py +++ b/app/routes.py @@ -5,7 +5,7 @@ import re from starlette.requests import Request from starlette.responses import JSONResponse, Response -from app import attachments, cache, config, db, direct, images, rediscli, s3, sessions, store +from app import attachments, cache, config, db, direct, images, jobs, rediscli, s3, sessions, store from app.cursor_codec import decode_cursor, encode_cursor from app.errors import BASIC_CHALLENGE, BEARER_CHALLENGE, AppError, error_response from app.passwords import dummy_hash, hash_password, verify_password @@ -309,7 +309,7 @@ async def upload_attachment(request: Request) -> Response: except images.ImageRejected as exc: return error_response(exc.status, exc.message, rid) created = await attachments.create_attachment(chat_id, user["id"], filename, content_type, data) - return _json(created, rid, 201) + return _json(created, rid, 202) async def begin_direct_upload(request: Request) -> Response: @@ -360,7 +360,7 @@ async def complete_direct_upload(request: Request) -> Response: if not isinstance(digest, str) or re.fullmatch(r"[a-f0-9]{64}", digest) is None: return error_response(400, "invalid checksum", rid) created = await direct.complete_upload(upload_id, user["id"], digest) - return _json(created, rid, 201) + return _json(created, rid, 202) async def get_attachment(request: Request) -> Response: @@ -372,7 +372,7 @@ async def get_attachment(request: Request) -> Response: row = await attachments.get_for_member(attachment_id, user["id"]) if row is None: return error_response(404, "attachment not found", rid) - return _json(attachments._public(row), rid) + return _json(await attachments.present(row), rid) async def download_attachment(request: Request) -> Response: @@ -381,14 +381,46 @@ async def download_attachment(request: Request) -> Response: attachment_id = request.path_params["attachment_id"] if validate_uuid(attachment_id): return error_response(400, "invalid attachment id", rid) - variant = request.query_params.get("variant") - if variant not in (None, "original"): + variant = request.query_params.get("variant") or "original" + if variant not in ("original", "thumbnail", "crop", "optimized"): return error_response(400, "invalid variant", rid) row = await attachments.get_for_member(attachment_id, user["id"]) if row is None: return error_response(404, "attachment not found", rid) - payload = await s3.get_bytes(row["object_key"]) - return Response(payload, status_code=200, media_type=row["content_type"], headers={"X-Request-Id": rid}) + if row["status"] != "ready": + return error_response(409, "attachment is not ready", rid) + if variant == "original": + payload = await s3.get_bytes(row["object_key"]) + media_type = row["content_type"] + else: + async with db.require().acquire() as conn: + found = await conn.fetchrow( + """ + SELECT content_type, object_key + FROM attachment_variants + WHERE attachment_id = $1 AND pipeline_version = 1 AND name = $2 + """, + row["id"], + variant, + ) + if found is None: + return error_response(404, "attachment not found", rid) + payload = await s3.get_bytes(found["object_key"]) + media_type = found["content_type"] + return Response(payload, status_code=200, media_type=media_type, headers={"X-Request-Id": rid}) + + +async def lab_replay(request: Request) -> Response: + rid = request.state.request_id + if not config.lab_mode(): + return error_response(404, "not found", rid) + attachment_id = request.path_params["attachment_id"] + if validate_uuid(attachment_id): + return error_response(400, "invalid attachment id", rid) + result = await jobs.replay_attachment(attachment_id) + if result["result"] == "missing": + return error_response(404, "attachment not found", rid) + return _json(result, rid) async def lab_sweep(request: Request) -> Response: diff --git a/app/store.py b/app/store.py index 2942d3e..63227b4 100644 --- a/app/store.py +++ b/app/store.py @@ -283,7 +283,7 @@ async def add_message( str(found[item]["chat_id"]) != chat_id for item in attachment_ids ): return None, "not_found" - if any(found[item]["status"] != "ready" for item in attachment_ids): + if any(found[item]["status"] == "failed" for item in attachment_ids): return None, "not_ready" resolved = attachment_ids message_id = uuid4() diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 0000000..1be412e --- /dev/null +++ b/app/worker.py @@ -0,0 +1,97 @@ +"""Separate process: consume durable jobs and publish image variants.""" + +from __future__ import annotations + +import asyncio +import logging +import signal +import sys +from uuid import UUID + +import aio_pika + +from app import broker, config, db, jobs +from app.broker import payload_from + +logger = logging.getLogger("messenger.worker") + + +def configure_logging() -> None: + logging.basicConfig( + level=getattr(logging, config.LOG_LEVEL.upper(), logging.INFO), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, + ) + + +async def _settle(message: aio_pika.IncomingMessage) -> None: + try: + payload = payload_from(message.body) + job_id = UUID(payload["job_id"]) + pipeline_version = int(payload["pipeline_version"]) + except (ValueError, KeyError, TypeError): + logger.warning("dropping a job message that does not match the schema") + await message.ack() + return + try: + waits = 0 + while True: + state = await jobs.process(job_id, pipeline_version) + if state == "busy": + waits += 1 + if waits > config.JOB_LEASE_SECONDS + 5: + await message.reject(requeue=True) + return + await asyncio.sleep(1) + continue + await message.ack() + return + except Exception: + logger.warning("job settle failed job_id=%s", payload.get("job_id")) + await message.reject(requeue=True) + + +async def _reap_loop(stop: asyncio.Event) -> None: + while not stop.is_set(): + try: + count = await jobs.reap_expired() + if count: + logger.info("requeued %s expired leases", count) + except Exception: + logger.warning("lease reaper failed") + try: + await asyncio.wait_for(stop.wait(), timeout=2) + except asyncio.TimeoutError: + continue + + +async def serve() -> None: + await db.connect() + stop = asyncio.Event() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, stop.set) + except NotImplementedError: + signal.signal(sig, lambda *_: stop.set()) + reaper = asyncio.create_task(_reap_loop(stop)) + queue, consumer = await broker.consume(_settle, config.WORKER_CONCURRENCY) + logger.info("worker consuming prefetch=%s", config.WORKER_CONCURRENCY) + await stop.wait() + await queue.cancel(consumer) + reaper.cancel() + try: + await reaper + except asyncio.CancelledError: + pass + await broker.close() + await db.close() + + +def main() -> None: + configure_logging() + asyncio.run(serve()) + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml index f1a337b..80fd5fe 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,4 +1,5 @@ -# Client -> lb:8080 -> api x2 -> Postgres + Redis + private SeaweedFS S3. +# Client -> lb:8080 -> api x2 -> Postgres + Redis + private SeaweedFS S3 + RabbitMQ. +# Worker is a separate process and is the only place that resizes images. # API replicas do not publish host ports. MinIO console is not published. # API replicas do not publish host ports. Migrations run once in `migrate`. # Check period: HAProxy inter 1s, fall 2, rise 1. Worst detection of a dead @@ -10,7 +11,7 @@ services: build: context: . dockerfile: Dockerfile - image: mtusi-messenger:lab4 + image: mtusi-messenger:lab5 command: ["python", "-m", "app.migrate"] environment: APP_ENV: lab @@ -30,7 +31,7 @@ services: build: context: . dockerfile: Dockerfile - image: mtusi-messenger:lab4 + image: mtusi-messenger:lab5 environment: APP_HOST: 0.0.0.0 APP_PORT: "8080" @@ -52,6 +53,10 @@ services: MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-120} + BROKER_URL: amqp://messenger:${RABBITMQ_PASSWORD}@rabbitmq:5672/ + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} S3_SWEEP_INTERVAL_SECONDS: ${S3_SWEEP_INTERVAL_SECONDS:-30} S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://127.0.0.1:9000} PRESIGN_TTL_SECONDS: ${PRESIGN_TTL_SECONDS:-60} @@ -69,6 +74,8 @@ services: condition: service_healthy s3-init: condition: service_completed_successfully + rabbitmq: + condition: service_healthy expose: - "8080" restart: unless-stopped @@ -148,7 +155,7 @@ services: build: context: . dockerfile: Dockerfile - image: mtusi-messenger:lab4 + image: mtusi-messenger:lab5 command: ["python", "-m", "app.s3bootstrap"] environment: S3_ENDPOINT: http://s3:9000 @@ -161,6 +168,66 @@ services: condition: service_healthy restart: "no" + rabbitmq: + image: rabbitmq:3.13.7-alpine + hostname: rabbitmq + environment: + RABBITMQ_DEFAULT_USER: messenger + RABBITMQ_DEFAULT_PASS: ${RABBITMQ_PASSWORD} + RABBITMQ_DEFAULT_VHOST: / + volumes: + - rabbitmq_data:/var/lib/rabbitmq + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 15s + restart: unless-stopped + mem_limit: 256m + cpus: "0.50" + + worker: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.worker"] + environment: + APP_ENV: lab + LOG_LEVEL: info + DATABASE_URL: postgresql://messenger:${POSTGRES_PASSWORD}@postgres:5432/messenger + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL: amqp://messenger:${RABBITMQ_PASSWORD}@rabbitmq:5672/ + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + DB_POOL_MIN: "1" + DB_POOL_MAX: "4" + MIGRATIONS_ON_START: "0" + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + s3-init: + condition: service_completed_successfully + restart: unless-stopped + mem_limit: 768m + cpus: "1.0" + stop_grace_period: 20s + healthcheck: + disable: true + redis: image: redis:7.4.2-alpine command: @@ -187,3 +254,4 @@ volumes: pg_data: redis_data: s3_data: + rabbitmq_data: diff --git a/contracts/extensions.openapi.json b/contracts/extensions.openapi.json index a91a973..bdfa215 100644 --- a/contracts/extensions.openapi.json +++ b/contracts/extensions.openapi.json @@ -14,7 +14,7 @@ "/api/v1/uploads/{upload_id}/complete": { "post": { "operationId": "completeDirectUpload", - "summary": "Check owner, size and sha256 of the object that arrived, then store ready metadata." + "summary": "Check owner, size and sha256, store the original, and enqueue processing. Returns 202 queued." } } } diff --git a/migrations/005_jobs.sql b/migrations/005_jobs.sql new file mode 100644 index 0000000..4a54f67 --- /dev/null +++ b/migrations/005_jobs.sql @@ -0,0 +1,67 @@ +-- Async attachment processing. Bytes stay in S3. The broker message carries only ids. +-- 202 is returned after the staging object and this job row commit. Publish is a +-- later step: published_at stays null until the broker confirms, so a crash +-- between COMMIT and publish is recovered by the outbox loop. + +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments + ADD CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')); + +ALTER TABLE attachments DROP CONSTRAINT attachments_ready_error; +ALTER TABLE attachments + ADD CONSTRAINT attachments_error CHECK ( + (status = 'failed' AND error_code IS NOT NULL AND error_code ~ '^[a-z0-9_]{1,64}$') + OR (status <> 'failed' AND error_code IS NULL) + ); + +CREATE TABLE attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1), + name TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL CHECK (size_bytes >= 1), + width INTEGER NOT NULL CHECK (width >= 1), + height INTEGER NOT NULL CHECK (height >= 1), + sha256 TEXT NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key TEXT NOT NULL UNIQUE, + PRIMARY KEY (attachment_id, pipeline_version, name), + CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')) +); + +CREATE TABLE attachment_jobs ( + job_id UUID PRIMARY KEY, + attachment_id UUID NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1), + attempt INTEGER NOT NULL CHECK (attempt >= 1), + status TEXT NOT NULL, + error_code TEXT, + lease_until TIMESTAMPTZ, + lease_token UUID, + next_attempt_at TIMESTAMPTZ, + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_error CHECK ( + error_code IS NULL OR error_code ~ '^[a-z0-9_]{1,64}$' + ), + CONSTRAINT attachment_jobs_failed_error CHECK (status <> 'failed' OR error_code IS NOT NULL) +); + +CREATE INDEX attachment_jobs_outbox_idx + ON attachment_jobs (created_at) + WHERE published_at IS NULL AND status = 'queued'; + +CREATE INDEX attachment_jobs_lease_idx + ON attachment_jobs (lease_until) + WHERE status = 'processing'; + +CREATE TABLE attachment_job_events ( + id BIGSERIAL PRIMARY KEY, + job_id UUID NOT NULL, + attachment_id UUID NOT NULL, + kind TEXT NOT NULL, + detail TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); diff --git a/requirements.txt b/requirements.txt index d764a41..f8e8853 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,4 @@ redis==5.2.1 argon2-cffi==23.1.0 boto3==1.35.99 pillow==11.1.0 +aio-pika==9.5.5 diff --git a/scripts/grade4_checks.py b/scripts/grade4_checks.py index 5789b10..716f5ce 100644 --- a/scripts/grade4_checks.py +++ b/scripts/grade4_checks.py @@ -1,18 +1,21 @@ #!/usr/bin/env python3 """Negative attachment cases. Prints statuses only, never tokens.""" +import os import base64 +from urllib.parse import urlparse import hashlib import json import socket import struct +import time import uuid import urllib.error import urllib.request import zlib from pathlib import Path -BASE = "http://127.0.0.1:8080" +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") ROOT = Path(__file__).resolve().parents[1] JPEG = base64.b64decode( "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAAIAAgDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD5soooroMD/9k=" @@ -69,6 +72,19 @@ class Api: return response.status, response.headers, parsed, payload +def wait_ready(api: Api, attachment_id: str, token: str): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + status, _, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token) + expect(200, status, "poll attachment") + if body["status"] == "failed": + raise SystemExit("FAIL attachment failed") + if body["status"] == "ready": + return body + time.sleep(0.3) + raise SystemExit("FAIL attachment did not become ready") + + def expect(status, got, label): if got == 500: raise SystemExit("FAIL %s returned 500" % label) @@ -93,15 +109,17 @@ def user(api: Api, label: str): def declared_length(token, chat_id): + parsed = urlparse(BASE) + host = parsed.netloc or "127.0.0.1:8080" request = ( "POST /api/v1/chats/%s/attachments?filename=big.png HTTP/1.1\r\n" - "Host: 127.0.0.1:8080\r\n" + "Host: %s\r\n" "Authorization: Bearer %s\r\n" "Content-Type: image/png\r\n" "Content-Length: 10485761\r\n" - "Connection: close\r\n\r\n" % (chat_id, token) + "Connection: close\r\n\r\n" % (chat_id, host, token) ).encode() + b"\x89PNG" - sock = socket.create_connection(("127.0.0.1", 8080), timeout=5) + sock = socket.create_connection((parsed.hostname or "127.0.0.1", parsed.port or 80), timeout=5) try: sock.sendall(request) data = b"" @@ -157,9 +175,10 @@ def main(): raw=good, content_type="image/png", ) - expect(201, status, "png upload") - if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "ready": + expect(202, status, "png upload") + if attachment["sha256"] != hashlib.sha256(good).hexdigest() or attachment["status"] != "queued": raise SystemExit("FAIL png metadata") + attachment = wait_ready(api, attachment["id"], bob_token) status, headers, _, downloaded = api.call( "GET", "/api/v1/attachments/%s/content" % attachment["id"], @@ -178,7 +197,7 @@ def main(): raw=JPEG, content_type="image/jpeg", ) - expect(201, status, "jpeg upload") + expect(202, status, "jpeg upload") if jpeg_meta["content_type"] != "image/jpeg" or jpeg_meta["sha256"] != hashlib.sha256(JPEG).hexdigest(): raise SystemExit("FAIL jpeg metadata") @@ -269,7 +288,7 @@ def main(): raw=good, content_type="image/png", ) - expect(201, status, "upload into other chat") + expect(202, status, "upload into other chat") status, _, _, _ = api.call( "POST", "/api/v1/chats/%s/messages" % chat_id, diff --git a/scripts/grade4_infra.py b/scripts/grade4_infra.py index 83a088b..341d56d 100644 --- a/scripts/grade4_infra.py +++ b/scripts/grade4_infra.py @@ -180,7 +180,7 @@ def memory_sample(): check=False, ) for line in stats.stdout.splitlines(): - if "mtusi2026lab4-api-" in line: + if "mtusi2026lab5-api-" in line: samples.append(line.strip()) time.sleep(0.4) @@ -192,7 +192,7 @@ def memory_sample(): for thread in threads: thread.join() watcher.join() - if codes.count(201) != 10: + if codes.count(202) != 10: raise SystemExit("concurrent uploads %s" % codes) def mib(line): diff --git a/scripts/lab5_checks.py b/scripts/lab5_checks.py new file mode 100644 index 0000000..3aa505c --- /dev/null +++ b/scripts/lab5_checks.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Lab5 fault checks: worker down, content 409, then the same upload reaches ready. + +Prints statuses and ids only. Does not print tokens. +""" + +import base64 +import hashlib +import json +import os +import subprocess +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +FIXTURE = ROOT / "tests" / "fixtures" / "sample.png" + + +def compose(*args: str) -> None: + command = ["docker", "compose", "--env-file", ".env"] + extra = os.environ.get("LAB5_COMPOSE_FILES", "") + if extra: + for name in extra.split(os.pathsep): + if name: + command.extend(["-f", name]) + subprocess.run([*command, *args], cwd=ROOT, check=True) + + +class Api: + def __init__(self) -> None: + self.opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + + def call(self, method, path, body=None, token=None, raw=None, content_type=None, headers=None): + hdrs = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + hdrs["Content-Type"] = "application/json" + if content_type: + hdrs["Content-Type"] = content_type + if token: + hdrs["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE + path, data=data, headers=hdrs, method=method) + try: + response = self.opener.open(request, timeout=30) + except urllib.error.HTTPError as error: + response = error + with response: + payload = response.read() + parsed = json.loads(payload) if payload and response.headers.get_content_type() == "application/json" else None + return response.status, parsed, payload + + +def expect(status, got, label): + if got != status: + raise SystemExit("FAIL %s expected %s got %s" % (label, status, got)) + print("PASS", label, got) + + +def user(api: Api): + username = "l5_" + uuid.uuid4().hex[:10] + password = "T3st-" + uuid.uuid4().hex + status, body, _ = api.call( + "POST", + "/api/v1/users", + {"username": username, "display_name": "Lab5", "password": password}, + ) + expect(201, status, "register") + basic = base64.b64encode((username + ":" + password).encode()).decode() + status, session, _ = api.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + expect(201, status, "login") + return body["id"], session["token"] + + +def wait_ready(api: Api, attachment_id: str, token: str): + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token) + expect(200, status, "poll") + if body["status"] == "failed": + raise SystemExit("FAIL attachment failed " + str(body.get("error_code"))) + if body["status"] == "ready": + return body + time.sleep(0.4) + raise SystemExit("FAIL not ready") + + +def main() -> None: + api = Api() + alice, alice_token = user(api) + bob, bob_token = user(api) + status, chat, _ = api.call( + "POST", + "/api/v1/chats", + {"title": "lab5", "member_ids": [alice, bob]}, + token=alice_token, + ) + expect(201, status, "chat") + chat_id = chat["id"] + print("chat", chat_id) + + compose("stop", "worker") + content = FIXTURE.read_bytes() + status, attachment, _ = api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=sample.png" % chat_id, + token=alice_token, + raw=content, + content_type="image/png", + ) + expect(202, status, "upload while worker is stopped") + if attachment["status"] != "queued" or attachment["sha256"] != hashlib.sha256(content).hexdigest(): + raise SystemExit("FAIL queued metadata") + print("attachment", attachment["id"]) + status, _, _ = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token) + expect(409, status, "content before ready") + status, message, _ = api.call( + "POST", + "/api/v1/chats/%s/messages" % chat_id, + {"text": "очередь не блокирует чат", "attachment_ids": [attachment["id"]]}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + expect(201, status, "message while queued") + if message["attachment_ids"] != [attachment["id"]]: + raise SystemExit("FAIL queued attachment was not linkable") + + compose("start", "worker") + ready = wait_ready(api, attachment["id"], bob_token) + thumbs = [item for item in ready["variants"] if item["name"] == "thumbnail"] + if len(thumbs) != 1 or (thumbs[0]["width"], thumbs[0]["height"]) != (256, 160): + raise SystemExit("FAIL thumbnail metadata %s" % thumbs) + names = sorted(item["name"] for item in ready["variants"]) + if names != ["crop", "optimized", "thumbnail"]: + raise SystemExit("FAIL variants %s" % names) + status, _, payload = api.call( + "GET", + "/api/v1/attachments/%s/content?variant=thumbnail" % attachment["id"], + token=bob_token, + ) + expect(200, status, "thumbnail bytes") + if hashlib.sha256(payload).hexdigest() != thumbs[0]["sha256"]: + raise SystemExit("FAIL thumbnail checksum") + print("PASS lab5 worker-down upload became ready", attachment["id"]) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh index c3ad93f..1b74925 100644 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -4,7 +4,7 @@ set -eu ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) cd "$ROOT" if [ -f .env ]; then - docker compose --env-file .env restart postgres redis s3 api + docker compose --env-file .env restart postgres redis s3 rabbitmq api worker else - docker compose restart postgres redis s3 api + docker compose restart postgres redis s3 rabbitmq api worker fi diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100644 index 0000000..57952af --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Stop the worker and leave it stopped. The broker, API and volumes keep running. +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +if [ -f .env ]; then + docker compose --env-file .env stop worker +else + docker compose stop worker +fi diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py index ff0fde8..65afd0d 100644 --- a/scripts/track1_experiment.py +++ b/scripts/track1_experiment.py @@ -28,6 +28,19 @@ def expect(status, got, label): raise SystemExit("FAIL %s returned 500" % label) if got != status: raise SystemExit("FAIL %s expected %s got %s" % (label, status, got)) + + +def wait_ready(api, attachment_id, token): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + status, body, _ = api.call("GET", "/api/v1/attachments/" + attachment_id, token=token) + expect(200, status, "poll attachment") + if body["status"] == "failed": + raise SystemExit("FAIL attachment failed") + if body["status"] == "ready": + return body + time.sleep(0.3) + raise SystemExit("FAIL attachment did not become ready") print("PASS", label, got) @@ -109,7 +122,7 @@ def peak_during(action): check=False, ) for line in stats.stdout.splitlines(): - if "mtusi2026lab4-api-" not in line: + if "mtusi2026lab5-api-" not in line: continue value = line.split()[1] number = float(value[:-3]) @@ -251,9 +264,10 @@ def negatives(api, alice_token, bob_token, eve_token, chat_id): {"sha256": digest}, token=alice_token, ) - expect(201, status, "complete happy path") - if attachment["sha256"] != digest or attachment["status"] != "ready": + expect(202, status, "complete happy path") + if attachment["sha256"] != digest or attachment["status"] != "queued": raise SystemExit("FAIL complete metadata") + attachment = wait_ready(api, attachment["id"], bob_token) status, _, downloaded = api.call("GET", "/api/v1/attachments/%s/content" % attachment["id"], token=bob_token) expect(200, status, "member downloads direct upload") if downloaded != small: @@ -272,7 +286,7 @@ def negatives(api, alice_token, bob_token, eve_token, chat_id): raw=small, content_type="image/png", ) - expect(201, status, "proxy endpoint still works") + expect(202, status, "proxy endpoint still works") return proxy["id"] @@ -289,7 +303,7 @@ def compare(api, token, chat_id): content_type="image/png", ) ) - if proxy["result"][0] != 201: + if proxy["result"][0] != 202: raise SystemExit("FAIL proxy %s %s" % (label, proxy["result"][0])) status, started, _ = api.call( "POST", @@ -309,7 +323,7 @@ def compare(api, token, chat_id): token=token, ) ) - if direct_complete["result"][0] != 201: + if direct_complete["result"][0] != 202: raise SystemExit("FAIL complete %s %s" % (label, direct_complete["result"][0])) rows.append( {