From b61a1cae9edb0fe6539fa6128c70464652dc1da3 Mon Sep 17 00:00:00 2001 From: ferraronp Date: Mon, 16 Nov 2026 16:08:39 +0300 Subject: [PATCH] Implement lab5 async image processing with RabbitMQ and outbox --- .env.example | 38 ++- Dockerfile | 10 + Makefile | 15 +- REPORT.md | 128 ++++++-- app/__init__.py | 1 + app/__main__.py | 4 + app/api.py | 193 ++++++++++++ app/attachments.py | 238 +++++++++++++++ app/auth.py | 104 +++++++ app/core.py | 79 +++++ app/image_worker.py | 115 +++++++ app/jobs.py | 386 ++++++++++++++++++++++++ app/main.py | 84 ++++++ app/migrate.py | 23 ++ app/models.py | 77 +++++ app/pipeline_selftest.py | 25 ++ app/reconcile.py | 132 ++++++++ app/reconcile_fault.py | 36 +++ app/reconcile_selftest.py | 61 ++++ app/s3_bootstrap.py | 35 +++ app/s3_policy.py | 32 ++ app/s3_probe.py | 40 +++ app/storage.py | 64 ++++ compose.yaml | 251 +++++++++++++++ db/migrations/001_initial.sql | 33 ++ db/migrations/002_idempotency.sql | 12 + db/migrations/003_attachments.sql | 25 ++ db/migrations/004_jobs.sql | 58 ++++ evidence/attachments-concurrent.json | 13 + evidence/verified_results_2026-11-15.md | 66 ++++ evidence/worker-resources.csv | 46 +++ infra/filer.toml | 4 + infra/haproxy.cfg | 32 ++ requirements.txt | 11 + scripts/restart.cmd | 11 + scripts/stop-one.cmd | 10 + scripts/wait_ready.py | 14 + tests/attachments_concurrent.py | 90 ++++++ tests/attachments_negative.py | 48 +++ tests/idempotency_parallel.py | 38 +++ tests/own_unit.py | 42 +++ tests/pipeline_extended.py | 233 ++++++++++++++ tests/pipeline_resilience.py | 108 +++++++ 43 files changed, 3020 insertions(+), 45 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/__main__.py create mode 100644 app/api.py create mode 100644 app/attachments.py create mode 100644 app/auth.py create mode 100644 app/core.py create mode 100644 app/image_worker.py create mode 100644 app/jobs.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/models.py create mode 100644 app/pipeline_selftest.py create mode 100644 app/reconcile.py create mode 100644 app/reconcile_fault.py create mode 100644 app/reconcile_selftest.py create mode 100644 app/s3_bootstrap.py create mode 100644 app/s3_policy.py create mode 100644 app/s3_probe.py create mode 100644 app/storage.py create mode 100644 compose.yaml create mode 100644 db/migrations/001_initial.sql create mode 100644 db/migrations/002_idempotency.sql create mode 100644 db/migrations/003_attachments.sql create mode 100644 db/migrations/004_jobs.sql create mode 100644 evidence/attachments-concurrent.json create mode 100644 evidence/verified_results_2026-11-15.md create mode 100644 evidence/worker-resources.csv create mode 100644 infra/filer.toml create mode 100644 infra/haproxy.cfg create mode 100644 requirements.txt create mode 100644 scripts/restart.cmd create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/wait_ready.py create mode 100644 tests/attachments_concurrent.py create mode 100644 tests/attachments_negative.py create mode 100644 tests/idempotency_parallel.py create mode 100644 tests/own_unit.py create mode 100644 tests/pipeline_extended.py create mode 100644 tests/pipeline_resilience.py diff --git a/.env.example b/.env.example index 119384b5..4a205c96 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,23 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Copy to .env and REPLACE all sample secrets with random unique values. +# For local synthetic data only. Never commit .env. +POSTGRES_DB=messenger +POSTGRES_USER=lab +POSTGRES_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_PASSWORD SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 -S3_REGION=us-east-1 +DB_POOL_MAX=8 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key -MAX_UPLOAD_BYTES=10485760 -MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +S3_ADMIN_ACCESS_KEY=adminlab4 +S3_ADMIN_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_ADMIN_SECRET +S3_ACCESS_KEY=messengerapi +S3_SECRET_KEY=REPLACE_WITH_DIFFERENT_RANDOM_API_SECRET + +# RabbitMQ persistent queue (different from session Redis). +BROKER_USER=labworker +BROKER_PASSWORD=REPLACE_WITH_RANDOM_LOCAL_BROKER_PASSWORD JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 -STAGING_RETENTION_SECONDS=86400 +JOB_BACKLOG_LIMIT=1000 +WORKER_PREFETCH=1 +# Only for deliberate crash experiment: once. Keep off for normal operation. +LAB5_FAULT_AFTER_S3=off diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..31979f18 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.12.10-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 +WORKDIR /srv +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 999 app && useradd -m --uid 999 --gid app app +COPY app ./app +COPY db ./db +USER app +EXPOSE 8080 +CMD ["python", "-m", "app"] diff --git a/Makefile b/Makefile index 5d86d171..d5e9dac0 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,14 @@ +ifeq ($(OS),Windows_NT) +PYTHON ?= python +else PYTHON ?= python3 +endif BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-own help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -22,9 +26,14 @@ load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) + $(PYTHON) -c "import pathlib,sys; sys.exit(0 if pathlib.Path(sys.argv[1]).is_file() else 2)" "$(ACTION_SCRIPT)" $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + $(PYTHON) -c "import pathlib,sys; sys.exit(0 if pathlib.Path(sys.argv[1]).is_file() else 2)" "$(ACTION_SCRIPT)" $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-own: + $(PYTHON) tests/own_unit.py + $(PYTHON) tests/idempotency_parallel.py + $(PYTHON) tests/attachments_negative.py diff --git a/REPORT.md b/REPORT.md index e98f75e9..f6ffd7ab 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,121 @@ -# Отчёт — лабораторная 5 +# Отчёт — лабораторная 5: очередь и асинхронная обработка изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +**Автор / группа:** Авзалов Артур Марсович / БПИ2301 +**Целевая оценка:** 4 +**Заявленная оценка:** 4; основной интеграционный набор выполнен на учебном Windows/Docker Desktop стенде (см. результаты и ограничения) +**Использование ИИ:** ChatGPT помогал проектировать и реализовать API, outbox, worker, инфраструктуру, тесты и этот отчёт. + +## Что перенесено и изменено + +Сохранён каркас Lab4: две stateless FastAPI-реплики за HAProxy, PostgreSQL, Redis-сессии, S3-совместимый SeaweedFS, авторизация по Bearer, идемпотентность сообщений, пагинация, проверки MIME/ACL и миграции. Публичные `contracts/` и `tests/smoke.py` взяты из **оригинального skeleton Lab5**, не из Lab4. + +Изменение Lab5: загрузка `POST /api/v1/chats/{id}/attachments?filename=...` отвечает `202` со снимком `status=queued`. Обещание на момент ответа: файл целиком проверен и находится в устойчивом S3 staging, а метаданные вложения, job и запись transactional outbox находятся в **одном коммите PostgreSQL**. Публикация RabbitMQ и обработка — после HTTP-ответа, в отдельных процессах. + +## Архитектура + +```text + HAProxy + / \ + api1 api2 GET status / content / messages + \ / + PostgreSQL (attachments, jobs, transactional outbox) + | ^ + outbox dispatcher worker (RabbitMQ consumer) + | ^ + RabbitMQ (durable queue, persistent messages, manual ack) + | + worker child (Pillow transforms, time budget) + | + SeaweedFS (private S3 bucket) + staging/ -> attachments/ + recovery/ + variants/thumbnail,crop,optimized +``` + +**Единственный источник истины по статусу** — PostgreSQL. В сообщении RabbitMQ только `job_id`, `attachment_id`, `pipeline_version`, `attempt`, дата и тип; байтов, клиентского filename, токенов и S3 credentials там нет. S3 staging не исчезает при перезапуске API/worker/broker. RabbitMQ имеет named volume и durable queue. Pubisher confirm фиксирует приём брокером; consumer ack вызывается после устойчивой фиксации результата в БД либо очередной попытки/failed. Доставка **at least once**, но не exactly-once. + +### Разрыв между COMMIT и publish + +`_admit_upload()` загружает исходник в `staging/{id}/original`, затем в одной транзакции создаёт `attachments(status=queued)`, `processing_jobs` и `job_outbox`. `dispatcher` читает ещё не опубликованные outbox-записи, публикует persistent RabbitMQ message с publisher confirms и только после подтверждения выставляет `published_at`. Если dispatcher падает до publish или после publish до обновления outbox — он повторяет отправку. Повторы безопасны благодаря `processing_jobs` и `attachment_variants` с уникальностью `(attachment_id,pipeline_version,name)`. + +### Worker, таймауты и retry + +Worker имеет ограниченный `prefetch` (по умолчанию 1), одну обработку за раз на процесс, и отдельный subprocess для каждой картинки с таймаутом `JOB_TIMEOUT_SECONDS=60`. Изображения ограничены 10 MiB и 40 млн пикселей; Docker memory limit worker — 768 MiB, CPU 1.0. Для большого формата subprocess может завершиться неуспешно: фиксируется ограниченный retry либо failed, но API не зависает синхронно. + +Worker захватывает job через строковую блокировку в PostgreSQL и создаёт lease на 105 с при таймауте 60 с. При падении lease истекает, dispatcher повторно ставит job в очередь. При временной ошибке: retry с экспоненциальным backoff + jitter, по умолчанию до пяти попыток. Повреждённый файл — постоянная ошибка `invalid_image`. По исчерпании — `retry_exhausted`. Для failed доступен **только операторский** `python -m app.jobs replay --id ...`, с аудитом в `job_replay_audit`. Job с `status=ready` не перезапускается по старому сообщению. + +### Варианты изображений + +- `original` — неизменённые исходные bytes, первичный ключ `attachments/...`, recovery-копия `recovery/...`. +- `thumbnail` — JPEG, вписанный в 256×256 без увеличения; например, fixture 320×200 → 256×160; quality 85. +- `crop` — JPEG 128×128, центрированный квадрат с допущенным увеличением; quality 85. +- `optimized` — WebP, вписанный в 1280×1280 без увеличения; quality 78, method 4. + +EXIF orientation применяется через `ImageOps.exif_transpose`; прозрачность накладывается на белый фон; метаданные не передаются при кодировании новых изображений. Контент queued/processing/failed отвечает `409`, готовые варианты — только через авторизованный API и проверку участника чата. + +### Безопасная очистка staging + +После `ready` worker удаляет `staging/` только после успешного commit. Если он умер раньше очистки, dispatcher повторяет безопасную уборку. Не привязанные к БД staging-объекты и staging failed-работ удерживаются не менее 86400 с; по истечении могут быть очищены, после чего replay требует нового staging и будет отклонён. S3 и PostgreSQL **не имеют общей транзакции**; очистка построена консервативно. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +1. Создать `.env` из `.env.example` и заменить учебные пароли БД, RabbitMQ и S3 на разные случайные значения. `.env` никогда не включать в Git. +2. Остановить Lab4, если он занимает `127.0.0.1:8080`, без удаления volumes. +3. В PowerShell из корня Lab5: -## Архитектура и решения +```powershell +$env:PYTHONUTF8 = '1' +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +docker compose --env-file .env ps +python scripts/check_contract.py +python tests/smoke.py +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Все зависимости создаются только этим Compose: PostgreSQL 16.10, Redis 7.4.5, RabbitMQ 4.1.8, SeaweedFS 4.53, HAProxy 3.1.7, Python 3.12.10. Для Windows также нужны Docker Desktop, PowerShell, Python 3.9+. + +**Команды тестов и их результаты приведены ниже в разделе доказательств критериев.** ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +**Фактически проверено пользователем 14–15 ноября 2026 года на Windows + Docker Desktop.** Команды и результаты ниже взяты из вывода реального запуска; отсутствующие результаты не обозначены PASS. Исправлены две обнаруженные проблемы: лишний `%s` в SQL `publish_result()` и устаревшее ожидание `404` thumbnail/`201` upload в собственных тестах из Lab4. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий | Команда / опыт | Результат | +| --- | --- | --- | +| Инфраструктура | `docker compose up -d --build --wait`; `docker compose ps` | **PASS**: API1/2, RabbitMQ, PostgreSQL, Redis, SeaweedFS, LB, worker, dispatcher поднялись; health основных сервисов зелёный | +| Структура/контракт | `python scripts/check_contract.py` | **PASS**, offline-only | +| Модели/limit/cursor | `python tests/own_unit.py` | **PASS** | +| API и вариантные файлы | `python tests/smoke.py` после SQL фикса | **PASS**: 202→ready | +| Негативные MIME, размеры, ACL, варианты | `python tests/attachments_negative.py` после актуализации ожиданий под Lab5 | **PASS** | +| Message idempotency | `python tests/idempotency_parallel.py` | **PASS**: 2 API, 201 replay и 409 для иного тела | +| Pillow, isolated | `python -m app.pipeline_selftest` внутри worker-контейнера | **PASS**: PNG→JPEG thumb 256×160, crop 128×128, WebP | +| Остановленный worker / рестарт API и RabbitMQ / ready | `python tests/pipeline_resilience.py --scenario paused` | **PASS**: данные, оригинал, 3 варианта доступны после восстановления | +| PostgreSQL COMMIT → падение dispatcher до publish | `... --scenario outbox-crash` | **PASS**: exit 78, восстановление из outbox | +| Worker после S3 effects, до DB commit/ack | `... --scenario worker-crash` | **PASS**: exit 77, восстановление после lease, 3 варианта | +| 10 одновременных загрузок → все ready | `python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json` | **PASS**: 10/10 accepted, 10/10 ready, 0 failed; повторный прогон: приём 2.391 s / 4.182 RPS, p50 2218.5 ms, max 2313 ms; все ready через 29.781 s, по 111817 bytes | +| Долговечность *уже опубликованного* RabbitMQ сообщения | `python tests/pipeline_extended.py --scenario broker-persistence` | **PASS**: подтверждённая публикация, ready=1 до и после рестарта брокера; dispatcher остановлен | +| Permanent failed / manual replay / аудит / запрет replay ready | `python tests/pipeline_extended.py --scenario failed-replay` | **PASS** (15.11): `invalid_image` без retry/вариантов → исправление метаданных → operator replay → `ready`; audit=1, replay `ready` отклонён | +| Очистка S3 staging | `python tests/pipeline_extended.py --scenario staging-cleanup` | **PASS**: `ready`, `staging_key=NULL`, staging-объект отсутствует | +| Использование памяти контейнеров под нагрузкой | 45-секундная серия `docker stats`, 15 отсчётов на контейнер | **ЗАМЕРЕНО**: наблюдаемый максимум памяти API1 163.10 MiB, API2 157.10 MiB, worker 128.90 MiB; это не peak RSS | +| Persistence Lab2/3 на Lab5 | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | **PASS** (15.11): ранее подтверждённые данные, байты вложения и действующая сессия сохранились после выполнения restart hook | +| Failover одной FastAPI-реплики | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | **PASS** (15.11): 2 экземпляра до отказа, 6 успешных записей, 1 неудачная попытка; первое успешное восстановление за 6.172 s; данные и сессия сохранены | +| Восстановление 2 API и повторный smoke | `docker compose ... up -d --no-deps --wait api1 api2`; `python tests/smoke.py`; `docker compose ps` | **PASS** (15.11): обе API healthy, smoke PASS, инфраструктура запущена | -## Эксперименты и отказы +`4.182 RPS` — скорость **приёма HTTP 202** во втором прогоне, а не производительность worker; `29.781 s` — время завершения всех 10 задач. Дополнительные прогоны 14–15.11: `broker-persistence`, `staging-cleanup`, `failed-replay`, `persistence` и `failover` — PASS после исправления тестового клиента. Первый прогон `failed-replay` падал из-за ошибки теста, затем перепроверен успешно. Новые проверки `pipeline_extended.py` работают только при `LAB5_ALLOW_FAILURE_TEST=yes`, не стирают volumes и не запускаются автоматически; их нельзя объявлять прошедшими без реального вывода. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Границы проверки restart/failover -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +`tests/resilience.py` **подавляет stdout/stderr hook** и выводит `Hook scope needs operator evidence`. Прочитанные `scripts/restart.cmd`/`scripts/stop-one.cmd` действительно содержат команды остановки и запуска соответствующих контейнеров; однако сам вывод `PASS` подтверждает **наблюдаемый эффект теста**, а не сам по себе независимую запись событий Docker. Для защиты удобно сохранить `docker compose ps`, показать сами `.cmd` и при необходимости провести повторный прогон с дополнительными логами Docker. Failover не бесшовный: **1 неудачная попытка записи**, восстановление в тесте около **6.172 s**. Сценарий с одним Docker-хостом не доказывает устойчивость к потере хоста. -## Ограничения и следующие шаги +Полный фактический результат последнего прогона: [`evidence/verified_results_2026-11-15.md`](evidence/verified_results_2026-11-15.md). Приведённые результаты — пользовательские логи на Windows; архив v6 подготовлен без повторного запуска Docker в среде сборки. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Ограничения + +- **Одиночный broker, S3, PostgreSQL, Redis и HAProxy** являются точками отказа. RabbitMQ volume выдерживает restart при сохранении диска, но не потерю хоста. +- Время обработки зависит от Pillow и S3. При нагрузке очередь растёт; учебный admission cap проверяется простым счётчиком, а не атомарным глобальным семафором. +- Доставка RabbitMQ at least once, поэтому возможны дубликаты публикаций и обработок, но БД не фиксирует второй набор вариантов. +- Worker lease обеспечивает восстановление после смерти; в этом кандидате нет периодического продления lease во время обработки. Время lease сделано больше budget subprocess; на сильно перегруженном узле возможны повторные попытки. +- Пакет создан на основе **предыдущего кандидата Lab4**, а не автоматически синхронизирован с последним Git-коммитом пользователя. Перед MR сравнить отличия с собственной Lab4. +- Набор тестов на текущем учебном стенде пройден, но не заменяет независимую проверку преподавателем и стресс-/долгосрочное тестирование. Однозначных доказательств устойчивости к отказу физического хоста или exact-once нет. + +## Полезные различия для защиты + +**Publisher confirm** — брокер подтвердил приём persistent сообщения. **Consumer ack** — worker завершил обработку и клиент брокера подтвердил удаление сообщения из очереди. **202** — уже сохранены staging bytes + metadata/job/outbox, но не варианты. **Idempotency key эффекта** — `(attachment_id,pipeline_version,variant)`. **Lease** — ограниченный срок права обработчика на работу; истёкший lease подбирает dispatcher. **Exactly-once** не обещаем: одна логическая операция может доставляться и исполняться несколько раз, итоговая запись идемпотентна. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e12a1933 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Lab 3 messenger API.""" diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..87740f8f --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,4 @@ +import os +import uvicorn +if __name__ == "__main__": + uvicorn.run("app.main:app", host="0.0.0.0", port=int(os.getenv("APP_PORT", "8080")), workers=1, timeout_graceful_shutdown=8) diff --git a/app/api.py b/app/api.py new file mode 100644 index 00000000..8f9ad0fa --- /dev/null +++ b/app/api.py @@ -0,0 +1,193 @@ +"""Messenger HTTP handlers; state lives only in PostgreSQL/Redis.""" +import hashlib +import json +import re +import uuid + +import psycopg +from argon2 import PasswordHasher +from fastapi import APIRouter, Request + +from .auth import current_user, hasher, login, logout, public_user +from .core import APIError, invalid, iso, make_cursor, parse_limit, parse_uuid, read_cursor +from .models import CreateChat, CreateMessage, CreateUser +from .storage import db + +router = APIRouter() +KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII) + + +def actor(req: Request) -> dict: + return getattr(req.state, "actor", None) or current_user(req) + + +def chat_detail(conn, chat: dict) -> dict: + members = conn.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position", (chat["id"],)).fetchall() + return {"id": str(chat["id"]), "title": bytes(chat["title"]).decode("utf-8"), "created_at": iso(chat["created_at"]), "member_ids": [str(x["user_id"]) for x in members]} + + +def message_detail(conn, row: dict) -> dict: + linked = conn.execute("SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position", (row["id"],)).fetchall() + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": bytes(row["text"]).decode("utf-8"), "created_at": iso(row["created_at"]), "attachment_ids": [str(x["attachment_id"]) for x in linked]} + + +def require_chat(conn, chat_id: str, actor_id: str) -> dict: + row = conn.execute("SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id WHERE c.id=%s AND cm.user_id=%s", (chat_id, actor_id)).fetchone() + if row is None: + raise APIError(404, "not_found", "Object not found") + return row + + +@router.get("/health/live") +def live(): + return {"status": "ok"} + + +@router.get("/health/ready") +def ready(): + from .storage import ready as storage_ready + if not storage_ready(): + raise APIError(503, "unavailable", "Service not ready") + return {"status": "ok"} + + +@router.post("/api/v1/users", status_code=201) +def register(body: CreateUser): + hashed = hasher.hash(body.password) + ident = str(uuid.uuid4()) + try: + with db() as conn: + row = conn.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING *", (ident, body.username, body.display_name.encode("utf-8"), hashed)).fetchone() + return public_user(row) + except psycopg.errors.UniqueViolation: + raise APIError(409, "conflict", "Username already exists") from None + + +@router.post("/api/v1/auth/sessions", status_code=201) +def sessions(req: Request): + return login(req) + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +def sessions_delete(req: Request): + logout(req) + return None + + +@router.get("/api/v1/users/me") +def users_me(req: Request): + return actor(req) + + +@router.post("/api/v1/chats", status_code=201) +def chats_create(req: Request, body: CreateChat): + ident = actor(req)["id"] + if ident not in body.member_ids: + raise invalid("Current user must belong to the chat") + chat_id = str(uuid.uuid4()) + try: + with db() as conn: + with conn.transaction(): + # Validate all members; FK constraints are the final authority for races. + found = conn.execute("SELECT id FROM users WHERE id = ANY(%s::uuid[])", (body.member_ids,)).fetchall() + if len(found) != len(body.member_ids): + raise invalid("Unknown member") + row = conn.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (chat_id, body.title.encode("utf-8"))).fetchone() + for pos, member in enumerate(body.member_ids): + conn.execute("INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)", (chat_id, member, pos)) + result = chat_detail(conn, row) + return result + except psycopg.errors.ForeignKeyViolation: + raise invalid("Unknown member") from None + + +@router.get("/api/v1/chats") +def chats_list(req: Request, limit: str | None = None, cursor: str | None = None): + who = actor(req)["id"] + n = parse_limit(limit) + mark = read_cursor(cursor, "chats", who) + with db() as conn: + if mark: + existing = conn.execute("SELECT 1 FROM chats c JOIN chat_members cm ON c.id=cm.chat_id WHERE c.id=%s AND c.created_at=%s AND cm.user_id=%s", (mark[1], mark[0], who)).fetchone() + if not existing: + raise invalid("Invalid cursor") + rows = conn.execute(""" + SELECT c.* FROM chats c JOIN chat_members cm ON cm.chat_id=c.id + WHERE cm.user_id=%s AND (%s::timestamptz IS NULL OR (c.created_at,c.id)>(%s::timestamptz,%s::uuid)) + ORDER BY c.created_at,c.id LIMIT %s + """, (who, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall() + items = [chat_detail(conn, row) for row in rows[:n]] + more = len(rows) > n + last = rows[n-1] if more else None + return {"items": items, "next_cursor": make_cursor("chats", who, last["created_at"], str(last["id"])) if last else None} + + +@router.get("/api/v1/chats/{chat_id}") +def chats_get(req: Request, chat_id: str): + who = actor(req)["id"] + ident = parse_uuid(chat_id) + with db() as conn: + row = require_chat(conn, ident, who) + return chat_detail(conn, row) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def messages_create(req: Request, chat_id: str, body: CreateMessage): + who = actor(req)["id"] + ident = parse_uuid(chat_id) + key = req.headers.get("Idempotency-Key") + if not key or not KEY_RE.fullmatch(key): + raise invalid("Invalid Idempotency-Key") + # Lab3 has only text; canonical JSON avoids unstable formatting/field ordering. + fingerprint = hashlib.sha256(json.dumps({"text": body.text, "attachment_ids": body.attachment_ids}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + lock_scope = f"{who}:POST:/api/v1/chats/{ident}/messages:{key}" + with db() as conn: + with conn.transaction(): + require_chat(conn, ident, who) + # Transaction-scoped advisory lock is GLOBAL across API replicas. + conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (lock_scope,)) + previous = conn.execute(""" + SELECT idem.body_sha256, m.* FROM message_idempotency idem + JOIN messages m ON m.id=idem.message_id + WHERE idem.actor_id=%s AND idem.chat_id=%s AND idem.method='POST' AND idem.idem_key=%s + """, (who, ident, key)).fetchone() + if previous: + if previous["body_sha256"] != fingerprint: + raise APIError(409, "conflict", "Idempotency key used with another body") + return message_detail(conn, previous) + # 404 for foreign/nonexistent IDs; check BEFORE any message insert. + if body.attachment_ids: + owned = conn.execute("SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s::uuid[])", (ident, body.attachment_ids)).fetchall() + if len(owned) != len(body.attachment_ids): + raise APIError(404, "not_found", "Attachment not found") + if any(item["status"] == "failed" for item in owned): + raise APIError(409, "conflict", "Failed attachment cannot be linked") + message_id = str(uuid.uuid4()) + row = conn.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (message_id, ident, who, body.text.encode("utf-8"))).fetchone() + for pos, attachment_id in enumerate(body.attachment_ids): + conn.execute("INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)", (message_id, attachment_id, pos)) + conn.execute("INSERT INTO message_idempotency(actor_id,chat_id,method,idem_key,body_sha256,message_id) VALUES (%s,%s,'POST',%s,%s,%s)", (who, ident, key, fingerprint, message_id)) + return message_detail(conn, row) + + +@router.get("/api/v1/chats/{chat_id}/messages") +def messages_list(req: Request, chat_id: str, limit: str | None = None, cursor: str | None = None): + who = actor(req)["id"] + ident = parse_uuid(chat_id) + n = parse_limit(limit) + mark = read_cursor(cursor, "messages", ident) + with db() as conn: + require_chat(conn, ident, who) + if mark: + existing = conn.execute("SELECT 1 FROM messages WHERE id=%s AND chat_id=%s AND created_at=%s", (mark[1], ident, mark[0])).fetchone() + if not existing: + raise invalid("Invalid cursor") + rows = conn.execute(""" + SELECT * FROM messages WHERE chat_id=%s + AND (%s::timestamptz IS NULL OR (created_at,id)>(%s::timestamptz,%s::uuid)) + ORDER BY created_at,id LIMIT %s + """, (ident, mark[0] if mark else None, mark[0] if mark else None, mark[1] if mark else None, n+1)).fetchall() + with db() as conn: + items = [message_detail(conn, row) for row in rows[:n]] + last = rows[n-1] if len(rows)>n else None + return {"items": items, "next_cursor": make_cursor("messages", ident, last["created_at"], str(last["id"])) if last else None} diff --git a/app/attachments.py b/app/attachments.py new file mode 100644 index 00000000..4e937a90 --- /dev/null +++ b/app/attachments.py @@ -0,0 +1,238 @@ +"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline. + +202 means verified bytes were staged to private S3 and metadata + a job + an +outbox event were committed in ONE PostgreSQL transaction. No broker availability +is required for admission; the dispatcher replays unsent intents. +""" +import hashlib +import os +import tempfile +import threading +import uuid +from contextlib import closing + +import boto3 +from boto3.s3.transfer import TransferConfig +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError +from fastapi import APIRouter, Query, Request +from fastapi.responses import StreamingResponse +from PIL import Image, ImageFile, UnidentifiedImageError +from starlette.concurrency import run_in_threadpool + +from .auth import current_user +from .core import APIError, invalid, iso, parse_uuid +from .storage import db + +router = APIRouter() +BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") +MAX_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +SPOOL_MEMORY_BYTES = 1024 * 1024 +S3_BUFFER_BYTES = 256 * 1024 +DECODE_LIMIT = threading.BoundedSemaphore(1) +Image.MAX_IMAGE_PIXELS = MAX_PIXELS +ImageFile.LOAD_TRUNCATED_IMAGES = False + + +def s3_client(): + return boto3.client( + "s3", endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_SECRET_KEY"], + region_name="us-east-1", use_ssl=False, + config=Config(s3={"addressing_style": "path"}, connect_timeout=3, + read_timeout=10, retries={"max_attempts": 1}), + ) + + +def s3_error(exc): + return APIError(503, "unavailable", "Object storage unavailable") + + +def format_attachment(row, variants=()): + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), "filename": row["filename"], + "content_type": row["content_type"], "size_bytes": row["size_bytes"], + "sha256": row["sha256"], "status": row["status"], + "variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")} + for item in variants] if row["status"] == "ready" else [], + "error_code": row["error_code"] if row["status"] == "failed" else None, + "created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]), + } + + +def require_member(conn, chat_id, user_id): + if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone(): + raise APIError(404, "not_found", "Object not found") + + +def attachment_row(conn, attachment_id, user_id): + row = conn.execute(""" + SELECT a.* FROM attachments a + JOIN chat_members cm ON cm.chat_id=a.chat_id + WHERE a.id=%s AND cm.user_id=%s + """, (attachment_id, user_id)).fetchone() + if not row: + raise APIError(404, "not_found", "Object not found") + return row + + +def validate_filename(filename: str): + if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename): + raise invalid("Invalid filename") + + +def validate_image(fileobj, mime: str): + """Validate MIME, magic, dimensions and complete decode; worker does the resize.""" + try: + fileobj.seek(0) + with Image.open(fileobj) as im: + if im.format not in ("PNG", "JPEG"): + raise APIError(415, "unsupported_media_type", "Unsupported image encoding") + actual = "image/png" if im.format == "PNG" else "image/jpeg" + if actual != mime: + raise APIError(415, "unsupported_media_type", "Content-Type does not match image") + if im.width * im.height > MAX_PIXELS: + raise APIError(413, "payload_too_large", "Image dimensions too large") + im.verify() + fileobj.seek(0) + with DECODE_LIMIT: + with Image.open(fileobj) as im: + im.load() + except APIError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise APIError(413, "payload_too_large", "Image dimensions too large") from None + except (UnidentifiedImageError, OSError, ValueError, SyntaxError): + raise APIError(415, "unsupported_media_type", "Invalid image") from None + finally: + fileobj.seek(0) + + +def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest): + from .jobs import job_payload + stage_key = f"staging/{attachment_id}/original" + key = f"attachments/{attachment_id}/original" + backup = f"recovery/{attachment_id}/original" + job_id = str(uuid.uuid4()) + outbox_id = str(uuid.uuid4()) + client = s3_client() + try: + spool.seek(0) + client.upload_fileobj(spool, BUCKET, stage_key, + ExtraArgs={"ContentType": mime}, + Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024, + io_chunksize=S3_BUFFER_BYTES)) + except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc: + raise s3_error(exc) from exc + + # Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the + # staged object: an accepted task may already exist; operator cleanup handles it. + with db() as conn: + with conn.transaction(): + require_member(conn, chat_id, user_id) + # Small educational admission bound; pre-existing queued tasks remain safe. + pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"] + if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")): + raise APIError(429, "rate_limited", "Processing queue is full") + row = conn.execute(""" + INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename, + content_type,size_bytes,sha256,status,staging_key,pipeline_version) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING * + """, (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone() + conn.execute(""" + INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt) + VALUES (%s,%s,1,'queued',1) + """, (job_id, attachment_id)) + payload = job_payload(job_id, attachment_id, 1, 1) + conn.execute(""" + INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb) + """, (outbox_id, job_id, __import__('json').dumps(payload))) + return format_attachment(row) # snapshot = queued, even if worker is very fast + + +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)): + from .api import actor, require_chat + who = actor(req)["id"] + chat_id = parse_uuid(chat_id) + validate_filename(filename) + mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if mime not in ("image/png", "image/jpeg"): + raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported") + with db() as conn: + require_chat(conn, chat_id, who) + announced = req.headers.get("content-length") + if announced: + try: + if int(announced) > MAX_BYTES: + raise APIError(413, "payload_too_large", "Image too large") + if int(announced) < 0: + raise invalid("Invalid Content-Length") + except ValueError: + raise invalid("Invalid Content-Length") from None + size = 0 + digest = hashlib.sha256() + with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool: + async for chunk in req.stream(): + size += len(chunk) + if size > MAX_BYTES: + raise APIError(413, "payload_too_large", "Image too large") + digest.update(chunk) + spool.write(chunk) + if not size: + raise APIError(415, "unsupported_media_type", "Empty image") + await run_in_threadpool(validate_image, spool, mime) + return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()), + chat_id=chat_id, user_id=who, filename=filename, + mime=mime, size=size, digest=digest.hexdigest()) + + +@router.get("/api/v1/attachments/{attachment_id}") +def get_attachment(req: Request, attachment_id: str): + who = getattr(req.state, "actor", None) or current_user(req) + with db() as conn: + row = attachment_row(conn, parse_uuid(attachment_id), who["id"]) + variants = conn.execute(""" + SELECT name,content_type,size_bytes,width,height,sha256 + FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s + ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END + """, (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else [] + return format_attachment(row, variants) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment(req: Request, attachment_id: str, variant: str = "original"): + who = getattr(req.state, "actor", None) or current_user(req) + with db() as conn: + row = attachment_row(conn, parse_uuid(attachment_id), who["id"]) + if row["status"] != "ready": + raise APIError(409, "conflict", "Attachment is not ready") + if variant == "original": + key, mime, size = row["object_key"], row["content_type"], row["size_bytes"] + else: + v = conn.execute(""" + SELECT * FROM attachment_variants + WHERE attachment_id=%s AND pipeline_version=%s AND name=%s + """, (row["id"], row["pipeline_version"], variant)).fetchone() + if not v: + raise APIError(404, "not_found", "Object not found") + key, mime, size = v["object_key"], v["content_type"], v["size_bytes"] + client = s3_client() + try: + response = client.get_object(Bucket=BUCKET, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"): + raise APIError(503, "unavailable", "Stored object missing") from exc + raise s3_error(exc) from exc + except BotoCoreError as exc: + raise s3_error(exc) from exc + + def iterator(): + with closing(response["Body"]) as body: + while chunk := body.read(S3_BUFFER_BYTES): + yield chunk + + return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)}) diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..8286fb31 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,104 @@ +"""HTTP Basic only on login; bearer sessions shared by all API replicas.""" +import base64 +import binascii +import hashlib +import re +import secrets +import uuid +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError +from fastapi import Request +from redis.exceptions import RedisError + +from .core import APIError, SESSION_TTL, iso +from .storage import cache, db + +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +PASSWORD_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +def public_user(row: dict): + return {"id": str(row["id"]), "username": row["username"], "display_name": bytes(row["display_name"]).decode("utf-8"), "created_at": iso(row["created_at"])} + + +def unauthorized() -> APIError: + return APIError(401, "unauthorized", "Unauthorized") + + +def challenge(req: Request) -> str: + return 'Basic realm="messenger", charset="UTF-8"' if req.url.path == "/api/v1/auth/sessions" else 'Bearer realm="messenger"' + + +def bearer_token(req: Request) -> str: + value = req.headers.get("Authorization", "") + if not value.startswith("Bearer ") or len(value) <= 7 or " " in value[7:] or len(value) > 512: + raise unauthorized() + try: + value[7:].encode("ascii") + except UnicodeEncodeError: + raise unauthorized() from None + return value[7:] + + +def session_key(token: str) -> str: + return "session:" + hashlib.sha256(token.encode("ascii")).hexdigest() + + +def current_user(req: Request) -> dict: + token = bearer_token(req) + try: + raw = cache().get(session_key(token)) + except (RedisError, UnicodeError) as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc + if not raw: + raise unauthorized() + try: + ident = str(uuid.UUID(raw)) + except (ValueError, TypeError): + try: + cache().delete(session_key(token)) + except RedisError as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc + raise unauthorized() from None + with db() as conn: + row = conn.execute("SELECT * FROM users WHERE id = %s", (ident,)).fetchone() + if row is None: + raise unauthorized() + return public_user(row) + + +def login(req: Request): + auth = req.headers.get("Authorization", "") + if not auth.startswith("Basic "): + raise unauthorized() + try: + decoded = base64.b64decode(auth[6:].encode("ascii"), validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + if not PASSWORD_RE.fullmatch(username): + raise ValueError() + except (ValueError, UnicodeError, binascii.Error): + raise unauthorized() from None + with db() as conn: + row = conn.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() + if not row: + raise unauthorized() + try: + hasher.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError): + raise unauthorized() from None + token = secrets.token_urlsafe(32) + try: + cache().set(session_key(token), str(row["id"]), ex=SESSION_TTL) + except RedisError as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc + return {"token": token, "token_type": "Bearer", "expires_at": iso(datetime.now(timezone.utc) + timedelta(seconds=SESSION_TTL)), "user": public_user(row)} + + +def logout(req: Request): + current_user(req) + try: + cache().delete(session_key(bearer_token(req))) + except RedisError as exc: + raise APIError(503, "unavailable", "Session store unavailable") from exc diff --git a/app/core.py b/app/core.py new file mode 100644 index 00000000..7a85e52a --- /dev/null +++ b/app/core.py @@ -0,0 +1,79 @@ +"""Configuration, HTTP errors and shared representation helpers.""" +import base64 +import json +import os +import uuid +from datetime import datetime, timezone + +from fastapi import Request + +INSTANCE_ID = os.getenv("API_INSTANCE_ID", uuid.uuid4().hex[:12]) +SESSION_TTL = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +IDEMPOTENCY_TTL = max(86400, int(os.getenv("IDEMPOTENCY_TTL_SECONDS", "86400"))) + +class APIError(Exception): + def __init__(self, status: int, code: str, message: str): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def iso(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def invalid(message: str = "Invalid request") -> APIError: + return APIError(400, "invalid_request", message) + + +def parse_uuid(value: str) -> str: + try: + return str(uuid.UUID(value)) + except (ValueError, TypeError, AttributeError): + raise invalid("Invalid UUID") from None + + +def parse_limit(raw: str | None) -> int: + if raw is None: + return 50 + try: + if not raw.isascii() or not raw.isdecimal(): + raise ValueError() + value = int(raw) + except (ValueError, AttributeError): + raise invalid("Invalid limit") from None + if not 1 <= value <= 100: + raise invalid("Invalid limit") + return value + + +def make_cursor(kind: str, scope: str, created: datetime, ident: str) -> str: + b = json.dumps({"kind": kind, "scope": scope, "created_at": iso(created), "id": ident}, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(b).decode().rstrip("=") + + +def read_cursor(raw: str | None, kind: str, scope: str): + if raw is None: + return None + try: + if not raw or len(raw) > 2048 or any(ch not in "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_" for ch in raw): + raise ValueError() + data = json.loads(base64.b64decode(raw + "=" * (-len(raw) % 4), altchars=b"-_", validate=True)) + if not isinstance(data, dict) or data.get("kind") != kind or data.get("scope") != scope: + raise ValueError() + created = datetime.fromisoformat(data["created_at"].replace("Z", "+00:00")) + if created.tzinfo is None: + raise ValueError() + ident = str(uuid.UUID(data["id"])) + return created, ident + except (ValueError, TypeError, KeyError, UnicodeError, json.JSONDecodeError): + raise invalid("Invalid cursor") from None + + +def request_id(req: Request) -> str: + return getattr(req.state, "request_id", uuid.uuid4().hex) + + +def error_body(req: Request, code: str, message: str): + return {"error": {"code": code, "message": message, "request_id": request_id(req)}} diff --git a/app/image_worker.py b/app/image_worker.py new file mode 100644 index 00000000..fbb058fa --- /dev/null +++ b/app/image_worker.py @@ -0,0 +1,115 @@ +"""Sandboxed-per-task image work. Process is killed after JOB_TIMEOUT_SECONDS. + +Outputs only JSON variant metadata. Original bytes are copied from durable staging, +never transformed. Pillow reopens/exif-transposes pixels for derivatives, stripping +metadata on re-encode. No API tokens or arbitrary user paths enter this command. +""" +import hashlib +import io +import json +import os +import sys + +from PIL import Image, ImageOps, UnidentifiedImageError +from botocore.exceptions import ClientError, BotoCoreError + +from .attachments import BUCKET, MAX_BYTES, MAX_PIXELS, s3_client + +Image.MAX_IMAGE_PIXELS = MAX_PIXELS + + +def rgb_image(src): + orient = ImageOps.exif_transpose(src) + orient.load() + if orient.width * orient.height > MAX_PIXELS: + raise ValueError("too many decoded pixels") + # Compose transparency on white, strip EXIF/ICC and do not enlarge. + rgba = orient.convert("RGBA") + white = Image.new("RGBA", rgba.size, (255, 255, 255, 255)) + white.alpha_composite(rgba) + return white.convert("RGB") + + +def transform(raw): + with Image.open(io.BytesIO(raw)) as image: + if image.format not in ("PNG", "JPEG"): + raise ValueError("invalid encoding") + rgb = rgb_image(image) + t = rgb.copy() + t.thumbnail((256, 256), resample=Image.Resampling.LANCZOS) + # Center crop fit scales up small images when needed. + crop = ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + optim = rgb.copy() + optim.thumbnail((1280, 1280), resample=Image.Resampling.LANCZOS) + return [ + ("thumbnail", t, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}), + ("crop", crop, "JPEG", "image/jpeg", {"quality": 85, "optimize": True}), + ("optimized", optim, "WEBP", "image/webp", {"quality": 78, "method": 4}), + ] + + +def run(attachment_id, pipeline_version): + import psycopg + from psycopg.rows import dict_row + + with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) as conn: + row = conn.execute(""" + SELECT * FROM attachments WHERE id=%s AND pipeline_version=%s + """, (attachment_id, pipeline_version)).fetchone() + if not row or not row["staging_key"]: + raise ValueError("staging metadata not found") + + client = s3_client() + body = client.get_object(Bucket=BUCKET, Key=row["staging_key"])["Body"] + try: + raw = body.read(MAX_BYTES + 1) + finally: + body.close() + if not raw or len(raw) > MAX_BYTES or len(raw) != row["size_bytes"]: + raise ValueError("staging size mismatch") + if hashlib.sha256(raw).hexdigest() != row["sha256"]: + raise ValueError("staging checksum mismatch") + + # Recompute every result if a prior run was interrupted. Keys are deterministic + # for (attachment_id,pipeline_version,variant), so duplicates don't multiply objects. + output = [] + for name, image, fmt, mime, options in transform(raw): + buf = io.BytesIO() + image.save(buf, format=fmt, **options) + encoded = buf.getvalue() + key = f"variants/{attachment_id}/v{pipeline_version}/{name}" + client.put_object(Bucket=BUCKET, Key=key, Body=encoded, ContentType=mime) + output.append({"name": name, "content_type": mime, + "size_bytes": len(encoded), "sha256": hashlib.sha256(encoded).hexdigest(), + "width": image.width, "height": image.height, "object_key": key}) + + # Only after derivatives exist, publish unchanged primary and recovery copy. + client.copy_object(Bucket=BUCKET, Key=row["object_key"], + CopySource={"Bucket": BUCKET, "Key": row["staging_key"]}, + MetadataDirective="REPLACE", ContentType=row["content_type"]) + client.copy_object(Bucket=BUCKET, Key=row["backup_key"], + CopySource={"Bucket": BUCKET, "Key": row["object_key"]}, + MetadataDirective="REPLACE", ContentType=row["content_type"]) + return output + + +def main(): + import uuid + if len(sys.argv) != 3: + raise SystemExit(2) + try: + ident = str(uuid.UUID(sys.argv[1])) + version = int(sys.argv[2]) + if version < 1: + raise ValueError("invalid version") + out = run(ident, version) + print(json.dumps({"variants": out}, ensure_ascii=False)) + except (ValueError, UnidentifiedImageError, Image.DecompressionBombError, Image.DecompressionBombWarning) as exc: + print("Permanent image processing failure", file=sys.stderr) + raise SystemExit(20) from None + except (BotoCoreError, ClientError, OSError): + print("Transient object storage failure", file=sys.stderr) + raise SystemExit(30) from None + +if __name__ == "__main__": + main() diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 00000000..e8fd2034 --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,386 @@ +"""Durable RabbitMQ delivery + PostgreSQL transactional outbox and worker leases. + +There is no exactly-once transport guarantee. We guarantee eventual recovery of +committed intent and idempotent DB results under at-least-once redelivery. +Commands: `python -m app.jobs dispatch`, `worker`, `replay --id UUID`, `status`. +""" +import argparse +from contextlib import contextmanager +import json +import logging +import os +import random +import subprocess +import sys +import time +import uuid +from datetime import datetime, timezone, timedelta + +import pika +import psycopg +from pika.exceptions import AMQPError +from psycopg.rows import dict_row + +log = logging.getLogger("lab5.jobs") +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +QUEUE = "attachment-processing.v1" +MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5")) +JOB_TIMEOUT = int(os.getenv("JOB_TIMEOUT_SECONDS", "60")) +LEASE_SECONDS = JOB_TIMEOUT + 45 + + +def connect_db(): + return psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row, connect_timeout=5) + + +def job_payload(job_id, attachment_id, version, attempt): + return { + "schema_version": 1, "job_id": str(job_id), "type": "attachment.process.v1", + "attachment_id": str(attachment_id), "pipeline_version": version, + "attempt": attempt, "created_at": datetime.now(timezone.utc).isoformat(), + } + + +@contextmanager +def connection(): + params = pika.ConnectionParameters( + host=os.getenv("BROKER_HOST", "broker"), port=5672, + credentials=pika.PlainCredentials(os.environ["BROKER_USER"], os.environ["BROKER_PASSWORD"]), + heartbeat=max(120, JOB_TIMEOUT + 60), blocked_connection_timeout=120, + socket_timeout=5, connection_attempts=3, retry_delay=1) + mq = pika.BlockingConnection(params) + try: + yield mq + finally: + try: + if mq.is_open: + mq.close() + except AMQPError: + pass + + +def setup_channel(conn): + ch = conn.channel() + ch.queue_declare(queue=QUEUE, durable=True) + return ch + + +def queue_outbox(db, job_id, attachment_id, version, attempt, available_at=None): + payload = job_payload(job_id, attachment_id, version, attempt) + db.execute(""" + INSERT INTO job_outbox(id,job_id,payload,available_at) + VALUES (%s,%s,%s::jsonb,COALESCE(%s,now())) + """, (str(uuid.uuid4()), job_id, json.dumps(payload), available_at)) + + +def cleanup_abandoned_staging(): + """Best effort cleanup of retained stages; never delete current processing work.""" + from .attachments import BUCKET, s3_client + client = s3_client() + retention = max(3600, int(os.getenv("STAGING_RETENTION_SECONDS", "86400"))) + with connect_db() as db: + rows = db.execute(""" + SELECT id,staging_key FROM attachments + WHERE staging_key IS NOT NULL AND + ((status='ready' AND updated_at < now()-interval '300 seconds') OR + (status='failed' AND updated_at < now()-(%s*interval '1 second'))) + LIMIT 50 + """, (retention,)).fetchall() + for row in rows: + with connect_db() as db: + with db.transaction(): + # Row-level lock serializes against operator replay. + current = db.execute("SELECT status,staging_key,updated_at FROM attachments WHERE id=%s FOR UPDATE", (row["id"],)).fetchone() + if not current or current["staging_key"] != row["staging_key"]: + continue + age = (datetime.now(timezone.utc) - current["updated_at"]).total_seconds() + if not (current["status"] == "ready" and age >= 300 or current["status"] == "failed" and age >= retention): + continue + client.delete_object(Bucket=BUCKET, Key=row["staging_key"]) + db.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND staging_key=%s", (row["id"], row["staging_key"])) + # Also prune staging uploads whose DB COMMIT failed, after a long retention. + from botocore.exceptions import ClientError + for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET, Prefix="staging/"): + for obj in page.get("Contents", []): + if (datetime.now(timezone.utc) - obj["LastModified"]).total_seconds() < retention: + continue + key = obj["Key"] + with connect_db() as db: + with db.transaction(): + referenced = db.execute("SELECT 1 FROM attachments WHERE staging_key=%s", (key,)).fetchone() + if not referenced: + client.delete_object(Bucket=BUCKET, Key=key) + +def rescue_expired_jobs(conn): + """Requeue leased jobs whose worker died (durably, no manual task recreation).""" + with conn.transaction(): + rows = conn.execute(""" + SELECT j.id,j.attachment_id,j.pipeline_version,j.attempt + FROM processing_jobs j + WHERE j.status='processing' AND j.leased_until < now() + FOR UPDATE SKIP LOCKED LIMIT 20 + """).fetchall() + for row in rows: + if row["attempt"] >= MAX_ATTEMPTS: + conn.execute("""UPDATE processing_jobs + SET status='failed',error_code='retry_exhausted',lease_token=NULL,leased_until=NULL,updated_at=now() + WHERE id=%s""", (row["id"],)) + conn.execute("""UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=now() + WHERE id=%s""", (row["attachment_id"],)) + else: + attempt = row["attempt"] + 1 + conn.execute("""UPDATE processing_jobs SET status='queued',attempt=%s, + lease_token=NULL,leased_until=NULL,next_run_at=now(),updated_at=now() + WHERE id=%s""", (attempt, row["id"])) + conn.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (row["attachment_id"],)) + queue_outbox(conn, row["id"], row["attachment_id"], row["pipeline_version"], attempt) + log.warning("reclaimed timed-out job %s attempt %s", row["id"], attempt) + + +def dispatch_loop(): + """Publish outbox rows with publisher confirm, mark only AFTER confirm.""" + while True: + try: + with connection() as mq: + ch = setup_channel(mq) + ch.confirm_delivery() + last_clean = time.monotonic() + while True: + # Periodic lease recovery works even if original message was consumed. + with connect_db() as conn: + rescue_expired_jobs(conn) + with conn.transaction(): + rows = conn.execute(""" + SELECT id,payload FROM job_outbox + WHERE published_at IS NULL AND available_at<=now() + ORDER BY available_at,created_at + FOR UPDATE SKIP LOCKED LIMIT 20 + """).fetchall() + for row in rows: + if os.getenv("LAB5_FAULT_BEFORE_PUBLISH") == "once": + # Persist one-time fault before hard exit; outbox still + # unmarked and dispatcher will publish on restart. + with connect_db() as faultdb: + inserted = faultdb.execute(""" + INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'before_publish') + ON CONFLICT DO NOTHING RETURNING job_id + """, (row["payload"]["job_id"],)).fetchone() + if inserted: + os._exit(78) + raw = json.dumps(row["payload"], separators=(",", ":")) + # Mandatory + confirm; an ambiguous publish will be retried. + ch.basic_publish(exchange="", routing_key=QUEUE, body=raw.encode("utf-8"), + mandatory=True, properties=pika.BasicProperties(content_type="application/json", delivery_mode=2)) + conn.execute("UPDATE job_outbox SET published_at=now() WHERE id=%s", (row["id"],)) + if time.monotonic() - last_clean > 60: + try: + cleanup_abandoned_staging() + except Exception: + log.warning("staging janitor failed; retry later") + last_clean = time.monotonic() + if not rows: + time.sleep(1) + except (AMQPError, OSError, psycopg.Error) as exc: + log.warning("dispatcher reconnect after %s", type(exc).__name__) + time.sleep(2) + + +def claim_job(db, message): + job_id = message["job_id"] + with db.transaction(): + job = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job_id,)).fetchone() + if job is None or job["status"] in ("ready", "failed") or job["attempt"] != message["attempt"] or job["pipeline_version"] != message["pipeline_version"]: + return "ignore", None + if job["status"] == "processing": + return "busy", None + if job["next_run_at"] > datetime.now(timezone.utc): + return "busy", None + lease = str(uuid.uuid4()) + db.execute("""UPDATE processing_jobs SET status='processing',lease_token=%s, + leased_until=now()+(%s*interval '1 second'),updated_at=now() + WHERE id=%s""", (lease, LEASE_SECONDS, job_id)) + db.execute("UPDATE attachments SET status='processing',updated_at=now() WHERE id=%s", (job["attachment_id"],)) + return "claimed", {"job": job, "lease": lease} + + +def publish_result(db, *, job, lease, variants): + """Final result is committed atomically; stale leases/versions cannot publish.""" + with db.transaction(): + current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone() + if current["status"] != "processing" or str(current["lease_token"]) != lease or current["pipeline_version"] != job["pipeline_version"]: + log.warning("stale worker result ignored %s", job["id"]) + return False + for v in variants: + db.execute(""" + INSERT INTO attachment_variants(attachment_id,pipeline_version,name,object_key,content_type,size_bytes,sha256,width,height) + VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) + ON CONFLICT(attachment_id,pipeline_version,name) DO UPDATE SET + object_key=EXCLUDED.object_key,content_type=EXCLUDED.content_type, + size_bytes=EXCLUDED.size_bytes,sha256=EXCLUDED.sha256, + width=EXCLUDED.width,height=EXCLUDED.height + """, (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], + v["content_type"], v["size_bytes"], v["sha256"], v["width"], v["height"])) + db.execute("""UPDATE attachments SET status='ready',error_code=NULL,updated_at=now() + WHERE id=%s AND pipeline_version=%s""", (job["attachment_id"], job["pipeline_version"])) + db.execute("""UPDATE processing_jobs SET status='ready',lease_token=NULL,leased_until=NULL, + error_code=NULL,updated_at=now() WHERE id=%s""", (job["id"],)) + return True + + +def fail_job(db, job, lease, *, permanent=False, code="processing_failed"): + with db.transaction(): + current = db.execute("SELECT * FROM processing_jobs WHERE id=%s FOR UPDATE", (job["id"],)).fetchone() + if current["status"] != "processing" or str(current["lease_token"]) != lease: + return + if permanent or job["attempt"] >= MAX_ATTEMPTS: + code = code if permanent else "retry_exhausted" + db.execute("""UPDATE processing_jobs SET status='failed',error_code=%s,lease_token=NULL, + leased_until=NULL,updated_at=now() WHERE id=%s""", (code, job["id"])) + db.execute("UPDATE attachments SET status='failed',error_code=%s,updated_at=now() WHERE id=%s", (code, job["attachment_id"])) + else: + attempt = job["attempt"] + 1 + delay = min(60.0, 2.0 * (2 ** (attempt - 2))) + random.uniform(0, 1.0) + run_at = datetime.now(timezone.utc) + timedelta(seconds=delay) + db.execute("""UPDATE processing_jobs SET status='queued',attempt=%s,next_run_at=%s, + lease_token=NULL,leased_until=NULL,updated_at=now() WHERE id=%s""", (attempt, run_at, job["id"])) + db.execute("UPDATE attachments SET status='queued',updated_at=now() WHERE id=%s", (job["attachment_id"],)) + queue_outbox(db, job["id"], job["attachment_id"], job["pipeline_version"], attempt, run_at) + log.warning("job %s scheduled retry %s after %.1fs", job["id"], attempt, delay) + + +def cleanup_staging(job): + from .attachments import BUCKET, s3_client + from botocore.exceptions import BotoCoreError, ClientError + try: + with connect_db() as conn: + row = conn.execute("SELECT staging_key,status FROM attachments WHERE id=%s", (job["attachment_id"],)).fetchone() + if row and row["status"] == "ready" and row["staging_key"]: + s3_client().delete_object(Bucket=BUCKET, Key=row["staging_key"]) + with connect_db() as conn: + conn.execute("UPDATE attachments SET staging_key=NULL WHERE id=%s AND status='ready' AND staging_key=%s", + (job["attachment_id"], row["staging_key"])) + except (BotoCoreError, ClientError, psycopg.Error, OSError): + log.warning("staging cleanup deferred for %s", job["id"]) + + +def process_message(raw): + try: + msg = json.loads(raw) + assert msg["schema_version"] == 1 and msg["type"] == "attachment.process.v1" + uuid.UUID(msg["job_id"]) + uuid.UUID(msg["attachment_id"]) + assert type(msg["attempt"]) is int and msg["attempt"] >= 1 + except (ValueError, KeyError, AssertionError, TypeError): + log.error("invalid internal broker message discarded") + return "ack" # no secrets printed, does not touch DB + with connect_db() as db: + status, state = claim_job(db, msg) + if status == "ignore": + return "ack" + if status == "busy": + return "requeue" + job, lease = state["job"], state["lease"] + try: + cmd = [sys.executable, "-m", "app.image_worker", str(job["attachment_id"]), str(job["pipeline_version"])] + proc = subprocess.run(cmd, capture_output=True, timeout=JOB_TIMEOUT, check=False) + if proc.returncode != 0: + with connect_db() as db: + fail_job(db, job, lease, permanent=(proc.returncode == 20), + code="invalid_image" if proc.returncode == 20 else "processing_failed") + return "ack" # next attempt is already safely in DB outbox + result = json.loads(proc.stdout) + variants = result["variants"] + if sorted(v["name"] for v in variants) != ["crop", "optimized", "thumbnail"]: + raise ValueError("incomplete variants") + if os.getenv("LAB5_FAULT_AFTER_S3") == "once": + # One-time operator-only injected hard death, after all S3 writes and + # before DB result commit/ack; SQL PK prevents endless failure loop. + with connect_db() as db: + with db.transaction(): + inserted = db.execute(""" + INSERT INTO job_fault_once(job_id,fault) VALUES (%s,'after_s3') + ON CONFLICT DO NOTHING RETURNING job_id + """, (job["id"],)).fetchone() + if inserted: + os._exit(77) + with connect_db() as db: + ok = publish_result(db, job=job, lease=lease, variants=variants) + if ok: + cleanup_staging(job) + return "ack" + except subprocess.TimeoutExpired: + with connect_db() as db: + fail_job(db, job, lease, code="processing_timeout") + return "ack" + except Exception: + log.exception("job processing failure (internal details not sent to API)") + # Leave lease for dispatcher recovery if even DB unavailable. + try: + with connect_db() as db: + fail_job(db, job, lease) + return "ack" + except psycopg.Error: + return "requeue" + + +def worker_loop(): + while True: + try: + with connection() as mq: + ch = setup_channel(mq) + ch.basic_qos(prefetch_count=max(1, int(os.getenv("WORKER_PREFETCH", "1")))) + + def callback(channel, method, props, body): + outcome = process_message(body) + if outcome == "ack": + channel.basic_ack(method.delivery_tag) + else: + channel.basic_nack(method.delivery_tag, requeue=True) + time.sleep(2) # prevent tight redelivery spin on an active lease + + ch.basic_consume(queue=QUEUE, on_message_callback=callback, auto_ack=False) + log.info("worker ready; max attempts=%d timeout=%ds", MAX_ATTEMPTS, JOB_TIMEOUT) + ch.start_consuming() + except (AMQPError, OSError, psycopg.Error) as exc: + log.warning("worker reconnect after %s", type(exc).__name__) + time.sleep(2) + + +def replay_failed(ident): + with connect_db() as db: + with db.transaction(): + row = db.execute("SELECT * FROM processing_jobs WHERE attachment_id=%s FOR UPDATE", (ident,)).fetchone() + if not row or row["status"] != "failed": + raise RuntimeError("Only failed jobs may be explicitly replayed") + attachment = db.execute("SELECT staging_key FROM attachments WHERE id=%s", (ident,)).fetchone() + if not attachment or not attachment["staging_key"]: + raise RuntimeError("Retained staging is required for replay") + db.execute("""UPDATE processing_jobs SET status='queued',attempt=1,next_run_at=now(), + lease_token=NULL,leased_until=NULL,error_code=NULL,updated_at=now() WHERE id=%s""", (row["id"],)) + db.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=now() WHERE id=%s", (ident,)) + queue_outbox(db, row["id"], ident, row["pipeline_version"], 1) + db.execute("INSERT INTO job_replay_audit(id,job_id) VALUES (%s,%s)", (str(uuid.uuid4()), row["id"])) + print("Replayed failed job", ident) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=["dispatch", "worker", "replay", "status"]) + parser.add_argument("--id", help="UUID for explicit operator replay/status") + args = parser.parse_args() + if args.mode == "dispatch": + dispatch_loop() + elif args.mode == "worker": + worker_loop() + elif args.mode in ("replay", "status"): + if not args.id: + parser.error("--id is required") + ident = str(uuid.UUID(args.id)) + if args.mode == "replay": + replay_failed(ident) + else: + with connect_db() as db: + print(json.dumps(db.execute("SELECT status,attempt,error_code FROM processing_jobs WHERE attachment_id=%s", (ident,)).fetchone(), default=str)) + +if __name__ == "__main__": + main() diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..de86771f --- /dev/null +++ b/app/main.py @@ -0,0 +1,84 @@ +"""FastAPI process: one stateless replica. No auto-migration on startup.""" +import logging +import uuid +from contextlib import asynccontextmanager + +import psycopg +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.concurrency import run_in_threadpool +from redis.exceptions import RedisError + +from .api import router +from .attachments import router as attachment_router +from .auth import challenge, current_user +from .core import APIError, INSTANCE_ID, error_body, invalid, request_id +from .storage import start, stop + +logging.basicConfig(level="INFO") +logger = logging.getLogger("lab5") + +@asynccontextmanager +async def lifespan(app: FastAPI): + start() + try: + yield + finally: + stop() + +app = FastAPI(title="MTUCI Messenger Lab 5", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan) +app.include_router(router) +app.include_router(attachment_router) + +PROTECTED = ("/api/v1/chats", "/api/v1/attachments", "/api/v1/users/me", "/api/v1/auth/sessions/current") +JSON_POST = ("/api/v1/users", "/api/v1/chats") + + +@app.middleware("http") +async def request_context(req: Request, call_next): + req.state.request_id = uuid.uuid4().hex + try: + path = req.url.path + # Authenticate before parsing the request body or resource-specific fields. + if any(path == p or path.startswith(p + "/") for p in PROTECTED): + req.state.actor = await run_in_threadpool(current_user, req) + if req.method == "POST" and (path in JSON_POST or (path.startswith("/api/v1/chats/") and path.endswith("/messages"))): + if req.headers.get("content-type", "").split(";", 1)[0].strip().lower() != "application/json": + raise APIError(415, "unsupported_media_type", "Content-Type must be application/json") + if req.method == "POST" and path == "/api/v1/auth/sessions" and await req.body(): + raise invalid("Login body is not allowed") + response = await call_next(req) + except APIError as exc: + response = api_error(req, exc) + except (RedisError, psycopg.OperationalError, psycopg.InterfaceError): + response = api_error(req, APIError(503, "unavailable", "Dependency unavailable")) + except Exception: + logger.exception("unexpected request failure id=%s", request_id(req)) + response = api_error(req, APIError(500, "internal_error", "Internal server error")) + response.headers["X-Request-Id"] = request_id(req) + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +def api_error(req: Request, exc: APIError) -> JSONResponse: + headers = {"WWW-Authenticate": challenge(req)} if exc.status == 401 else {} + return JSONResponse(status_code=exc.status, content=error_body(req, exc.code, exc.message), headers=headers) + + +@app.exception_handler(APIError) +async def api_error_handler(req: Request, exc: APIError): + return api_error(req, exc) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(req: Request, exc: RequestValidationError): + return api_error(req, invalid()) + + +@app.exception_handler(Exception) +async def unexpected_error_handler(req: Request, exc: Exception): + logger.exception("handler failure id=%s", request_id(req)) + if isinstance(exc, (psycopg.OperationalError, psycopg.InterfaceError, RedisError)): + return api_error(req, APIError(503, "unavailable", "Dependency unavailable")) + return api_error(req, APIError(500, "internal_error", "Internal server error")) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..5f624e33 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,23 @@ +"""Run once before API instances; transactional and advisory-lock coordinated migrations.""" +import os +from pathlib import Path +import psycopg + +MIGRATIONS = Path(__file__).resolve().parents[1] / "db" / "migrations" + +def main(): + with psycopg.connect(os.environ["DATABASE_URL"], autocommit=False) as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(62132103)") + conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations(version TEXT PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT now())") + for path in sorted(MIGRATIONS.glob("[0-9]*.sql")): + applied = conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (path.name,)).fetchone() + if applied: + continue + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (path.name,)) + print("Applied", path.name, flush=True) + print("Migrations up to date", flush=True) + +if __name__ == "__main__": + main() diff --git a/app/models.py b/app/models.py new file mode 100644 index 00000000..d0018c58 --- /dev/null +++ b/app/models.py @@ -0,0 +1,77 @@ +"""HTTP request models; all strings use Unicode code point limits.""" +import re +import uuid +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + +class CreateUser(StrictModel): + username: str + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("username") + @classmethod + def username_valid(cls, value: str) -> str: + if not USERNAME_RE.fullmatch(value): + raise ValueError("Invalid username") + return value + + @field_validator("display_name", "password") + @classmethod + def non_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("Blank value") + value.encode("utf-8", errors="strict") + return value + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def non_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("Blank title") + value.encode("utf-8", errors="strict") + return value + + @field_validator("member_ids") + @classmethod + def uuid_members(cls, value: list[str]) -> list[str]: + for member in value: + if not isinstance(member, str) or str(uuid.UUID(member)) != member: + raise ValueError("Invalid member UUID") + if len(value) != len(set(value)): + raise ValueError("Duplicate member") + return value + +class CreateMessage(StrictModel): + text: str = Field(min_length=0, max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("text") + @classmethod + def non_blank(cls, value: str) -> str: + value.encode("utf-8", errors="strict") + return value + + @field_validator("attachment_ids") + @classmethod + def valid_attachments(cls, value: list[str]) -> list[str]: + if len(value) != len(set(value)): + raise ValueError("Duplicate attachment IDs") + for item in value: + if not isinstance(item, str) or str(uuid.UUID(item)) != item: + raise ValueError("Invalid attachment UUID") + return value + + @model_validator(mode="after") + def nonempty_message(self): + if not self.text.strip() and not self.attachment_ids: + raise ValueError("Empty message") + return self diff --git a/app/pipeline_selftest.py b/app/pipeline_selftest.py new file mode 100644 index 00000000..af559cbd --- /dev/null +++ b/app/pipeline_selftest.py @@ -0,0 +1,25 @@ +"""Offline image conversion tests inside the application image; no S3 or DB.""" +import io +from PIL import Image +from .image_worker import transform + + +def main(): + source = Image.new('RGB',(320,200), (255,0,0)) + buf=io.BytesIO();source.save(buf,'PNG') + result=transform(buf.getvalue()) + items={name:(img,fmt,mime,opts) for name,img,fmt,mime,opts in result} + assert set(items)=={'thumbnail','crop','optimized'} + assert items['thumbnail'][0].size==(256,160) + assert items['crop'][0].size==(128,128) + assert items['optimized'][0].size==(320,200) + for name,(im,fmt,mime,opts) in items.items(): + data=io.BytesIO();im.save(data,format=fmt,**opts) + with Image.open(io.BytesIO(data.getvalue())) as decoded: + assert decoded.size==im.size + assert decoded.format==fmt + assert 'exif' not in decoded.info + print('PASS: PNG 320x200 -> JPEG thumb 256x160, crop 128x128, optimized WebP, metadata stripped') + +if __name__=='__main__': + main() diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..0f316828 --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,132 @@ +"""Lab4 grade-5 track 2: repeatable DB/S3 reconciliation. + +Dry-run is default. Apply never drops metadata when BOTH object copies are gone: +without a trustworthy copy or external backup, deletion would destroy evidence. +""" +import argparse +import json +import sys +import time +from datetime import datetime, timezone + +import psycopg +from psycopg.rows import dict_row +from botocore.exceptions import BotoCoreError, ClientError + +from .attachments import BUCKET, s3_client + + +def inventory(client): + result = {} + for page in client.get_paginator("list_objects_v2").paginate(Bucket=BUCKET): + for obj in page.get("Contents", []): + # Only reconciliation-owned prefixes; never mutate arbitrary S3 keys. + key = obj["Key"] + if key.startswith(("attachments/", "recovery/")): + result[key] = obj["LastModified"] + return result + + +def get_db_rows(conn): + return conn.execute("SELECT id, object_key, backup_key, status FROM attachments").fetchall() + + +def current_reference(conn, key): + return conn.execute( + "SELECT 1 FROM attachments WHERE object_key=%s OR backup_key=%s", (key, key) + ).fetchone() is not None + + +def reconcile(*, apply=False, grace_seconds=300, now=None, client=None, connect=None): + if grace_seconds < 300 and apply: + raise ValueError("Apply requires >=300 s grace period to protect in-flight uploads") + started = time.monotonic() + now = now or datetime.now(timezone.utc) + client = client or s3_client() + connect = connect or (lambda: psycopg.connect(__import__('os').environ['DATABASE_URL'], row_factory=dict_row)) + objects = inventory(client) + stats = { + "mode": "apply" if apply else "dry-run", "grace_seconds": grace_seconds, + "s3_keys_scanned": len(objects), "db_rows_scanned": 0, + "orphan_keys": [], "orphan_keys_young": [], + "missing_primary_repairable": [], "missing_backup_repairable": [], + "missing_both_unrecoverable": [], "actions": [], "errors": [], + } + with connect() as conn: + rows = get_db_rows(conn) + stats["db_rows_scanned"] = len(rows) + references = {key for row in rows for key in (row["object_key"], row["backup_key"])} + for row in rows: + if row["status"] != "ready": + continue # queued/processing have durable staging, not final variants yet + primary, backup = row["object_key"], row["backup_key"] + have_primary, have_backup = primary in objects, backup in objects + rid = str(row["id"]) + if have_primary and have_backup: + continue + if not have_primary and not have_backup: + stats["missing_both_unrecoverable"].append(rid) + continue + dest, source, typ = (primary, backup, "missing_primary_repairable") if not have_primary else (backup, primary, "missing_backup_repairable") + stats[typ].append(rid) + if apply: + # Defensive recheck. Object may have reappeared after inventory. + try: + client.head_object(Bucket=BUCKET, Key=dest) + stats["actions"].append({"action": "already-restored", "key": dest}) + continue + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code") + if code not in ("404", "NotFound", "NoSuchKey"): + raise + try: + client.copy_object(Bucket=BUCKET, Key=dest, + CopySource={"Bucket": BUCKET, "Key": source}) + stats["actions"].append({"action": "restore", "key": dest, "from": source}) + except (BotoCoreError, ClientError) as exc: + stats["errors"].append({"action": "restore", "key": dest, "error_type": type(exc).__name__}) + for key, modified in objects.items(): + if key in references: + continue + age = (now - modified).total_seconds() + if age < grace_seconds: + stats["orphan_keys_young"].append(key) + continue + stats["orphan_keys"].append(key) + if apply: + # Query again immediately before destructive action; never delete + # an object now referenced by a committed metadata row. + if current_reference(conn, key): + stats["actions"].append({"action": "skip-now-referenced", "key": key}) + continue + try: + client.delete_object(Bucket=BUCKET, Key=key) + stats["actions"].append({"action": "delete-old-orphan", "key": key}) + except (BotoCoreError, ClientError) as exc: + stats["errors"].append({"action": "delete-old-orphan", "key": key, "error_type": type(exc).__name__}) + stats["scan_elapsed_seconds"] = round(time.monotonic() - started, 3) + return stats + + +def main(): + parser = argparse.ArgumentParser(description="Reconcile S3 attachments vs PostgreSQL (dry-run default)") + parser.add_argument("--apply", action="store_true", help="Make safe changes; orphans older than 300 seconds only") + parser.add_argument("--grace-seconds", type=int, default=300) + parser.add_argument("--output", help="Write JSON summary (no credentials)") + args = parser.parse_args() + try: + result = reconcile(apply=args.apply, grace_seconds=args.grace_seconds) + except Exception as exc: + # No exception string: botocore errors may contain signed credentials/URLs. + print(json.dumps({"status":"failed", "error_type":type(exc).__name__}), file=sys.stderr) + raise SystemExit(2) from None + rendered = json.dumps(result, ensure_ascii=False, indent=2) + print(rendered) + if args.output: + with open(args.output, "w", encoding="utf-8") as fh: + fh.write(rendered + "\n") + if result["errors"] or result["missing_both_unrecoverable"]: + raise SystemExit(1) + +if __name__ == "__main__": + main() diff --git a/app/reconcile_fault.py b/app/reconcile_fault.py new file mode 100644 index 00000000..fce1eea4 --- /dev/null +++ b/app/reconcile_fault.py @@ -0,0 +1,36 @@ +"""Operator-only fault injection in a LOCAL TEST stack; never exposed as HTTP.""" +import argparse +import json +import os +import uuid +import psycopg +from psycopg.rows import dict_row +from .attachments import BUCKET, s3_client + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("action", choices=["delete-primary", "delete-backup", "create-orphan"]) + p.add_argument("--id", help="Attachment UUID for the delete actions") + args = p.parse_args() + if os.environ.get("LAB4_ALLOW_FAULT_INJECTION") != "yes": + raise SystemExit("Fault injection disabled; use LAB4_ALLOW_FAULT_INJECTION=yes in isolated test") + client = s3_client() + if args.action == "create-orphan": + key = f"attachments/{uuid.uuid4()}/original" + client.put_object(Bucket=BUCKET, Key=key, Body=b"synthetic orphan") + print(json.dumps({"fault":"orphan-created", "object_key":key})) + return + if not args.id: + raise SystemExit("--id is required") + ident = str(uuid.UUID(args.id)) + with psycopg.connect(os.environ["DATABASE_URL"], row_factory=dict_row) as conn: + row = conn.execute("SELECT object_key,backup_key FROM attachments WHERE id=%s", (ident,)).fetchone() + if not row: + raise SystemExit("Attachment does not exist") + key = row["object_key"] if args.action == "delete-primary" else row["backup_key"] + client.delete_object(Bucket=BUCKET, Key=key) + print(json.dumps({"fault":args.action, "object_key":key})) + +if __name__ == "__main__": + main() diff --git a/app/reconcile_selftest.py b/app/reconcile_selftest.py new file mode 100644 index 00000000..e9d2c81d --- /dev/null +++ b/app/reconcile_selftest.py @@ -0,0 +1,61 @@ +"""Fast isolated tests for reconciliation safety; run in application image.""" +import uuid +from datetime import datetime, timezone, timedelta +from .reconcile import reconcile +from .attachments import BUCKET + +NOW = datetime.now(timezone.utc) + +class FakeConn: + def __init__(self, rows): self.rows=rows + def __enter__(self): return self + def __exit__(self,*a): return False + def execute(self, sql, args=None): + self.sql=sql + self.args=args + return self + def fetchall(self): return self.rows + def fetchone(self): + key=self.args[0] + return {'id': 1} if any(key in (row['object_key'],row['backup_key']) for row in self.rows) else None + +class FakeS3: + def __init__(self,objects): self.objects=objects + def get_paginator(self,op): return self + def paginate(self,**kwargs): + yield {'Contents':[{'Key':k,'LastModified':t} for k,t in self.objects.items()]} + def head_object(self,**kw): + if kw['Key'] not in self.objects: + from botocore.exceptions import ClientError + raise ClientError({'Error':{'Code':'404'}},'HeadObject') + def copy_object(self,**kw): + assert kw['CopySource']['Key'] in self.objects + self.objects[kw['Key']]=NOW + def delete_object(self,**kw): + self.objects.pop(kw['Key'],None) + + +def test(): + i=str(uuid.uuid4()) + primary=f'attachments/{i}/original';backup=f'recovery/{i}/original' + young=f'attachments/{uuid.uuid4()}/original' + old=f'attachments/{uuid.uuid4()}/original' + rows=[{'id':i,'object_key':primary,'backup_key':backup,'status':'ready'}] + fake=FakeS3({backup: NOW-timedelta(days=1),young:NOW-timedelta(seconds=10),old:NOW-timedelta(days=1)}) + kwargs={'now':NOW,'client':fake,'connect':lambda:FakeConn(rows)} + first=reconcile(apply=False,**kwargs) + assert first['missing_primary_repairable']==[i] + assert old in first['orphan_keys'] and young in first['orphan_keys_young'] + assert primary not in fake.objects and old in fake.objects, 'Dry run mutated objects' + second=reconcile(apply=True,**kwargs) + assert primary in fake.objects and old not in fake.objects and young in fake.objects + assert any(x['action']=='restore' for x in second['actions']) + third=reconcile(apply=True,**kwargs) + assert not third['actions'], 'Re-run unexpectedly mutated objects' + assert rows[0]['object_key']==primary, 'Metadata destroyed' + try: reconcile(apply=True, grace_seconds=0, **kwargs) + except ValueError: pass + else: raise AssertionError('Unsafe zero-grace apply accepted') + print('PASS: dry-run, orphan grace, recover primary from backup, apply idempotence') + +if __name__ == '__main__': test() diff --git a/app/s3_bootstrap.py b/app/s3_bootstrap.py new file mode 100644 index 00000000..da81f64e --- /dev/null +++ b/app/s3_bootstrap.py @@ -0,0 +1,35 @@ +"""Idempotently create the private S3 bucket with bootstrap-only credentials.""" +import os +import time +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError, BotoCoreError + + +def main(): + bucket = os.environ["S3_BUCKET"] + client = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], + aws_access_key_id=os.environ["S3_ADMIN_ACCESS_KEY"], + aws_secret_access_key=os.environ["S3_ADMIN_SECRET_KEY"], + region_name="us-east-1", + config=Config(s3={"addressing_style": "path"}, + connect_timeout=2, read_timeout=4, retries={"max_attempts": 1})) + for attempt in range(70): + try: + try: + client.head_bucket(Bucket=bucket) + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + if code not in ("404", "NoSuchBucket", "NotFound"): + raise + client.create_bucket(Bucket=bucket) + # PRIVATE: no public bucket policy, ACL or anonymous identity. + print(f"Private bucket {bucket} ready", flush=True) + return + except (ClientError, BotoCoreError, OSError) as exc: + if attempt == 69: + raise RuntimeError("S3 bootstrap failed; inspect S3 container") from exc + time.sleep(1) + +if __name__ == "__main__": + main() diff --git a/app/s3_policy.py b/app/s3_policy.py new file mode 100644 index 00000000..b51dead4 --- /dev/null +++ b/app/s3_policy.py @@ -0,0 +1,32 @@ +"""One-off creation of private SeaweedFS S3 identities; secrets stay in Docker volume.""" +import json +import os +from pathlib import Path + + +def main(): + bucket = os.environ["S3_BUCKET"] + identities = [ + {"name": "bucket-bootstrap", "credentials": [{"accessKey": os.environ["S3_ADMIN_ACCESS_KEY"], + "secretKey": os.environ["S3_ADMIN_SECRET_KEY"]}], + "actions": ["Admin", "Read", "Write", "List"]}, + {"name": "messenger-api", "credentials": [{"accessKey": os.environ["S3_ACCESS_KEY"], + "secretKey": os.environ["S3_SECRET_KEY"]}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ] + if len({entry["credentials"][0]["accessKey"] for entry in identities}) != 2: + raise ValueError("Admin and API credentials must differ") + path = Path("/s3-config/config.json") + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + old_umask = os.umask(0o077) + try: + tmp.write_text(json.dumps({"identities": identities}, separators=(",", ":")), encoding="utf-8") + os.chmod(tmp, 0o644) # readable by the SeaweedFS runtime UID; Docker volume stays private + os.replace(tmp, path) + finally: + os.umask(old_umask) + print("S3 identities configured (no secrets printed)", flush=True) + +if __name__ == "__main__": + main() diff --git a/app/s3_probe.py b/app/s3_probe.py new file mode 100644 index 00000000..ef6d441e --- /dev/null +++ b/app/s3_probe.py @@ -0,0 +1,40 @@ +"""Verify bucket is private at S3 boundary (not merely hidden behind HTTP API).""" +import json +import os +import boto3 +from botocore import UNSIGNED +from botocore.config import Config +from botocore.exceptions import ClientError +from .attachments import BUCKET, s3_client + + +def main(): + client = s3_client() + client.head_bucket(Bucket=BUCKET) # restricted app identity works + unauth = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"], + region_name="us-east-1", config=Config(signature_version=UNSIGNED, + s3={"addressing_style":"path"})) + try: + unauth.list_objects_v2(Bucket=BUCKET, MaxKeys=1) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status not in (401,403): + raise RuntimeError("Unexpected anonymous S3 response") from None + else: + raise RuntimeError("Private bucket leaked object list to anonymous caller") + listing = client.list_objects_v2(Bucket=BUCKET, MaxKeys=1) + keys = [entry["Key"] for entry in listing.get("Contents", [])] + if keys: + try: + unauth.get_object(Bucket=BUCKET, Key=keys[0]) + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status not in (401, 403): + raise RuntimeError("Unexpected anonymous object GET response") from None + else: + raise RuntimeError("Private S3 object downloadable anonymously") + print(json.dumps({"private_bucket": True, "anonymous_listing": "denied", + "anonymous_object_get": "denied" if keys else "not_tested_empty_bucket"})) + +if __name__ == "__main__": + main() diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..2ab0d05a --- /dev/null +++ b/app/storage.py @@ -0,0 +1,64 @@ +"""Shared external state: psycopg connection pool and Redis session store.""" +import os +from contextlib import contextmanager + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +from redis import Redis +from redis.exceptions import RedisError + +from .core import APIError + +pool: ConnectionPool | None = None +redis_client: Redis | None = None + + +def start() -> None: + global pool, redis_client + url = os.environ["DATABASE_URL"] + size = int(os.getenv("DB_POOL_MAX", "8")) + pool = ConnectionPool(conninfo=url, min_size=1, max_size=size, timeout=3, kwargs={"row_factory": dict_row, "connect_timeout": 5, "options": "-c statement_timeout=5000"}, open=False) + pool.open(wait=True, timeout=30) + redis_client = Redis.from_url(os.environ.get("REDIS_URL", "redis://redis:6379/0"), socket_connect_timeout=2, socket_timeout=2, decode_responses=True) + redis_client.ping() + + +def stop() -> None: + global pool, redis_client + if pool is not None: + pool.close() + pool = None + if redis_client is not None: + redis_client.close() + redis_client = None + + +@contextmanager +def db(): + if pool is None: + raise APIError(503, "unavailable", "Database unavailable") + try: + with pool.connection() as conn: + yield conn + except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout) as exc: + raise APIError(503, "unavailable", "Database unavailable") from exc + + +def cache() -> Redis: + if redis_client is None: + raise APIError(503, "unavailable", "Session store unavailable") + return redis_client + + +def ready() -> bool: + try: + with db() as conn: + conn.execute("SELECT 1").fetchone() + cache().ping() + # S3 is needed for the mandatory synchronous upload/download path. + from .attachments import s3_client, BUCKET + s3_client().head_bucket(Bucket=BUCKET) + return True + except Exception: + return False diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..592804e3 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,251 @@ +services: + postgres: + image: postgres:16.10-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-lab} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: ["postgres_data:/var/lib/postgresql/data"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1.0 + restart: unless-stopped + networks: [storage] + + redis: + image: redis:7.4.5-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "60", "1"] + volumes: ["redis_data:/data"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 256m + cpus: 0.5 + restart: unless-stopped + networks: [storage] + + broker: + image: rabbitmq:4.1.8-management-alpine + hostname: lab5-broker + environment: + RABBITMQ_DEFAULT_USER: ${BROKER_USER:-labworker} + RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env} + volumes: ["rabbitmq_data:/var/lib/rabbitmq"] + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 25 + start_period: 15s + mem_limit: 512m + cpus: 0.75 + restart: unless-stopped + networks: [storage] + + s3-policy: + build: . + user: "0:0" + command: ["python", "-m", "app.s3_policy"] + environment: + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY} + S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + volumes: ["s3_config:/s3-config"] + restart: "no" + networks: [storage] + + s3: + image: chrislusf/seaweedfs:4.53 + command: ["server", "-dir=/data", "-ip=s3", "-ip.bind=0.0.0.0", "-s3", "-s3.port=8333", "-s3.iam=false", "-s3.config=/s3-config/config.json"] + volumes: + - s3_data:/data + - s3_config:/s3-config:ro + - ./infra/filer.toml:/etc/seaweedfs/filer.toml:ro + depends_on: + s3-policy: + condition: service_completed_successfully + expose: ["8333"] + mem_limit: 640m + cpus: 1.0 + restart: unless-stopped + networks: [storage] + + s3-bootstrap: + build: . + command: ["python", "-m", "app.s3_bootstrap"] + environment: + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY: ${S3_ADMIN_ACCESS_KEY:?Set S3_ADMIN_ACCESS_KEY} + S3_ADMIN_SECRET_KEY: ${S3_ADMIN_SECRET_KEY:?Set S3_ADMIN_SECRET_KEY} + depends_on: + s3: + condition: service_started + restart: "no" + networks: [storage] + + migrate: + build: . + command: ["python", "-m", "app.migrate"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + depends_on: + postgres: + condition: service_healthy + restart: "no" + networks: [storage] + + reconcile: + profiles: [maintenance] + build: . + command: ["python", "-m", "app.reconcile"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + depends_on: + postgres: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + restart: "no" + networks: [storage] + + dispatcher: + build: . + command: ["python", "-m", "app.jobs", "dispatch"] + environment: &job_env + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + BROKER_HOST: broker + BROKER_USER: ${BROKER_USER:-labworker} + BROKER_PASSWORD: ${BROKER_PASSWORD:?Set BROKER_PASSWORD in .env} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + depends_on: + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + mem_limit: 192m + cpus: 0.5 + restart: unless-stopped + networks: [storage] + + worker: + build: . + command: ["python", "-m", "app.jobs", "worker"] + environment: + <<: *job_env + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + LAB5_FAULT_AFTER_S3: ${LAB5_FAULT_AFTER_S3:-off} + depends_on: + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + mem_limit: 768m + cpus: 1.0 + stop_grace_period: 75s + restart: unless-stopped + networks: [storage] + + api1: + build: . + environment: &api_env + APP_PORT: "8080" + API_INSTANCE_ID: api1 + DATABASE_URL: postgresql://${POSTGRES_USER:-lab}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX: ${DB_POOL_MAX:-8} + JOB_BACKLOG_LIMIT: ${JOB_BACKLOG_LIMIT:-1000} + S3_ENDPOINT: http://s3:8333 + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:?Set S3_ACCESS_KEY} + S3_SECRET_KEY: ${S3_SECRET_KEY:?Set S3_SECRET_KEY} + depends_on: &api_depends + postgres: + condition: service_healthy + redis: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-bootstrap: + condition: service_completed_successfully + expose: ["8080"] + healthcheck: &api_health + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 2s + timeout: 3s + retries: 15 + start_period: 10s + mem_limit: 384m + cpus: 0.75 + stop_grace_period: 12s + restart: unless-stopped + networks: [frontend, storage] + + api2: + build: . + environment: + <<: *api_env + API_INSTANCE_ID: api2 + depends_on: *api_depends + expose: ["8080"] + healthcheck: *api_health + mem_limit: 384m + cpus: 0.75 + stop_grace_period: 12s + restart: unless-stopped + networks: [frontend, storage] + + lb: + image: haproxy:3.1.7-alpine + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + ports: ["127.0.0.1:8080:8080"] + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 + mem_limit: 128m + cpus: 0.5 + restart: unless-stopped + networks: [frontend] + +networks: + frontend: + storage: + internal: true +volumes: + postgres_data: + redis_data: + s3_data: + s3_config: + rabbitmq_data: diff --git a/db/migrations/001_initial.sql b/db/migrations/001_initial.sql new file mode 100644 index 00000000..203b012f --- /dev/null +++ b/db/migrations/001_initial.sql @@ -0,0 +1,33 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE, + display_name BYTEA NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_display_bytes CHECK (octet_length(display_name) BETWEEN 1 AND 400) +); +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_bytes CHECK (octet_length(title) BETWEEN 1 AND 400) +); +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id), + user_id UUID NOT NULL REFERENCES users(id), + position INTEGER NOT NULL, + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); +CREATE INDEX IF NOT EXISTS idx_members_user ON chat_members (user_id, chat_id); +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + sender_id UUID NOT NULL, + text BYTEA NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id), + CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 1 AND 16000) +); +CREATE INDEX IF NOT EXISTS idx_messages_chat_history ON messages (chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_chats_history ON chats (created_at, id); diff --git a/db/migrations/002_idempotency.sql b/db/migrations/002_idempotency.sql new file mode 100644 index 00000000..04a000ad --- /dev/null +++ b/db/migrations/002_idempotency.sql @@ -0,0 +1,12 @@ +-- Same transaction creates message and idempotency receipt. Unique key arbitrates concurrent replicas. +CREATE TABLE IF NOT EXISTS message_idempotency ( + actor_id UUID NOT NULL REFERENCES users(id), + chat_id UUID NOT NULL REFERENCES chats(id), + method VARCHAR(8) NOT NULL DEFAULT 'POST', + idem_key VARCHAR(128) NOT NULL, + body_sha256 CHAR(64) NOT NULL, + message_id UUID NOT NULL REFERENCES messages(id), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (actor_id, chat_id, method, idem_key) +); +CREATE INDEX IF NOT EXISTS idx_idempotency_age ON message_idempotency (created_at); diff --git a/db/migrations/003_attachments.sql b/db/migrations/003_attachments.sql new file mode 100644 index 00000000..ed8e0ea9 --- /dev/null +++ b/db/migrations/003_attachments.sql @@ -0,0 +1,25 @@ +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_bytes; +ALTER TABLE messages ADD CONSTRAINT messages_text_bytes CHECK (octet_length(text) BETWEEN 0 AND 16000); +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + object_key TEXT NOT NULL UNIQUE, + backup_key TEXT NOT NULL UNIQUE, + filename TEXT NOT NULL, + content_type VARCHAR(32) NOT NULL CHECK(content_type IN ('image/png','image/jpeg')), + size_bytes BIGINT NOT NULL CHECK(size_bytes BETWEEN 1 AND 10485760), + sha256 CHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id) +); +CREATE INDEX IF NOT EXISTS idx_attachments_chat ON attachments(chat_id); +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id), + attachment_id UUID NOT NULL REFERENCES attachments(id), + position INTEGER NOT NULL CHECK(position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id,position) +); +CREATE INDEX IF NOT EXISTS idx_message_attachments_attachment ON message_attachments(attachment_id); diff --git a/db/migrations/004_jobs.sql b/db/migrations/004_jobs.sql new file mode 100644 index 00000000..b7099927 --- /dev/null +++ b/db/migrations/004_jobs.sql @@ -0,0 +1,58 @@ +-- Lab5: stage durably, then transact job and outbox atomically. +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS status TEXT NOT NULL DEFAULT 'ready' + CHECK (status IN ('queued','processing','ready','failed')); +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code TEXT; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS pipeline_version INTEGER NOT NULL DEFAULT 1; +CREATE UNIQUE INDEX IF NOT EXISTS idx_attachments_staging ON attachments(staging_key) WHERE staging_key IS NOT NULL; +CREATE INDEX IF NOT EXISTS idx_attachments_status ON attachments(status); + +CREATE TABLE IF NOT EXISTS attachment_variants ( + attachment_id UUID NOT NULL REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL CHECK (pipeline_version >= 1), + name TEXT NOT NULL CHECK (name IN ('thumbnail','crop','optimized')), + object_key TEXT NOT NULL UNIQUE, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL CHECK (size_bytes > 0), + sha256 CHAR(64) NOT NULL, + width INTEGER NOT NULL CHECK (width > 0), + height INTEGER NOT NULL CHECK (height > 0), + PRIMARY KEY (attachment_id,pipeline_version,name) +); + +CREATE TABLE IF NOT EXISTS processing_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id), + pipeline_version INTEGER NOT NULL DEFAULT 1, + status TEXT NOT NULL DEFAULT 'queued' CHECK (status IN ('queued','processing','ready','failed')), + attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1), + next_run_at TIMESTAMPTZ NOT NULL DEFAULT now(), + lease_token UUID, + leased_until TIMESTAMPTZ, + error_code TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_jobs_leases ON processing_jobs(status, leased_until); + +CREATE TABLE IF NOT EXISTS job_outbox ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES processing_jobs(id), + payload JSONB NOT NULL, + available_at TIMESTAMPTZ NOT NULL DEFAULT now(), + published_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS idx_outbox_due ON job_outbox(available_at, created_at) WHERE published_at IS NULL; + +CREATE TABLE IF NOT EXISTS job_replay_audit ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES processing_jobs(id), + replayed_at TIMESTAMPTZ NOT NULL DEFAULT now(), + action TEXT NOT NULL DEFAULT 'operator_replay' +); +CREATE TABLE IF NOT EXISTS job_fault_once ( + job_id UUID NOT NULL REFERENCES processing_jobs(id), + fault TEXT NOT NULL, + PRIMARY KEY (job_id,fault) +); diff --git a/evidence/attachments-concurrent.json b/evidence/attachments-concurrent.json new file mode 100644 index 00000000..1a58338f --- /dev/null +++ b/evidence/attachments-concurrent.json @@ -0,0 +1,13 @@ +{ + "concurrency": 10, + "accepted_uploads": 10, + "ready_attachments": 10, + "failed_requests": 0, + "acceptance_elapsed_seconds": 2.391, + "acceptance_rps": 4.182, + "upload_latency_p50_ms": 2218.5, + "upload_latency_max_ms": 2313.0, + "all_ready_elapsed_seconds": 29.781, + "bytes_per_file": 111817, + "note": "HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats." +} \ No newline at end of file diff --git a/evidence/verified_results_2026-11-15.md b/evidence/verified_results_2026-11-15.md new file mode 100644 index 00000000..133533cf --- /dev/null +++ b/evidence/verified_results_2026-11-15.md @@ -0,0 +1,66 @@ +# Lab5 — итоговая сводка подтверждённых запусков (14–15.11.2026) + +Источник: фактические выводы Windows PowerShell / Docker Desktop, присланные автором 14–15 ноября 2026 года. Архив v6 обновляет документацию; новые контейнерные тесты при упаковке **не запускались**. Для повторяемости команды приведены в [`../LAB5_TEST_COMMANDS.md`](../LAB5_TEST_COMMANDS.md). + +## Контракт и функциональность + +| Команда / проверка | Фактически наблюдалось | +| --- | --- | +| `docker compose up -d --build --wait`, `docker compose ps` | Основная инфраструктура запустилась; API1/API2/broker/lb/DB/Redis/S3 healthy там, где определён healthcheck | +| `python scripts/check_contract.py` | PASS (только структура/синтаксис, не интеграционный прогон) | +| `python tests/own_unit.py` | PASS: модели, cursor scoping, limits, Unicode U+0000 | +| `python tests/smoke.py` | PASS после SQL-исправления и повторно после восстановления обеих API | +| `python tests/attachments_negative.py` | PASS: image/MIME/size/filename/ACL/variants/cross-chat после исправления ожиданий Lab5 | +| `python tests/idempotency_parallel.py` | PASS: 2 реплики, одно сообщение, replay `201`, конфликт `409` | +| `python -m app.pipeline_selftest` в worker | PASS: PNG 320×200 → JPEG 256×160, crop 128×128, WebP, stripped metadata | + +## Очередь, worker и восстановление + +| Сценарий | Фактически наблюдалось | +| --- | --- | +| `pipeline_resilience.py --scenario paused` | PASS: durable `queued → ready`; оригинал и 3 варианта, без дублей | +| `pipeline_resilience.py --scenario outbox-crash` | PASS: exit 78 после outbox commit, до RabbitMQ publish; восстановление | +| `pipeline_resilience.py --scenario worker-crash` | PASS: exit 77 после S3 effects, до result/ack; восстановление | +| `pipeline_extended.py --scenario broker-persistence` | PASS: publisher confirmed; `ready=1` до рестарта и после, dispatcher остановлен, нет republish из outbox | +| `pipeline_extended.py --scenario staging-cleanup` | PASS: `ready`, DB `staging_key=NULL`, staging object отсутствует в S3 | +| `pipeline_extended.py --scenario failed-replay` | PASS (15.11): permanent `invalid_image`, retry не запущен, variants отсутствуют; затем исправление метаданных, replay → `ready`, `audit=1`, replay готового задания отклонён | + +Первый запуск `failed-replay` упал с `TypeError` из-за отсутствующего `schema='Attachment'` при вызове тестового клиента. После исправления повторный тест **PASS**. Это исправление теста, а не подтверждённый дефект API. + +## Параллельность и ресурсы + +Повторный запуск `python tests/attachments_concurrent.py --concurrency 10 --output evidence/attachments-concurrent.json`: + +- **PASS**: HTTP `202` 10/10, `ready` 10/10, 0 ошибок, 3 варианта у каждого. +- Acceptance: **2.391 s** и **4.182 requests/s**; это throughput приёма HTTP, **не** скорость обработки worker. +- HTTP upload latency: p50 **2218.5 ms**, max **2313 ms**. +- Время завершения всех 10 задач от старта теста: **29.781 s** (не p95 и не среднее время одного задания). +- Размер каждого файла: **111817 bytes**. +- Параллельная 45-секундная выборка `docker stats`: по 15 отсчётов на контейнер. Наблюдавшийся максимум памяти: API1 **163.10 MiB**, API2 **157.10 MiB**, worker **128.90 MiB**. Это **не истинный peak RSS**. + +## Persistence, failover и финальная проверка (15.11) + +```text +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 6, "failed_attempts": 1, "first_recovery_seconds": 6.172} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +api1: healthy; api2: healthy +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +``` + +- Persistence: `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd` — **PASS** сохранности ранее подтверждённых данных, вложения и существующей сессии после возврата сервиса. +- Failover: `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` — **PASS** сохранности и работы оставшейся реплики. **Не zero-downtime:** один неудачный запрос и около **6.172 s** до первого успеха. +- После сценариев `docker compose up -d --no-deps --wait api1 api2`, оба API healthy, повторный `smoke.py` — **PASS**, остальные runtime-контейнеры подняты. + +**Оговорка про hook:** в тесте stdout/stderr скриптов подавлен. Исходники `.cmd` показывают их область действия: `restart.cmd` останавливает/поднимает названные Compose-сервисы, `stop-one.cmd` останавливает одну целевую API. Поэтому статус PASS означает успешную проверку клиентского поведения при выполненном hook; для демонстрации масштаба самого hook приложить его исходник и при необходимости независимый лог Docker. + +## Что не следует утверждать + +- Нет доказательства отказоустойчивости при потере физического хоста: учебный Compose работает на одном Docker-хосте. +- Система использует **at-least-once**, не exactly-once delivery; идемпотентность реализована на уровне устойчивого состояния/эффектов. +- Нагрузка 10 файлов не измеряет устойчивую производительность при тысячах заданий; sampled memory — не точный максимум. +- Оценку назначает преподаватель, а не тесты; это набор технических evidence для защиты уровня 4. diff --git a/evidence/worker-resources.csv b/evidence/worker-resources.csv new file mode 100644 index 00000000..ecf8a9d5 --- /dev/null +++ b/evidence/worker-resources.csv @@ -0,0 +1,46 @@ +"time","container","memory_mib","cpu_percent" +"2026-11-14T20:57:07.6236562Z","mtusi2026lab5-api1-1","150,6","39,55" +"2026-11-14T20:57:07.7200386Z","mtusi2026lab5-api2-1","142,7","1,61" +"2026-11-14T20:57:07.7211119Z","mtusi2026lab5-worker-1","67,54","0,01" +"2026-11-14T20:57:10.5614383Z","mtusi2026lab5-api1-1","150,4","2,6" +"2026-11-14T20:57:10.5624387Z","mtusi2026lab5-api2-1","155,7","35,95" +"2026-11-14T20:57:10.5634403Z","mtusi2026lab5-worker-1","67,54","0" +"2026-11-14T20:57:13.6800147Z","mtusi2026lab5-api1-1","163,1","34,99" +"2026-11-14T20:57:13.6805548Z","mtusi2026lab5-api2-1","146,3","9,56" +"2026-11-14T20:57:13.6805548Z","mtusi2026lab5-worker-1","67,54","0" +"2026-11-14T20:57:16.7854336Z","mtusi2026lab5-api1-1","150,5","29,25" +"2026-11-14T20:57:16.7860143Z","mtusi2026lab5-api2-1","143,3","3,53" +"2026-11-14T20:57:16.7860143Z","mtusi2026lab5-worker-1","67,54","0" +"2026-11-14T20:57:19.8297024Z","mtusi2026lab5-api1-1","150,4","1,84" +"2026-11-14T20:57:19.8302414Z","mtusi2026lab5-api2-1","143,3","6,74" +"2026-11-14T20:57:19.8302414Z","mtusi2026lab5-worker-1","67,54","0" +"2026-11-14T20:57:22.9337889Z","mtusi2026lab5-api1-1","150,5","3,43" +"2026-11-14T20:57:22.9343583Z","mtusi2026lab5-api2-1","143,5","28,06" +"2026-11-14T20:57:22.9343583Z","mtusi2026lab5-worker-1","67,54","0" +"2026-11-14T20:57:26.0226361Z","mtusi2026lab5-api1-1","150,5","1,22" +"2026-11-14T20:57:26.0226361Z","mtusi2026lab5-api2-1","143,5","17,2" +"2026-11-14T20:57:26.0226361Z","mtusi2026lab5-worker-1","67,54","0" +"2026-11-14T20:57:29.1343125Z","mtusi2026lab5-api1-1","151,8","22,45" +"2026-11-14T20:57:29.1343125Z","mtusi2026lab5-api2-1","144,4","1,13" +"2026-11-14T20:57:29.1348670Z","mtusi2026lab5-worker-1","84,8","54,15" +"2026-11-14T20:57:32.2450510Z","mtusi2026lab5-api1-1","151,7","32,87" +"2026-11-14T20:57:32.2455994Z","mtusi2026lab5-api2-1","144,4","1,24" +"2026-11-14T20:57:32.2463896Z","mtusi2026lab5-worker-1","91,36","93,69" +"2026-11-14T20:57:35.3731553Z","mtusi2026lab5-api1-1","151,6","2,21" +"2026-11-14T20:57:35.3744212Z","mtusi2026lab5-api2-1","144,4","1,75" +"2026-11-14T20:57:35.3750375Z","mtusi2026lab5-worker-1","128,9","95,95" +"2026-11-14T20:57:38.4925792Z","mtusi2026lab5-api1-1","151,5","1,51" +"2026-11-14T20:57:38.4925792Z","mtusi2026lab5-api2-1","144,3","47,55" +"2026-11-14T20:57:38.4925792Z","mtusi2026lab5-worker-1","118,6","98,38" +"2026-11-14T20:57:41.5595716Z","mtusi2026lab5-api1-1","151,7","73,27" +"2026-11-14T20:57:41.5605761Z","mtusi2026lab5-api2-1","144,3","47,12" +"2026-11-14T20:57:41.5615747Z","mtusi2026lab5-worker-1","85,18","91,04" +"2026-11-14T20:57:44.6992845Z","mtusi2026lab5-api1-1","151,5","25,29" +"2026-11-14T20:57:44.6998215Z","mtusi2026lab5-api2-1","144,3","1,55" +"2026-11-14T20:57:44.6998215Z","mtusi2026lab5-worker-1","81,51","92,75" +"2026-11-14T20:57:47.8226967Z","mtusi2026lab5-api1-1","151,5","32,64" +"2026-11-14T20:57:47.8226967Z","mtusi2026lab5-api2-1","144,6","1,55" +"2026-11-14T20:57:47.8233538Z","mtusi2026lab5-worker-1","68,29","0" +"2026-11-14T20:57:50.8657247Z","mtusi2026lab5-api1-1","151,5","1,54" +"2026-11-14T20:57:50.8662684Z","mtusi2026lab5-api2-1","157,1","27,07" +"2026-11-14T20:57:50.8662684Z","mtusi2026lab5-worker-1","68,29","0,01" diff --git a/infra/filer.toml b/infra/filer.toml new file mode 100644 index 00000000..f3f385a4 --- /dev/null +++ b/infra/filer.toml @@ -0,0 +1,4 @@ +# Persistent SeaweedFS filer metadata for the single-node lab stack. +[leveldb2] +enabled = true +dir = "/data/filerldb2" diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..62b0dba5 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,32 @@ +global + log stdout format raw local0 + maxconn 1000 + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 2s + timeout client 60s + timeout server 60s + timeout http-request 5s + +resolvers docker + nameserver localdns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + +frontend messenger + bind *:8080 + default_backend api_nodes + +backend api_nodes + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 slowstart 2s + server api1 api1:8080 check resolvers docker init-addr none + server api2 api2:8080 check resolvers docker init-addr none diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..3628b896 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +fastapi==0.115.12 +uvicorn==0.34.2 +pydantic==2.11.4 +psycopg[binary]==3.2.6 +psycopg-pool==3.2.6 +redis==5.2.1 +argon2-cffi==23.1.0 + +boto3==1.40.0 +Pillow==11.3.0 +pika==1.3.2 diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..04aba9d5 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,11 @@ +@echo off +setlocal +cd /d "%~dp0.." +docker compose --env-file .env stop --timeout 5 lb api1 api2 worker dispatcher broker s3 redis postgres +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --wait postgres redis broker s3 s3-bootstrap +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --no-deps api1 api2 lb dispatcher worker +if errorlevel 1 exit /b 1 +python scripts\wait_ready.py +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..e3f21f0e --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,10 @@ +@echo off +setlocal +cd /d "%~dp0.." +if "%TARGET_INSTANCE_ID%"=="api1" goto stop +if "%TARGET_INSTANCE_ID%"=="api2" goto stop +echo Unexpected TARGET_INSTANCE_ID 1>&2 +exit /b 2 +:stop +docker compose --env-file .env stop --timeout 1 %TARGET_INSTANCE_ID% +exit /b %ERRORLEVEL% diff --git a/scripts/wait_ready.py b/scripts/wait_ready.py new file mode 100644 index 00000000..86d0318b --- /dev/null +++ b/scripts/wait_ready.py @@ -0,0 +1,14 @@ +"""Operator-facing readiness wait after container replacement.""" +import time +import urllib.request +for _ in range(120): + try: + with urllib.request.urlopen("http://localhost:8080/health/ready", timeout=2) as response: + if response.status == 200: + print("[restart] all services ready") + break + except Exception: + pass + time.sleep(.5) +else: + raise SystemExit("[restart] health timeout") diff --git a/tests/attachments_concurrent.py b/tests/attachments_concurrent.py new file mode 100644 index 00000000..91f6d6d8 --- /dev/null +++ b/tests/attachments_concurrent.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Concurrent Lab5 uploads (HTTP 202) and eventual processing completion. + +Admission metrics measure upload/enqueue requests, not worker throughput. +Completion is checked separately so an accepted-but-stuck job cannot pass. +""" +import argparse +import concurrent.futures +import json +import statistics +import time +from pathlib import Path + +from client import Client, check, run + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--output', default='evidence/attachments-concurrent.json') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--wait-seconds', type=int, default=180) + args = parser.parse_args() + check(args.concurrency > 0, 'concurrency must be positive') + check(args.wait_seconds > 0, 'wait-seconds must be positive') + + client = Client(args.base_url) + alice, bob, eve, chat = client.bootstrap() + source = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + path = '/api/v1/chats/' + chat['id'] + '/attachments?filename=parallel.png' + + def upload(_): + cli = Client(args.base_url) + response = cli.call('POST', path, 202, actor=alice, raw=source, + headers={'Content-Type': 'image/png'}, schema='Attachment') + att = response['json'] + check(att['status'] == 'queued', '202 must return a queued attachment') + check(att['size_bytes'] == len(source), 'size mismatch') + check(att['chat_id'] == chat['id'] and att['owner_id'] == alice['id'], + 'incorrect attachment ownership') + return att['id'], response['elapsed'] + + started = time.monotonic() + with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool: + results = list(pool.map(upload, range(args.concurrency))) + accepted_seconds = time.monotonic() - started + ids = [att_id for att_id, _ in results] + timings = [duration for _, duration in results] + check(len(set(ids)) == args.concurrency, 'duplicate attachment IDs') + + # A 202 response alone only proves acceptance, not successful processing. + pending = set(ids) + deadline = time.monotonic() + args.wait_seconds + while pending and time.monotonic() < deadline: + for att_id in list(pending): + att = client.call('GET', '/api/v1/attachments/' + att_id, + actor=alice, schema='Attachment')['json'] + check(att['status'] != 'failed', 'attachment reached failed: ' + att_id) + if att['status'] == 'ready': + variants = [v['name'] for v in att['variants']] + check(len(variants) == 3 and set(variants) == {'thumbnail', 'crop', 'optimized'}, + 'missing/duplicate image variants: ' + att_id) + pending.remove(att_id) + if pending: + time.sleep(0.5) + check(not pending, 'attachments not ready before deadline: ' + str(len(pending))) + complete_seconds = time.monotonic() - started + + data = { + 'concurrency': args.concurrency, + 'accepted_uploads': len(results), + 'ready_attachments': args.concurrency, + 'failed_requests': 0, + 'acceptance_elapsed_seconds': round(accepted_seconds, 3), + 'acceptance_rps': round(len(results) / accepted_seconds, 3), + 'upload_latency_p50_ms': round(statistics.median(timings) * 1000, 1), + 'upload_latency_max_ms': round(max(timings) * 1000, 1), + 'all_ready_elapsed_seconds': round(complete_seconds, 3), + 'bytes_per_file': len(source), + 'note': 'HTTP 202 = queued; readiness checked for all uploads. Capture API peak RSS separately via docker stats.', + } + output = Path(args.output) + output.parent.mkdir(exist_ok=True, parents=True) + output.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding='utf-8') + print('PASS: %d concurrent HTTP 202 uploads -> ready, 3 variants each' % args.concurrency) + print(json.dumps(data, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + run(main) diff --git a/tests/attachments_negative.py b/tests/attachments_negative.py new file mode 100644 index 00000000..594f103b --- /dev/null +++ b/tests/attachments_negative.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Lab5 negative API tests with isolated synthetic users (requires live stack).""" +from urllib.parse import quote +import uuid +from client import Client, arguments, run, check +from pathlib import Path + + +def main(): + args = arguments('Lab5 negative file/ACL/security tests') + c = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = c.bootstrap() + base = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + data = (Path(__file__).parent / 'fixtures/sample.png').read_bytes() + c.call('POST', base+'big.png', 413, actor=alice, raw=data+b'\x00'*(10*1024*1024+1-len(data)), headers={'Content-Type':'image/png'}) + c.call('POST', base+'bad.png', 415, actor=alice, raw=b'not-an-image', headers={'Content-Type':'image/png'}) + c.call('POST', base+'fake.jpg', 415, actor=alice, raw=data, headers={'Content-Type':'image/jpeg'}) + c.call('POST', base+quote('../escape.png',safe=''), 400, actor=alice, raw=data, headers={'Content-Type':'image/png'}) + c.call('POST', base+'x.png', 404, actor=eve, raw=data, headers={'Content-Type':'image/png'}) + c.call('POST', base+'x.png', 401, raw=data, headers={'Content-Type':'image/png'}) + uploaded = c.attachment(alice,chat)[0] + ready = c.wait_ready(alice, uploaded) + unknown = str(uuid.uuid4()) + c.call('GET','/api/v1/attachments/'+unknown,404,actor=alice) + c.call('GET','/api/v1/attachments/'+uploaded['id'],404,actor=eve) + c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content',404,actor=eve) + # A ready Lab5 attachment has a downloadable thumbnail for chat members. + # A non-member still gets 404, and an unknown variant must not resolve. + c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',404,actor=eve) + c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=nonexistent',404,actor=alice) + check(any(v['name'] == 'thumbnail' for v in ready['variants']), 'missing thumbnail variant') + thumb = c.call('GET','/api/v1/attachments/'+uploaded['id']+'/content?variant=thumbnail',200,actor=alice) + check(thumb['headers'].get_content_type() == 'image/jpeg', 'wrong thumbnail MIME') + other_chat = c.call('POST','/api/v1/chats',201,actor=alice, + body={'title':'Другой чат','member_ids':[alice['id'],bob['id']]},schema='Chat')['json'] + path = '/api/v1/chats/'+other_chat['id']+'/messages' + c.call('POST',path,404,actor=alice, + body={'text':'cross-chat','attachment_ids':[uploaded['id']]}, + headers={'Idempotency-Key':uuid.uuid4().hex}) + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',404,actor=alice, + body={'text':'unknown','attachment_ids':[unknown]}, + headers={'Idempotency-Key':uuid.uuid4().hex}) + ok = c.message(bob,chat,'',attachments=[uploaded['id']])['json'] + check(ok['attachment_ids']==[uploaded['id']], 'valid attachment not linked') + print('PASS: lab5 negative image, MIME, size, filename, ACL, variants and cross-chat checks') + +if __name__=='__main__': + run(main) diff --git a/tests/idempotency_parallel.py b/tests/idempotency_parallel.py new file mode 100644 index 00000000..3edffd75 --- /dev/null +++ b/tests/idempotency_parallel.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Lab3 own test: 30 requests using identical actor/chat/key/body across both replicas.""" +import concurrent.futures +import sys +import uuid +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments("Check atomic idempotency across API replicas") + c = Client(args.base_url, args.ca_file) + alice, bob, _, chat = c.bootstrap() + key = uuid.uuid4().hex + body = {"text": "Concurrent replay " + key} + path = f"/api/v1/chats/{chat['id']}/messages" + + def worker(_): + # Independent request client; one shared bearer token and idempotency key. + client = Client(args.base_url, args.ca_file) + return client.call("POST", path, expected=201, actor=alice, body=body, + headers={"Idempotency-Key": key}, schema="Message") + + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as executor: + results = list(executor.map(worker, range(30))) + ids = {result["json"]["id"] for result in results} + instances = {result["headers"]["X-Instance-Id"] for result in results} + check(len(ids) == 1, "parallel replay created different messages") + check(len(instances) >= 2, "LB did not serve requests via both API replicas") + check(all(r["json"] == results[0]["json"] for r in results), "replay returned altered result") + c.call("POST", path, 409, actor=alice, body={"text": "different body"}, headers={"Idempotency-Key": key}) + c.call("POST", path, 201, actor=bob, body=body, headers={"Idempotency-Key": key}, schema="Message") + history = c.call("GET", path + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"] + check(sum(x["id"] in ids for x in history) == 1, "message duplicated in history") + print("PASS: parallel idempotency across", len(instances), "replicas; one message, replay 201, conflicting body 409") + + +if __name__ == "__main__": + run(main) diff --git a/tests/own_unit.py b/tests/own_unit.py new file mode 100644 index 00000000..bad78c17 --- /dev/null +++ b/tests/own_unit.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""Fast non-Docker checks for input contracts and cursor scoping.""" +import sys +from pathlib import Path +from datetime import datetime, timezone +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app.core import make_cursor, read_cursor, parse_limit, APIError +from app.models import CreateUser, CreateChat, CreateMessage +from pydantic import ValidationError + + +def reject(fn): + try: + fn() + except (APIError, ValidationError, ValueError): + return + raise AssertionError("Expected invalid input") + + +def main(): + token = str(uuid.uuid4()) + dt = datetime.now(timezone.utc) + cursor = make_cursor("messages", token, dt, str(uuid.uuid4())) + assert read_cursor(cursor, "messages", token)[0] == dt + reject(lambda: read_cursor(cursor, "messages", str(uuid.uuid4()))) + reject(lambda: read_cursor(cursor, "chats", token)) + assert parse_limit(None) == 50 + reject(lambda: parse_limit("0")) + reject(lambda: parse_limit("101")) + reject(lambda: CreateUser(username="user_name", display_name=" \t", password="Long-enough-secret")) + reject(lambda: CreateUser(username="user_name", display_name="Test", password=" " * 3)) + assert CreateMessage(text="A\u0000B").text == "A\u0000B" + reject(lambda: CreateMessage(text=" \t")) + reject(lambda: CreateChat(title="title", member_ids=[token] * 2)) + reject(lambda: CreateChat(title="title", member_ids=[token] * 101)) + print("PASS: models, cursor scoping, limits, Unicode U+0000") + + +if __name__ == "__main__": + main() diff --git a/tests/pipeline_extended.py b/tests/pipeline_extended.py new file mode 100644 index 00000000..d1206448 --- /dev/null +++ b/tests/pipeline_extended.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Additional *live* Lab5 fault tests; operator-controlled, synthetic records only. + +broker-persistence: require a publisher-confirmed RabbitMQ message to survive +broker restart without a running dispatcher or worker. This differs from the +outbox-restoration tests where dispatcher publishes *after* broker recovery. + +failed-replay: safely inject a deterministic checksum mismatch into one fresh +synthetic attachment's DB metadata, verify permanent failure, restore the +original hash, replay via the operator-only CLI, and verify success plus audit. + +staging-cleanup: verify the staged S3 object is absent after ready (with a +matching cleared staging_key in the DB). + +No volumes are deleted. The test may stop/restart worker/dispatcher/broker; +run only against a disposable LAB5 environment and never concurrently. +""" +import argparse +import hashlib +import os +import re +import subprocess +import time +import uuid + +from client import Client, Failure, check, run +from pipeline_resilience import verify_ready, wait_api + +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args, expected=(0,), timeout=240): + proc = subprocess.run( + ['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + capture_output=True, text=True, encoding='utf-8', errors='replace', timeout=timeout, + ) + if proc.returncode not in expected: + # Don't print env vars, tokens or command stdout/stderr by default. + raise Failure('docker compose %s failed (exit %d); inspect ps/logs' % + (' '.join(args[:2]), proc.returncode)) + return proc.stdout.strip() + + +def pg(sql): + """Read-only or deliberately scoped fault SQL; UUIDs never user-supplied here.""" + user = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_USER').strip() + dbname = compose('exec', '-T', 'postgres', 'printenv', 'POSTGRES_DB').strip() + return compose('exec', '-T', 'postgres', 'psql', '-X', '-q', '-A', '-t', + '-v', 'ON_ERROR_STOP=1', '-U', user, '-d', dbname, '-c', sql) + + +def scalar(sql): + return pg(sql).splitlines()[-1].strip() + + +def wait_for(label, action, predicate, seconds=30, interval=0.5): + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + last = action() + if predicate(last): + return last + except Failure: + pass + time.sleep(interval) + raise Failure('timed out waiting for ' + label + ', last=' + str(last)[:100]) + + +def job_query(att_id, fields): + uuid.UUID(att_id) + return scalar('SELECT %s FROM processing_jobs WHERE attachment_id = \'%s\'' % + (fields, att_id)) + + +def queue_ready(): + out = compose('exec', '-T', 'broker', 'rabbitmqctl', '-q', 'list_queues', + 'name', 'messages_ready', 'messages_unacknowledged') + for line in out.splitlines(): + parts = re.split(r'\s+', line.strip()) + if parts[0] == 'attachment-processing.v1' and len(parts) >= 3: + return int(parts[1]), int(parts[2]) + raise Failure('RabbitMQ durable queue missing') + + +def make_upload(c): + alice, bob, _eve, chat = c.bootstrap() + att, original = c.attachment(alice, chat) + check(att['status'] == 'queued', 'upload must return 202 queued') + uuid.UUID(att['id']) + return alice, bob, att, original + + +def wait_staging_cleared(att_id, seconds=20): + return wait_for('staging_key cleared after ready', + lambda: scalar("SELECT COALESCE(staging_key,'') FROM attachments WHERE id='%s'" % att_id), + lambda value: value == '', seconds=seconds) + + +def verify_staging_absent(att_id): + # This executes *inside* the worker container with its normal S3 credentials. + # Never expose S3 secrets to the host or write them to evidence files. + code = '''import sys +from botocore.exceptions import ClientError +from app.attachments import s3_client, BUCKET +key='staging/'+sys.argv[1]+'/original' +try: + s3_client().head_object(Bucket=BUCKET,Key=key) +except ClientError as exc: + if exc.response.get('ResponseMetadata',{}).get('HTTPStatusCode')==404: + print('ABSENT') + else: + raise +else: + print('STILL_PRESENT')''' + result = compose('exec', '-T', 'worker', 'python', '-c', code, att_id) + check(result == 'ABSENT', 'staging S3 object still present: ' + result[:70]) + + +def broker_persistence(c): + compose('stop', 'worker') + compose('up', '-d', 'dispatcher') + baseline = queue_ready()[0] + alice, _bob, att, original = make_upload(c) + print('202 queued:', att['id'], flush=True) + sql = "SELECT COALESCE(bool_and(published_at IS NOT NULL),false) FROM job_outbox WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % att['id'] + wait_for('outbox publisher confirm', lambda: scalar(sql), lambda v: v == 't', seconds=30) + wait_for('message in RabbitMQ ready queue', lambda: queue_ready()[0], + lambda n: n >= baseline + 1, seconds=15) + compose('stop', 'dispatcher') + before = queue_ready()[0] + check(before >= baseline + 1, 'published message disappeared before restart') + print('Publisher confirmed; dispatcher and worker stopped; broker queue ready=%d' % before, flush=True) + compose('restart', 'broker') + compose('up', '-d', '--wait', 'broker') + after = wait_for('RabbitMQ queue restoration', lambda: queue_ready()[0], + lambda n: n >= before, seconds=40, interval=1) + check(scalar(sql) == 't', 'outbox publish confirmation lost') + print('Broker restarted; ready messages persisted=%d' % after, flush=True) + # CRITICAL: do NOT restart dispatcher yet; only the restored RabbitMQ message + # can deliver this job to the worker. + compose('up', '-d', 'worker') + verify_ready(c, alice, att, original, seconds=180) + check(job_query(att['id'], 'status') == 'ready', 'job not committed ready') + print('PASS: broker-persistence, confirmed RabbitMQ message survives broker restart without outbox republish') + + +def failed_replay(c): + compose('stop', 'worker', 'dispatcher') + alice, _bob, att, original = make_upload(c) + ident = att['id'] + original_sha = hashlib.sha256(original).hexdigest() + check(original_sha == att['sha256'], 'fixture hash mismatch') + check(scalar("SELECT sha256 FROM attachments WHERE id='%s'" % ident) == original_sha, + 'DB hash unexpected before controlled fault') + fault_sha = '0' * 64 if original_sha != '0' * 64 else 'f' * 64 + changed = False + try: + updated = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='queued' RETURNING id" % + (fault_sha, ident)) + check(ident in updated, 'failed to inject synthetic checksum mismatch') + changed = True + compose('up', '-d', 'dispatcher', 'worker') + + def get_state(): + return c.call('GET', '/api/v1/attachments/' + ident, actor=alice, schema='Attachment')['json'] + failed = wait_for('permanent failed status', get_state, + lambda a: a['status'] == 'failed', seconds=90) + check(failed['error_code'] == 'invalid_image', + 'expected permanent invalid_image, got ' + str(failed['error_code'])) + check(job_query(ident, 'status') == 'failed', 'processing job must fail') + check(job_query(ident, 'attempt') == '1', 'permanent error must not retry') + check(scalar("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % ident) == '0', + 'failed job created variants') + print('Permanent invalid_image confirmed; no retry and no variant rows', flush=True) + # Repair only the generated synthetic row before explicit operator replay. + restored = pg("UPDATE attachments SET sha256='%s' WHERE id='%s' AND status='failed' RETURNING id" % + (original_sha, ident)) + check(ident in restored, 'synthetic metadata could not be restored') + changed = False + compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', 'replay', '--id', ident) + verify_ready(c, alice, att, original, seconds=180) + audit = scalar("SELECT count(*) FROM job_replay_audit WHERE job_id=(SELECT id FROM processing_jobs WHERE attachment_id='%s')" % ident) + check(audit == '1', 'operator replay audit count incorrect: ' + audit) + check(job_query(ident, 'status') == 'ready', 'replayed job not ready') + attempt = job_query(ident, 'attempt') + check(attempt == '1', 'replay did not reset attempt: ' + attempt) + invalid = compose('exec', '-T', 'dispatcher', 'python', '-m', 'app.jobs', + 'replay', '--id', ident, expected=(1,)) + print('PASS: failed-replay; permanent failure → metadata repaired → replay → ready; audit=1; replay of ready rejected') + finally: + if changed: + # Never leave the synthetic attachment with altered metadata. + pg("UPDATE attachments SET sha256='%s' WHERE id='%s'" % (original_sha, ident)) + + +def staging_cleanup(c): + compose('up', '-d', 'dispatcher', 'worker') + alice, _bob, att, original = make_upload(c) + verify_ready(c, alice, att, original, seconds=180) + wait_staging_cleared(att['id'], seconds=30) + verify_staging_absent(att['id']) + print('PASS: staging-cleanup; ready, DB staging_key=NULL, original staging object absent in S3') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--scenario', required=True, + choices=['broker-persistence', 'failed-replay', 'staging-cleanup']) + args = parser.parse_args() + if os.getenv('LAB5_ALLOW_FAILURE_TEST') != 'yes': + raise Failure('Set LAB5_ALLOW_FAILURE_TEST=yes; run only on your isolated Lab5 stack') + c = Client('http://localhost:8080') + try: + wait_api() + if args.scenario == 'broker-persistence': + broker_persistence(c) + elif args.scenario == 'failed-replay': + failed_replay(c) + else: + staging_cleanup(c) + finally: + # Restore normal services even when an assertion fails. NO volume deletion. + try: + compose('up', '-d', '--wait', 'broker') + compose('up', '-d', 'dispatcher', 'worker') + except Failure: + print('WARNING: inspect docker compose ps and restore dispatcher/worker manually') + + +if __name__ == '__main__': + run(main) diff --git a/tests/pipeline_resilience.py b/tests/pipeline_resilience.py new file mode 100644 index 00000000..68be85d0 --- /dev/null +++ b/tests/pipeline_resilience.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +"""Explicit operator evidence for Lab5 queue durability and failure windows. + +Runs only on an isolated lab5 Compose stack. Never resets named volumes. The test +preserves user/session secrets strictly in process memory and prints no tokens. +""" +import argparse +import hashlib +import os +import subprocess +import sys +import time +from pathlib import Path + +from client import Client, Failure, check, run + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args, expected=(0,)): + p = subprocess.run(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, timeout=240) + if p.returncode not in expected: + raise Failure('docker compose command failed: %s (exit %s); inspect docker compose ps/logs' + % (' '.join(args[:2]), p.returncode)) + return p + + +def wait_api(timeout=60): + import urllib.request + deadline=time.monotonic()+timeout + while time.monotonic()ready; original/3 variants verified; no duplicates') + finally: + # Restore the normal topology even after an assertion failure. + compose('up','-d','dispatcher','worker') + + +if __name__=='__main__': + run(main)