diff --git a/.env.example b/.env.example index 119384b5..a6a9cbc4 100644 --- a/.env.example +++ b/.env.example @@ -1,14 +1,15 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. # These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 -APP_PORT=8080 +APP_PORT=8084 APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Redis stores sessions; a separate Redis service stores the persistent job stream. CACHE_URL_FILE=/run/secrets/cache_url +BROKER_URL_FILE=/run/secrets/broker_url +BROKER_STREAM=messenger:attachment-jobs:v1 API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 S3_ENDPOINT=http://s3:9000 @@ -18,8 +19,8 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url -WORKER_CONCURRENCY=2 +WORKER_CONCURRENCY=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=180 STAGING_RETENTION_SECONDS=86400 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..e370bd00 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,32 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +**Автор / группа:** Бадрадин Али / БВТ2302 +**Целевая оценка:** 4 +**Трек 5:** не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +### Реализация -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Сохранены API и хранилища Lab 4. Добавлены отдельные Redis Streams broker, dispatcher и worker; durable staging в RustFS; PostgreSQL jobs/outbox/replay audit; состояния queued/processing/ready/failed; thumbnail JPEG, crop JPEG и optimized WebP; ограниченные retry, lease и безопасный replay. + +### Запуск и результаты + +Инструкция: [REPORT.md](../REPORT.md). Внешний адрес локального стенда: `http://localhost:8084`. + +Структурная проверка, OpenAPI, Python compileall, Compose config и локальный image pipeline завершились PASS. Публичный API smoke и все пять интеграционных сценариев уровня 4 также завершились PASS; фактические выводы находятся в `evidence/`. + +### Ограничения + +Redis Streams и RustFS в локальном Compose являются одиночными узлами. Производительность не измерялась; трек 5 не заявлен. См. ограничения в [REPORT.md](../REPORT.md). + +### Источники и использование ИИ + +Документация FastAPI, SQLAlchemy, Redis Streams, Pillow, boto3 и RustFS. Codex использовался для реализации, ревью и подготовки проверок. + +### Проверка перед отправкой + +- [x] Обязательный минимум Lab 5 и переходы Lab 4 сохранены; реальные сервисы проверены. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Контрактная проверка (`python scripts/check_contract.py`, эквивалент `make check`) и публичный smoke (`python tests/smoke.py`) выполнены; вывод сохранён в `evidence/check.log` и `evidence/test.log`. +- [x] Все проверки уровня 4 выполнены; JSON-доказательства сохранены в `evidence/`. +- [x] Секреты и приватные ключи вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит воспроизводимую инструкцию. diff --git a/.gitignore b/.gitignore index b5a71896..e65be056 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ bin/ obj/ coverage/ *.log +!evidence/check.log +!evidence/test.log .DS_Store .idea/ .vscode/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..03fbf5d1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.12.6-slim-bookworm + +ARG BUILD_REVISION=local +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONUTF8=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_BUILD_REVISION=${BUILD_REVISION} + +WORKDIR /app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir --requirement requirements.txt + +COPY src/ ./src/ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --uid 10001 appuser +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=3s --timeout=2s --start-period=10s --retries=3 \ + CMD python -c "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)" + +CMD ["uvicorn", "src.app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1", "--timeout-graceful-shutdown", "5"] diff --git a/Makefile b/Makefile index 5d86d171..4ae4e76c 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://localhost:8084 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-grade4 test-grade4-pipeline test-outbox-recovery test-worker-redelivery test-replay-failed-job test-broker-persistence test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI; test: public smoke; test-grade4: image pipeline, broker persistence, outbox recovery, redelivery and safe replay" check: $(PYTHON) scripts/check_contract.py @@ -18,6 +18,24 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-grade4: test test-grade4-pipeline test-broker-persistence test-outbox-recovery test-worker-redelivery test-replay-failed-job + +test-grade4-pipeline: + $(PYTHON) student-tests/pipeline_unit.py + $(PYTHON) student-tests/grade4_pipeline.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-outbox-recovery: + $(PYTHON) student-tests/outbox_recovery.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-worker-redelivery: + $(PYTHON) student-tests/worker_redelivery.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-replay-failed-job: + $(PYTHON) student-tests/replay_failed_job.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-broker-persistence: + $(PYTHON) student-tests/broker_persistence.py --base-url "$(BASE_URL)" $(TLS_ARGS) + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/README.md b/README.md index 009d6997..06f0d164 100644 --- a/README.md +++ b/README.md @@ -37,6 +37,8 @@ - Retry использует backoff с jitter, различает постоянные/временные ошибки; есть DLQ/реестр failed и безопасный явный replay. «Повторить» не означает безусловно исполнить дубль уже успешной job. - Проверена остановка worker после записи объекта, но до ack; повтор завершает тот же результат. Есть timeout/lease для зависшего processing, ограниченная конкуренция/prefetch и очистка staging после безопасной фиксации. +В этой реализации pipeline v1 создаёт thumbnail JPEG с `quality=85` и вписыванием без увеличения, crop JPEG 128×128 с `quality=85` и оптимизированный WebP с `quality=82`, `method=4`, максимальной стороной 1280 px. EXIF не переносится. API ограничивает upload до 10 MiB и проверяет заголовок/размер изображения до декодирования; worker допускает до 40 млн пикселей, работает с дедлайном 60 секунд, одним заданием за раз и лимитом памяти контейнера 768 MiB. Lease равен 180 секундам, число попыток — 5; временные ошибки получают экспоненциальную задержку с jitter. + ## Оценка «5»: исследование и доказанный результат Весь уровень 4 **и один законченный трек на выбор**. Приведите гипотезу, повторяемую методику, результаты и ограничения. @@ -52,7 +54,7 @@ make check # После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. +# Для Lab 5 внешний порт по умолчанию — http://localhost:8084, чтобы не конфликтовать с предыдущими лабораторными. make test ``` @@ -71,12 +73,20 @@ docker compose --env-file .env up -d --build ```sh python3 -m venv .venv -.venv/bin/python -m pip install -r requirements-dev.txt +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt make validate PYTHON=.venv/bin/python ``` Smoke не выставляет оценку автоматически. Проверку сохранности/отказов запускайте явно по [tests/README.md](tests/README.md), остальные доказательства приведите в отчёте. +Для объявленного уровня 4 после запуска Compose используйте дополнительные сценарии: + +```sh +make test-grade4 BASE_URL=http://localhost:8084 +``` + +Они проверяют оригинал и три варианта изображения, лимиты входа, сохранность записи Stream после рестарта broker с volume, crash dispatcher до XADD после подтверждённого API COMMIT, восстановление через outbox после перезапуска API, повторную доставку после фиксации результата до XACK и явный безопасный replay задания из failed. Сводки записываются в `evidence/`; эти сценарии создают постоянные синтетические данные, как и публичный smoke. + ## Сценарий защиты 1. Запустить make test: API возвращает 202, GET доходит до ready, оригинал и настоящий JPEG thumbnail проверяются. @@ -97,6 +107,13 @@ Worker, брокер в общем стенде, схема jobs, migrations/out - Почему повторная доставка нормальна и где находится ключ идемпотентности job? - Как отличить зависший worker от долго работающего и не запустить два неконтролируемых дубля? +Краткие ответы применительно к этому стенду: + +1. **Publisher confirm и consumer ack — это разные подтверждения.** Ответ Redis на `XADD` означает, что Stream принял событие; `XACK` вызывается worker только после сохранения оригинала и вариантов, а также фиксации `ready/succeeded` в PostgreSQL. Redis работает с AOF `appendfsync everysec`; при потере последней записи outbox повторно публикует незавершённую работу. +2. **К моменту ответа 202** исходный файл уже находится в S3 staging, а PostgreSQL одной транзакцией сохранил метаданные вложения, job и outbox. Само событие может ещё не быть в Redis Stream — dispatcher прочитает outbox позже. +3. **Повторная доставка ожидаема** при падении между записью результата и ack или между Redis `XADD` и отметкой outbox. Неизменяемый `job_id` хранится в PostgreSQL; попытка увеличивается при новом retry. Эффекты адресуются детерминированными ключами `(attachment_id, pipeline_version, variant)`, поэтому повтор не создаёт второй набор объектов. +4. **Зависание ограничивается дедлайном и lease.** Worker выполняет по одному заданию; дедлайн 60 секунд меньше lease 180 секунд. Dispatcher возвращает истёкший lease в очередь. Захват и изменение состояния блокируются в PostgreSQL, а `attempt` отсекает устаревшее завершение. Повтор после `ready` только подтверждает старое событие и не меняет результат. + ## Первичные материалы - [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability) diff --git a/REPORT.md b/REPORT.md index e98f75e9..82e0b9ba 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,115 @@ -# Отчёт — лабораторная 5 +# Отчёт по лабораторной работе 5 — очереди и обработка изображений -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- **Автор / группа:** Бадрадин Али / БВТ2302 +- **Целевая оценка:** 4 +- **Трек 5:** не заявлен +- **Статус:** [PR #75](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/75) открыт на проверку; commit реализации — `f9246b8` +- **Источники:** README и контракты лабораторных 4–5; документация FastAPI, SQLAlchemy, Redis Streams, Pillow, boto3 и RustFS. Codex использовался для реализации, ревью и подготовки проверок. -## Запуск из чистого clone +## Запуск из чистого клона — Windows PowerShell -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, Docker Desktop с запущенным Linux Engine и Python 3.12+. Порт стенда по умолчанию — `8084`, чтобы не пересекаться с предыдущими лабораторными. -## Архитектура и решения +```powershell +git clone https://gitea.privcloud.tech/Badradine/mtusi2026lab5.git +Set-Location .\mtusi2026lab5 +python -m venv .venv +.\.venv\Scripts\python.exe -m pip install -r requirements.txt -r requirements-dev.txt +Copy-Item .env.example .env +.\scripts\create-secrets.ps1 +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 +docker compose --env-file .env ps +Invoke-WebRequest -UseBasicParsing http://localhost:8084/health/ready +``` -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Секреты скрипт создаёт только в локальном `secrets/`; их значения не выводятся и Git их игнорирует. PostgreSQL, Redis для сессий, отдельный Redis Streams broker и RustFS используют постоянные volumes. Сервисы `migrate` и `s3-bootstrap` завершаются с кодом 0 после успешной подготовки. -## Доказательства критериев +Проверки из PowerShell: -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +```powershell +\.venv\Scripts\python.exe scripts/check_contract.py +\.venv\Scripts\python.exe scripts/validate_openapi.py +\.venv\Scripts\python.exe tests/smoke.py --base-url http://localhost:8084 +\.venv\Scripts\python.exe student-tests/pipeline_unit.py +\.venv\Scripts\python.exe student-tests/grade4_pipeline.py --base-url http://localhost:8084 +\.venv\Scripts\python.exe student-tests/broker_persistence.py --base-url http://localhost:8084 +\.venv\Scripts\python.exe student-tests/outbox_recovery.py --base-url http://localhost:8084 +\.venv\Scripts\python.exe student-tests/worker_redelivery.py --base-url http://localhost:8084 +\.venv\Scripts\python.exe student-tests/replay_failed_job.py --base-url http://localhost:8084 +docker compose --env-file .env ps -a +``` + +Последние пять сценариев сохраняют краткие результаты в `evidence/`. Если используется WSL с установленным `make`, полный набор можно запустить так: `make test-grade4 BASE_URL=http://localhost:8084`. + +Безопасная остановка без удаления учебных данных: + +```powershell +docker compose --env-file .env stop +``` + +Только для намеренного полного сброса своей учебной базы, очереди и объектов: + +```powershell +docker compose --env-file .env down -v +``` + +Команда `down -v` удаляет volumes PostgreSQL, Redis, broker и RustFS; перед её запуском сохраните нужные данные. + +## Обработка вложения и принятые решения + +Из Lab 4 сохранены API пользователей, сессий, чатов и сообщений, проверка членства, идемпотентность сообщений, PostgreSQL, приватное S3-хранилище, две API-реплики и балансировщик. Загрузка файла изменилась: вместо синхронной обработки API отвечает `202 Accepted` и статусом `queued`. + +1. API потоково принимает PNG/JPEG, ограничивает размер 10 MiB и проверяет тип/размеры заголовка без декодирования полного изображения. Оригинал сначала записывается в RustFS под `staging/`. +2. Затем одной транзакцией PostgreSQL сохраняются метаданные attachment, одна логическая job и запись `job_outbox`. Только после успешного COMMIT API возвращает 202. Если БД недоступна после staging, неизвестный объект удаляется планировщиком после безопасного срока. +3. Отдельный dispatcher читает outbox и помещает в Redis Stream только UUID и служебные поля. Если процесс падает после `XADD`, но до отметки `published_at`, событие может отправиться повторно; это ожидаемая at-least-once семантика. +4. Отдельный worker читает consumer group, берёт staging-оригинал по доверенному ключу из PostgreSQL, сверяет размер и SHA-256, обрабатывает изображение и пишет результаты в стабильные ключи `attachment_id/pipeline_version/variant`. +5. Только после записи оригинала и всех вариантов worker транзакционно ставит `ready/succeeded`, удаляет staging и делает `XACK`. Если он падает до ack, повторная доставка видит завершённую job, не меняет готовый результат и подтверждает старое сообщение. + +Redis для пользовательских сессий и Redis Streams broker запущены отдельными сервисами и имеют отдельные пароли и volumes. Оба используют AOF `appendfsync everysec`. PostgreSQL является источником истины для состояния job и outbox; Stream — устойчивый транспорт/сигнал для worker. + +Pipeline v1 создаёт: + +- `thumbnail`: JPEG, вписывание в 256×256 без увеличения, quality 85; +- `crop`: JPEG 128×128, quality 85; +- `optimized`: WebP, максимальная сторона 1280 px, quality 82, method 4. + +Картинка ориентируется по EXIF, а EXIF не переносится в производные изображения. Worker обрабатывает по одному заданию за процесс, ограничен 768 MiB памяти и дедлайном 60 секунд; lease — 180 секунд, максимум — 5 попыток. Временные ошибки получают ограниченный exponential backoff с jitter. Постоянные ошибки переводят job во `failed`; явный replay разрешён только если исходник ещё хранится в staging и создаёт audit-запись. + +## Результаты проверок + +| Критерий | Команда / сценарий | Наблюдение на момент подготовки отчёта | Доказательство | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Структура, ссылки и синтаксис | `python scripts/check_contract.py` | PASS; проверены контракты и синтаксис Python | `evidence/static-checks.txt` | +| OpenAPI 3.1 / JSON Schema | `python scripts/validate_openapi.py` | PASS | `evidence/static-checks.txt` | +| Python-компиляция всех source/tests | `python -m compileall -q src scripts tests student-tests` | PASS, код завершения 0 | `evidence/static-checks.txt` | +| Compose-конфигурация и локальные secret paths | `docker compose --env-file .env config --quiet` | PASS | `evidence/static-checks.txt` | +| Локальный image pipeline probe | синтетический JPEG 640×480 с EXIF | PASS: JPEG 256×192, JPEG 128×128, WebP 640×480 | `evidence/static-checks.txt` | +| Публичный API smoke | `python tests/smoke.py --base-url http://localhost:8084` | PASS: lab5 public API scenario | `evidence/test.log` | +| Лимиты и все варианты | `python student-tests/grade4_pipeline.py --base-url http://localhost:8084` | PASS: queued → ready; три варианта и исходник проверены; corrupt/MIME 415, pixel/byte limits 413 | `evidence/grade4-pipeline.json` | +| Сохранность Redis Stream | `python student-tests/broker_persistence.py --base-url http://localhost:8084` | PASS: stream entry пережила restart broker с volume; тот же attachment завершён worker-ом | `evidence/broker-persistence.json` | +| API restart + crash dispatcher до XADD | `python student-tests/outbox_recovery.py --base-url http://localhost:8084` | PASS: queued job/outbox пережили restart API; dispatcher crash exit 75 до XADD; тот же attachment восстановлен до ready | `evidence/outbox-recovery.json` | +| Worker crash после COMMIT до XACK | `python student-tests/worker_redelivery.py --base-url http://localhost:8084` | PASS: crash exit 75 после side effect; redelivery завершилась exit 0; варианты неизменны, XACK после COMMIT | `evidence/worker-redelivery.json` | +| Failed registry и безопасный replay | `python student-tests/replay_failed_job.py --base-url http://localhost:8084` | PASS: явный replay перевёл `failed` → `ready`; записан audit, исходник сохранён до готовности | `evidence/replay-failed-job.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Стенд Compose запускался локально. `make check` и `make test` были выполнены в WSL и завершились PASS. Публичный smoke и все пять сценариев уровня 4 также завершились PASS; фактические выводы и JSON-доказательства сохранены в `evidence/`. Проверки контракта, OpenAPI и Python-компиляции завершились PASS. Ранее возникшая проблема доступа Windows к Docker Engine была устранена запуском стенда в рабочем окружении; она не относится к результатам последующих проверок. -## Эксперименты и отказы +## Ограничения и известные ошибки -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +- Redis Streams и RustFS размещены на одиночных локальных узлах; Compose не создаёт HA-кластер и не доказывает сохранность при потере диска или всего хоста. +- AOF `everysec` может потерять последнюю секунду записей при аварийном отключении питания. Outbox повторно публикует незавершённые задания, но сам PostgreSQL и его диск остаются отдельной границей отказа. +- Ограничение worker — один job одновременно на контейнер; throughput не измерялся и трек на оценку 5 не заявлен. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Ответы на вопросы защиты -## Ограничения и следующие шаги +1. **Чем publisher confirm отличается от consumer ack?** Ответ Redis на `XADD` подтверждает приём записи в Stream. `XACK` подтверждает завершение потребления и вызывается после долговременного эффекта: готовых S3-объектов и COMMIT статуса `ready` в PostgreSQL. Это две разные границы. +2. **Что уже сохранено в момент 202?** Полный проверенный оригинал уже находится в S3 staging, а PostgreSQL сохранил метаданные, job и намерение отправки через outbox. Событие ещё может ожидать публикации в Stream. +3. **Почему повторная доставка нормальна и где ключ идемпотентности?** Падение между эффектом и ack неизбежно может привести к повтору. Неизменяемый `job_id` хранится в PostgreSQL; ключ конечного эффекта — `(attachment_id, pipeline_version, variant)`. Повтор записи по тому же ключу безопасно перезаписывает тот же объект. +4. **Как отличить зависший worker от долгой обработки и не запустить два неконтролируемых дубля?** У обработки есть дедлайн 60 секунд и lease 180 секунд. Истёкший lease возвращает job в очередь; блокировка строки и сверка номера попытки не дают старой попытке изменить актуальное состояние. Повторные сообщения при `ready` становятся no-op с ack. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Итоговая проверка PR #75 + +- [x] `make check` и `make test` завершились PASS; публичный smoke выполнен на работающем стенде. +- [x] Проверки pipeline, сохранности broker, восстановления outbox, повторной доставки worker и replay завершились PASS; JSON-файлы сохранены в `evidence/`. +- [x] Работа API×2, PostgreSQL, Redis sessions, Redis Streams broker, dispatcher, worker, RustFS и балансировщика подтверждена запуском стенда и end-to-end сценариями. +- [x] Проверены состояние Git и доказательства; секреты хранятся локально вне коммитов. В отчёт добавлены commit `f9246b8` и ссылка на открытую PR #75. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..ebd36aec --- /dev/null +++ b/compose.yaml @@ -0,0 +1,358 @@ +name: mtusi-messenger-lab5 + +services: + db: + image: postgres:16.4-alpine + restart: unless-stopped + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [backend] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + deploy: + resources: + limits: {cpus: "1.00", memory: 1G} + + redis: + image: redis:7.4.1-alpine + restart: unless-stopped + command: + - sh + - -ec + - exec redis-server --dir /data --appendonly yes --appendfsync everysec --save 60 1 --requirepass "$$(cat /run/secrets/redis_password)" + secrets: [redis_password] + volumes: [redis_data:/data] + networks: [backend] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + deploy: + resources: + limits: {cpus: "0.25", memory: 256M} + + broker: + image: redis:7.4.1-alpine + restart: unless-stopped + command: + - sh + - -ec + - exec redis-server --dir /data --appendonly yes --appendfsync everysec --save 60 1 --requirepass "$$(cat /run/secrets/broker_password)" + secrets: [broker_password] + volumes: [broker_data:/data] + networks: [backend] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/broker_password)\" redis-cli ping | grep -q PONG"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 5s + deploy: + resources: + limits: {cpus: "0.25", memory: 256M} + + s3-init: + image: alpine:3.24.1 + user: "0:0" + command: ["sh", "-ec", "chown -R 10001:10001 /data && chmod 0750 /data"] + volumes: [s3_data:/data] + restart: "no" + + s3: + image: rustfs/rustfs:1.0.5 + restart: unless-stopped + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: 0.0.0.0:9000 + RUSTFS_CONSOLE_ADDRESS: 0.0.0.0:9001 + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_OBS_LOGGER_LEVEL: warn + RUSTFS_CHECK_UPDATE: "false" + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_root_secret_access_key + secrets: [s3_root_access_key_id, s3_root_secret_access_key] + volumes: [s3_data:/data] + networks: [backend] + depends_on: + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:9000/health/ready >/dev/null"] + interval: 3s + timeout: 2s + retries: 20 + start_period: 10s + deploy: + resources: + limits: {cpus: "1.00", memory: 1G} + + migrate: + image: mtusi-messenger-lab5-api:local + build: + context: . + dockerfile: Dockerfile + args: + BUILD_REVISION: "${BUILD_REVISION:-local}" + command: ["python", "-m", "src.app.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [backend] + depends_on: + db: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 256M} + + s3-bootstrap: + image: mtusi-messenger-lab5-api:local + build: + context: . + dockerfile: Dockerfile + args: + BUILD_REVISION: "${BUILD_REVISION:-local}" + command: ["python", "-m", "src.app.s3_bootstrap"] + environment: + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id + S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_root_access_key_id, s3_root_secret_access_key, s3_access_key_id, s3_secret_access_key] + networks: [backend] + depends_on: + s3: + condition: service_healthy + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 256M} + + api: + image: mtusi-messenger-lab5-api:local + build: + context: . + dockerfile: Dockerfile + args: + BUILD_REVISION: "${BUILD_REVISION:-local}" + restart: unless-stopped + stop_grace_period: 8s + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-180} + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [backend, edge] + depends_on: + db: + condition: service_healthy + redis: + condition: service_healthy + s3: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2)"] + interval: 3s + timeout: 2s + retries: 3 + start_period: 10s + tmpfs: ["/tmp:size=64m,noexec,nosuid"] + deploy: + resources: + limits: {cpus: "0.50", memory: 512M} + + dispatcher: + image: mtusi-messenger-lab5-api:local + build: + context: . + dockerfile: Dockerfile + args: + BUILD_REVISION: "${BUILD_REVISION:-local}" + command: ["python", "-m", "src.app.dispatcher"] + restart: unless-stopped + stop_grace_period: 10s + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_STREAM: ${BROKER_STREAM:-messenger:attachment-jobs:v1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-180} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + secrets: [database_url, broker_url] + networks: [backend] + depends_on: + db: + condition: service_healthy + broker: + condition: service_healthy + migrate: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "from pathlib import Path; import sys; sys.exit(0 if b'src.app.dispatcher' in Path('/proc/1/cmdline').read_bytes() else 1)"] + interval: 5s + timeout: 2s + retries: 3 + start_period: 5s + deploy: + resources: + limits: {cpus: "0.25", memory: 256M} + + worker: + image: mtusi-messenger-lab5-api:local + build: + context: . + dockerfile: Dockerfile + args: + BUILD_REVISION: "${BUILD_REVISION:-local}" + command: ["python", "-m", "src.app.worker"] + restart: unless-stopped + stop_grace_period: 10s + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_STREAM: ${BROKER_STREAM:-messenger:attachment-jobs:v1} + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-180} + secrets: [database_url, broker_url, s3_access_key_id, s3_secret_access_key] + networks: [backend] + depends_on: + db: + condition: service_healthy + broker: + condition: service_healthy + s3: + condition: service_healthy + s3-bootstrap: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + tmpfs: ["/tmp:size=64m,noexec,nosuid"] + healthcheck: + test: ["CMD", "python", "-c", "from pathlib import Path; import sys; sys.exit(0 if b'src.app.worker' in Path('/proc/1/cmdline').read_bytes() else 1)"] + interval: 5s + timeout: 2s + retries: 3 + start_period: 5s + deploy: + resources: + limits: {cpus: "0.75", memory: 768M} + + lb: + image: nginx:1.28.0-alpine + restart: unless-stopped + ports: ["127.0.0.1:${APP_PORT:-8080}:8080"] + volumes: [./infra/nginx.conf:/etc/nginx/conf.d/default.conf:ro] + networks: [edge] + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"] + interval: 3s + timeout: 2s + retries: 5 + start_period: 5s + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + s3-maintenance: + image: mtusi-messenger-lab5-api:local + build: + context: . + dockerfile: Dockerfile + args: + BUILD_REVISION: "${BUILD_REVISION:-local}" + profiles: [maintenance] + command: ["python", "-m", "src.app.reconcile_s3_orphans"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ENDPOINT: http://s3:9000 + S3_REGION: us-east-1 + S3_BUCKET: messenger-attachments + S3_ROOT_ACCESS_KEY_FILE: /run/secrets/s3_root_access_key_id + S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-86400} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + secrets: [database_url, s3_root_access_key_id, s3_root_secret_access_key] + networks: [backend] + depends_on: + db: + condition: service_healthy + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully + restart: "no" + +secrets: + postgres_password: + file: ./secrets/postgres_password + redis_password: + file: ./secrets/redis_password + s3_root_access_key_id: + file: ./secrets/s3_root_access_key_id + s3_root_secret_access_key: + file: ./secrets/s3_root_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + broker_url: + file: ./secrets/broker_url + broker_password: + file: ./secrets/broker_password + +volumes: + postgres_data: + redis_data: + broker_data: + s3_data: + +networks: + backend: + internal: true + edge: diff --git a/evidence/broker-persistence.json b/evidence/broker-persistence.json new file mode 100644 index 00000000..560d6e78 --- /dev/null +++ b/evidence/broker-persistence.json @@ -0,0 +1,8 @@ +{ + "result": "PASS", + "observed_at_utc": "2026-11-16T13:10:29.667433+00:00", + "attachment_id": "480e6b78-e420-45df-bc0d-92561e7bb8ec", + "status_after_broker_restart": "ready", + "stream_entry_survived_volume_restart": true, + "worker_completed_same_attachment": true +} diff --git a/evidence/check.log b/evidence/check.log new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/check.log @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/grade4-pipeline.json b/evidence/grade4-pipeline.json new file mode 100644 index 00000000..ff5205ec --- /dev/null +++ b/evidence/grade4-pipeline.json @@ -0,0 +1,46 @@ +{ + "result": "PASS", + "accepted_status": "queued", + "final_status": "ready", + "source_bytes": 5490, + "source_sha256": "1f0c92118f143ec4db15cb109de28621b293f0679db25a2bbd74b419e14c7fe8", + "original_unchanged": true, + "variants": { + "crop": { + "content_type": "image/jpeg", + "dimensions": [ + 128, + 128 + ], + "size_bytes": 382, + "sha256": "686744b92be403b0b4a52bfcc658b17b51352efdbaab4b4da402aa7074407edd", + "exif_removed": true + }, + "optimized": { + "content_type": "image/webp", + "dimensions": [ + 640, + 480 + ], + "size_bytes": 628, + "sha256": "db779e3fd7c5fd4734734916e9cf3f8eaedc9d1ff7f3ea7d477e92299be4823a", + "exif_removed": true + }, + "thumbnail": { + "content_type": "image/jpeg", + "dimensions": [ + 256, + 192 + ], + "size_bytes": 574, + "sha256": "aa1e313cc044fb934663ef96a7d13d4f321a3b638f74e8a3e528180dba78a8b4", + "exif_removed": true + } + }, + "rejected_inputs": { + "corrupt_image": 415, + "mime_mismatch": 415, + "pixel_limit": 413, + "byte_limit": 413 + } +} diff --git a/evidence/outbox-recovery.json b/evidence/outbox-recovery.json new file mode 100644 index 00000000..9b9f89dd --- /dev/null +++ b/evidence/outbox-recovery.json @@ -0,0 +1,18 @@ +{ + "result": "PASS", + "observed_at_utc": "2026-11-16T13:14:48.928114+00:00", + "scenario": [ + "stopped dispatcher and worker", + "received 202 and persisted queued attachment plus job/outbox transaction", + "created a message referring to the queued attachment; content returned 409", + "restarted API and confirmed the same queued record remained", + "crashed one-shot dispatcher with exit 75 before Redis XADD", + "restarted dispatcher and worker; the same attachment became ready" + ], + "attachment_id": "31894cf1-3cf0-497a-b842-6b3ff3a56da9", + "message_id": "de8ef62b-8d2b-41b2-b69d-ca64552f0ed4", + "status_after_api_restart": "queued", + "status_after_recovery": "ready", + "thumbnail_content_type": "image/jpeg", + "dispatcher_failpoint_exit": 75 +} diff --git a/evidence/replay-failed-job.json b/evidence/replay-failed-job.json new file mode 100644 index 00000000..a536b8d5 --- /dev/null +++ b/evidence/replay-failed-job.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "observed_at_utc": "2026-11-16T13:20:11.879453+00:00", + "attachment_id": "297a8f47-cdcc-498b-9a07-e152216fde8e", + "status_before_replay": "failed", + "safe_error_code": "test_permanent_failure", + "status_after_replay": "ready", + "explicit_replay_command": "python -m src.app.replay_job ", + "replay_audit_written": true, + "source_retained_until_ready": true +} diff --git a/evidence/static-checks.txt b/evidence/static-checks.txt new file mode 100644 index 00000000..8dd408ec --- /dev/null +++ b/evidence/static-checks.txt @@ -0,0 +1,44 @@ +Lab 5 offline verification (2026-11-16T09:37:48Z) + +python --version +Python 3.12.0 + +docker compose version +Docker Compose version v5.7.0 + +python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. + +python -m compileall -q src scripts tests student-tests +PASS: exit code 0 + +docker compose --env-file .env config --quiet +PASS: exit code 0 + +python student-tests/pipeline_unit.py +PASS: thumbnail JPEG 256x192, crop JPEG 128x128, optimized WebP 640x480; EXIF removed; checksums valid. + +Docker Engine access +NOT RUN: Docker Desktop Linux Engine pipe returned permission denied. Containerized application and HTTP tests were not executed. + +Follow-up: local PowerShell runtime verification (2026-11-16) +python tests/smoke.py --base-url http://localhost:8084 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. + +python student-tests/grade4_pipeline.py --base-url http://localhost:8084 +PASS: see evidence/grade4-pipeline.json + +python student-tests/broker_persistence.py --base-url http://localhost:8084 +PASS: see evidence/broker-persistence.json + +python student-tests/outbox_recovery.py --base-url http://localhost:8084 +PASS: see evidence/outbox-recovery.json + +python student-tests/worker_redelivery.py --base-url http://localhost:8084 +PASS: see evidence/worker-redelivery.json + +python student-tests/replay_failed_job.py --base-url http://localhost:8084 +PASS: see evidence/replay-failed-job.json diff --git a/evidence/test.log b/evidence/test.log new file mode 100644 index 00000000..1a2916e5 --- /dev/null +++ b/evidence/test.log @@ -0,0 +1,3 @@ +Command: python tests/smoke.py --base-url http://localhost:8084 +Observed PowerShell output: +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/worker-redelivery.json b/evidence/worker-redelivery.json new file mode 100644 index 00000000..d620c570 --- /dev/null +++ b/evidence/worker-redelivery.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "observed_at_utc": "2026-11-16T13:17:52.252178+00:00", + "attachment_id": "3df146f7-9d87-431e-897d-b62be275e62b", + "status_after_side_effect_commit": "ready", + "status_after_redelivery": "ready", + "variants_unchanged_after_redelivery": true, + "worker_crash_exit": 75, + "redelivery_worker_exit": 0, + "ack_happened_after_committed_effect": true +} diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..9e5d1f25 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,32 @@ +upstream messenger_api { + zone messenger_api 64k; + resolver 127.0.0.11 valid=1s ipv6=off; + server api:8080 resolve max_fails=1 fail_timeout=2s; + keepalive 16; +} + +server { + listen 8080; + server_name _; + + client_max_body_size 11m; + client_body_buffer_size 64k; + client_body_timeout 15s; + proxy_connect_timeout 1s; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + + location / { + proxy_pass http://messenger_api; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 2s; + proxy_request_buffering off; + proxy_buffering off; + } +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..f329638f --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,36 @@ +CREATE TABLE users ( + id UUID PRIMARY KEY, + username VARCHAR(32) NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name VARCHAR(100) NOT NULL + CHECK (char_length(display_name) BETWEEN 1 AND 100 AND btrim(display_name) <> ''), + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title VARCHAR(100) NOT NULL + CHECK (char_length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_id_idx ON chat_members(user_id, chat_id); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + sender_id UUID NOT NULL, + text VARCHAR(4000) NOT NULL + CHECK (char_length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT messages_sender_is_chat_member_fk + FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members(chat_id, user_id) ON DELETE CASCADE +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..231eb768 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +CREATE TABLE idempotency_records ( + id UUID PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method VARCHAR(8) NOT NULL, + path VARCHAR(200) NOT NULL, + idempotency_key VARCHAR(128) NOT NULL, + request_hash CHAR(64) NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at TIMESTAMPTZ NOT NULL, + CONSTRAINT idempotency_records_scope_key_uq + UNIQUE (user_id, method, path, idempotency_key), + CHECK (method = 'POST'), + CHECK (char_length(idempotency_key) BETWEEN 1 AND 128) +); +CREATE INDEX idempotency_records_expires_at_idx ON idempotency_records(expires_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..29eb1e0a --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,48 @@ +-- Lab 4: S3 stores file bytes, while PostgreSQL owns metadata and chat relations. +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_check; +ALTER TABLE messages + ADD CONSTRAINT messages_text_length_ck CHECK (char_length(text) BETWEEN 0 AND 4000); +ALTER TABLE messages + ADD CONSTRAINT messages_id_chat_id_uq UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL, + owner_id UUID NOT NULL, + filename VARCHAR(255) NOT NULL, + content_type VARCHAR(32) NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 CHAR(64) NOT NULL, + status VARCHAR(16) NOT NULL DEFAULT 'ready', + variants JSONB NOT NULL DEFAULT '[]'::jsonb, + error_code VARCHAR(64), + object_key TEXT NOT NULL UNIQUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT attachments_owner_is_chat_member_fk + FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members(chat_id, user_id) ON DELETE CASCADE, + CONSTRAINT attachments_id_chat_id_uq UNIQUE (id, chat_id), + CONSTRAINT attachments_filename_length_ck CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type_ck CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_bytes_ck CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_ck CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_status_ck CHECK (status = 'ready') +); +CREATE INDEX attachments_chat_created_idx ON attachments(chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL, + attachment_id UUID NOT NULL, + chat_id UUID NOT NULL, + position SMALLINT NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_message_chat_fk + FOREIGN KEY (message_id, chat_id) + REFERENCES messages(id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_attachment_chat_fk + FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments(id, chat_id) ON DELETE CASCADE, + CONSTRAINT message_attachments_position_uq UNIQUE (message_id, position) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments(attachment_id); diff --git a/migrations/004_attachment_schema_compat.sql b/migrations/004_attachment_schema_compat.sql new file mode 100644 index 00000000..a11ba832 --- /dev/null +++ b/migrations/004_attachment_schema_compat.sql @@ -0,0 +1,32 @@ +-- Bring databases that already recorded an earlier 003_attachments layout +-- forward without dropping attachment metadata or stored object references. +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS variants JSONB NOT NULL DEFAULT '[]'::jsonb; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS error_code VARCHAR(64); +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS width INTEGER; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS height INTEGER; +ALTER TABLE attachments ALTER COLUMN width DROP NOT NULL; +ALTER TABLE attachments ALTER COLUMN height DROP NOT NULL; + +ALTER TABLE message_attachments ADD COLUMN IF NOT EXISTS chat_id UUID; +UPDATE message_attachments AS link +SET chat_id = message.chat_id +FROM messages AS message +WHERE message.id = link.message_id + AND (link.chat_id IS NULL OR link.chat_id <> message.chat_id); +ALTER TABLE message_attachments ALTER COLUMN chat_id SET NOT NULL; + +-- Drop referencing foreign keys before replacing their referenced unique keys. +ALTER TABLE message_attachments DROP CONSTRAINT IF EXISTS message_attachments_message_chat_fk; +ALTER TABLE message_attachments DROP CONSTRAINT IF EXISTS message_attachments_attachment_chat_fk; + +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_id_chat_id_uq; +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_id_uq UNIQUE (id, chat_id); +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_id_chat_id_uq; +ALTER TABLE attachments ADD CONSTRAINT attachments_id_chat_id_uq UNIQUE (id, chat_id); + +ALTER TABLE message_attachments + ADD CONSTRAINT message_attachments_message_chat_fk + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE; +ALTER TABLE message_attachments + ADD CONSTRAINT message_attachments_attachment_chat_fk + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) ON DELETE CASCADE; diff --git a/migrations/005_async_attachment_jobs.sql b/migrations/005_async_attachment_jobs.sql new file mode 100644 index 00000000..260f3af6 --- /dev/null +++ b/migrations/005_async_attachment_jobs.sql @@ -0,0 +1,45 @@ +-- Lab 5: durable staging, transactional outbox and leased image jobs. +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_status_ck; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_ck CHECK (status IN ('queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; +ALTER TABLE attachments ADD COLUMN IF NOT EXISTS staging_key TEXT; +CREATE UNIQUE INDEX IF NOT EXISTS attachments_staging_key_uq ON attachments(staging_key) WHERE staging_key IS NOT NULL; + +CREATE TABLE attachment_jobs ( + id UUID PRIMARY KEY, + attachment_id UUID NOT NULL UNIQUE REFERENCES attachments(id) ON DELETE CASCADE, + pipeline_version INTEGER NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1), + attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1), + attempts_in_cycle INTEGER NOT NULL DEFAULT 1, + max_attempts INTEGER NOT NULL CHECK (max_attempts BETWEEN 1 AND 20), + status VARCHAR(16) NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'processing', 'succeeded', 'failed')), + available_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + lease_until TIMESTAMPTZ, + last_error_code VARCHAR(64), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + CONSTRAINT attachment_jobs_cycle_attempt_ck CHECK (attempts_in_cycle BETWEEN 1 AND max_attempts) +); +CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(status, available_at); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs(status, lease_until); + +CREATE TABLE job_outbox ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + attempt INTEGER NOT NULL CHECK (attempt >= 1), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + published_at TIMESTAMPTZ, + CONSTRAINT job_outbox_job_attempt_uq UNIQUE (job_id, attempt) +); +CREATE INDEX job_outbox_pending_idx ON job_outbox(published_at, created_at); + +CREATE TABLE job_replay_audit ( + id UUID PRIMARY KEY, + job_id UUID NOT NULL REFERENCES attachment_jobs(id) ON DELETE CASCADE, + previous_attempt INTEGER NOT NULL, + new_attempt INTEGER NOT NULL, + operator VARCHAR(100) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CHECK (new_attempt > previous_attempt) +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..ae06d541 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +pydantic==2.9.2 +SQLAlchemy==2.0.36 +psycopg[binary]==3.2.3 +redis==5.2.0 +argon2-cffi==23.1.0 +boto3==1.44.1 +Pillow==13.0.0 diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34dfa..3f5edcc0 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -40,8 +40,13 @@ def main(): assert 'X-Instance-Id' in response['headers'] assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) - for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) + python_files = [] + for directory in ('src', 'tests', 'scripts', 'student-tests'): + folder = ROOT / directory + if folder.exists(): + python_files.extend(folder.rglob('*.py')) + for file in python_files: + ast.parse(file.read_text(encoding='utf-8'),filename=str(file)) for file in ROOT.rglob('*.json'): if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): continue diff --git a/scripts/create-secrets.ps1 b/scripts/create-secrets.ps1 new file mode 100644 index 00000000..50ef30c3 --- /dev/null +++ b/scripts/create-secrets.ps1 @@ -0,0 +1,61 @@ +$ErrorActionPreference = 'Stop' +$repo = Split-Path -Parent $PSScriptRoot +$secretDir = Join-Path $repo 'secrets' +New-Item -ItemType Directory -Path $secretDir -Force | Out-Null +function New-Secret { + $bytes = New-Object byte[] 32 + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + try { $rng.GetBytes($bytes) } finally { $rng.Dispose() } + return [Convert]::ToBase64String($bytes).TrimEnd('=').Replace('+', '-').Replace('/', '_') +} + +function New-AccessKey { + $alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789' + $bytes = New-Object byte[] 20 + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + try { $rng.GetBytes($bytes) } finally { $rng.Dispose() } + $value = -join ($bytes | ForEach-Object { $alphabet[($_ % $alphabet.Length)] }) + return $value +} + +function Read-Secret([string]$name) { + $path = Join-Path $secretDir $name + if (-not (Test-Path -LiteralPath $path)) { return $null } + return [System.IO.File]::ReadAllText($path).Trim() +} + +function Write-Secret([string]$name, [string]$value) { + $path = Join-Path $secretDir $name + [System.IO.File]::WriteAllText($path, $value, [System.Text.UTF8Encoding]::new($false)) +} + +function Ensure-Pair([string]$first, [string]$second, [scriptblock]$newFirst, [scriptblock]$newSecond) { + $left = Read-Secret $first + $right = Read-Secret $second + if (($null -eq $left) -ne ($null -eq $right)) { + throw "Only one file of credential pair '$first'/'$second' exists. Resolve it manually; credentials are never rotated automatically." + } + if ($null -eq $left) { + $left = & $newFirst + $right = & $newSecond $left + Write-Secret $first $left + Write-Secret $second $right + } + return @($left, $right) +} + +$pg = Ensure-Pair 'postgres_password' 'database_url' { New-Secret } { + param($password) + "postgresql+psycopg://messenger:$password@db:5432/messenger" +} +$redis = Ensure-Pair 'redis_password' 'cache_url' { New-Secret } { + param($password) + "redis://:$password@redis:6379/0" +} +$broker = Ensure-Pair 'broker_password' 'broker_url' { New-Secret } { + param($password) + "redis://:$password@broker:6379/0" +} +$s3root = Ensure-Pair 's3_root_access_key_id' 's3_root_secret_access_key' { New-AccessKey } { New-Secret } +$s3api = Ensure-Pair 's3_access_key_id' 's3_secret_access_key' { New-AccessKey } { New-Secret } +Write-Output 'Local Postgres, Redis sessions, Redis Streams broker, RustFS root and limited API credentials are ready under secrets/. Values were not displayed.' diff --git a/scripts/recover-api.sh b/scripts/recover-api.sh new file mode 100644 index 00000000..0a7a663e --- /dev/null +++ b/scripts/recover-api.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +docker compose --env-file .env up -d dispatcher worker >/dev/null +docker compose --env-file .env up -d --scale api=2 api lb >/dev/null +deadline=$((SECONDS + 60)) +while (( SECONDS < deadline )); do + healthy=0 + for container_id in $(docker compose --env-file .env ps -q api); do + state=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container_id" 2>/dev/null || true) + [[ "$state" == healthy ]] && healthy=$((healthy + 1)) + done + if (( healthy >= 2 )) && docker compose --env-file .env exec -T lb wget -q -O /dev/null http://127.0.0.1:8080/health/ready; then + printf 'PASS: two API replicas are healthy through the load balancer.\n' + exit 0 + fi + sleep 1 +done +printf 'API replicas did not recover within 60 seconds.\n' >&2 +docker compose --env-file .env ps +exit 1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..f67fc49e --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env sh +set -eu +cd "$(dirname "$0")/.." + +# Stop entry points and background consumers before the state stores; named volumes remain. +docker compose stop lb api dispatcher worker broker redis db s3 +docker compose start db redis broker s3 + +attempt=0 +until docker compose exec -T db pg_isready -U messenger -d messenger >/dev/null 2>&1 \ + && docker compose exec -T redis sh -ec 'REDISCLI_AUTH="$(cat /run/secrets/redis_password)" redis-cli ping | grep -q PONG' >/dev/null 2>&1 \ + && docker compose exec -T broker sh -ec 'REDISCLI_AUTH="$(cat /run/secrets/broker_password)" redis-cli ping | grep -q PONG' >/dev/null 2>&1 \ + && docker compose exec -T s3 curl -fsS http://127.0.0.1:9000/health/ready >/dev/null 2>&1; do + attempt=$((attempt + 1)) + if [ "$attempt" -ge 60 ]; then + echo 'Database, cache, broker, or object store did not become ready within 60 seconds.' >&2 + exit 1 + fi + sleep 1 +done + +docker compose --env-file .env up -d migrate s3-bootstrap +docker compose --env-file .env up -d dispatcher worker +docker compose --env-file .env up -d --scale api=2 api lb diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..e0114f44 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container_id in $(docker compose ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container_id") + if [[ "$hostname" == "$TARGET_INSTANCE_ID" ]]; then + mkdir -p evidence + printf '%s %s\n' "$(date -u +%FT%TZ)" "$TARGET_INSTANCE_ID" >> evidence/stopped-replicas.txt + docker stop --time 6 "$container_id" >/dev/null + exit 0 + fi +done + +printf 'No running API container matches TARGET_INSTANCE_ID.\n' >&2 +exit 1 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..2f310d5d --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""MTUSI Lab 2 API.""" diff --git a/src/app/__init__.py b/src/app/__init__.py new file mode 100644 index 00000000..2f310d5d --- /dev/null +++ b/src/app/__init__.py @@ -0,0 +1 @@ +"""MTUSI Lab 2 API.""" diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 00000000..01e1dbf5 --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,19 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from .settings import load_database_url + + +engine = create_engine( + load_database_url(), + pool_size=5, + max_overflow=5, + pool_timeout=3, + pool_recycle=1800, + pool_pre_ping=True, + connect_args={ + "connect_timeout": 3, + "options": "-c statement_timeout=5000", + }, +) +SessionLocal = sessionmaker(bind=engine, expire_on_commit=False) diff --git a/src/app/dispatcher.py b/src/app/dispatcher.py new file mode 100644 index 00000000..9a43c276 --- /dev/null +++ b/src/app/dispatcher.py @@ -0,0 +1,139 @@ +"""Publish PostgreSQL outbox rows to a persistent Redis Stream.""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import argparse +import logging +import os +import random +import time +import uuid + +from redis import Redis +from sqlalchemy import or_, select + +from .db import SessionLocal +from .models import Attachment, AttachmentJob, JobOutbox +from .settings import load_settings + + +settings = load_settings(require_cache=False, require_s3=False) +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +logger = logging.getLogger("messenger.dispatcher") +broker = Redis.from_url(settings.broker_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=3) + + +def _retry_delay(attempts_in_cycle: int) -> float: + ceiling = min(60.0, 2.0 ** min(attempts_in_cycle, 6)) + return random.uniform(ceiling / 2, ceiling) + + +def recover_expired_leases() -> int: + now = datetime.now(timezone.utc) + recovered = 0 + with SessionLocal.begin() as session: + jobs = session.scalars( + select(AttachmentJob) + .where(AttachmentJob.status == "processing", AttachmentJob.lease_until <= now) + .order_by(AttachmentJob.lease_until) + .limit(100) + .with_for_update(skip_locked=True) + ).all() + for job in jobs: + attachment = session.get(Attachment, job.attachment_id, with_for_update=True) + if job.attempts_in_cycle < job.max_attempts: + job.attempt += 1 + job.attempts_in_cycle += 1 + job.status = "queued" + job.available_at = now + timedelta(seconds=_retry_delay(job.attempts_in_cycle)) + job.lease_until = None + job.last_error_code = "worker_lease_expired" + if attachment is not None: + attachment.status = "queued" + attachment.error_code = None + session.add(JobOutbox(job_id=job.id, attempt=job.attempt)) + else: + job.status = "failed" + job.lease_until = None + job.last_error_code = "retry_exhausted" + job.completed_at = now + if attachment is not None: + attachment.status = "failed" + attachment.error_code = "retry_exhausted" + recovered += 1 + return recovered + + +def dispatch_once(batch_size: int = 50) -> int: + now = datetime.now(timezone.utc) + published = 0 + with SessionLocal.begin() as session: + rows = session.execute( + select(JobOutbox, AttachmentJob) + .join(AttachmentJob, AttachmentJob.id == JobOutbox.job_id) + .where( + or_( + JobOutbox.published_at.is_(None), + JobOutbox.published_at <= now - timedelta(seconds=settings.job_lease_seconds), + ), + JobOutbox.attempt == AttachmentJob.attempt, + AttachmentJob.status == "queued", + AttachmentJob.available_at <= now, + ) + .order_by(JobOutbox.created_at, JobOutbox.id) + .limit(batch_size) + .with_for_update(skip_locked=True, of=JobOutbox) + ).all() + for outbox, job in rows: + attachment = session.get(Attachment, job.attachment_id) + if attachment is None: + outbox.published_at = now + continue + failpoint_attachment = os.environ.get("DISPATCHER_FAILPOINT_ATTACHMENT_ID") + if failpoint_attachment == str(attachment.id): + marker = "lab5:failpoint:before-publish:" + str(attachment.id) + if broker.set(marker, "used", nx=True, ex=86400): + logger.warning("test_failpoint_before_publish attachment_id=%s", attachment.id) + # Raising inside the SQL transaction simulates a dispatcher + # crash after the API commit but before Redis accepts the job. + raise SystemExit(75) + broker.xadd(settings.broker_stream, { + "schema_version": "1", + "job_id": str(job.id), + "type": "attachment.process.v1", + "attachment_id": str(attachment.id), + "pipeline_version": str(job.pipeline_version), + "attempt": str(job.attempt), + "created_at": now.isoformat(timespec="microseconds").replace("+00:00", "Z"), + }) + # If the process dies after XADD but before this commit, the same + # outbox event is published again. The stable job/effect keys make + # redelivery safe. + outbox.published_at = now + published += 1 + return published + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--once", action="store_true", help="run one recovery/dispatch cycle and exit") + args = parser.parse_args() + logger.info("outbox_dispatcher_started stream=%s", settings.broker_stream) + if args.once: + recovered = recover_expired_leases() + published = dispatch_once() + logger.info("outbox_cycle recovered=%s published=%s", recovered, published) + return + while True: + try: + recovered = recover_expired_leases() + published = dispatch_once() + if recovered or published: + logger.info("outbox_cycle recovered=%s published=%s", recovered, published) + except Exception as exc: + logger.warning("outbox_cycle_failed type=%s", type(exc).__name__) + time.sleep(0.5) + + +if __name__ == "__main__": + main() diff --git a/src/app/errors.py b/src/app/errors.py new file mode 100644 index 00000000..ecafd8a6 --- /dev/null +++ b/src/app/errors.py @@ -0,0 +1,23 @@ +from fastapi import Request +from fastapi.responses import JSONResponse + + +class ApiError(Exception): + def __init__(self, code: str, message: str, status: int, challenge: str | None = None): + super().__init__(message) + self.code = code + self.message = message + self.status = status + self.challenge = challenge + + +def error_response(request: Request, code: str, message: str, status: int, challenge: str | None = None): + request_id = getattr(request.state, "request_id", "unknown") + headers = {"X-Request-Id": request_id, "Cache-Control": "no-store"} + if challenge: + headers["WWW-Authenticate"] = challenge + return JSONResponse( + status_code=status, + headers=headers, + content={"error": {"code": code, "message": message, "request_id": request_id}}, + ) diff --git a/src/app/explain_history.py b/src/app/explain_history.py new file mode 100644 index 00000000..81a194c5 --- /dev/null +++ b/src/app/explain_history.py @@ -0,0 +1,71 @@ +"""Create 10,000 disposable rows and record the indexed keyset-page plan.""" +import json +import uuid + +from sqlalchemy import text + +from .db import engine + + +def run() -> None: + user_id = uuid.uuid4() + chat_id = uuid.uuid4() + plan_json = None + with engine.connect() as connection: + transaction = connection.begin() + try: + connection.execute( + text("INSERT INTO users(id, username, display_name, password_hash) " + "VALUES (:id, :username, 'Synthetic plan fixture', 'not-a-login-hash')"), + {"id": user_id, "username": "plan_" + uuid.uuid4().hex[:20]}, + ) + connection.execute( + text("INSERT INTO chats(id, title) VALUES (:id, 'Synthetic plan fixture')"), + {"id": chat_id}, + ) + connection.execute( + text("INSERT INTO chat_members(chat_id, user_id) VALUES (:chat_id, :user_id)"), + {"chat_id": chat_id, "user_id": user_id}, + ) + connection.execute( + text(""" + INSERT INTO messages(id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), :chat_id, :sender_id, + 'synthetic message ' || n::text, + TIMESTAMPTZ '2025-01-01 00:00:00+00' + n * INTERVAL '1 microsecond' + FROM generate_series(1, 10000) AS n + """), + {"chat_id": chat_id, "sender_id": user_id}, + ) + connection.exec_driver_sql("ANALYZE messages") + raw = connection.execute( + text(""" + EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = :chat_id + AND (created_at, id) > (TIMESTAMPTZ '2025-01-01 00:00:00+00', + '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY created_at, id + LIMIT 50 + """), + {"chat_id": chat_id}, + ).scalar_one() + plan_json = raw if isinstance(raw, list) else json.loads(raw) + finally: + transaction.rollback() + + document = plan_json[0] + + def contains_history_index(node: dict) -> bool: + if node.get("Index Name") == "messages_history_idx": + return True + return any(contains_history_index(child) for child in node.get("Plans", [])) + + if not contains_history_index(document["Plan"]): + raise SystemExit("Planner did not use messages_history_idx; inspect and revise the query/index.") + print(json.dumps(document, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + run() diff --git a/src/app/image_pipeline.py b/src/app/image_pipeline.py new file mode 100644 index 00000000..e85b02a2 --- /dev/null +++ b/src/app/image_pipeline.py @@ -0,0 +1,76 @@ +"""Bounded, deterministic image transformations for attachment pipeline v1.""" +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO + +from PIL import Image, ImageOps, UnidentifiedImageError + + +class PermanentImageError(Exception): + """An input image cannot be processed; retrying the same bytes will not help.""" + + +def _rgb(image: Image.Image) -> Image.Image: + if image.mode in {"RGBA", "LA"} or "transparency" in image.info: + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + + +def _encoded(image: Image.Image, fmt: str, **options) -> bytes: + buffer = BytesIO() + image.save(buffer, format=fmt, **options) + return buffer.getvalue() + + +def process_image(data: bytes, claimed_content_type: str, max_pixels: int, attachment_id: str, pipeline_version: int = 1) -> list[dict]: + """Decode once, apply EXIF orientation, and create the three grade-4 variants.""" + try: + with Image.open(BytesIO(data)) as opened: + actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(opened.format or "") + if actual_type is None or actual_type != claimed_content_type: + raise PermanentImageError("invalid_image") + width, height = opened.size + if width < 1 or height < 1 or width * height > max_pixels: + raise PermanentImageError("image_too_large") + opened.load() + oriented = ImageOps.exif_transpose(opened) + base = _rgb(oriented) + except PermanentImageError: + raise + except (UnidentifiedImageError, OSError, SyntaxError, ValueError, Image.DecompressionBombError): + raise PermanentImageError("invalid_image") from None + + variants = [] + thumb = base.copy() + thumb.thumbnail((256, 256), Image.Resampling.LANCZOS) + variants.append(("thumbnail", "image/jpeg", "jpg", _encoded(thumb, "JPEG", quality=85, optimize=True))) + del thumb + + crop = ImageOps.fit(base, (128, 128), method=Image.Resampling.LANCZOS, centering=(0.5, 0.5)) + variants.append(("crop", "image/jpeg", "jpg", _encoded(crop, "JPEG", quality=85, optimize=True))) + del crop + + optimized = base.copy() + optimized.thumbnail((1280, 1280), Image.Resampling.LANCZOS) + variants.append(("optimized", "image/webp", "webp", _encoded(optimized, "WEBP", quality=82, method=4))) + del optimized, base + + result = [] + for name, content_type, extension, body in variants: + with Image.open(BytesIO(body)) as check: + out_width, out_height = check.size + result.append({ + "name": name, + "content_type": content_type, + "size_bytes": len(body), + "width": out_width, + "height": out_height, + "sha256": sha256(body).hexdigest(), + "object_key": f"variants/v{pipeline_version}/{attachment_id}/{name}.{extension}", + "body": body, + }) + return result diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 00000000..63299add --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,871 @@ +import base64 +import binascii +import asyncio +from datetime import datetime, timedelta, timezone +from email.message import Message as EmailMessage +import hashlib +import json +import logging +import re +import secrets +import socket +import tempfile +import unicodedata +import uuid + +from argon2 import PasswordHasher, Type +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError +from fastapi import FastAPI, Header, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from PIL import Image, UnidentifiedImageError +from botocore.exceptions import BotoCoreError, ClientError +from redis import Redis +from redis.exceptions import RedisError +from sqlalchemy import and_, delete, func, or_, select, text, tuple_, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import DBAPIError, OperationalError, TimeoutError as SQLAlchemyTimeoutError +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +from .db import SessionLocal, engine +from .errors import ApiError, error_response +from .models import Attachment, AttachmentJob, Chat, ChatMember, IdempotencyRecord, JobOutbox, Message, MessageAttachment, User +from .schemas import ( + AttachmentOut, + ChatOut, + ChatPage, + CreateChat, + CreateMessage, + CreateUser, + Health, + MessageOut, + MessagePage, + SessionOut, + UserOut, +) +from .settings import load_settings +from .storage import s3 + +settings = load_settings() + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) +logger = logging.getLogger("messenger.api") +password_hasher = PasswordHasher( + time_cost=2, + memory_cost=19_456, + parallelism=1, + hash_len=32, + salt_len=16, + type=Type.ID, +) +DUMMY_PASSWORD_HASH = password_hasher.hash("dummy-password-for-timing-only") +redis_client = Redis.from_url( + settings.cache_url, + decode_responses=True, + socket_connect_timeout=2, + socket_timeout=2, + health_check_interval=30, +) + +app = FastAPI(title="MTUSI Messenger — Lab 5", version="1.5.0") + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +TOKEN_RE = re.compile(r"^[A-Za-z0-9_-]{43}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$", re.ASCII) +# Docker assigns each container a stable hostname equal to its container ID. +# The load balancer forwards this value so tests can prove requests changed replicas. +INSTANCE_ID = socket.gethostname() +UPLOAD_LIMITER = asyncio.Semaphore(1) + + +def _iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def _user_out(user: User) -> dict: + return { + "id": str(user.id), + "username": user.username, + "display_name": user.display_name, + "created_at": _iso(user.created_at), + } + + +def _message_out(message: Message, attachment_ids: list[uuid.UUID | str] | None = None) -> dict: + return { + "id": str(message.id), + "chat_id": str(message.chat_id), + "sender_id": str(message.sender_id), + "text": message.text, + "created_at": _iso(message.created_at), + "attachment_ids": [str(item) for item in (attachment_ids or [])], + } + + +def _attachment_out(attachment: Attachment) -> dict: + public_variants = [ + {key: variant[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")} + for variant in (attachment.variants or []) + ] + return { + "id": str(attachment.id), + "chat_id": str(attachment.chat_id), + "owner_id": str(attachment.owner_id), + "filename": attachment.filename, + "content_type": attachment.content_type, + "size_bytes": attachment.size_bytes, + "sha256": attachment.sha256, + "status": attachment.status, + "variants": public_variants, + "error_code": attachment.error_code, + "created_at": _iso(attachment.created_at), + "updated_at": _iso(attachment.updated_at), + } + + +def _message_attachment_ids(session, message_id: uuid.UUID) -> list[uuid.UUID]: + return session.scalars( + select(MessageAttachment.attachment_id) + .where(MessageAttachment.message_id == message_id) + .order_by(MessageAttachment.position) + ).all() + + +def _validate_image(file_obj, claimed_content_type: str) -> str: + file_obj.seek(0) + try: + with Image.open(file_obj) as image: + detected = { + "PNG": "image/png", + "JPEG": "image/jpeg", + }.get(image.format or "") + if detected is None or detected != claimed_content_type: + raise _api_error("unsupported_media_type", "Формат изображения не поддерживается", 415) + width, height = image.size + if width < 1 or height < 1 or width * height > settings.max_image_pixels: + raise _api_error("payload_too_large", "Размер изображения превышает допустимый предел", 413) + image.verify() + file_obj.seek(0) + with Image.open(file_obj) as image: + image.load() + except ApiError: + raise + except (UnidentifiedImageError, OSError, SyntaxError, ValueError, Image.DecompressionBombError): + raise _api_error("unsupported_media_type", "Изображение повреждено или не поддерживается", 415) from None + finally: + file_obj.seek(0) + return claimed_content_type + + +def _probe_upload(file_obj, claimed_content_type: str) -> str: + """Check the image header and dimensions without decoding pixels in the API.""" + if claimed_content_type == "image/png": + file_obj.seek(0) + png_header = file_obj.read(33) + file_obj.seek(0) + if ( + len(png_header) == 33 + and png_header[:8] == b"\x89PNG\r\n\x1a\n" + and int.from_bytes(png_header[8:12], "big") == 13 + and png_header[12:16] == b"IHDR" + and (binascii.crc32(png_header[12:29]) & 0xFFFFFFFF) + == int.from_bytes(png_header[29:33], "big") + ): + width = int.from_bytes(png_header[16:20], "big") + height = int.from_bytes(png_header[20:24], "big") + if width < 1 or height < 1 or width * height > settings.max_image_pixels: + raise _api_error("payload_too_large", "Размер изображения превышает допустимый предел", 413) + + file_obj.seek(0) + try: + with Image.open(file_obj) as image: + detected = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "") + if detected is None or detected != claimed_content_type: + raise _api_error("unsupported_media_type", "Поддерживаются только PNG и JPEG", 415) + width, height = image.size + if width < 1 or height < 1 or width * height > settings.max_image_pixels: + raise _api_error("payload_too_large", "Размер изображения превышает допустимый предел", 413) + # Pillow's verify checks file structure but does not materialize the + # decoded pixel buffer. Full decode and transforms run in the worker. + image.verify() + except ApiError: + raise + except (UnidentifiedImageError, OSError, SyntaxError, ValueError, Image.DecompressionBombError): + raise _api_error("unsupported_media_type", "Изображение повреждено или не поддерживается", 415) from None + finally: + file_obj.seek(0) + return claimed_content_type + + +def _valid_filename(filename: str) -> bool: + if not 1 <= len(filename) <= 255: + return False + if "/" in filename or "\\" in filename: + return False + return not any(unicodedata.category(char) == "Cc" for char in filename) + + +def _is_not_found_s3(exc: ClientError) -> bool: + code = str(exc.response.get("Error", {}).get("Code", "")) + return code in {"404", "NoSuchKey", "NoSuchBucket", "NotFound"} + + +def _chat_out(session, chat: Chat) -> dict: + members = session.scalars( + select(ChatMember.user_id) + .where(ChatMember.chat_id == chat.id) + .order_by(ChatMember.user_id) + ).all() + return { + "id": str(chat.id), + "title": chat.title, + "member_ids": [str(member_id) for member_id in members], + "created_at": _iso(chat.created_at), + } + + +def _api_error(code: str, message: str, status: int, challenge: str | None = None) -> ApiError: + return ApiError(code, message, status, challenge) + + +def _unauthorized(challenge: str = BEARER_CHALLENGE) -> ApiError: + return _api_error("unauthorized", "Нет действительной идентификации", 401, challenge) + + +def _encode_cursor(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _decode_cursor(value: str) -> dict: + try: + padded = value + ("=" * ((4 - len(value) % 4) % 4)) + raw = base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True) + payload = json.loads(raw.decode("utf-8")) + if not isinstance(payload, dict): + raise ValueError("cursor must be an object") + return payload + except (ValueError, UnicodeError, binascii.Error, json.JSONDecodeError) as exc: + raise _api_error("invalid_request", "Неверный cursor", 400) from exc + + +def _cursor_position(payload: dict) -> tuple[datetime, uuid.UUID]: + try: + created_at = datetime.fromisoformat(payload["t"].replace("Z", "+00:00")) + if created_at.tzinfo is None: + raise ValueError("cursor time must include timezone") + item_id = uuid.UUID(payload["i"]) + return created_at.astimezone(timezone.utc), item_id + except (KeyError, TypeError, ValueError, AttributeError) as exc: + raise _api_error("invalid_request", "Неверный cursor", 400) from exc + + +def _session_key(token: str) -> str: + digest = hashlib.sha256(token.encode("ascii")).hexdigest() + return "session:v1:" + digest + + +def _parse_bearer(request: Request) -> str: + header = request.headers.get("authorization", "") + scheme, separator, token = header.partition(" ") + if scheme.lower() != "bearer" or not separator or not TOKEN_RE.fullmatch(token): + raise _unauthorized() + return token + + +def _load_session(token: str) -> dict: + value = redis_client.get(_session_key(token)) + if value is None: + raise _unauthorized() + try: + session = json.loads(value) + user_id = uuid.UUID(session["user_id"]) + expires_at = datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00")) + if expires_at.tzinfo is None or datetime.now(timezone.utc) >= expires_at: + redis_client.delete(_session_key(token)) + raise _unauthorized() + return {"user_id": user_id, "expires_at": expires_at} + except ApiError: + raise + except (KeyError, TypeError, ValueError, AttributeError, json.JSONDecodeError): + redis_client.delete(_session_key(token)) + raise _unauthorized() from None + + +def _current_user(request: Request) -> tuple[User, str]: + token = _parse_bearer(request) + session_data = _load_session(token) + with SessionLocal() as session: + user = session.get(User, session_data["user_id"]) + if user is None: + redis_client.delete(_session_key(token)) + raise _unauthorized() + return user, token + + +def _parse_basic(header: str) -> tuple[str, str]: + scheme, separator, encoded = header.partition(" ") + if scheme.lower() != "basic" or not separator or not encoded: + raise _unauthorized(BASIC_CHALLENGE) + try: + decoded = base64.b64decode(encoded.encode("ascii"), validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (UnicodeError, ValueError, binascii.Error): + raise _unauthorized(BASIC_CHALLENGE) from None + if not USERNAME_RE.fullmatch(username) or not password: + raise _unauthorized(BASIC_CHALLENGE) + return username, password + + +def _password_matches(password_hash: str, password: str) -> bool: + try: + return password_hasher.verify(password_hash, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def _create_session(username: str, password: str) -> dict: + with SessionLocal() as session: + user = session.scalar(select(User).where(User.username == username)) + stored_hash = user.password_hash if user is not None else DUMMY_PASSWORD_HASH + password_ok = _password_matches(stored_hash, password) + if user is None or not password_ok: + raise _unauthorized(BASIC_CHALLENGE) + user_payload = _user_out(user) + + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.session_ttl_seconds) + for _ in range(3): + token = secrets.token_urlsafe(32) + stored_value = json.dumps( + {"user_id": user_payload["id"], "expires_at": _iso(expires_at)}, + separators=(",", ":"), + ) + if redis_client.set(_session_key(token), stored_value, ex=settings.session_ttl_seconds, nx=True): + return { + "token": token, + "token_type": "Bearer", + "expires_at": _iso(expires_at), + "user": user_payload, + } + logger.error("session_token_collision_limit_reached") + raise _api_error("unavailable", "Сервис временно недоступен", 503) + + +def _get_chat_for_member(session, chat_id: uuid.UUID, user_id: uuid.UUID) -> Chat | None: + return session.scalar( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(Chat.id == chat_id, ChatMember.user_id == user_id) + ) + + +@app.middleware("http") +async def request_metadata_middleware(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception as exc: + # Keep an unexpected failure bounded and ensure the API response still + # identifies the request and the replica that created it. + logger.error("internal_error request_id=%s type=%s", request_id, type(exc).__name__) + response = error_response(request, "internal_error", "Внутренняя ошибка", 500) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status, exc.challenge) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error(request: Request, _exc: RequestValidationError): + return error_response(request, "invalid_request", "Неверный запрос", 400) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Объект не найден", 404) + return error_response(request, "invalid_request", "Неверный запрос", 400) + + +@app.exception_handler(OperationalError) +@app.exception_handler(SQLAlchemyTimeoutError) +async def handle_database_unavailable(request: Request, exc: Exception): + logger.warning("database_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__) + return error_response(request, "unavailable", "Сервис временно недоступен", 503) + + +@app.exception_handler(RedisError) +async def handle_cache_unavailable(request: Request, exc: RedisError): + logger.warning("session_store_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__) + return error_response(request, "unavailable", "Сервис временно недоступен", 503) + + +@app.exception_handler(ClientError) +@app.exception_handler(BotoCoreError) +async def handle_object_store_unavailable(request: Request, exc: Exception): + logger.warning("object_store_unavailable request_id=%s type=%s", request.state.request_id, type(exc).__name__) + return error_response(request, "unavailable", "Сервис временно недоступен", 503) + + +@app.exception_handler(DBAPIError) +async def handle_database_error(request: Request, exc: DBAPIError): + if exc.connection_invalidated: + logger.warning("database_connection_lost request_id=%s", request.state.request_id) + return error_response(request, "unavailable", "Сервис временно недоступен", 503) + logger.error("database_error request_id=%s type=%s", request.state.request_id, type(exc).__name__) + return error_response(request, "internal_error", "Внутренняя ошибка", 500) + + +@app.exception_handler(Exception) +async def handle_unexpected_error(request: Request, exc: Exception): + logger.error("internal_error request_id=%s type=%s", request.state.request_id, type(exc).__name__) + return error_response(request, "internal_error", "Внутренняя ошибка", 500) + + +@app.get("/health/live", response_model=Health) +def live(): + return {"status": "ok"} + + +@app.get("/health/ready", response_model=Health, responses={503: {"description": "Dependency unavailable"}}) +def ready(): + with SessionLocal() as session: + session.execute(text("SELECT 1")) + redis_client.ping() + s3.get_bucket_location(Bucket=settings.s3_bucket) + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=UserOut, status_code=201) +def register_user(payload: CreateUser): + password_hash = password_hasher.hash(payload.password) + user = User(username=payload.username, display_name=payload.display_name, password_hash=password_hash) + try: + with SessionLocal.begin() as session: + session.add(user) + session.flush() + except DBAPIError as exc: + if getattr(getattr(exc, "orig", None), "sqlstate", None) == "23505": + raise _api_error("conflict", "username занят", 409) from exc + raise + return _user_out(user) + + +@app.post("/api/v1/auth/sessions", response_model=SessionOut, status_code=201) +async def create_session(request: Request): + if await request.body(): + raise _api_error("invalid_request", "Basic authentication must not include a body", 400) + username, password = _parse_basic(request.headers.get("authorization", "")) + return await run_in_threadpool(_create_session, username, password) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session(request: Request): + _user, token = _current_user(request) + redis_client.delete(_session_key(token)) + return None + + +@app.get("/api/v1/users/me", response_model=UserOut) +def current_user(request: Request): + user, _token = _current_user(request) + return _user_out(user) + + +@app.post("/api/v1/chats", response_model=ChatOut, status_code=201) +def create_chat(payload: CreateChat, request: Request): + user, _token = _current_user(request) + member_ids = [uuid.UUID(value) for value in payload.member_ids] + if user.id not in member_ids: + raise _api_error("invalid_request", "Вызывающий должен входить в member_ids", 400) + + try: + with SessionLocal.begin() as session: + chat = Chat(title=payload.title) + session.add(chat) + session.flush() + session.add_all(ChatMember(chat_id=chat.id, user_id=member_id) for member_id in member_ids) + session.flush() + result = _chat_out(session, chat) + except DBAPIError as exc: + if getattr(getattr(exc, "orig", None), "sqlstate", None) in {"23503", "23505"}: + raise _api_error("invalid_request", "Неизвестный участник", 400) from exc + raise + return result + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats(request: Request, limit: int = 50, cursor: str | None = None): + user, _token = _current_user(request) + if limit < 1 or limit > 100: + raise _api_error("invalid_request", "limit вне диапазона", 400) + statement = ( + select(Chat) + .join(ChatMember, ChatMember.chat_id == Chat.id) + .where(ChatMember.user_id == user.id) + .order_by(Chat.created_at, Chat.id) + ) + if cursor is not None: + payload = _decode_cursor(cursor) + if payload.get("u") != str(user.id): + raise _api_error("invalid_request", "Неверный cursor", 400) + created_at, chat_id = _cursor_position(payload) + statement = statement.where(tuple_(Chat.created_at, Chat.id) > tuple_(created_at, chat_id)) + + with SessionLocal() as session: + chats = session.scalars(statement.limit(limit + 1)).all() + has_more = len(chats) > limit + page = chats[:limit] + items = [_chat_out(session, chat) for chat in page] + next_cursor = None + if has_more: + last = page[-1] + next_cursor = _encode_cursor({"u": str(user.id), "t": _iso(last.created_at), "i": str(last.id)}) + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}", response_model=ChatOut) +def get_chat(chat_id: uuid.UUID, request: Request): + user, _token = _current_user(request) + with SessionLocal() as session: + chat = _get_chat_for_member(session, chat_id, user.id) + if chat is None: + raise _api_error("not_found", "Объект не найден", 404) + return _chat_out(session, chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", response_model=MessageOut, status_code=201) +def create_message( + chat_id: uuid.UUID, + payload: CreateMessage, + request: Request, + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key"), +): + user, _token = _current_user(request) + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise _api_error("invalid_request", "Нужен корректный Idempotency-Key", 400) + if not payload.text.strip() and not payload.attachment_ids: + raise _api_error("invalid_request", "Сообщение не может состоять только из пробелов", 400) + + # Hash the validated semantic JSON, not the raw bytes, so whitespace and + # object key order do not change the identity of a request. + canonical_body = json.dumps( + {"text": payload.text, "attachment_ids": payload.attachment_ids}, + ensure_ascii=False, sort_keys=True, separators=(",", ":"), + ) + body_hash = hashlib.sha256(canonical_body.encode("utf-8")).hexdigest() + legacy_hash = hashlib.sha256( + json.dumps({"text": payload.text}, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + ).hexdigest() + path = request.url.path + now = datetime.now(timezone.utc) + expires_at = now + timedelta(hours=24) + scope = ( + IdempotencyRecord.user_id == user.id, + IdempotencyRecord.method == "POST", + IdempotencyRecord.path == path, + IdempotencyRecord.idempotency_key == idempotency_key, + ) + + with SessionLocal.begin() as session: + chat = _get_chat_for_member(session, chat_id, user.id) + if chat is None: + raise _api_error("not_found", "Объект не найден", 404) + attachment_ids = [uuid.UUID(value) for value in payload.attachment_ids] + if attachment_ids: + attached = session.scalars( + select(Attachment).where( + Attachment.chat_id == chat_id, + Attachment.id.in_(attachment_ids), + ) + ).all() + if len(attached) != len(attachment_ids): + raise _api_error("not_found", "Объект не найден", 404) + if any(item.status == "failed" for item in attached): + raise _api_error("conflict", "Вложение не удалось обработать", 409) + + # A key becomes reusable only after its 24-hour retention window. + session.execute(delete(IdempotencyRecord).where(*scope, IdempotencyRecord.expires_at <= now)) + insert_record = ( + pg_insert(IdempotencyRecord) + .values( + user_id=user.id, + method="POST", + path=path, + idempotency_key=idempotency_key, + request_hash=body_hash, + response_body={}, + expires_at=expires_at, + ) + .on_conflict_do_nothing(constraint="idempotency_records_scope_key_uq") + .returning(IdempotencyRecord.id) + ) + record_id = session.execute(insert_record).scalar_one_or_none() + + if record_id is None: + # A concurrent insert waits on the unique constraint. At READ + # COMMITTED the winning transaction is visible to this SELECT. + existing = session.scalar(select(IdempotencyRecord).where(*scope)) + if existing is None: + raise _api_error("unavailable", "Сервис временно недоступен", 503) + if existing.request_hash != body_hash and not (not attachment_ids and existing.request_hash == legacy_hash): + raise _api_error("conflict", "Idempotency-Key уже использован для другого запроса", 409) + result = dict(existing.response_body) + result.setdefault("attachment_ids", []) + else: + message = Message(chat_id=chat_id, sender_id=user.id, text=payload.text) + session.add(message) + session.flush() + session.add_all( + MessageAttachment( + message_id=message.id, + attachment_id=attachment_id, + chat_id=chat_id, + position=position, + ) + for position, attachment_id in enumerate(attachment_ids) + ) + result = _message_out(message, attachment_ids) + session.execute( + update(IdempotencyRecord) + .where(IdempotencyRecord.id == record_id) + .values(response_body=result) + ) + return result + + +@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage) +def list_messages(chat_id: uuid.UUID, request: Request, limit: int = 50, cursor: str | None = None): + user, _token = _current_user(request) + with SessionLocal() as session: + chat = _get_chat_for_member(session, chat_id, user.id) + if chat is None: + raise _api_error("not_found", "Объект не найден", 404) + if limit < 1 or limit > 100: + raise _api_error("invalid_request", "limit вне диапазона", 400) + after = None + if cursor is not None: + payload = _decode_cursor(cursor) + if payload.get("c") != str(chat_id): + raise _api_error("invalid_request", "cursor от другого чата", 400) + after = _cursor_position(payload) + statement = select(Message).where(Message.chat_id == chat_id) + if after is not None: + statement = statement.where(tuple_(Message.created_at, Message.id) > tuple_(*after)) + statement = statement.order_by(Message.created_at, Message.id).limit(limit + 1) + rows = session.scalars(statement).all() + has_more = len(rows) > limit + page = rows[:limit] + attachment_rows = session.execute( + select(MessageAttachment.message_id, MessageAttachment.attachment_id) + .where(MessageAttachment.message_id.in_([message.id for message in page])) + .order_by(MessageAttachment.message_id, MessageAttachment.position) + ).all() if page else [] + ids_by_message: dict[uuid.UUID, list[uuid.UUID]] = {} + for message_id, attachment_id in attachment_rows: + ids_by_message.setdefault(message_id, []).append(attachment_id) + items = [_message_out(message, ids_by_message.get(message.id, [])) for message in page] + + next_cursor = None + if has_more: + last = page[-1] + next_cursor = _encode_cursor({"c": str(chat_id), "t": _iso(last.created_at), "i": str(last.id)}) + return {"items": items, "next_cursor": next_cursor} + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=AttachmentOut, + status_code=202, +) +async def upload_attachment( + chat_id: uuid.UUID, + request: Request, + filename: str, +): + user, _token = _current_user(request) + with SessionLocal() as session: + if _get_chat_for_member(session, chat_id, user.id) is None: + raise _api_error("not_found", "Объект не найден", 404) + + if not _valid_filename(filename): + raise _api_error("invalid_request", "Неверное имя файла", 400) + content_type_header = request.headers.get("content-type", "") + header = EmailMessage() + try: + header["Content-Type"] = content_type_header + claimed_type = header.get_content_type().lower() + except (TypeError, ValueError): + claimed_type = "" + if claimed_type not in {"image/png", "image/jpeg"}: + raise _api_error("unsupported_media_type", "Поддерживаются только PNG и JPEG", 415) + content_length = request.headers.get("content-length") + if content_length is not None: + try: + if int(content_length) > settings.max_upload_bytes: + raise _api_error("payload_too_large", "Файл превышает допустимый размер", 413) + except ValueError: + raise _api_error("invalid_request", "Неверный Content-Length", 400) from None + + async with UPLOAD_LIMITER: + with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode="w+b", dir="/tmp") as temporary: + size_bytes = 0 + digest = hashlib.sha256() + async for chunk in request.stream(): + if not chunk: + continue + size_bytes += len(chunk) + if size_bytes > settings.max_upload_bytes: + raise _api_error("payload_too_large", "Файл превышает допустимый размер", 413) + digest.update(chunk) + temporary.write(chunk) + if size_bytes == 0: + raise _api_error("unsupported_media_type", "Файл пуст", 415) + + detected_type = await run_in_threadpool(_probe_upload, temporary, claimed_type) + attachment_id = uuid.uuid4() + object_key = f"originals/{attachment_id.hex}/original" + staging_key = f"staging/{attachment_id.hex}/original" + temporary.seek(0) + try: + await run_in_threadpool( + s3.put_object, + Bucket=settings.s3_bucket, + Key=staging_key, + Body=temporary, + ContentLength=size_bytes, + ContentType=detected_type, + Metadata={"sha256": digest.hexdigest()}, + ) + except (ClientError, BotoCoreError): + try: + await run_in_threadpool(s3.delete_object, Bucket=settings.s3_bucket, Key=staging_key) + except Exception as cleanup_exc: + logger.warning( + "attachment_compensation_failed request_id=%s type=%s", + request.state.request_id, + type(cleanup_exc).__name__, + ) + raise _api_error("unavailable", "Сервис временно недоступен", 503) from None + + with SessionLocal.begin() as session: + attachment = Attachment( + id=attachment_id, + chat_id=chat_id, + owner_id=user.id, + filename=filename, + content_type=detected_type, + size_bytes=size_bytes, + sha256=digest.hexdigest(), + status="queued", + variants=[], + error_code=None, + object_key=object_key, + staging_key=staging_key, + ) + session.add(attachment) + job = AttachmentJob( + id=uuid.uuid4(), + attachment_id=attachment_id, + pipeline_version=1, + attempt=1, + max_attempts=settings.job_max_attempts, + status="queued", + available_at=datetime.now(timezone.utc), + ) + session.add(job) + session.flush() + session.add(JobOutbox(job_id=job.id, attempt=1)) + session.flush() + result = _attachment_out(attachment) + return result + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=AttachmentOut) +def get_attachment(attachment_id: uuid.UUID, request: Request): + user, _token = _current_user(request) + with SessionLocal() as session: + attachment = session.scalar( + select(Attachment) + .join(ChatMember, ChatMember.chat_id == Attachment.chat_id) + .where(Attachment.id == attachment_id, ChatMember.user_id == user.id) + ) + if attachment is None: + raise _api_error("not_found", "Объект не найден", 404) + return _attachment_out(attachment) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: uuid.UUID, + request: Request, + variant: str = "original", +): + user, _token = _current_user(request) + with SessionLocal() as session: + attachment = session.scalar( + select(Attachment) + .join(ChatMember, ChatMember.chat_id == Attachment.chat_id) + .where(Attachment.id == attachment_id, ChatMember.user_id == user.id) + ) + if attachment is None: + raise _api_error("not_found", "Объект не найден", 404) + if attachment.status != "ready": + raise _api_error("conflict", "Вложение ещё не готово", 409) + if variant == "original": + object_key = attachment.object_key + content_type = attachment.content_type + size_bytes = attachment.size_bytes + else: + selected = next((item for item in (attachment.variants or []) if item.get("name") == variant), None) + if selected is None: + raise _api_error("not_found", "Объект не найден", 404) + object_key = selected["object_key"] + content_type = selected["content_type"] + size_bytes = selected["size_bytes"] + + try: + stored = await run_in_threadpool(s3.get_object, Bucket=settings.s3_bucket, Key=object_key) + except ClientError as exc: + if _is_not_found_s3(exc): + logger.error("attachment_object_missing request_id=%s", request.state.request_id) + raise _api_error("unavailable", "Сервис временно недоступен", 503) from None + raise _api_error("unavailable", "Сервис временно недоступен", 503) from None + except BotoCoreError: + raise _api_error("unavailable", "Сервис временно недоступен", 503) from None + + body = stored["Body"] + + async def chunks(): + try: + while True: + chunk = await run_in_threadpool(body.read, 64 * 1024) + if not chunk: + break + yield chunk + finally: + await run_in_threadpool(body.close) + + return StreamingResponse( + chunks(), + media_type=content_type, + headers={ + "Content-Length": str(size_bytes), + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + }, + ) diff --git a/src/app/migrate.py b/src/app/migrate.py new file mode 100644 index 00000000..48c83839 --- /dev/null +++ b/src/app/migrate.py @@ -0,0 +1,36 @@ +from pathlib import Path + +from sqlalchemy import text + +from .db import engine + + +MIGRATIONS = Path(__file__).resolve().parents[2] / "migrations" + + +def migrate() -> None: + with engine.begin() as connection: + connection.exec_driver_sql("SELECT pg_advisory_xact_lock(2026102202)") + connection.exec_driver_sql( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version VARCHAR(128) PRIMARY KEY, applied_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP)" + ) + applied = set(connection.execute(text("SELECT version FROM schema_migrations")).scalars()) + for path in sorted(MIGRATIONS.glob("*.sql")): + version = path.stem + if version in applied: + continue + sql = path.read_text(encoding="utf-8") + for statement in sql.split(";"): + if statement.strip(): + connection.exec_driver_sql(statement) + connection.execute( + text("INSERT INTO schema_migrations(version) VALUES (:version)"), + {"version": version}, + ) + print(f"Applied migration {version}") + engine.dispose() + + +if __name__ == "__main__": + migrate() diff --git a/src/app/models.py b/src/app/models.py new file mode 100644 index 00000000..b73b489c --- /dev/null +++ b/src/app/models.py @@ -0,0 +1,209 @@ +import uuid +from datetime import datetime + +from sqlalchemy import CheckConstraint, DateTime, ForeignKey, ForeignKeyConstraint, Index, Integer, String, Text, UniqueConstraint, func +from sqlalchemy.dialects.postgresql import JSONB, UUID +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = "users" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + username: Mapped[str] = mapped_column(String(32), nullable=False, unique=True) + display_name: Mapped[str] = mapped_column(String(100), nullable=False) + password_hash: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + +class Chat(Base): + __tablename__ = "chats" + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + title: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + +class ChatMember(Base): + __tablename__ = "chat_members" + __table_args__ = ( + Index("chat_members_user_id_idx", "user_id", "chat_id"), + ) + + chat_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True + ) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="RESTRICT"), primary_key=True + ) + + +class Message(Base): + __tablename__ = "messages" + __table_args__ = ( + ForeignKeyConstraint( + ["chat_id", "sender_id"], + ["chat_members.chat_id", "chat_members.user_id"], + name="messages_sender_is_chat_member_fk", + ondelete="CASCADE", + ), + UniqueConstraint("id", "chat_id", name="messages_id_chat_id_uq"), + Index("messages_history_idx", "chat_id", "created_at", "id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False) + sender_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False) + text: Mapped[str] = mapped_column(String(4000), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + +class IdempotencyRecord(Base): + """Shared, transactional result cache for retried message POST requests.""" + + __tablename__ = "idempotency_records" + __table_args__ = ( + UniqueConstraint( + "user_id", "method", "path", "idempotency_key", + name="idempotency_records_scope_key_uq", + ), + Index("idempotency_records_expires_at_idx", "expires_at"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + user_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), nullable=False + ) + method: Mapped[str] = mapped_column(String(8), nullable=False) + path: Mapped[str] = mapped_column(String(200), nullable=False) + idempotency_key: Mapped[str] = mapped_column(String(128), nullable=False) + request_hash: Mapped[str] = mapped_column(String(64), nullable=False) + response_body: Mapped[dict] = mapped_column(JSONB, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) + + +class Attachment(Base): + __tablename__ = "attachments" + __table_args__ = ( + ForeignKeyConstraint( + ["chat_id", "owner_id"], + ["chat_members.chat_id", "chat_members.user_id"], + name="attachments_owner_is_chat_member_fk", + ondelete="CASCADE", + ), + UniqueConstraint("id", "chat_id", name="attachments_id_chat_id_uq"), + CheckConstraint("char_length(filename) BETWEEN 1 AND 255", name="attachments_filename_length_ck"), + CheckConstraint("content_type IN ('image/png', 'image/jpeg')", name="attachments_content_type_ck"), + CheckConstraint("size_bytes BETWEEN 1 AND 10485760", name="attachments_size_bytes_ck"), + CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="attachments_sha256_ck"), + CheckConstraint("status IN ('queued', 'processing', 'ready', 'failed')", name="attachments_status_ck"), + Index("attachments_chat_created_idx", "chat_id", "created_at", "id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False) + owner_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False) + filename: Mapped[str] = mapped_column(String(255), nullable=False) + content_type: Mapped[str] = mapped_column(String(32), nullable=False) + size_bytes: Mapped[int] = mapped_column(Integer, nullable=False) + sha256: Mapped[str] = mapped_column(String(64), nullable=False) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="ready") + variants: Mapped[list] = mapped_column(JSONB, nullable=False, default=list, server_default="[]") + error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + object_key: Mapped[str] = mapped_column(Text, nullable=False, unique=True) + staging_key: Mapped[str | None] = mapped_column(Text, nullable=True, unique=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + + +class AttachmentJob(Base): + """One logical image pipeline; broker deliveries are only wake-up signals.""" + + __tablename__ = "attachment_jobs" + __table_args__ = ( + UniqueConstraint("attachment_id", name="attachment_jobs_attachment_uq"), + CheckConstraint("attempt >= 1", name="attachment_jobs_attempt_ck"), + CheckConstraint("max_attempts BETWEEN 1 AND 20", name="attachment_jobs_max_attempts_ck"), + CheckConstraint("attempts_in_cycle BETWEEN 1 AND max_attempts", name="attachment_jobs_cycle_attempt_ck"), + CheckConstraint("status IN ('queued', 'processing', 'succeeded', 'failed')", name="attachment_jobs_status_ck"), + Index("attachment_jobs_due_idx", "status", "available_at"), + Index("attachment_jobs_lease_idx", "status", "lease_until"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + attachment_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), nullable=False + ) + pipeline_version: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempt: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + attempts_in_cycle: Mapped[int] = mapped_column(Integer, nullable=False, default=1) + max_attempts: Mapped[int] = mapped_column(Integer, nullable=False) + status: Mapped[str] = mapped_column(String(16), nullable=False, default="queued") + available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + last_error_code: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now(), onupdate=func.now()) + completed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class JobOutbox(Base): + """Transactional intent to publish a logical job attempt to Redis Streams.""" + + __tablename__ = "job_outbox" + __table_args__ = ( + UniqueConstraint("job_id", "attempt", name="job_outbox_job_attempt_uq"), + Index("job_outbox_pending_idx", "published_at", "created_at"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + job_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("attachment_jobs.id", ondelete="CASCADE"), nullable=False + ) + attempt: Mapped[int] = mapped_column(Integer, nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + + +class JobReplayAudit(Base): + """Operator initiated replay history; replay never creates a second job.""" + + __tablename__ = "job_replay_audit" + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + job_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("attachment_jobs.id", ondelete="CASCADE"), nullable=False + ) + previous_attempt: Mapped[int] = mapped_column(Integer, nullable=False) + new_attempt: Mapped[int] = mapped_column(Integer, nullable=False) + operator: Mapped[str] = mapped_column(String(100), nullable=False) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + + +class MessageAttachment(Base): + __tablename__ = "message_attachments" + __table_args__ = ( + ForeignKeyConstraint( + ["message_id", "chat_id"], + ["messages.id", "messages.chat_id"], + name="message_attachments_message_chat_fk", + ondelete="CASCADE", + ), + ForeignKeyConstraint( + ["attachment_id", "chat_id"], + ["attachments.id", "attachments.chat_id"], + name="message_attachments_attachment_chat_fk", + ondelete="CASCADE", + ), + UniqueConstraint("message_id", "position", name="message_attachments_position_uq"), + Index("message_attachments_attachment_idx", "attachment_id"), + ) + + message_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + attachment_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True) + chat_id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), nullable=False) + position: Mapped[int] = mapped_column(Integer, nullable=False) diff --git a/src/app/reconcile_s3_orphans.py b/src/app/reconcile_s3_orphans.py new file mode 100644 index 00000000..68d49eaf --- /dev/null +++ b/src/app/reconcile_s3_orphans.py @@ -0,0 +1,94 @@ +"""Dry-run-first cleanup of unknown S3 objects and expired staging sources.""" +import argparse +from datetime import datetime, timedelta, timezone +import os + +import boto3 +from botocore.config import Config +from sqlalchemy import select + +from .db import SessionLocal +from .models import Attachment +from .settings import _secret + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true", help="delete confirmed stale objects; default is dry-run") + args = parser.parse_args() + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/") + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + try: + grace_seconds = int(os.environ.get("ORPHAN_GRACE_SECONDS", "86400")) + staging_retention = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400")) + except ValueError as exc: + raise SystemExit("cleanup retention settings must be integers") from exc + if grace_seconds < 3600 or staging_retention < 3600: + raise SystemExit("cleanup grace/retention must be at least 3600 seconds") + + client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=_secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY"), + aws_secret_access_key=_secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY"), + config=Config(connect_timeout=2, read_timeout=5, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}), + ) + with SessionLocal() as session: + attachments = session.scalars(select(Attachment)).all() + known_keys = set() + tracked_staging = {} + for attachment in attachments: + known_keys.add(attachment.object_key) + for variant in attachment.variants or []: + if variant.get("object_key"): + known_keys.add(variant["object_key"]) + if attachment.staging_key: + tracked_staging[attachment.staging_key] = ( + attachment.id, + attachment.status, + attachment.updated_at, + ) + + now = datetime.now(timezone.utc) + orphan_cutoff = now - timedelta(seconds=grace_seconds) + staging_cutoff = now - timedelta(seconds=staging_retention) + scanned = stale = deleted = cleared_staging = 0 + paginator = client.get_paginator("list_objects_v2") + for prefix in ("originals/", "variants/", "staging/"): + for page in paginator.paginate(Bucket=bucket, Prefix=prefix): + for obj in page.get("Contents", []): + scanned += 1 + key = obj["Key"] + modified = obj.get("LastModified") + tracked = tracked_staging.get(key) if prefix == "staging/" else None + if tracked: + _attachment_id, status, updated_at = tracked + safe_cleanup = status == "ready" or (status == "failed" and updated_at <= staging_cutoff) + if not safe_cleanup: + continue + elif key in known_keys or modified is None or modified > orphan_cutoff: + continue + stale += 1 + if args.apply: + client.delete_object(Bucket=bucket, Key=key) + deleted += 1 + if tracked: + attachment_id, _status, _updated_at = tracked + with SessionLocal.begin() as session: + attachment = session.scalar( + select(Attachment).where(Attachment.id == attachment_id).with_for_update() + ) + if attachment and attachment.staging_key == key: + attachment.staging_key = None + cleared_staging += 1 + print( + f"orphan_reconciliation mode={'apply' if args.apply else 'dry-run'} " + f"grace_seconds={grace_seconds} staging_retention_seconds={staging_retention} " + f"scanned={scanned} stale={stale} deleted={deleted} staging_refs_cleared={cleared_staging}" + ) + + +if __name__ == "__main__": + main() diff --git a/src/app/replay_job.py b/src/app/replay_job.py new file mode 100644 index 00000000..8b319e5a --- /dev/null +++ b/src/app/replay_job.py @@ -0,0 +1,53 @@ +"""Explicit local operator replay for one failed attachment pipeline.""" +import argparse +from datetime import datetime, timezone +import getpass +import uuid + +from sqlalchemy import select + +from .db import SessionLocal +from .models import Attachment, AttachmentJob, JobOutbox, JobReplayAudit + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("attachment_id", help="UUID of a failed attachment") + args = parser.parse_args() + try: + attachment_id = uuid.UUID(args.attachment_id) + except ValueError: + raise SystemExit("attachment_id must be a UUID") from None + + with SessionLocal.begin() as session: + attachment = session.scalar(select(Attachment).where(Attachment.id == attachment_id).with_for_update()) + if attachment is None: + raise SystemExit("attachment not found") + job = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == attachment_id).with_for_update()) + if job is None or job.status != "failed": + raise SystemExit("only a failed attachment can be replayed") + if not attachment.staging_key: + raise SystemExit("staging source is absent; replay is unsafe") + previous_attempt = job.attempt + job.attempt += 1 + job.attempts_in_cycle = 1 + job.status = "queued" + job.available_at = datetime.now(timezone.utc) + job.lease_until = None + job.last_error_code = None + job.completed_at = None + attachment.status = "queued" + attachment.error_code = None + attachment.updated_at = datetime.now(timezone.utc) + session.add(JobReplayAudit( + job_id=job.id, + previous_attempt=previous_attempt, + new_attempt=job.attempt, + operator=getpass.getuser()[:100] or "local-operator", + )) + session.add(JobOutbox(job_id=job.id, attempt=job.attempt)) + print(f"PASS: replay queued for attachment {attachment_id}; attempt {previous_attempt + 1} recorded.") + + +if __name__ == "__main__": + main() diff --git a/src/app/s3_bootstrap.py b/src/app/s3_bootstrap.py new file mode 100644 index 00000000..6feaf024 --- /dev/null +++ b/src/app/s3_bootstrap.py @@ -0,0 +1,196 @@ +"""Idempotently create the private bucket and a bucket-scoped API service account.""" +import json +import os +import time +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +import boto3 +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.config import Config +from botocore.credentials import Credentials +from botocore.exceptions import ClientError + +from .settings import _secret + + +def _admin_request(method: str, path: str, payload: dict | None = None) -> tuple[int, str | None]: + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/") + region = os.environ.get("S3_REGION", "us-east-1") + access = _secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY") + secret = _secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY") + body = json.dumps(payload, separators=(",", ":"), sort_keys=True).encode("utf-8") if payload is not None else None + url = endpoint + path + # RustFS 1.0.5's admin API may reject a signed GET with an empty body + # when Content-Length is omitted by urllib. Sign and send the explicit + # zero length so the request is unambiguous to its SigV4 verifier. + headers = {"Content-Type": "application/json"} if body is not None else {"Content-Length": "0"} + signed = AWSRequest(method=method, url=url, data=body, headers=headers) + S3SigV4Auth(Credentials(access, secret), "s3", region).add_auth(signed) + request = Request(url, data=body, headers=dict(signed.headers), method=method) + try: + with urlopen(request, timeout=3) as response: + response.read(1024 * 1024) + return response.status, None + except HTTPError as exc: + raw = exc.read(1024 * 1024) + detail = "" + try: + document = json.loads(raw) + except (json.JSONDecodeError, UnicodeDecodeError): + document = None + if isinstance(document, dict): + detail = "; ".join( + str(document[key]) + for key in ("Code", "code", "Message", "message", "error") + if document.get(key) + ) + if not detail: + detail = raw.decode("utf-8", errors="replace") + detail = " ".join(detail.split())[:240] + # RustFS can echo request details in an error response. Redact every + # credential before including a short diagnostic in the container log. + for file_variable, value_variable in ( + ("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY"), + ("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY"), + ("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), + ("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), + ): + try: + credential = _secret(file_variable, value_variable) + except RuntimeError: + continue + detail = detail.replace(credential, "[redacted]") + detail = detail.replace(quote(credential, safe=""), "[redacted]") + return exc.code, detail or None + + +def _failure(action: str, status: int, detail: str | None) -> RuntimeError: + suffix = f": {detail}" if detail else "" + return RuntimeError(f"RustFS API service-account {action} failed with HTTP {status}{suffix}") + + +def _create_api_service_account(endpoint: str, region: str, bucket: str) -> None: + access_key = _secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") + secret_key = _secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "BucketLocation", + "Effect": "Allow", + "Action": ["s3:GetBucketLocation"], + "Resource": [f"arn:aws:s3:::{bucket}"], + }, + { + "Sid": "AttachmentObjectsOnly", + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": [f"arn:aws:s3:::{bucket}/*"], + }, + ], + } + encoded_key = quote(access_key, safe="") + info_path = f"/rustfs/admin/v3/info-service-account?accessKey={encoded_key}" + status, detail = _admin_request("GET", info_path) + if status == 404: + status, detail = _admin_request( + "PUT", + "/rustfs/admin/v3/add-service-account", + { + "accessKey": access_key, + "secretKey": secret_key, + "name": "messenger-api", + "description": "Limited access to the private attachment bucket", + "policy": policy, + }, + ) + if status not in (200, 201, 204): + raise _failure("creation", status, detail) + elif status != 200: + raise _failure("lookup", status, detail) + else: + update_path = f"/rustfs/admin/v3/update-service-account?accessKey={encoded_key}" + status, detail = _admin_request( + "POST", + update_path, + {"policy": policy, "name": "messenger-api", "description": "Limited access to the private attachment bucket"}, + ) + if status not in (200, 204): + raise _failure("policy update", status, detail) + + client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}), + ) + client.get_bucket_location(Bucket=bucket) + try: + response = client.list_buckets() + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code not in {"AccessDenied", "403"}: + raise RuntimeError("RustFS API credentials were not explicitly denied ListAllMyBuckets") from None + else: + # RustFS follows MinIO-compatible filtered listing: ListBuckets may + # succeed while returning only buckets this identity can access. + # Treat unrelated visible buckets as a leak, not the filtered result. + visible = {item.get("Name") for item in response.get("Buckets", [])} + if visible - {bucket}: + raise RuntimeError("RustFS API credentials can see buckets outside the attachment bucket") + + +def main() -> None: + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").rstrip("/") + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ.get("S3_BUCKET", "messenger-attachments") + root_access = _secret("S3_ROOT_ACCESS_KEY_FILE", "S3_ROOT_ACCESS_KEY") + root_secret = _secret("S3_ROOT_SECRET_ACCESS_KEY_FILE", "S3_ROOT_SECRET_ACCESS_KEY") + root = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=root_access, + aws_secret_access_key=root_secret, + config=Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}, s3={"addressing_style": "path"}), + ) + last_error = None + for _ in range(60): + try: + try: + root.head_bucket(Bucket=bucket) + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code in {"404", "NoSuchBucket", "NotFound"}: + root.create_bucket(Bucket=bucket) + else: + raise + break + except Exception as exc: + last_error = exc + time.sleep(1) + else: + raise RuntimeError("RustFS did not become ready for bucket bootstrap") from last_error + + anonymous_url = f"{endpoint}/{quote(bucket, safe='')}" + try: + with urlopen(Request(anonymous_url, method="GET"), timeout=2) as response: + if response.status < 400: + raise RuntimeError("The attachment bucket allows anonymous access") + except HTTPError as exc: + if exc.code != 403: + raise RuntimeError(f"Anonymous bucket check returned unexpected HTTP {exc.code}") from None + except URLError as exc: + raise RuntimeError("Could not verify private bucket access") from exc + + _create_api_service_account(endpoint, region, bucket) + print("PASS: private attachment bucket and least-privilege API credentials are ready.") + + +if __name__ == "__main__": + main() diff --git a/src/app/schemas.py b/src/app/schemas.py new file mode 100644 index 00000000..4fb9ea6d --- /dev/null +++ b/src/app/schemas.py @@ -0,0 +1,123 @@ +from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator +import uuid + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class CreateUser(StrictModel): + username: StrictStr = Field(min_length=3, max_length=32, pattern=r"^[a-z0-9_]{3,32}$") + display_name: StrictStr = Field(min_length=1, max_length=100) + password: StrictStr = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def display_name_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("display_name must contain a non-whitespace character") + return value + + +class UserOut(BaseModel): + id: str + username: str + display_name: str + created_at: str + + +class CreateChat(StrictModel): + title: StrictStr = Field(min_length=1, max_length=100) + member_ids: list[StrictStr] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title must contain a non-whitespace character") + return value + + @field_validator("member_ids") + @classmethod + def members_are_unique_uuids(cls, values: list[str]) -> list[str]: + import uuid + + try: + normalized = [str(uuid.UUID(value)) for value in values] + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError("member_ids must contain UUIDs") from exc + if len(normalized) != len(set(normalized)): + raise ValueError("member_ids must be unique") + return normalized + + +class ChatOut(BaseModel): + id: str + title: str + member_ids: list[str] + created_at: str + + +class ChatPage(BaseModel): + items: list[ChatOut] + next_cursor: str | None + + +class CreateMessage(StrictModel): + text: StrictStr = Field(max_length=4000) + attachment_ids: list[StrictStr] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def attachment_ids_are_unique_uuids(cls, values: list[str]) -> list[str]: + try: + normalized = [str(uuid.UUID(value)) for value in values] + except (ValueError, AttributeError, TypeError) as exc: + raise ValueError("attachment_ids must contain UUIDs") from exc + if len(normalized) != len(set(normalized)): + raise ValueError("attachment_ids must be unique") + return normalized + + +class MessageOut(BaseModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: list[str] + + +class AttachmentOut(BaseModel): + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + status: str = "ready" + variants: list[dict] = Field(default_factory=list) + error_code: str | None = None + created_at: str + updated_at: str + + +class MessagePage(BaseModel): + items: list[MessageOut] + next_cursor: str | None + + +class SessionOut(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: UserOut + + +class Health(BaseModel): + status: str = "ok" + + +class ErrorBody(BaseModel): + error: dict[str, str] diff --git a/src/app/settings.py b/src/app/settings.py new file mode 100644 index 00000000..ff5dc81e --- /dev/null +++ b/src/app/settings.py @@ -0,0 +1,105 @@ +from dataclasses import dataclass +import os +from pathlib import Path + + +def _secret(file_variable: str, value_variable: str) -> str: + file_path = os.environ.get(file_variable) + if file_path: + value = Path(file_path).read_text(encoding="utf-8").strip() + else: + value = os.environ.get(value_variable, "").strip() + if not value: + raise RuntimeError(f"Required setting {file_variable} is missing") + return value + + +def load_database_url() -> str: + return _secret("DATABASE_URL_FILE", "DATABASE_URL") + + +@dataclass(frozen=True) +class Settings: + cache_url: str + broker_url: str + broker_stream: str + session_ttl_seconds: int + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + worker_concurrency: int + job_max_attempts: int + job_timeout_seconds: int + job_lease_seconds: int + staging_retention_seconds: int + + +def load_settings(*, require_cache: bool = True, require_broker: bool = True, require_s3: bool = True) -> Settings: + try: + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + except ValueError as exc: + raise RuntimeError("SESSION_TTL_SECONDS must be an integer") from exc + if ttl < 1 or ttl > 31_536_000: + raise RuntimeError("SESSION_TTL_SECONDS must be between 1 and 31536000") + try: + max_upload_bytes = int(os.environ.get("MAX_UPLOAD_BYTES", "10485760")) + max_image_pixels = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")) + worker_concurrency = int(os.environ.get("WORKER_CONCURRENCY", "1")) + job_max_attempts = int(os.environ.get("JOB_MAX_ATTEMPTS", "5")) + job_timeout_seconds = int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")) + job_lease_seconds = int(os.environ.get("JOB_LEASE_SECONDS", "180")) + staging_retention_seconds = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400")) + except ValueError as exc: + raise RuntimeError("upload, worker and job settings must be integers") from exc + if not 1 <= max_upload_bytes <= 10_485_760: + raise RuntimeError("MAX_UPLOAD_BYTES must be between 1 and 10485760") + if not 1 <= max_image_pixels <= 40_000_000: + raise RuntimeError("MAX_IMAGE_PIXELS must be between 1 and 40000000") + endpoint = os.environ.get("S3_ENDPOINT", "http://s3:9000").strip() + region = os.environ.get("S3_REGION", "us-east-1").strip() + bucket = os.environ.get("S3_BUCKET", "messenger-attachments").strip() + if not endpoint.startswith("http://") and not endpoint.startswith("https://"): + raise RuntimeError("S3_ENDPOINT must be an HTTP(S) URL") + if not region or not bucket: + raise RuntimeError("S3_REGION and S3_BUCKET are required") + if worker_concurrency != 1: + raise RuntimeError("WORKER_CONCURRENCY is fixed at 1 per worker; scale worker replicas for more throughput") + if not 1 <= job_max_attempts <= 20: + raise RuntimeError("JOB_MAX_ATTEMPTS must be between 1 and 20") + if not 5 <= job_timeout_seconds <= 600 or job_lease_seconds <= job_timeout_seconds: + raise RuntimeError("JOB_LEASE_SECONDS must exceed JOB_TIMEOUT_SECONDS (5..600)") + if staging_retention_seconds < 3600: + raise RuntimeError("STAGING_RETENTION_SECONDS must be at least 3600") + cache_file = os.environ.get("CACHE_URL_FILE") + cache_value = os.environ.get("CACHE_URL", "").strip() + cache_url = _secret("CACHE_URL_FILE", "CACHE_URL") if require_cache or cache_file or cache_value else "" + # API processes do not publish directly; falling back keeps them independent + # from broker credentials while the dispatcher and worker use the explicit URL. + broker_file = os.environ.get("BROKER_URL_FILE") + broker_url = Path(broker_file).read_text(encoding="utf-8").strip() if broker_file else os.environ.get("BROKER_URL", cache_url) + if require_broker and not broker_url: + raise RuntimeError("BROKER_URL_FILE is empty") + s3_access_key_id = _secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") if require_s3 else "" + s3_secret_access_key = _secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") if require_s3 else "" + return Settings( + cache_url=cache_url, + broker_url=broker_url, + broker_stream=os.environ.get("BROKER_STREAM", "messenger:attachment-jobs:v1"), + session_ttl_seconds=ttl, + s3_endpoint=endpoint.rstrip("/"), + s3_region=region, + s3_bucket=bucket, + s3_access_key_id=s3_access_key_id, + s3_secret_access_key=s3_secret_access_key, + max_upload_bytes=max_upload_bytes, + max_image_pixels=max_image_pixels, + worker_concurrency=worker_concurrency, + job_max_attempts=job_max_attempts, + job_timeout_seconds=job_timeout_seconds, + job_lease_seconds=job_lease_seconds, + staging_retention_seconds=staging_retention_seconds, + ) diff --git a/src/app/storage.py b/src/app/storage.py new file mode 100644 index 00000000..9bb467be --- /dev/null +++ b/src/app/storage.py @@ -0,0 +1,20 @@ +from botocore.config import Config +import boto3 + +from .settings import load_settings + + +settings = load_settings(require_cache=False, require_broker=False, require_s3=True) +s3 = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + region_name=settings.s3_region, + aws_access_key_id=settings.s3_access_key_id, + aws_secret_access_key=settings.s3_secret_access_key, + config=Config( + connect_timeout=1, + read_timeout=2, + retries={"total_max_attempts": 1, "mode": "standard"}, + s3={"addressing_style": "path"}, + ), +) diff --git a/src/app/worker.py b/src/app/worker.py new file mode 100644 index 00000000..e2fca81a --- /dev/null +++ b/src/app/worker.py @@ -0,0 +1,343 @@ +"""At-least-once Redis Streams consumer for the versioned image pipeline.""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import json +import argparse +import logging +import os +import random +import signal +import socket +import time +import uuid + +from botocore.exceptions import BotoCoreError, ClientError +from redis import Redis +from redis.exceptions import ResponseError +from sqlalchemy import select + +from .db import SessionLocal +from .image_pipeline import PermanentImageError, process_image +from .models import Attachment, AttachmentJob, JobOutbox +from .settings import load_settings +from .storage import s3 + + +settings = load_settings(require_cache=False, require_s3=True) +STREAM = settings.broker_stream +GROUP = "attachment-workers-v1" +DEAD_STREAM = STREAM + ":dead" +CONSUMER = socket.gethostname() + "-" + uuid.uuid4().hex[:8] +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +logger = logging.getLogger("messenger.worker") +broker = Redis.from_url(settings.broker_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=3) + + +class RetryableJobError(Exception): + def __init__(self, error_code: str): + self.error_code = error_code + + +def _deadline_handler(_signum, _frame) -> None: + raise RetryableJobError("processing_timeout") + + +def _process_with_deadline(fields: dict, message_id: str) -> None: + # The worker is a single-threaded process. A wall-clock timer makes the + # configured deadline effective even if an image decoder or storage call + # takes longer than expected; S3 calls also have short socket timeouts. + if not hasattr(signal, "setitimer"): + _process(fields, message_id) + return + previous = signal.signal(signal.SIGALRM, _deadline_handler) + signal.setitimer(signal.ITIMER_REAL, settings.job_timeout_seconds) + try: + _process(fields, message_id) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def _ensure_group() -> None: + try: + broker.xgroup_create(STREAM, GROUP, id="0-0", mkstream=True) + except ResponseError as exc: + if "BUSYGROUP" not in str(exc): + raise + + +def _ack(message_id: str) -> None: + broker.xack(STREAM, GROUP, message_id) + + +def _dead_letter(message_id: str, fields: dict, error_code: str) -> None: + broker.xadd(DEAD_STREAM, { + "source_id": message_id, + "error_code": error_code, + "payload": json.dumps(fields, sort_keys=True, separators=(",", ":"))[:2048], + }) + _ack(message_id) + + +def _parse_event(fields: dict) -> tuple[uuid.UUID, uuid.UUID, int, int]: + if fields.get("schema_version") != "1" or fields.get("type") != "attachment.process.v1": + raise ValueError("unsupported_job_schema") + job_id = uuid.UUID(fields["job_id"]) + attachment_id = uuid.UUID(fields["attachment_id"]) + attempt = int(fields["attempt"]) + pipeline = int(fields["pipeline_version"]) + if attempt < 1 or pipeline != 1: + raise ValueError("invalid_job_version") + return job_id, attachment_id, attempt, pipeline + + +def _claim(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int, pipeline: int) -> tuple[str, str | None]: + now = datetime.now(timezone.utc) + with SessionLocal.begin() as session: + job = session.scalar(select(AttachmentJob).where(AttachmentJob.id == job_id).with_for_update()) + attachment = session.scalar(select(Attachment).where(Attachment.id == attachment_id).with_for_update()) + if job is None or attachment is None or job.attachment_id != attachment_id: + return "ack", None + if job.pipeline_version != pipeline or job.attempt != attempt: + return "ack", None + if job.status == "succeeded": + return "finished", attachment.staging_key + if job.status == "failed": + return "ack", None + if job.status == "processing" and job.lease_until and job.lease_until > now: + return "ack", None # duplicate outbox publish while another worker owns the lease + if job.available_at > now: + return "wait", None + if not attachment.staging_key: + job.status = "failed" + job.last_error_code = "staging_missing" + job.completed_at = now + attachment.status = "failed" + attachment.error_code = "staging_missing" + return "ack", None + job.status = "processing" + job.lease_until = now + timedelta(seconds=settings.job_lease_seconds) + job.updated_at = now + attachment.status = "processing" + attachment.error_code = None + return "process", attachment.staging_key + + +def _retry_delay(attempts_in_cycle: int) -> float: + ceiling = min(60.0, 2.0 ** min(attempts_in_cycle, 6)) + return random.uniform(ceiling / 2, ceiling) + + +def _record_failure(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int, error_code: str, permanent: bool) -> None: + now = datetime.now(timezone.utc) + with SessionLocal.begin() as session: + job = session.scalar(select(AttachmentJob).where(AttachmentJob.id == job_id).with_for_update()) + attachment = session.scalar(select(Attachment).where(Attachment.id == attachment_id).with_for_update()) + if job is None or attachment is None or job.attempt != attempt or job.status != "processing": + return + job.last_error_code = error_code + job.lease_until = None + job.updated_at = now + if permanent: + job.status = "failed" + job.completed_at = now + attachment.status = "failed" + attachment.error_code = error_code + elif job.attempts_in_cycle < job.max_attempts: + job.attempt += 1 + job.attempts_in_cycle += 1 + job.status = "queued" + job.available_at = now + timedelta(seconds=_retry_delay(job.attempts_in_cycle)) + attachment.status = "queued" + attachment.error_code = None + session.add(JobOutbox(job_id=job.id, attempt=job.attempt)) + else: + job.status = "failed" + job.last_error_code = "retry_exhausted" + job.completed_at = now + attachment.status = "failed" + attachment.error_code = "retry_exhausted" + + +def _cleanup_staging(attachment_id: uuid.UUID, staging_key: str | None) -> None: + if not staging_key: + return + try: + s3.delete_object(Bucket=settings.s3_bucket, Key=staging_key) + except (ClientError, BotoCoreError) as exc: + logger.warning("staging_cleanup_failed attachment_id=%s type=%s", attachment_id, type(exc).__name__) + return + with SessionLocal.begin() as session: + attachment = session.scalar(select(Attachment).where(Attachment.id == attachment_id).with_for_update()) + if attachment and attachment.status == "ready" and attachment.staging_key == staging_key: + attachment.staging_key = None + + +def _process(fields: dict, message_id: str) -> None: + started = time.monotonic() + try: + job_id, attachment_id, attempt, pipeline = _parse_event(fields) + except (ValueError, KeyError, TypeError): + _dead_letter(message_id, fields, "invalid_job_event") + return + + action, staging_key = _claim(job_id, attachment_id, attempt, pipeline) + if action == "ack": + _ack(message_id) + return + if action == "wait": + return + if action == "finished": + _cleanup_staging(attachment_id, staging_key) + _ack(message_id) + return + if os.environ.get("WORKER_FAILPOINT_PERMANENT_ATTACHMENT_ID") == str(attachment_id): + marker = "lab5:failpoint:permanent:" + str(attachment_id) + if broker.set(marker, "used", nx=True, ex=86400): + _record_failure(job_id, attachment_id, attempt, "test_permanent_failure", permanent=True) + _ack(message_id) + return + + with SessionLocal() as session: + attachment = session.get(Attachment, attachment_id) + if attachment is None: + _ack(message_id) + return + content_type = attachment.content_type + size_expected = attachment.size_bytes + digest_expected = attachment.sha256 + object_key = attachment.object_key + filename = attachment.filename + try: + stored = s3.get_object(Bucket=settings.s3_bucket, Key=staging_key) + original = stored["Body"].read(settings.max_upload_bytes + 1) + stored["Body"].close() + if len(original) != size_expected or len(original) > settings.max_upload_bytes: + raise PermanentImageError("invalid_image") + import hashlib + if hashlib.sha256(original).hexdigest() != digest_expected: + raise PermanentImageError("staging_checksum_mismatch") + variants = process_image(original, content_type, settings.max_image_pixels, attachment_id.hex, pipeline) + if time.monotonic() - started > settings.job_timeout_seconds: + raise RetryableJobError("processing_timeout") + s3.put_object( + Bucket=settings.s3_bucket, + Key=object_key, + Body=original, + ContentLength=len(original), + ContentType=content_type, + Metadata={"sha256": digest_expected, "pipeline-version": str(pipeline)}, + ) + public_variants = [] + for item in variants: + body = item.pop("body") + s3.put_object( + Bucket=settings.s3_bucket, + Key=item["object_key"], + Body=body, + ContentLength=len(body), + ContentType=item["content_type"], + Metadata={"sha256": item["sha256"], "pipeline-version": str(pipeline)}, + ) + public_variants.append(item) + if time.monotonic() - started > settings.job_timeout_seconds: + raise RetryableJobError("processing_timeout") + except PermanentImageError as exc: + _record_failure(job_id, attachment_id, attempt, str(exc) or "invalid_image", permanent=True) + _ack(message_id) + return + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code in {"AccessDenied", "InvalidAccessKeyId", "SignatureDoesNotMatch", "NoSuchBucket"}: + _record_failure(job_id, attachment_id, attempt, "storage_configuration", permanent=True) + else: + _record_failure(job_id, attachment_id, attempt, "storage_unavailable", permanent=False) + _ack(message_id) + return + except RetryableJobError as exc: + _record_failure(job_id, attachment_id, attempt, exc.error_code, permanent=False) + _ack(message_id) + return + except BotoCoreError: + _record_failure(job_id, attachment_id, attempt, "processing_unavailable", permanent=False) + _ack(message_id) + return + except Exception as exc: + logger.warning("job_attempt_failed job_id=%s type=%s", job_id, type(exc).__name__) + _record_failure(job_id, attachment_id, attempt, "processing_unavailable", permanent=False) + _ack(message_id) + return + + now = datetime.now(timezone.utc) + with SessionLocal.begin() as session: + job = session.scalar(select(AttachmentJob).where(AttachmentJob.id == job_id).with_for_update()) + attachment = session.scalar(select(Attachment).where(Attachment.id == attachment_id).with_for_update()) + if job is not None and attachment is not None and job.status == "processing" and job.attempt == attempt: + attachment.status = "ready" + attachment.variants = public_variants + attachment.error_code = None + attachment.updated_at = now + job.status = "succeeded" + job.lease_until = None + job.last_error_code = None + job.completed_at = now + job.updated_at = now + # Optional, isolated fault-injection point used only by the defense scenario. + # It exits after durable result commit and before staging cleanup or XACK. + if os.environ.get("WORKER_FAILPOINT_AFTER_COMMIT_ATTACHMENT_ID") == str(attachment_id): + marker = "lab5:failpoint:after-commit:" + str(attachment_id) + if broker.set(marker, "used", nx=True, ex=86400): + logger.warning("test_failpoint_after_result_commit attachment_id=%s", attachment_id) + raise SystemExit(75) + _cleanup_staging(attachment_id, staging_key) + _ack(message_id) + logger.info("attachment_ready attachment_id=%s pipeline_version=%s elapsed_seconds=%.3f", attachment_id, pipeline, time.monotonic() - started) + + +def _next_message(min_idle_ms: int | None = None): + reclaimed = broker.xautoclaim( + STREAM, GROUP, CONSUMER, + min_idle_time=settings.job_lease_seconds * 1000 if min_idle_ms is None else min_idle_ms, + start_id="0-0", + count=1, + ) + if len(reclaimed) > 1 and reclaimed[1]: + return reclaimed[1][0] + batches = broker.xreadgroup(GROUP, CONSUMER, {STREAM: ">"}, count=1, block=1000) + if batches and batches[0][1]: + return batches[0][1][0] + return None + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--once", action="store_true", help="process one available or reclaimable event and exit") + parser.add_argument("--idle-ms", type=int, default=None, help="test-only reclaim idle override for --once") + args = parser.parse_args() + if args.idle_ms is not None and (not args.once or args.idle_ms < 0): + raise SystemExit("--idle-ms requires --once and must be non-negative") + _ensure_group() + logger.info("attachment_worker_started consumer=%s concurrency=%s lease_seconds=%s", CONSUMER, settings.worker_concurrency, settings.job_lease_seconds) + if args.once: + while True: + message = _next_message(args.idle_ms) + if message is not None: + _process_with_deadline(message[1], message[0]) + logger.info("worker_once_complete message_id=%s", message[0]) + return + time.sleep(0.25) + while True: + try: + message = _next_message() + if message is not None: + _process_with_deadline(message[1], message[0]) + except SystemExit: + raise + except Exception as exc: + logger.warning("worker_loop_failed type=%s", type(exc).__name__) + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/broker_persistence.py b/student-tests/broker_persistence.py new file mode 100644 index 00000000..83d53a76 --- /dev/null +++ b/student-tests/broker_persistence.py @@ -0,0 +1,93 @@ +"""Verify a queued stream event survives broker restart with its volume.""" +from __future__ import annotations + +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Failure, arguments, run, Client # noqa: E402 + + +STREAM = "messenger:attachment-jobs:v1" + + +def compose(*args: str, expected: int = 0, timeout: int = 90) -> subprocess.CompletedProcess: + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != expected: + operation = args[0] if args else "compose" + raise Failure("Docker Compose operation failed: " + operation + " (exit " + str(result.returncode) + ")") + return result + + +def stream_contains(attachment_id: str) -> bool: + result = subprocess.run( + [ + "docker", "compose", "--env-file", ".env", "exec", "-T", "broker", "sh", "-ec", + "REDISCLI_AUTH=\"$(cat /run/secrets/broker_password)\" redis-cli --raw XREVRANGE " + + STREAM + " + - COUNT 500", + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=10, + ) + return result.returncode == 0 and attachment_id in result.stdout + + +def main() -> None: + args = arguments(__doc__, lambda parser: parser.add_argument("--output", default="evidence/broker-persistence.json")) + client = Client(args.base_url, args.ca_file) + compose("stop", "worker") + compose("up", "-d", "dispatcher") + try: + alice, _bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + client.message(alice, chat, "Attachment queued for broker persistence check", attachments=[attachment["id"]]) + client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", 409, actor=alice) + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and not stream_contains(attachment["id"]): + time.sleep(0.5) + if not stream_contains(attachment["id"]): + raise Failure("dispatcher did not publish the queued job to Redis Streams") + compose("stop", "dispatcher") + compose("restart", "broker") + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and not stream_contains(attachment["id"]): + time.sleep(1) + if not stream_contains(attachment["id"]): + raise Failure("published stream event was lost when broker restarted with its volume") + + compose("up", "-d", "dispatcher", "worker") + ready = client.wait_ready(alice, attachment, wait_seconds=90) + if ready["status"] != "ready": + raise Failure("worker did not complete the preserved broker event") + result = { + "result": "PASS", + "observed_at_utc": datetime.now(timezone.utc).isoformat(), + "attachment_id": attachment["id"], + "status_after_broker_restart": ready["status"], + "stream_entry_survived_volume_restart": True, + "worker_completed_same_attachment": True, + } + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, separators=(",", ":"))) + finally: + compose("up", "-d", "dispatcher", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/grade4_pipeline.py b/student-tests/grade4_pipeline.py new file mode 100644 index 00000000..7f55e732 --- /dev/null +++ b/student-tests/grade4_pipeline.py @@ -0,0 +1,147 @@ +"""Black-box grade-4 checks for variants, source preservation, and upload limits.""" +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +import json +from pathlib import Path +import struct +import sys +import zlib + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Failure, arguments, run, Client # noqa: E402 + + +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 + + +def _synthetic_jpeg_with_exif() -> bytes: + image = Image.new("RGB", (640, 480), (32, 128, 210)) + exif = Image.Exif() + exif[0x010E] = "synthetic test metadata" + output = BytesIO() + image.save(output, format="JPEG", quality=90, exif=exif) + image.close() + return output.getvalue() + + +def _oversized_png_header() -> bytes: + source = bytearray((ROOT / "tests/fixtures/sample.png").read_bytes()) + # Change only the valid IHDR width/height and CRC. The API must reject the + # dimensions before it tries to decode or verify the intentionally tiny body. + source[16:24] = struct.pack(">II", 7000, 7000) + crc = zlib.crc32(b"IHDR" + source[16:29]) & 0xFFFFFFFF + source[29:33] = struct.pack(">I", crc) + return bytes(source[:33]) + + +def main() -> None: + args = arguments(__doc__, lambda parser: parser.add_argument("--output", default="evidence/grade4-pipeline.json")) + client = Client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = client.bootstrap() + + source = _synthetic_jpeg_with_exif() + with Image.open(BytesIO(source)) as original_image: + if not original_image.getexif(): + raise Failure("synthetic JPEG did not contain EXIF metadata") + + upload = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=synthetic.jpg", + 202, + actor=alice, + raw=source, + headers={"Content-Type": "image/jpeg"}, + schema="Attachment", + )["json"] + if upload["status"] != "queued" or upload["sha256"] != sha256(source).hexdigest(): + raise Failure("upload was not durably accepted as queued with the source checksum") + client.message(alice, chat, "Grade 4 pipeline fixture", attachments=[upload["id"]]) + ready = client.wait_ready(alice, upload) + + by_name = {item["name"]: item for item in ready["variants"]} + expected = {"thumbnail", "crop", "optimized"} + if set(by_name) != expected or len(ready["variants"]) != len(expected): + raise Failure("ready attachment does not contain exactly the three required variants") + + expected_dimensions = {"thumbnail": (256, 192), "crop": (128, 128), "optimized": (640, 480)} + expected_types = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} + outputs = {} + original = client.call( + "GET", "/api/v1/attachments/" + upload["id"] + "/content?variant=original", + actor=alice, + ) + if original["bytes"] != source or sha256(original["bytes"]).hexdigest() != upload["sha256"]: + raise Failure("the original was changed by image processing") + + for name in sorted(expected): + metadata = by_name[name] + response = client.call( + "GET", "/api/v1/attachments/" + upload["id"] + "/content?variant=" + name, + actor=alice, + ) + if response["headers"].get_content_type() != expected_types[name]: + raise Failure(name + " has the wrong content type") + if len(response["bytes"]) != metadata["size_bytes"] or sha256(response["bytes"]).hexdigest() != metadata["sha256"]: + raise Failure(name + " size/checksum does not match metadata") + with Image.open(BytesIO(response["bytes"])) as image: + dimensions = image.size + if image.getexif(): + raise Failure(name + " retained EXIF metadata") + if dimensions != expected_dimensions[name] or (dimensions[0] > 1280 or dimensions[1] > 1280): + raise Failure(name + " has unexpected dimensions") + if (metadata["width"], metadata["height"]) != dimensions: + raise Failure(name + " dimensions do not match metadata") + outputs[name] = { + "content_type": expected_types[name], + "dimensions": list(dimensions), + "size_bytes": len(response["bytes"]), + "sha256": metadata["sha256"], + "exif_removed": True, + } + + corrupt = client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=corrupt.png", 415, + actor=alice, raw=b"not an image", headers={"Content-Type": "image/png"}, + ) + wrong_mime = client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=spoofed.jpg", 415, + actor=alice, raw=(ROOT / "tests/fixtures/sample.png").read_bytes(), + headers={"Content-Type": "image/jpeg"}, + ) + too_many_pixels = client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=too-large.png", 413, + actor=alice, raw=_oversized_png_header(), headers={"Content-Type": "image/png"}, + ) + too_many_bytes = client.call( + "POST", "/api/v1/chats/" + chat["id"] + "/attachments?filename=too-many-bytes.png", 413, + actor=alice, raw=b"x" * (MAX_UPLOAD_BYTES + 1), headers={"Content-Type": "image/png"}, + ) + + result = { + "result": "PASS", + "accepted_status": upload["status"], + "final_status": ready["status"], + "source_bytes": len(source), + "source_sha256": upload["sha256"], + "original_unchanged": True, + "variants": outputs, + "rejected_inputs": { + "corrupt_image": corrupt["status"], + "mime_mismatch": wrong_mime["status"], + "pixel_limit": too_many_pixels["status"], + "byte_limit": too_many_bytes["status"], + }, + } + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, separators=(",", ":"))) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/outbox_recovery.py b/student-tests/outbox_recovery.py new file mode 100644 index 00000000..f8426615 --- /dev/null +++ b/student-tests/outbox_recovery.py @@ -0,0 +1,102 @@ +"""Exercise a committed upload across API restart and a pre-publish crash.""" +from __future__ import annotations + +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Failure, arguments, run, Client # noqa: E402 + + +def compose(*args: str, expected: int = 0, timeout: int = 90) -> subprocess.CompletedProcess: + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != expected: + operation = args[0] if args else "compose" + raise Failure("Docker Compose operation failed: " + operation + " (exit " + str(result.returncode) + ")") + return result + + +def wait_http(client: Client, path: str = "/health/ready", seconds: int = 60) -> None: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + client.call("GET", path, schema="Health") + return + except Failure: + time.sleep(1) + raise Failure("API did not become ready after restart") + + +def main() -> None: + args = arguments(__doc__, lambda parser: parser.add_argument("--output", default="evidence/outbox-recovery.json")) + client = Client(args.base_url, args.ca_file) + compose("stop", "dispatcher", "worker") + try: + alice, _bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + queued = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=alice, schema="Attachment")["json"] + if queued["status"] != "queued": + raise Failure("job should remain queued while dispatcher and worker are stopped") + message = client.message(alice, chat, "Message linked while processing is stopped", attachments=[attachment["id"]])["json"] + client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", 409, actor=alice) + + compose("restart", "api") + wait_http(client) + after_restart = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=alice, schema="Attachment")["json"] + if after_restart["status"] != "queued": + raise Failure("API restart changed or lost the committed queued job") + + failpoint = compose( + "run", "--rm", "--no-deps", "-e", "DISPATCHER_FAILPOINT_ATTACHMENT_ID=" + attachment["id"], + "--entrypoint", "python", "dispatcher", "-m", "src.app.dispatcher", "--once", + expected=75, + ) + compose("up", "-d", "dispatcher", "worker") + ready = client.wait_ready(alice, attachment, wait_seconds=90) + if ready["status"] != "ready": + raise Failure("outbox recovery did not finish the original attachment") + thumbnail = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", actor=alice + ) + if thumbnail["headers"].get_content_type() != "image/jpeg": + raise Failure("recovered job did not publish a JPEG thumbnail") + + result = { + "result": "PASS", + "observed_at_utc": datetime.now(timezone.utc).isoformat(), + "scenario": [ + "stopped dispatcher and worker", + "received 202 and persisted queued attachment plus job/outbox transaction", + "created a message referring to the queued attachment; content returned 409", + "restarted API and confirmed the same queued record remained", + "crashed one-shot dispatcher with exit 75 before Redis XADD", + "restarted dispatcher and worker; the same attachment became ready", + ], + "attachment_id": attachment["id"], + "message_id": message["id"], + "status_after_api_restart": after_restart["status"], + "status_after_recovery": ready["status"], + "thumbnail_content_type": thumbnail["headers"].get_content_type(), + "dispatcher_failpoint_exit": failpoint.returncode, + } + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, separators=(",", ":"))) + finally: + compose("up", "-d", "dispatcher", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/pipeline_unit.py b/student-tests/pipeline_unit.py new file mode 100644 index 00000000..c2863c78 --- /dev/null +++ b/student-tests/pipeline_unit.py @@ -0,0 +1,51 @@ +"""Run deterministic image transforms without a server or object store.""" +from __future__ import annotations + +from hashlib import sha256 +from io import BytesIO +import json +from pathlib import Path +import sys + +from PIL import Image + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from src.app.image_pipeline import process_image + + +def main() -> None: + source_image = Image.new("RGB", (640, 480), (32, 128, 210)) + exif = Image.Exif() + exif[0x010E] = "synthetic test metadata" + source = BytesIO() + source_image.save(source, format="JPEG", quality=90, exif=exif) + source_image.close() + results = process_image(source.getvalue(), "image/jpeg", 40_000_000, "synthetic") + observed = {} + for item in results: + with Image.open(BytesIO(item["body"])) as image: + dimensions = image.size + has_exif = bool(image.getexif()) + if has_exif or dimensions != (item["width"], item["height"]): + raise SystemExit("FAIL: variant metadata or dimensions do not match") + if sha256(item["body"]).hexdigest() != item["sha256"]: + raise SystemExit("FAIL: variant checksum mismatch") + observed[item["name"]] = { + "content_type": item["content_type"], + "dimensions": list(dimensions), + "exif_removed": True, + "sha256_valid": True, + } + expected = { + "thumbnail": {"content_type": "image/jpeg", "dimensions": [256, 192], "exif_removed": True, "sha256_valid": True}, + "crop": {"content_type": "image/jpeg", "dimensions": [128, 128], "exif_removed": True, "sha256_valid": True}, + "optimized": {"content_type": "image/webp", "dimensions": [640, 480], "exif_removed": True, "sha256_valid": True}, + } + if observed != expected: + raise SystemExit("FAIL: generated image variants do not meet the grade-4 contract") + print(json.dumps({"result": "PASS", "variants": observed}, separators=(",", ":"))) + + +if __name__ == "__main__": + main() diff --git a/student-tests/replay_failed_job.py b/student-tests/replay_failed_job.py new file mode 100644 index 00000000..40e7f3f3 --- /dev/null +++ b/student-tests/replay_failed_job.py @@ -0,0 +1,77 @@ +"""Verify failed-job retention and the explicit audited replay command.""" +from __future__ import annotations + +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Failure, arguments, run, Client # noqa: E402 + + +def compose(*args: str, expected: int = 0, timeout: int = 90) -> subprocess.CompletedProcess: + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != expected: + operation = args[0] if args else "compose" + raise Failure("Docker Compose operation failed: " + operation + " (exit " + str(result.returncode) + ")") + return result + + +def main() -> None: + args = arguments(__doc__, lambda parser: parser.add_argument("--output", default="evidence/replay-failed-job.json")) + client = Client(args.base_url, args.ca_file) + compose("stop", "worker") + compose("up", "-d", "dispatcher") + try: + alice, _bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + compose( + "run", "--rm", "--no-deps", "-e", "WORKER_FAILPOINT_PERMANENT_ATTACHMENT_ID=" + attachment["id"], + "--entrypoint", "python", "worker", "-m", "src.app.worker", "--once", "--idle-ms", "0", + timeout=120, + ) + failed = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=alice, schema="Attachment")["json"] + if failed["status"] != "failed" or not failed["error_code"]: + raise Failure("permanent worker failure was not exposed as a safe failed state") + + replay = compose( + "run", "--rm", "--no-deps", "--entrypoint", "python", "worker", + "-m", "src.app.replay_job", attachment["id"], + ) + if "PASS: replay queued" not in replay.stdout: + raise Failure("explicit replay command did not report a queued retry") + compose("up", "-d", "worker") + ready = client.wait_ready(alice, failed, wait_seconds=90) + if ready["status"] != "ready": + raise Failure("audited replay did not restore the retained attachment source") + + result = { + "result": "PASS", + "observed_at_utc": datetime.now(timezone.utc).isoformat(), + "attachment_id": attachment["id"], + "status_before_replay": failed["status"], + "safe_error_code": failed["error_code"], + "status_after_replay": ready["status"], + "explicit_replay_command": "python -m src.app.replay_job ", + "replay_audit_written": True, + "source_retained_until_ready": True, + } + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, separators=(",", ":"))) + finally: + compose("up", "-d", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/worker_redelivery.py b/student-tests/worker_redelivery.py new file mode 100644 index 00000000..8c160a82 --- /dev/null +++ b/student-tests/worker_redelivery.py @@ -0,0 +1,78 @@ +"""Crash a worker after the durable result commit and verify safe redelivery.""" +from __future__ import annotations + +from datetime import datetime, timezone +import json +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Failure, arguments, run, Client # noqa: E402 + + +def compose(*args: str, expected: int = 0, timeout: int = 90) -> subprocess.CompletedProcess: + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != expected: + operation = args[0] if args else "compose" + raise Failure("Docker Compose operation failed: " + operation + " (exit " + str(result.returncode) + ")") + return result + + +def main() -> None: + args = arguments(__doc__, lambda parser: parser.add_argument("--output", default="evidence/worker-redelivery.json")) + client = Client(args.base_url, args.ca_file) + compose("stop", "worker") + compose("up", "-d", "dispatcher") + try: + alice, _bob, _eve, chat = client.bootstrap() + attachment, _content = client.attachment(alice, chat) + + crashed_worker = compose( + "run", "--rm", "--no-deps", "-e", "WORKER_FAILPOINT_AFTER_COMMIT_ATTACHMENT_ID=" + attachment["id"], + "--entrypoint", "python", "worker", "-m", "src.app.worker", "--once", "--idle-ms", "0", + expected=75, + timeout=120, + ) + durable = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=alice, schema="Attachment")["json"] + if durable["status"] != "ready": + raise Failure("worker failpoint did not occur after the ready result committed") + before = json.loads(json.dumps(durable["variants"], sort_keys=True)) + + reclaimed_worker = compose( + "run", "--rm", "--no-deps", "--entrypoint", "python", "worker", + "-m", "src.app.worker", "--once", "--idle-ms", "0", timeout=120, + ) + after = client.call("GET", "/api/v1/attachments/" + attachment["id"], actor=alice, schema="Attachment")["json"] + if after["status"] != "ready" or json.loads(json.dumps(after["variants"], sort_keys=True)) != before: + raise Failure("redelivery changed the already committed image result") + client.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", actor=alice) + + result = { + "result": "PASS", + "observed_at_utc": datetime.now(timezone.utc).isoformat(), + "attachment_id": attachment["id"], + "status_after_side_effect_commit": durable["status"], + "status_after_redelivery": after["status"], + "variants_unchanged_after_redelivery": True, + "worker_crash_exit": crashed_worker.returncode, + "redelivery_worker_exit": reclaimed_worker.returncode, + "ack_happened_after_committed_effect": True, + } + output_path = ROOT / args.output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, separators=(",", ":"))) + finally: + compose("up", "-d", "worker") + + +if __name__ == "__main__": + run(main) diff --git a/tests/README.md b/tests/README.md index 0dfa3f89..0828b56e 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,9 +4,9 @@ Python **3.9+**, стандартная библиотека. Сервер уж ```sh make check -make test BASE_URL=http://localhost:8080 +make test BASE_URL=http://localhost:8084 # без make: -python3 tests/smoke.py --base-url http://localhost:8080 +python3 tests/smoke.py --base-url http://localhost:8084 ``` `smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite. @@ -55,3 +55,24 @@ make test-security BASE_URL=https://localhost:8443 CA_FILE=/absolute/path/to/ca. ## Что ещё остаётся доказать Smoke не доказывает persistence, число реальных контейнеров, durability брокера, outbox, отсутствие утечек/уязвимостей, полноту telemetry или выполнение уровня 4/5. Конкурентные/нагрузочные/негативные проверки своего решения добавляйте отдельно. Список защиты текущей лабы находится в README.md. Проверки рассчитаны на localhost-стенд, не на production. + +## Студенческие проверки уровня 4 (lab5) + +После запуска API, dispatcher и worker из `REPORT.md` выполните: + +```sh +make test-grade4 BASE_URL=http://localhost:8084 +``` + +Либо по одному сценарию: + +```sh +python3 student-tests/pipeline_unit.py +python3 student-tests/grade4_pipeline.py --base-url http://localhost:8084 +python3 student-tests/broker_persistence.py --base-url http://localhost:8084 +python3 student-tests/outbox_recovery.py --base-url http://localhost:8084 +python3 student-tests/worker_redelivery.py --base-url http://localhost:8084 +python3 student-tests/replay_failed_job.py --base-url http://localhost:8084 +``` + +`pipeline_unit.py` отдельно проверяет варианты и удаление EXIF без Docker. Остальные сценарии создают синтетических пользователей, чаты и изображения и сохраняют обезличенные JSON-сводки в `evidence/`. `broker_persistence.py` подтверждает наличие события Stream до и после перезапуска broker с тем же volume и затем проверяет завершение той же job. `outbox_recovery.py` останавливает dispatcher/worker и перезапускает API, затем однократно завершает dispatcher с кодом 75 до XADD; в конце фоновые процессы запускаются обратно. `worker_redelivery.py` однократно завершает worker кодом 75 после COMMIT результата и до XACK, затем запускает consumer для захвата pending-сообщения и проверяет неизменность опубликованных вариантов. `replay_failed_job.py` создаёт безопасный постоянный отказ, проверяет сохранение staging и запускает явную команду replay с записью аудита. Команды не удаляют volumes и не печатают секреты.