ЛР5- Максимова БВТ2303 - на 4 #41
@@ -18,3 +18,7 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
infra/secrets
|
||||
Защита.md
|
||||
ЗАЩИТА.md
|
||||
|
||||
+17
-6
@@ -1,13 +1,12 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Runtime interfaces; fixed internal addresses are mapped in compose.yaml.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
SESSION_STORE=valkey
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
@@ -19,7 +18,19 @@ S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_CONCURRENCY=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
JOB_TIMEOUT_SECONDS=30
|
||||
JOB_TOTAL_TIMEOUT_SECONDS=90
|
||||
JOB_LEASE_SECONDS=120
|
||||
WORKER_MEMORY_MB=768
|
||||
RETRY_BASE_SECONDS=1
|
||||
STAGING_RETENTION_SECONDS=604800
|
||||
LB_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
APP_VERSION=dev
|
||||
UPLOAD_GRACE_SECONDS=900
|
||||
JANITOR_INTERVAL_SECONDS=60
|
||||
PUBLISHER_FAULT=
|
||||
WORKER_FAULT=
|
||||
FAULT_INJECTION=
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
*.sh text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.py text eol=lf
|
||||
@@ -1,18 +1,22 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Максимова Мария, БВТ2303.
|
||||
Целевая оценка: 4.
|
||||
Трек 5: не заявляется.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
После загрузки API сохраняет исходник в S3 staging, а вложение queued и задание outbox — одной транзакцией PostgreSQL; клиент получает 202. Отдельные publisher и RabbitMQ доставляют работу worker, который создаёт thumbnail, crop и WebP. Реализованы ограниченные повторы с backoff/jitter, lease, идемпотентные результаты, failed registry, явный replay с аудитом и безопасная очистка staging. API, сессии, сообщения, ACL, две API-реплики и приватное S3 перенесены из lab4.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Запуск из чистого clone и команды PowerShell/Linux: [REPORT.md](../REPORT.md#запуск-из-чистого-clone). После старта стенда: `python scripts/test_lab5.py`; быстрые проверки без остановок: `python scripts/test_lab5.py --quick`.
|
||||
|
||||
Результаты и методика: [REPORT.md](../REPORT.md#доказательства-критериев), [evidence/](../evidence/). Понятное объяснение и сценарий защиты: [EXPLANATION.md](../EXPLANATION.md).
|
||||
|
||||
Ограничения: один Docker host без репликации хранилищ и резервного восстановления; один последовательный worker; только pipeline v1, без исследования перегрузки. Replay возможен до очистки failed-исходника через семь суток. Исторические ready-вложения lab4 автоматически не получают варианты.
|
||||
|
||||
Источники: задание, контракты курса, собственная lab4 и документация RabbitMQ, указанная в отчёте. Codex использован для реализации, проверки и оформления объяснения.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] check и smoke выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
name: Application
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
application:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: sh scripts/init-secrets.sh
|
||||
- run: cp .env.example .env
|
||||
- run: docker compose config --quiet
|
||||
- run: docker compose up -d --build --wait --wait-timeout 180
|
||||
- run: python scripts/test_lab5.py
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: lab5-evidence
|
||||
path: evidence/*.txt
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
@@ -28,5 +28,7 @@ coverage/
|
||||
.DS_Store
|
||||
.idea/
|
||||
.vscode/
|
||||
# Personal defense notes (kept only on this machine).
|
||||
/Защита.md
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
# Messenger lab5: one image for API, publisher, worker, migrations and bootstrap.
|
||||
FROM python:3.12.7-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
COPY scripts/migrate.py scripts/s3_bootstrap.py ./scripts/
|
||||
COPY migrations ./migrations
|
||||
|
||||
RUN useradd --create-home --uid 10001 appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Exec form: python is PID 1 and receives SIGTERM from "docker stop" itself.
|
||||
# (lab2 used "sh -c uvicorn ...": sh stayed PID 1, uvicorn never got SIGTERM
|
||||
# and was killed together with the container instead of draining.)
|
||||
CMD ["python", "-m", "app.server"]
|
||||
@@ -1,8 +0,0 @@
|
||||
# Teaching outline, not a runnable Dockerfile. Implement your own Dockerfile.
|
||||
# 1. Choose a runtime/toolchain version and pin the base image tag/digest.
|
||||
# 2. Copy dependency manifests first; install from the language lockfile.
|
||||
# 3. Copy source and build. For compiled languages use a separate runtime stage.
|
||||
# 4. Run the HTTP process as PID 1; listen on 0.0.0.0:8080.
|
||||
# 5. Add non-root, signals and healthcheck according to the current rubric.
|
||||
# 6. Pass runtime config/secrets at startup, never COPY .env or secret files.
|
||||
# This file deliberately has no FROM/CMD: selecting/implementing them is the task.
|
||||
@@ -1,37 +1,267 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Максимова Мария, БВТ2303.
|
||||
- Целевая оценка: 4. Исследовательский трек на 5 не выбирается.
|
||||
- Ветка: `solution/maksimova_bvt2303`. Commit и PR ещё не опубликованы.
|
||||
- Основа: собственная реализация lab4; задание, `COURSE.md`, `contracts/` и публичные проверки текущей lab5 сохранены.
|
||||
- Библиотеки: FastAPI 0.115.0, SQLAlchemy 2.0.36/asyncpg 0.30.0, argon2-cffi 23.1.0, redis-py 5.0.8, aiobotocore 2.15.2/botocore 1.35.36, Pillow 11.0.0, aio-pika 9.4.3. Полный список закреплён в `requirements.txt`.
|
||||
- Источники: задание и контракт этой лабораторной, собственная lab4; ссылки из задания на [надёжность RabbitMQ](https://www.rabbitmq.com/docs/reliability) и [подтверждения публикации и обработки](https://www.rabbitmq.com/docs/confirms). Помощь ИИ (Codex): адаптация API, реализация очереди и worker, проверка кода, собственные тесты, оформление отчёта и объяснения для защиты. В таблицах ниже указаны реальные прогоны.
|
||||
|
||||
## О чём работа, зачем и почему
|
||||
|
||||
В lab4 сервер принимал картинку и сохранял готовое вложение. Теперь нужно ещё сделать уменьшенную копию, квадратную обрезку и WebP. Если выполнять это внутри HTTP-запроса, клиент дольше ждёт, а процесс API занят изображением вместо ответов на сообщения.
|
||||
|
||||
В этой работе приём файла и обработка разделены. API принимает все байты и надёжно сохраняет исходник вместе с заданием. Отдельный процесс — worker — получает работу через очередь и создаёт варианты. Клиент сначала получает `202 queued`, а готовность узнаёт отдельным GET. Смысл 202: «файл принят и работа сохранена».
|
||||
|
||||
Очередь нужна ещё и для восстановления. Worker может остановиться, сервер может перезапуститься, задание может прийти повторно. Нужно сохранить принятую работу и получить один правильный результат. Поэтому здесь важны порядок записи в БД, подтверждения, ограниченные повторы и защита от дублей.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Стенд проверки: Windows 11, Docker Desktop 29.3.0 с Linux-контейнерами, 14 vCPU и около 7.4 GiB RAM для Docker; Python 3.13.3 на хосте. Образ приложения использует Python 3.12.7. Нужны запущенный Docker Desktop и Docker Compose. Python на хосте нужен для публичных проверок; собственные проверки с управлением контейнерами запускаются в `tester`.
|
||||
|
||||
| Роль | Service | Версия/образ | Готовность и сохранность |
|
||||
|-----------------|-------------|----------------------------------------------------------------|---------------------------------------------------------------------------------|
|
||||
| Балансировщик | `lb` | `haproxy:3.0.11-alpine` | `/lb-health`; проверяет `/health/ready` API; внешний порт по умолчанию 8080 |
|
||||
| API ×2 | `api` | `messenger-lab5-api:dev`, сборка из `python:3.12.7-slim` | `/health/ready`; собственных volumes нет |
|
||||
| БД | `db` | `postgres:16.4-alpine` | `pg_isready`; volume `postgres_data` |
|
||||
| Кэш сессий | `valkey` | `valkey/valkey:8.0.1-alpine` | `PING`; исходные сессии в PostgreSQL, кэш можно потерять |
|
||||
| Приватное S3 | `s3` | `chrislusf/seaweedfs:3.97` | `/healthz`; volume `s3_data` |
|
||||
| Брокер | `broker` | `rabbitmq:3.13.7-alpine` | `rabbitmq-diagnostics ping`; volume `broker_data`, постоянное имя узла `broker` |
|
||||
| Публикация | `publisher` | тот же образ приложения | `python -m app.jobs`; повторяет подключение при отказах |
|
||||
| Обработка | `worker` | тот же образ приложения | `python -m app.worker`; одна активная задача, `prefetch=1` |
|
||||
| Миграции | `migrate` | тот же образ приложения | однократный запуск; применённые SQL повторно пропускаются |
|
||||
| Создание bucket | `s3-init` | тот же образ приложения | однократный запуск, проверяет приватность и права S3 |
|
||||
| Проверки | `tester` | `python:3.12.11-alpine3.22`, Docker CLI из `docker:28.5.1-cli` | профиль `tools`; make и Pillow |
|
||||
|
||||
PowerShell, из корня репозитория:
|
||||
|
||||
```powershell
|
||||
powershell -ExecutionPolicy Bypass -File scripts/init-secrets.ps1
|
||||
if (-not (Test-Path .env)) { Copy-Item .env.example .env }
|
||||
# Если 8080 занят, изменить в .env LB_PORT=8080 на LB_PORT=8081.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait --wait-timeout 180
|
||||
docker compose ps
|
||||
|
||||
# Все проверки API и уровня 4 одной командой; Python на хосте без дополнительных пакетов.
|
||||
# Тесты запускаются в Docker, логи автоматически сохраняются в evidence/.
|
||||
python -X utf8 scripts/test_lab5.py
|
||||
# Короткая проверка без остановок сервисов:
|
||||
# python -X utf8 scripts/test_lab5.py --quick
|
||||
|
||||
# Ниже порт проведённых прогонов: 8081. Для стандартного порта указать 8080.
|
||||
python -X utf8 scripts/check_contract.py
|
||||
python -X utf8 tests/smoke.py --base-url http://localhost:8081
|
||||
|
||||
# Полная проверка OpenAPI и JSON Schema:
|
||||
python -m venv .venv
|
||||
.venv/Scripts/python -m pip install -r requirements-dev.txt
|
||||
.venv/Scripts/python -X utf8 scripts/validate_openapi.py
|
||||
|
||||
# Собственные проверки и публичные сценарии с POSIX hooks:
|
||||
docker compose --profile tools build tester
|
||||
docker compose --profile tools run --rm tester python3 student-tests/lab5_images.py
|
||||
docker compose --profile tools run --rm tester python3 student-tests/lab5_recovery.py
|
||||
Get-Content -Raw student-tests/lab5_limits.py | docker compose exec -T worker python
|
||||
docker compose --profile tools run --rm tester make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://lb:8080
|
||||
docker compose --profile tools run --rm tester make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://lb:8080
|
||||
docker compose --profile tools run --rm tester sh scripts/start-api.sh
|
||||
```
|
||||
|
||||
На машине проверки 8080 был занят, поэтому локальный `.env` задаёт `LB_PORT=8081`. В `.env.example` оставлен обычный 8080. Внутри Docker адрес всегда `http://lb:8080`; `tester` получает его автоматически вместе с именем проекта `messenger-lab5`.
|
||||
|
||||
`scripts/test_lab5.py` подходит и для Windows, и для Linux. Он проверяет, что сервисы уже запущены, собирает tester и последовательно выполняет проверку контракта, smoke, семь тестов валидатора, проверку вариантов и ограничений декодера. Полный запуск дополнительно воспроизводит шесть сценариев recovery, публичные persistence и failover. После failover вторая API-реплика запускается обратно. Файлы доказательств содержат UTC-время, команду и код завершения; `container-events.txt` фиксирует состояния контейнеров до и после публичных отказов. Полная валидация OpenAPI выполняется отдельно с `requirements-dev.txt`, как показано в блоке команд.
|
||||
|
||||
Workflow `.github/workflows/application.yml` запускает полный runner и сохраняет текстовые логи как артефакт. Схемы проверяет отдельный workflow `contracts.yml`. Эти workflows подготовлены для CI; результат удалённого GitHub Actions в локальные доказательства не включается.
|
||||
|
||||
В Linux вместо PowerShell: `sh scripts/init-secrets.sh`, `cp .env.example .env`. Для публичных проверок — `make check`, `make test BASE_URL=http://localhost:8080`; для схемы — `make validate PYTHON=.venv/bin/python` после установки `requirements-dev.txt`. Проверка ресурсов: `docker compose exec -T worker python < student-tests/lab5_limits.py`.
|
||||
|
||||
Скрипты создания secrets генерируют случайные значения локально, не печатают их и не перезаписывают существующие файлы. Они лежат в `infra/secrets/`, исключённом из Git; `.env` тоже исключён. При старте ожидаются хранилища, затем выполняются `migrate` и `s3-init`, после них запускаются API, publisher и worker; LB ждёт готовности API. Новая миграция — `migrations/005_async_pipeline.sql`.
|
||||
|
||||
Безопасная остановка с сохранением данных:
|
||||
|
||||
```sh
|
||||
docker compose stop
|
||||
# Возобновить работу с теми же volumes:
|
||||
docker compose up -d --wait --wait-timeout 180
|
||||
```
|
||||
|
||||
Отдельная команда **намеренного удаления всех учебных данных**, включая пользователей, сессии, файлы и очередь:
|
||||
|
||||
```sh
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент → API → исходник в S3 staging + вложение/outbox в PostgreSQL → 202
|
||||
↓
|
||||
publisher
|
||||
↓
|
||||
RabbitMQ
|
||||
↓
|
||||
worker
|
||||
↓
|
||||
конечные файлы в S3 + ready в PostgreSQL
|
||||
↓
|
||||
GET клиента видит ready
|
||||
```
|
||||
|
||||
Из lab4 сохранены регистрация, Basic-вход и Bearer-сессии, авторизация участников чата, сообщения с идемпотентным POST, пагинация, две API-реплики, `X-Request-Id`/`X-Instance-Id`, HAProxy, graceful shutdown, приватное S3 и скачивание через API. Изменение upload `201 ready` → `202 queued` предусмотрено контрактом lab5. `queued` и `processing` разрешено прикреплять к сообщению; `failed` — нельзя, ответ 409. Скачать содержимое до `ready` тоже нельзя: 409.
|
||||
|
||||
Источник истины. PostgreSQL хранит принадлежность файла чату, исходные размер и SHA-256, состояние, попытку и намерение обработать файл. S3 хранит байты. RabbitMQ доставляет задания, а outbox позволяет восстановить ещё не опубликованное задание. Valkey ускоряет проверку сессий, но не заменяет PostgreSQL.
|
||||
|
||||
### Что происходит до ответа 202
|
||||
|
||||
1. API проверяет пользователя и членство в чате, имя файла, MIME, размер и изображение по содержимому. Разрешены PNG/JPEG до 10 MiB и 40 млн пикселей. Имя от клиента используется для отображения; путём S3 оно не становится.
|
||||
2. Тело читается частями во временный spool: в памяти до 1 MiB на загрузку, остальное во временном файле. SHA-256 считается во время чтения. На реплику разрешено до восьми одновременных загрузок; ожидание слота ограничено 10 секундами, приём тела — 60 секундами.
|
||||
3. В БД фиксируется `upload_intent`: пометка для поиска незавершённой загрузки. Затем все байты записываются в приватный S3-prefix `staging/`.
|
||||
4. Одной транзакцией PostgreSQL удаляется intent, создаётся вложение `queued` и строка `outbox`. Только после COMMIT возвращается 202 со снимком `queued` и пустым списком вариантов.
|
||||
|
||||
Staging здесь уже находится в S3. Это не временная память API и не готовый результат: worker позже читает объект, проверяет исходный SHA-256 и пишет оригинал в конечный prefix вместе с вариантами. Поэтому выключение API после 202 не теряет принятые байты.
|
||||
|
||||
Если S3-запись прошла, а транзакция вложения не завершилась, остаётся intent. Сохраняется уборка из lab4: незавершённые загрузки удаляются после `UPLOAD_GRACE_SECONDS=900`. Блокировка строки intent защищает от одновременного завершения upload и удаления файла уборщиком. При неизвестном исходе COMMIT сначала проверяется intent, чтобы не удалить уже принятое вложение.
|
||||
|
||||
### Зачем outbox и чем confirm отличается от ack
|
||||
|
||||
Нельзя одним обычным COMMIT одновременно записать PostgreSQL и отправить сообщение RabbitMQ. Если записать БД и упасть перед отправкой, запись есть, а worker о ней не узнает. Поэтому намерение отправить задание хранится в `outbox` вместе с вложением.
|
||||
|
||||
Publisher выбирает строку под `FOR UPDATE SKIP LOCKED`, публикует persistent-сообщение в durable-очередь `attachments`, ждёт publisher confirm и отмечает строку опубликованной. Confirm означает: брокер принял сообщение. При падении после отправки, но до отметки в БД отправка повторится — это допустимый дубль.
|
||||
|
||||
Consumer ack отправляет worker после устойчивой фиксации результата или сохранённого решения о повторе/ошибке. Ack означает: эту доставку можно убрать из очереди. Принятие брокером ещё не означает, что картинка обработана.
|
||||
|
||||
Сообщение соответствует `contracts/jobs.schema.json`: версия схемы, `job_id`, `attachment_id`, `pipeline_version`, `attempt`, тип и время создания. В нём нет байтов, паролей или клиентского URL. Доверенный ключ объекта worker получает из БД.
|
||||
|
||||
### Состояния, повторы и защита от дублей
|
||||
|
||||
| Состояние | Что означает | Что происходит дальше |
|
||||
|--------------|-----------------------------------------------------------|--------------------------------------------------------------------|
|
||||
| `queued` | Исходник и задание сохранены | Publisher доставляет задание, worker захватывает его |
|
||||
| `processing` | Worker получил аренду задачи — lease | Выполняется ограниченная по времени обработка |
|
||||
| `ready` | Оригинал и три варианта записаны, метаданные опубликованы | Можно скачивать; дубль не меняет результат |
|
||||
| `failed` | Постоянная ошибка либо закончились попытки | Безопасный error_code, реестр для оператора; возможен явный replay |
|
||||
|
||||
Worker обрабатывает одну задачу за раз, `prefetch=1` не позволяет заранее забрать большую пачку. При захвате увеличивается счётчик попыток и выдаётся случайный `lease_token` на 120 секунд. Общий дедлайн — 90 секунд, меньше lease. Записать результат может только владелец ещё действующего token. Если worker погиб и остался `processing`, publisher находит истёкшую аренду и сохраняет повтор в БД/outbox.
|
||||
|
||||
Логическая работа имеет постоянный `job_id`; управляемая попытка увеличивает `attempt`. Ключ результата — `(attachment_id, pipeline_version, variant)`. Здесь `pipeline_version=1`: конечные ключи имеют вид `originals/<attachment_id>/v1/original`, `thumbnail.jpg`, `crop.jpg`, `optimized.webp`. Повтор пишет те же байты по тем же ключам, а список вариантов фиксируется одной транзакцией. Дубль `ready` не запускает resize. Устаревшая попытка не может перезаписать состояние новой благодаря проверке attempt и lease token.
|
||||
|
||||
При временном отказе, например недоступности S3, сохраняются следующая попытка и её срок. Максимум — `JOB_MAX_ATTEMPTS=5` на один запуск или явный replay. Backoff растёт экспоненциально; случайная задержка (**jitter**) выбирается между половиной и полным значением верхней границы. Для базы 1 секунда первые интервалы: 0.5–1, 1–2, 2–4, 4–8 секунд; верхняя граница ограничена 60 секундами. Это распределяет повторы во времени. Повреждённый исходник, неверная контрольная сумма и ошибка декодирования считаются постоянными и автоматически не повторяются.
|
||||
|
||||
Реестр failed — строки `attachments WHERE status='failed'`. История обработки и replay хранится в `job_events`; некорректные сообщения брокера отправляются в durable `attachments.dlq`. Публичный API выдаёт краткий код, например `source_checksum_mismatch` или `retry_exhausted`, без traceback и внутренних адресов.
|
||||
|
||||
### Правила обработки и ограничения ресурсов
|
||||
|
||||
| Результат | Правило |
|
||||
|-------------|--------------------------------------------------------------------------------------------------------------|
|
||||
| Original | Исходные байты без перекодирования; исходный SHA-256 сохраняется |
|
||||
| Thumbnail | JPEG, вписан в 256×256 без увеличения; пропорции сохранены, размеры округлены вниз, минимум 1 px |
|
||||
| Crop | JPEG, квадрат из центра, 128×128; маленький исходник разрешено увеличить |
|
||||
| Optimized | WebP, вписан в 1280×1280 без увеличения; размер файла не обязан уменьшиться для каждого маленького исходника |
|
||||
|
||||
Для вариантов применяется EXIF orientation. В JPEG прозрачность накладывается на белый фон. JPEG сохраняется с `quality=85, optimize=True`; WebP — `quality=80, method=4`. EXIF и прочие метаданные из вариантов удаляются; исходник остаётся неизменным.
|
||||
|
||||
Декодирование worker выполняется дочерним процессом без импорта настроек/секретов приложения. Он получает Linux-ограничение адресного пространства `RLIMIT_AS=768 MiB`, ограничение CPU и максимум 30 секунд по часам. Родитель при превышении времени убивает и ожидает завершения дочернего процесса. На весь worker дополнительно действует лимит контейнера 1 GiB; общий job ограничен 90 секундами. Лимит 40 млн пикселей проверяется до полной распаковки: маленький сжатый файл тоже может требовать много памяти.
|
||||
|
||||
Ограничения проверены отдельно: декодирование настоящего изображения 16 млн пикселей при временном тестовом лимите 48 MiB останавливается по памяти; при тестовом дедлайне 0.001 секунды дочерний процесс убивается и ожидается его завершение. Проверка меняет параметры только внутри своего `docker exec`-процесса; штатные 768 MiB и 30 секунд остаются прежними. Это проверка механизма ограничения, а не замер максимальной производительности или гарантия обработки каждого изображения 40 млн пикселей.
|
||||
|
||||
### Очистка и действия оператора
|
||||
|
||||
После COMMIT `ready` publisher удаляет staging-объект и ставит `staging_deleted=true`. До фиксации результата исходник не удаляется. Для failed исходник и возможные частичные результаты сохраняются семь суток (`STAGING_RETENTION_SECONDS=604800`), чтобы оператор мог устранить причину и явно повторить обработку. После срока уборка удаляет их; replay такого исходника отклоняется. Сбой очистки S3 не блокирует публикацию outbox.
|
||||
|
||||
Команды оператора выполняются внутри контейнера; отдельные HTTP-endpoint к обязательному контракту не добавлены:
|
||||
|
||||
```sh
|
||||
docker compose exec --index 1 api python -m app.operator failed
|
||||
docker compose exec --index 1 api python -m app.operator replay <attachment_id> --reason "Причина устранена"
|
||||
docker compose exec --index 1 api python -m app.operator redeliver <attachment_id>
|
||||
```
|
||||
|
||||
Replay разрешён для `failed`, сохраняет `job_id`, увеличивает номер попытки, сбрасывает бюджет автоматических повторов и пишет причину в аудит. Для готового результата replay возвращает `NO-OP`; `redeliver` отправляет дубль той же работы и позволяет проверить повторную доставку.
|
||||
|
||||
Аудит `operator_replay` сохраняет номер новой попытки. Базовая задержка `RETRY_BASE_SECONDS` должна быть конечным положительным числом; ноль, отрицательные значения, NaN и infinity отклоняются при запуске pipeline.
|
||||
|
||||
### Сети, готовность и конфигурация
|
||||
|
||||
Внешние порты есть у LB; у API, PostgreSQL, Valkey, S3 и RabbitMQ они не публикуются. Данные и AMQP идут по внутренней сети `backend`. API дополнительно подключён к `edge` вместе с LB. Bucket приватен; сервис получает ограниченные S3-ключи, административные используются при bootstrap. Членство в чате проверяется в PostgreSQL при каждом скачивании; bucket сам его проверить не может.
|
||||
|
||||
Для небольшого контейнера RabbitMQ задано два Erlang scheduler (`RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=+S 2:2`), для его диагностических CLI — один (`RABBITMQ_CTL_ERL_ARGS=+S 1:1`). Это согласует число потоков с CPU-квотой брокера вместо числа ядер всего хоста.
|
||||
|
||||
Первые публичные persistence-прогоны выявили, что штатная остановка и запуск хранилищ на Docker Desktop могут исчерпать срок hook ([лог до исправления](evidence/persistence-before-fix.txt)). Итоговый `scripts/restart.sh` сначала останавливает LB, API, publisher и worker, затем **аварийно завершает PostgreSQL, Valkey, S3 и RabbitMQ через SIGKILL**. После этого запускаются те же контейнеры с прежними volumes; hook ждёт здорового состояния всех компонентов. Это явная проверка сохранности после аварии хранилищ; обычная команда `docker compose stop` остаётся отдельной штатной остановкой. Ограничения времени публичного теста и hook не увеличивались. Точная причина задержки внутри Docker не установлена; её наблюдение не доказывает потерю данных.
|
||||
|
||||
`/health/ready` API зависит от PostgreSQL и состояния graceful drain. Недоступность worker, брокера, Valkey или S3 не выключает обычные сообщения: кэш обходится, задания ждут восстановления, вложения при недоступности S3 получают 503. Это выбранная граница готовности синхронного пути.
|
||||
|
||||
| Настройки | Где применяются |
|
||||
|--------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------|
|
||||
| `SESSION_TTL_SECONDS`, `API_REPLICAS`, `IDEMPOTENCY_TTL_SECONDS`, `LB_PORT`, `LB_STATS_PORT`, `APP_VERSION` | Параметры Compose; TTL сессии по умолчанию 3600 секунд, API-реплик две |
|
||||
| `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `S3_*_FILE`, `BROKER_URL_FILE` | Файлы Docker secrets; приложение читает значение по пути, секретов в репозитории нет |
|
||||
| `S3_ENDPOINT`, `S3_REGION`, `S3_BUCKET`, `APP_HOST/APP_PORT`, `SESSION_STORE`, byte/pixel limits | В этом стенде заданы явно в `compose.yaml` под именами, которые читает приложение |
|
||||
| `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `JOB_TOTAL_TIMEOUT_SECONDS`, `JOB_LEASE_SECONDS`, `WORKER_MEMORY_MB`, `RETRY_BASE_SECONDS`, `STAGING_RETENTION_SECONDS` | Параметры pipeline из `.env`; значения и смысл описаны выше |
|
||||
| `WORKER_CONCURRENCY=1` | Объявленный режим: код worker последовательный, `prefetch=1`; произвольное изменение строки не увеличивает параллелизм |
|
||||
| `UPLOAD_GRACE_SECONDS`, `JANITOR_INTERVAL_SECONDS` | Очистка intents: grace 900 секунд, проход примерно раз в минуту с jitter |
|
||||
| `PUBLISHER_FAULT`, `WORKER_FAULT`, `FAULT_INJECTION` | Только явные эксперименты; в обычном `.env` пусты |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Прогоны 10.11.2026 на описанном стенде, внешний адрес `http://localhost:8081`. Базовый HEAD и состояние рабочего дерева описаны в [run-context.txt](evidence/run-context.txt): реализация ещё не закоммичена, поэтому один HEAD не определяет проверенные исходники. Публичные `tests/` и `contracts/` не изменялись. Smoke подтверждает обязательный сценарий API; для отказов и вариантов есть отдельные проверки. В таблице имена `lab5_*.py` означают файлы из `student-tests/`, запуск в `tester`, если не указано иначе.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
|----------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------|
|
||||
| Контракт и структура skeleton сохранены | `python -X utf8 scripts/check_contract.py` | PASS: структура, локальные ссылки и синтаксис публичных Python-проверок | [check.txt](evidence/check.txt) |
|
||||
| Схемы OpenAPI и jobs валидны | `.venv/Scripts/python -X utf8 scripts/validate_openapi.py` | PASS: полная валидация OpenAPI 3.1 и JSON Schema | [validate.txt](evidence/validate.txt) |
|
||||
| 3: прежний API, upload 202, GET до ready | `python -X utf8 tests/smoke.py --base-url http://localhost:8081` | PASS: также сообщение с queued-вложением, исходные байты и настоящий JPEG thumbnail 256×160 | [smoke.txt](evidence/smoke.txt) |
|
||||
| 3: сохранность при остановленном worker и рестарте API/broker | `lab5_recovery.py --mode persistence` | PASS: те же ID и сессия, content409 до worker, messages201, после старта ready; подтверждён реальный restart контейнеров | [recovery.txt](evidence/recovery.txt) |
|
||||
| 3: restart всех процессов и хранилищ с volumes | Публичный `tests/resilience.py --mode persistence` с `scripts/restart.sh` (тот же тест, что `make test-persistence`) | PASS: после аварийного перезапуска хранилищ сохранены сессия, подтверждённые сообщения и исходные байты файла; StartedAt всех сервисов изменился | [persistence.txt](evidence/persistence.txt), [container-events.txt](evidence/container-events.txt) |
|
||||
| 3: две API-реплики, отказ обслужившей реплики | Публичный `tests/resilience.py --mode failover` с `scripts/stop-one.sh` (тот же тест, что `make test-failover`) | PASS: подтверждён отказ нужной реплики, данные и сессия сохранены; после теста обе реплики healthy | [failover.txt](evidence/failover.txt), [container-events.txt](evidence/container-events.txt) |
|
||||
| 3: состояния, ограниченный retry, безопасный error_code | `lab5_recovery.py --mode replay` и `--mode retry` | PASS: постоянная ошибка после одной попытки; S3 outage → пять попыток → retry_exhausted | [recovery.txt](evidence/recovery.txt) |
|
||||
| 3: повтор не размножает варианты | `lab5_recovery.py --mode replay` | PASS: ready до/после redeliver совпадает, вариантов три | [recovery.txt](evidence/recovery.txt) |
|
||||
| 4: crop, WebP, EXIF, alpha, отсутствие увеличения и округление | `lab5_images.py` | PASS: декодированы варианты, проверены реальные размеры, SHA-256 и исходные байты | [images.txt](evidence/images.txt) |
|
||||
| 4: bytes/pixels, невалидные изображения | `lab5_images.py` | PASS: >10 MiB и >40 млн пикселей → 413; повреждение/MIME mismatch → 415; опасные имена → 400 | [images.txt](evidence/images.txt) |
|
||||
| 4: ограничения времени и памяти worker | `Get-Content -Raw student-tests/lab5_limits.py \| docker compose exec -T worker python` | PASS: реальное декодирование 16MP ограничено 48 MiB; timeout убивает и ожидает дочерний процесс | [limits.txt](evidence/limits.txt) |
|
||||
| 4: разрыв COMMIT/publish закрыт outbox | `lab5_recovery.py --mode outbox` | PASS: одноразовый crash, прежнее вложение ready после восстановления, без нового задания вручную | [recovery.txt](evidence/recovery.txt) |
|
||||
| 4: backoff/jitter, реестр failed, явный replay с аудитом | `lab5_recovery.py --mode retry` и `--mode replay` | PASS: пять сохранённых попыток, failed виден оператору, replay сохраняет job_id и пишет operator_replay с номером новой попытки | [recovery.txt](evidence/recovery.txt) |
|
||||
| 4: crash после объектов, восстановление processing по lease | `lab5_recovery.py --mode objects` | PASS: processing и записанный S3-объект, естественное истечение lease, вторая попытка, прежний результат | [recovery.txt](evidence/recovery.txt) |
|
||||
| 4: crash после COMMIT до ack | `lab5_recovery.py --mode ack` | PASS: прежняя ready-задача, исходник и три варианта сохранены | [recovery.txt](evidence/recovery.txt) |
|
||||
| 4: concurrency/prefetch и безопасная очистка staging | Последовательный worker, `prefetch=1`; `--mode objects`, `--mode ack` | PASS: после ready staging_deleted=true, staging-объекта нет; конечные файлы доступны | [recovery.txt](evidence/recovery.txt), `app/worker.py`, `app/pipeline.py` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Это функциональные проверки отказов для уровня 4. Нагрузочное исследование и выбор оптимальной concurrency на 5 не заявляются. RPS и p50/p95/p99 не измерялись: они не нужны, чтобы показать восстановление одной принятой задачи.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Методика: уникальные синтетические пользователи и чат; небольшой PNG 320×200; один worker, `prefetch=1`; указанные выше лимиты. В тесте изображений дополнительно используются 400×200, 11×7, 513×201, 1701×1003, прозрачный PNG и JPEG с EXIF orientation. Проверяются скачанные байты, а не только JSON. Проверка ограничений использует отдельный 16MP PNG и намеренно уменьшенные лимиты.
|
||||
|
||||
`lab5_recovery.py` имеет режимы `persistence`, `outbox`, `objects`, `ack`, `replay`, `retry`. Без `--mode` выполняет все. Тест управляет контейнерами своего Compose-проекта через Docker labels, не удаляет volumes и в `finally` возвращает остановленные сервисы. Токены существуют только в памяти клиента и не печатаются.
|
||||
|
||||
Fault injection включается явно через CLI и сохраняется в `fault_injections`. Каждая точка срабатывает один раз: перед аварийным `os._exit(70)` фиксируется счётчик hits. После автоматического перезапуска она не убивает процесс снова. Так подтверждается нужное окно отказа, а не обычный успешный путь.
|
||||
|
||||
```sh
|
||||
# Включить один сбой для следующей подходящей задачи:
|
||||
docker compose exec --index 1 api python -m app.operator fault crash_after_commit_before_publish
|
||||
# Другие точки:
|
||||
# crash_after_objects_before_commit
|
||||
# crash_after_commit_before_ack
|
||||
```
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
|
||||
|----------------------------------------------------------------------|------------------------------------------------------------------|-------------------------------------------------------|-----------------------------------------------------------------------|
|
||||
| Worker остановлен, upload, сообщение с вложением, restart API+broker | Upload202, content409; после старта тот же ID ready | PASS; обычные сообщения продолжают работать | Прежние байты и сессия сохранены |
|
||||
| Publisher погиб после COMMIT до publish | Pending outbox остаётся; новый publisher отправит прежнюю работу | PASS, fault hits=1 | 2.7 с от наблюдения сбоя до ready; принятое вложение не потеряно |
|
||||
| Worker погиб после записи объектов до COMMIT | S3 уже записано, processing; lease истечёт и запустит повтор | PASS, естественный lease120, без ручного изменения БД | 119.5 с от старта ожидания после crash; прежние байты, три варианта |
|
||||
| Worker погиб после COMMIT ready до ack | Повторная доставка безопасна для ready | PASS, fault hits=1 | 0.3 с до наблюдения ready; второй набор не появился |
|
||||
| Испорчен staging одной задачи | Постоянная ошибка без retry, content/link409 | PASS, failed после одной попытки, виден в реестре | После возврата исходных байтов явный replay с аудитом, job_id прежний |
|
||||
| S3 остановлено после приёма файла | Временные повторы ограничены | PASS, пять попыток и retry_exhausted | После старта S3 явный replay завершил ту же работу; исходник сохранён |
|
||||
| Лимит декодера уменьшен до 48 MiB / 0.001 с | Превышение не оставляет работающий декодер | PASS на реальном изображении 16MP | Ошибка памяти и timeout обработаны, дочерний процесс завершён |
|
||||
|
||||
Времена 2.7/119.5/0.3 секунды — результаты единичных функциональных прогонов на ноутбуке, а не статистическая оценка задержки. В ack-сценарии `ready` фиксируется до аварийного выхода; быстрое чтение ready не означает повторной обработки.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker host. PostgreSQL, SeaweedFS, RabbitMQ и LB в одном экземпляре. Перезапуск с volumes предусмотрен; потеря диска/host или удаление volume не компенсируются. Резервного копирования и репликации хранилищ нет.
|
||||
- Стенд предназначен для запуска из чистого clone и проверен на перезапусках данных lab5. Миграция старого lab4 не создаёт варианты для ранее загруженных ready-вложений: для них требуется новая загрузка; автоматическая обработка исторических файлов не реализована.
|
||||
- Доставка — at least once: сообщение может прийти несколько раз. Идемпотентная запись делает повтор безопасным, но не превращает доставку в exactly once. Несколько версий pipeline и исследование устаревших версий результата относятся к уровню 5; здесь только v1.
|
||||
- Один worker выполняет одну задачу. Большая очередь увеличит ожидание ready. Порог smoke 60 секунд относится к свободному локальному worker и небольшому изображению, а не к любой нагрузке.
|
||||
- Глобальный лимит длины очереди и исследование перегрузки не реализованы. Число загрузок ограничено на API-реплику, время и память обработки — на worker/дочерний процесс.
|
||||
- Failed-исходник доступен для replay семь суток. После очистки нужна новая загрузка: восстановить удалённые байты из метаданных невозможно. Успешные конечные объекты сохраняются; outbox и аудит не имеют автоматической архивной ротации.
|
||||
- Readiness API отражает работу сообщений и PostgreSQL. Отдельных HTTP health-endpoint у publisher/worker нет; их состояние видно по процессу, журналу, outbox, статусам и срокам lease.
|
||||
- `tester` получает Docker socket для явных учебных отказов. Его профиль запускается оператором; постоянной частью приложения он не является.
|
||||
- Проведённые функциональные проверки доказывают перечисленные сценарии на этом стенде. Они не заменяют длительную нагрузку, проверку потери носителя и резервного восстановления.
|
||||
|
||||
@@ -0,0 +1,467 @@
|
||||
"""Attachments: durable upload to staging, queued metadata + outbox (lab5).
|
||||
|
||||
Upload, in this order (all checks before the bytes are stored):
|
||||
|
||||
1. filename (400) -> chat membership (404) -> Content-Type header (415)
|
||||
-> Content-Length (413 without reading the body);
|
||||
2. an upload slot: at most MAX_CONCURRENT_UPLOADS per replica, else 429;
|
||||
3. the body is read in chunks into a SpooledTemporaryFile (only
|
||||
UPLOAD_SPOOL_MEMORY_BYTES in RAM), counted and hashed on the fly; more
|
||||
than MAX_UPLOAD_BYTES -> 413, slower than UPLOAD_TIMEOUT_SECONDS -> 400;
|
||||
4. content validation in a thread (app.images): format by magic bytes,
|
||||
pixel limit from the header (413), integrity (415);
|
||||
5. upload intent COMMIT -> staging S3 PUT -> one transaction "delete intent +
|
||||
insert queued attachment + outbox" -> 202. Worker produces all variants.
|
||||
|
||||
Failure between S3 and PostgreSQL (the answer is 503 at once, cleanup runs
|
||||
in the background):
|
||||
- PUT failed or timed out: the outcome is unknown - S3 may still finish
|
||||
writing the object AFTER our DELETE. So the object is deleted best
|
||||
effort, but the intent is KEPT; the janitor deletes the object again
|
||||
after UPLOAD_GRACE_SECONDS and only then the intent;
|
||||
- COMMIT failed: the outcome is unknown too - COMMIT may have succeeded
|
||||
and only the answer was lost. Under the intent's row lock: no intent ->
|
||||
the row is committed, keep the object; else delete object, then intent;
|
||||
- the process died after PUT: the intent stays; the janitor deletes the
|
||||
object and the intent after UPLOAD_GRACE_SECONDS.
|
||||
So an object without a row exists only while its intent exists, and an
|
||||
object of a committed row is never deleted.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import tempfile
|
||||
import unicodedata
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from fastapi import Request
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
from starlette.requests import ClientDisconnect
|
||||
|
||||
from app import repositories
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.errors import (
|
||||
ApiError,
|
||||
Conflict,
|
||||
InvalidRequest,
|
||||
NotFound,
|
||||
PayloadTooLarge,
|
||||
RateLimited,
|
||||
Unavailable,
|
||||
UnsupportedMediaType,
|
||||
)
|
||||
from app.images import InvalidImage, TooManyPixels, validate_image
|
||||
from app.storage import ObjectMissing, StorageUnavailable, storage
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.attachments")
|
||||
|
||||
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
||||
# Characters that can turn a display name into a path or hide its real
|
||||
# extension: separators, NUL/control characters, bidi overrides etc.
|
||||
FORBIDDEN_CATEGORIES = {"Cc", "Cf", "Cs", "Co", "Cn", "Zl", "Zp"}
|
||||
|
||||
upload_slots = asyncio.Semaphore(settings.max_concurrent_uploads)
|
||||
# Full integrity checks are CPU work too. In particular, progressive JPEG
|
||||
# retains decoder coefficients even with draft(); never run eight at once.
|
||||
validation_slot = asyncio.Semaphore(1)
|
||||
# Strong references to background compensation tasks (asyncio keeps only
|
||||
# weak ones); if the process dies first, the intent stays for the janitor.
|
||||
_compensations: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
def _to_attachment(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"owner_id": row["owner_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": [
|
||||
{key: value for key, value in variant.items() if key != "object_key"}
|
||||
for variant in _variants(row)
|
||||
],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
|
||||
|
||||
def _variants(row: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
value = row.get("variants") or []
|
||||
return json.loads(value) if isinstance(value, str) else value
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> str:
|
||||
if not 1 <= len(filename) <= 255 or not filename.strip():
|
||||
raise InvalidRequest("filename must be 1-255 characters, not blank.")
|
||||
|
||||
if "/" in filename or "\\" in filename:
|
||||
raise InvalidRequest("filename must not contain path separators.")
|
||||
|
||||
if any(unicodedata.category(ch) in FORBIDDEN_CATEGORIES for ch in filename):
|
||||
raise InvalidRequest("filename must not contain control characters.")
|
||||
|
||||
if filename.strip(". ") == "":
|
||||
raise InvalidRequest("filename must not be a relative path.")
|
||||
|
||||
return filename
|
||||
|
||||
|
||||
def declared_content_type(request: Request) -> str:
|
||||
raw = request.headers.get("Content-Type", "")
|
||||
media_type = raw.split(";", 1)[0].strip().lower()
|
||||
|
||||
if media_type not in ALLOWED_CONTENT_TYPES:
|
||||
raise UnsupportedMediaType(
|
||||
"Content-Type must be image/png or image/jpeg (raw bytes, not multipart)."
|
||||
)
|
||||
|
||||
return media_type
|
||||
|
||||
|
||||
def _too_large(message: str) -> PayloadTooLarge:
|
||||
error = PayloadTooLarge(message)
|
||||
# The rest of the body is not read: the connection cannot be reused.
|
||||
error.headers["Connection"] = "close"
|
||||
return error
|
||||
|
||||
|
||||
def check_declared_length(request: Request) -> None:
|
||||
raw = request.headers.get("Content-Length")
|
||||
if raw is None:
|
||||
return # chunked: the limit is enforced while reading
|
||||
|
||||
if not raw.isdigit():
|
||||
raise InvalidRequest("Invalid Content-Length.")
|
||||
|
||||
if int(raw) > settings.max_upload_bytes:
|
||||
raise _too_large("File is larger than 10 MiB.")
|
||||
|
||||
|
||||
async def _receive_body(request: Request, spool) -> tuple[int, str]:
|
||||
"""Stream the body into the spool; return (size, sha256)."""
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(settings.upload_timeout_seconds):
|
||||
async for chunk in request.stream():
|
||||
if not chunk:
|
||||
continue
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise _too_large("File is larger than 10 MiB.")
|
||||
digest.update(chunk)
|
||||
spool.write(chunk)
|
||||
except TimeoutError:
|
||||
raise ApiError(
|
||||
400,
|
||||
"invalid_request",
|
||||
"Upload was not completed in time.",
|
||||
{"Connection": "close"},
|
||||
) from None
|
||||
except ClientDisconnect:
|
||||
# Nobody will read this answer; nothing was stored anywhere.
|
||||
logger.info("upload interrupted by the client after %d bytes", size)
|
||||
raise InvalidRequest("Upload was interrupted.") from None
|
||||
|
||||
return size, digest.hexdigest()
|
||||
|
||||
|
||||
async def _acquire_slot() -> None:
|
||||
try:
|
||||
async with asyncio.timeout(settings.upload_slot_wait_seconds):
|
||||
await upload_slots.acquire()
|
||||
except TimeoutError:
|
||||
raise RateLimited("Too many concurrent uploads, retry later.", 2) from None
|
||||
|
||||
|
||||
async def _discard_after_failed_put(object_key: str) -> None:
|
||||
"""Best effort only; the intent stays for the janitor (late PUT)."""
|
||||
try:
|
||||
await storage.delete(object_key)
|
||||
except StorageUnavailable:
|
||||
logger.warning("compensation: S3 delete failed, janitor will retry")
|
||||
|
||||
|
||||
async def _compensate_failed_commit(object_key: str) -> None:
|
||||
"""Delete the object only if its upload really did not commit."""
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
row = await repositories.lock_upload_intent(db, object_key)
|
||||
if row is None:
|
||||
# COMMIT succeeded after all (or the janitor finished it).
|
||||
await db.rollback()
|
||||
return
|
||||
if not row["committed"]:
|
||||
try:
|
||||
await storage.delete(object_key)
|
||||
except StorageUnavailable:
|
||||
await db.rollback()
|
||||
logger.warning("compensation: S3 delete failed, janitor will retry")
|
||||
return
|
||||
await repositories.delete_upload_intent(db, object_key)
|
||||
except Exception as exc:
|
||||
logger.warning("compensation failed, janitor will retry: %s", type(exc).__name__)
|
||||
|
||||
|
||||
async def upload_attachment(
|
||||
request: Request,
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
filename: str,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
validate_filename(filename)
|
||||
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db, chat_id=chat_id, user_id=current_user["id"]
|
||||
)
|
||||
if chat is None:
|
||||
raise NotFound()
|
||||
# Do not keep a pooled connection idle in a transaction while the
|
||||
# client uploads up to 10 MiB.
|
||||
await db.rollback()
|
||||
|
||||
content_type = declared_content_type(request)
|
||||
check_declared_length(request)
|
||||
|
||||
await _acquire_slot()
|
||||
try:
|
||||
with tempfile.SpooledTemporaryFile(
|
||||
max_size=settings.upload_spool_memory_bytes
|
||||
) as spool:
|
||||
size, sha256 = await _receive_body(request, spool)
|
||||
if size == 0:
|
||||
raise InvalidRequest("Request body with image bytes is required.")
|
||||
|
||||
try:
|
||||
async with validation_slot:
|
||||
validation = asyncio.create_task(asyncio.to_thread(
|
||||
validate_image, spool, content_type, settings.max_image_pixels
|
||||
))
|
||||
try:
|
||||
info = await asyncio.shield(validation)
|
||||
except asyncio.CancelledError:
|
||||
# A thread cannot be killed: keep its spool and the
|
||||
# validation slot alive until it has really stopped.
|
||||
await asyncio.gather(validation, return_exceptions=True)
|
||||
raise
|
||||
except TooManyPixels:
|
||||
raise PayloadTooLarge("Image has more than 40 million pixels.") from None
|
||||
except InvalidImage as exc:
|
||||
raise UnsupportedMediaType(str(exc)) from None
|
||||
|
||||
return await _store(
|
||||
db,
|
||||
spool=spool,
|
||||
chat_id=chat_id,
|
||||
owner_id=current_user["id"],
|
||||
filename=filename,
|
||||
content_type=info.content_type,
|
||||
size=size,
|
||||
sha256=sha256,
|
||||
width=info.width,
|
||||
height=info.height,
|
||||
)
|
||||
finally:
|
||||
upload_slots.release()
|
||||
|
||||
|
||||
async def _store(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
spool,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
) -> dict[str, Any]:
|
||||
attachment_id = uuid4()
|
||||
# Server-generated random key, unrelated to the filename and to the
|
||||
# public attachment ID.
|
||||
object_key = f"staging/{uuid4().hex}"
|
||||
|
||||
await repositories.create_upload_intent(
|
||||
db, object_key=object_key, attachment_id=attachment_id, chat_id=chat_id
|
||||
)
|
||||
|
||||
try:
|
||||
await storage.put(object_key, spool, size, content_type)
|
||||
except StorageUnavailable as exc:
|
||||
logger.warning("S3 PUT failed: %s", exc)
|
||||
_in_background(_discard_after_failed_put(object_key))
|
||||
raise Unavailable() from None
|
||||
|
||||
if settings.fault_injection == "crash_after_s3_put":
|
||||
logger.error("FAULT_INJECTION: process exits after S3 PUT, before COMMIT")
|
||||
os._exit(1)
|
||||
|
||||
try:
|
||||
if settings.fault_injection == "db_error_after_s3_put":
|
||||
raise ConnectionError("FAULT_INJECTION: database lost after S3 PUT")
|
||||
|
||||
row = await repositories.finalize_attachment(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
chat_id=chat_id,
|
||||
owner_id=owner_id,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=size,
|
||||
sha256=sha256,
|
||||
width=width,
|
||||
height=height,
|
||||
object_key=object_key,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("attachment COMMIT failed after S3 PUT: %s", type(exc).__name__)
|
||||
_in_background(_compensate_failed_commit(object_key))
|
||||
raise Unavailable() from None
|
||||
|
||||
if row is None:
|
||||
# The janitor already claimed this intent (upload slower than the
|
||||
# grace period): the object is (being) deleted, report a failure.
|
||||
logger.warning("upload intent expired before COMMIT")
|
||||
raise Unavailable()
|
||||
|
||||
return _to_attachment(row)
|
||||
|
||||
|
||||
async def get_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
row = await repositories.get_accessible_attachment(
|
||||
db, attachment_id=attachment_id, user_id=current_user["id"]
|
||||
)
|
||||
if row is None:
|
||||
raise NotFound()
|
||||
return _to_attachment(row)
|
||||
|
||||
|
||||
async def open_attachment_content(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
variant: str,
|
||||
current_user: dict[str, Any],
|
||||
) -> tuple[AsyncIterator[bytes], dict[str, str]]:
|
||||
"""Authorize in PostgreSQL, then stream the bytes from S3 (no redirect)."""
|
||||
row = await repositories.get_accessible_attachment(
|
||||
db, attachment_id=attachment_id, user_id=current_user["id"]
|
||||
)
|
||||
if row is None:
|
||||
raise NotFound()
|
||||
await db.rollback()
|
||||
|
||||
if row["status"] != "ready":
|
||||
raise Conflict("Attachment content is not ready.")
|
||||
|
||||
content = row
|
||||
if variant != "original":
|
||||
content = next((item for item in _variants(row) if item["name"] == variant), None)
|
||||
if content is None:
|
||||
raise NotFound()
|
||||
|
||||
try:
|
||||
chunks, length = await storage.open_download(content["object_key"])
|
||||
except StorageUnavailable as exc:
|
||||
logger.warning("S3 GET failed: %s", exc)
|
||||
raise Unavailable() from None
|
||||
except ObjectMissing:
|
||||
# Metadata says "ready" but the bytes are gone: an inconsistency,
|
||||
# not a user error (logged as ERROR).
|
||||
logger.error("object of attachment %s is missing in S3", attachment_id)
|
||||
raise
|
||||
|
||||
if length != content["size_bytes"]:
|
||||
logger.error("object of attachment %s has a wrong size", attachment_id)
|
||||
raise ObjectMissing(content["object_key"])
|
||||
|
||||
headers = {
|
||||
"Content-Type": content["content_type"],
|
||||
"Content-Length": str(length),
|
||||
"Content-Disposition": "inline; filename*=UTF-8''"
|
||||
+ quote(row["filename"], safe=""),
|
||||
"ETag": f'"{content["sha256"]}"',
|
||||
"Cache-Control": "private, no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
}
|
||||
return chunks, headers
|
||||
|
||||
|
||||
def _in_background(compensation) -> None:
|
||||
"""Do not keep the client waiting for compensation: with S3 down each
|
||||
attempt takes up to S3_OPERATION_TIMEOUT_SECONDS, and the answer is 503
|
||||
anyway."""
|
||||
task = asyncio.create_task(compensation)
|
||||
_compensations.add(task)
|
||||
task.add_done_callback(_compensations.discard)
|
||||
|
||||
|
||||
# --- Janitor ----------------------------------------------------------------
|
||||
|
||||
async def cleanup_stale_uploads(grace_seconds: int, limit: int = 100) -> int:
|
||||
"""Delete objects of upload intents older than the grace period.
|
||||
|
||||
One intent per transaction: lock it (SKIP LOCKED), delete the object in
|
||||
S3 (idempotent), delete the intent, COMMIT. If S3 is down, roll back and
|
||||
stop: the intent stays and the next run retries.
|
||||
"""
|
||||
cleaned = 0
|
||||
for _ in range(limit):
|
||||
async with SessionLocal() as db:
|
||||
row = await repositories.lock_stale_intent(db, grace_seconds)
|
||||
if row is None:
|
||||
await db.rollback()
|
||||
break
|
||||
|
||||
if not row["committed"]:
|
||||
try:
|
||||
await storage.delete(row["object_key"])
|
||||
except StorageUnavailable:
|
||||
await db.rollback()
|
||||
logger.warning("janitor: S3 unavailable, will retry")
|
||||
break
|
||||
|
||||
await repositories.delete_upload_intent(db, row["object_key"])
|
||||
cleaned += 1
|
||||
logger.warning(
|
||||
"janitor: removed abandoned upload %s (object %s)",
|
||||
row["object_key"],
|
||||
"kept, row committed" if row["committed"] else "deleted",
|
||||
)
|
||||
return cleaned
|
||||
|
||||
|
||||
async def janitor_loop() -> None:
|
||||
while True:
|
||||
# Jitter: replicas do not all wake up at the same moment.
|
||||
await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2))
|
||||
try:
|
||||
await cleanup_stale_uploads(settings.upload_grace_seconds)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("janitor run failed: %s", type(exc).__name__)
|
||||
+179
@@ -0,0 +1,179 @@
|
||||
"""Valkey session cache.
|
||||
|
||||
PostgreSQL is the source of truth for sessions. Valkey only keeps a short
|
||||
lived copy of "token hash -> user" to avoid a SQL lookup on every request:
|
||||
|
||||
- key: messenger:session:<sha256(token)>, the raw token is never stored;
|
||||
- value TTL: min(SESSION_CACHE_TTL_SECONDS, remaining session lifetime),
|
||||
and expires_at is checked again on every read;
|
||||
- logout writes a "revoked" tombstone (SET ... EX, overwrite), while cache
|
||||
population uses SET ... NX, so a concurrent request cannot resurrect a
|
||||
revoked session in the cache;
|
||||
- any Valkey error or timeout is a cache miss: requests go to PostgreSQL and
|
||||
the cache is skipped for CACHE_RETRY_AFTER_SECONDS. After such an error,
|
||||
and on every API process start, the cache is flushed before its next use,
|
||||
because an invalidation may have been lost while Valkey was unreachable
|
||||
(also by a previous API process).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from redis.asyncio import ConnectionPool, Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.cache")
|
||||
|
||||
KEY_PREFIX = "messenger:session:"
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class Revoked:
|
||||
"""Marker returned when the cache knows the session was revoked."""
|
||||
|
||||
|
||||
REVOKED_SESSION = Revoked()
|
||||
|
||||
|
||||
def _key(token_hash: str) -> str:
|
||||
return KEY_PREFIX + token_hash
|
||||
|
||||
|
||||
def _serialize(user: dict[str, Any], expires_at: datetime) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": str(user["id"]),
|
||||
"username": user["username"],
|
||||
"display_name": user["display_name"],
|
||||
"created_at": user["created_at"].isoformat(),
|
||||
"expires_at": expires_at.isoformat(),
|
||||
},
|
||||
separators=(",", ":"),
|
||||
)
|
||||
|
||||
|
||||
def _deserialize(raw: str) -> tuple[dict[str, Any], datetime]:
|
||||
payload = json.loads(raw)
|
||||
user = {
|
||||
"id": UUID(payload["id"]),
|
||||
"username": payload["username"],
|
||||
"display_name": payload["display_name"],
|
||||
"created_at": datetime.fromisoformat(payload["created_at"]),
|
||||
}
|
||||
return user, datetime.fromisoformat(payload["expires_at"])
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url: str | None) -> None:
|
||||
self._client: Redis | None = None
|
||||
self._unavailable_until = 0.0
|
||||
# A process never trusts a cache it has not flushed itself: an
|
||||
# earlier process may have lost an invalidation (logout while
|
||||
# Valkey was unreachable) and that knowledge died with it.
|
||||
# Cleared only after a successful FLUSHDB.
|
||||
self._needs_flush = True
|
||||
|
||||
if url:
|
||||
pool = ConnectionPool.from_url(
|
||||
url,
|
||||
max_connections=settings.cache_max_connections,
|
||||
socket_timeout=settings.cache_timeout_seconds,
|
||||
socket_connect_timeout=settings.cache_timeout_seconds,
|
||||
decode_responses=True,
|
||||
)
|
||||
self._client = Redis(connection_pool=pool)
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self._client is not None
|
||||
|
||||
async def _run(self, operation):
|
||||
"""Run one cache operation; return (ok, result)."""
|
||||
if self._client is None:
|
||||
return False, None
|
||||
|
||||
if time.monotonic() < self._unavailable_until:
|
||||
return False, None
|
||||
|
||||
try:
|
||||
async with asyncio.timeout(settings.cache_timeout_seconds * 2):
|
||||
if self._needs_flush:
|
||||
await self._client.flushdb()
|
||||
self._needs_flush = False
|
||||
logger.warning("session cache flushed before use")
|
||||
return True, await operation(self._client)
|
||||
except (RedisError, OSError, TimeoutError) as exc:
|
||||
self._unavailable_until = (
|
||||
time.monotonic() + settings.cache_retry_after_seconds
|
||||
)
|
||||
self._needs_flush = True
|
||||
logger.warning(
|
||||
"session cache unavailable, falling back to PostgreSQL: %s",
|
||||
type(exc).__name__,
|
||||
)
|
||||
return False, None
|
||||
|
||||
async def get(
|
||||
self,
|
||||
token_hash: str,
|
||||
) -> dict[str, Any] | Revoked | None:
|
||||
ok, raw = await self._run(lambda client: client.get(_key(token_hash)))
|
||||
|
||||
if not ok or raw is None:
|
||||
return None
|
||||
|
||||
if raw == REVOKED:
|
||||
return REVOKED_SESSION
|
||||
|
||||
try:
|
||||
user, expires_at = _deserialize(raw)
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return None
|
||||
|
||||
if expires_at <= datetime.now(timezone.utc):
|
||||
return None
|
||||
|
||||
return user
|
||||
|
||||
async def put(
|
||||
self,
|
||||
token_hash: str,
|
||||
user: dict[str, Any],
|
||||
expires_at: datetime,
|
||||
) -> None:
|
||||
remaining = (expires_at - datetime.now(timezone.utc)).total_seconds()
|
||||
ttl = min(settings.session_cache_ttl_seconds, math.floor(remaining))
|
||||
|
||||
if ttl < 1:
|
||||
return
|
||||
|
||||
value = _serialize(user, expires_at)
|
||||
await self._run(
|
||||
lambda client: client.set(_key(token_hash), value, ex=ttl, nx=True)
|
||||
)
|
||||
|
||||
async def revoke(self, token_hash: str) -> None:
|
||||
await self._run(
|
||||
lambda client: client.set(
|
||||
_key(token_hash),
|
||||
REVOKED,
|
||||
ex=settings.session_cache_ttl_seconds,
|
||||
)
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
|
||||
|
||||
session_cache = SessionCache(settings.cache_url)
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
import os
|
||||
import socket
|
||||
|
||||
|
||||
def read_value(name: str, default: str | None = None) -> str:
|
||||
file_name = os.getenv(f"{name}_FILE")
|
||||
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
|
||||
value = os.getenv(name, default)
|
||||
|
||||
if value is None:
|
||||
raise RuntimeError(f"{name} is required.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def read_positive_int(name: str, default: str) -> int:
|
||||
value = int(os.getenv(name, default))
|
||||
|
||||
if value < 1:
|
||||
raise RuntimeError(f"{name} must be positive.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def read_positive_float(name: str, default: str) -> float:
|
||||
value = float(os.getenv(name, default))
|
||||
|
||||
if value <= 0:
|
||||
raise RuntimeError(f"{name} must be positive.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
app_host: str
|
||||
app_port: int
|
||||
database_url: str
|
||||
session_ttl_seconds: int
|
||||
# PostgreSQL connection pool and timeouts.
|
||||
db_pool_size: int
|
||||
db_max_overflow: int
|
||||
db_pool_timeout_seconds: float
|
||||
db_connect_timeout_seconds: float
|
||||
db_statement_timeout_ms: int
|
||||
# Valkey is a session cache; PostgreSQL stays the source of truth.
|
||||
cache_url: str | None
|
||||
session_cache_ttl_seconds: int
|
||||
cache_timeout_seconds: float
|
||||
cache_max_connections: int
|
||||
cache_retry_after_seconds: float
|
||||
# Lab3: X-Instance-Id is the container hostname (short Docker ID), stable
|
||||
# for the process lifetime; scripts/stop-one.sh maps it to a container.
|
||||
instance_id: str
|
||||
idempotency_ttl_seconds: int
|
||||
# Graceful shutdown: readiness answers 503 for shutdown_drain_seconds so
|
||||
# the LB drops this replica, then uvicorn closes the listener and waits
|
||||
# for in-flight requests up to shutdown_timeout_seconds.
|
||||
shutdown_drain_seconds: float
|
||||
shutdown_timeout_seconds: int
|
||||
# Lab4: attachments. Metadata lives in PostgreSQL, bytes in a private
|
||||
# S3 bucket; the API uses its own restricted credentials (not root).
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key_id: str
|
||||
s3_secret_access_key: str
|
||||
s3_object_prefix: str
|
||||
s3_connect_timeout_seconds: float
|
||||
s3_read_timeout_seconds: float
|
||||
# Deadline of one whole S3 call (DNS + retries included), below the LB
|
||||
# "timeout server" (10 s): a stopped S3 container makes Docker DNS hang
|
||||
# ~8 s per lookup, which connect_timeout does not cover.
|
||||
s3_operation_timeout_seconds: float
|
||||
# PUT gets operation timeout + size / this rate: 10 MiB under load is
|
||||
# slower than a HEAD. HAProxy allows 40 s for an upload POST.
|
||||
s3_put_min_bytes_per_second: int
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
# One upload may not take unlimited RAM/time: the body is spooled to a
|
||||
# temporary file (only upload_spool_memory_bytes stay in RAM), receiving
|
||||
# it is limited by upload_timeout_seconds, and at most
|
||||
# max_concurrent_uploads uploads per replica run at once.
|
||||
upload_spool_memory_bytes: int
|
||||
upload_timeout_seconds: float
|
||||
max_concurrent_uploads: int
|
||||
upload_slot_wait_seconds: float
|
||||
# Janitor: an upload intent older than upload_grace_seconds means the
|
||||
# request died between S3 PUT and COMMIT; its object is deleted.
|
||||
upload_grace_seconds: int
|
||||
janitor_interval_seconds: float
|
||||
# Experiments only (see REPORT.md): "" | crash_after_s3_put | db_error_after_s3_put
|
||||
fault_injection: str
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
session_store = os.getenv("SESSION_STORE", "postgres").lower()
|
||||
|
||||
if session_store not in {"postgres", "valkey", "redis"}:
|
||||
raise RuntimeError("SESSION_STORE must be postgres, valkey or redis.")
|
||||
|
||||
cache_url = None
|
||||
if session_store != "postgres":
|
||||
cache_url = read_value("CACHE_URL")
|
||||
|
||||
return Settings(
|
||||
app_host=os.getenv("APP_HOST", "0.0.0.0"),
|
||||
app_port=int(os.getenv("APP_PORT", "8080")),
|
||||
database_url=read_value("DATABASE_URL"),
|
||||
session_ttl_seconds=read_positive_int("SESSION_TTL_SECONDS", "3600"),
|
||||
db_pool_size=read_positive_int("DB_POOL_SIZE", "10"),
|
||||
db_max_overflow=int(os.getenv("DB_MAX_OVERFLOW", "5")),
|
||||
db_pool_timeout_seconds=read_positive_float("DB_POOL_TIMEOUT_SECONDS", "3"),
|
||||
db_connect_timeout_seconds=read_positive_float(
|
||||
"DB_CONNECT_TIMEOUT_SECONDS", "3"
|
||||
),
|
||||
db_statement_timeout_ms=read_positive_int("DB_STATEMENT_TIMEOUT_MS", "5000"),
|
||||
cache_url=cache_url,
|
||||
session_cache_ttl_seconds=read_positive_int(
|
||||
"SESSION_CACHE_TTL_SECONDS", "60"
|
||||
),
|
||||
cache_timeout_seconds=read_positive_float("CACHE_TIMEOUT_SECONDS", "0.25"),
|
||||
cache_max_connections=read_positive_int("CACHE_MAX_CONNECTIONS", "20"),
|
||||
cache_retry_after_seconds=read_positive_float(
|
||||
"CACHE_RETRY_AFTER_SECONDS", "5"
|
||||
),
|
||||
instance_id=os.getenv("INSTANCE_ID") or socket.gethostname(),
|
||||
idempotency_ttl_seconds=read_positive_int(
|
||||
"IDEMPOTENCY_TTL_SECONDS", "86400"
|
||||
),
|
||||
shutdown_drain_seconds=read_positive_float(
|
||||
"SHUTDOWN_DRAIN_SECONDS", "3"
|
||||
),
|
||||
shutdown_timeout_seconds=read_positive_int(
|
||||
"SHUTDOWN_TIMEOUT_SECONDS", "8"
|
||||
),
|
||||
s3_endpoint=read_value("S3_ENDPOINT", "http://s3:9000"),
|
||||
s3_region=os.getenv("S3_REGION", "us-east-1"),
|
||||
s3_bucket=read_value("S3_BUCKET", "messenger-attachments"),
|
||||
s3_access_key_id=read_value("S3_ACCESS_KEY_ID"),
|
||||
s3_secret_access_key=read_value("S3_SECRET_ACCESS_KEY"),
|
||||
s3_object_prefix=os.getenv("S3_OBJECT_PREFIX", "originals/"),
|
||||
s3_connect_timeout_seconds=read_positive_float(
|
||||
"S3_CONNECT_TIMEOUT_SECONDS", "2"
|
||||
),
|
||||
s3_read_timeout_seconds=read_positive_float(
|
||||
"S3_READ_TIMEOUT_SECONDS", "5"
|
||||
),
|
||||
s3_operation_timeout_seconds=read_positive_float(
|
||||
"S3_OPERATION_TIMEOUT_SECONDS", "6"
|
||||
),
|
||||
s3_put_min_bytes_per_second=read_positive_int(
|
||||
"S3_PUT_MIN_BYTES_PER_SECOND", "1048576"
|
||||
),
|
||||
max_upload_bytes=read_positive_int("MAX_UPLOAD_BYTES", "10485760"),
|
||||
max_image_pixels=read_positive_int("MAX_IMAGE_PIXELS", "40000000"),
|
||||
upload_spool_memory_bytes=read_positive_int(
|
||||
"UPLOAD_SPOOL_MEMORY_BYTES", "1048576"
|
||||
),
|
||||
upload_timeout_seconds=read_positive_float(
|
||||
"UPLOAD_TIMEOUT_SECONDS", "60"
|
||||
),
|
||||
max_concurrent_uploads=read_positive_int("MAX_CONCURRENT_UPLOADS", "8"),
|
||||
upload_slot_wait_seconds=read_positive_float(
|
||||
"UPLOAD_SLOT_WAIT_SECONDS", "10"
|
||||
),
|
||||
upload_grace_seconds=read_positive_int("UPLOAD_GRACE_SECONDS", "900"),
|
||||
janitor_interval_seconds=read_positive_float(
|
||||
"JANITOR_INTERVAL_SECONDS", "60"
|
||||
),
|
||||
fault_injection=os.getenv("FAULT_INJECTION", ""),
|
||||
)
|
||||
|
||||
|
||||
settings = load_settings()
|
||||
@@ -0,0 +1,57 @@
|
||||
from collections.abc import AsyncGenerator
|
||||
|
||||
from sqlalchemy.ext.asyncio import (
|
||||
AsyncSession,
|
||||
async_sessionmaker,
|
||||
create_async_engine,
|
||||
)
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
database_url = settings.database_url.replace(
|
||||
"postgresql://",
|
||||
"postgresql+asyncpg://",
|
||||
1,
|
||||
)
|
||||
|
||||
# Bounded pool: at most db_pool_size + db_max_overflow connections per API
|
||||
# process. Waiting for a free connection, connecting and every statement are
|
||||
# limited, so an unavailable database produces a fast 503 instead of a hang.
|
||||
engine = create_async_engine(
|
||||
database_url,
|
||||
pool_pre_ping=True,
|
||||
pool_size=settings.db_pool_size,
|
||||
max_overflow=settings.db_max_overflow,
|
||||
pool_timeout=settings.db_pool_timeout_seconds,
|
||||
pool_recycle=1800,
|
||||
connect_args={
|
||||
"timeout": settings.db_connect_timeout_seconds,
|
||||
"command_timeout": settings.db_statement_timeout_ms / 1000 + 1,
|
||||
"server_settings": {
|
||||
"statement_timeout": str(settings.db_statement_timeout_ms),
|
||||
"idle_in_transaction_session_timeout": "10000",
|
||||
"application_name": "messenger-api",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
SessionLocal = async_sessionmaker(
|
||||
bind=engine,
|
||||
expire_on_commit=False,
|
||||
autoflush=False,
|
||||
)
|
||||
|
||||
|
||||
async def get_db() -> AsyncGenerator[AsyncSession, None]:
|
||||
async with SessionLocal() as session:
|
||||
yield session
|
||||
|
||||
|
||||
async def check_database_ready() -> bool:
|
||||
try:
|
||||
async with engine.connect() as connection:
|
||||
await connection.exec_driver_sql("SELECT 1")
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
"""
|
||||
FastAPI dependencies for Lab2:
|
||||
- Bearer authentication;
|
||||
- pagination query parameters.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, Query, Request, Security
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBasic, HTTPBearer
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import get_db
|
||||
from app.errors import Unauthorized
|
||||
from app.services import current_user_from_bearer_token
|
||||
|
||||
|
||||
# Security schemes make the "Authorize" button in /docs send real
|
||||
# Authorization headers. Errors keep the contract format: auto_error=False
|
||||
# and our own 401 responses.
|
||||
bearer_scheme = HTTPBearer(
|
||||
scheme_name="Bearer",
|
||||
auto_error=False,
|
||||
description="Opaque session token from POST /api/v1/auth/sessions.",
|
||||
)
|
||||
|
||||
|
||||
class BasicForDocs(HTTPBasic):
|
||||
"""Documents Basic auth; the header is parsed by app.security."""
|
||||
|
||||
async def __call__(self, request: Request) -> None: # type: ignore[override]
|
||||
return None
|
||||
|
||||
|
||||
basic_scheme = BasicForDocs(
|
||||
scheme_name="Basic",
|
||||
auto_error=False,
|
||||
description="Only for POST /api/v1/auth/sessions.",
|
||||
)
|
||||
|
||||
|
||||
async def require_user(
|
||||
credentials: Optional[HTTPAuthorizationCredentials] = Security(
|
||||
bearer_scheme
|
||||
),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
) -> dict:
|
||||
if credentials is None or not credentials.credentials.strip():
|
||||
raise Unauthorized("Bearer")
|
||||
|
||||
return await current_user_from_bearer_token(
|
||||
db,
|
||||
credentials.credentials.strip(),
|
||||
)
|
||||
|
||||
|
||||
def pagination_params(
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
) -> tuple[int, Optional[str]]:
|
||||
return limit, cursor
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
import re
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from uuid import uuid4
|
||||
|
||||
from app.config import settings
|
||||
from app.lifecycle import draining
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
code: str,
|
||||
message: str,
|
||||
headers: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class InvalidRequest(ApiError):
|
||||
def __init__(self, message: str = "Invalid request.") -> None:
|
||||
super().__init__(400, "invalid_request", message)
|
||||
|
||||
|
||||
class Unauthorized(ApiError):
|
||||
def __init__(self, scheme: str = "Bearer") -> None:
|
||||
if scheme == "Basic":
|
||||
challenge = 'Basic realm="messenger", charset="UTF-8"'
|
||||
else:
|
||||
challenge = 'Bearer realm="messenger"'
|
||||
|
||||
super().__init__(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication is required.",
|
||||
{"WWW-Authenticate": challenge},
|
||||
)
|
||||
|
||||
|
||||
class NotFound(ApiError):
|
||||
def __init__(self, message: str = "Object not found.") -> None:
|
||||
super().__init__(404, "not_found", message)
|
||||
|
||||
|
||||
class Conflict(ApiError):
|
||||
def __init__(self, message: str = "Conflict.") -> None:
|
||||
super().__init__(409, "conflict", message)
|
||||
|
||||
|
||||
class Unavailable(ApiError):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(503, "unavailable", "Service is unavailable.")
|
||||
|
||||
|
||||
class PayloadTooLarge(ApiError):
|
||||
def __init__(self, message: str = "Payload is too large.") -> None:
|
||||
super().__init__(413, "payload_too_large", message)
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
def __init__(
|
||||
self,
|
||||
message: str = "Only PNG and JPEG images are accepted.",
|
||||
) -> None:
|
||||
super().__init__(415, "unsupported_media_type", message)
|
||||
|
||||
|
||||
class RateLimited(ApiError):
|
||||
def __init__(self, message: str, retry_after_seconds: int = 1) -> None:
|
||||
super().__init__(
|
||||
429,
|
||||
"rate_limited",
|
||||
message,
|
||||
{"Retry-After": str(retry_after_seconds)},
|
||||
)
|
||||
|
||||
# X-Request-Id from HAProxy (or the client) is reused only if it is short and
|
||||
# printable; anything else is replaced, so it cannot inject into logs.
|
||||
REQUEST_ID_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
def incoming_request_id(request: Request) -> str:
|
||||
value = request.headers.get("X-Request-Id", "")
|
||||
if REQUEST_ID_PATTERN.fullmatch(value):
|
||||
return value
|
||||
return str(uuid4())
|
||||
|
||||
|
||||
def request_id(request: Request) -> str:
|
||||
return getattr(request.state, "request_id", str(uuid4()))
|
||||
|
||||
|
||||
def common_headers(request: Request) -> dict[str, str]:
|
||||
"""Headers of every API response, including errors (lab3: X-Instance-Id)."""
|
||||
headers = {
|
||||
"X-Request-Id": request_id(request),
|
||||
"X-Instance-Id": settings.instance_id,
|
||||
}
|
||||
if draining.is_set():
|
||||
# Ask HAProxy not to reuse the connection to a replica that is stopping.
|
||||
headers["Connection"] = "close"
|
||||
return headers
|
||||
|
||||
|
||||
async def api_error_handler(
|
||||
request: Request,
|
||||
exc: ApiError,
|
||||
) -> JSONResponse:
|
||||
rid = request_id(request)
|
||||
|
||||
headers = {
|
||||
**common_headers(request),
|
||||
**exc.headers,
|
||||
}
|
||||
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": exc.code,
|
||||
"message": exc.message,
|
||||
"request_id": rid,
|
||||
}
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Disposable, resource-limited image decoder. No app credentials/config.
|
||||
|
||||
Usage: python -m app.image_worker INPUT OUTPUT_DIR. Result is manifest.json.
|
||||
All input comes from a trusted staging object selected by the worker in PG.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import warnings
|
||||
|
||||
|
||||
def fit_size(width: int, height: int, limit: int) -> tuple[int, int]:
|
||||
largest = max(width, height)
|
||||
if largest <= limit:
|
||||
return width, height
|
||||
return max(1, width * limit // largest), max(1, height * limit // largest)
|
||||
|
||||
|
||||
def convert(source: Path, directory: Path) -> list[dict]:
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
Image.MAX_IMAGE_PIXELS = int(os.getenv("MAX_IMAGE_PIXELS", "40000000"))
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
with Image.open(source) as opened:
|
||||
if opened.format not in {"PNG", "JPEG"}:
|
||||
raise ValueError("invalid_image")
|
||||
if opened.width * opened.height > Image.MAX_IMAGE_PIXELS:
|
||||
raise ValueError("image_too_large")
|
||||
opened.load()
|
||||
oriented = ImageOps.exif_transpose(opened)
|
||||
# Detach metadata and use fixed output modes. The original file
|
||||
# itself is never rewritten; EXIF orientation affects variants only.
|
||||
image = oriented.convert("RGBA" if "A" in oriented.getbands() or
|
||||
"transparency" in oriented.info else "RGB")
|
||||
oriented.close()
|
||||
image.info.clear()
|
||||
if image.mode == "RGBA":
|
||||
rgb = Image.new("RGB", image.size, "white")
|
||||
rgb.paste(image, mask=image.getchannel("A"))
|
||||
else:
|
||||
rgb = image
|
||||
|
||||
variants = []
|
||||
|
||||
def save(name, result, filename, content_type, **options):
|
||||
path = directory / filename
|
||||
result.info.clear()
|
||||
result.save(path, **options)
|
||||
data = path.read_bytes()
|
||||
variants.append({"name": name, "filename": filename, "content_type": content_type,
|
||||
"width": result.width, "height": result.height,
|
||||
"size_bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()})
|
||||
if result is not rgb and result is not image:
|
||||
result.close()
|
||||
|
||||
save("thumbnail", rgb.resize(fit_size(*rgb.size, 256), Image.Resampling.LANCZOS),
|
||||
"thumbnail.jpg", "image/jpeg", format="JPEG", quality=85, optimize=True)
|
||||
save("crop", ImageOps.fit(rgb, (128, 128), method=Image.Resampling.LANCZOS,
|
||||
centering=(0.5, 0.5)),
|
||||
"crop.jpg", "image/jpeg", format="JPEG", quality=85, optimize=True)
|
||||
save("optimized", image.resize(fit_size(*image.size, 1280), Image.Resampling.LANCZOS),
|
||||
"optimized.webp", "image/webp", format="WEBP", quality=80, method=4)
|
||||
if rgb is not image:
|
||||
rgb.close()
|
||||
image.close()
|
||||
return variants
|
||||
|
||||
|
||||
def main() -> int:
|
||||
# Linux container limits are inherited by every native decoder. Parent
|
||||
# enforces a wall timeout and kills/reaps this process on expiration.
|
||||
if sys.platform == "linux":
|
||||
import resource
|
||||
memory = int(os.getenv("WORKER_MEMORY_MB", "768")) * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
cpu = math.ceil(float(os.getenv("JOB_TIMEOUT_SECONDS", "30"))) + 1
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu))
|
||||
try:
|
||||
source, directory = Path(sys.argv[1]), Path(sys.argv[2])
|
||||
manifest = convert(source, directory)
|
||||
(directory / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
||||
return 0
|
||||
except MemoryError:
|
||||
return 21
|
||||
except Exception:
|
||||
return 20
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
"""Content-based validation of uploaded PNG/JPEG images.
|
||||
|
||||
The declared Content-Type is never trusted: the format is detected from the
|
||||
first bytes and must match it. Dimensions are read from the header and the
|
||||
pixel limit is applied BEFORE any decoding, so a small "pixel bomb" file
|
||||
cannot make the API allocate gigabytes.
|
||||
|
||||
- PNG is validated by our own streaming reader: critical chunks, CRC of every
|
||||
chunk, IHDR, and the IDAT zlib stream is decompressed in 64 KiB pieces.
|
||||
Scanline sizes and filter bytes (also in Adam7 passes) are checked without
|
||||
retaining any row. Memory stays O(64 KiB) for any image.
|
||||
- JPEG: Pillow reads the header (lazy open); then the image is decoded with
|
||||
draft() at 1/8 scale, which makes libjpeg read the whole entropy-coded
|
||||
stream (a truncated/corrupted file raises) while allocating only 1/64 of
|
||||
the full-size pixel buffer. Large progressive JPEGs can still retain DCT
|
||||
coefficients; the caller must bound concurrent validation and memory.
|
||||
Their entropy stream is checked as well: there is no header-only bypass.
|
||||
|
||||
The functions are synchronous (CPU work); call them via asyncio.to_thread.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
from typing import BinaryIO
|
||||
|
||||
from PIL import Image
|
||||
|
||||
# Our own limit is applied before Pillow decodes anything.
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SIGNATURE = b"\xff\xd8\xff"
|
||||
CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class InvalidImage(Exception):
|
||||
"""Not a PNG/JPEG, a corrupted file or a Content-Type mismatch (-> 415)."""
|
||||
|
||||
|
||||
class TooManyPixels(Exception):
|
||||
"""Decoded image would exceed MAX_IMAGE_PIXELS (-> 413)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ImageInfo:
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
def sniff_content_type(head: bytes) -> str | None:
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return "image/png"
|
||||
if head.startswith(JPEG_SIGNATURE):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def validate_image(
|
||||
file: BinaryIO,
|
||||
declared_content_type: str,
|
||||
max_pixels: int,
|
||||
) -> ImageInfo:
|
||||
file.seek(0)
|
||||
detected = sniff_content_type(file.read(len(PNG_SIGNATURE)))
|
||||
file.seek(0)
|
||||
|
||||
if detected is None:
|
||||
raise InvalidImage("Content is not a PNG or JPEG image.")
|
||||
|
||||
if detected != declared_content_type:
|
||||
raise InvalidImage("Content-Type does not match the image content.")
|
||||
|
||||
if detected == "image/png":
|
||||
width, height = _validate_png(file, max_pixels)
|
||||
else:
|
||||
width, height = _validate_jpeg(file, max_pixels)
|
||||
|
||||
return ImageInfo(detected, width, height)
|
||||
|
||||
|
||||
# --- PNG -------------------------------------------------------------------
|
||||
|
||||
# color type -> (channels, allowed bit depths)
|
||||
_PNG_COLOR_TYPES = {
|
||||
0: (1, {1, 2, 4, 8, 16}),
|
||||
2: (3, {8, 16}),
|
||||
3: (1, {1, 2, 4, 8}),
|
||||
4: (2, {8, 16}),
|
||||
6: (4, {8, 16}),
|
||||
}
|
||||
_ADAM7_PASSES = (
|
||||
(0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4),
|
||||
(0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2),
|
||||
)
|
||||
|
||||
|
||||
def _read_exact(file: BinaryIO, size: int) -> bytes:
|
||||
data = file.read(size)
|
||||
if len(data) != size:
|
||||
raise InvalidImage("Image file is truncated.")
|
||||
return data
|
||||
|
||||
|
||||
def _png_scanlines(width: int, height: int, bits_per_pixel: int,
|
||||
interlace: int) -> list[tuple[int, int]]:
|
||||
"""At most seven (packed row bytes, row count) pairs; no row allocation."""
|
||||
if interlace == 0:
|
||||
return [((width * bits_per_pixel + 7) // 8, height)]
|
||||
passes = []
|
||||
for x0, y0, dx, dy in _ADAM7_PASSES:
|
||||
w = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
h = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
if w and h:
|
||||
passes.append(((w * bits_per_pixel + 7) // 8, h))
|
||||
return passes
|
||||
|
||||
|
||||
class _PngScanlines:
|
||||
"""Validate filter positions across arbitrary decompression boundaries."""
|
||||
def __init__(self, passes: list[tuple[int, int]]) -> None:
|
||||
self.passes = iter(passes)
|
||||
self.row_bytes, self.rows_left = next(self.passes)
|
||||
self.row_remaining = 0
|
||||
self.expected = sum((row_bytes + 1) * count for row_bytes, count in passes)
|
||||
self.produced = 0
|
||||
|
||||
def feed(self, data: bytes) -> None:
|
||||
self.produced += len(data)
|
||||
if self.produced > self.expected:
|
||||
raise InvalidImage("PNG image data is larger than IHDR declares.")
|
||||
position = 0
|
||||
while position < len(data):
|
||||
if self.row_remaining == 0:
|
||||
if self.rows_left == 0:
|
||||
try:
|
||||
self.row_bytes, self.rows_left = next(self.passes)
|
||||
except StopIteration:
|
||||
raise InvalidImage("PNG has too many scanlines.") from None
|
||||
if data[position] > 4:
|
||||
raise InvalidImage("Invalid PNG scanline filter.")
|
||||
position += 1
|
||||
self.rows_left -= 1
|
||||
self.row_remaining = self.row_bytes
|
||||
consumed = min(self.row_remaining, len(data) - position)
|
||||
self.row_remaining -= consumed
|
||||
position += consumed
|
||||
|
||||
|
||||
def _validate_png(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
_read_exact(file, len(PNG_SIGNATURE))
|
||||
|
||||
width = height = 0
|
||||
scanlines = None
|
||||
inflater = zlib.decompressobj()
|
||||
color_type = -1
|
||||
bit_depth = 0
|
||||
seen_plte = False
|
||||
palette_entries = 0
|
||||
seen_trns = False
|
||||
idat_state = "before" # before -> inside -> after
|
||||
first = True
|
||||
|
||||
while True:
|
||||
header = _read_exact(file, 8)
|
||||
length, chunk_type = struct.unpack(">I4s", header)
|
||||
|
||||
if length > 0x7FFFFFFF:
|
||||
raise InvalidImage("Invalid PNG chunk length.")
|
||||
|
||||
if first and chunk_type != b"IHDR":
|
||||
raise InvalidImage("PNG must start with IHDR.")
|
||||
if not all(65 <= ch <= 90 or 97 <= ch <= 122 for ch in chunk_type) \
|
||||
or not 65 <= chunk_type[2] <= 90:
|
||||
raise InvalidImage("Invalid PNG chunk type.")
|
||||
if chunk_type == b"IHDR" and (not first or length != 13):
|
||||
# Check before reading/buffering: an attacker cannot turn a
|
||||
# bogus multi-megabyte IHDR into a large allocation.
|
||||
raise InvalidImage("Invalid or duplicate PNG IHDR.")
|
||||
if chunk_type[0] < 97 and chunk_type not in {b"IHDR", b"PLTE", b"IDAT", b"IEND"}:
|
||||
raise InvalidImage("Unknown critical PNG chunk.")
|
||||
if chunk_type == b"IEND" and length != 0:
|
||||
raise InvalidImage("Invalid PNG IEND.")
|
||||
if chunk_type == b"PLTE":
|
||||
if seen_plte or seen_trns or idat_state != "before" or color_type in {0, 4} \
|
||||
or not 3 <= length <= 768 or length % 3:
|
||||
raise InvalidImage("Invalid PNG palette.")
|
||||
palette_entries = length // 3
|
||||
if color_type == 3 and palette_entries > 1 << bit_depth:
|
||||
raise InvalidImage("PNG palette exceeds its bit depth.")
|
||||
if chunk_type == b"tRNS":
|
||||
valid_length = ((color_type == 0 and length == 2) or
|
||||
(color_type == 2 and length == 6) or
|
||||
(color_type == 3 and seen_plte and 1 <= length <= palette_entries))
|
||||
if seen_trns or idat_state != "before" or not valid_length:
|
||||
raise InvalidImage("Invalid PNG transparency chunk.")
|
||||
seen_trns = True
|
||||
|
||||
crc = zlib.crc32(chunk_type)
|
||||
is_idat = chunk_type == b"IDAT"
|
||||
|
||||
if is_idat:
|
||||
if idat_state == "after":
|
||||
raise InvalidImage("PNG IDAT chunks are not consecutive.")
|
||||
if color_type == 3 and not seen_plte:
|
||||
raise InvalidImage("PNG palette image without PLTE.")
|
||||
idat_state = "inside"
|
||||
elif idat_state == "inside":
|
||||
idat_state = "after"
|
||||
|
||||
data = b""
|
||||
remaining = length
|
||||
while remaining:
|
||||
piece = _read_exact(file, min(CHUNK, remaining))
|
||||
remaining -= len(piece)
|
||||
crc = zlib.crc32(piece, crc)
|
||||
|
||||
if is_idat:
|
||||
_inflate_scanlines(inflater, piece, scanlines)
|
||||
elif chunk_type == b"IHDR":
|
||||
data += piece
|
||||
|
||||
(stored_crc,) = struct.unpack(">I", _read_exact(file, 4))
|
||||
if stored_crc != crc:
|
||||
raise InvalidImage("PNG chunk CRC mismatch.")
|
||||
|
||||
if first:
|
||||
first = False
|
||||
(width, height, bit_depth, color_type,
|
||||
compression, filter_method, interlace) = struct.unpack(">IIBBBBB", data)
|
||||
|
||||
if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF:
|
||||
raise InvalidImage("Invalid PNG dimensions.")
|
||||
if width * height > max_pixels:
|
||||
# Before a single IDAT byte is decompressed.
|
||||
raise TooManyPixels()
|
||||
if color_type not in _PNG_COLOR_TYPES:
|
||||
raise InvalidImage("Invalid PNG color type.")
|
||||
channels, depths = _PNG_COLOR_TYPES[color_type]
|
||||
if bit_depth not in depths or compression != 0 or filter_method != 0 \
|
||||
or interlace not in (0, 1):
|
||||
raise InvalidImage("Invalid PNG IHDR.")
|
||||
scanlines = _PngScanlines(
|
||||
_png_scanlines(width, height, channels * bit_depth, interlace)
|
||||
)
|
||||
elif chunk_type == b"PLTE":
|
||||
seen_plte = True
|
||||
elif chunk_type == b"IEND":
|
||||
break
|
||||
|
||||
if idat_state == "before":
|
||||
raise InvalidImage("PNG has no image data.")
|
||||
|
||||
# decompress(max_length) drains every unconsumed_tail above. No flush()
|
||||
# is needed (and its output would not have a strict allocation bound).
|
||||
if not inflater.eof or scanlines.produced != scanlines.expected:
|
||||
raise InvalidImage("PNG image data is corrupted or truncated.")
|
||||
|
||||
return width, height
|
||||
|
||||
|
||||
def _inflate_scanlines(inflater, data: bytes, scanlines: _PngScanlines) -> None:
|
||||
"""Decompress at most 64 KiB at once, validate, then discard."""
|
||||
try:
|
||||
while data:
|
||||
if inflater.eof:
|
||||
raise InvalidImage("PNG has data after the end of the zlib stream.")
|
||||
out = inflater.decompress(data, CHUNK)
|
||||
scanlines.feed(out)
|
||||
if inflater.unused_data:
|
||||
raise InvalidImage("PNG has data after the end of the zlib stream.")
|
||||
data = inflater.unconsumed_tail
|
||||
except zlib.error:
|
||||
raise InvalidImage("PNG image data is corrupted.") from None
|
||||
|
||||
|
||||
# --- JPEG ------------------------------------------------------------------
|
||||
|
||||
def _validate_jpeg(file: BinaryIO, max_pixels: int) -> tuple[int, int]:
|
||||
try:
|
||||
with Image.open(file, formats=["JPEG"]) as image:
|
||||
width, height = image.size
|
||||
|
||||
if width < 1 or height < 1:
|
||||
raise InvalidImage("Invalid JPEG dimensions.")
|
||||
if width * height > max_pixels:
|
||||
# Header only: no pixel has been decoded yet.
|
||||
raise TooManyPixels()
|
||||
|
||||
image.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
image.load()
|
||||
except (InvalidImage, TooManyPixels):
|
||||
raise
|
||||
except Exception:
|
||||
# Pillow raises OSError/SyntaxError/ValueError/... for broken data.
|
||||
raise InvalidImage("JPEG image is corrupted or truncated.") from None
|
||||
|
||||
return width, height
|
||||
|
||||
|
||||
+134
@@ -0,0 +1,134 @@
|
||||
"""Transactional outbox relay, expired-lease recovery and staging janitor.
|
||||
|
||||
DB COMMIT and RabbitMQ publish cannot be one transaction. Keep outbox rows
|
||||
until a publisher confirm, accepting harmless duplicate delivery on crash.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import SessionLocal, engine
|
||||
from app.pipeline import (RETENTION_SECONDS, arm_from_env, broker_channel,
|
||||
crash_if_armed, envelope, publish, retry_or_fail)
|
||||
from app.storage import storage
|
||||
|
||||
logger = logging.getLogger("messenger.jobs")
|
||||
|
||||
|
||||
async def recover_leases() -> int:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments WHERE status='processing' AND lease_until<now()
|
||||
ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED
|
||||
"""))
|
||||
rows = list(result.mappings())
|
||||
for row in rows:
|
||||
await retry_or_fail(db, row, "lease_expired")
|
||||
await db.commit()
|
||||
if rows:
|
||||
logger.warning("recovered expired leases count=%s", len(rows))
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def cleanup_staging() -> None:
|
||||
# One object transaction at a time: S3's bounded deadline is shorter
|
||||
# than idle_in_transaction_session_timeout. Replay takes the same lock.
|
||||
for _ in range(10):
|
||||
async with SessionLocal() as db:
|
||||
# Failed retention may remove five objects. Keep its row lock
|
||||
# until all bounded deletes finish, so replay cannot race it.
|
||||
await db.execute(text("SET LOCAL idle_in_transaction_session_timeout='60s'"))
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments
|
||||
WHERE staging_key IS NOT NULL AND NOT staging_deleted AND
|
||||
(status='ready' OR (status='failed' AND updated_at <
|
||||
now()-make_interval(secs => :retention)))
|
||||
ORDER BY updated_at LIMIT 1 FOR UPDATE SKIP LOCKED
|
||||
"""), {"retention": RETENTION_SECONDS})
|
||||
row = result.mappings().first()
|
||||
if row is None:
|
||||
await db.rollback()
|
||||
return
|
||||
await storage.delete(row["staging_key"])
|
||||
# Partial final objects of an exhausted job are no longer
|
||||
# useful after retention; ready objects are never deleted.
|
||||
if row["status"] == "failed":
|
||||
prefix = row["object_key"].rsplit("/", 1)[0]
|
||||
for suffix in ("original", "thumbnail.jpg", "crop.jpg", "optimized.webp"):
|
||||
await storage.delete(f"{prefix}/{suffix}")
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET staging_deleted=true WHERE id=:id
|
||||
"""), {"id": row["id"]})
|
||||
await db.commit()
|
||||
logger.info("staging cleaned attachment=%s", row["id"])
|
||||
|
||||
|
||||
async def relay_one(channel) -> bool:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT o.*,a.pipeline_version FROM outbox o
|
||||
JOIN attachments a ON a.id=o.attachment_id
|
||||
WHERE o.published_at IS NULL AND o.available_at<=now()
|
||||
ORDER BY o.id LIMIT 1 FOR UPDATE OF o SKIP LOCKED
|
||||
"""))
|
||||
row = result.mappings().first()
|
||||
if row is None:
|
||||
await db.rollback()
|
||||
return False
|
||||
# The upload COMMIT already exists; the pending row is untouched
|
||||
# by this crash, so the restarted relay finds it automatically.
|
||||
await crash_if_armed("crash_after_commit_before_publish")
|
||||
await publish(channel, envelope(row))
|
||||
await db.execute(text("UPDATE outbox SET published_at=now() WHERE id=:id"),
|
||||
{"id": row["id"]})
|
||||
await db.commit()
|
||||
logger.info("published job=%s attempt=%s", row["job_id"], row["attempt"])
|
||||
return True
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
for name in ("aio_pika", "aiormq"):
|
||||
logging.getLogger(name).setLevel(logging.CRITICAL)
|
||||
await storage.start()
|
||||
connection = None
|
||||
try:
|
||||
await arm_from_env("PUBLISHER_FAULT")
|
||||
while True:
|
||||
try:
|
||||
connection, channel, _ = await broker_channel()
|
||||
last_maintenance = 0.0
|
||||
while True:
|
||||
now = asyncio.get_running_loop().time()
|
||||
if now - last_maintenance > 1:
|
||||
await recover_leases()
|
||||
try:
|
||||
await cleanup_staging()
|
||||
except Exception as exc:
|
||||
# Retention is best effort. S3 failure must not
|
||||
# starve outbox delivery or exhaustible retries.
|
||||
logger.warning("staging cleanup deferred: %s", type(exc).__name__)
|
||||
last_maintenance = now
|
||||
if not await relay_one(channel):
|
||||
await asyncio.sleep(0.25)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
# Exception text can contain connection URLs. Log type only.
|
||||
logger.warning("relay unavailable: %s", type(exc).__name__)
|
||||
if connection:
|
||||
await connection.close()
|
||||
connection = None
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
if connection:
|
||||
await connection.close()
|
||||
await storage.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
"""Process lifecycle state shared by the server and request handlers."""
|
||||
import threading
|
||||
|
||||
# Set on the first SIGTERM/SIGINT. While draining, /health/ready answers 503
|
||||
# and responses carry "Connection: close", but requests are still served.
|
||||
draining = threading.Event()
|
||||
+460
@@ -0,0 +1,460 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import UUID
|
||||
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import Depends, FastAPI, Header, Query, Request, Response, Security
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from sqlalchemy import exc as sa_exc
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.attachments import (
|
||||
get_attachment,
|
||||
janitor_loop,
|
||||
open_attachment_content,
|
||||
upload_attachment,
|
||||
)
|
||||
from app.cache import session_cache
|
||||
from app.db import check_database_ready, engine, get_db
|
||||
from app.deps import basic_scheme, pagination_params, require_user
|
||||
from app.errors import (
|
||||
ApiError,
|
||||
InvalidRequest,
|
||||
Unavailable,
|
||||
api_error_handler,
|
||||
common_headers,
|
||||
incoming_request_id,
|
||||
)
|
||||
from app.lifecycle import draining
|
||||
from app.models import (
|
||||
Attachment,
|
||||
Chat,
|
||||
ChatPage,
|
||||
CreateChat,
|
||||
CreateMessage,
|
||||
CreateUser,
|
||||
Health,
|
||||
Message,
|
||||
MessagePage,
|
||||
Session,
|
||||
User,
|
||||
)
|
||||
from app.storage import storage
|
||||
from app.services import (
|
||||
create_chat,
|
||||
create_message,
|
||||
create_user_session,
|
||||
get_chat,
|
||||
list_chats,
|
||||
list_messages,
|
||||
logout_current_session,
|
||||
register_user,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI) -> AsyncIterator[None]:
|
||||
await storage.start()
|
||||
janitor = asyncio.create_task(janitor_loop())
|
||||
yield
|
||||
janitor.cancel()
|
||||
await asyncio.gather(janitor, return_exceptions=True)
|
||||
await storage.close()
|
||||
await session_cache.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="MTUSI Messenger — лабораторная 5",
|
||||
version="1.5.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_request_id(request: Request, call_next):
|
||||
request.state.request_id = incoming_request_id(request)
|
||||
|
||||
response = await call_next(request)
|
||||
response.headers.update(common_headers(request))
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
if not isinstance(exc, ApiError):
|
||||
raise exc
|
||||
|
||||
return await api_error_handler(request, exc)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_error_handler(
|
||||
request: Request,
|
||||
exc: StarletteHTTPException,
|
||||
) -> JSONResponse:
|
||||
# Unknown routes/methods: the contract error format instead of
|
||||
# FastAPI's default {"detail": ...}.
|
||||
if exc.status_code == 404:
|
||||
error = ApiError(404, "not_found", "Route not found.")
|
||||
else:
|
||||
error = ApiError(exc.status_code, "invalid_request", "Invalid request.")
|
||||
|
||||
return await api_error_handler(request, error)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(
|
||||
request: Request,
|
||||
_exc: RequestValidationError,
|
||||
) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=400,
|
||||
content={
|
||||
"error": {
|
||||
"code": "invalid_request",
|
||||
"message": "Invalid request.",
|
||||
"request_id": common_headers(request)["X-Request-Id"],
|
||||
}
|
||||
},
|
||||
headers=common_headers(request),
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.api")
|
||||
|
||||
|
||||
def is_dependency_failure(exc: Exception) -> bool:
|
||||
"""PostgreSQL is unreachable, restarting, overloaded or timed out."""
|
||||
if isinstance(
|
||||
exc,
|
||||
(
|
||||
sa_exc.OperationalError,
|
||||
sa_exc.InterfaceError,
|
||||
sa_exc.TimeoutError, # connection pool exhausted
|
||||
OSError,
|
||||
TimeoutError,
|
||||
),
|
||||
):
|
||||
return True
|
||||
|
||||
return isinstance(exc, sa_exc.DBAPIError) and exc.connection_invalidated
|
||||
|
||||
|
||||
async def dependency_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
if not is_dependency_failure(exc):
|
||||
return await unexpected_error_handler(request, exc)
|
||||
|
||||
logger.warning("database unavailable: %s", type(exc).__name__)
|
||||
return await api_error_handler(request, Unavailable())
|
||||
|
||||
|
||||
for _exc_class in (sa_exc.DBAPIError, sa_exc.TimeoutError, OSError, TimeoutError):
|
||||
app.add_exception_handler(_exc_class, dependency_error_handler)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unexpected_error_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
) -> JSONResponse:
|
||||
headers = common_headers(request)
|
||||
# Only the type: SQL parameters and headers may contain secrets.
|
||||
logger.error("unexpected error: %s", type(exc).__name__)
|
||||
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={
|
||||
"error": {
|
||||
"code": "internal_error",
|
||||
"message": "Internal server error.",
|
||||
"request_id": headers["X-Request-Id"],
|
||||
}
|
||||
},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/health/live",
|
||||
response_model=Health,
|
||||
)
|
||||
async def liveness() -> dict[str, str]:
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get(
|
||||
"/health/ready",
|
||||
response_model=Health,
|
||||
)
|
||||
async def readiness() -> dict[str, str]:
|
||||
# A draining replica reports "not ready" so HAProxy stops sending new
|
||||
# requests to it before the listener is closed.
|
||||
# S3 is deliberately NOT checked: without it only attachments degrade
|
||||
# (503), users/chats/messages keep working. If readiness depended on S3,
|
||||
# an S3 outage would take every replica out of the LB.
|
||||
if draining.is_set() or not await check_database_ready():
|
||||
raise Unavailable()
|
||||
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/users",
|
||||
status_code=201,
|
||||
response_model=User,
|
||||
)
|
||||
async def register_user_route(
|
||||
payload: CreateUser,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await register_user(
|
||||
db,
|
||||
username=payload.username,
|
||||
display_name=payload.display_name,
|
||||
password=payload.password,
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/auth/sessions",
|
||||
status_code=201,
|
||||
response_model=Session,
|
||||
)
|
||||
async def login(
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_basic: None = Security(basic_scheme),
|
||||
):
|
||||
# Contract: login takes only "Authorization: Basic", no request body.
|
||||
# Any body (malformed JSON, "{}", ...) is rejected before the password check.
|
||||
if await request.body():
|
||||
raise InvalidRequest("POST /auth/sessions does not accept a request body.")
|
||||
|
||||
return await create_user_session(
|
||||
db,
|
||||
request.headers.get("Authorization"),
|
||||
)
|
||||
|
||||
|
||||
@app.delete(
|
||||
"/api/v1/auth/sessions/current",
|
||||
status_code=204,
|
||||
)
|
||||
async def logout(
|
||||
request: Request,
|
||||
db: AsyncSession = Depends(get_db),
|
||||
_current_user: dict = Depends(require_user),
|
||||
) -> Response:
|
||||
_, _, token = request.headers["Authorization"].partition(" ")
|
||||
|
||||
await logout_current_session(
|
||||
db,
|
||||
token.strip(),
|
||||
)
|
||||
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/users/me",
|
||||
response_model=User,
|
||||
)
|
||||
async def current_user_route(
|
||||
current_user: dict = Depends(require_user),
|
||||
):
|
||||
return current_user
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats",
|
||||
status_code=201,
|
||||
response_model=Chat,
|
||||
)
|
||||
async def create_chat_route(
|
||||
payload: CreateChat,
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await create_chat(
|
||||
db,
|
||||
title=payload.title,
|
||||
member_ids=payload.member_ids,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats",
|
||||
response_model=ChatPage,
|
||||
)
|
||||
async def list_chats_route(
|
||||
page: tuple[int, str | None] = Depends(pagination_params),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
limit, cursor = page
|
||||
|
||||
return await list_chats(
|
||||
db,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats/{chat_id}",
|
||||
response_model=Chat,
|
||||
)
|
||||
async def get_chat_route(
|
||||
chat_id: UUID,
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await get_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
status_code=201,
|
||||
response_model=Message,
|
||||
)
|
||||
async def create_message_route(
|
||||
chat_id: UUID,
|
||||
payload: CreateMessage,
|
||||
response: Response,
|
||||
idempotency_key: str | None = Header(
|
||||
default=None,
|
||||
alias="Idempotency-Key",
|
||||
),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
message, replayed = await create_message(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
text_value=payload.text,
|
||||
attachment_ids=payload.attachment_ids,
|
||||
idempotency_key=idempotency_key,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
if replayed:
|
||||
response.headers["Idempotent-Replayed"] = "true"
|
||||
|
||||
return message
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
response_model=MessagePage,
|
||||
)
|
||||
async def list_messages_route(
|
||||
chat_id: UUID,
|
||||
page: tuple[int, str | None] = Depends(pagination_params),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
limit, cursor = page
|
||||
|
||||
return await list_messages(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
limit=limit,
|
||||
cursor=cursor,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
UPLOAD_BODY = {
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"description": "Raw image bytes, not multipart and not JSON/base64.",
|
||||
"content": {
|
||||
"image/png": {"schema": {"type": "string", "format": "binary"}},
|
||||
"image/jpeg": {"schema": {"type": "string", "format": "binary"}},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
status_code=202,
|
||||
response_model=Attachment,
|
||||
openapi_extra=UPLOAD_BODY,
|
||||
)
|
||||
async def upload_attachment_route(
|
||||
chat_id: UUID,
|
||||
request: Request,
|
||||
filename: str = Query(min_length=1, max_length=255),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await upload_attachment(
|
||||
request,
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
filename=filename,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/attachments/{attachment_id}",
|
||||
response_model=Attachment,
|
||||
)
|
||||
async def get_attachment_route(
|
||||
attachment_id: UUID,
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
return await get_attachment(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
current_user=current_user,
|
||||
)
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/attachments/{attachment_id}/content",
|
||||
response_class=StreamingResponse,
|
||||
responses={200: {"content": {"image/png": {}, "image/jpeg": {}, "image/webp": {}}}},
|
||||
)
|
||||
async def download_attachment_route(
|
||||
attachment_id: UUID,
|
||||
# Known variant names are accepted so that a ready attachment without
|
||||
# that variant answers 404 (contract); any other value is 400.
|
||||
variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(
|
||||
default="original"
|
||||
),
|
||||
current_user: dict = Depends(require_user),
|
||||
db: AsyncSession = Depends(get_db),
|
||||
):
|
||||
chunks, headers = await open_attachment_content(
|
||||
db,
|
||||
attachment_id=attachment_id,
|
||||
variant=variant,
|
||||
current_user=current_user,
|
||||
)
|
||||
return StreamingResponse(
|
||||
chunks,
|
||||
media_type=headers.pop("Content-Type"),
|
||||
headers=headers,
|
||||
)
|
||||
+193
@@ -0,0 +1,193 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
|
||||
|
||||
class ApiModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class CreateUser(ApiModel):
|
||||
username: str = Field(
|
||||
min_length=3,
|
||||
max_length=32,
|
||||
pattern=r"^[a-z0-9_]{3,32}$",
|
||||
)
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
@classmethod
|
||||
def username_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("username must not contain only spaces.")
|
||||
return value
|
||||
|
||||
@field_validator("display_name")
|
||||
@classmethod
|
||||
def display_name_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError(
|
||||
"display_name must not contain only spaces."
|
||||
)
|
||||
return value
|
||||
|
||||
@field_validator("password")
|
||||
@classmethod
|
||||
def password_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("password must not contain only spaces.")
|
||||
return value
|
||||
|
||||
|
||||
class User(ApiModel):
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def user_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("created_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class CreateChat(ApiModel):
|
||||
# Contract: 1-100 code points (lab2 allowed 255 by mistake).
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
# Contract: 1-100 members (minItems/maxItems in openapi.json).
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def title_is_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("title must not contain only spaces.")
|
||||
return value
|
||||
|
||||
|
||||
class Chat(ApiModel):
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: datetime
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def chat_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("created_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class ChatPage(ApiModel):
|
||||
items: list[Chat]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class CreateMessage(ApiModel):
|
||||
# Lab4: text may be "" when the message carries attachments.
|
||||
text: str = Field(max_length=4000)
|
||||
# 0-10 unique IDs of attachments of the same chat; missing == [].
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator("attachment_ids")
|
||||
@classmethod
|
||||
def attachment_ids_are_unique(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError("attachment_ids must be unique.")
|
||||
return value
|
||||
|
||||
@model_validator(mode="after")
|
||||
def text_or_attachments(self) -> "CreateMessage":
|
||||
if not self.attachment_ids and not self.text.strip():
|
||||
raise ValueError(
|
||||
"text must not be blank when there are no attachments."
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class Message(ApiModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[UUID] = Field(default_factory=list)
|
||||
|
||||
@field_validator("created_at")
|
||||
@classmethod
|
||||
def message_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("created_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class MessagePage(ApiModel):
|
||||
items: list[Message]
|
||||
next_cursor: str | None = None
|
||||
|
||||
|
||||
class Session(ApiModel):
|
||||
token: str = Field(min_length=1)
|
||||
token_type: Literal["Bearer"]
|
||||
expires_at: datetime
|
||||
user: User
|
||||
|
||||
@field_validator("expires_at")
|
||||
@classmethod
|
||||
def expiry_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("expires_at must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class Variant(ApiModel):
|
||||
name: Literal["thumbnail", "crop", "optimized"]
|
||||
content_type: Literal["image/jpeg", "image/png", "image/webp"]
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class Attachment(ApiModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: Literal["image/png", "image/jpeg"]
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: Literal["queued", "processing", "ready", "failed"]
|
||||
variants: list[Variant]
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
@field_validator("created_at", "updated_at")
|
||||
@classmethod
|
||||
def attachment_time_is_utc(cls, value: datetime) -> datetime:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("timestamps must include UTC timezone.")
|
||||
return value.astimezone(timezone.utc)
|
||||
|
||||
|
||||
class Health(ApiModel):
|
||||
status: Literal["ok"]
|
||||
|
||||
|
||||
class ErrorInner(ApiModel):
|
||||
code: str = Field(min_length=1)
|
||||
message: str = Field(min_length=1)
|
||||
request_id: str = Field(min_length=1)
|
||||
|
||||
|
||||
class Error(ApiModel):
|
||||
error: ErrorInner
|
||||
@@ -0,0 +1,92 @@
|
||||
"""Explicit local operator commands; no new public HTTP endpoints."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.db import SessionLocal, engine
|
||||
from app.pipeline import event
|
||||
|
||||
POINTS = ("crash_after_commit_before_publish", "crash_after_objects_before_commit",
|
||||
"crash_after_commit_before_ack")
|
||||
|
||||
|
||||
async def run(args) -> None:
|
||||
try:
|
||||
async with SessionLocal() as db:
|
||||
if args.command == "fault":
|
||||
await db.execute(text("""
|
||||
INSERT INTO fault_injections(point,remaining) VALUES (:point,1)
|
||||
ON CONFLICT(point) DO UPDATE SET remaining=1,updated_at=now()
|
||||
"""), {"point": args.point})
|
||||
await db.commit()
|
||||
print(f"ARMED one-shot fault: {args.point}")
|
||||
return
|
||||
if args.command == "failed":
|
||||
result = await db.execute(text("""
|
||||
SELECT id,job_id,attempt,attempts_run,error_code,staging_deleted,updated_at
|
||||
FROM attachments WHERE status='failed' ORDER BY updated_at
|
||||
"""))
|
||||
print(json.dumps([dict(row) for row in result.mappings()], default=str, indent=2))
|
||||
return
|
||||
result = await db.execute(text("SELECT * FROM attachments WHERE id=:id FOR UPDATE"),
|
||||
{"id": args.attachment_id})
|
||||
row = result.mappings().first()
|
||||
if row is None or row["job_id"] is None:
|
||||
raise ValueError("Unknown asynchronous attachment")
|
||||
if args.command == "replay":
|
||||
if row["status"] == "ready":
|
||||
print("NO-OP: attachment is already ready")
|
||||
return
|
||||
if row["status"] != "failed":
|
||||
raise ValueError("Replay is allowed only for failed attachments")
|
||||
if row["staging_deleted"]:
|
||||
raise ValueError("Replay source expired; upload the original again")
|
||||
if not args.reason.strip() or len(args.reason) > 200:
|
||||
raise ValueError("Replay reason must have 1-200 characters")
|
||||
result = await db.execute(text("""
|
||||
UPDATE attachments SET status='queued',attempt=attempt+1,attempts_run=0,
|
||||
error_code=NULL,lease_token=NULL,lease_until=NULL,
|
||||
next_attempt_at=now(),updated_at=now() WHERE id=:id RETURNING *
|
||||
"""), {"id": row["id"]})
|
||||
row = result.mappings().one()
|
||||
await event(db, row, "operator_replay", args.reason)
|
||||
else:
|
||||
# Real duplicate envelope: same job ID and current attempt.
|
||||
# Ready effects are checked in PG by the worker and acked.
|
||||
await event(db, row, "operator_redeliver")
|
||||
await db.execute(text("""
|
||||
INSERT INTO outbox(job_id,attachment_id,attempt,available_at)
|
||||
SELECT job_id,id,attempt,GREATEST(now(),next_attempt_at)
|
||||
FROM attachments WHERE id=:id
|
||||
"""), {"id": row["id"]})
|
||||
await db.commit()
|
||||
print(f"QUEUED: {args.command} attachment={row['id']} job={row['job_id']}")
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
commands.add_parser("failed")
|
||||
fault = commands.add_parser("fault")
|
||||
fault.add_argument("point", choices=POINTS)
|
||||
replay = commands.add_parser("replay")
|
||||
replay.add_argument("attachment_id", type=UUID)
|
||||
replay.add_argument("--reason", required=True)
|
||||
redeliver = commands.add_parser("redeliver")
|
||||
redeliver.add_argument("attachment_id", type=UUID)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
asyncio.run(run(args))
|
||||
except ValueError as exc:
|
||||
parser.exit(2, f"REFUSED: {exc}\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
"""Shared durable-job plumbing, deliberately separate from the HTTP API."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
import aio_pika
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import read_value
|
||||
from app.db import SessionLocal
|
||||
|
||||
logger = logging.getLogger("messenger.pipeline")
|
||||
QUEUE = "attachments"
|
||||
MAX_ATTEMPTS = int(os.getenv("JOB_MAX_ATTEMPTS", "5"))
|
||||
LEASE_SECONDS = int(os.getenv("JOB_LEASE_SECONDS", "120"))
|
||||
IMAGE_TIMEOUT = float(os.getenv("JOB_TIMEOUT_SECONDS", "30"))
|
||||
TOTAL_TIMEOUT = float(os.getenv("JOB_TOTAL_TIMEOUT_SECONDS", "90"))
|
||||
RETENTION_SECONDS = int(os.getenv("STAGING_RETENTION_SECONDS", "604800"))
|
||||
RETRY_BASE_SECONDS = float(os.getenv("RETRY_BASE_SECONDS", "1"))
|
||||
if (MAX_ATTEMPTS < 1 or not math.isfinite(IMAGE_TIMEOUT) or IMAGE_TIMEOUT <= 0
|
||||
or not 0 < TOTAL_TIMEOUT < LEASE_SECONDS):
|
||||
raise RuntimeError("Invalid job limits; total timeout must be below lease.")
|
||||
if not math.isfinite(RETRY_BASE_SECONDS) or RETRY_BASE_SECONDS <= 0:
|
||||
raise RuntimeError("RETRY_BASE_SECONDS must be finite and positive.")
|
||||
|
||||
|
||||
def retry_delay(attempts_run: int) -> float:
|
||||
# Equal jitter: never zero, capped exponential delay, one durable due date.
|
||||
upper = min(60.0, RETRY_BASE_SECONDS * 2 ** min(max(attempts_run - 1, 0), 10))
|
||||
return random.uniform(upper / 2, upper)
|
||||
|
||||
|
||||
async def broker_channel():
|
||||
connection = await aio_pika.connect_robust(read_value("BROKER_URL"), timeout=5)
|
||||
try:
|
||||
channel = await connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
await channel.set_qos(prefetch_count=1)
|
||||
dead = await channel.declare_exchange("attachments.dead", aio_pika.ExchangeType.FANOUT,
|
||||
durable=True)
|
||||
dlq = await channel.declare_queue("attachments.dlq", durable=True)
|
||||
await dlq.bind(dead)
|
||||
queue = await channel.declare_queue(
|
||||
QUEUE, durable=True, arguments={"x-dead-letter-exchange": "attachments.dead"}
|
||||
)
|
||||
return connection, channel, queue
|
||||
except BaseException:
|
||||
await connection.close()
|
||||
raise
|
||||
|
||||
|
||||
def envelope(row) -> dict:
|
||||
return {
|
||||
"schema_version": 1, "job_id": str(row["job_id"]),
|
||||
"type": "attachment.process.v1", "attachment_id": str(row["attachment_id"]),
|
||||
"pipeline_version": int(row.get("pipeline_version", 1)),
|
||||
"attempt": int(row["attempt"]),
|
||||
"created_at": row["created_at"].isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def parse_envelope(raw: bytes) -> dict:
|
||||
job = json.loads(raw)
|
||||
required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version",
|
||||
"attempt", "created_at"}
|
||||
if not isinstance(job, dict) or not required <= job.keys() or (
|
||||
job.keys() - required - {"traceparent", "tracestate"}
|
||||
):
|
||||
raise ValueError("invalid job fields")
|
||||
if type(job["schema_version"]) is not int or job["schema_version"] != 1 or job["type"] != "attachment.process.v1":
|
||||
raise ValueError("unsupported job schema")
|
||||
if type(job["attempt"]) is not int or job["attempt"] < 1:
|
||||
raise ValueError("invalid attempt")
|
||||
if type(job["pipeline_version"]) is not int or job["pipeline_version"] != 1:
|
||||
raise ValueError("unsupported pipeline")
|
||||
UUID(job["job_id"])
|
||||
UUID(job["attachment_id"])
|
||||
if datetime.fromisoformat(job["created_at"].replace("Z", "+00:00")).tzinfo is None:
|
||||
raise ValueError("job timestamp must include timezone")
|
||||
if "traceparent" in job and not re.fullmatch(
|
||||
r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}", job["traceparent"]
|
||||
):
|
||||
raise ValueError("invalid traceparent")
|
||||
if "tracestate" in job and (not isinstance(job["tracestate"], str) or
|
||||
len(job["tracestate"]) > 512):
|
||||
raise ValueError("invalid tracestate")
|
||||
return job
|
||||
|
||||
|
||||
async def publish(channel, job: dict) -> None:
|
||||
await channel.default_exchange.publish(
|
||||
aio_pika.Message(json.dumps(job).encode("utf-8"), content_type="application/json",
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=job["job_id"]),
|
||||
routing_key=QUEUE, mandatory=True, timeout=5,
|
||||
)
|
||||
|
||||
|
||||
async def event(db, row, name: str, detail: str = "") -> None:
|
||||
await db.execute(text("""
|
||||
INSERT INTO job_events(attachment_id,event,attempt,detail)
|
||||
VALUES (:id,:event,:attempt,:detail)
|
||||
"""), {"id": row["id"], "attempt": row["attempt"], "event": name, "detail": detail})
|
||||
|
||||
|
||||
async def arm_from_env(variable: str) -> None:
|
||||
point = os.getenv(variable, "")
|
||||
if point:
|
||||
async with SessionLocal() as db:
|
||||
await db.execute(text("""
|
||||
INSERT INTO fault_injections(point,remaining) VALUES (:point,1)
|
||||
ON CONFLICT (point) DO NOTHING
|
||||
"""), {"point": point})
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def crash_if_armed(point: str) -> None:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
UPDATE fault_injections SET remaining=remaining-1,hits=hits+1,updated_at=now()
|
||||
WHERE point=:point AND remaining>0 RETURNING point
|
||||
"""), {"point": point})
|
||||
hit = result.first() is not None
|
||||
await db.commit()
|
||||
if hit:
|
||||
logger.error("FAULT: %s (one-shot durable hit)", point)
|
||||
os._exit(70)
|
||||
|
||||
|
||||
async def retry_or_fail(db, row, code: str, *, permanent: bool = False) -> None:
|
||||
"""Caller owns the attachment row lock. Commit before acking old delivery."""
|
||||
if permanent or row["attempts_run"] >= MAX_ATTEMPTS:
|
||||
final_code = code if permanent else "retry_exhausted"
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET status='failed',error_code=:code,
|
||||
lease_token=NULL,lease_until=NULL,updated_at=now()
|
||||
WHERE id=:id
|
||||
"""), {"id": row["id"], "code": final_code})
|
||||
await event(db, row, "failed", final_code)
|
||||
logger.warning("failed attachment=%s code=%s", row["id"], final_code)
|
||||
return
|
||||
delay = retry_delay(row["attempts_run"])
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET status='queued',error_code=NULL,attempt=attempt+1,
|
||||
lease_token=NULL,lease_until=NULL,
|
||||
next_attempt_at=now()+make_interval(secs => CAST(:delay AS double precision)),
|
||||
updated_at=now() WHERE id=:id
|
||||
"""), {"id": row["id"], "delay": delay})
|
||||
await db.execute(text("""
|
||||
INSERT INTO outbox(job_id,attachment_id,attempt,available_at)
|
||||
SELECT job_id,id,attempt,next_attempt_at FROM attachments WHERE id=:id
|
||||
"""), {"id": row["id"]})
|
||||
await event(db, row, "retry", f"{code}; delay={delay:.3f}s")
|
||||
logger.info("retry attachment=%s after=%.3fs", row["id"], delay)
|
||||
@@ -0,0 +1,718 @@
|
||||
"""PostgreSQL repository functions for MTUSI Messenger Lab4."""
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
|
||||
async def create_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
username: str,
|
||||
display_name: str,
|
||||
password_hash: str,
|
||||
) -> dict[str, Any] | None:
|
||||
user_id = uuid4()
|
||||
try:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (:id, :username, :display_name, :password_hash)
|
||||
RETURNING id, username, display_name, created_at
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": user_id,
|
||||
"username": username,
|
||||
"display_name": display_name,
|
||||
"password_hash": password_hash,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
return dict(result.mappings().one())
|
||||
except IntegrityError:
|
||||
await db.rollback()
|
||||
return None
|
||||
|
||||
|
||||
async def get_user_by_username(
|
||||
db: AsyncSession,
|
||||
username: str,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, username, display_name, password_hash, created_at
|
||||
FROM users
|
||||
WHERE username = :username
|
||||
"""
|
||||
),
|
||||
{"username": username},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def get_user_by_id(
|
||||
db: AsyncSession,
|
||||
user_id: UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, username, display_name, created_at
|
||||
FROM users
|
||||
WHERE id = :user_id
|
||||
"""
|
||||
),
|
||||
{"user_id": user_id},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def user_ids_that_exist(
|
||||
db: AsyncSession,
|
||||
member_ids: Iterable[UUID],
|
||||
) -> set[UUID]:
|
||||
values = list(member_ids)
|
||||
if not values:
|
||||
return set()
|
||||
|
||||
result = await db.execute(
|
||||
text("SELECT id FROM users WHERE id = ANY(CAST(:member_ids AS uuid[]))"),
|
||||
{"member_ids": values},
|
||||
)
|
||||
return {row.id for row in result}
|
||||
|
||||
|
||||
async def create_session(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: UUID,
|
||||
token_hash: str,
|
||||
expires_at: datetime,
|
||||
) -> None:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO sessions (id, user_id, token_hash, expires_at)
|
||||
VALUES (:id, :user_id, :token_hash, :expires_at)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": uuid4(),
|
||||
"user_id": user_id,
|
||||
"token_hash": token_hash,
|
||||
"expires_at": expires_at,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def get_active_session_user(
|
||||
db: AsyncSession,
|
||||
token_hash: str,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT
|
||||
s.id AS session_id,
|
||||
s.token_hash,
|
||||
s.expires_at,
|
||||
u.id,
|
||||
u.username,
|
||||
u.display_name,
|
||||
u.created_at
|
||||
FROM sessions AS s
|
||||
JOIN users AS u ON u.id = s.user_id
|
||||
WHERE s.token_hash = :token_hash
|
||||
AND s.revoked_at IS NULL
|
||||
AND s.expires_at > now()
|
||||
"""
|
||||
),
|
||||
{"token_hash": token_hash},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def revoke_active_session(
|
||||
db: AsyncSession,
|
||||
token_hash: str,
|
||||
) -> bool:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE sessions
|
||||
SET revoked_at = now()
|
||||
WHERE token_hash = :token_hash
|
||||
AND revoked_at IS NULL
|
||||
AND expires_at > now()
|
||||
"""
|
||||
),
|
||||
{"token_hash": token_hash},
|
||||
)
|
||||
await db.commit()
|
||||
return result.rowcount == 1
|
||||
|
||||
|
||||
async def create_chat_with_members(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
title: str,
|
||||
member_ids: Iterable[UUID],
|
||||
) -> dict[str, Any]:
|
||||
chat_id = uuid4()
|
||||
members = list(member_ids)
|
||||
|
||||
try:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO chats (id, title)
|
||||
VALUES (:id, :title)
|
||||
RETURNING id, title, created_at
|
||||
"""
|
||||
),
|
||||
{"id": chat_id, "title": title},
|
||||
)
|
||||
chat = dict(result.mappings().one())
|
||||
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO chat_members (chat_id, user_id)
|
||||
VALUES (:chat_id, :user_id)
|
||||
"""
|
||||
),
|
||||
[{"chat_id": chat_id, "user_id": user_id} for user_id in members],
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
chat["member_ids"] = members
|
||||
return chat
|
||||
|
||||
|
||||
async def get_accessible_chat(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
user_id: UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats AS c
|
||||
JOIN chat_members AS own_membership
|
||||
ON own_membership.chat_id = c.id
|
||||
AND own_membership.user_id = :user_id
|
||||
WHERE c.id = :chat_id
|
||||
"""
|
||||
),
|
||||
{"chat_id": chat_id, "user_id": user_id},
|
||||
)
|
||||
chat_row = result.mappings().first()
|
||||
if chat_row is None:
|
||||
return None
|
||||
|
||||
members_result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT user_id
|
||||
FROM chat_members
|
||||
WHERE chat_id = :chat_id
|
||||
ORDER BY user_id ASC
|
||||
"""
|
||||
),
|
||||
{"chat_id": chat_id},
|
||||
)
|
||||
chat = dict(chat_row)
|
||||
chat["member_ids"] = [row.user_id for row in members_result]
|
||||
return chat
|
||||
|
||||
|
||||
async def list_chats_for_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
user_id: UUID,
|
||||
limit: int,
|
||||
cursor_created_at: datetime | None,
|
||||
cursor_id: UUID | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
query = """
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats AS c
|
||||
JOIN chat_members AS own_membership
|
||||
ON own_membership.chat_id = c.id
|
||||
AND own_membership.user_id = :user_id \
|
||||
"""
|
||||
params: dict[str, Any] = {
|
||||
"user_id": user_id,
|
||||
"limit_plus_one": limit + 1,
|
||||
}
|
||||
|
||||
if cursor_created_at is not None and cursor_id is not None:
|
||||
query += """
|
||||
WHERE (c.created_at, c.id) > (:cursor_created_at, :cursor_id)
|
||||
"""
|
||||
params["cursor_created_at"] = cursor_created_at
|
||||
params["cursor_id"] = cursor_id
|
||||
|
||||
query += " ORDER BY c.created_at ASC, c.id ASC LIMIT :limit_plus_one"
|
||||
|
||||
result = await db.execute(text(query), params)
|
||||
chats = [dict(row) for row in result.mappings().all()]
|
||||
if not chats:
|
||||
return []
|
||||
|
||||
chat_ids = [chat["id"] for chat in chats]
|
||||
members_result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT chat_id, user_id
|
||||
FROM chat_members
|
||||
WHERE chat_id = ANY(CAST(:chat_ids AS uuid[]))
|
||||
ORDER BY chat_id ASC, user_id ASC
|
||||
"""
|
||||
),
|
||||
{"chat_ids": chat_ids},
|
||||
)
|
||||
|
||||
members_by_chat: dict[UUID, list[UUID]] = {chat_id: [] for chat_id in chat_ids}
|
||||
for row in members_result:
|
||||
members_by_chat[row.chat_id].append(row.user_id)
|
||||
|
||||
for chat in chats:
|
||||
chat["member_ids"] = members_by_chat[chat["id"]]
|
||||
|
||||
return chats
|
||||
|
||||
|
||||
# Attachment IDs of a message in the order of the request.
|
||||
MESSAGE_ATTACHMENT_IDS = """
|
||||
COALESCE(
|
||||
(SELECT array_agg(ma.attachment_id ORDER BY ma.position)
|
||||
FROM message_attachments AS ma
|
||||
WHERE ma.message_id = m.id),
|
||||
CAST('{}' AS uuid[])
|
||||
) AS attachment_ids
|
||||
"""
|
||||
|
||||
|
||||
class AttachmentsNotFound(Exception):
|
||||
"""Some attachment_ids do not exist or belong to another chat (-> 404)."""
|
||||
|
||||
|
||||
class AttachmentFailed(Exception):
|
||||
"""A failed attachment cannot be added to a new message (-> 409)."""
|
||||
|
||||
|
||||
async def create_message_idempotent(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text_value: str,
|
||||
attachment_ids: list[UUID],
|
||||
idempotency_key: str,
|
||||
request_hash: str,
|
||||
ttl_seconds: int,
|
||||
) -> tuple[dict[str, Any], bool] | None:
|
||||
"""Insert a message at most once per idempotency key; one transaction.
|
||||
|
||||
1. INSERT the key row. Its primary key (user, chat, method, key) lives in
|
||||
the shared PostgreSQL, so a concurrent request with the same key on
|
||||
ANY replica blocks on this INSERT until the first transaction ends.
|
||||
2. If the key was claimed: check that every attachment belongs to this
|
||||
chat, INSERT the message and its attachment links, link the message
|
||||
to the key and COMMIT everything together. A failure before COMMIT
|
||||
(also AttachmentsNotFound) rolls back all of it, so an error is never
|
||||
stored as the final answer and a retry with the same key runs again.
|
||||
3. If the key already exists (committed by someone else): return the
|
||||
stored message, or None when the body differs (-> 409).
|
||||
|
||||
A key older than ttl_seconds is reclaimed by the ON CONFLICT ... WHERE.
|
||||
Returns (message, replayed) or None on a body mismatch.
|
||||
"""
|
||||
key_params = {
|
||||
"user_id": sender_id,
|
||||
"chat_id": chat_id,
|
||||
"key": idempotency_key,
|
||||
}
|
||||
|
||||
try:
|
||||
claimed = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO idempotency_keys AS k
|
||||
(user_id, chat_id, method, key, request_hash)
|
||||
VALUES (:user_id, :chat_id, 'POST', :key, :request_hash)
|
||||
ON CONFLICT (user_id, chat_id, method, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash,
|
||||
message_id = NULL,
|
||||
created_at = now()
|
||||
WHERE k.created_at < now() - make_interval(secs => :ttl)
|
||||
RETURNING 1
|
||||
"""
|
||||
),
|
||||
{**key_params, "request_hash": request_hash, "ttl": ttl_seconds},
|
||||
)
|
||||
|
||||
if claimed.first() is not None:
|
||||
if attachment_ids:
|
||||
# Unknown and foreign IDs look the same (404): the response
|
||||
# does not reveal that an attachment of another chat exists.
|
||||
found = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, status FROM attachments
|
||||
WHERE chat_id = :chat_id
|
||||
AND id = ANY(CAST(:ids AS uuid[]))
|
||||
FOR SHARE
|
||||
"""
|
||||
),
|
||||
{"chat_id": chat_id, "ids": attachment_ids},
|
||||
)
|
||||
attachments = list(found)
|
||||
if {row.id for row in attachments} != set(attachment_ids):
|
||||
raise AttachmentsNotFound()
|
||||
if any(row.status == "failed" for row in attachments):
|
||||
raise AttachmentFailed()
|
||||
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO messages (id, chat_id, sender_id, text)
|
||||
VALUES (:id, :chat_id, :sender_id, :text)
|
||||
RETURNING id, chat_id, sender_id, text, created_at
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": uuid4(),
|
||||
"chat_id": chat_id,
|
||||
"sender_id": sender_id,
|
||||
"text": text_value,
|
||||
},
|
||||
)
|
||||
message = dict(result.mappings().one())
|
||||
|
||||
if attachment_ids:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO message_attachments
|
||||
(message_id, attachment_id, position)
|
||||
VALUES (:message_id, :attachment_id, :position)
|
||||
"""
|
||||
),
|
||||
[
|
||||
{
|
||||
"message_id": message["id"],
|
||||
"attachment_id": attachment_id,
|
||||
"position": position,
|
||||
}
|
||||
for position, attachment_id in enumerate(attachment_ids)
|
||||
],
|
||||
)
|
||||
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE idempotency_keys
|
||||
SET message_id = :message_id
|
||||
WHERE user_id = :user_id
|
||||
AND chat_id = :chat_id
|
||||
AND method = 'POST'
|
||||
AND key = :key
|
||||
"""
|
||||
),
|
||||
{**key_params, "message_id": message["id"]},
|
||||
)
|
||||
await db.commit()
|
||||
message["attachment_ids"] = list(attachment_ids)
|
||||
return message, False
|
||||
|
||||
stored = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT k.request_hash,
|
||||
m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{MESSAGE_ATTACHMENT_IDS}
|
||||
FROM idempotency_keys AS k
|
||||
JOIN messages AS m ON m.id = k.message_id
|
||||
WHERE k.user_id = :user_id
|
||||
AND k.chat_id = :chat_id
|
||||
AND k.method = 'POST'
|
||||
AND k.key = :key
|
||||
"""
|
||||
),
|
||||
key_params,
|
||||
)
|
||||
row = stored.mappings().first()
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
if row is None:
|
||||
# Not reachable while the key row and the message commit together.
|
||||
raise RuntimeError("idempotency key without a message")
|
||||
|
||||
message = dict(row)
|
||||
if message.pop("request_hash") != request_hash:
|
||||
return None
|
||||
|
||||
return message, True
|
||||
|
||||
|
||||
async def list_messages_keyset(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
limit: int,
|
||||
cursor_created_at: datetime | None,
|
||||
cursor_id: UUID | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
query = f"""
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
{MESSAGE_ATTACHMENT_IDS}
|
||||
FROM messages AS m
|
||||
WHERE m.chat_id = :chat_id
|
||||
"""
|
||||
params: dict[str, Any] = {
|
||||
"chat_id": chat_id,
|
||||
"limit_plus_one": limit + 1,
|
||||
}
|
||||
|
||||
if cursor_created_at is not None and cursor_id is not None:
|
||||
query += """
|
||||
AND (m.created_at, m.id) > (:cursor_created_at, :cursor_id)
|
||||
"""
|
||||
params["cursor_created_at"] = cursor_created_at
|
||||
params["cursor_id"] = cursor_id
|
||||
|
||||
query += " ORDER BY m.created_at ASC, m.id ASC LIMIT :limit_plus_one"
|
||||
result = await db.execute(text(query), params)
|
||||
return [dict(row) for row in result.mappings().all()]
|
||||
|
||||
|
||||
# --- Attachments ------------------------------------------------------------
|
||||
|
||||
ATTACHMENT_COLUMNS = """
|
||||
a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes,
|
||||
a.sha256, a.status, a.error_code, a.object_key, a.variants, a.created_at, a.updated_at
|
||||
"""
|
||||
|
||||
|
||||
async def create_upload_intent(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
object_key: str,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
) -> None:
|
||||
"""Committed BEFORE the S3 PUT: from now on the object can be found and
|
||||
cleaned up even if this process dies right after the PUT."""
|
||||
try:
|
||||
await db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO upload_intents (object_key, attachment_id, chat_id)
|
||||
VALUES (:object_key, :attachment_id, :chat_id)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"object_key": object_key,
|
||||
"attachment_id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
},
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def finalize_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
chat_id: UUID,
|
||||
owner_id: UUID,
|
||||
filename: str,
|
||||
content_type: str,
|
||||
size_bytes: int,
|
||||
sha256: str,
|
||||
width: int,
|
||||
height: int,
|
||||
object_key: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Delete the intent, insert queued metadata + outbox in ONE transaction.
|
||||
|
||||
Returns None if the intent is already gone: the janitor decided this
|
||||
upload was abandoned and deleted the object, so the row must not be
|
||||
created. The DELETE waits on the janitor's row lock, so the two can
|
||||
never both succeed.
|
||||
"""
|
||||
try:
|
||||
claimed = await db.execute(
|
||||
text(
|
||||
"DELETE FROM upload_intents WHERE object_key = :object_key RETURNING 1"
|
||||
),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
if claimed.first() is None:
|
||||
await db.rollback()
|
||||
return None
|
||||
|
||||
job_id = uuid4()
|
||||
final_key = f"originals/{attachment_id}/v1/original"
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
INSERT INTO attachments AS a
|
||||
(id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, width, height, object_key, staging_key, job_id, status)
|
||||
VALUES (:id, :chat_id, :owner_id, :filename, :content_type,
|
||||
:size_bytes, :sha256, :width, :height, :final_key, :object_key, :job_id, 'queued')
|
||||
RETURNING {ATTACHMENT_COLUMNS}
|
||||
"""
|
||||
),
|
||||
{
|
||||
"id": attachment_id,
|
||||
"chat_id": chat_id,
|
||||
"owner_id": owner_id,
|
||||
"filename": filename,
|
||||
"content_type": content_type,
|
||||
"size_bytes": size_bytes,
|
||||
"sha256": sha256,
|
||||
"width": width,
|
||||
"height": height,
|
||||
"object_key": object_key,
|
||||
"final_key": final_key,
|
||||
"job_id": job_id,
|
||||
},
|
||||
)
|
||||
row = dict(result.mappings().one())
|
||||
await db.execute(
|
||||
text("INSERT INTO outbox (job_id, attachment_id, attempt) VALUES (:job_id, :id, 1)"),
|
||||
{"job_id": job_id, "id": attachment_id},
|
||||
)
|
||||
await db.commit()
|
||||
return row
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def delete_upload_intent(db: AsyncSession, object_key: str) -> None:
|
||||
try:
|
||||
await db.execute(
|
||||
text("DELETE FROM upload_intents WHERE object_key = :object_key"),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
await db.commit()
|
||||
except Exception:
|
||||
await db.rollback()
|
||||
raise
|
||||
|
||||
|
||||
async def get_accessible_attachment(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
attachment_id: UUID,
|
||||
user_id: UUID,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Only members of the attachment's chat see it; others get None (404)."""
|
||||
result = await db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT {ATTACHMENT_COLUMNS}
|
||||
FROM attachments AS a
|
||||
JOIN chat_members AS cm
|
||||
ON cm.chat_id = a.chat_id
|
||||
AND cm.user_id = :user_id
|
||||
WHERE a.id = :attachment_id
|
||||
"""
|
||||
),
|
||||
{"attachment_id": attachment_id, "user_id": user_id},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def lock_upload_intent(
|
||||
db: AsyncSession,
|
||||
object_key: str,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Lock the intent of one upload (waits for a finishing transaction);
|
||||
the transaction stays open. None: no intent, nothing to compensate."""
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT i.object_key,
|
||||
EXISTS (SELECT 1 FROM attachments AS a
|
||||
WHERE a.staging_key = i.object_key OR a.object_key = i.object_key) AS committed
|
||||
FROM upload_intents AS i
|
||||
WHERE i.object_key = :object_key
|
||||
FOR UPDATE OF i
|
||||
"""
|
||||
),
|
||||
{"object_key": object_key},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def lock_stale_intent(
|
||||
db: AsyncSession,
|
||||
grace_seconds: int,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Lock ONE intent older than the grace period; the transaction stays open.
|
||||
|
||||
SKIP LOCKED: an upload that is finishing right now holds the row lock in
|
||||
finalize_attachment and is skipped; several replicas can run the janitor
|
||||
at once without touching the same intent.
|
||||
"""
|
||||
result = await db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT i.object_key,
|
||||
EXISTS (SELECT 1 FROM attachments AS a
|
||||
WHERE a.staging_key = i.object_key OR a.object_key = i.object_key) AS committed
|
||||
FROM upload_intents AS i
|
||||
WHERE i.created_at < now() - make_interval(secs => :grace)
|
||||
ORDER BY i.created_at
|
||||
LIMIT 1
|
||||
FOR UPDATE OF i SKIP LOCKED
|
||||
"""
|
||||
),
|
||||
{"grace": grace_seconds},
|
||||
)
|
||||
row = result.mappings().first()
|
||||
return dict(row) if row is not None else None
|
||||
|
||||
|
||||
async def attachment_keys(db: AsyncSession) -> set[str]:
|
||||
result = await db.execute(text("SELECT object_key FROM attachments"))
|
||||
return {row.object_key for row in result}
|
||||
|
||||
|
||||
async def intent_keys(db: AsyncSession) -> dict[str, datetime]:
|
||||
result = await db.execute(
|
||||
text("SELECT object_key, created_at FROM upload_intents")
|
||||
)
|
||||
return {row.object_key: row.created_at for row in result}
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerifyMismatchError
|
||||
|
||||
|
||||
password_hasher = PasswordHasher()
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return password_hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return password_hasher.verify(password_hash, password)
|
||||
except (VerifyMismatchError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def generate_session_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def hash_session_token(token: str) -> str:
|
||||
return hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def parse_basic_authorization(value: str | None) -> tuple[str, str] | None:
|
||||
if value is None or not value.startswith("Basic "):
|
||||
return None
|
||||
|
||||
try:
|
||||
encoded_credentials = value[len("Basic "):]
|
||||
decoded = base64.b64decode(
|
||||
encoded_credentials,
|
||||
validate=True,
|
||||
).decode("utf-8")
|
||||
|
||||
username, password = decoded.split(":", 1)
|
||||
except (
|
||||
ValueError,
|
||||
UnicodeDecodeError,
|
||||
binascii.Error,
|
||||
):
|
||||
return None
|
||||
|
||||
if not username or not username.isascii():
|
||||
return None
|
||||
|
||||
return username, password
|
||||
|
||||
|
||||
def _encode_cursor_payload(payload: dict[str, str]) -> str:
|
||||
raw = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
).encode("utf-8")
|
||||
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii")
|
||||
|
||||
|
||||
def _decode_cursor_payload(cursor: str) -> dict[str, str] | None:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
|
||||
payload = json.loads(raw.decode("utf-8"))
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
return None
|
||||
|
||||
return payload
|
||||
except (
|
||||
ValueError,
|
||||
TypeError,
|
||||
UnicodeDecodeError,
|
||||
binascii.Error,
|
||||
):
|
||||
return None
|
||||
|
||||
|
||||
def _utc_timestamp(value: datetime) -> str:
|
||||
if value.tzinfo is None:
|
||||
raise ValueError("Cursor datetime must be timezone-aware.")
|
||||
|
||||
return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def encode_chat_cursor(
|
||||
created_at: datetime,
|
||||
chat_id: UUID,
|
||||
) -> str:
|
||||
return _encode_cursor_payload(
|
||||
{
|
||||
"kind": "chat",
|
||||
"created_at": _utc_timestamp(created_at),
|
||||
"id": str(chat_id),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def decode_chat_cursor(
|
||||
cursor: str,
|
||||
) -> tuple[datetime, UUID] | None:
|
||||
payload = _decode_cursor_payload(cursor)
|
||||
|
||||
if payload is None or payload.get("kind") != "chat":
|
||||
return None
|
||||
|
||||
try:
|
||||
created_at = datetime.fromisoformat(
|
||||
payload["created_at"].replace("Z", "+00:00")
|
||||
)
|
||||
chat_id = UUID(payload["id"])
|
||||
except (KeyError, ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
return created_at, chat_id
|
||||
|
||||
|
||||
def encode_message_cursor(
|
||||
chat_id: UUID,
|
||||
created_at: datetime,
|
||||
message_id: UUID,
|
||||
) -> str:
|
||||
return _encode_cursor_payload(
|
||||
{
|
||||
"kind": "message",
|
||||
"chat_id": str(chat_id),
|
||||
"created_at": _utc_timestamp(created_at),
|
||||
"id": str(message_id),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def decode_message_cursor(
|
||||
cursor: str,
|
||||
) -> tuple[UUID, datetime, UUID] | None:
|
||||
payload = _decode_cursor_payload(cursor)
|
||||
|
||||
if payload is None or payload.get("kind") != "message":
|
||||
return None
|
||||
|
||||
try:
|
||||
chat_id = UUID(payload["chat_id"])
|
||||
created_at = datetime.fromisoformat(
|
||||
payload["created_at"].replace("Z", "+00:00")
|
||||
)
|
||||
message_id = UUID(payload["id"])
|
||||
except (KeyError, ValueError, AttributeError):
|
||||
return None
|
||||
|
||||
return chat_id, created_at, message_id
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Entry point: uvicorn with a drain phase before the graceful shutdown.
|
||||
|
||||
Plain uvicorn stops accepting connections immediately on SIGTERM. Behind
|
||||
HAProxy that turns every stop into a burst of refused connections until the
|
||||
active health check notices. Instead, on the first signal:
|
||||
|
||||
t = 0 readiness -> 503, keep serving (Connection: close);
|
||||
t = drain (3 s) HAProxy has marked the server DOWN (inter 1 s x fall 2),
|
||||
uvicorn closes the listener and waits for in-flight
|
||||
requests up to SHUTDOWN_TIMEOUT_SECONDS (8 s).
|
||||
|
||||
stop_grace_period in compose.yaml (15 s) is larger than 3 + 8 s.
|
||||
A second signal skips the drain phase.
|
||||
"""
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from app.config import settings
|
||||
from app.lifecycle import draining
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.server")
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
def handle_exit(self, sig, frame) -> None:
|
||||
if draining.is_set():
|
||||
super().handle_exit(sig, frame)
|
||||
return
|
||||
|
||||
draining.set()
|
||||
logger.warning(
|
||||
"signal %s: draining for %.1f s before shutdown",
|
||||
sig,
|
||||
settings.shutdown_drain_seconds,
|
||||
)
|
||||
# Set should_exit directly instead of super().handle_exit(): uvicorn
|
||||
# would re-raise the captured SIGTERM after shutdown and the process
|
||||
# would exit with 143 instead of a clean 0.
|
||||
timer = threading.Timer(
|
||||
settings.shutdown_drain_seconds,
|
||||
self._begin_shutdown,
|
||||
)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
def _begin_shutdown(self) -> None:
|
||||
logger.warning("drain finished: closing listener")
|
||||
self.should_exit = True
|
||||
|
||||
|
||||
def main() -> None:
|
||||
config = uvicorn.Config(
|
||||
"app.main:app",
|
||||
host=settings.app_host,
|
||||
port=settings.app_port,
|
||||
proxy_headers=False,
|
||||
# Longer than HAProxy "timeout http-keep-alive 10s": the LB closes an
|
||||
# idle connection first, so it never reuses one the API just closed.
|
||||
timeout_keep_alive=30,
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout_seconds,
|
||||
log_level="info",
|
||||
)
|
||||
DrainingServer(config).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+432
@@ -0,0 +1,432 @@
|
||||
"""Business rules for MTUSI Messenger Lab4 (users, chats, messages)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app import repositories
|
||||
from app.cache import REVOKED_SESSION, session_cache
|
||||
from app.config import settings
|
||||
from app.errors import ApiError
|
||||
from app.security import (
|
||||
decode_chat_cursor,
|
||||
decode_message_cursor,
|
||||
encode_chat_cursor,
|
||||
encode_message_cursor,
|
||||
generate_session_token,
|
||||
hash_password,
|
||||
hash_session_token,
|
||||
parse_basic_authorization,
|
||||
verify_password,
|
||||
)
|
||||
|
||||
|
||||
def _utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _to_user(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
|
||||
|
||||
def _to_chat(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"title": row["title"],
|
||||
"member_ids": row["member_ids"],
|
||||
"created_at": row["created_at"],
|
||||
}
|
||||
|
||||
|
||||
def _to_message(row: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"sender_id": row["sender_id"],
|
||||
"text": row["text"],
|
||||
"created_at": row["created_at"],
|
||||
"attachment_ids": list(row.get("attachment_ids") or []),
|
||||
}
|
||||
|
||||
|
||||
def _bearer_unauthorized() -> ApiError:
|
||||
return ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Authentication is required.",
|
||||
headers={
|
||||
"WWW-Authenticate": 'Bearer realm="messenger"',
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _basic_unauthorized() -> ApiError:
|
||||
return ApiError(
|
||||
401,
|
||||
"unauthorized",
|
||||
"Invalid credentials.",
|
||||
headers={
|
||||
"WWW-Authenticate": (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
# Lab3 contract: ASCII [A-Za-z0-9._:-]{1,128}.
|
||||
IDEMPOTENCY_KEY_PATTERN = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
def _invalid_request(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def _not_found() -> ApiError:
|
||||
return ApiError(404, "not_found", "Object not found.")
|
||||
|
||||
|
||||
async def register_user(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
username: str,
|
||||
display_name: str,
|
||||
password: str,
|
||||
) -> dict[str, Any]:
|
||||
# Argon2 is CPU-bound (~0.5-1 s); off the event loop so /health/ready keeps answering.
|
||||
password_hash = await asyncio.to_thread(hash_password, password)
|
||||
user = await repositories.create_user(
|
||||
db,
|
||||
username=username,
|
||||
display_name=display_name,
|
||||
password_hash=password_hash,
|
||||
)
|
||||
|
||||
if user is None:
|
||||
raise ApiError(
|
||||
409,
|
||||
"conflict",
|
||||
"Username is already taken.",
|
||||
)
|
||||
|
||||
return _to_user(user)
|
||||
|
||||
|
||||
async def create_user_session(
|
||||
db: AsyncSession,
|
||||
authorization: str | None,
|
||||
) -> dict[str, Any]:
|
||||
credentials = parse_basic_authorization(authorization)
|
||||
|
||||
if credentials is None:
|
||||
raise _basic_unauthorized()
|
||||
|
||||
username, password = credentials
|
||||
user = await repositories.get_user_by_username(db, username)
|
||||
|
||||
if user is None or not await asyncio.to_thread(
|
||||
verify_password,
|
||||
user["password_hash"],
|
||||
password,
|
||||
):
|
||||
raise _basic_unauthorized()
|
||||
|
||||
token = generate_session_token()
|
||||
expires_at = _utc_now() + timedelta(
|
||||
seconds=settings.session_ttl_seconds
|
||||
)
|
||||
|
||||
await repositories.create_session(
|
||||
db,
|
||||
user_id=user["id"],
|
||||
token_hash=hash_session_token(token),
|
||||
expires_at=expires_at,
|
||||
)
|
||||
|
||||
return {
|
||||
"token": token,
|
||||
"token_type": "Bearer",
|
||||
"expires_at": expires_at,
|
||||
"user": _to_user(user),
|
||||
}
|
||||
|
||||
|
||||
async def current_user_from_bearer_token(
|
||||
db: AsyncSession,
|
||||
token: str,
|
||||
) -> dict[str, Any]:
|
||||
token_hash = hash_session_token(token)
|
||||
cached = await session_cache.get(token_hash)
|
||||
|
||||
if cached is REVOKED_SESSION:
|
||||
raise _bearer_unauthorized()
|
||||
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
# Cache miss or cache unavailable: PostgreSQL decides.
|
||||
session_user = await repositories.get_active_session_user(db, token_hash)
|
||||
|
||||
if session_user is None:
|
||||
raise _bearer_unauthorized()
|
||||
|
||||
user = _to_user(session_user)
|
||||
await session_cache.put(token_hash, user, session_user["expires_at"])
|
||||
return user
|
||||
|
||||
|
||||
async def logout_current_session(
|
||||
db: AsyncSession,
|
||||
token: str,
|
||||
) -> None:
|
||||
token_hash = hash_session_token(token)
|
||||
revoked = await repositories.revoke_active_session(db, token_hash)
|
||||
|
||||
if not revoked:
|
||||
raise _bearer_unauthorized()
|
||||
|
||||
# Invalidate after COMMIT in the source of truth.
|
||||
await session_cache.revoke(token_hash)
|
||||
|
||||
|
||||
async def create_chat(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
title: str,
|
||||
member_ids: list[UUID],
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
unique_member_ids = set(member_ids)
|
||||
|
||||
if len(unique_member_ids) != len(member_ids):
|
||||
raise _invalid_request(
|
||||
"member_ids must not contain duplicates."
|
||||
)
|
||||
|
||||
if current_user["id"] not in unique_member_ids:
|
||||
raise _invalid_request(
|
||||
"The current user must be included in member_ids."
|
||||
)
|
||||
|
||||
existing_ids = await repositories.user_ids_that_exist(
|
||||
db,
|
||||
unique_member_ids,
|
||||
)
|
||||
|
||||
if existing_ids != unique_member_ids:
|
||||
raise _invalid_request(
|
||||
"One or more member_ids do not exist."
|
||||
)
|
||||
|
||||
chat = await repositories.create_chat_with_members(
|
||||
db,
|
||||
title=title,
|
||||
member_ids=member_ids,
|
||||
)
|
||||
|
||||
return _to_chat(chat)
|
||||
|
||||
|
||||
async def get_chat(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
user_id=current_user["id"],
|
||||
)
|
||||
|
||||
if chat is None:
|
||||
raise _not_found()
|
||||
|
||||
return _to_chat(chat)
|
||||
|
||||
|
||||
async def list_chats(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
limit: int,
|
||||
cursor: str | None,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
cursor_created_at: datetime | None = None
|
||||
cursor_id: UUID | None = None
|
||||
|
||||
if cursor is not None:
|
||||
decoded = decode_chat_cursor(cursor)
|
||||
|
||||
if decoded is None:
|
||||
raise _invalid_request("Invalid cursor.")
|
||||
|
||||
cursor_created_at, cursor_id = decoded
|
||||
|
||||
rows = await repositories.list_chats_for_user(
|
||||
db,
|
||||
user_id=current_user["id"],
|
||||
limit=limit,
|
||||
cursor_created_at=cursor_created_at,
|
||||
cursor_id=cursor_id,
|
||||
)
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
next_cursor = encode_chat_cursor(
|
||||
last["created_at"],
|
||||
last["id"],
|
||||
)
|
||||
|
||||
return {
|
||||
"items": [_to_chat(row) for row in page],
|
||||
"next_cursor": next_cursor,
|
||||
}
|
||||
|
||||
|
||||
def _request_fingerprint(text_value: str, attachment_ids: list[UUID]) -> str:
|
||||
"""Hash of the validated body: key order and whitespace do not matter,
|
||||
a missing attachment_ids equals []. Without attachments the canonical
|
||||
form is the lab3 one, so keys stored by lab3 still replay correctly."""
|
||||
body: dict[str, Any] = {"text": text_value}
|
||||
if attachment_ids:
|
||||
body["attachment_ids"] = [str(item) for item in attachment_ids]
|
||||
canonical = json.dumps(
|
||||
body,
|
||||
ensure_ascii=False,
|
||||
separators=(",", ":"),
|
||||
sort_keys=True,
|
||||
)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
async def create_message(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
text_value: str,
|
||||
attachment_ids: list[UUID],
|
||||
idempotency_key: str | None,
|
||||
current_user: dict[str, Any],
|
||||
) -> tuple[dict[str, Any], bool]:
|
||||
"""Send a message once per (user, POST, chat, Idempotency-Key).
|
||||
|
||||
Returns (message, replayed). A replay re-checks the session (done by
|
||||
require_user) and chat membership before returning the stored result.
|
||||
"""
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY_PATTERN.fullmatch(
|
||||
idempotency_key
|
||||
):
|
||||
raise _invalid_request(
|
||||
"Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}."
|
||||
)
|
||||
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
user_id=current_user["id"],
|
||||
)
|
||||
|
||||
if chat is None:
|
||||
raise _not_found()
|
||||
|
||||
try:
|
||||
result = await repositories.create_message_idempotent(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
sender_id=current_user["id"],
|
||||
text_value=text_value,
|
||||
attachment_ids=attachment_ids,
|
||||
idempotency_key=idempotency_key,
|
||||
request_hash=_request_fingerprint(text_value, attachment_ids),
|
||||
ttl_seconds=settings.idempotency_ttl_seconds,
|
||||
)
|
||||
except repositories.AttachmentsNotFound:
|
||||
raise _not_found() from None
|
||||
except repositories.AttachmentFailed:
|
||||
raise ApiError(409, "conflict", "Failed attachment cannot be attached.") from None
|
||||
|
||||
if result is None:
|
||||
raise ApiError(
|
||||
409,
|
||||
"conflict",
|
||||
"Idempotency-Key was already used with a different body.",
|
||||
)
|
||||
|
||||
message, replayed = result
|
||||
return _to_message(message), replayed
|
||||
|
||||
|
||||
async def list_messages(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
chat_id: UUID,
|
||||
limit: int,
|
||||
cursor: str | None,
|
||||
current_user: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
chat = await repositories.get_accessible_chat(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
user_id=current_user["id"],
|
||||
)
|
||||
|
||||
if chat is None:
|
||||
raise _not_found()
|
||||
|
||||
cursor_created_at: datetime | None = None
|
||||
cursor_id: UUID | None = None
|
||||
|
||||
if cursor is not None:
|
||||
decoded = decode_message_cursor(cursor)
|
||||
|
||||
if decoded is None:
|
||||
raise _invalid_request("Invalid cursor.")
|
||||
|
||||
cursor_chat_id, cursor_created_at, cursor_id = decoded
|
||||
|
||||
if cursor_chat_id != chat_id:
|
||||
raise _invalid_request(
|
||||
"Cursor belongs to another chat."
|
||||
)
|
||||
|
||||
rows = await repositories.list_messages_keyset(
|
||||
db,
|
||||
chat_id=chat_id,
|
||||
limit=limit,
|
||||
cursor_created_at=cursor_created_at,
|
||||
cursor_id=cursor_id,
|
||||
)
|
||||
|
||||
has_more = len(rows) > limit
|
||||
page = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if has_more and page:
|
||||
last = page[-1]
|
||||
next_cursor = encode_message_cursor(
|
||||
chat_id,
|
||||
last["created_at"],
|
||||
last["id"],
|
||||
)
|
||||
|
||||
return {
|
||||
"items": [_to_message(row) for row in page],
|
||||
"next_cursor": next_cursor,
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
"""Private S3 bucket for attachment bytes (SeaweedFS S3 gateway).
|
||||
|
||||
One long-lived aiobotocore client per API process, path-style addressing,
|
||||
short connect/read timeouts, at most 2 attempts and an overall deadline
|
||||
S3_OPERATION_TIMEOUT_SECONDS per call, so an unavailable S3 turns into a
|
||||
fast 503 on the attachment endpoints only. Any client-side or
|
||||
network error is raised as StorageUnavailable; a missing key is
|
||||
ObjectMissing. Object keys are generated by the server (see services.py);
|
||||
the user's filename never becomes part of a key.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import AsyncExitStack
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, BinaryIO
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app.config import settings
|
||||
|
||||
|
||||
logger = logging.getLogger("messenger.storage")
|
||||
|
||||
DOWNLOAD_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class StorageUnavailable(Exception):
|
||||
"""S3 is unreachable, timed out or refused the request."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""The key does not exist in the bucket."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class StoredObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified: Any
|
||||
|
||||
|
||||
class ObjectStorage:
|
||||
def __init__(self) -> None:
|
||||
self._stack: AsyncExitStack | None = None
|
||||
self._client = None
|
||||
self.bucket = settings.s3_bucket
|
||||
|
||||
async def start(self) -> None:
|
||||
self._stack = AsyncExitStack()
|
||||
session = get_session()
|
||||
self._client = await self._stack.enter_async_context(
|
||||
session.create_client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.s3_access_key_id,
|
||||
aws_secret_access_key=settings.s3_secret_access_key,
|
||||
config=AioConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=settings.s3_connect_timeout_seconds,
|
||||
read_timeout=settings.s3_read_timeout_seconds,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
max_pool_connections=settings.max_concurrent_uploads * 2 + 4,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._stack is not None:
|
||||
await self._stack.aclose()
|
||||
self._stack = None
|
||||
self._client = None
|
||||
|
||||
@staticmethod
|
||||
def _deadline(size: int = 0):
|
||||
return asyncio.timeout(
|
||||
settings.s3_operation_timeout_seconds
|
||||
+ size / settings.s3_put_min_bytes_per_second
|
||||
)
|
||||
|
||||
def _require_client(self):
|
||||
if self._client is None:
|
||||
raise StorageUnavailable("S3 client is not started")
|
||||
return self._client
|
||||
|
||||
async def put(
|
||||
self,
|
||||
key: str,
|
||||
body: BinaryIO,
|
||||
size: int,
|
||||
content_type: str,
|
||||
) -> None:
|
||||
"""Upload a seekable file. SigV4 signs the payload SHA-256, so S3
|
||||
itself rejects bytes that were damaged on the way."""
|
||||
client = self._require_client()
|
||||
body.seek(0)
|
||||
try:
|
||||
async with self._deadline(size):
|
||||
await client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=key,
|
||||
Body=body,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
)
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
async def open_download(self, key: str) -> tuple[AsyncIterator[bytes], int]:
|
||||
"""GET the object; S3 errors surface here, before any byte is sent."""
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
response = await client.get_object(Bucket=self.bucket, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
raise ObjectMissing(key) from exc
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
body = response["Body"]
|
||||
|
||||
async def chunks() -> AsyncIterator[bytes]:
|
||||
try:
|
||||
async for chunk in body.iter_chunks(DOWNLOAD_CHUNK):
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return chunks(), int(response["ContentLength"])
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
"""Idempotent: deleting a missing key succeeds in S3."""
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
await client.delete_object(Bucket=self.bucket, Key=key)
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
client = self._require_client()
|
||||
try:
|
||||
async with self._deadline():
|
||||
await client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404", "NotFound"}:
|
||||
return False
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
except (BotoCoreError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
async def list_objects(self, prefix: str) -> AsyncIterator[StoredObject]:
|
||||
client = self._require_client()
|
||||
paginator = client.get_paginator("list_objects_v2")
|
||||
try:
|
||||
async for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix):
|
||||
for item in page.get("Contents", []):
|
||||
yield StoredObject(item["Key"], item["Size"], item["LastModified"])
|
||||
except (BotoCoreError, ClientError, OSError, asyncio.TimeoutError) as exc:
|
||||
raise StorageUnavailable(type(exc).__name__) from exc
|
||||
|
||||
|
||||
storage = ObjectStorage()
|
||||
+210
@@ -0,0 +1,210 @@
|
||||
"""RabbitMQ consumer: one bounded job at a time, explicit ack after COMMIT."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal, engine
|
||||
from app.pipeline import (IMAGE_TIMEOUT, LEASE_SECONDS, TOTAL_TIMEOUT,
|
||||
arm_from_env, broker_channel, crash_if_armed,
|
||||
event, parse_envelope, retry_or_fail)
|
||||
from app.storage import ObjectMissing, storage
|
||||
|
||||
logger = logging.getLogger("messenger.worker")
|
||||
|
||||
|
||||
class PermanentFailure(Exception):
|
||||
pass
|
||||
|
||||
|
||||
async def claim(job: dict) -> dict | None:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments WHERE id=:id FOR UPDATE
|
||||
"""), {"id": UUID(job["attachment_id"])})
|
||||
row = result.mappings().first()
|
||||
if row is None or row["job_id"] != UUID(job["job_id"]) or (
|
||||
row["pipeline_version"] != job["pipeline_version"] or
|
||||
row["attempt"] != job["attempt"] or row["status"] != "queued"
|
||||
):
|
||||
await db.rollback()
|
||||
return None
|
||||
# An early/manual duplicate cannot bypass the durable retry date.
|
||||
result = await db.execute(text("""
|
||||
UPDATE attachments SET status='processing',attempts_run=attempts_run+1,
|
||||
lease_token=:token,lease_until=now()+make_interval(secs => :lease),
|
||||
updated_at=now()
|
||||
WHERE id=:id AND next_attempt_at<=now() RETURNING *
|
||||
"""), {"id": row["id"], "token": uuid4(), "lease": LEASE_SECONDS})
|
||||
claimed = result.mappings().first()
|
||||
if claimed is None:
|
||||
await db.rollback()
|
||||
return None
|
||||
claimed = dict(claimed)
|
||||
await event(db, claimed, "processing")
|
||||
await db.commit()
|
||||
return claimed
|
||||
|
||||
|
||||
async def generate(source: Path, directory: Path) -> list[dict]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "app.image_worker", str(source), str(directory),
|
||||
stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.DEVNULL,
|
||||
)
|
||||
try:
|
||||
try:
|
||||
async with asyncio.timeout(IMAGE_TIMEOUT):
|
||||
code = await process.wait()
|
||||
except TimeoutError:
|
||||
raise PermanentFailure("image_timeout") from None
|
||||
if code == 21:
|
||||
raise PermanentFailure("image_resource_limit")
|
||||
if code != 0:
|
||||
raise PermanentFailure("invalid_image")
|
||||
return json.loads((directory / "manifest.json").read_text(encoding="utf-8"))
|
||||
finally:
|
||||
# Includes cancellation of the entire job. Native image code cannot
|
||||
# continue using CPU/RAM after its parent timed out.
|
||||
if process.returncode is None:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
|
||||
|
||||
async def build_objects(row: dict) -> list[dict]:
|
||||
with tempfile.TemporaryDirectory(prefix="attachment-") as temporary:
|
||||
directory = Path(temporary)
|
||||
source = directory / "source"
|
||||
if not row["staging_key"] or row["staging_deleted"]:
|
||||
raise PermanentFailure("source_missing")
|
||||
try:
|
||||
chunks, declared_length = await storage.open_download(row["staging_key"])
|
||||
except ObjectMissing:
|
||||
raise PermanentFailure("source_missing") from None
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
try:
|
||||
with source.open("wb") as output:
|
||||
async for chunk in chunks:
|
||||
size += len(chunk)
|
||||
if size > settings.max_upload_bytes:
|
||||
raise PermanentFailure("source_too_large")
|
||||
digest.update(chunk)
|
||||
output.write(chunk)
|
||||
finally:
|
||||
await chunks.aclose()
|
||||
if size != declared_length or size != row["size_bytes"] or digest.hexdigest() != row["sha256"]:
|
||||
raise PermanentFailure("source_checksum_mismatch")
|
||||
variants = await generate(source, directory)
|
||||
# API never publishes directly to this final prefix. Repeating one
|
||||
# v1 job writes the same bytes to the same four keys, never appends.
|
||||
with source.open("rb") as body:
|
||||
await storage.put(row["object_key"], body, size, row["content_type"])
|
||||
prefix = row["object_key"].rsplit("/", 1)[0]
|
||||
for variant in variants:
|
||||
filename = variant.pop("filename")
|
||||
key = f"{prefix}/{filename}"
|
||||
with (directory / filename).open("rb") as body:
|
||||
await storage.put(key, body, variant["size_bytes"], variant["content_type"])
|
||||
variant["object_key"] = key
|
||||
await crash_if_armed("crash_after_objects_before_commit")
|
||||
return variants
|
||||
|
||||
|
||||
async def finish(row: dict, *, variants=None, code: str | None = None,
|
||||
permanent: bool = False) -> bool:
|
||||
async with SessionLocal() as db:
|
||||
result = await db.execute(text("""
|
||||
SELECT * FROM attachments WHERE id=:id AND status='processing'
|
||||
AND lease_token=:token AND lease_until>now() FOR UPDATE
|
||||
"""), {"id": row["id"], "token": row["lease_token"]})
|
||||
current = result.mappings().first()
|
||||
if current is None:
|
||||
# Lease recovery or explicit operator action already owns a
|
||||
# newer attempt. Never overwrite its state with an old worker.
|
||||
await db.rollback()
|
||||
return False
|
||||
if code is not None:
|
||||
await retry_or_fail(db, current, code, permanent=permanent)
|
||||
else:
|
||||
await db.execute(text("""
|
||||
UPDATE attachments SET status='ready',variants=CAST(:variants AS jsonb),
|
||||
error_code=NULL,lease_token=NULL,lease_until=NULL,updated_at=now()
|
||||
WHERE id=:id
|
||||
"""), {"id": row["id"], "variants": json.dumps(variants)})
|
||||
await event(db, current, "ready")
|
||||
await db.commit()
|
||||
return True
|
||||
|
||||
|
||||
async def handle(message) -> None:
|
||||
try:
|
||||
job = parse_envelope(message.body)
|
||||
except (ValueError, TypeError, KeyError, AttributeError):
|
||||
logger.warning("invalid envelope sent to durable DLQ")
|
||||
await message.reject(requeue=False)
|
||||
return
|
||||
row = await claim(job)
|
||||
if row is None:
|
||||
# A ready/failed/stale attempt is a durable no-op. A duplicate of
|
||||
# active processing is covered by the DB lease recovery, so it too
|
||||
# can be acknowledged without running a second concurrent decoder.
|
||||
await message.ack()
|
||||
return
|
||||
try:
|
||||
async with asyncio.timeout(TOTAL_TIMEOUT):
|
||||
variants = await build_objects(row)
|
||||
except PermanentFailure as exc:
|
||||
await finish(row, code=str(exc), permanent=True)
|
||||
except Exception as exc:
|
||||
logger.warning("temporary processing failure attachment=%s type=%s",
|
||||
row["id"], type(exc).__name__)
|
||||
await finish(row, code="processing_failed")
|
||||
else:
|
||||
if await finish(row, variants=variants):
|
||||
await crash_if_armed("crash_after_commit_before_ack")
|
||||
logger.info("ready attachment=%s", row["id"])
|
||||
# If finish failed, the exception propagates and the connection closes:
|
||||
# RabbitMQ redelivers while PG keeps the durable lease/retry intention.
|
||||
await message.ack()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
for name in ("aio_pika", "aiormq"):
|
||||
logging.getLogger(name).setLevel(logging.CRITICAL)
|
||||
await storage.start()
|
||||
connection = None
|
||||
try:
|
||||
await arm_from_env("WORKER_FAULT")
|
||||
while True:
|
||||
try:
|
||||
connection, _, queue = await broker_channel()
|
||||
async with queue.iterator() as deliveries:
|
||||
async for message in deliveries:
|
||||
await handle(message)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
logger.warning("worker reconnect: %s", type(exc).__name__)
|
||||
if connection:
|
||||
await connection.close()
|
||||
connection = None
|
||||
await asyncio.sleep(1)
|
||||
finally:
|
||||
if connection:
|
||||
await connection.close()
|
||||
await storage.close()
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -1,14 +0,0 @@
|
||||
# Topology worksheet, deliberately NOT a runnable stack (no application supplied).
|
||||
# Implement compose.yaml or stack.yaml; do not submit this empty file as deployment.
|
||||
# Service names below are logical roles, not imposed container names.
|
||||
x-lab-number: 5
|
||||
x-required-roles:
|
||||
- api
|
||||
- postgres
|
||||
- load_balancer
|
||||
- s3
|
||||
- broker
|
||||
- worker
|
||||
# TODO: implement real images/builds, network wiring, volumes, healthchecks,
|
||||
# credentials and bootstrap. Details: contracts/infrastructure.json and README.md.
|
||||
services: {}
|
||||
+361
@@ -0,0 +1,361 @@
|
||||
#file: noinspection ComposeUnknownValues
|
||||
# Lab5: API x2 -> durable S3 staging + PostgreSQL outbox -> publisher
|
||||
# -> RabbitMQ -> worker -> final private S3 objects + PostgreSQL metadata.
|
||||
# Start: see REPORT.md, "Запуск из чистого clone".
|
||||
name: messenger-lab5
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}"
|
||||
SESSION_STORE: valkey
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_CACHE_TTL_SECONDS: "60"
|
||||
CACHE_TIMEOUT_SECONDS: "0.25"
|
||||
CACHE_MAX_CONNECTIONS: "20"
|
||||
CACHE_RETRY_AFTER_SECONDS: "5"
|
||||
DB_POOL_SIZE: "10"
|
||||
DB_MAX_OVERFLOW: "5"
|
||||
DB_POOL_TIMEOUT_SECONDS: "3"
|
||||
DB_CONNECT_TIMEOUT_SECONDS: "3"
|
||||
DB_STATEMENT_TIMEOUT_MS: "5000"
|
||||
IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}"
|
||||
SHUTDOWN_DRAIN_SECONDS: "3"
|
||||
SHUTDOWN_TIMEOUT_SECONDS: "8"
|
||||
# Lab4: S3 with the API's own restricted credentials (not admin).
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_OBJECT_PREFIX: originals/
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_CONNECT_TIMEOUT_SECONDS: "2"
|
||||
S3_READ_TIMEOUT_SECONDS: "5"
|
||||
S3_OPERATION_TIMEOUT_SECONDS: "6"
|
||||
S3_PUT_MIN_BYTES_PER_SECOND: "1048576"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
UPLOAD_SPOOL_MEMORY_BYTES: "1048576"
|
||||
UPLOAD_TIMEOUT_SECONDS: "60"
|
||||
MAX_CONCURRENT_UPLOADS: "8"
|
||||
UPLOAD_SLOT_WAIT_SECONDS: "10"
|
||||
UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}"
|
||||
JANITOR_INTERVAL_SECONDS: "${JANITOR_INTERVAL_SECONDS:-60}"
|
||||
JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}"
|
||||
JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-30}"
|
||||
JOB_TOTAL_TIMEOUT_SECONDS: "${JOB_TOTAL_TIMEOUT_SECONDS:-90}"
|
||||
WORKER_MEMORY_MB: "${WORKER_MEMORY_MB:-768}"
|
||||
JOB_LEASE_SECONDS: "${JOB_LEASE_SECONDS:-120}"
|
||||
RETRY_BASE_SECONDS: "${RETRY_BASE_SECONDS:-1}"
|
||||
STAGING_RETENTION_SECONDS: "${STAGING_RETENTION_SECONDS:-604800}"
|
||||
PUBLISHER_FAULT: "${PUBLISHER_FAULT:-}"
|
||||
WORKER_FAULT: "${WORKER_FAULT:-}"
|
||||
# API fault-injection is disabled during normal operation.
|
||||
FAULT_INJECTION: "${FAULT_INJECTION:-}"
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 5s
|
||||
networks: [backend]
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 512M }
|
||||
|
||||
# Session cache only (lab2). PostgreSQL stays the source of truth, so the
|
||||
# cache is intentionally not persisted. It is shared by all API replicas:
|
||||
# a logout on one replica writes the "revoked" tombstone seen by all.
|
||||
valkey:
|
||||
image: valkey/valkey:8.0.1-alpine
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
exec valkey-server
|
||||
--requirepass "$$(cat /run/secrets/valkey_password)"
|
||||
--save ""
|
||||
--appendonly no
|
||||
--maxmemory 64mb
|
||||
--maxmemory-policy allkeys-lru
|
||||
secrets: [valkey_password]
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD-SHELL
|
||||
- valkey-cli -a "$$(cat /run/secrets/valkey_password)" --no-auth-warning ping | grep -q PONG
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.25", memory: 128M }
|
||||
|
||||
# S3: SeaweedFS all-in-one. No "ports": reachable only from the internal
|
||||
# backend network; its admin web UIs are bound to 127.0.0.1 inside the
|
||||
# container (infra/seaweedfs/start.sh). Bytes persist in the s3_data volume.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.97
|
||||
entrypoint: ["/bin/sh", "/opt/lab/start.sh"]
|
||||
environment:
|
||||
# Soft limit of the Go heap: without it the GC lets SeaweedFS grow to
|
||||
# the container limit under parallel 10 MiB PUTs (measured, REPORT.md).
|
||||
GOMEMLIMIT: 600MiB
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
- ./infra/seaweedfs:/opt/lab:ro
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:9000/healthz"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 60s
|
||||
networks: [backend]
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
# 512M was measured to be too little: 12 parallel 10 MiB PUTs pushed
|
||||
# SeaweedFS to the limit and the kernel killed it (REPORT.md).
|
||||
limits: { cpus: "1.0", memory: 1G }
|
||||
|
||||
# Idempotent bucket bootstrap + self-checks (private bucket, least
|
||||
# privilege of the API key). The only container with the S3 admin key.
|
||||
s3-init:
|
||||
build: .
|
||||
image: messenger-lab5-api:${APP_VERSION:-dev}
|
||||
command: ["python", "scripts/s3_bootstrap.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_OBJECT_PREFIX: originals/
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
s3: { condition: service_healthy }
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128M }
|
||||
|
||||
# Runs the migrations once, before any API replica starts (no race of
|
||||
# replicas). A repeated run only prints SKIP.
|
||||
migrate:
|
||||
build: .
|
||||
image: messenger-lab5-api:${APP_VERSION:-dev}
|
||||
command: ["python", "scripts/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
db: { condition: service_healthy }
|
||||
networks: [backend]
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128M }
|
||||
|
||||
api:
|
||||
build: .
|
||||
image: messenger-lab5-api:${APP_VERSION:-dev}
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
# No "ports": replicas are reachable only through the LB.
|
||||
# No "container_name": the service is scaled by deploy.replicas / --scale.
|
||||
networks: [edge, backend]
|
||||
depends_on:
|
||||
db: { condition: service_healthy }
|
||||
valkey: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 2
|
||||
start_period: 5s
|
||||
stop_signal: SIGTERM
|
||||
# > SHUTDOWN_DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (3 + 8 s).
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
|
||||
# A stable nodename keeps durable queues in the same RabbitMQ database
|
||||
# even after the container is re-created. No AMQP/management port is public.
|
||||
broker:
|
||||
image: rabbitmq:3.13.7-alpine
|
||||
hostname: broker
|
||||
entrypoint: ["/bin/sh", "-ec"]
|
||||
command:
|
||||
- 'export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"; exec docker-entrypoint.sh rabbitmq-server'
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: messenger
|
||||
RABBITMQ_DEFAULT_VHOST: /
|
||||
# Bound Erlang schedulers to this small container, rather than the
|
||||
# host's core count; health-check CLI VMs need only one scheduler.
|
||||
RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2"
|
||||
RABBITMQ_CTL_ERL_ARGS: "+S 1:1"
|
||||
secrets: [broker_password]
|
||||
volumes:
|
||||
- broker_data:/var/lib/rabbitmq
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 3s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
start_period: 15s
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 512M }
|
||||
|
||||
publisher:
|
||||
build: .
|
||||
image: messenger-lab5-api:${APP_VERSION:-dev}
|
||||
command: ["python", "-m", "app.jobs"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
broker: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 256M }
|
||||
|
||||
worker:
|
||||
build: .
|
||||
image: messenger-lab5-api:${APP_VERSION:-dev}
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [backend]
|
||||
depends_on:
|
||||
broker: { condition: service_healthy }
|
||||
migrate: { condition: service_completed_successfully }
|
||||
s3-init: { condition: service_completed_successfully }
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
# One active job and one reserved delivery; image subprocess has its own
|
||||
# address-space/time limits in addition to this container memory limit.
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "1.0", memory: 1G }
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.11-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "${LB_PORT:-8080}:8080"
|
||||
# Backend state page for the operator, only on the host loopback.
|
||||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
||||
networks: [edge]
|
||||
depends_on:
|
||||
api: { condition: service_healthy }
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: { cpus: "0.5", memory: 128M }
|
||||
|
||||
# Runs the public and own checks in Linux (needed on Windows: the hooks are
|
||||
# POSIX .sh scripts that call the docker CLI). Profile "tools": not started
|
||||
# by a plain "up". The Docker socket grants control over the Docker host —
|
||||
# use it only on the lab stand.
|
||||
tester:
|
||||
profiles: [tools]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: student-tests/Dockerfile.tester
|
||||
working_dir: /work
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
COMPOSE_PROJECT: messenger-lab5
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
volumes:
|
||||
- ./:/work
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
# backend: only for the "private bucket from outside the API" check.
|
||||
networks: [edge, backend]
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
# PostgreSQL, Valkey and S3: no route outside, the LB is not attached.
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
postgres_data: {}
|
||||
s3_data: {}
|
||||
broker_data: {}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./infra/secrets/postgres_password.txt
|
||||
database_url:
|
||||
file: ./infra/secrets/database_url.txt
|
||||
valkey_password:
|
||||
file: ./infra/secrets/valkey_password.txt
|
||||
cache_url:
|
||||
file: ./infra/secrets/cache_url.txt
|
||||
s3_admin_access_key_id:
|
||||
file: ./infra/secrets/s3_admin_access_key_id.txt
|
||||
s3_admin_secret_access_key:
|
||||
file: ./infra/secrets/s3_admin_secret_access_key.txt
|
||||
s3_access_key_id:
|
||||
file: ./infra/secrets/s3_access_key_id.txt
|
||||
s3_secret_access_key:
|
||||
file: ./infra/secrets/s3_secret_access_key.txt
|
||||
broker_password:
|
||||
file: ./infra/secrets/broker_password.txt
|
||||
broker_url:
|
||||
file: ./infra/secrets/broker_url.txt
|
||||
@@ -0,0 +1,8 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:07Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u scripts/check_contract.py
|
||||
Container messenger-lab5-tester-run-2ab3f1ca0a72 Creating
|
||||
Container messenger-lab5-tester-run-2ab3f1ca0a72 Created
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:13Z
|
||||
@@ -0,0 +1,170 @@
|
||||
|
||||
Before public persistence (crash-restart stores) — 2026-11-10T11:42:41Z
|
||||
|
||||
UTC: 2026-11-10T11:42:41Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0
|
||||
a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369
|
||||
c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b
|
||||
93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd
|
||||
b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3
|
||||
891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e
|
||||
971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398
|
||||
fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c
|
||||
5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b
|
||||
d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e
|
||||
3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:42:41Z
|
||||
|
||||
UTC: 2026-11-10T11:42:41Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0 a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369 c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b 93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3 891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e 971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398 fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c 5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e 3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
/messenger-lab5-api-1 status=running started=2026-11-10T11:40:02.214726211Z finished=2026-11-10T11:38:22.342123315Z
|
||||
/messenger-lab5-api-2 status=running started=2026-11-10T11:40:02.682626597Z finished=2026-11-10T11:38:22.49969344Z
|
||||
/messenger-lab5-broker-1 status=running started=2026-11-10T11:39:39.164939692Z finished=2026-11-10T11:39:23.590473655Z
|
||||
/messenger-lab5-db-1 status=running started=2026-11-10T11:39:38.128469939Z finished=2026-11-10T11:38:28.455904616Z
|
||||
/messenger-lab5-lb-1 status=running started=2026-11-10T11:40:08.719971782Z finished=2026-11-10T11:38:17.968115638Z
|
||||
/messenger-lab5-migrate-1 status=exited started=2026-11-10T11:39:44.355210584Z finished=2026-11-10T11:39:45.414817313Z
|
||||
/messenger-lab5-publisher-1 status=running started=2026-11-10T11:40:02.205235899Z finished=2026-11-10T11:38:28.010120877Z
|
||||
/messenger-lab5-s3-1 status=running started=2026-11-10T11:39:38.900710259Z finished=2026-11-10T11:39:37.759418922Z
|
||||
/messenger-lab5-s3-init-1 status=exited started=2026-11-10T11:39:59.844406543Z finished=2026-11-10T11:40:01.82853823Z
|
||||
/messenger-lab5-valkey-1 status=running started=2026-11-10T11:39:38.502256743Z finished=2026-11-10T11:38:28.447469147Z
|
||||
/messenger-lab5-worker-1 status=running started=2026-11-10T11:40:02.218275767Z finished=2026-11-10T11:38:28.004446154Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:42:41Z
|
||||
|
||||
After public persistence — 2026-11-10T11:43:48Z
|
||||
|
||||
UTC: 2026-11-10T11:43:48Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0
|
||||
a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369
|
||||
c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b
|
||||
93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd
|
||||
b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3
|
||||
891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e
|
||||
971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398
|
||||
fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c
|
||||
5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b
|
||||
d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e
|
||||
3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:43:49Z
|
||||
|
||||
UTC: 2026-11-10T11:43:49Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0 a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369 c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b 93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3 891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e 971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398 fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c 5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e 3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
/messenger-lab5-api-1 status=running started=2026-11-10T11:43:35.752420255Z finished=2026-11-10T11:42:52.498323088Z
|
||||
/messenger-lab5-api-2 status=running started=2026-11-10T11:43:36.099337159Z finished=2026-11-10T11:42:51.726168665Z
|
||||
/messenger-lab5-broker-1 status=running started=2026-11-10T11:42:59.998455808Z finished=2026-11-10T11:42:58.631778022Z
|
||||
/messenger-lab5-db-1 status=running started=2026-11-10T11:42:59.206360394Z finished=2026-11-10T11:42:58.520564305Z
|
||||
/messenger-lab5-lb-1 status=running started=2026-11-10T11:43:42.758385817Z finished=2026-11-10T11:42:47.893068685Z
|
||||
/messenger-lab5-migrate-1 status=exited started=2026-11-10T11:39:44.355210584Z finished=2026-11-10T11:39:45.414817313Z
|
||||
/messenger-lab5-publisher-1 status=running started=2026-11-10T11:43:35.235890224Z finished=2026-11-10T11:42:58.049693356Z
|
||||
/messenger-lab5-s3-1 status=running started=2026-11-10T11:42:59.710691335Z finished=2026-11-10T11:42:58.532420918Z
|
||||
/messenger-lab5-s3-init-1 status=exited started=2026-11-10T11:39:59.844406543Z finished=2026-11-10T11:40:01.82853823Z
|
||||
/messenger-lab5-valkey-1 status=running started=2026-11-10T11:42:59.473694935Z finished=2026-11-10T11:42:58.500195842Z
|
||||
/messenger-lab5-worker-1 status=running started=2026-11-10T11:43:35.475881026Z finished=2026-11-10T11:42:58.056034905Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:43:49Z
|
||||
|
||||
Before public failover — 2026-11-10T11:43:49Z
|
||||
|
||||
UTC: 2026-11-10T11:43:49Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0
|
||||
a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369
|
||||
c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b
|
||||
93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd
|
||||
b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3
|
||||
891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e
|
||||
971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398
|
||||
fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c
|
||||
5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b
|
||||
d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e
|
||||
3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:43:49Z
|
||||
|
||||
UTC: 2026-11-10T11:43:49Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0 a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369 c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b 93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3 891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e 971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398 fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c 5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e 3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
/messenger-lab5-api-1 status=running started=2026-11-10T11:43:35.752420255Z finished=2026-11-10T11:42:52.498323088Z
|
||||
/messenger-lab5-api-2 status=running started=2026-11-10T11:43:36.099337159Z finished=2026-11-10T11:42:51.726168665Z
|
||||
/messenger-lab5-broker-1 status=running started=2026-11-10T11:42:59.998455808Z finished=2026-11-10T11:42:58.631778022Z
|
||||
/messenger-lab5-db-1 status=running started=2026-11-10T11:42:59.206360394Z finished=2026-11-10T11:42:58.520564305Z
|
||||
/messenger-lab5-lb-1 status=running started=2026-11-10T11:43:42.758385817Z finished=2026-11-10T11:42:47.893068685Z
|
||||
/messenger-lab5-migrate-1 status=exited started=2026-11-10T11:39:44.355210584Z finished=2026-11-10T11:39:45.414817313Z
|
||||
/messenger-lab5-publisher-1 status=running started=2026-11-10T11:43:35.235890224Z finished=2026-11-10T11:42:58.049693356Z
|
||||
/messenger-lab5-s3-1 status=running started=2026-11-10T11:42:59.710691335Z finished=2026-11-10T11:42:58.532420918Z
|
||||
/messenger-lab5-s3-init-1 status=exited started=2026-11-10T11:39:59.844406543Z finished=2026-11-10T11:40:01.82853823Z
|
||||
/messenger-lab5-valkey-1 status=running started=2026-11-10T11:42:59.473694935Z finished=2026-11-10T11:42:58.500195842Z
|
||||
/messenger-lab5-worker-1 status=running started=2026-11-10T11:43:35.475881026Z finished=2026-11-10T11:42:58.056034905Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:43:49Z
|
||||
|
||||
After public failover, before API restoration — 2026-11-10T11:44:14Z
|
||||
|
||||
UTC: 2026-11-10T11:44:14Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0
|
||||
a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369
|
||||
c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b
|
||||
93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd
|
||||
b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3
|
||||
891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e
|
||||
971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398
|
||||
fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c
|
||||
5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b
|
||||
d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e
|
||||
3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:44:14Z
|
||||
|
||||
UTC: 2026-11-10T11:44:14Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0 a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369 c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b 93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3 891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e 971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398 fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c 5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e 3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
/messenger-lab5-api-1 status=exited started=2026-11-10T11:43:35.752420255Z finished=2026-11-10T11:44:02.81426728Z
|
||||
/messenger-lab5-api-2 status=running started=2026-11-10T11:43:36.099337159Z finished=2026-11-10T11:42:51.726168665Z
|
||||
/messenger-lab5-broker-1 status=running started=2026-11-10T11:42:59.998455808Z finished=2026-11-10T11:42:58.631778022Z
|
||||
/messenger-lab5-db-1 status=running started=2026-11-10T11:42:59.206360394Z finished=2026-11-10T11:42:58.520564305Z
|
||||
/messenger-lab5-lb-1 status=running started=2026-11-10T11:43:42.758385817Z finished=2026-11-10T11:42:47.893068685Z
|
||||
/messenger-lab5-migrate-1 status=exited started=2026-11-10T11:39:44.355210584Z finished=2026-11-10T11:39:45.414817313Z
|
||||
/messenger-lab5-publisher-1 status=running started=2026-11-10T11:43:35.235890224Z finished=2026-11-10T11:42:58.049693356Z
|
||||
/messenger-lab5-s3-1 status=running started=2026-11-10T11:42:59.710691335Z finished=2026-11-10T11:42:58.532420918Z
|
||||
/messenger-lab5-s3-init-1 status=exited started=2026-11-10T11:39:59.844406543Z finished=2026-11-10T11:40:01.82853823Z
|
||||
/messenger-lab5-valkey-1 status=running started=2026-11-10T11:42:59.473694935Z finished=2026-11-10T11:42:58.500195842Z
|
||||
/messenger-lab5-worker-1 status=running started=2026-11-10T11:43:35.475881026Z finished=2026-11-10T11:42:58.056034905Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:44:14Z
|
||||
|
||||
After API restoration — 2026-11-10T11:44:22Z
|
||||
|
||||
UTC: 2026-11-10T11:44:22Z
|
||||
Command: docker compose --profile tools ps -a -q
|
||||
17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0
|
||||
a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369
|
||||
c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b
|
||||
93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd
|
||||
b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3
|
||||
891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e
|
||||
971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398
|
||||
fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c
|
||||
5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b
|
||||
d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e
|
||||
3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:44:23Z
|
||||
|
||||
UTC: 2026-11-10T11:44:23Z
|
||||
Command: docker inspect --format "{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}" 17151567e8aa2d963d5adc4dc7eee37caae1487d419a72e3b6d6c0440cfbcdf0 a5ac10d7ad3e604c2f7ab2b8e1480ff38ff0751f9b760488b19ff9622e674369 c2f4318d250b957561ccaa99ad2162c21ee0ccabf12a386e931f8107df943f3b 93a1911bc1b41ec117e4822ff9cdad0166ecba0d3fed7f751fca328b18cd0bbd b90556b888302efc27c3517b7d3320c40eed0189e3bfbff801b96d02c7188dd3 891e3cb3b01b36b3326f2e3c7d8982d231735e5f54baf39a8681e92b6a0dab2e 971c52d70d1cf2cac8394e2b0dc76bc4c40784d1cf3688c5ac6b45420fe28398 fcb93dd47cb29d1338fb835421bab524d757090d1e9b16577fb94415facbe00c 5c0ee9009edeb66d3dbc59c64d7039aab9b3089e8b8134a50a4c0115a77e4a9b d1ccd5d9db422ea785ca8cdfaddd8be29b0d61a2d8ebe02e2bbad0e551d93d4e 3c07c8953cc4670c7356a0d5ce9546bda3daff603004c365f0bf48b29427ea49
|
||||
/messenger-lab5-api-1 status=running started=2026-11-10T11:44:15.758201101Z finished=2026-11-10T11:44:02.81426728Z
|
||||
/messenger-lab5-api-2 status=running started=2026-11-10T11:43:36.099337159Z finished=2026-11-10T11:42:51.726168665Z
|
||||
/messenger-lab5-broker-1 status=running started=2026-11-10T11:42:59.998455808Z finished=2026-11-10T11:42:58.631778022Z
|
||||
/messenger-lab5-db-1 status=running started=2026-11-10T11:42:59.206360394Z finished=2026-11-10T11:42:58.520564305Z
|
||||
/messenger-lab5-lb-1 status=running started=2026-11-10T11:43:42.758385817Z finished=2026-11-10T11:42:47.893068685Z
|
||||
/messenger-lab5-migrate-1 status=exited started=2026-11-10T11:39:44.355210584Z finished=2026-11-10T11:39:45.414817313Z
|
||||
/messenger-lab5-publisher-1 status=running started=2026-11-10T11:43:35.235890224Z finished=2026-11-10T11:42:58.049693356Z
|
||||
/messenger-lab5-s3-1 status=running started=2026-11-10T11:42:59.710691335Z finished=2026-11-10T11:42:58.532420918Z
|
||||
/messenger-lab5-s3-init-1 status=exited started=2026-11-10T11:39:59.844406543Z finished=2026-11-10T11:40:01.82853823Z
|
||||
/messenger-lab5-valkey-1 status=running started=2026-11-10T11:42:59.473694935Z finished=2026-11-10T11:42:58.500195842Z
|
||||
/messenger-lab5-worker-1 status=running started=2026-11-10T11:43:35.475881026Z finished=2026-11-10T11:42:58.056034905Z
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:44:23Z
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
UTC: 2026-11-10T11:43:49Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode failover --action-script scripts/stop-one.sh
|
||||
Container messenger-lab5-tester-run-e77463b85f85 Creating
|
||||
Container messenger-lab5-tester-run-e77463b85f85 Created
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 108, "failed_attempts": 0, "first_recovery_seconds": 4.697}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:44:14Z
|
||||
|
||||
UTC: 2026-11-10T11:44:14Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester sh scripts/start-api.sh
|
||||
Container messenger-lab5-tester-run-b663461ce1d4 Creating
|
||||
Container messenger-lab5-tester-run-b663461ce1d4 Created
|
||||
started api container 17151567e8aa
|
||||
running api replicas: 2
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:44:22Z
|
||||
@@ -0,0 +1,16 @@
|
||||
UTC: 2026-11-10T11:45:13.736656+00:00
|
||||
Tested working tree: see run-context.txt.
|
||||
PASS: Contract structure; evidence/check.txt
|
||||
PASS: OpenAPI and JSON Schema; evidence/validate.txt
|
||||
PASS: Public API smoke; evidence/smoke.txt
|
||||
PASS: 7 image validation regressions; evidence/image-validation.txt
|
||||
PASS: Real image variants and ACL; evidence/images.txt
|
||||
PASS: Decoder memory and timeout; evidence/limits.txt
|
||||
PASS: All 6 recovery scenarios; evidence/recovery.txt
|
||||
PASS: Public persistence after store crash-restart; evidence/persistence.txt
|
||||
PASS: Public failover and API restoration; evidence/failover.txt
|
||||
PASS: Retry configuration and jitter; evidence/pipeline-validation.txt
|
||||
Quick runner and full recovery/public scenarios were run separately.
|
||||
Public restart hook intentionally SIGKILLs stores after stopping application writers.
|
||||
Original unsuccessful restart log retained in persistence-before-fix.txt.
|
||||
Final readiness: HTTP 200, status=ok; both API replicas healthy; persistent volumes kept.
|
||||
@@ -0,0 +1,19 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:21Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/image_validation_unit.py
|
||||
Container messenger-lab5-tester-run-2f25c0612b0e Creating
|
||||
Container messenger-lab5-tester-run-2f25c0612b0e Created
|
||||
test_adam7_pass_filter_boundaries (__main__.ImageValidationTests.test_adam7_pass_filter_boundaries) ... ok
|
||||
test_crc_compressed_tail_and_pixel_bomb (__main__.ImageValidationTests.test_crc_compressed_tail_and_pixel_bomb) ... ok
|
||||
test_critical_headers_and_chunk_order (__main__.ImageValidationTests.test_critical_headers_and_chunk_order) ... ok
|
||||
test_filter_values_and_row_boundaries (__main__.ImageValidationTests.test_filter_values_and_row_boundaries) ... ok
|
||||
test_jpeg_and_truncated_progressive_above_old_bypass_limit (__main__.ImageValidationTests.test_jpeg_and_truncated_progressive_above_old_bypass_limit) ... ok
|
||||
test_palette_shape_and_transparency (__main__.ImageValidationTests.test_palette_shape_and_transparency) ... ok
|
||||
test_valid_png_and_trailing_padding (__main__.ImageValidationTests.test_valid_png_and_trailing_padding) ... ok
|
||||
|
||||
----------------------------------------------------------------------
|
||||
Ran 7 tests in 0.459s
|
||||
|
||||
OK
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:24Z
|
||||
@@ -0,0 +1,14 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:24Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u student-tests/lab5_images.py
|
||||
Container messenger-lab5-tester-run-8332865285ab Creating
|
||||
Container messenger-lab5-tester-run-8332865285ab Created
|
||||
PASS centered crop, three real variants, checksums, original bytes and ACL
|
||||
PASS no upscaling and floor rounding for thumbnail/optimized
|
||||
PASS transparent PNG composited onto white JPEG
|
||||
PASS EXIF orientation applied to variants and metadata removed
|
||||
PASS malformed PNG filter and truncated >16MP progressive JPEG rejected before 202
|
||||
PASS invalid/MIME mismatch 415, >10MiB and >40M pixels 413, unsafe filenames 400
|
||||
PASS: lab5 grade4 image checks
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:36Z
|
||||
@@ -0,0 +1,8 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:36Z
|
||||
Command: docker compose --profile tools exec -T worker python -u - < student-tests/lab5_limits.py
|
||||
PASS memory: real 16MP image decoder stopped at RLIMIT_AS=48MiB
|
||||
PASS timeout: decoder killed and reaped; parent continues
|
||||
PASS: lab5 decoder resource limits
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:38Z
|
||||
@@ -0,0 +1,15 @@
|
||||
Date: 2026-11-10 UTC
|
||||
Git HEAD: fec72a0189d8be39cdcfb4ab9cf99d9815114776
|
||||
Branch: solution/maksimova_bvt2303
|
||||
Working tree was clean when the command started; subsequent changes are submission documentation only.
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester make check test BASE_URL=http://lb:8080
|
||||
|
||||
Container messenger-lab5-tester-run-0aa025c6093a Creating
|
||||
Container messenger-lab5-tester-run-0aa025c6093a Created
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://lb:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
RESULT: PASS (exit 0)
|
||||
This run covers make check and make test. Earlier failure/security scenarios are recorded separately in recovery.txt, images.txt, limits.txt, persistence.txt and failover.txt.
|
||||
@@ -0,0 +1,10 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:54Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode persistence --action-script scripts/restart.sh
|
||||
Container messenger-lab5-tester-run-a986bb1db1f1 Creating
|
||||
Container messenger-lab5-tester-run-a986bb1db1f1 Created
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
FAIL: restart hook failed (inspect your script locally)
|
||||
|
||||
|
||||
RESULT: FAIL (exit 1) at 2026-11-10T11:34:26Z
|
||||
@@ -0,0 +1,9 @@
|
||||
|
||||
UTC: 2026-11-10T11:42:41Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/resilience.py --mode persistence --action-script scripts/restart.sh
|
||||
Container messenger-lab5-tester-run-6871f7ab3b9a Creating
|
||||
Container messenger-lab5-tester-run-6871f7ab3b9a Created
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:43:48Z
|
||||
@@ -0,0 +1,8 @@
|
||||
UTC: 2026-11-10T11:37:53.098883+00:00
|
||||
Command: docker compose exec -T --index 1 api python (inline configuration/jitter assertions)
|
||||
PASS: invalid retry base rejected: 0
|
||||
PASS: invalid retry base rejected: -1
|
||||
PASS: invalid retry base rejected: nan
|
||||
PASS: invalid retry base rejected: inf
|
||||
PASS: configured retry jitter and 60 second cap
|
||||
Exit: 0
|
||||
Binary file not shown.
@@ -0,0 +1,11 @@
|
||||
UTC: 2026-11-10T11:27:14.855858+00:00
|
||||
$ docker compose --profile tools run --rm tester python3 student-tests/lab5_recovery.py
|
||||
Container messenger-lab5-tester-run-ce9027d69d17 Creating
|
||||
Container messenger-lab5-tester-run-ce9027d69d17 Created
|
||||
PASS persistence: attachment=01bac600-7535-4a3e-ae86-caf27d6577c6 same session/ID after API+broker restart; volumes preserved; queued content409; messages201; ready
|
||||
PASS crash_after_commit_before_publish: attachment=21351fb0-a029-4ad8-a528-e8d13c2a7178 hits=1 recovered_seconds=2.7; same original; three variants
|
||||
PASS crash_after_objects_before_commit: attachment=e9d2d8c1-483a-4a98-a8b4-ded94a3a72fe hits=1 recovered_seconds=119.5; same original; three variants
|
||||
PASS crash_after_commit_before_ack: attachment=e6673c1d-c26c-4088-afaa-17c01b407add hits=1 recovered_seconds=0.3; same original; three variants
|
||||
PASS failed/replay: attachment=d1fe0bd8-9df7-4aa4-9fcd-8faacb78ef2c; permanent error fails once; failed registry; content/link409; audited replay preserves job_id; ready duplicate no-op
|
||||
PASS transient S3 outage: attachment=1dc9ad14-4c3a-4af0-b1af-7171d0e2cdba; attempts=5; durable retries, retry_exhausted, explicit replay after recovery
|
||||
PASS: lab5 grade4 recovery checks (all)
|
||||
@@ -0,0 +1,10 @@
|
||||
UTC: 2026-11-10T11:31:44.505443+00:00
|
||||
Host: Windows 11
|
||||
Host Python: 3.13.3
|
||||
Git base HEAD: 276dbdad8a34da4c5b57fbf1879ca169b581d75b
|
||||
Branch: solution/maksimova_bvt2303
|
||||
Implementation includes uncommitted working-tree changes; HEAD alone is not the tested version.
|
||||
Docker=29.3.0 CPU=14 MemoryBytes=7965708288
|
||||
LB binding: 0.0.0.0:8081
|
||||
Compose memory/CPU/job limits: compose.yaml and REPORT.md. Worker concurrency=1, prefetch=1.
|
||||
All test data are synthetic. Existing volumes are preserved.
|
||||
@@ -0,0 +1,129 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:04Z
|
||||
Command: docker compose --profile tools ps --status running --services
|
||||
api
|
||||
broker
|
||||
db
|
||||
lb
|
||||
publisher
|
||||
s3
|
||||
valkey
|
||||
worker
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:04Z
|
||||
|
||||
UTC: 2026-11-10T11:32:05Z
|
||||
Command: docker compose --profile tools build tester
|
||||
Image messenger-lab5-tester Building
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 587B 0.0s done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [internal] load build definition from Dockerfile.tester
|
||||
#2 transferring dockerfile: 405B 0.0s done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/docker:28.5.1-cli
|
||||
#3 ...
|
||||
|
||||
#4 [internal] load metadata for docker.io/library/python:3.12.11-alpine3.22
|
||||
#4 DONE 0.2s
|
||||
|
||||
#3 [internal] load metadata for docker.io/library/docker:28.5.1-cli
|
||||
#3 DONE 0.2s
|
||||
|
||||
#5 [internal] load .dockerignore
|
||||
#5 transferring context: 243B done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [stage-1 1/5] FROM docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757
|
||||
#6 resolve docker.io/library/python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 0.1s done
|
||||
#6 DONE 0.1s
|
||||
|
||||
#7 [docker-cli 1/1] FROM docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad
|
||||
#7 resolve docker.io/library/docker:28.5.1-cli@sha256:9190b0613792e658a7783cf14b2d5ace5941bb68ede7276922ea36ee457d76ad 0.1s done
|
||||
#7 DONE 0.1s
|
||||
|
||||
#8 [stage-1 2/5] RUN apk add --no-cache make
|
||||
#8 CACHED
|
||||
|
||||
#9 [stage-1 3/5] RUN pip install --no-cache-dir Pillow==11.0.0
|
||||
#9 CACHED
|
||||
|
||||
#10 [stage-1 4/5] COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
|
||||
#10 CACHED
|
||||
|
||||
#11 [stage-1 5/5] WORKDIR /work
|
||||
#11 CACHED
|
||||
|
||||
#12 exporting to image
|
||||
#12 exporting layers done
|
||||
#12 exporting manifest sha256:7120867ac411a644ce6a37f369e42c49abd4b27d4de8895a44d98606b1377ba7 done
|
||||
#12 exporting config sha256:901695a7e540d71d60e4a269b7c85ab5164fa732591e725d1e9eec78b948b7b8 done
|
||||
#12 exporting attestation manifest sha256:d1b87aa0599d2c09eec65682d480479ecf2ef5c5ab10faf972df22b047eee851
|
||||
#12 exporting attestation manifest sha256:d1b87aa0599d2c09eec65682d480479ecf2ef5c5ab10faf972df22b047eee851 0.1s done
|
||||
#12 exporting manifest list sha256:e126c7a385f8843f487c92bcabe0284499ecfe085a0c8a2ba46cc69b9e4ef494
|
||||
#12 exporting manifest list sha256:e126c7a385f8843f487c92bcabe0284499ecfe085a0c8a2ba46cc69b9e4ef494 0.1s done
|
||||
#12 naming to docker.io/library/messenger-lab5-tester:latest done
|
||||
#12 unpacking to docker.io/library/messenger-lab5-tester:latest 0.0s done
|
||||
#12 DONE 0.4s
|
||||
|
||||
#13 resolving provenance for metadata file
|
||||
#13 DONE 0.0s
|
||||
Image messenger-lab5-tester Built
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:07Z
|
||||
|
||||
UTC: 2026-11-10T11:32:52Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester chmod +x scripts/restart.sh scripts/stop-one.sh
|
||||
Container messenger-lab5-tester-run-a914ec3f149c Creating
|
||||
Container messenger-lab5-tester-run-a914ec3f149c Created
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:53Z
|
||||
|
||||
UTC: 2026-11-10T11:39:43Z
|
||||
Command: docker compose --profile tools start db valkey s3 broker api lb publisher worker
|
||||
Container messenger-lab5-s3-1 Waiting
|
||||
Container messenger-lab5-db-1 Waiting
|
||||
Container messenger-lab5-db-1 Healthy
|
||||
Container messenger-lab5-migrate-1 Starting
|
||||
Container messenger-lab5-migrate-1 Started
|
||||
Container messenger-lab5-s3-1 Healthy
|
||||
Container messenger-lab5-s3-init-1 Starting
|
||||
Container messenger-lab5-s3-init-1 Started
|
||||
Container messenger-lab5-db-1 Waiting
|
||||
Container messenger-lab5-valkey-1 Waiting
|
||||
Container messenger-lab5-s3-init-1 Waiting
|
||||
Container messenger-lab5-migrate-1 Waiting
|
||||
Container messenger-lab5-broker-1 Waiting
|
||||
Container messenger-lab5-s3-init-1 Waiting
|
||||
Container messenger-lab5-migrate-1 Waiting
|
||||
Container messenger-lab5-s3-init-1 Waiting
|
||||
Container messenger-lab5-broker-1 Waiting
|
||||
Container messenger-lab5-migrate-1 Waiting
|
||||
Container messenger-lab5-db-1 Healthy
|
||||
Container messenger-lab5-migrate-1 Exited
|
||||
Container messenger-lab5-migrate-1 Exited
|
||||
Container messenger-lab5-valkey-1 Healthy
|
||||
Container messenger-lab5-broker-1 Healthy
|
||||
Container messenger-lab5-migrate-1 Exited
|
||||
Container messenger-lab5-broker-1 Healthy
|
||||
Container messenger-lab5-s3-init-1 Exited
|
||||
Container messenger-lab5-publisher-1 Starting
|
||||
Container messenger-lab5-s3-init-1 Exited
|
||||
Container messenger-lab5-worker-1 Starting
|
||||
Container messenger-lab5-s3-init-1 Exited
|
||||
Container messenger-lab5-api-1 Starting
|
||||
Container messenger-lab5-publisher-1 Started
|
||||
Container messenger-lab5-worker-1 Started
|
||||
Container messenger-lab5-api-1 Started
|
||||
Container messenger-lab5-api-2 Starting
|
||||
Container messenger-lab5-api-2 Started
|
||||
Container messenger-lab5-api-1 Waiting
|
||||
Container messenger-lab5-api-2 Waiting
|
||||
Container messenger-lab5-api-1 Healthy
|
||||
Container messenger-lab5-api-2 Healthy
|
||||
Container messenger-lab5-lb-1 Starting
|
||||
Container messenger-lab5-lb-1 Started
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:40:09Z
|
||||
@@ -0,0 +1,8 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:13Z
|
||||
Command: docker compose --profile tools run --rm -T --no-deps tester python3 -u tests/smoke.py
|
||||
Container messenger-lab5-tester-run-7aab6dac8e41 Creating
|
||||
Container messenger-lab5-tester-run-7aab6dac8e41 Created
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:21Z
|
||||
@@ -0,0 +1,51 @@
|
||||
|
||||
UTC: 2026-11-10T11:32:04Z
|
||||
Command: docker compose --profile tools ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab5-api-1 messenger-lab5-api:dev "python -m app.server" api 6 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
messenger-lab5-api-2 messenger-lab5-api:dev "python -m app.server" api 6 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
messenger-lab5-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 2 days ago Up 4 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab5-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 2 days ago Up 7 minutes (healthy) 5432/tcp
|
||||
messenger-lab5-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 2 days ago Up 7 minutes (healthy) 127.0.0.1:8404->8404/tcp, 0.0.0.0:8081->8080/tcp, [::]:8081->8080/tcp
|
||||
messenger-lab5-migrate-1 messenger-lab5-api:dev "python scripts/migr…" migrate 6 minutes ago Exited (0) 6 minutes ago
|
||||
messenger-lab5-publisher-1 messenger-lab5-api:dev "python -m app.jobs" publisher 6 minutes ago Up 3 minutes 8080/tcp
|
||||
messenger-lab5-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 2 days ago Up 44 seconds (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab5-s3-init-1 messenger-lab5-api:dev "python scripts/s3_b…" s3-init 6 minutes ago Exited (0) 6 minutes ago
|
||||
messenger-lab5-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 days ago Up 7 minutes (healthy) 6379/tcp
|
||||
messenger-lab5-worker-1 messenger-lab5-api:dev "python -m app.worker" worker 6 minutes ago Up About a minute 8080/tcp
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:05Z
|
||||
|
||||
UTC: 2026-11-10T11:32:38Z
|
||||
Command: docker compose --profile tools ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab5-api-1 messenger-lab5-api:dev "python -m app.server" api 6 minutes ago Up 5 minutes (healthy) 8080/tcp
|
||||
messenger-lab5-api-2 messenger-lab5-api:dev "python -m app.server" api 6 minutes ago Up 5 minutes (healthy) 8080/tcp
|
||||
messenger-lab5-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 2 days ago Up 5 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab5-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 2 days ago Up 8 minutes (healthy) 5432/tcp
|
||||
messenger-lab5-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 2 days ago Up 7 minutes (healthy) 127.0.0.1:8404->8404/tcp, 0.0.0.0:8081->8080/tcp, [::]:8081->8080/tcp
|
||||
messenger-lab5-migrate-1 messenger-lab5-api:dev "python scripts/migr…" migrate 6 minutes ago Exited (0) 6 minutes ago
|
||||
messenger-lab5-publisher-1 messenger-lab5-api:dev "python -m app.jobs" publisher 6 minutes ago Up 4 minutes 8080/tcp
|
||||
messenger-lab5-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 2 days ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab5-s3-init-1 messenger-lab5-api:dev "python scripts/s3_b…" s3-init 6 minutes ago Exited (0) 6 minutes ago
|
||||
messenger-lab5-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 days ago Up 8 minutes (healthy) 6379/tcp
|
||||
messenger-lab5-worker-1 messenger-lab5-api:dev "python -m app.worker" worker 6 minutes ago Up About a minute 8080/tcp
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:32:39Z
|
||||
|
||||
UTC: 2026-11-10T11:44:23Z
|
||||
Command: docker compose --profile tools ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab5-api-1 messenger-lab5-api:dev "python -m app.server" api 7 minutes ago Up 7 seconds (healthy) 8080/tcp
|
||||
messenger-lab5-api-2 messenger-lab5-api:dev "python -m app.server" api 7 minutes ago Up 47 seconds (healthy) 8080/tcp
|
||||
messenger-lab5-broker-1 rabbitmq:3.13.7-alpine "/bin/sh -ec 'export…" broker 7 minutes ago Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
messenger-lab5-db-1 postgres:16.4-alpine "docker-entrypoint.s…" db 2 days ago Up About a minute (healthy) 5432/tcp
|
||||
messenger-lab5-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb 2 days ago Up 40 seconds (healthy) 127.0.0.1:8404->8404/tcp, 0.0.0.0:8081->8080/tcp, [::]:8081->8080/tcp
|
||||
messenger-lab5-migrate-1 messenger-lab5-api:dev "python scripts/migr…" migrate 7 minutes ago Exited (0) 4 minutes ago
|
||||
messenger-lab5-publisher-1 messenger-lab5-api:dev "python -m app.jobs" publisher 7 minutes ago Up 48 seconds 8080/tcp
|
||||
messenger-lab5-s3-1 chrislusf/seaweedfs:3.97 "/bin/sh /opt/lab/st…" s3 2 days ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp
|
||||
messenger-lab5-s3-init-1 messenger-lab5-api:dev "python scripts/s3_b…" s3-init 7 minutes ago Exited (0) 4 minutes ago
|
||||
messenger-lab5-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 2 days ago Up About a minute (healthy) 6379/tcp
|
||||
messenger-lab5-worker-1 messenger-lab5-api:dev "python -m app.worker" worker 7 minutes ago Up 48 seconds 8080/tcp
|
||||
|
||||
RESULT: PASS (exit 0) at 2026-11-10T11:44:23Z
|
||||
@@ -0,0 +1,4 @@
|
||||
UTC: 2026-11-10T11:31:55.081159+00:00
|
||||
Command: .venv/Scripts/python -X utf8 scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
Exit: 0
|
||||
@@ -0,0 +1,102 @@
|
||||
# HAProxy 3.0 — единственная публичная точка входа (порт 8080).
|
||||
# Backend-адреса берутся из встроенного DNS Docker: имя сервиса `api` возвращает A-запись
|
||||
# каждого работающего контейнера-реплики. Реплики не публикуют порты наружу.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2048
|
||||
|
||||
# Встроенный DNS Docker. hold valid 2s — как часто перечитывается список реплик
|
||||
# (новые контейнеры при scale/rolling update появляются в пуле за ≤ 2–3 s).
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold other 2s
|
||||
hold refused 2s
|
||||
hold nx 2s
|
||||
hold timeout 2s
|
||||
hold obsolete 2s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
# Таймауты согласованы с приложением:
|
||||
# connect 1s — недоступная реплика быстро даёт ошибку соединения -> retry на другую;
|
||||
# server 10s — больше statement_timeout БД (5 s) + ожидания пула (3 s);
|
||||
# http-keep-alive 10s < timeout_keep_alive uvicorn (30 s): LB закрывает простой первым.
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 10s
|
||||
timeout queue 5s
|
||||
retries 2
|
||||
option redispatch 1
|
||||
|
||||
frontend http_in
|
||||
bind :8080
|
||||
# X-Request-Id создаётся на входе (если клиент не прислал корректный), передаётся в API
|
||||
# и попадает в лог LB — один ID в логах LB и реплики.
|
||||
unique-id-format %{+X}o\ %ci:%cp_%fi:%fp_%Ts_%rt:%pid
|
||||
acl has_valid_rid req.hdr(X-Request-Id) -m reg ^[A-Za-z0-9._:-]{1,128}$
|
||||
http-request set-header X-Request-Id %[unique-id] unless has_valid_rid
|
||||
http-request set-var(txn.rid) req.hdr(X-Request-Id)
|
||||
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.rid)] \"%HM %HPO\""
|
||||
default_backend api
|
||||
|
||||
# Ошибки, созданные самим LB (нет живых реплик / timeout), — в формате контракта.
|
||||
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
|
||||
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет доступных реплик API\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
|
||||
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Реплика API не ответила вовремя\",\"request_id\":\"%[var(txn.rid)]\"}}" hdr X-Request-Id "%[var(txn.rid)]"
|
||||
|
||||
backend api
|
||||
# Обычный round-robin без sticky session: корректность обеспечивает общее состояние в PostgreSQL.
|
||||
balance roundrobin
|
||||
|
||||
# Активная проверка готовности каждые 1 s; 2 неудачи подряд -> DOWN, 2 успеха -> UP.
|
||||
# Максимальное время обнаружения отказа «процесс жив, но не готов»:
|
||||
# inter × fall + timeout check = 1 s × 2 + 1 s ≈ 3 s.
|
||||
# Упавший процесс (connection refused) обнаруживается быстрее: observe layer4 + error-limit 1
|
||||
# переводит сервер в DOWN по первой реальной ошибке соединения, а запрос уходит на другую реплику.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker init-addr none resolve-prefer ipv4 check
|
||||
timeout check 1s
|
||||
|
||||
# Повторы на другую реплику:
|
||||
# - ошибка установления соединения повторяется всегда (запрос не дошёл до приложения);
|
||||
# - L7-повторы (пустой ответ / 503 от дренирующейся реплики) разрешены только для
|
||||
# безопасных GET и для POST сообщения С Idempotency-Key — его повтор не создаст дубль.
|
||||
# POST /users, /chats, /auth/sessions после отправки не повторяются (нет ключа идемпотентности).
|
||||
retry-on conn-failure empty-response 503
|
||||
acl is_message_post path_reg ^/api/v1/chats/[^/]+/messages$
|
||||
acl has_idem_key req.hdr(Idempotency-Key) -m found
|
||||
http-request disable-l7-retry if METH_POST !is_message_post
|
||||
http-request disable-l7-retry if METH_POST !has_idem_key
|
||||
http-request disable-l7-retry if METH_DELETE
|
||||
# Lab4: 503 вложения означает отказ общего S3 — на другой реплике тот же S3, повтор
|
||||
# только утроил бы ожидание клиента (3 × S3_OPERATION_TIMEOUT_SECONDS). conn-failure остаётся.
|
||||
http-request disable-l7-retry if { path_beg /api/v1/attachments/ }
|
||||
# Upload: после приёма тела API проверяет PNG/JPEG и делает S3 PUT с дедлайном
|
||||
# 6 s + size / 1 MiB/s (<= 16 s для 10 MiB) — 10 s «timeout server» мало.
|
||||
http-request set-timeout server 40s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ }
|
||||
|
||||
# До 8 реплик (scale/rolling update); пустые слоты остаются в MAINT, пока DNS не вернёт адрес.
|
||||
server-template api 8 api:8080
|
||||
|
||||
# Служебная страница состояния backend-ов (UP/DOWN, счётчики ошибок/повторов).
|
||||
# В compose.yaml публикуется только на 127.0.0.1 хоста — не часть публичного API.
|
||||
frontend stats
|
||||
bind :8404
|
||||
http-request return status 200 content-type text/plain string "ok" if { path /lb-health }
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
stats show-legends
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"identities": [
|
||||
{
|
||||
"name": "admin",
|
||||
"credentials": [
|
||||
{ "accessKey": "@ADMIN_ACCESS_KEY@", "secretKey": "@ADMIN_SECRET_KEY@" }
|
||||
],
|
||||
"actions": ["Admin", "Read", "List", "Tagging", "Write"]
|
||||
},
|
||||
{
|
||||
"name": "messenger-api",
|
||||
"credentials": [
|
||||
{ "accessKey": "@API_ACCESS_KEY@", "secretKey": "@API_SECRET_KEY@" }
|
||||
],
|
||||
"actions": [
|
||||
"Read:messenger-attachments",
|
||||
"Write:messenger-attachments",
|
||||
"List:messenger-attachments"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
# SeaweedFS all-in-one (master + volume + filer + S3 gateway) for the lab stand.
|
||||
#
|
||||
# - master/volume/filer and their web UIs listen on 127.0.0.1 INSIDE the
|
||||
# container only (-ip.bind): their HTTP API has no authentication and must
|
||||
# not be reachable from the Docker network. Only the S3 gateway (9000)
|
||||
# listens on all interfaces, and it requires SigV4 credentials.
|
||||
# - Identities and rights come from s3.json.template; the secret values are
|
||||
# substituted at start from Docker secrets into a file readable by root
|
||||
# only (never stored in Git or in the image).
|
||||
set -eu
|
||||
umask 077
|
||||
s() { cat "/run/secrets/$1"; }
|
||||
sed -e "s|@ADMIN_ACCESS_KEY@|$(s s3_admin_access_key_id)|" \
|
||||
-e "s|@ADMIN_SECRET_KEY@|$(s s3_admin_secret_access_key)|" \
|
||||
-e "s|@API_ACCESS_KEY@|$(s s3_access_key_id)|" \
|
||||
-e "s|@API_SECRET_KEY@|$(s s3_secret_access_key)|" \
|
||||
/opt/lab/s3.json.template > /tmp/s3.json
|
||||
|
||||
exec weed -logtostderr=true server \
|
||||
-dir=/data \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.volumeSizeLimitMB=256 -volume.max=0 \
|
||||
-s3 -s3.port=9000 -s3.ip.bind=0.0.0.0 -s3.config=/tmp/s3.json
|
||||
@@ -0,0 +1,48 @@
|
||||
CREATE TABLE IF NOT EXISTS users (
|
||||
id UUID PRIMARY KEY,
|
||||
username VARCHAR(32) NOT NULL UNIQUE,
|
||||
display_name VARCHAR(100) NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT users_username_format_check
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
|
||||
CONSTRAINT users_display_name_nonblank_check
|
||||
CHECK (char_length(btrim(display_name)) >= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title VARCHAR(255) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT chats_title_nonblank_check
|
||||
CHECK (char_length(btrim(title)) >= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
text VARCHAR(4000) NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT messages_text_nonblank_check
|
||||
CHECK (char_length(btrim(text)) >= 1)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id UUID PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
token_hash CHAR(64) NOT NULL UNIQUE,
|
||||
expires_at TIMESTAMPTZ NOT NULL,
|
||||
revoked_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE INDEX IF NOT EXISTS messages_keyset_idx
|
||||
ON messages (chat_id, created_at ASC, id ASC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chats_keyset_idx
|
||||
ON chats (created_at ASC, id ASC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS chat_members_user_chat_idx
|
||||
ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS sessions_active_token_idx
|
||||
ON sessions (token_hash, expires_at)
|
||||
WHERE revoked_at IS NULL;
|
||||
@@ -0,0 +1,18 @@
|
||||
-- Lab3: shared idempotency of POST /chats/{chat_id}/messages.
|
||||
-- Scope: user + method + chat path + key. The key row and the message are
|
||||
-- written in one transaction, so either both exist or neither does.
|
||||
-- Expand-only change: the running lab2/lab3 API keeps working with it.
|
||||
CREATE TABLE IF NOT EXISTS idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
method VARCHAR(8) NOT NULL,
|
||||
key VARCHAR(128) NOT NULL,
|
||||
request_hash CHAR(64) NOT NULL,
|
||||
message_id UUID REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, method, key)
|
||||
);
|
||||
|
||||
-- For a periodic cleanup of keys older than IDEMPOTENCY_TTL_SECONDS.
|
||||
CREATE INDEX IF NOT EXISTS idempotency_keys_created_idx
|
||||
ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,68 @@
|
||||
-- Lab4: attachments. PostgreSQL keeps metadata, the bytes live in a private
|
||||
-- S3 bucket under a server-generated object_key. Expand-only migration: the
|
||||
-- lab3 API keeps working on this schema.
|
||||
|
||||
-- Text may be empty when the message has attachments; "not blank without
|
||||
-- attachments" is checked by the API (one message + its links are inserted
|
||||
-- in one transaction).
|
||||
ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_nonblank_check;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS attachments (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE RESTRICT,
|
||||
owner_id UUID NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
|
||||
-- Display name only; never used as a path or an S3 key.
|
||||
filename VARCHAR(255) NOT NULL,
|
||||
content_type VARCHAR(32) NOT NULL,
|
||||
size_bytes BIGINT NOT NULL,
|
||||
sha256 CHAR(64) NOT NULL,
|
||||
width INTEGER NOT NULL,
|
||||
height INTEGER NOT NULL,
|
||||
-- Lab4 has only "ready": the row is inserted after the object is stored.
|
||||
status VARCHAR(16) NOT NULL DEFAULT 'ready',
|
||||
error_code VARCHAR(64),
|
||||
object_key TEXT NOT NULL UNIQUE,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
|
||||
CONSTRAINT attachments_content_type_check
|
||||
CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size_check
|
||||
CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256_check
|
||||
CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('ready'))
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id);
|
||||
|
||||
-- A message references 0-10 attachments of its own chat; "position" keeps
|
||||
-- the order of attachment_ids. One attachment may be referenced by several
|
||||
-- messages of the chat.
|
||||
CREATE TABLE IF NOT EXISTS message_attachments (
|
||||
message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT,
|
||||
position SMALLINT NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9),
|
||||
CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx
|
||||
ON message_attachments (attachment_id);
|
||||
|
||||
-- Upload intent: committed BEFORE the S3 PUT and deleted in the same
|
||||
-- transaction that inserts the attachment row. An intent older than the
|
||||
-- grace period therefore means "the object may exist in S3, but the request
|
||||
-- never committed" (crash, lost DB, client gone) -> the janitor deletes the
|
||||
-- object and then the intent.
|
||||
CREATE TABLE IF NOT EXISTS upload_intents (
|
||||
object_key TEXT PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL,
|
||||
chat_id UUID NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS upload_intents_created_idx
|
||||
ON upload_intents (created_at);
|
||||
@@ -0,0 +1,49 @@
|
||||
-- Lab5: the database owns the work intent; RabbitMQ carries only its IDs.
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN staging_key TEXT,
|
||||
ADD COLUMN pipeline_version INTEGER NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
|
||||
ADD COLUMN job_id UUID UNIQUE,
|
||||
ADD COLUMN attempt INTEGER NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
ADD COLUMN attempts_run INTEGER NOT NULL DEFAULT 0,
|
||||
ADD COLUMN lease_token UUID,
|
||||
ADD COLUMN lease_until TIMESTAMPTZ,
|
||||
ADD COLUMN next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
ADD COLUMN staging_deleted BOOLEAN NOT NULL DEFAULT false,
|
||||
ADD COLUMN variants JSONB NOT NULL DEFAULT '[]'::jsonb;
|
||||
|
||||
CREATE TABLE outbox (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id UUID NOT NULL,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
attempt INTEGER NOT NULL DEFAULT 1,
|
||||
available_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
published_at TIMESTAMPTZ,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX outbox_pending_idx ON outbox(available_at, id) WHERE published_at IS NULL;
|
||||
CREATE INDEX attachments_lease_idx ON attachments(lease_until) WHERE status = 'processing';
|
||||
CREATE INDEX attachments_staging_idx ON attachments(updated_at)
|
||||
WHERE staging_key IS NOT NULL AND NOT staging_deleted;
|
||||
|
||||
-- Failed registry = attachments WHERE status='failed'. This append-only
|
||||
-- journal explains retry, lease recovery and explicit operator replay.
|
||||
CREATE TABLE job_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
attachment_id UUID NOT NULL REFERENCES attachments(id),
|
||||
event TEXT NOT NULL,
|
||||
attempt INTEGER NOT NULL,
|
||||
detail TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_events_attachment_idx ON job_events(attachment_id, id);
|
||||
|
||||
-- Internal one-shot crash injection, never exposed through public HTTP.
|
||||
CREATE TABLE fault_injections (
|
||||
point TEXT PRIMARY KEY,
|
||||
remaining INTEGER NOT NULL DEFAULT 0 CHECK (remaining >= 0),
|
||||
hits INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
);
|
||||
@@ -0,0 +1,15 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
pydantic==2.9.2
|
||||
SQLAlchemy==2.0.36
|
||||
asyncpg==0.30.0
|
||||
argon2-cffi==23.1.0
|
||||
redis==5.0.8
|
||||
# Lab4: S3 client (async) and image header/JPEG decoding
|
||||
aiobotocore==2.15.2
|
||||
botocore==1.35.36
|
||||
aiohttp==3.10.10
|
||||
Pillow==11.0.0
|
||||
|
||||
# Lab5: durable RabbitMQ messages and publisher confirmations.
|
||||
aio-pika==9.4.3
|
||||
@@ -0,0 +1,32 @@
|
||||
# Creates the same local secret files as init-secrets.sh on Windows.
|
||||
# Existing values are preserved and never printed.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
$secretDirectory = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '../infra/secrets'))
|
||||
[IO.Directory]::CreateDirectory($secretDirectory) | Out-Null
|
||||
function New-RandomHex {
|
||||
$bytes = New-Object byte[] 24
|
||||
$randomGenerator = [Security.Cryptography.RandomNumberGenerator]::Create()
|
||||
try { $randomGenerator.GetBytes($bytes) } finally { $randomGenerator.Dispose() }
|
||||
return ([BitConverter]::ToString($bytes)).Replace('-', '').ToLowerInvariant()
|
||||
}
|
||||
function Write-NewSecret([string]$Name, [string]$Value) {
|
||||
$secretPath = Join-Path $secretDirectory $Name
|
||||
if (Test-Path -LiteralPath $secretPath) { Write-Output "keep infra/secrets/$Name" }
|
||||
else {
|
||||
[IO.File]::WriteAllText($secretPath, $Value, (New-Object Text.UTF8Encoding $false))
|
||||
Write-Output "create infra/secrets/$Name"
|
||||
}
|
||||
}
|
||||
Write-NewSecret 'postgres_password.txt' (New-RandomHex)
|
||||
$databasePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'postgres_password.txt')).Trim()
|
||||
Write-NewSecret 'database_url.txt' "postgresql://messenger:${databasePassword}@db:5432/messenger"
|
||||
Write-NewSecret 'valkey_password.txt' (New-RandomHex)
|
||||
$cachePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'valkey_password.txt')).Trim()
|
||||
Write-NewSecret 'cache_url.txt' "redis://:${cachePassword}@valkey:6379/0"
|
||||
Write-NewSecret 's3_admin_access_key_id.txt' ('admin' + (New-RandomHex).Substring(0,15))
|
||||
Write-NewSecret 's3_admin_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
|
||||
Write-NewSecret 's3_access_key_id.txt' ('api' + (New-RandomHex).Substring(0,17))
|
||||
Write-NewSecret 's3_secret_access_key.txt' ((New-RandomHex) + (New-RandomHex))
|
||||
Write-NewSecret 'broker_password.txt' (New-RandomHex)
|
||||
$queuePassword = [IO.File]::ReadAllText((Join-Path $secretDirectory 'broker_password.txt')).Trim()
|
||||
Write-NewSecret 'broker_url.txt' "amqp://messenger:${queuePassword}@broker:5672/"
|
||||
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
# Creates local Docker secrets for a synthetic lab stand.
|
||||
# Existing files are never overwritten; values are never printed.
|
||||
set -eu
|
||||
|
||||
dir="$(cd "$(dirname "$0")/.." && pwd)/infra/secrets"
|
||||
mkdir -p "$dir"
|
||||
# Host access is limited by the directory mode; the files themselves must stay
|
||||
# readable by the non-root users inside containers (bind-mounted secrets).
|
||||
chmod 700 "$dir" 2>/dev/null || true
|
||||
|
||||
random() {
|
||||
od -An -tx1 -N24 /dev/urandom | tr -d ' \n'
|
||||
}
|
||||
|
||||
write_new() {
|
||||
if [ -f "$dir/$1" ]; then
|
||||
echo "keep infra/secrets/$1"
|
||||
else
|
||||
printf '%s' "$2" > "$dir/$1"
|
||||
chmod 644 "$dir/$1" 2>/dev/null || true
|
||||
echo "create infra/secrets/$1"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ ! -f "$dir/postgres_password.txt" ]; then
|
||||
write_new postgres_password.txt "$(random)"
|
||||
fi
|
||||
pg_password="$(cat "$dir/postgres_password.txt")"
|
||||
write_new database_url.txt "postgresql://messenger:${pg_password}@db:5432/messenger"
|
||||
|
||||
if [ ! -f "$dir/valkey_password.txt" ]; then
|
||||
write_new valkey_password.txt "$(random)"
|
||||
fi
|
||||
valkey_password="$(cat "$dir/valkey_password.txt")"
|
||||
write_new cache_url.txt "redis://:${valkey_password}@valkey:6379/0"
|
||||
|
||||
# Lab4: S3 (SeaweedFS). Admin credentials are used only by the s3-init
|
||||
# bootstrap; the API gets its own key pair restricted to one bucket.
|
||||
write_new s3_admin_access_key_id.txt "admin$(random | cut -c1-15)"
|
||||
write_new s3_admin_secret_access_key.txt "$(random)$(random)"
|
||||
write_new s3_access_key_id.txt "api$(random | cut -c1-17)"
|
||||
write_new s3_secret_access_key.txt "$(random)$(random)"
|
||||
|
||||
# Lab5: RabbitMQ uses a separate local password. The URL is a Docker secret;
|
||||
# no credentials appear in compose.yaml or the report.
|
||||
if [ ! -f "$dir/broker_password.txt" ]; then
|
||||
write_new broker_password.txt "$(random)"
|
||||
fi
|
||||
broker_password="$(cat "$dir/broker_password.txt")"
|
||||
write_new broker_url.txt "amqp://messenger:${broker_password}@broker:5672/"
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Вариант жёсткого отказа (для сравнения): SIGKILL без graceful shutdown.
|
||||
# Используется в собственных экспериментах; публичный тест запускается со stop-one.sh.
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?target API instance alias required}"
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
for id in $(service_ids api); do
|
||||
case "$id" in "$TARGET_INSTANCE_ID"*)
|
||||
docker kill --signal KILL "$id" >/dev/null
|
||||
echo "killed api container $(short "$id") at $(date -u +%H:%M:%S)"
|
||||
exit 0 ;;
|
||||
esac
|
||||
done
|
||||
echo "instance $TARGET_INSTANCE_ID is not a running api container" >&2
|
||||
exit 3
|
||||
@@ -0,0 +1,33 @@
|
||||
#!/bin/sh
|
||||
# Общие функции операторских скриптов. Используют только docker CLI и метки Compose,
|
||||
# поэтому работают и на хосте, и в контейнере tester (docker.sock).
|
||||
PROJECT="${COMPOSE_PROJECT:-messenger-lab5}"
|
||||
|
||||
# ID контейнеров сервиса (по умолчанию только работающих); -a — включая остановленные.
|
||||
service_ids() {
|
||||
# $1 — сервис, $2 — необязательный флаг -a
|
||||
docker ps ${2:-} -q --no-trunc \
|
||||
--filter "label=com.docker.compose.project=$PROJECT" \
|
||||
--filter "label=com.docker.compose.service=$1"
|
||||
}
|
||||
|
||||
# Ждать, пока все переданные контейнеры станут healthy (timeout в секундах первым аргументом).
|
||||
# В POSIX sh переменные функций глобальны: имена с префиксом _wh_, чтобы не затереть
|
||||
# переменные вызывающего цикла.
|
||||
wait_healthy() {
|
||||
_wh_timeout="$1"; shift
|
||||
_wh_deadline=$(( $(date +%s) + _wh_timeout ))
|
||||
for _wh_id in "$@"; do
|
||||
while :; do
|
||||
_wh_state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$_wh_id" 2>/dev/null || echo missing)
|
||||
case "$_wh_state" in healthy|running) break ;; esac
|
||||
if [ "$(date +%s)" -ge "$_wh_deadline" ]; then
|
||||
echo "container $(echo "$_wh_id" | cut -c1-12) not healthy (state=$_wh_state)" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
short() { echo "$1" | cut -c1-12; }
|
||||
@@ -0,0 +1,99 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MIGRATIONS_DIR = ROOT / "migrations"
|
||||
# Lab3: migrations run once in the "migrate" service before the API replicas
|
||||
# start. The advisory lock still serialises two accidental parallel runs
|
||||
# (e.g. a manual run during a rolling update); the second one then sees all
|
||||
# versions in schema_migrations and only prints SKIP.
|
||||
MIGRATION_LOCK_ID = 727003
|
||||
|
||||
|
||||
def read_database_url() -> str:
|
||||
file_name = os.getenv("DATABASE_URL_FILE")
|
||||
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
|
||||
value = os.getenv("DATABASE_URL")
|
||||
|
||||
if not value:
|
||||
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required.")
|
||||
|
||||
return value
|
||||
|
||||
|
||||
def plain_postgres_url(url: str) -> str:
|
||||
if url.startswith("postgresql+asyncpg://"):
|
||||
return url.replace("postgresql+asyncpg://", "postgresql://", 1)
|
||||
|
||||
return url
|
||||
|
||||
|
||||
async def connect_with_retry(url: str, attempts: int = 30):
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
return await asyncpg.connect(url, timeout=3)
|
||||
except (OSError, asyncpg.PostgresError) as exc:
|
||||
if attempt == attempts:
|
||||
raise
|
||||
print(f"WAIT database ({type(exc).__name__}), retry {attempt}")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def migrate() -> None:
|
||||
connection = await connect_with_retry(
|
||||
plain_postgres_url(read_database_url())
|
||||
)
|
||||
|
||||
try:
|
||||
await connection.execute(
|
||||
"SELECT pg_advisory_lock($1)",
|
||||
MIGRATION_LOCK_ID,
|
||||
)
|
||||
await connection.execute(
|
||||
"""
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
)
|
||||
|
||||
applied_rows = await connection.fetch(
|
||||
"SELECT version FROM schema_migrations"
|
||||
)
|
||||
applied = {row["version"] for row in applied_rows}
|
||||
|
||||
for migration_file in sorted(MIGRATIONS_DIR.glob("*.sql")):
|
||||
version = migration_file.name
|
||||
|
||||
if version in applied:
|
||||
print(f"SKIP {version}")
|
||||
continue
|
||||
|
||||
sql = migration_file.read_text(encoding="utf-8")
|
||||
|
||||
print(f"APPLY {version}")
|
||||
|
||||
async with connection.transaction():
|
||||
await connection.execute(sql)
|
||||
await connection.execute(
|
||||
"INSERT INTO schema_migrations (version) VALUES ($1)",
|
||||
version,
|
||||
)
|
||||
|
||||
print("PASS: migrations applied")
|
||||
finally:
|
||||
await connection.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(migrate())
|
||||
@@ -0,0 +1,31 @@
|
||||
#!/bin/sh
|
||||
# Crash-restart every persistent store after draining the application.
|
||||
# No re-seed or volume removal; the whole hook is bounded to 85 seconds.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
api=$(service_ids api); lb=$(service_ids lb); db=$(service_ids db)
|
||||
valkey=$(service_ids valkey); s3=$(service_ids s3); broker=$(service_ids broker)
|
||||
worker=$(service_ids worker); publisher=$(service_ids publisher)
|
||||
[ -n "$api" ] && [ -n "$db" ] && [ -n "$lb" ] && [ -n "$valkey" ] && [ -n "$s3" ] && [ -n "$broker" ] && [ -n "$worker" ] && [ -n "$publisher" ] || { echo "stack is not running" >&2; exit 2; }
|
||||
_restart_deadline=$(( $(date +%s) + 85 ))
|
||||
remaining() {
|
||||
_restart_left=$(( _restart_deadline - $(date +%s) ))
|
||||
[ "$_restart_left" -gt 0 ] || { echo "restart deadline exceeded" >&2; exit 1; }
|
||||
echo "$_restart_left"
|
||||
}
|
||||
echo "stopping lb + api + publisher + worker, then all stores"
|
||||
docker stop --time 10 $lb $api $publisher $worker >/dev/null
|
||||
# Deliberate crash, not a graceful shutdown of the stores. This tests that
|
||||
# confirmed data survives process death and avoids spending the public
|
||||
# hook's entire deadline on S3/RabbitMQ internal shutdown procedures.
|
||||
# Writers are stopped first; persistent volumes are kept unchanged.
|
||||
docker kill --signal KILL $db $valkey $s3 $broker >/dev/null
|
||||
echo "starting db + valkey + s3 + broker with existing volumes"
|
||||
docker start $db $valkey $s3 $broker >/dev/null
|
||||
wait_healthy "$(remaining)" $db $valkey $s3 $broker
|
||||
echo "starting publisher + worker + api"
|
||||
docker start $publisher $worker $api >/dev/null
|
||||
wait_healthy "$(remaining)" $api
|
||||
docker start $lb >/dev/null
|
||||
wait_healthy "$(remaining)" $lb
|
||||
echo "restart complete: PostgreSQL + Valkey + S3 + RabbitMQ + publisher + worker + API replicas + LB"
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Idempotent bootstrap of the private attachments bucket (service s3-init).
|
||||
|
||||
Runs with the S3 ADMIN credentials (only this one-shot container gets them),
|
||||
before any API replica starts:
|
||||
|
||||
1. wait until the S3 gateway answers;
|
||||
2. create the bucket if it does not exist (a repeated run prints "keep");
|
||||
3. self-checks, the stand does not start if one fails:
|
||||
- anonymous (unsigned) GET/PUT/LIST of the bucket -> 403: bucket is private;
|
||||
- API credentials can PUT/GET/DELETE under the prefix;
|
||||
- API credentials can NOT create/delete buckets or touch other buckets.
|
||||
|
||||
The API identity and its rights are declared in infra/seaweedfs/s3.json.template.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import botocore.session
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
file_name = os.getenv(f"{name}_FILE")
|
||||
if file_name:
|
||||
return Path(file_name).read_text(encoding="utf-8").strip()
|
||||
value = os.getenv(name)
|
||||
if not value:
|
||||
raise RuntimeError(f"{name} or {name}_FILE is required")
|
||||
return value
|
||||
|
||||
|
||||
ENDPOINT = os.getenv("S3_ENDPOINT", "http://s3:9000")
|
||||
BUCKET = os.getenv("S3_BUCKET", "messenger-attachments")
|
||||
PREFIX = os.getenv("S3_OBJECT_PREFIX", "originals/")
|
||||
|
||||
|
||||
def client(access_key: str, secret_key: str):
|
||||
return botocore.session.get_session().create_client(
|
||||
"s3",
|
||||
endpoint_url=ENDPOINT,
|
||||
region_name=os.getenv("S3_REGION", "us-east-1"),
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=2,
|
||||
read_timeout=5,
|
||||
retries={"max_attempts": 1},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def error_code(exc: ClientError) -> str:
|
||||
return exc.response.get("Error", {}).get("Code", "")
|
||||
|
||||
|
||||
failures = 0
|
||||
|
||||
|
||||
def check(label: str, ok: bool) -> None:
|
||||
global failures
|
||||
failures += not ok
|
||||
print(("PASS " if ok else "FAIL ") + label, flush=True)
|
||||
|
||||
|
||||
def denied(call) -> bool:
|
||||
try:
|
||||
call()
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden", "AllAccessDisabled"}
|
||||
return False
|
||||
|
||||
|
||||
def anonymous_status(method: str, path: str, data: bytes | None = None) -> int:
|
||||
request = urllib.request.Request(ENDPOINT + path, data=data, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
admin = client(secret("S3_ADMIN_ACCESS_KEY_ID"), secret("S3_ADMIN_SECRET_ACCESS_KEY"))
|
||||
api = client(secret("S3_ACCESS_KEY_ID"), secret("S3_SECRET_ACCESS_KEY"))
|
||||
|
||||
for attempt in range(1, 61):
|
||||
try:
|
||||
admin.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
if attempt == 60:
|
||||
raise
|
||||
print(f"WAIT s3 ({type(exc).__name__}), retry {attempt}", flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
admin.head_bucket(Bucket=BUCKET)
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
except ClientError as exc:
|
||||
if error_code(exc) not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
raise
|
||||
try:
|
||||
admin.create_bucket(Bucket=BUCKET)
|
||||
print(f"create bucket {BUCKET}", flush=True)
|
||||
except ClientError as create_exc:
|
||||
# Two bootstrap runs at the same time: the other one won.
|
||||
if error_code(create_exc) not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
raise
|
||||
print(f"keep bucket {BUCKET}", flush=True)
|
||||
|
||||
probe = f"{PREFIX}bootstrap-probe-{uuid.uuid4().hex}"
|
||||
|
||||
check("api credentials: PUT object", not denied(
|
||||
lambda: api.put_object(Bucket=BUCKET, Key=probe, Body=b"probe")))
|
||||
check("api credentials: GET object",
|
||||
api.get_object(Bucket=BUCKET, Key=probe)["Body"].read() == b"probe")
|
||||
|
||||
check("anonymous GET object -> 403", anonymous_status("GET", f"/{BUCKET}/{probe}") == 403)
|
||||
check("anonymous LIST bucket -> 403", anonymous_status("GET", f"/{BUCKET}?list-type=2") == 403)
|
||||
check("anonymous PUT object -> 403",
|
||||
anonymous_status("PUT", f"/{BUCKET}/{PREFIX}anonymous-{uuid.uuid4().hex}", b"x") == 403)
|
||||
|
||||
other = f"forbidden-{uuid.uuid4().hex[:12]}"
|
||||
check("api credentials: CreateBucket denied",
|
||||
denied(lambda: api.create_bucket(Bucket=other)))
|
||||
check("api credentials: DeleteBucket denied",
|
||||
denied(lambda: api.delete_bucket(Bucket=BUCKET)))
|
||||
check("api credentials: DELETE object", not denied(
|
||||
lambda: api.delete_object(Bucket=BUCKET, Key=probe)))
|
||||
|
||||
def foreign_buckets_hidden() -> bool:
|
||||
# SeaweedFS denies ListBuckets to a bucket-scoped identity; other
|
||||
# servers may answer with only the allowed bucket. Both are fine.
|
||||
try:
|
||||
names = [b["Name"] for b in api.list_buckets().get("Buckets", [])]
|
||||
except ClientError as exc:
|
||||
return error_code(exc) in {"AccessDenied", "403", "Forbidden"}
|
||||
return all(name == BUCKET for name in names)
|
||||
|
||||
check("api credentials: ListBuckets shows no foreign buckets", foreign_buckets_hidden())
|
||||
|
||||
if failures:
|
||||
print(f"FAIL: {failures} bucket checks failed; the API must not start", flush=True)
|
||||
return 1
|
||||
|
||||
print("PASS: bucket bootstrap complete (idempotent, private, least privilege)", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
# Восстановление после stop-one.sh: запускает остановленные API-реплики и ждёт healthy.
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
all=$(service_ids api -a)
|
||||
running=$(service_ids api)
|
||||
started=""
|
||||
for id in $all; do
|
||||
case " $running " in *" $id "*) continue ;; esac
|
||||
docker start "$id" >/dev/null
|
||||
started="$started $id"
|
||||
echo "started api container $(short "$id")"
|
||||
done
|
||||
[ -z "$started" ] || wait_healthy 60 $started
|
||||
echo "running api replicas: $(service_ids api | wc -l | tr -d ' ')"
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
worker=$(service_ids worker -a)
|
||||
[ -n "$worker" ] || { echo "worker has not been created: run docker compose up -d" >&2; exit 2; }
|
||||
docker start $worker >/dev/null
|
||||
echo "worker started; pending durable jobs will be consumed"
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Failover hook: останавливает ОДНУ API-реплику, обслужившую тест, и оставляет её остановленной.
|
||||
# TARGET_INSTANCE_ID = X-Instance-Id = hostname контейнера = префикс его Docker ID.
|
||||
# БД, LB и volumes не затрагиваются. Восстановление: scripts/start-api.sh.
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]* | "") echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
target=""
|
||||
for id in $(service_ids api); do
|
||||
case "$id" in "$TARGET_INSTANCE_ID"*) target="$id" ;; esac
|
||||
done
|
||||
if [ -z "$target" ]; then
|
||||
echo "instance $TARGET_INSTANCE_ID is not a running api container of project $PROJECT" >&2
|
||||
exit 3
|
||||
fi
|
||||
# docker stop = SIGTERM -> graceful drain (~3 s) -> выход; SIGKILL только после 15 s.
|
||||
docker stop --time 15 "$target" >/dev/null
|
||||
echo "stopped api container $(short "$target") at $(date -u +%H:%M:%S)"
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
. "$(dirname "$0")/lib.sh"
|
||||
worker=$(service_ids worker)
|
||||
[ -n "$worker" ] || { echo "worker is already stopped"; exit 0; }
|
||||
docker stop --time 10 $worker >/dev/null
|
||||
echo "worker stopped; publisher and API remain available"
|
||||
@@ -0,0 +1,182 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run lab5 checks from Windows or Linux with only host Python and Docker.
|
||||
|
||||
Start the Compose stand first. Run ``python scripts/test_lab5.py --quick``
|
||||
for image/API checks, or omit --quick to include deliberate service failures.
|
||||
Commands run from the repository root regardless of the current directory.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
COMPOSE = ["docker", "compose", "--profile", "tools"]
|
||||
SERVICES = ("db", "valkey", "s3", "broker", "api", "lb", "publisher", "worker")
|
||||
|
||||
|
||||
class CheckFailed(Exception):
|
||||
"""A check or a Docker command returned a nonzero exit code."""
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def command_text(command: list[str]) -> str:
|
||||
return subprocess.list2cmdline(command) if os.name == "nt" else shlex.join(command)
|
||||
|
||||
|
||||
def run(command: list[str], evidence: str, *, append: bool = False,
|
||||
stdin_file: Path | None = None) -> str:
|
||||
"""Stream combined stdout/stderr to the terminal and a UTF-8 evidence log."""
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
with (EVIDENCE / evidence).open("a" if append else "w", encoding="utf-8", newline="\n") as log:
|
||||
def write(value: str) -> None:
|
||||
print(value, end="", flush=True)
|
||||
log.write(value)
|
||||
log.flush()
|
||||
|
||||
display = command_text(command)
|
||||
if stdin_file is not None:
|
||||
display += " < " + stdin_file.relative_to(ROOT).as_posix()
|
||||
write(f"\nUTC: {utc_now()}\nCommand: {display}\n")
|
||||
output: list[str] = []
|
||||
try:
|
||||
# Pass an actual file as stdin: no shell redirection, quoting or
|
||||
# PowerShell encoding conversions are involved.
|
||||
with stdin_file.open("rb") if stdin_file else open(os.devnull, "rb") as source:
|
||||
process = subprocess.Popen(
|
||||
command, cwd=ROOT, stdin=source, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT, text=True, encoding="utf-8", errors="replace",
|
||||
)
|
||||
try:
|
||||
assert process.stdout is not None
|
||||
for line in process.stdout:
|
||||
write(line)
|
||||
output.append(line)
|
||||
code = process.wait()
|
||||
except BaseException:
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait()
|
||||
raise
|
||||
finally:
|
||||
if process.stdout is not None:
|
||||
process.stdout.close()
|
||||
except (OSError, KeyboardInterrupt) as error:
|
||||
write(f"\nRESULT: ERROR ({type(error).__name__}) at {utc_now()}\n")
|
||||
raise
|
||||
write(f"\nRESULT: {'PASS' if code == 0 else 'FAIL'} (exit {code}) at {utc_now()}\n")
|
||||
if code:
|
||||
raise CheckFailed(f"{display}: exit {code}; see evidence/{evidence}")
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def tester(*command: str) -> list[str]:
|
||||
return [*COMPOSE, "run", "--rm", "-T", "--no-deps", "tester", *command]
|
||||
|
||||
|
||||
def snapshot(stage: str, *, append: bool = True) -> None:
|
||||
"""Record hook scope without dumping container environment or secrets."""
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
with (EVIDENCE / "container-events.txt").open("a" if append else "w", encoding="utf-8") as log:
|
||||
log.write(f"\n{stage} — {utc_now()}\n")
|
||||
identifiers = run([*COMPOSE, "ps", "-a", "-q"], "container-events.txt", append=True).split()
|
||||
if identifiers:
|
||||
run(["docker", "inspect", "--format",
|
||||
"{{.Name}} status={{.State.Status}} started={{.State.StartedAt}} finished={{.State.FinishedAt}}",
|
||||
*identifiers], "container-events.txt", append=True)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--quick", action="store_true",
|
||||
help="skip recovery, full-stack restart and API failover checks")
|
||||
args = parser.parse_args()
|
||||
for stream in (sys.stdout, sys.stderr):
|
||||
if hasattr(stream, "reconfigure"):
|
||||
stream.reconfigure(encoding="utf-8", errors="replace")
|
||||
|
||||
destructive_started = False
|
||||
complete = False
|
||||
exit_code = 0
|
||||
try:
|
||||
running = run([*COMPOSE, "ps", "--status", "running", "--services"], "runner-setup.txt")
|
||||
missing = set(SERVICES) - set(running.split())
|
||||
if missing:
|
||||
raise CheckFailed(
|
||||
"Start the Compose stand first: missing running services "
|
||||
+ ", ".join(sorted(missing))
|
||||
+ ". Run docker compose up -d --build --wait --wait-timeout 180."
|
||||
)
|
||||
run([*COMPOSE, "ps", "-a"], "stand.txt")
|
||||
run([*COMPOSE, "build", "tester"], "runner-setup.txt", append=True)
|
||||
run(tester("python3", "-u", "scripts/check_contract.py"), "check.txt")
|
||||
run(tester("python3", "-u", "tests/smoke.py"), "smoke.txt")
|
||||
run(tester("python3", "-u", "student-tests/image_validation_unit.py"), "image-validation.txt")
|
||||
run(tester("python3", "-u", "student-tests/lab5_images.py"), "images.txt")
|
||||
run([*COMPOSE, "exec", "-T", "worker", "python", "-u", "-"], "limits.txt",
|
||||
stdin_file=ROOT / "student-tests" / "lab5_limits.py")
|
||||
|
||||
if not args.quick:
|
||||
# Git clones on Linux may lack executable bits on newly added
|
||||
# hooks; the public test explicitly requires executable scripts.
|
||||
run(tester("chmod", "+x", "scripts/restart.sh", "scripts/stop-one.sh"),
|
||||
"runner-setup.txt", append=True)
|
||||
destructive_started = True
|
||||
run(tester("python3", "-u", "student-tests/lab5_recovery.py"), "recovery.txt")
|
||||
snapshot("Before public persistence", append=False)
|
||||
try:
|
||||
run(tester("python3", "-u", "tests/resilience.py", "--mode", "persistence",
|
||||
"--action-script", "scripts/restart.sh"), "persistence.txt")
|
||||
finally:
|
||||
snapshot("After public persistence")
|
||||
snapshot("Before public failover")
|
||||
try:
|
||||
run(tester("python3", "-u", "tests/resilience.py", "--mode", "failover",
|
||||
"--action-script", "scripts/stop-one.sh"), "failover.txt")
|
||||
finally:
|
||||
# Failover deliberately leaves an API replica stopped.
|
||||
# Restore it after success as well as after a failed check.
|
||||
try:
|
||||
snapshot("After public failover, before API restoration")
|
||||
finally:
|
||||
run(tester("sh", "scripts/start-api.sh"), "failover.txt", append=True)
|
||||
snapshot("After API restoration")
|
||||
run([*COMPOSE, "ps", "-a"], "stand.txt", append=True)
|
||||
complete = True
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted. Completed command logs remain in evidence/.", file=sys.stderr)
|
||||
exit_code = 130
|
||||
except (CheckFailed, OSError) as error:
|
||||
print(f"\nFAIL: {error}", file=sys.stderr)
|
||||
exit_code = 1
|
||||
finally:
|
||||
if destructive_started and not complete:
|
||||
try:
|
||||
# Preserve all volumes. Compose may also restart the
|
||||
# idempotent migration/bootstrap dependencies.
|
||||
run([*COMPOSE, "start", *SERVICES], "runner-setup.txt", append=True)
|
||||
except (CheckFailed, OSError, KeyboardInterrupt) as error:
|
||||
print(f"Service restoration failed: {error}", file=sys.stderr)
|
||||
exit_code = exit_code or 1
|
||||
|
||||
if exit_code == 0:
|
||||
mode = "quick" if args.quick else "full"
|
||||
print(f"\nPASS: lab5 {mode} checks. UTF-8 logs: {EVIDENCE}")
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,7 @@
|
||||
# Окружение для запуска проверок: Python 3.12 (stdlib-тесты), make, docker CLI (action-скрипты).
|
||||
FROM docker:28.5.1-cli AS docker-cli
|
||||
FROM python:3.12.11-alpine3.22
|
||||
RUN apk add --no-cache make
|
||||
RUN pip install --no-cache-dir Pillow==11.0.0
|
||||
COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker
|
||||
WORKDIR /work
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Offline regressions for malformed uploads; run with the app's Pillow installed.
|
||||
|
||||
python student-tests/image_validation_unit.py
|
||||
No HTTP, Docker, credentials or test data in the application are required.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
from pathlib import Path
|
||||
import struct
|
||||
import sys
|
||||
import unittest
|
||||
import zlib
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app.images import InvalidImage, PNG_SIGNATURE, TooManyPixels, validate_image
|
||||
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return (struct.pack(">I", len(data)) + kind + data +
|
||||
struct.pack(">I", zlib.crc32(kind + data)))
|
||||
|
||||
|
||||
def ihdr(width=1, height=1, depth=8, color=2, interlace=0) -> bytes:
|
||||
return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace))
|
||||
|
||||
|
||||
def png(raw=b"\x00\xff\x00\x00", *, header=None, before=b"", after=b"") -> bytes:
|
||||
return (PNG_SIGNATURE + (header or ihdr()) + before + chunk(b"IDAT", zlib.compress(raw)) +
|
||||
after + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
class ImageValidationTests(unittest.TestCase):
|
||||
def validate(self, data: bytes, mime="image/png"):
|
||||
return validate_image(io.BytesIO(data), mime, 40_000_000)
|
||||
|
||||
def assert_invalid(self, data: bytes, mime="image/png"):
|
||||
with self.assertRaises(InvalidImage):
|
||||
self.validate(data, mime)
|
||||
|
||||
def test_valid_png_and_trailing_padding(self):
|
||||
image = self.validate(png() + b"\x00" * 100)
|
||||
self.assertEqual((image.width, image.height), (1, 1))
|
||||
|
||||
def test_filter_values_and_row_boundaries(self):
|
||||
for filter_byte in range(5):
|
||||
self.validate(png(bytes([filter_byte, 0, 0, 0])))
|
||||
self.assert_invalid(png(b"\x05\x00\x00\x00"))
|
||||
# Second filter is after a row larger than the inflater's 64 KiB
|
||||
# output piece. It must be checked despite that boundary.
|
||||
self.assert_invalid(png(b"\x00" + bytes(90000) + b"\xff" + bytes(90000),
|
||||
header=ihdr(30000, 2)))
|
||||
|
||||
def test_critical_headers_and_chunk_order(self):
|
||||
# Wrong IHDR length is rejected before any payload read/allocation.
|
||||
self.assert_invalid(PNG_SIGNATURE + struct.pack(">I4s", 10_000_000, b"IHDR"))
|
||||
self.assert_invalid(png(before=ihdr()))
|
||||
self.assert_invalid(png(before=chunk(b"ABCD", b"unknown critical")))
|
||||
self.assert_invalid(png(before=chunk(b"abca", b"reserved lowercase")))
|
||||
self.assert_invalid(png()[:-12] + chunk(b"IEND", b"invalid"))
|
||||
self.assert_invalid(png(after=chunk(b"PLTE", b"\x00\x00\x00")))
|
||||
self.assert_invalid(png(before=chunk(b"PLTE", b"\x00\x00")))
|
||||
self.assert_invalid(png(before=chunk(b"PLTE", b"\x00\x00\x00") * 2))
|
||||
|
||||
def test_palette_shape_and_transparency(self):
|
||||
header = ihdr(depth=1, color=3)
|
||||
palette = chunk(b"PLTE", b"\xff\x00\x00")
|
||||
self.validate(png(b"\x00\x00", header=header, before=palette))
|
||||
self.assert_invalid(png(b"\x00\x00", header=header))
|
||||
self.assert_invalid(png(b"\x00\x00", header=header,
|
||||
before=chunk(b"PLTE", bytes(9))))
|
||||
self.assert_invalid(png(before=chunk(b"tRNS", b"\x00")))
|
||||
|
||||
def test_crc_compressed_tail_and_pixel_bomb(self):
|
||||
damaged = bytearray(png())
|
||||
damaged[-1] ^= 1
|
||||
self.assert_invalid(bytes(damaged))
|
||||
self.assert_invalid(PNG_SIGNATURE + ihdr() +
|
||||
chunk(b"IDAT", zlib.compress(b"\x00\xff\x00\x00") + b"junk") +
|
||||
chunk(b"IEND", b""))
|
||||
with self.assertRaises(TooManyPixels):
|
||||
self.validate(PNG_SIGNATURE + ihdr(10000, 10000))
|
||||
|
||||
def test_adam7_pass_filter_boundaries(self):
|
||||
width, height = 9, 7
|
||||
passes = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8),
|
||||
(2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
raw = bytearray()
|
||||
pass_starts = []
|
||||
for x, y, dx, dy in passes:
|
||||
w = (width - x + dx - 1) // dx if width > x else 0
|
||||
h = (height - y + dy - 1) // dy if height > y else 0
|
||||
if w and h:
|
||||
pass_starts.append(len(raw))
|
||||
for _ in range(h):
|
||||
raw.extend(b"\x00" + bytes(w * 4))
|
||||
header = ihdr(width, height, color=6, interlace=1)
|
||||
valid = png(bytes(raw), header=header)
|
||||
self.validate(valid)
|
||||
with Image.open(io.BytesIO(valid)) as decoded:
|
||||
decoded.load()
|
||||
self.assertEqual(decoded.size, (width, height))
|
||||
for position in pass_starts:
|
||||
corrupted = bytearray(raw)
|
||||
corrupted[position] = 5
|
||||
self.assert_invalid(png(bytes(corrupted), header=header))
|
||||
|
||||
def test_jpeg_and_truncated_progressive_above_old_bypass_limit(self):
|
||||
image = Image.new("RGB", (4100, 4000), "white")
|
||||
data = io.BytesIO()
|
||||
image.save(data, format="JPEG", progressive=True)
|
||||
image.close()
|
||||
encoded = data.getvalue()
|
||||
result = self.validate(encoded, "image/jpeg")
|
||||
self.assertEqual((result.width, result.height), (4100, 4000))
|
||||
# Keep SOF/header and final EOI, but truncate the first scan's
|
||||
# header. The former >16 MP branch only inspected EOI and accepted
|
||||
# this malformed file. Actual entropy decoding must reject it.
|
||||
scan = encoded.index(b"\xff\xda")
|
||||
corrupted = encoded[:scan + 4] + b"\xff\xd9"
|
||||
self.assert_invalid(corrupted, "image/jpeg")
|
||||
self.assert_invalid(encoded[:-2], "image/jpeg")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,158 @@
|
||||
"""Shared helpers for the student's explicit lab5 checks; never log tokens."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from uuid import UUID
|
||||
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
|
||||
def encoded(image: Image.Image, format: str = "PNG", **options) -> bytes:
|
||||
output = io.BytesIO()
|
||||
image.save(output, format=format, **options)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def upload(client, actor, chat, content, *, filename="sample.png", mime="image/png"):
|
||||
from urllib.parse import quote
|
||||
result = client.call(
|
||||
"POST", f"/api/v1/chats/{chat['id']}/attachments?filename={quote(filename, safe='')}",
|
||||
202, actor=actor, raw=content, headers={"Content-Type": mime}, schema="Attachment",
|
||||
)["json"]
|
||||
check(result["status"] == "queued", "upload must return its queued snapshot")
|
||||
check(result["variants"] == [], "queued snapshot must not expose partial variants")
|
||||
return result
|
||||
|
||||
|
||||
def metadata(client, actor, attachment_id):
|
||||
return client.call("GET", f"/api/v1/attachments/{attachment_id}", actor=actor,
|
||||
schema="Attachment")["json"]
|
||||
|
||||
|
||||
def wait_until(predicate, message, timeout=90, interval=0.3):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
value = predicate()
|
||||
if value:
|
||||
return value
|
||||
time.sleep(interval)
|
||||
raise Failure(message)
|
||||
|
||||
|
||||
class Stand:
|
||||
"""Control only containers bearing this lab's Compose project label."""
|
||||
|
||||
def __init__(self):
|
||||
self.project = os.environ.get("COMPOSE_PROJECT", "messenger-lab5")
|
||||
|
||||
@staticmethod
|
||||
def docker(*args, stdin=None, timeout=70):
|
||||
process = subprocess.run(["docker", *args], input=stdin, text=True,
|
||||
capture_output=True, timeout=timeout)
|
||||
# Docker command output can contain secrets: do not echo it on errors.
|
||||
if process.returncode:
|
||||
raise Failure(f"docker {args[0]} failed (exit {process.returncode})")
|
||||
return process.stdout.strip()
|
||||
|
||||
def ids(self, service, *, all=True):
|
||||
args = ["ps", "-q"]
|
||||
if all:
|
||||
args.append("-a")
|
||||
args += ["--filter", f"label=com.docker.compose.project={self.project}",
|
||||
"--filter", f"label=com.docker.compose.service={service}"]
|
||||
return self.docker(*args).split()
|
||||
|
||||
def one(self, service):
|
||||
ids = self.ids(service)
|
||||
check(len(ids) == 1, f"expected one {service} container")
|
||||
return ids[0]
|
||||
|
||||
def stop(self, *services):
|
||||
ids = [cid for service in services for cid in self.ids(service)]
|
||||
check(bool(ids), "containers to stop were not found")
|
||||
self.docker("stop", "--time", "3", *ids)
|
||||
|
||||
def start(self, *services):
|
||||
ids = [cid for service in services for cid in self.ids(service)]
|
||||
check(bool(ids), "containers to start were not found")
|
||||
self.docker("start", *ids)
|
||||
|
||||
def restart(self, *services):
|
||||
ids = [cid for service in services for cid in self.ids(service)]
|
||||
self.docker("restart", "--time", "3", *ids)
|
||||
|
||||
def sql(self, statement):
|
||||
return self.docker("exec", "-i", self.one("db"), "psql", "-U", "messenger",
|
||||
"-d", "messenger", "-v", "ON_ERROR_STOP=1", "-tA",
|
||||
stdin=statement)
|
||||
|
||||
def row(self, attachment_id):
|
||||
aid = str(UUID(attachment_id))
|
||||
value = self.sql(f"SELECT row_to_json(a) FROM attachments a WHERE id='{aid}';")
|
||||
check(bool(value), "attachment row disappeared")
|
||||
return json.loads(value)
|
||||
|
||||
def operator(self, *args):
|
||||
# Use a running API image so stopping worker/publisher is supported.
|
||||
apis = self.ids("api", all=False)
|
||||
check(bool(apis), "no API available for operator command")
|
||||
return self.docker("exec", apis[0], "python", "-m", "app.operator", *args)
|
||||
|
||||
def objects_exist(self, keys):
|
||||
code = """
|
||||
import asyncio, json, sys
|
||||
from app.storage import storage
|
||||
async def main():
|
||||
await storage.start()
|
||||
try:
|
||||
print(json.dumps([await storage.exists(key) for key in json.load(sys.stdin)]))
|
||||
finally:
|
||||
await storage.close()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
apis = self.ids("api", all=False)
|
||||
return json.loads(self.docker("exec", "-i", apis[0], "python", "-c", code,
|
||||
stdin=json.dumps(keys)))
|
||||
|
||||
def replace_object(self, key, content, mime="image/png"):
|
||||
"""Explicit fault-test operation; key must come from this test's DB row."""
|
||||
import base64
|
||||
code = """
|
||||
import asyncio, base64, io, json, sys
|
||||
from app.storage import storage
|
||||
async def main():
|
||||
key, encoded, mime = json.load(sys.stdin)
|
||||
content = base64.b64decode(encoded)
|
||||
await storage.start()
|
||||
try:
|
||||
await storage.put(key, io.BytesIO(content), len(content), mime)
|
||||
finally:
|
||||
await storage.close()
|
||||
asyncio.run(main())
|
||||
"""
|
||||
self.docker("exec", "-i", self.ids("api", all=False)[0], "python", "-c", code,
|
||||
stdin=json.dumps([key, base64.b64encode(content).decode(), mime]))
|
||||
|
||||
def ready(self):
|
||||
for service in ("db", "broker", "api", "lb"):
|
||||
for cid in self.ids(service):
|
||||
raw = self.docker("inspect", "--format", "{{json .State}}", cid)
|
||||
state = json.loads(raw)
|
||||
if not state.get("Running"):
|
||||
return False
|
||||
if "Health" in state and state["Health"]["Status"] != "healthy":
|
||||
return False
|
||||
return True
|
||||
|
||||
def wait_ready(self):
|
||||
wait_until(self.ready, "stand did not recover readiness", timeout=90)
|
||||
@@ -0,0 +1,145 @@
|
||||
"""Grade 4 image semantics, actual decoded bytes and upload rejection checks.
|
||||
|
||||
Run explicitly against the running lab5 stand:
|
||||
python student-tests/lab5_images.py --base-url http://localhost:8080
|
||||
Requires Pillow, also present in the application image.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from lab5_helpers import Client, Failure, check, encoded, upload
|
||||
|
||||
|
||||
def assert_variants(client, actor, att, source, expected_thumb, expected_optimized,
|
||||
*, white_corner=False, green_crop=False):
|
||||
ready = client.wait_ready(actor, att)
|
||||
check(ready["sha256"] == hashlib.sha256(source).hexdigest(), "original SHA-256 changed")
|
||||
check(ready["size_bytes"] == len(source), "original byte count changed")
|
||||
check(ready["error_code"] is None, "ready has error_code")
|
||||
by_name = {item["name"]: item for item in ready["variants"]}
|
||||
check(len(ready["variants"]) == 3 and set(by_name) == {"thumbnail", "crop", "optimized"},
|
||||
"ready must contain exactly the three promised variants")
|
||||
check(not any(k in json.dumps(ready) for k in ("object_key", "staging_key", "broker_url")),
|
||||
"storage internals leaked into metadata")
|
||||
original = client.call("GET", f"/api/v1/attachments/{att['id']}/content", actor=actor)
|
||||
check(original["bytes"] == source, "original was re-encoded or changed")
|
||||
check(original["headers"].get_content_type() == ready["content_type"],
|
||||
"original Content-Type changed")
|
||||
for name, expected_size, expected_format, mime in (
|
||||
("thumbnail", expected_thumb, "JPEG", "image/jpeg"),
|
||||
("crop", (128, 128), "JPEG", "image/jpeg"),
|
||||
("optimized", expected_optimized, "WEBP", "image/webp"),
|
||||
):
|
||||
variant = by_name[name]
|
||||
response = client.call("GET", f"/api/v1/attachments/{att['id']}/content?variant={name}",
|
||||
actor=actor)
|
||||
data = response["bytes"]
|
||||
check(response["headers"].get_content_type() == mime, f"{name}: wrong MIME")
|
||||
check(variant["content_type"] == mime, f"{name}: wrong metadata MIME")
|
||||
check(variant["size_bytes"] == len(data), f"{name}: wrong size_bytes")
|
||||
check(variant["sha256"] == hashlib.sha256(data).hexdigest(), f"{name}: wrong SHA-256")
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
image.load()
|
||||
check(image.format == expected_format, f"{name}: wrong actual format")
|
||||
check(image.size == expected_size, f"{name}: wrong actual dimensions {image.size}")
|
||||
check((variant["width"], variant["height"]) == expected_size,
|
||||
f"{name}: dimensions metadata differs from bytes")
|
||||
check(not image.getexif(), f"{name}: EXIF was retained")
|
||||
if white_corner and name == "thumbnail":
|
||||
check(min(image.convert("RGB").getpixel((0, 0))) > 245,
|
||||
"transparent JPEG background is not white")
|
||||
if green_crop and name == "crop":
|
||||
pixel = image.convert("RGB").getpixel((10, 64))
|
||||
check(pixel[1] > 180 and pixel[0] < 40 and pixel[2] < 40,
|
||||
"crop is not taken from the centered square")
|
||||
return ready
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default=None)
|
||||
args = parser.parse_args()
|
||||
client = Client(args.base_url, timeout=40)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
|
||||
image = Image.new("RGB", (400, 200), (0, 220, 0))
|
||||
image.paste((220, 0, 0), (0, 0, 100, 200))
|
||||
image.paste((0, 0, 220), (300, 0, 400, 200))
|
||||
source = encoded(image)
|
||||
att = upload(client, alice, chat, source)
|
||||
assert_variants(client, bob, att, source, (256, 128), (400, 200), green_crop=True)
|
||||
for suffix in ("", "/content", "/content?variant=thumbnail", "/content?variant=crop",
|
||||
"/content?variant=optimized"):
|
||||
client.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, actor=eve)
|
||||
print("PASS centered crop, three real variants, checksums, original bytes and ACL", flush=True)
|
||||
|
||||
for size, thumb, optimized in (
|
||||
((11, 7), (11, 7), (11, 7)),
|
||||
((513, 201), (256, 100), (513, 201)),
|
||||
((1701, 1003), (256, 150), (1280, 754)),
|
||||
):
|
||||
source = encoded(Image.new("RGB", size, (30, 80, 170)))
|
||||
att = upload(client, alice, chat, source)
|
||||
assert_variants(client, bob, att, source, thumb, optimized)
|
||||
print("PASS no upscaling and floor rounding for thumbnail/optimized", flush=True)
|
||||
|
||||
source = encoded(Image.new("RGBA", (320, 200), (200, 30, 30, 0)))
|
||||
att = upload(client, alice, chat, source)
|
||||
assert_variants(client, bob, att, source, (256, 160), (320, 200), white_corner=True)
|
||||
print("PASS transparent PNG composited onto white JPEG", flush=True)
|
||||
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
exif[270] = "student test metadata must be stripped"
|
||||
source = encoded(Image.new("RGB", (320, 200), (90, 20, 170)), "JPEG", exif=exif)
|
||||
att = upload(client, alice, chat, source, filename="rotated.jpg", mime="image/jpeg")
|
||||
assert_variants(client, bob, att, source, (160, 256), (200, 320))
|
||||
print("PASS EXIF orientation applied to variants and metadata removed", flush=True)
|
||||
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename=bad.png"
|
||||
for data, mime, expected in ((b"this is not a PNG", "image/png", 415),
|
||||
(source, "image/png", 415),
|
||||
(b"x" * (10485760 + 1), "image/png", 413)):
|
||||
client.call("POST", path, expected, actor=alice, raw=data, headers={"Content-Type": mime})
|
||||
def png_chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data))
|
||||
bad_filter = (b"\x89PNG\r\n\x1a\n"
|
||||
+ png_chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0))
|
||||
+ png_chunk(b"IDAT", zlib.compress(b"\x05\x00\x00\x00"))
|
||||
+ png_chunk(b"IEND", b""))
|
||||
client.call("POST", path, 415, actor=alice, raw=bad_filter,
|
||||
headers={"Content-Type": "image/png"})
|
||||
with Image.new("RGB", (4100, 4000), "blue") as large:
|
||||
progressive = encoded(large, "JPEG", progressive=True)
|
||||
scan = progressive.index(b"\xff\xda")
|
||||
client.call("POST", path, 415, actor=alice, raw=progressive[:scan + 4] + b"\xff\xd9",
|
||||
headers={"Content-Type": "image/jpeg"})
|
||||
print("PASS malformed PNG filter and truncated >16MP progressive JPEG rejected before 202", flush=True)
|
||||
ihdr = struct.pack(">IIBBBBB", 10001, 4000, 8, 2, 0, 0, 0)
|
||||
chunk = b"IHDR" + ihdr
|
||||
oversized = b"\x89PNG\r\n\x1a\n" + struct.pack(">I", len(ihdr)) + chunk + struct.pack(">I", zlib.crc32(chunk))
|
||||
client.call("POST", path, 413, actor=alice, raw=oversized,
|
||||
headers={"Content-Type": "image/png"})
|
||||
from urllib.parse import quote
|
||||
for bad_name in ("../x.png", "a\\b.png", "a\x00.png", "a\n.png"):
|
||||
client.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={quote(bad_name, safe='')}",
|
||||
400, actor=alice, raw=encoded(Image.new("RGB", (2, 2))),
|
||||
headers={"Content-Type": "image/png"})
|
||||
print("PASS invalid/MIME mismatch 415, >10MiB and >40M pixels 413, unsafe filenames 400", flush=True)
|
||||
print("PASS: lab5 grade4 image checks")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
print("FAIL:", str(exc), flush=True)
|
||||
raise SystemExit(1)
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Run inside the worker image; deliberately small limits, no stack changes.
|
||||
|
||||
PowerShell: Get-Content -Raw student-tests/lab5_limits.py | docker compose exec -T worker python
|
||||
Linux: docker compose exec -T worker python < student-tests/lab5_limits.py
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app import worker
|
||||
|
||||
|
||||
async def main():
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
directory = Path(temporary)
|
||||
source = directory / "large.png"
|
||||
with Image.new("RGB", (4000, 4000), "green") as image:
|
||||
image.save(source)
|
||||
old_memory = os.environ.get("WORKER_MEMORY_MB")
|
||||
old_timeout = worker.IMAGE_TIMEOUT
|
||||
try:
|
||||
os.environ["WORKER_MEMORY_MB"] = "48"
|
||||
try:
|
||||
await worker.generate(source, directory)
|
||||
raise AssertionError("decoder exceeded its deliberately small memory budget")
|
||||
except worker.PermanentFailure as error:
|
||||
assert str(error) == "image_resource_limit", str(error)
|
||||
print("PASS memory: real 16MP image decoder stopped at RLIMIT_AS=48MiB", flush=True)
|
||||
os.environ["WORKER_MEMORY_MB"] = "768"
|
||||
worker.IMAGE_TIMEOUT = 0.001
|
||||
started = time.monotonic()
|
||||
try:
|
||||
await worker.generate(source, directory)
|
||||
raise AssertionError("decoder exceeded wall timeout")
|
||||
except worker.PermanentFailure as error:
|
||||
assert str(error) == "image_timeout", str(error)
|
||||
assert time.monotonic() - started < 3, "timed-out child was not promptly reaped"
|
||||
print("PASS timeout: decoder killed and reaped; parent continues", flush=True)
|
||||
finally:
|
||||
worker.IMAGE_TIMEOUT = old_timeout
|
||||
if old_memory is None:
|
||||
os.environ.pop("WORKER_MEMORY_MB", None)
|
||||
else:
|
||||
os.environ["WORKER_MEMORY_MB"] = old_memory
|
||||
print("PASS: lab5 decoder resource limits", flush=True)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,218 @@
|
||||
"""Explicit grade4 failure checks; changes only this local lab stand.
|
||||
|
||||
Run after public smoke; services are restored in finally blocks:
|
||||
docker compose --profile tools run --rm tester python3 student-tests/lab5_recovery.py
|
||||
|
||||
Modes can be run separately: --mode persistence|outbox|objects|ack|replay|retry.
|
||||
Faults are one-shot DB records; API tokens are never printed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from uuid import UUID
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from lab5_helpers import Client, Failure, Stand, check, encoded, metadata, upload, wait_until
|
||||
|
||||
|
||||
def wait_status(client, actor, attachment_id, target, timeout=180):
|
||||
def poll():
|
||||
value = metadata(client, actor, attachment_id)
|
||||
if target != "failed":
|
||||
check(value["status"] != "failed", "attachment unexpectedly failed")
|
||||
return value if value["status"] == target else None
|
||||
return wait_until(poll, f"attachment did not reach {target}", timeout=timeout, interval=0.6)
|
||||
|
||||
|
||||
def assert_original(client, actor, att, content):
|
||||
result = client.call("GET", f"/api/v1/attachments/{att['id']}/content", actor=actor)
|
||||
check(result["bytes"] == content, "accepted original bytes were lost")
|
||||
check(hashlib.sha256(content).hexdigest() == att["sha256"], "original checksum changed")
|
||||
|
||||
|
||||
def fault_hits(stand, point):
|
||||
result = stand.sql(f"SELECT hits FROM fault_injections WHERE point='{point}';")
|
||||
return int(result or 0)
|
||||
|
||||
|
||||
def persistence(stand, client, alice, bob, chat, content):
|
||||
stand.stop("worker")
|
||||
try:
|
||||
att = upload(client, alice, chat, content)
|
||||
linked = client.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||||
check(linked["attachment_ids"] == [att["id"]], "queued attachment not linkable")
|
||||
client.call("GET", f"/api/v1/attachments/{att['id']}/content", 409, actor=bob)
|
||||
client.message(bob, chat, "Messages work while worker is stopped")
|
||||
# Ensure there really is a durable broker message before its restart.
|
||||
aid = str(UUID(att["id"]))
|
||||
wait_until(lambda: stand.sql(
|
||||
f"SELECT count(*) FROM outbox WHERE attachment_id='{aid}' AND published_at IS NOT NULL;"
|
||||
) != "0", "outbox did not publish before broker restart", timeout=30)
|
||||
before = {service: {cid: stand.docker("inspect", "-f", "{{.State.StartedAt}}", cid)
|
||||
for cid in stand.ids(service)} for service in ("api", "broker")}
|
||||
stand.restart("api", "broker")
|
||||
stand.wait_ready()
|
||||
for service, containers in before.items():
|
||||
for cid, started in containers.items():
|
||||
check(stand.docker("inspect", "-f", "{{.State.StartedAt}}", cid) != started,
|
||||
f"{service} did not actually restart")
|
||||
check(metadata(client, bob, att["id"])["status"] == "queued", "worker ran while stopped")
|
||||
stand.start("worker")
|
||||
ready = wait_status(client, bob, att["id"], "ready")
|
||||
assert_original(client, bob, ready, content)
|
||||
print(f"PASS persistence: attachment={att['id']} same session/ID after API+broker restart; "
|
||||
"volumes preserved; queued content409; messages201; ready", flush=True)
|
||||
finally:
|
||||
stand.start("worker", "broker", "api")
|
||||
|
||||
|
||||
def injected(stand, client, alice, bob, chat, content, point):
|
||||
service = "publisher" if point == "crash_after_commit_before_publish" else "worker"
|
||||
stand.stop(service)
|
||||
try:
|
||||
before = fault_hits(stand, point)
|
||||
stand.operator("fault", point)
|
||||
att = upload(client, alice, chat, content)
|
||||
aid = str(UUID(att["id"]))
|
||||
if service == "publisher":
|
||||
check(stand.sql(f"SELECT count(*) FROM outbox WHERE attachment_id='{aid}' "
|
||||
"AND published_at IS NULL;") != "0", "no durable unpublished outbox")
|
||||
check(metadata(client, bob, aid)["status"] == "queued", "outbox gap not established")
|
||||
stand.start(service)
|
||||
wait_until(lambda: fault_hits(stand, point) > before,
|
||||
f"fault {point} was not reached", timeout=60)
|
||||
if point == "crash_after_objects_before_commit":
|
||||
row = stand.row(aid)
|
||||
check(row["status"] == "processing", "objects crash did not leave processing")
|
||||
check(stand.objects_exist([row["object_key"]]) == [True],
|
||||
"no durable original exists at the injected crash boundary")
|
||||
check(row["lease_until"] is not None, "processing has no lease")
|
||||
# Restart policy normally does this; explicit start also works without one.
|
||||
stand.start(service)
|
||||
started = time.monotonic()
|
||||
ready = wait_status(client, bob, aid, "ready", timeout=200)
|
||||
assert_original(client, bob, ready, content)
|
||||
check(len(ready["variants"]) == 3, "recovery duplicated/lost variants")
|
||||
row = stand.row(aid)
|
||||
if point == "crash_after_objects_before_commit":
|
||||
check(row["attempts_run"] >= 2, "expired processing was not retried")
|
||||
print(f"PASS {point}: attachment={aid} hits={fault_hits(stand, point)-before} "
|
||||
f"recovered_seconds={time.monotonic()-started:.1f}; same original; three variants", flush=True)
|
||||
# No staging removal is allowed before ready, but it must eventually be cleaned.
|
||||
wait_until(lambda: stand.row(aid)["staging_deleted"],
|
||||
"ready staging was not cleaned", timeout=90)
|
||||
check(stand.objects_exist([row["staging_key"]]) == [False], "ready staging object remains")
|
||||
finally:
|
||||
stand.sql(f"UPDATE fault_injections SET remaining=0 WHERE point='{point}';")
|
||||
stand.start(service)
|
||||
|
||||
|
||||
def replay(stand, client, alice, bob, chat, content):
|
||||
stand.stop("worker")
|
||||
try:
|
||||
att = upload(client, alice, chat, content)
|
||||
aid = att["id"]
|
||||
row = stand.row(aid)
|
||||
original_job = row["job_id"]
|
||||
# Damage only this test's staged object. The checksum mismatch is permanent.
|
||||
stand.replace_object(row["staging_key"], b"deliberately damaged staged source")
|
||||
stand.start("worker")
|
||||
failed = wait_status(client, bob, aid, "failed", timeout=120)
|
||||
check(failed["error_code"] is not None, "failed has no safe error code")
|
||||
check(stand.row(aid)["attempts_run"] == 1, "permanent error was repeatedly retried")
|
||||
client.call("GET", f"/api/v1/attachments/{aid}/content", 409, actor=bob)
|
||||
import uuid
|
||||
client.call("POST", f"/api/v1/chats/{chat['id']}/messages", 409,
|
||||
actor=alice, body={"text": "", "attachment_ids": [aid]},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
registry = stand.operator("failed")
|
||||
check(aid in registry, "failed attachment not visible to operator")
|
||||
stand.replace_object(row["staging_key"], content)
|
||||
stand.operator("replay", aid, "--reason", "test source restored after simulated corruption")
|
||||
ready = wait_status(client, bob, aid, "ready")
|
||||
check(stand.row(aid)["job_id"] == original_job, "replay changed logical job ID")
|
||||
assert_original(client, bob, ready, content)
|
||||
check(stand.sql(f"SELECT count(*) FROM job_events WHERE attachment_id='{aid}' AND event='operator_replay';") != "0",
|
||||
"explicit replay was not audited")
|
||||
check(stand.sql(f"SELECT attempt FROM job_events WHERE attachment_id='{aid}' "
|
||||
"AND event='operator_replay' ORDER BY id DESC LIMIT 1;") ==
|
||||
str(stand.row(aid)["attempt"]), "replay audit refers to the previous attempt")
|
||||
snapshot = metadata(client, bob, aid)
|
||||
stand.operator("redeliver", aid)
|
||||
wait_until(lambda: stand.sql(f"SELECT count(*) FROM outbox WHERE attachment_id='{aid}' "
|
||||
"AND published_at IS NULL;") == "0",
|
||||
"ready duplicate was not published", timeout=30)
|
||||
time.sleep(2)
|
||||
check(metadata(client, bob, aid) == snapshot, "ready redelivery modified completed result")
|
||||
check(len(snapshot["variants"]) == 3, "ready redelivery duplicated variants")
|
||||
print(f"PASS failed/replay: attachment={aid}; permanent error fails once; failed registry; "
|
||||
"content/link409; audited replay preserves job_id; ready duplicate no-op", flush=True)
|
||||
finally:
|
||||
stand.start("worker")
|
||||
|
||||
|
||||
def retry(stand, client, alice, bob, chat, content):
|
||||
"""Temporary S3 failure must persist retries and eventually exhaust the budget."""
|
||||
stand.stop("worker")
|
||||
try:
|
||||
att = upload(client, alice, chat, content)
|
||||
aid = att["id"]
|
||||
stand.stop("s3")
|
||||
stand.start("worker")
|
||||
failed = wait_status(client, bob, aid, "failed", timeout=240)
|
||||
check(failed["error_code"] == "retry_exhausted", "transient errors did not exhaust retry budget")
|
||||
row = stand.row(aid)
|
||||
check(row["attempts_run"] > 1, "temporary outage was not retried")
|
||||
attempts = stand.sql(f"SELECT count(DISTINCT attempt) FROM outbox WHERE attachment_id='{aid}';")
|
||||
check(int(attempts) == row["attempts_run"], "retry attempt was not saved in outbox")
|
||||
stand.start("s3")
|
||||
# Wait storage health separately; Stand.ready covers API/broker/db.
|
||||
wait_until(lambda: stand.docker("inspect", "-f", "{{.State.Health.Status}}",
|
||||
stand.one("s3")) == "healthy", "S3 did not recover", timeout=90)
|
||||
stand.operator("replay", aid, "--reason", "S3 recovered after bounded retry exhaustion")
|
||||
ready = wait_status(client, bob, aid, "ready")
|
||||
assert_original(client, bob, ready, content)
|
||||
print(f"PASS transient S3 outage: attachment={aid}; attempts={row['attempts_run']}; "
|
||||
"durable retries, retry_exhausted, explicit replay after recovery", flush=True)
|
||||
finally:
|
||||
stand.start("s3", "worker")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--mode", default="all", choices=("all", "persistence", "outbox", "objects",
|
||||
"ack", "replay", "retry"))
|
||||
args = parser.parse_args()
|
||||
stand = Stand()
|
||||
stand.wait_ready()
|
||||
client = Client(args.base_url, timeout=40)
|
||||
alice, bob, _eve, chat = client.bootstrap()
|
||||
content = encoded(Image.new("RGB", (320, 200), (40, 100, 190)))
|
||||
scenarios = (
|
||||
("persistence", lambda: persistence(stand, client, alice, bob, chat, content)),
|
||||
("outbox", lambda: injected(stand, client, alice, bob, chat, content,
|
||||
"crash_after_commit_before_publish")),
|
||||
("objects", lambda: injected(stand, client, alice, bob, chat, content,
|
||||
"crash_after_objects_before_commit")),
|
||||
("ack", lambda: injected(stand, client, alice, bob, chat, content,
|
||||
"crash_after_commit_before_ack")),
|
||||
("replay", lambda: replay(stand, client, alice, bob, chat, content)),
|
||||
("retry", lambda: retry(stand, client, alice, bob, chat, content)),
|
||||
)
|
||||
for name, action in scenarios:
|
||||
if args.mode in ("all", name):
|
||||
action()
|
||||
print(f"PASS: lab5 grade4 recovery checks ({args.mode})", flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except (Failure, ValueError, OSError) as exc:
|
||||
print("FAIL:", str(exc), flush=True)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user