ЛР 5 - БВТ2301 - Ромашко - на 5 #26
+15
-16
@@ -1,25 +1,24 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
# Compose supplies internal PostgreSQL, Redis, broker and S3 addresses.
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
MAX_PENDING_ATTACHMENTS=64
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_MEMORY_MB=768
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
|
||||
# Choose credentials locally.
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=replace-with-local-password
|
||||
CACHE_TTL_SECONDS=30
|
||||
MAX_REQUEST_BYTES=65536
|
||||
MAX_INFLIGHT=32
|
||||
MINIO_ROOT_USER=local-admin
|
||||
MINIO_ROOT_PASSWORD=replace-with-admin-password
|
||||
S3_ACCESS_KEY_ID=messenger-api
|
||||
S3_SECRET_ACCESS_KEY=replace-with-api-password
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# Shell scripts are mounted into Linux containers, including on Windows hosts.
|
||||
*.sh text eol=lf
|
||||
@@ -1,18 +1,22 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Ромашко Владимир, БВТ2301.
|
||||
Целевая оценка (3/4/5): **5**.
|
||||
Трек 5, если заявлен: **1 — очередь под перегрузкой**.
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: перенесён мессенджер lab4; добавлены Redis Streams, outbox/dispatcher, отдельный worker, thumbnail/crop/WebP, retry с backoff/jitter, lease, failed-реестр/replay, cleanup и общий лимит очереди.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone: `cp .env.example .env`, задать локальные пароли, затем `docker compose --env-file .env up -d --build`. После готовности API: `python -X utf8 scripts/check_contract.py` и `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` (эквиваленты `make check`/`make test`).
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md: smoke, 5 тестов изображений, сбои/SIGKILL, retry/replay и timeout прошли. Трек 1: 18 замеров, 324 успешные загрузки; проверены backpressure и восстановление принятых ID. Подробности: [REPORT.md](../REPORT.md), [проверки](../evidence/defense-check.md), [эксперимент](../evidence/queue-results.md).
|
||||
|
||||
Ограничения / известные ошибки: хранилища по одному экземпляру; потеря volumes не покрывается. Два повтора нагрузки не дают универсального SLA; чистый clone отдельно не проверялся.
|
||||
|
||||
Источники и использование ИИ: задание и контракт репозитория; FastAPI, SQLAlchemy, Redis, boto3, Pillow, MinIO. Codex использовался для реализации, проверок и оформления.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] Выполнены эквивалентные Python-команды `make check` и `make test`; результаты приведены в отчёте.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+24
@@ -0,0 +1,24 @@
|
||||
FROM python:3.12.8-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY src ./src
|
||||
COPY migrations ./migrations
|
||||
|
||||
RUN useradd --create-home --shell /usr/sbin/nologin appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()"
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "-m", "src"]
|
||||
@@ -1,37 +1,65 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Ромашко Владимир, БВТ2301.
|
||||
- Целевая оценка; выбранный трек 5: **5, трек 1 — очередь под перегрузкой**.
|
||||
- Commit/PR: проверенная реализация — `ff6d7c1`
|
||||
- Источники, библиотеки и помощь ИИ: [задание](README.md), [контракт](contracts/README.md); FastAPI, SQLAlchemy, Redis, boto3, Pillow, MinIO. Codex использовался для реализации, тестов и оформления.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker с Compose, Python на хосте и свободный порт 8080. В контейнерах: Python 3.12.8, PostgreSQL 17.4, Redis 7.4.2, Nginx 1.28.0; MinIO закреплён digest в `compose.yaml`, зависимости — в `requirements.txt`.
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
# Задать свои POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD и S3_SECRET_ACCESS_KEY.
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
# Дождаться 200 {"status":"ok"}:
|
||||
curl http://127.0.0.1:8080/health/ready
|
||||
python -X utf8 scripts/check_contract.py
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
```
|
||||
|
||||
Последние две команды эквивалентны `make check` и `make test`. Миграции выполняет сервис `migrate` до запуска API/worker. Остановка с сохранением данных: `docker compose down`. Намеренное удаление учебных данных: `docker compose down -v`. `.env` не входит в Git.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Из lab4 перенесены API, две реплики за Nginx, PostgreSQL, Redis-сессии и private MinIO. Добавлены Redis Streams broker, dispatcher и worker. PostgreSQL — источник истины: после сохранения оригинала в staging вложение и job/outbox записываются одной транзакцией; затем возвращается `202 queued`.
|
||||
|
||||
Dispatcher публикует job в брокер с AOF/`appendfsync=always`. Worker сохраняет оригинал и варианты, затем атомарно фиксирует ready. Доставка at least once; состояние job, lease-token и постоянные ключи объектов защищают от дублей. Ответ на XADD подтверждает приём брокером, XACK — обработку потребителем.
|
||||
|
||||
Thumbnail — JPEG до 256×256, crop — JPEG 128×128, quality=85; optimized — WebP до 1280×1280, quality=80. Применяется EXIF orientation, прозрачность заменяется белым, EXIF/ICC удаляются; оригинал неизменен.
|
||||
|
||||
Настройки `.env.example` передаются через Compose: 2 обработчика, prefetch=1, 5 попыток с backoff/jitter, timeout 60 с, lease 75 с, 768 MiB на процесс. Файл ограничен 10 MiB и 40 млн пикселей. Общий лимит — 64 queued/processing; превышение даёт 429 с `Retry-After`. Cleanup удаляет ready-staging и просроченные failed/orphan-объекты, retention — сутки. Команды failed/replay и детали — в [infra/async.md](infra/async.md).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Ниже `pipeline` — `python -X utf8 student-tests/pipeline.py`, `edge_cases` — `python -X utf8 student-tests/edge_cases.py`.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke, 202/queued, доступ, оригинал | Команда smoke выше | PASS, thumbnail соответствует контракту | [Проверка](evidence/defense-check.md) |
|
||||
| Отдельные broker/worker и сохранность | `pipeline`: остановка worker, рестарт API/broker | Прежний ID становится ready | [Вывод](evidence/defense-pipeline.txt) |
|
||||
| Ограниченные попытки и дубли | `pipeline` | 5 попыток → failed; дубль ready — no-op | [Вывод](evidence/defense-pipeline.txt) |
|
||||
| Варианты, EXIF, alpha, размеры | Unittest из `infra/async.md`, `pipeline` | 5/5 OK; проверены варианты и checksum | [Проверка](evidence/defense-check.md) |
|
||||
| Лимиты и постоянные ошибки | `edge_cases`, unittest | Невалидные файлы отклонены, timeout завершает процесс | [Вывод](evidence/defense-edge-cases.txt) |
|
||||
| Outbox: crash до publish | `pipeline --outbox-only`: SIGKILL API | Задание восстановлено без нового upload | [Вывод](evidence/defense-outbox-crash.txt) |
|
||||
| Retry, failed-реестр и replay | `pipeline` | Replay с audit, ready повторять нельзя | [Вывод](evidence/defense-pipeline.txt) |
|
||||
| Crash после S3, lease и cleanup | `pipeline`, `edge_cases` | Нет лишних вариантов; активный staging сохранён | [Вывод](evidence/defense-pipeline.txt) |
|
||||
| Трек 1: нагрузка и backpressure | `python -X utf8 scripts/queue_experiment.py` | 18 замеров; при лимите 12 приняты 12/24; все принятые ID восстановлены | [Результаты](evidence/queue-results.md) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотеза: 2 обработчика ускорят обработку относительно 1; 4 дадут меньшую отдачу при большем RSS. Docker: 8 CPU, около 7.76 GiB RAM; worker — 2 CPU и 2 GiB. Профиль: concurrency 1/2/4, prefetch=1; JPEG 320×200, 1280×800, 2560×1600; по 18 файлов, 6 клиентов, 2 повтора, прогрев 6 файлами. Все 324 измеряемые загрузки успешны с первой попытки.
|
||||
|
||||
При concurrency=2 средний throughput — 0.53–0.62 файла/с, p95 до ready — 28.74–32.25 с, пиковый RSS — 357–423 MiB. При concurrency=4 — 0.52–0.58 файла/с и до 713 MiB. Оставлено 2 обработчика. Длительности, queue lag и замеры: [CSV](evidence/queue-load.csv), [JSON](evidence/queue-load.json), [методика](evidence/queue-method.md). p50/p99 и фактическая загрузка CPU отдельно не измерялись.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| SIGKILL API до publish | Outbox сохранит работу | Прежний ID стал ready | После запуска dispatcher/worker; потерь нет |
|
||||
| Crash после S3 / до ACK | Безопасный повтор | Нет лишних объектов; ready не обрабатывается заново | Незавершённая попытка — после lease 75 с и backoff; потерь нет |
|
||||
| Перегрузка, worker остановлен | Ограничение очереди | 12 ответов 202, 12 ответов 429; после рестартов все принятые ID готовы | Наблюдаемое опустошение 18.63 с; потерь нет |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Хранилища работают по одному экземпляру; потеря volumes не покрывается. Лимит очереди действует после приёма HTTP bytes. После истечения failed-retention нужен новый upload. Всего два повтора нагрузки; p95 по 18 файлам равен максимуму, RSS может повторно учитывать общие страницы. Новый чистый clone отдельно не разворачивался. Следующий шаг — более длинный эксперимент на разных изображениях и CPU-профилирование.
|
||||
|
||||
+228
@@ -0,0 +1,228 @@
|
||||
x-api: &api
|
||||
image: mtusi2026lab5-api:local
|
||||
dns_opt:
|
||||
- timeout:1
|
||||
- attempts:1
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
POSTGRES_HOST: postgres
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30}
|
||||
MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536}
|
||||
MAX_INFLIGHT: ${MAX_INFLIGHT:-32}
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
MAX_PENDING_ATTACHMENTS: ${MAX_PENDING_ATTACHMENTS:-64}
|
||||
BROKER_URL: redis://broker:6379/0
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_MEMORY_MB: ${JOB_MEMORY_MB:-768}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
networks: [backend]
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
stop_grace_period: 10s
|
||||
|
||||
services:
|
||||
broker:
|
||||
image: redis:7.4.2-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 128mb, --maxmemory-policy, noeviction]
|
||||
volumes:
|
||||
- broker_data:/data
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
|
||||
dispatcher:
|
||||
<<: *api
|
||||
command: [python, -m, src.jobs, dispatch]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
mem_limit: 128m
|
||||
|
||||
worker:
|
||||
<<: *api
|
||||
command: [python, -m, src.worker]
|
||||
init: true
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
cpus: 2.0
|
||||
mem_limit: 2g
|
||||
pids_limit: 64
|
||||
|
||||
api:
|
||||
<<: *api
|
||||
build: .
|
||||
deploy:
|
||||
replicas: 2
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=1).read()"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
start_period: 5s
|
||||
retries: 2
|
||||
|
||||
migrate:
|
||||
<<: *api
|
||||
command: [python, -m, src.database]
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: 'no'
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
s3:
|
||||
# RELEASE.2025-09-07T16-13-09Z, pinned by immutable image digest.
|
||||
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
command: [server, /data, --console-address, '127.0.0.1:9001']
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
MINIO_BROWSER: 'off'
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: [CMD, curl, -f, http://127.0.0.1:9000/minio/health/live]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
|
||||
s3-init:
|
||||
# The pinned server image also contains mc; no second image is needed.
|
||||
image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
|
||||
entrypoint: [sh, /bootstrap.sh]
|
||||
environment:
|
||||
MINIO_ROOT_USER: ${MINIO_ROOT_USER:?Set MINIO_ROOT_USER in .env}
|
||||
MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:?Set MINIO_ROOT_PASSWORD in .env}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
volumes:
|
||||
- ./infra/s3-bootstrap.sh:/bootstrap.sh:ro
|
||||
- ./infra/s3-policy.json:/policy.json:ro
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
networks: [backend]
|
||||
restart: 'no'
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
|
||||
cleanup:
|
||||
<<: *api
|
||||
command: [python, -m, src.cleanup]
|
||||
healthcheck:
|
||||
disable: true
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
cpus: 0.25
|
||||
mem_limit: 128m
|
||||
|
||||
nginx:
|
||||
image: nginx:1.28.0-alpine
|
||||
entrypoint: [sh, /etc/nginx/readiness.sh]
|
||||
dns_opt:
|
||||
- timeout:1
|
||||
- attempts:1
|
||||
ports:
|
||||
- '127.0.0.1:${APP_PORT:-8080}:8080'
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/readiness.sh:/etc/nginx/readiness.sh:ro
|
||||
networks: [edge, backend]
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'wget -q -O /dev/null http://127.0.0.1:8080/health/ready']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
cpus: 0.5
|
||||
mem_limit: 128m
|
||||
stop_grace_period: 10s
|
||||
|
||||
postgres:
|
||||
image: postgres:17.4-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 1.0
|
||||
mem_limit: 512m
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-bookworm
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
cpus: 0.5
|
||||
mem_limit: 256m
|
||||
|
||||
networks:
|
||||
edge:
|
||||
backend:
|
||||
internal: true
|
||||
volumes:
|
||||
broker_data:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
@@ -0,0 +1,37 @@
|
||||
# Проверка перед защитой — 6 ноября 2026
|
||||
|
||||
Техническая реализация покрывает уровни 3, 4 и трек 1 уровня 5.
|
||||
Результаты проверки включены в [REPORT.md](../REPORT.md) и шаблон PR.
|
||||
Оценку выставляет преподаватель после защиты.
|
||||
|
||||
| Требование | Проверка и результат |
|
||||
|---|---|
|
||||
| Публичный API, 202/queued, доступ, оригинал, thumbnail | `python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080` — PASS |
|
||||
| Структура и неизменность контрактов | `python -X utf8 scripts/check_contract.py` — PASS; контракт и публичные тесты не изменены |
|
||||
| Размеры, округление, EXIF, alpha, pixel limit | `docker compose run --rm --no-deps -v ./student-tests:/checks:ro worker python -m unittest discover -s /checks` — 5/5 OK |
|
||||
| Outbox, сохранность при остановках, lease, дубли, retry/replay, cleanup | `python -X utf8 student-tests/pipeline.py` — [вывод](defense-pipeline.txt) |
|
||||
| Постоянные ошибки, retention, идемпотентность сообщения, реальный timeout процесса | `python -X utf8 student-tests/edge_cases.py` — [вывод](defense-edge-cases.txt) |
|
||||
| Жёсткий сбой после COMMIT до publish | `python -X utf8 student-tests/pipeline.py --outbox-only` — API завершается через SIGKILL при остановленном dispatcher; [вывод](defense-outbox-crash.txt) |
|
||||
| Трек 1: три настройки, разные размеры, throughput/lag/p95/RSS | Проверена матрица 18 замеров / 324 файла и SHA-256 текущих исходников приложения/эксперимента; [результаты](queue-results.md), [методика](queue-method.md) |
|
||||
| Backpressure и восстановление | При лимите 12: 12 ответов 202, 12 ответов 429; после перезапусков все 12 ID готовы, исходники и варианты сохранены; новая загрузка принята |
|
||||
|
||||
Полный нагрузочный эксперимент при этой проверке не повторялся: сохранённые
|
||||
данные соответствуют текущим исходникам, матрица и результаты перепроверены.
|
||||
Новый чистый clone отдельно не разворачивался. Исправлен обнаруженный риск
|
||||
Windows-checkout: `.gitattributes` сохраняет LF для `.sh`; Git-фильтры проверены
|
||||
с `core.autocrlf=true`. Прикладной код и измеренные параметры не менялись.
|
||||
|
||||
## Что объяснить на защите
|
||||
|
||||
- До 202 сохранены staging и транзакция PostgreSQL с вложением и outbox-job.
|
||||
- Ответ Redis на XADD при AOF/appendfsync=always подтверждает запись в брокер;
|
||||
XACK подтверждает обработку потребителем. Это разные границы.
|
||||
- Доставка at least once: дубль возможен. Постоянный job_id, ключи вариантов,
|
||||
транзакционное состояние и lease-token делают результат идемпотентным.
|
||||
- Timeout ограничивает отдельный процесс, lease восстанавливает работу после
|
||||
его смерти; токен и блокировка job защищают фиксацию результата.
|
||||
- Для этого стенда concurrency=2 даёт меньший RSS, чем 4, без устойчивой потери
|
||||
throughput. Два повтора и p95 по 18 файлам не доказывают универсальный SLA.
|
||||
|
||||
Автор/группа, целевая оценка, трек, запуск, доказательства, ограничения
|
||||
и использование ИИ указаны в отчёте и шаблоне PR.
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: upload validation, active staging retention, permanent failure, message replay, expired staging
|
||||
PASS: real subprocess timeout, termination and successful retry
|
||||
@@ -0,0 +1 @@
|
||||
PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart
|
||||
@@ -0,0 +1,5 @@
|
||||
PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart
|
||||
PASS: crash after_objects, redelivery and stable variants
|
||||
PASS: crash after_commit, redelivery and stable variants
|
||||
PASS: bounded retries, failed registry, explicit audited replay, ready replay rejected
|
||||
PASS: staging cleanup preserves ready objects
|
||||
@@ -0,0 +1,19 @@
|
||||
repeat,concurrency,prefetch_per_consumer,width,height,bytes,requests,clients,elapsed_s,throughput_per_s,ready_p95_s,queue_wait_p95_s,queue_lag_max_jobs,queue_oldest_max_s,processing_max,rss_peak_mib
|
||||
1,1,1,320,200,49354,18,6,45.69717665200005,0.39389742033903496,45.332322,43.354063,18,43.352613,1,220.434
|
||||
1,1,1,1280,800,774325,18,6,46.36494216900064,0.38822436000005855,45.690003,43.478215,17,43.483165,1,220.574
|
||||
1,1,1,2560,1600,3095842,18,6,50.051750227999946,0.3596277836040675,48.61938,46.288583,17,46.243644,1,254.23
|
||||
1,2,1,320,200,49354,18,6,24.970797072000096,0.7208420279136186,24.151489,21.722265,16,21.622827,2,357.164
|
||||
1,2,1,1280,800,774325,18,6,27.756625333000557,0.6484938202699787,26.906551,24.179169,16,24.151913,2,357.484
|
||||
1,2,1,2560,1600,3095842,18,6,30.89121619800062,0.5826899104466149,29.276858,26.462352,16,26.398178,2,398.773
|
||||
1,4,1,320,200,49354,18,6,30.706496757999957,0.5861951671615051,30.198425,27.003696,16,27.005221,4,624.723
|
||||
1,4,1,1280,800,774325,18,6,32.03026625700022,0.56196847867495,30.771213,27.799776,14,27.734374,4,629.391
|
||||
1,4,1,2560,1600,3095842,18,6,33.556769314999656,0.5364044384318641,31.874479,28.928972,14,28.955958,4,700.137
|
||||
2,4,1,2560,1600,3095842,18,6,36.27555272599966,0.4962019500008588,34.628852,31.143277,14,31.074291,4,712.543
|
||||
2,4,1,1280,800,774325,18,6,32.433771702999366,0.5549770826787752,31.417533,28.421323,14,28.33212,4,627.523
|
||||
2,4,1,320,200,49354,18,6,31.647972769999797,0.568756808874116,30.943396,27.535648,16,27.47535,4,565.867
|
||||
2,2,1,2560,1600,3095842,18,6,36.96276580199992,0.4869765454355173,35.219853,31.503015,16,31.49645,2,423.055
|
||||
2,2,1,1280,800,774325,18,6,36.61566027800018,0.4915929376484563,35.487861,31.857229,16,31.797539,2,357.262
|
||||
2,2,1,320,200,49354,18,6,34.0682360790006,0.5283513933113508,33.333839,28.783239,16,28.790205,2,356.77
|
||||
2,1,1,2560,1600,3095842,18,6,61.558325450000666,0.2924056148119248,59.328557,56.653392,17,56.578126,1,254.727
|
||||
2,1,1,1280,800,774325,18,6,56.564085449999766,0.3182231243871384,55.765589,52.567299,17,52.510874,1,221.125
|
||||
2,1,1,320,200,49354,18,6,52.09092416000021,0.3455496382577507,51.537452,49.49779,18,49.4865,1,221.094
|
||||
|
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,95 @@
|
||||
# Трек 1: очередь под перегрузкой
|
||||
|
||||
## Гипотеза
|
||||
|
||||
При фиксированном лимите worker в 2 CPU увеличение concurrency с 1 до 2
|
||||
повысит throughput и сократит ожидание. Увеличение до 4 даст меньший выигрыш
|
||||
и увеличит RSS. Для крупных изображений сильнее проявится ограничение CPU;
|
||||
для маленьких заметнее накладные расходы запуска отдельного процесса.
|
||||
Общий лимит незавершённых вложений должен удерживать очередь в заданных
|
||||
границах при остановленном worker, не теряя уже подтверждённые загрузки.
|
||||
|
||||
## Воспроизведение
|
||||
|
||||
Нужны Docker Compose, Python на хосте, локальный `.env` и свободный порт 8080.
|
||||
Эксперимент запускается без другой нагрузки, после завершения текущих задач.
|
||||
Он создаёт только синтетические данные и не удаляет volumes.
|
||||
|
||||
```sh
|
||||
docker compose up -d --build
|
||||
python -X utf8 scripts/queue_experiment.py
|
||||
```
|
||||
|
||||
По умолчанию выполняются два повтора, в каждом по 18 загрузок для каждой
|
||||
комбинации. Настройки `(concurrency, prefetch на потребителя)`:
|
||||
`(1, 1)`, `(2, 1)`, `(4, 1)`. Prefetch оставлен равным 1: каждый потребитель
|
||||
читает одно сообщение и подтверждает его перед чтением следующего.
|
||||
Меняется только concurrency, чтобы сравнение не смешивало два фактора.
|
||||
|
||||
Три JPEG: 320×200, 1280×800, 2560×1600. RGB-шум генерируется
|
||||
`random.Random(202605)`, JPEG quality=85; файлы генерируются до измерения.
|
||||
Размеры и SHA-256 входят в результаты. Это труднее для кодека, чем однотонная
|
||||
картинка, но не репрезентативная коллекция фотографий.
|
||||
|
||||
Шесть HTTP-клиентов одновременно загружают файлы через nginx на обе API-реплики.
|
||||
Для обычных замеров лимит очереди равен 64. Перед каждой конфигурацией —
|
||||
прогрев шестью маленькими изображениями. Во втором повторе порядок
|
||||
concurrency и размеров обращается, чтобы уменьшить влияние порядка/прогрева.
|
||||
Worker сохраняет одинаковые лимиты: 2 CPU, 2 GiB контейнеру, 768 MiB процессу.
|
||||
Все сервисы, данные и настройки обработки изображений одинаковы.
|
||||
|
||||
Скрипт временно переопределяет настройки через отдельный Compose-файл;
|
||||
в `finally` возвращает настройки базового `compose.yaml`/`.env` и запускает cleanup.
|
||||
Не меняет `.env`, отчёт, контракт или публичные тесты.
|
||||
|
||||
## Метрики
|
||||
|
||||
- Throughput: число готовых файлов / время от начала первой отправки до
|
||||
наблюдения готовности всего пакета. Включает HTTP upload и ожидание очереди.
|
||||
- p95 до ready: `attachments.updated_at - attachments.created_at`, от
|
||||
устойчивой записи метаданных до публикации готовности; передача bytes
|
||||
до этой записи сюда не входит. Nearest-rank p95; при 18 файлах это максимум.
|
||||
- p95 ожидания: `jobs.first_started_at - jobs.created_at`, включая outbox
|
||||
и ожидание worker. Время первого захвата сохраняется в БД.
|
||||
- Queue lag: максимум числа queued и максимальный возраст старейшей queued
|
||||
записи; processing учитывается отдельно. Опрос состояния каждые ~100 ms.
|
||||
- RSS: максимум суммы `VmRSS` процессов внутри контейнера worker из `/proc`.
|
||||
Процесс измерения исключён. Это RSS, а не `docker stats`/page cache;
|
||||
общие страницы разных процессов могут учитываться повторно.
|
||||
|
||||
Драйвер запускается через `docker compose exec` внутри worker и не требует
|
||||
дополнительных библиотек на хосте. Его HTTP/SQL-опросы делят CPU-квоту с worker;
|
||||
это одинаковая, но не нулевая погрешность для всех конфигураций. Между отсчётами
|
||||
можно пропустить краткий пик RSS. Для точных эксплуатационных SLA этих данных мало.
|
||||
|
||||
## Перегрузка и восстановление
|
||||
|
||||
Worker и cleanup останавливаются; лимит уменьшается до 12. Двенадцать
|
||||
параллельных клиентов отправляют 24 файла. Проверяются ровно 12 ответов 202
|
||||
и 12 ответов 429 с `Retry-After`, отсутствие записей/объектов для отказов,
|
||||
доступность health и отправки сообщений, content=409 для queued.
|
||||
Проверка охватывает обе API-реплики.
|
||||
|
||||
API и broker перезапускаются при остановленном worker. После запуска worker
|
||||
все первоначально принятые ID должны стать ready с тремя вариантами и исходным
|
||||
SHA-256. Новая загрузка после опустошения очереди должна снова получить 202
|
||||
и завершиться. Это проверяет восстановление после перегрузки, а не только
|
||||
ограничение числа одновременно исполняемых HTTP-запросов.
|
||||
|
||||
Admission использует PostgreSQL advisory lock до COMMIT, поэтому две реплики
|
||||
не могут одновременно пересчитать свободные места и превысить лимит.
|
||||
Проверка выполняется до S3 PUT, но после приёма/валидации HTTP bytes:
|
||||
защищены объём принятой работы и staging, а не входящий сетевой трафик.
|
||||
При максимуме 10 MiB на файл активный staging ограничен примерно
|
||||
`MAX_PENDING_ATTACHMENTS × 10 MiB`; старые ready/failed объекты живут по
|
||||
правилам cleanup и в этот лимит не входят. Операторский replay также
|
||||
проверяет доступную ёмкость. При отказе клиент самостоятельно решает,
|
||||
когда повторить upload; прозрачных повторов POST нет.
|
||||
|
||||
## Артефакты
|
||||
|
||||
`queue-load.json` содержит профиль, время запуска UTC, commit базы,
|
||||
SHA-256 исходников эксперимента/приложения, ID Docker-образа, ресурсы Docker,
|
||||
все замеры, временные ряды и проверки восстановления.
|
||||
`queue-load.csv` — компактная таблица замеров. `queue-results.md` — разбор
|
||||
полученных чисел и ограничений. Секреты, session tokens и пароли не сохраняются.
|
||||
@@ -0,0 +1,37 @@
|
||||
# Результаты трека 1
|
||||
|
||||
Дата UTC: 2026-11-06T16:08:28.320053+00:00. База: `482e7ff477c8cdd9c7e6ceae7833ba19f61d9d98`.
|
||||
|
||||
Методика: [queue-method.md](queue-method.md). Исходные измерения: [JSON](queue-load.json), [CSV](queue-load.csv).
|
||||
|
||||
18 замеров: 3 настройки concurrency, 3 размера, 2 повтора; 324 измеряемые загрузки. Prefetch на потребителя = 1. Все загрузки получили 202 и завершились с первой попытки.
|
||||
|
||||
Throughput и p95 усреднены по двум прогонам; lag и RSS — максимумы двух прогонов. В скобках указан диапазон throughput.
|
||||
|
||||
| Размер | Concurrency | Файлов/с (диапазон) | p95 ready, с | p95 ожидания, с | Max queued | Peak RSS, MiB |
|
||||
|---|---:|---:|---:|---:|---:|---:|
|
||||
| 320x200 | 1 | 0.37 (0.35–0.39) | 48.43 | 46.43 | 18 | 221.1 |
|
||||
| 320x200 | 2 | 0.62 (0.53–0.72) | 28.74 | 25.25 | 16 | 357.2 |
|
||||
| 320x200 | 4 | 0.58 (0.57–0.59) | 30.57 | 27.27 | 16 | 624.7 |
|
||||
| 1280x800 | 1 | 0.35 (0.32–0.39) | 50.73 | 48.02 | 17 | 221.1 |
|
||||
| 1280x800 | 2 | 0.57 (0.49–0.65) | 31.20 | 28.02 | 16 | 357.5 |
|
||||
| 1280x800 | 4 | 0.56 (0.55–0.56) | 31.09 | 28.11 | 14 | 629.4 |
|
||||
| 2560x1600 | 1 | 0.33 (0.29–0.36) | 53.97 | 51.47 | 17 | 254.7 |
|
||||
| 2560x1600 | 2 | 0.53 (0.49–0.58) | 32.25 | 28.98 | 16 | 423.1 |
|
||||
| 2560x1600 | 4 | 0.52 (0.50–0.54) | 33.25 | 30.04 | 14 | 712.5 |
|
||||
|
||||
## Перегрузка
|
||||
|
||||
При остановленном worker и лимите 12 обе API-реплики приняли ровно 12 из 24 загрузок; 12 получили 429 с Retry-After. Отказанные загрузки не создали метаданных или S3-объектов. Health и отправка сообщений оставались доступны.
|
||||
|
||||
После рестарта API/broker и запуска worker все 12 первоначальных ID стали ready. Проверены SHA-256 оригиналов и наличие трёх вариантов. Наблюдаемое время опустошения: 18.63 с (от запуска проверяющего процесса после старта worker). Новая загрузка после восстановления получила 202 и тоже завершилась.
|
||||
|
||||
## Вывод и ограничения
|
||||
|
||||
Переход с 1 на 2 обработчика повышает средний throughput и сокращает ожидание на всех трёх размерах. Четыре обработчика не дают устойчивого выигрыша над двумя, но увеличивают RSS примерно до 626–713 MiB. Для этого стенда оставлено concurrency=2: это разумный компромисс задержки и памяти.
|
||||
|
||||
Гипотеза о пользе двух обработчиков подтверждается. Увеличение до четырёх имеет убывающую отдачу; на крупных файлах диапазоны throughput перекрываются. Результат совместим с ограничением worker в 2 CPU и накладными расходами запуска отдельного процесса на файл, но без CPU-профилирования разделить эти причины нельзя.
|
||||
|
||||
Docker: 8 CPU, примерно 7.76 GiB RAM; worker ограничен 2 CPU и 2 GiB. Всего два повтора, заметен разброс, доверительные интервалы не оценивались. Nearest-rank p95 для пакета из 18 файлов фактически равен максимуму. Это локальное сравнение, не универсальный SLA и не доказательство оптимальности concurrency=2 на любом оборудовании. Драйвер делит CPU-квоту с worker; RSS суммирует процессы и может повторно учитывать общие страницы. Полные ограничения и определения метрик приведены в методике.
|
||||
|
||||
Backpressure доказан для принятой незавершённой работы: advisory lock не позволил двум репликам превысить общий лимит. Он не ограничивает уже переданные HTTP bytes и общий архив готовых/failed вложений. После освобождения очереди приём автоматически возобновился.
|
||||
@@ -0,0 +1,74 @@
|
||||
# Локальный pipeline lab5
|
||||
|
||||
Запуск: скопировать `.env.example` в `.env`, задать локальные пароли,
|
||||
выполнить `docker compose up -d --build`. API: `http://localhost:8080`.
|
||||
Миграции запускаются сервисом `migrate` перед API/worker.
|
||||
|
||||
Upload проверяет формат, bytes и пиксели, сохраняет неизменный оригинал
|
||||
в приватный S3 `staging/`, затем одной транзакцией записывает вложение и job.
|
||||
Ответ 202 содержит queued. Изменение размера выполняется только worker.
|
||||
Job в PostgreSQL одновременно служит outbox: dispatcher отправляет его
|
||||
в отдельный Redis Streams, после ответа XADD отмечает публикацию.
|
||||
Redis использует volume, AOF и `appendfsync always`, eviction отключён.
|
||||
Сбой до/после XADD даёт повторную доставку, а не потерю принятого задания.
|
||||
Readiness API зависит от PostgreSQL, сессионного Redis и S3;
|
||||
недоступность broker/worker не мешает сохранять работу в outbox.
|
||||
При достижении `MAX_PENDING_ATTACHMENTS` (64 queued/processing суммарно
|
||||
по всем репликам) upload получает 429 с `Retry-After`, без нового S3-объекта.
|
||||
Лимит также проверяется при операторском replay. Описание и воспроизводимый
|
||||
эксперимент перегрузки: [evidence/queue-method.md](../evidence/queue-method.md).
|
||||
|
||||
Worker: два потребителя, у каждого не больше одной выполняемой задачи.
|
||||
Каждая попытка работает в отдельном процессе: timeout 60 секунд,
|
||||
лимит адресного пространства 768 MiB, контейнер 2 GiB.
|
||||
Настройки: `WORKER_CONCURRENCY` (1–4), `JOB_TIMEOUT_SECONDS`,
|
||||
`JOB_MEMORY_MB`, `JOB_MAX_ATTEMPTS` (по умолчанию 5).
|
||||
При повышении concurrency нужно соответственно увеличить память контейнера.
|
||||
Lease равен timeout + 15 секунд. Dispatcher возвращает просроченный
|
||||
processing в очередь. Токен lease и блокировка job защищают публикацию
|
||||
от устаревшей попытки. ACK выполняется после устойчивого результата
|
||||
либо сохранённого решения о retry/failed. Pending читаются через XAUTOCLAIM.
|
||||
|
||||
Оригинал публикуется в `originals/`, варианты — по стабильным ключам
|
||||
`variants/<attachment_id>/v1/<variant>`. Готовность и метаданные трёх
|
||||
вариантов фиксируются одной транзакцией после S3. Дубль ready — no-op.
|
||||
JPEG thumbnail: вписывание в 256×256, округление вниз, без увеличения.
|
||||
JPEG crop: центрированный квадрат 128×128. JPEG quality=85.
|
||||
WebP optimized: вписывание в 1280×1280 без увеличения, quality=80.
|
||||
Применяется EXIF orientation, прозрачность — на белом; EXIF/ICC удаляются.
|
||||
Оригинал остаётся побайтно неизменным.
|
||||
|
||||
Временные ошибки получают backoff `min(60, 2^failures)` секунд с jitter
|
||||
0.5–1.5. Повреждённое изображение/checksum — `invalid_image`, нехватка
|
||||
памяти — `resource_limit`, без retry. Исчерпание попыток — `retry_exhausted`.
|
||||
Последняя причина ошибки и время следующей попытки сохраняются в PostgreSQL.
|
||||
Реестр failed и replay доступны только локальному оператору через Compose:
|
||||
|
||||
```sh
|
||||
docker compose exec api python -m src.jobs failed
|
||||
docker compose exec api python -m src.jobs replay JOB_UUID --reason "причина повтора"
|
||||
```
|
||||
|
||||
Replay допускается только для failed с доступным staging, сохраняет audit,
|
||||
job_id и общий номер попытки, выдаёт новый бюджет retry. Ready повторять нельзя.
|
||||
Cleanup запускается раз в минуту: staging ready удаляется после публикации;
|
||||
failed хранится `STAGING_RETENTION_SECONDS` (86400) после ошибки.
|
||||
Частично записанные конечные объекты failed очищаются после того же срока.
|
||||
Бесхозные объекты удаляются после того же retention. Queued/processing
|
||||
не удаляются независимо от возраста. После удаления failed staging нужен
|
||||
новый upload. Реестр ошибок остаётся в PostgreSQL.
|
||||
|
||||
Проверки (из корня проекта):
|
||||
|
||||
```sh
|
||||
python -X utf8 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
python -X utf8 student-tests/pipeline.py
|
||||
python -X utf8 student-tests/edge_cases.py
|
||||
docker compose run --rm --no-deps -v ./student-tests:/checks:ro worker python -m unittest discover -s /checks
|
||||
```
|
||||
|
||||
Pipeline-тест останавливает/возвращает сервисы только этого Compose-проекта,
|
||||
создаёт синтетические данные, проверяет разрыв COMMIT/publish, рестарт broker,
|
||||
crash после S3 и после COMMIT до ack, lease, дубли, retry/replay и cleanup.
|
||||
Точки `JOB_FAULT=after_objects|after_commit` используются только тестом,
|
||||
в обычной конфигурации выключены. Контракты и публичные тесты не изменены.
|
||||
@@ -0,0 +1,38 @@
|
||||
worker_processes 1;
|
||||
worker_shutdown_timeout 8s;
|
||||
events { worker_connections 1024; }
|
||||
http {
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
include /etc/nginx/ready-backends.conf;
|
||||
}
|
||||
log_format upstream '$request_method $uri $status upstream=$upstream_addr '
|
||||
'upstream_status=$upstream_status time=$request_time';
|
||||
access_log /var/log/nginx/access.log upstream;
|
||||
server {
|
||||
listen 8080;
|
||||
# API enforces the byte limit and returns the contracted JSON error.
|
||||
client_max_body_size 0;
|
||||
client_body_timeout 5s;
|
||||
client_header_timeout 5s;
|
||||
keepalive_timeout 5s;
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_request_buffering off;
|
||||
proxy_buffering off;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header Connection close;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 35s;
|
||||
proxy_read_timeout 120s;
|
||||
# A 503 is an API response (for example, S3 is unavailable), so
|
||||
# preserve it for the client instead of marking every replica down.
|
||||
proxy_next_upstream error timeout http_502 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 8s;
|
||||
# Never transparently retry a POST already sent to an API.
|
||||
# A client retries messages using the original Idempotency-Key.
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
# Only ready API addresses enter the upstream, including at nginx startup.
|
||||
update_backends() {
|
||||
next=/etc/nginx/ready-backends.conf.next
|
||||
: > "$next"
|
||||
for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do
|
||||
if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then
|
||||
echo "server $address:8080;" >> "$next"
|
||||
fi
|
||||
done
|
||||
# If every running API temporarily loses a shared dependency (for example
|
||||
# S3), retain the last addresses. The API can then return the contracted
|
||||
# 503 instead of Nginx replacing it with a 502. At initial startup there
|
||||
# is no previous healthy set, so use an inert placeholder.
|
||||
if [ ! -s "$next" ]; then
|
||||
if [ -s /etc/nginx/ready-backends.conf ] && ! grep -q '127.0.0.1:9 down' /etc/nginx/ready-backends.conf; then
|
||||
rm -f "$next"
|
||||
return
|
||||
fi
|
||||
echo 'server 127.0.0.1:9 down;' > "$next"
|
||||
fi
|
||||
if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then
|
||||
mv "$next" /etc/nginx/ready-backends.conf
|
||||
if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then
|
||||
nginx -s reload
|
||||
fi
|
||||
else
|
||||
rm -f "$next"
|
||||
fi
|
||||
}
|
||||
|
||||
update_backends
|
||||
(
|
||||
while sleep 2; do
|
||||
update_backends reload
|
||||
done
|
||||
) &
|
||||
exec nginx -g 'daemon off;'
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
test "$MINIO_ROOT_USER" != "$S3_ACCESS_KEY_ID"
|
||||
mc alias set local http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null
|
||||
mc mb --ignore-existing local/messenger-attachments
|
||||
mc anonymous set none local/messenger-attachments
|
||||
mc admin user add local "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null
|
||||
mc admin policy info local attachments >/dev/null 2>&1 || mc admin policy create local attachments /policy.json
|
||||
mc admin policy attach local attachments --user "$S3_ACCESS_KEY_ID"
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/originals/*",
|
||||
"arn:aws:s3:::messenger-attachments/staging/*",
|
||||
"arn:aws:s3:::messenger-attachments/variants/*"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
CREATE INDEX chats_order_idx ON chats (created_at, id);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0;
|
||||
@@ -0,0 +1,9 @@
|
||||
-- Additive migration: no rewrite of existing lab2 tables or messages.
|
||||
CREATE TABLE message_idempotency (
|
||||
actor_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (actor_id, chat_id, key)
|
||||
);
|
||||
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id);
|
||||
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CHECK (char_length(text) <= 4000);
|
||||
ALTER TABLE messages ADD CHECK (length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0);
|
||||
ALTER TABLE messages ADD CHECK (cardinality(attachment_ids) <= 10);
|
||||
@@ -0,0 +1,23 @@
|
||||
ALTER TABLE attachments
|
||||
ADD COLUMN updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
ADD COLUMN status text NOT NULL DEFAULT 'ready'
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
ADD COLUMN staging_key text,
|
||||
ADD COLUMN variants jsonb NOT NULL DEFAULT '[]',
|
||||
ADD COLUMN error_code text;
|
||||
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
pipeline_version bigint NOT NULL DEFAULT 1 CHECK (pipeline_version = 1),
|
||||
attempt bigint NOT NULL DEFAULT 0,
|
||||
failures bigint NOT NULL DEFAULT 0,
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
lease_until timestamptz,
|
||||
lease_token uuid,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
replay_audit jsonb NOT NULL DEFAULT '[]'
|
||||
);
|
||||
CREATE INDEX attachment_jobs_due_idx ON attachment_jobs(available_at);
|
||||
@@ -0,0 +1 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN last_error text;
|
||||
@@ -0,0 +1,3 @@
|
||||
ALTER TABLE attachment_jobs ADD COLUMN first_started_at timestamptz;
|
||||
CREATE INDEX attachments_pending_idx ON attachments(status)
|
||||
WHERE status IN ('queued', 'processing');
|
||||
@@ -0,0 +1,19 @@
|
||||
fastapi==0.115.6
|
||||
uvicorn==0.34.0
|
||||
pydantic==2.13.5
|
||||
starlette==0.41.3
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
click==8.5.0
|
||||
colorama==0.4.6; sys_platform == "win32"
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
pydantic_core==2.46.5
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
psycopg[binary]==3.2.6
|
||||
SQLAlchemy==2.0.41
|
||||
redis==5.2.1
|
||||
argon2-cffi==23.1.0
|
||||
boto3==1.40.70
|
||||
Pillow==12.0.0
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Track 1: reproducible local benchmark and overload/recovery proof. Uses Docker Compose."""
|
||||
import argparse
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
PROBE = (ROOT / 'student-tests/queue_probe.py').read_text(encoding='utf-8')
|
||||
|
||||
|
||||
def run(args, input=None):
|
||||
result = subprocess.run(args, cwd=ROOT, input=input, text=True, encoding='utf-8',
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if result.returncode:
|
||||
raise RuntimeError('Command failed: ' + ' '.join(args[:4]) + '\n' + result.stderr[-4000:])
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, override=None, input=None):
|
||||
command = ['docker', 'compose', '-f', str(ROOT / 'compose.yaml')]
|
||||
if override:
|
||||
command += ['-f', str(override)]
|
||||
return run(command + list(args), input)
|
||||
|
||||
|
||||
def ready():
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) as response:
|
||||
if response.status == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError('API did not become ready')
|
||||
|
||||
|
||||
def probe(*args, service='worker'):
|
||||
return json.loads(compose('exec', '-T', service, 'python', '-', *map(str, args), input=PROBE))
|
||||
|
||||
|
||||
def configure(path, concurrency, limit, api=False):
|
||||
path.write_text(json.dumps({'services': {
|
||||
'worker': {'environment': {'WORKER_CONCURRENCY': str(concurrency)}},
|
||||
'api': {'environment': {'MAX_PENDING_ATTACHMENTS': str(limit)}},
|
||||
}}), encoding='utf-8')
|
||||
services = ['worker', 'api'] if api else ['worker']
|
||||
compose('up', '-d', '--no-deps', '--force-recreate', *services, override=path)
|
||||
ready()
|
||||
|
||||
|
||||
def write_results(data):
|
||||
path = ROOT / 'evidence/queue-load.json'
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
fields = ['repeat', 'concurrency', 'prefetch_per_consumer', 'width', 'height', 'bytes',
|
||||
'requests', 'clients', 'elapsed_s', 'throughput_per_s', 'ready_p95_s',
|
||||
'queue_wait_p95_s', 'queue_lag_max_jobs', 'queue_oldest_max_s',
|
||||
'processing_max', 'rss_peak_mib']
|
||||
with (ROOT / 'evidence/queue-load.csv').open('w', encoding='utf-8', newline='') as output:
|
||||
writer = csv.DictWriter(output, fieldnames=fields, extrasaction='ignore')
|
||||
writer.writeheader()
|
||||
writer.writerows(data['runs'])
|
||||
|
||||
|
||||
def source_hash():
|
||||
digest = hashlib.sha256()
|
||||
paths = [ROOT / 'compose.yaml', ROOT / 'scripts/queue_experiment.py', ROOT / 'student-tests/queue_probe.py']
|
||||
paths += list((ROOT / 'src').rglob('*.py')) + list((ROOT / 'migrations').glob('*.sql'))
|
||||
for path in sorted(paths):
|
||||
digest.update(path.relative_to(ROOT).as_posix().encode())
|
||||
digest.update(path.read_bytes().replace(b'\r\n', b'\n'))
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--count', type=int, default=18)
|
||||
parser.add_argument('--repeats', type=int, default=2)
|
||||
args = parser.parse_args()
|
||||
if not 6 <= args.count <= 48 or args.repeats < 1:
|
||||
parser.error('count must be 6..48; repeats must be positive')
|
||||
ready()
|
||||
active = compose('exec', '-T', 'api', 'python', '-c',
|
||||
"from src.database import create_db_engine; from sqlalchemy import text; "
|
||||
"e=create_db_engine(); c=e.connect(); print(c.execute(text(\"SELECT count(*) FROM attachments WHERE status IN ('queued','processing')\")).scalar())")
|
||||
if active != '0':
|
||||
raise RuntimeError('Wait for existing queued/processing work before this isolated experiment')
|
||||
data = {
|
||||
'started_at': datetime.now(timezone.utc).isoformat(),
|
||||
'base_commit': run(['git', 'rev-parse', 'HEAD']), 'source_sha256': source_hash(),
|
||||
'host_platform': platform.platform(),
|
||||
'docker': json.loads(run(['docker', 'info', '--format', '{"cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}}}'])),
|
||||
'image_id': run(['docker', 'image', 'inspect', 'mtusi2026lab5-api:local', '--format', '{{.Id}}']),
|
||||
'profile': {'sizes': [[320, 200], [1280, 800], [2560, 1600]], 'concurrency': [1, 2, 4],
|
||||
'prefetch_per_consumer': 1, 'requests_per_run': args.count, 'clients': 6,
|
||||
'repeats': args.repeats, 'seed': 202605, 'jpeg_quality': 85,
|
||||
'worker_cpus': 2, 'worker_memory_mib': 2048, 'sampling_seconds': 0.1,
|
||||
'baseline_admission_limit': 64, 'overload_admission_limit': 12},
|
||||
'runs': [],
|
||||
}
|
||||
with tempfile.TemporaryDirectory(prefix='lab5-queue-') as directory:
|
||||
override = Path(directory) / 'compose.json'
|
||||
try:
|
||||
for repeat in range(1, args.repeats + 1):
|
||||
order = [1, 2, 4] if repeat % 2 else [4, 2, 1]
|
||||
sizes = data['profile']['sizes'] if repeat % 2 else list(reversed(data['profile']['sizes']))
|
||||
for concurrency in order:
|
||||
print(f'Configuration: repeat={repeat}, concurrency={concurrency}, prefetch=1', flush=True)
|
||||
configure(override, concurrency, 64, api=repeat == 1 and concurrency == 1)
|
||||
probe('measure', '--count', 6, '--clients', 6) # Same warmup for every configuration.
|
||||
for width, height in sizes:
|
||||
result = probe('measure', '--width', width, '--height', height, '--count', args.count)
|
||||
assert result['processing_max'] <= concurrency
|
||||
result.update(repeat=repeat, concurrency=concurrency, prefetch_per_consumer=1)
|
||||
data['runs'].append(result)
|
||||
write_results(data)
|
||||
print(f" {width}x{height}: {result['throughput_per_s']:.2f}/s, "
|
||||
f"p95={result['ready_p95_s']:.2f}s, RSS={result['rss_peak_mib']:.1f} MiB", flush=True)
|
||||
print('Overload: stop worker, 24 concurrent uploads, shared limit=12', flush=True)
|
||||
configure(override, 2, 12, api=True)
|
||||
compose('stop', 'worker', 'cleanup')
|
||||
overloaded = probe('overload', '--count', 24, '--clients', 12, '--limit', 12, service='api')
|
||||
data['overload'] = overloaded
|
||||
write_results(data)
|
||||
# Accepted work must survive restarts while the worker is still stopped.
|
||||
compose('restart', 'api', 'broker')
|
||||
ready()
|
||||
compose('start', 'worker')
|
||||
data['recovery'] = probe('recover', '--expected', json.dumps(overloaded))
|
||||
data['completed_at'] = datetime.now(timezone.utc).isoformat()
|
||||
write_results(data)
|
||||
print('PASS: 12 accepted, 12 rejected with Retry-After; all accepted IDs recovered; new upload accepted', flush=True)
|
||||
finally:
|
||||
# Restore the user's base Compose settings, including any values in .env.
|
||||
compose('up', '-d', '--no-deps', '--force-recreate', 'api', 'worker')
|
||||
compose('start', 'cleanup')
|
||||
ready()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,13 @@
|
||||
import os
|
||||
|
||||
import uvicorn
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
uvicorn.run(
|
||||
'src.main:app',
|
||||
host=os.getenv('APP_HOST', '0.0.0.0'),
|
||||
port=int(os.getenv('APP_PORT', '8080')),
|
||||
workers=1,
|
||||
timeout_graceful_shutdown=5,
|
||||
)
|
||||
@@ -0,0 +1,21 @@
|
||||
"""A shared admission limit for unfinished attachments across all API replicas."""
|
||||
import os
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from .errors import APIError
|
||||
from .models import Attachment
|
||||
|
||||
MAX_PENDING = int(os.getenv('MAX_PENDING_ATTACHMENTS', '64'))
|
||||
if MAX_PENDING < 1:
|
||||
raise ValueError('MAX_PENDING_ATTACHMENTS must be positive')
|
||||
|
||||
|
||||
def reserve_capacity(session):
|
||||
# Keep the lock until the new queued row commits. Counting without this lock
|
||||
# would allow simultaneous uploads on different replicas to exceed the limit.
|
||||
session.execute(select(func.pg_advisory_xact_lock(202605)))
|
||||
count = session.scalar(select(func.count()).select_from(Attachment).where(
|
||||
Attachment.status.in_(['queued', 'processing'])))
|
||||
if count >= MAX_PENDING:
|
||||
raise APIError(429, 'rate_limited', 'Очередь обработки заполнена; повторите загрузку позже')
|
||||
@@ -0,0 +1,87 @@
|
||||
import os
|
||||
import unicodedata
|
||||
from uuid import uuid4
|
||||
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from .errors import APIError
|
||||
from .admission import reserve_capacity
|
||||
from .models import Attachment, AttachmentJob, ChatMember
|
||||
from .object_storage import BUCKET, PREFIX, s3_client
|
||||
from .store import ensure_uuid, serialize, store
|
||||
|
||||
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
|
||||
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
|
||||
if MAX_BYTES < 1 or MAX_PIXELS < 1:
|
||||
raise ValueError('Upload limits must be positive')
|
||||
|
||||
|
||||
def validate_filename(filename):
|
||||
if (not filename or not filename.strip() or len(filename) > 255
|
||||
or any(c in '/\\' or unicodedata.category(c) == 'Cc' for c in filename)):
|
||||
raise APIError(400, 'invalid_request', 'Неверное имя файла')
|
||||
|
||||
|
||||
def validate_image(source, content_type):
|
||||
try:
|
||||
source.seek(0)
|
||||
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||
if picture.width * picture.height > MAX_PIXELS:
|
||||
raise APIError(413, 'payload_too_large', 'Слишком много пикселей')
|
||||
if Image.MIME[picture.format] != content_type:
|
||||
raise APIError(415, 'unsupported_media_type', 'Тип изображения не совпадает с Content-Type')
|
||||
picture.verify()
|
||||
source.seek(0)
|
||||
with Image.open(source, formats=['PNG', 'JPEG']) as picture:
|
||||
picture.load() # verify alone does not decode JPEG or detect truncated pixel data.
|
||||
except Image.DecompressionBombError as exc:
|
||||
raise APIError(413, 'payload_too_large', 'Слишком много пикселей') from exc
|
||||
except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as exc:
|
||||
raise APIError(415, 'unsupported_media_type', 'Повреждённое изображение') from exc
|
||||
source.seek(0)
|
||||
|
||||
|
||||
def lock_object(session, key):
|
||||
# Shared by upload and cleanup: no deletion while a COMMIT is in flight.
|
||||
session.execute(select(func.pg_advisory_xact_lock(func.hashtextextended(key, 0))))
|
||||
|
||||
|
||||
def metadata(attachment):
|
||||
result = serialize(attachment, 'id', 'chat_id', 'owner_id', 'filename',
|
||||
'content_type', 'size_bytes', 'sha256', 'created_at',
|
||||
'updated_at', 'status', 'error_code')
|
||||
return {**result, 'variants': [{k: v for k, v in item.items() if k != 'object_key'}
|
||||
for item in attachment.variants]}
|
||||
|
||||
|
||||
def save_attachment(source, chat_id, actor_id, filename, content_type, size, digest):
|
||||
validate_image(source, content_type)
|
||||
attachment_id = str(uuid4())
|
||||
key = PREFIX + attachment_id
|
||||
staging_key = 'staging/' + attachment_id
|
||||
with store.sessions.begin() as session:
|
||||
store._chat(session, chat_id, actor_id)
|
||||
reserve_capacity(session)
|
||||
lock_object(session, staging_key)
|
||||
s3_client().put_object(Bucket=BUCKET, Key=staging_key, Body=source,
|
||||
ContentLength=size, ContentType=content_type)
|
||||
attachment = Attachment(id=attachment_id, chat_id=chat_id, owner_id=actor_id,
|
||||
object_key=key, filename=filename, content_type=content_type,
|
||||
size_bytes=size, sha256=digest, staging_key=staging_key)
|
||||
session.add(attachment)
|
||||
session.flush()
|
||||
session.add(AttachmentJob(attachment_id=attachment_id))
|
||||
result = metadata(attachment)
|
||||
# An uncertain COMMIT is not followed by DELETE: cleanup checks the DB later.
|
||||
return result
|
||||
|
||||
|
||||
def get_attachment(attachment_id, actor_id):
|
||||
with store.sessions() as session:
|
||||
attachment = session.scalar(select(Attachment).join(
|
||||
ChatMember, ChatMember.chat_id == Attachment.chat_id).where(
|
||||
Attachment.id == ensure_uuid(attachment_id), ChatMember.user_id == actor_id))
|
||||
if attachment is None:
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
return attachment
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
from redis import Redis
|
||||
|
||||
from .errors import APIError
|
||||
|
||||
password_hasher = PasswordHasher()
|
||||
dummy_hash = password_hasher.hash(secrets.token_urlsafe(32))
|
||||
|
||||
|
||||
def verify_password(encoded, password):
|
||||
try:
|
||||
return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None
|
||||
except VerificationError:
|
||||
return False
|
||||
|
||||
|
||||
def unauthorized():
|
||||
return APIError(401, 'unauthorized', 'Нет действительной идентификации')
|
||||
|
||||
|
||||
class Sessions:
|
||||
def __init__(self):
|
||||
self.client = Redis.from_url(
|
||||
os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||
socket_connect_timeout=1, socket_timeout=1, max_connections=20,
|
||||
retry_on_timeout=False,
|
||||
)
|
||||
self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600'))
|
||||
if self.ttl < 1:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||
|
||||
@staticmethod
|
||||
def key(token):
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise unauthorized()
|
||||
return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest()
|
||||
|
||||
def create(self, user):
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl)
|
||||
expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}),
|
||||
pxat=int(expires.timestamp() * 1000))
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user}
|
||||
|
||||
def user_id(self, token):
|
||||
raw = self.client.get(self.key(token))
|
||||
if raw is None:
|
||||
raise unauthorized()
|
||||
session = json.loads(raw)
|
||||
if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc):
|
||||
raise unauthorized()
|
||||
return session['user_id']
|
||||
|
||||
def delete(self, token):
|
||||
if not self.client.delete(self.key(token)):
|
||||
raise unauthorized()
|
||||
|
||||
|
||||
sessions = Sessions()
|
||||
@@ -0,0 +1,40 @@
|
||||
"""History pages are disposable; PostgreSQL owns both data and their version."""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .auth import sessions
|
||||
|
||||
|
||||
class HistoryCache:
|
||||
def __init__(self, client, ttl=30, prefix='history'):
|
||||
self.client = client
|
||||
self.ttl = ttl
|
||||
self.prefix = prefix
|
||||
if ttl < 1:
|
||||
raise ValueError('CACHE_TTL_SECONDS must be positive')
|
||||
|
||||
def key(self, chat_id, version, limit, after):
|
||||
page = json.dumps([limit, after], separators=(',', ':'))
|
||||
digest = hashlib.sha256(page.encode()).hexdigest()
|
||||
return f'{self.prefix}:{chat_id}:{version}:{digest}'
|
||||
|
||||
def get_or_load(self, chat_id, version, limit, after, load):
|
||||
key = self.key(chat_id, version, limit, after)
|
||||
try:
|
||||
cached = self.client.get(key)
|
||||
except RedisError:
|
||||
return load()
|
||||
if cached is not None:
|
||||
return json.loads(cached)
|
||||
page = load()
|
||||
try:
|
||||
self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl)
|
||||
except RedisError:
|
||||
pass
|
||||
return page
|
||||
|
||||
|
||||
history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30')), prefix='history-v4')
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Remove old S3 objects without committed metadata. Never delete on a DB error."""
|
||||
import argparse
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .attachments import lock_object
|
||||
from .database import create_db_engine
|
||||
from .models import Attachment
|
||||
from .object_storage import BUCKET, s3_client
|
||||
|
||||
GRACE = timedelta(seconds=max(1, int(os.getenv('STAGING_RETENTION_SECONDS', '86400'))))
|
||||
|
||||
|
||||
def cleanup(sessions, client, now=None):
|
||||
cutoff = (now or datetime.now(timezone.utc)) - GRACE
|
||||
deleted = 0
|
||||
for page in client.get_paginator('list_objects_v2').paginate(Bucket=BUCKET):
|
||||
for obj in page.get('Contents', []):
|
||||
key = obj['Key']
|
||||
if not key.startswith(('staging/', 'originals/', 'variants/')):
|
||||
continue
|
||||
with sessions.begin() as session:
|
||||
lock_object(session, key)
|
||||
if key.startswith('staging/'):
|
||||
attachment = session.scalar(select(Attachment).where(
|
||||
Attachment.staging_key == key).with_for_update())
|
||||
if attachment:
|
||||
if attachment.status in ('queued', 'processing'):
|
||||
continue
|
||||
if attachment.status == 'failed' and attachment.updated_at >= cutoff:
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
attachment.staging_key = None
|
||||
deleted += 1
|
||||
continue
|
||||
else:
|
||||
# All final keys belong to a server-generated attachment UUID.
|
||||
from uuid import UUID
|
||||
try:
|
||||
attachment_id = str(UUID(key.split('/')[1]))
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
attachment = session.get(Attachment, attachment_id, with_for_update=True)
|
||||
if attachment:
|
||||
if attachment.status != 'failed' or attachment.updated_at >= cutoff:
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
deleted += 1
|
||||
continue
|
||||
# Recheck age after taking the same lock as the uploader.
|
||||
if client.head_object(Bucket=BUCKET, Key=key)['LastModified'] >= cutoff:
|
||||
continue
|
||||
client.delete_object(Bucket=BUCKET, Key=key)
|
||||
deleted += 1
|
||||
return deleted
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--once', action='store_true')
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
count = cleanup(sessionmaker(engine), s3_client())
|
||||
logging.info('Cleanup completed: deleted=%d', count)
|
||||
except Exception as exc:
|
||||
logging.error('Cleanup deferred: %s', type(exc).__name__)
|
||||
if args.once:
|
||||
raise SystemExit(1) from None
|
||||
if args.once:
|
||||
break
|
||||
time.sleep(60)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,51 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import create_engine, func, insert, select
|
||||
from sqlalchemy.engine import URL, make_url
|
||||
|
||||
from .models import SchemaMigration
|
||||
|
||||
|
||||
def database_url():
|
||||
if os.getenv('DATABASE_URL'):
|
||||
url = make_url(os.environ['DATABASE_URL'])
|
||||
if url.drivername in ('postgres', 'postgresql'):
|
||||
url = url.set(drivername='postgresql+psycopg')
|
||||
if url.drivername != 'postgresql+psycopg':
|
||||
raise ValueError('DATABASE_URL must use PostgreSQL with psycopg')
|
||||
return url
|
||||
return URL.create(
|
||||
'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'),
|
||||
database=os.getenv('POSTGRES_DB', 'messenger'),
|
||||
username=os.getenv('POSTGRES_USER', 'messenger'),
|
||||
password=os.environ['POSTGRES_PASSWORD'],
|
||||
)
|
||||
|
||||
|
||||
def create_db_engine():
|
||||
return create_engine(
|
||||
database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True,
|
||||
connect_args={'connect_timeout': 2,
|
||||
'options': '-c statement_timeout=3000 -c lock_timeout=2000'},
|
||||
)
|
||||
|
||||
|
||||
def migrate(engine):
|
||||
with engine.begin() as conn:
|
||||
conn.execute(select(func.pg_advisory_xact_lock(202602)))
|
||||
SchemaMigration.__table__.create(conn, checkfirst=True)
|
||||
applied = set(conn.scalars(select(SchemaMigration.version)))
|
||||
for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')):
|
||||
if path.name not in applied:
|
||||
conn.exec_driver_sql(path.read_text(encoding='utf-8'))
|
||||
conn.execute(insert(SchemaMigration).values(version=path.name))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
migrate(engine)
|
||||
print('Migrations applied')
|
||||
finally:
|
||||
engine.dispose()
|
||||
@@ -0,0 +1,31 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi import Depends, Header, Query
|
||||
|
||||
from .auth import sessions, unauthorized
|
||||
from .errors import APIError
|
||||
from .store import store
|
||||
|
||||
|
||||
def page_limit(limit: str = Query(default="50")) -> int:
|
||||
if not limit.isascii() or not limit.isdecimal() or len(limit) > 3:
|
||||
raise APIError(400, "invalid_request", "Неверный limit")
|
||||
number = int(limit)
|
||||
if not 1 <= number <= 100:
|
||||
raise APIError(400, "invalid_request", "Неверный limit")
|
||||
return number
|
||||
|
||||
|
||||
|
||||
def bearer_token(authorization: Optional[str] = Header(default=None)) -> str:
|
||||
parts = (authorization or '').split()
|
||||
if len(parts) != 2 or parts[0].lower() != 'bearer':
|
||||
raise unauthorized()
|
||||
return parts[1]
|
||||
|
||||
|
||||
def current_user(token: str = Depends(bearer_token)) -> dict:
|
||||
user = store.get_user(sessions.user_id(token))
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
@@ -0,0 +1,60 @@
|
||||
from fastapi import Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from .middleware import INSTANCE_ID
|
||||
|
||||
|
||||
class APIError(Exception):
|
||||
def __init__(self, status_code: int, code: str, message: str):
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
|
||||
|
||||
def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse:
|
||||
request_id = getattr(request.state, "request_id", "")
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID},
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def api_error_handler(request: Request, exc: APIError) -> JSONResponse:
|
||||
response = error_response(request, exc.status_code, exc.code, exc.message)
|
||||
if exc.status_code == 429:
|
||||
response.headers['Retry-After'] = '1'
|
||||
if exc.status_code == 401:
|
||||
response.headers['WWW-Authenticate'] = (
|
||||
'Basic realm="messenger", charset="UTF-8"'
|
||||
if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"'
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно')
|
||||
|
||||
|
||||
def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return error_response(request, 400, "invalid_request", "Неверный запрос")
|
||||
|
||||
|
||||
def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||
if exc.status_code == 404:
|
||||
return error_response(request, 404, "not_found", "Объект не найден")
|
||||
if exc.status_code == 405:
|
||||
return error_response(request, 400, "invalid_request", "Метод не поддерживается")
|
||||
else:
|
||||
return error_response(request, exc.status_code, "invalid_request", "Неверный запрос")
|
||||
|
||||
|
||||
def internal_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
return error_response(request, 500, "internal_error", "Внутренняя ошибка")
|
||||
@@ -0,0 +1,116 @@
|
||||
"""One isolated process per attempt; quality: JPEG 85, WebP 80; no EXIF/ICC."""
|
||||
import hashlib
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
|
||||
from PIL import Image, ImageOps, UnidentifiedImageError
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .attachments import MAX_BYTES, MAX_PIXELS
|
||||
from .database import create_db_engine
|
||||
from .jobs import now
|
||||
from .models import Attachment, AttachmentJob
|
||||
from .object_storage import BUCKET, s3_client
|
||||
|
||||
|
||||
def fit_size(size, bound):
|
||||
width, height = size
|
||||
if max(size) <= bound:
|
||||
return size
|
||||
return max(1, width * bound // max(size)), max(1, height * bound // max(size))
|
||||
|
||||
|
||||
def variants(data):
|
||||
with Image.open(io.BytesIO(data), formats=['PNG', 'JPEG']) as original:
|
||||
if original.width * original.height > MAX_PIXELS:
|
||||
raise ValueError('pixel limit')
|
||||
original.load()
|
||||
oriented = ImageOps.exif_transpose(original)
|
||||
rgba = oriented.convert('RGBA')
|
||||
rgb = Image.new('RGB', rgba.size, 'white')
|
||||
rgb.paste(rgba, mask=rgba.getchannel('A'))
|
||||
rgba.close()
|
||||
oriented.close()
|
||||
rgb.info.clear()
|
||||
try:
|
||||
for name, image, fmt, mime, quality in (
|
||||
('thumbnail', rgb.resize(fit_size(rgb.size, 256), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
|
||||
('crop', ImageOps.fit(rgb, (128, 128), Image.Resampling.LANCZOS), 'JPEG', 'image/jpeg', 85),
|
||||
('optimized', rgb.resize(fit_size(rgb.size, 1280), Image.Resampling.LANCZOS), 'WEBP', 'image/webp', 80),
|
||||
):
|
||||
with image, io.BytesIO() as output:
|
||||
image.save(output, format=fmt, quality=quality)
|
||||
blob = output.getvalue()
|
||||
yield {'name': name, 'content_type': mime, 'width': image.width,
|
||||
'height': image.height, 'size_bytes': len(blob),
|
||||
'sha256': hashlib.sha256(blob).hexdigest()}, blob
|
||||
finally:
|
||||
rgb.close()
|
||||
|
||||
|
||||
def process(sessions, job_id, token):
|
||||
with sessions() as session:
|
||||
job = session.get(AttachmentJob, job_id)
|
||||
if job.lease_token != token:
|
||||
return
|
||||
attachment = session.get(Attachment, job.attachment_id)
|
||||
staging_key = attachment.staging_key
|
||||
client = s3_client()
|
||||
response = client.get_object(Bucket=BUCKET, Key=staging_key)
|
||||
with response['Body'] as body:
|
||||
data = body.read(MAX_BYTES + 1)
|
||||
if len(data) != attachment.size_bytes or hashlib.sha256(data).hexdigest() != attachment.sha256:
|
||||
raise ValueError('staging checksum mismatch')
|
||||
try:
|
||||
rendered = list(variants(data))
|
||||
except OSError as exc:
|
||||
raise ValueError('invalid image') from exc
|
||||
# Job lock fences the final writes, cleanup and publication against lease recovery.
|
||||
# Work before this transaction is CPU-only. Keys are stable across retries.
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||
if job.lease_token != token or job.lease_until <= now():
|
||||
return
|
||||
current = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if current.status != 'processing':
|
||||
return
|
||||
client.put_object(Bucket=BUCKET, Key=current.object_key, Body=data,
|
||||
ContentType=current.content_type)
|
||||
saved = []
|
||||
for meta, blob in rendered:
|
||||
key = f'variants/{current.id}/v{job.pipeline_version}/{meta["name"]}'
|
||||
client.put_object(Bucket=BUCKET, Key=key, Body=blob, ContentType=meta['content_type'])
|
||||
saved.append({**meta, 'object_key': key})
|
||||
# Test-only crash injection; never enabled by compose.yaml.
|
||||
if os.getenv('JOB_FAULT') == 'after_objects':
|
||||
os._exit(91)
|
||||
current.variants = saved
|
||||
current.status, current.error_code = 'ready', None
|
||||
current.updated_at = now()
|
||||
job.lease_token = job.lease_until = None
|
||||
if os.getenv('JOB_FAULT') == 'after_commit':
|
||||
os._exit(92)
|
||||
|
||||
|
||||
def main():
|
||||
# Linux worker: bounded address space as well as the container memory limit.
|
||||
import resource
|
||||
memory = int(os.getenv('JOB_MEMORY_MB', '768')) * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
process(sessionmaker(engine, expire_on_commit=False), sys.argv[1], sys.argv[2])
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError, ValueError, SyntaxError):
|
||||
return 20
|
||||
except MemoryError:
|
||||
return 21
|
||||
except Exception:
|
||||
return 22 # No credentials or decoder/database internals in logs.
|
||||
finally:
|
||||
engine.dispose()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
+180
@@ -0,0 +1,180 @@
|
||||
"""Small PostgreSQL outbox + Redis Streams queue; PostgreSQL owns job state."""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from redis import Redis
|
||||
from redis.exceptions import ResponseError
|
||||
from sqlalchemy import and_, or_, select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .database import create_db_engine
|
||||
from .admission import reserve_capacity
|
||||
from .models import Attachment, AttachmentJob
|
||||
from .object_storage import BUCKET, s3_client
|
||||
|
||||
STREAM = 'attachment-jobs'
|
||||
GROUP = 'image-workers'
|
||||
MAX_ATTEMPTS = max(1, int(os.getenv('JOB_MAX_ATTEMPTS', '5')))
|
||||
TIMEOUT = max(1, int(os.getenv('JOB_TIMEOUT_SECONDS', '60')))
|
||||
LEASE_SECONDS = TIMEOUT + 15
|
||||
|
||||
|
||||
def now():
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def broker():
|
||||
return Redis.from_url(os.getenv('BROKER_URL', 'redis://broker:6379/0'),
|
||||
decode_responses=True, socket_connect_timeout=2, socket_timeout=5)
|
||||
|
||||
|
||||
def ensure_group(client):
|
||||
try:
|
||||
client.xgroup_create(STREAM, GROUP, id='0', mkstream=True)
|
||||
except ResponseError as exc:
|
||||
if not str(exc).startswith('BUSYGROUP'):
|
||||
raise
|
||||
|
||||
|
||||
def envelope(job):
|
||||
return {'schema_version': 1, 'job_id': job.id, 'type': 'attachment.process.v1',
|
||||
'attachment_id': job.attachment_id, 'pipeline_version': job.pipeline_version,
|
||||
'attempt': job.attempt + 1,
|
||||
'created_at': job.created_at.isoformat().replace('+00:00', 'Z')}
|
||||
|
||||
|
||||
def fail(session, job, attachment, code, permanent=False):
|
||||
job.failures += 1
|
||||
job.last_error = code
|
||||
job.lease_token = job.lease_until = job.published_at = None
|
||||
attachment.updated_at = now()
|
||||
if permanent or job.failures >= MAX_ATTEMPTS:
|
||||
attachment.status = 'failed'
|
||||
attachment.error_code = code if permanent else 'retry_exhausted'
|
||||
else:
|
||||
attachment.status = 'queued'
|
||||
attachment.error_code = None
|
||||
delay = min(60, 2 ** min(job.failures, 6)) * random.uniform(0.5, 1.5)
|
||||
job.available_at = now() + timedelta(seconds=delay)
|
||||
logging.info('job=%s attempt=%s result=%s code=%s', job.id, job.attempt, attachment.status, code)
|
||||
|
||||
|
||||
def dispatch_once(sessions, client):
|
||||
# A crash after XADD but before COMMIT deliberately results in redelivery.
|
||||
with sessions.begin() as session:
|
||||
jobs = list(session.scalars(select(AttachmentJob).join(Attachment).where(
|
||||
or_(and_(Attachment.status == 'queued', AttachmentJob.published_at.is_(None)),
|
||||
and_(Attachment.status == 'processing', AttachmentJob.lease_until <= now())),
|
||||
AttachmentJob.available_at <= now()).order_by(AttachmentJob.available_at)
|
||||
.with_for_update(skip_locked=True, of=AttachmentJob).limit(50)))
|
||||
for job in jobs:
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if attachment.status == 'processing':
|
||||
if job.lease_until <= now():
|
||||
fail(session, job, attachment, 'lease_expired')
|
||||
continue
|
||||
if job.published_at is None:
|
||||
client.xadd(STREAM, {'job': json.dumps(envelope(job))})
|
||||
job.published_at = now()
|
||||
|
||||
|
||||
def claim(sessions, message):
|
||||
if (message.get('schema_version') != 1 or message.get('type') != 'attachment.process.v1'
|
||||
or message.get('pipeline_version') != 1):
|
||||
raise ValueError('unsupported job envelope')
|
||||
job_id = str(UUID(message['job_id']))
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||
if job is None or job.attachment_id != message.get('attachment_id'):
|
||||
raise ValueError('unknown job')
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if attachment.status != 'queued' or job.available_at > now():
|
||||
return None
|
||||
job.attempt += 1
|
||||
if job.first_started_at is None:
|
||||
job.first_started_at = now()
|
||||
job.lease_token = str(uuid4())
|
||||
job.lease_until = now() + timedelta(seconds=LEASE_SECONDS)
|
||||
attachment.status = 'processing'
|
||||
attachment.updated_at = now()
|
||||
return job.id, job.lease_token
|
||||
|
||||
|
||||
def record_failure(sessions, job_id, token, code, permanent=False):
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, job_id, with_for_update=True)
|
||||
if job.lease_token == token:
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
fail(session, job, attachment, code, permanent)
|
||||
|
||||
|
||||
def replay(sessions, job_id, reason):
|
||||
if not reason.strip() or len(reason) > 200:
|
||||
raise ValueError('Give a nonempty reason of at most 200 characters')
|
||||
with sessions.begin() as session:
|
||||
job = session.get(AttachmentJob, str(UUID(job_id)), with_for_update=True)
|
||||
if job is None:
|
||||
raise ValueError('Job not found')
|
||||
attachment = session.get(Attachment, job.attachment_id, with_for_update=True)
|
||||
if attachment.status != 'failed':
|
||||
raise ValueError('Replay is allowed only for failed jobs')
|
||||
if not attachment.staging_key:
|
||||
raise ValueError('Staging retention expired; upload the file again')
|
||||
s3_client().head_object(Bucket=BUCKET, Key=attachment.staging_key)
|
||||
reserve_capacity(session)
|
||||
job.replay_audit = job.replay_audit + [{'at': now().isoformat(), 'reason': reason,
|
||||
'after_attempt': job.attempt,
|
||||
'last_error': job.last_error}]
|
||||
job.failures = 0
|
||||
job.available_at = now()
|
||||
job.published_at = job.lease_until = job.lease_token = None
|
||||
attachment.status, attachment.error_code = 'queued', None
|
||||
attachment.updated_at = now()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
sub = parser.add_subparsers(dest='command', required=True)
|
||||
sub.add_parser('dispatch')
|
||||
sub.add_parser('failed')
|
||||
retry = sub.add_parser('replay')
|
||||
retry.add_argument('job_id')
|
||||
retry.add_argument('--reason', required=True)
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
engine = create_db_engine()
|
||||
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
try:
|
||||
if args.command == 'failed':
|
||||
with sessions() as session:
|
||||
for job, att in session.execute(select(AttachmentJob, Attachment).join(
|
||||
Attachment, Attachment.id == AttachmentJob.attachment_id)
|
||||
.where(Attachment.status == 'failed')):
|
||||
print(json.dumps({'job_id': job.id, 'attachment_id': att.id,
|
||||
'attempt': job.attempt, 'error_code': att.error_code,
|
||||
'last_error': job.last_error,
|
||||
'replay_available': bool(att.staging_key)}))
|
||||
elif args.command == 'replay':
|
||||
replay(sessions, args.job_id, args.reason)
|
||||
print('Job queued for replay')
|
||||
else:
|
||||
client = broker()
|
||||
while True:
|
||||
try:
|
||||
ensure_group(client)
|
||||
dispatch_once(sessions, client)
|
||||
except Exception as exc:
|
||||
logging.warning('Dispatch deferred: %s', type(exc).__name__)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,99 @@
|
||||
import asyncio
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
from starlette.requests import Request
|
||||
|
||||
from .errors import error_response
|
||||
from .middleware import is_upload
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Limits:
|
||||
max_users: int = 1000
|
||||
max_chats: int = 1000
|
||||
max_messages: int = 10000
|
||||
max_text_bytes: int = 16 * 1024 * 1024
|
||||
max_request_bytes: int = 64 * 1024
|
||||
max_inflight: int = 32
|
||||
message_rate: int = 1000
|
||||
message_burst: int = 1000
|
||||
|
||||
@classmethod
|
||||
def from_env(cls):
|
||||
values = {}
|
||||
for name, default in cls().__dict__.items():
|
||||
value = int(os.getenv(name.upper(), str(default)))
|
||||
if value < 1:
|
||||
raise ValueError(f'{name.upper()} must be positive')
|
||||
values[name] = value
|
||||
return cls(**values)
|
||||
|
||||
|
||||
limits = Limits.from_env()
|
||||
|
||||
|
||||
class ResourceLimitMiddleware:
|
||||
def __init__(self, app, settings=limits):
|
||||
self.app = app
|
||||
self.settings = settings
|
||||
self.active = 0
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope['type'] != 'http' or scope['path'].startswith('/health/'):
|
||||
return await self.app(scope, receive, send)
|
||||
|
||||
request = Request(scope)
|
||||
|
||||
async def reject(status, code, message):
|
||||
response = error_response(request, status, code, message)
|
||||
if status == 503:
|
||||
response.headers['Retry-After'] = '1'
|
||||
await response(scope, receive, send)
|
||||
|
||||
if self.active >= self.settings.max_inflight:
|
||||
return await reject(503, 'unavailable', 'Слишком много одновременных запросов')
|
||||
self.active += 1
|
||||
try:
|
||||
# The upload endpoint authenticates and bounds its stream itself.
|
||||
if is_upload(scope['method'], scope['path']):
|
||||
return await self.app(scope, receive, send)
|
||||
length = request.headers.get('content-length')
|
||||
if length is not None:
|
||||
try:
|
||||
size = int(length)
|
||||
if size < 0:
|
||||
raise ValueError()
|
||||
except ValueError:
|
||||
return await reject(400, 'invalid_request', 'Неверный Content-Length')
|
||||
if size > self.settings.max_request_bytes:
|
||||
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
|
||||
|
||||
body = bytearray()
|
||||
try:
|
||||
async with asyncio.timeout(5):
|
||||
while True:
|
||||
message = await receive()
|
||||
if message['type'] == 'http.disconnect':
|
||||
return
|
||||
chunk = message.get('body', b'')
|
||||
if len(body) + len(chunk) > self.settings.max_request_bytes:
|
||||
return await reject(413, 'payload_too_large', 'Тело запроса слишком большое')
|
||||
body.extend(chunk)
|
||||
if not message.get('more_body', False):
|
||||
break
|
||||
except TimeoutError:
|
||||
return await reject(503, 'unavailable', 'Истекло время чтения запроса')
|
||||
|
||||
delivered = False
|
||||
|
||||
async def buffered_receive():
|
||||
nonlocal delivered
|
||||
if not delivered:
|
||||
delivered = True
|
||||
return {'type': 'http.request', 'body': bytes(body), 'more_body': False}
|
||||
return await receive()
|
||||
|
||||
await self.app(scope, buffered_receive, send)
|
||||
finally:
|
||||
self.active -= 1
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
from fastapi import FastAPI
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from starlette.exceptions import HTTPException
|
||||
from contextlib import asynccontextmanager
|
||||
from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from redis.exceptions import RedisError
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler
|
||||
from .database import create_db_engine
|
||||
from .store import store
|
||||
from .auth import sessions
|
||||
from .middleware import RequestIdMiddleware
|
||||
from .limits import ResourceLimitMiddleware
|
||||
from .routers.health import router as health_router
|
||||
from .routers.users import router as users_router
|
||||
from .routers.chats import router as chats_router
|
||||
from .routers.auth import router as auth_router
|
||||
from .routers.attachments import router as attachments_router
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
store.sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
yield
|
||||
finally:
|
||||
engine.dispose()
|
||||
sessions.client.close()
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger", lifespan=lifespan)
|
||||
app.add_middleware(ResourceLimitMiddleware)
|
||||
app.add_middleware(RequestIdMiddleware)
|
||||
app.add_exception_handler(APIError, api_error_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||
app.add_exception_handler(HTTPException, http_error_handler)
|
||||
app.add_exception_handler(Exception, internal_error_handler)
|
||||
for error in (OperationalError, InterfaceError, PoolTimeout, RedisError, BotoCoreError, ClientError):
|
||||
app.add_exception_handler(error, dependency_error_handler)
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(users_router)
|
||||
app.include_router(chats_router)
|
||||
app.include_router(auth_router)
|
||||
app.include_router(attachments_router)
|
||||
@@ -0,0 +1,43 @@
|
||||
from uuid import uuid4
|
||||
from socket import gethostname
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
from starlette.requests import Request
|
||||
import re
|
||||
|
||||
|
||||
def is_upload(method, path):
|
||||
return method == 'POST' and re.fullmatch(r'/api/v1/chats/[^/]+/attachments', path) is not None
|
||||
|
||||
INSTANCE_ID = gethostname()
|
||||
|
||||
|
||||
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4())
|
||||
request.state.request_id = request_id
|
||||
|
||||
if (request.method in {"POST", "PUT", "PATCH"}
|
||||
and request.url.path.startswith("/api/")
|
||||
and not is_upload(request.method, request.url.path)
|
||||
and request.url.path != '/api/v1/auth/sessions'):
|
||||
content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||||
if content_type != "application/json":
|
||||
response = JSONResponse(
|
||||
status_code=415,
|
||||
content={
|
||||
"error": {
|
||||
"code": "unsupported_media_type",
|
||||
"message": "Ожидается Content-Type application/json",
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = INSTANCE_ID
|
||||
return response
|
||||
+108
@@ -0,0 +1,108 @@
|
||||
from datetime import datetime
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
from sqlalchemy.dialects.postgresql import ARRAY, JSONB
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = 'users'
|
||||
__table_args__ = (
|
||||
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"),
|
||||
CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
username: Mapped[str] = mapped_column(Text, unique=True)
|
||||
display_name: Mapped[str] = mapped_column(Text)
|
||||
password_hash: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = 'chats'
|
||||
__table_args__ = (
|
||||
CheckConstraint('char_length(title) BETWEEN 1 AND 100'),
|
||||
Index('chats_order_idx', 'created_at', 'id'),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
history_version: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = 'chat_members'
|
||||
__table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),)
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||
user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = 'messages'
|
||||
__table_args__ = (
|
||||
CheckConstraint('char_length(text) <= 4000'),
|
||||
CheckConstraint("length(btrim(text)) > 0 OR cardinality(attachment_ids) > 0"),
|
||||
Index('messages_history_idx', 'chat_id', 'created_at', 'id'),
|
||||
)
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||
sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
attachment_ids: Mapped[list[str]] = mapped_column(ARRAY(Uuid(as_uuid=False)), default=list, server_default='{}')
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
|
||||
|
||||
class SchemaMigration(Base):
|
||||
__tablename__ = 'schema_migrations'
|
||||
version: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True)
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = 'attachments'
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True)
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'))
|
||||
owner_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'))
|
||||
object_key: Mapped[str] = mapped_column(Text, unique=True)
|
||||
filename: Mapped[str] = mapped_column(Text)
|
||||
content_type: Mapped[str] = mapped_column(Text)
|
||||
size_bytes: Mapped[int] = mapped_column(BigInteger)
|
||||
sha256: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
status: Mapped[str] = mapped_column(Text, server_default='queued')
|
||||
staging_key: Mapped[str | None] = mapped_column(Text)
|
||||
variants: Mapped[list] = mapped_column(JSONB, server_default='[]')
|
||||
error_code: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class AttachmentJob(Base):
|
||||
"""The job row is also the transactional outbox and failed-job registry."""
|
||||
__tablename__ = 'attachment_jobs'
|
||||
id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4()))
|
||||
attachment_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('attachments.id'), unique=True)
|
||||
pipeline_version: Mapped[int] = mapped_column(BigInteger, server_default='1')
|
||||
attempt: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||
failures: Mapped[int] = mapped_column(BigInteger, server_default='0')
|
||||
last_error: Mapped[str | None] = mapped_column(Text)
|
||||
first_started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
available_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
lease_until: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
lease_token: Mapped[str | None] = mapped_column(Uuid(as_uuid=False))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
replay_audit: Mapped[list] = mapped_column(JSONB, server_default='[]')
|
||||
|
||||
|
||||
class IdempotencyRecord(Base):
|
||||
__tablename__ = 'message_idempotency'
|
||||
actor_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True)
|
||||
chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
message_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('messages.id', ondelete='CASCADE'))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp())
|
||||
@@ -0,0 +1,23 @@
|
||||
import os
|
||||
from functools import lru_cache
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
|
||||
BUCKET = os.getenv('S3_BUCKET', 'messenger-attachments')
|
||||
PREFIX = 'originals/'
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def s3_client():
|
||||
return boto3.client(
|
||||
's3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
||||
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
||||
aws_access_key_id=os.environ['S3_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key=os.environ['S3_SECRET_ACCESS_KEY'],
|
||||
config=Config(connect_timeout=2, read_timeout=5,
|
||||
retries={'total_max_attempts': 1},
|
||||
s3={'addressing_style': 'path'},
|
||||
request_checksum_calculation='when_required',
|
||||
response_checksum_validation='when_required'),
|
||||
)
|
||||
@@ -0,0 +1,90 @@
|
||||
import asyncio
|
||||
import hashlib
|
||||
from tempfile import TemporaryFile
|
||||
from urllib.parse import quote
|
||||
|
||||
from fastapi import APIRouter, Depends, Query, Request
|
||||
from starlette.background import BackgroundTask
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from ..attachments import MAX_BYTES, get_attachment, metadata, save_attachment, validate_filename
|
||||
from ..dependencies import current_user
|
||||
from ..errors import APIError
|
||||
from ..object_storage import BUCKET, s3_client
|
||||
from ..store import ensure_uuid, store
|
||||
|
||||
router = APIRouter(prefix='/api/v1', tags=['attachments'])
|
||||
# At most one decoded image (<=40M pixels) per API process; incoming bodies use disk.
|
||||
decode_slot = asyncio.Semaphore(1)
|
||||
|
||||
|
||||
@router.post('/chats/{chat_id}/attachments', status_code=202)
|
||||
async def upload(chat_id: str, request: Request, filename: str = Query(...),
|
||||
user: dict = Depends(current_user)):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
await run_in_threadpool(store.get_chat_for_user, chat_id, user['id'])
|
||||
validate_filename(filename)
|
||||
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
|
||||
if content_type not in ('image/png', 'image/jpeg'):
|
||||
raise APIError(415, 'unsupported_media_type', 'Ожидается PNG или JPEG')
|
||||
length = request.headers.get('content-length')
|
||||
if length is not None:
|
||||
try:
|
||||
length = int(length)
|
||||
if length < 0:
|
||||
raise ValueError()
|
||||
except ValueError as exc:
|
||||
raise APIError(400, 'invalid_request', 'Неверный Content-Length') from exc
|
||||
if length > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||
size = 0
|
||||
digest = hashlib.sha256()
|
||||
with TemporaryFile() as source:
|
||||
try:
|
||||
async with asyncio.timeout(30):
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise APIError(413, 'payload_too_large', 'Файл слишком большой')
|
||||
digest.update(chunk)
|
||||
await run_in_threadpool(source.write, chunk)
|
||||
except (TimeoutError, ClientDisconnect) as exc:
|
||||
raise APIError(400, 'invalid_request', 'Загрузка прервана') from exc
|
||||
if length is not None and size != length:
|
||||
raise APIError(400, 'invalid_request', 'Неполное тело запроса')
|
||||
async with decode_slot:
|
||||
return await run_in_threadpool(save_attachment, source, chat_id, user['id'],
|
||||
filename, content_type, size, digest.hexdigest())
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}')
|
||||
def attachment_info(attachment_id: str, user: dict = Depends(current_user)):
|
||||
return metadata(get_attachment(attachment_id, user['id']))
|
||||
|
||||
|
||||
@router.get('/attachments/{attachment_id}/content')
|
||||
def download(attachment_id: str, variant: str = 'original', user: dict = Depends(current_user)):
|
||||
attachment = get_attachment(attachment_id, user['id'])
|
||||
if attachment.status != 'ready':
|
||||
raise APIError(409, 'conflict', 'Вложение ещё не готово')
|
||||
key, content_type, size = attachment.object_key, attachment.content_type, attachment.size_bytes
|
||||
if variant != 'original':
|
||||
item = next((v for v in attachment.variants if v['name'] == variant), None)
|
||||
if item is None:
|
||||
raise APIError(404, 'not_found', 'Вариант не найден')
|
||||
key, content_type, size = item['object_key'], item['content_type'], item['size_bytes']
|
||||
body = s3_client().get_object(Bucket=BUCKET, Key=key)['Body']
|
||||
|
||||
def chunks():
|
||||
try:
|
||||
yield from body.iter_chunks(chunk_size=64 * 1024)
|
||||
finally:
|
||||
body.close()
|
||||
|
||||
return StreamingResponse(chunks(), media_type=content_type,
|
||||
headers={'Content-Length': str(size),
|
||||
'Content-Disposition': "attachment; filename*=UTF-8''" + quote(attachment.filename, safe=''),
|
||||
'X-Content-Type-Options': 'nosniff'},
|
||||
background=BackgroundTask(body.close))
|
||||
@@ -0,0 +1,34 @@
|
||||
import base64
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Request, Response
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from ..auth import sessions, unauthorized
|
||||
from ..dependencies import bearer_token, current_user
|
||||
from ..errors import APIError
|
||||
from ..store import store
|
||||
|
||||
router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth'])
|
||||
|
||||
|
||||
@router.post('', status_code=201)
|
||||
async def create_session(request: Request, authorization: str = Header(default='')):
|
||||
if await request.body():
|
||||
raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса')
|
||||
try:
|
||||
scheme, encoded = authorization.split()
|
||||
if scheme.lower() != 'basic':
|
||||
raise ValueError()
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username:
|
||||
raise ValueError()
|
||||
except (ValueError, UnicodeError):
|
||||
raise unauthorized() from None
|
||||
user = await run_in_threadpool(store.login, username, password)
|
||||
return await run_in_threadpool(sessions.create, user)
|
||||
|
||||
|
||||
@router.delete('/current', status_code=204)
|
||||
def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)):
|
||||
sessions.delete(token)
|
||||
return Response(status_code=204)
|
||||
@@ -0,0 +1,49 @@
|
||||
from typing import Optional
|
||||
import re
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, Query, status
|
||||
from ..errors import APIError
|
||||
|
||||
from ..dependencies import current_user, page_limit
|
||||
from ..schemas.chat import Chat, ChatPage, CreateChat
|
||||
from ..schemas.message import CreateMessage, Message, MessagePage
|
||||
from ..store import store
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
|
||||
@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED)
|
||||
def create_chat(payload: CreateChat, user: dict = Depends(current_user)):
|
||||
return store.create_chat(user["id"], payload.title, payload.member_ids)
|
||||
|
||||
|
||||
@router.get("", response_model=ChatPage)
|
||||
def list_chats(
|
||||
limit: int = Depends(page_limit),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
user: dict = Depends(current_user),
|
||||
):
|
||||
return store.list_chats(user["id"], limit, cursor)
|
||||
|
||||
|
||||
@router.get("/{chat_id}", response_model=Chat)
|
||||
def get_chat(chat_id: str, user: dict = Depends(current_user)):
|
||||
return store.get_chat_for_user(chat_id, user["id"])
|
||||
|
||||
|
||||
@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED)
|
||||
def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user),
|
||||
idempotency_key: Optional[str] = Header(default=None)):
|
||||
if not idempotency_key or not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', idempotency_key):
|
||||
raise APIError(400, 'invalid_request', 'Требуется корректный Idempotency-Key')
|
||||
return store.create_message(chat_id, user["id"], payload.text, idempotency_key, payload.attachment_ids)
|
||||
|
||||
|
||||
@router.get("/{chat_id}/messages", response_model=MessagePage)
|
||||
def list_messages(
|
||||
chat_id: str,
|
||||
limit: int = Depends(page_limit),
|
||||
cursor: Optional[str] = Query(default=None, min_length=1),
|
||||
user: dict = Depends(current_user),
|
||||
):
|
||||
return store.list_messages(chat_id, user["id"], limit, cursor)
|
||||
@@ -0,0 +1,23 @@
|
||||
from fastapi import APIRouter
|
||||
from sqlalchemy import select
|
||||
|
||||
from ..schemas.common import Health
|
||||
from ..auth import sessions
|
||||
from ..store import store
|
||||
from ..object_storage import BUCKET, s3_client
|
||||
|
||||
router = APIRouter(prefix='/health')
|
||||
|
||||
|
||||
@router.get("/live", response_model=Health)
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@router.get("/ready", response_model=Health)
|
||||
def health_ready():
|
||||
with store.sessions() as session:
|
||||
session.execute(select(1))
|
||||
sessions.client.ping()
|
||||
s3_client().head_bucket(Bucket=BUCKET)
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,19 @@
|
||||
from fastapi import APIRouter, Depends, status
|
||||
|
||||
from ..dependencies import current_user
|
||||
from ..schemas.user import CreateUser, User
|
||||
from ..store import store
|
||||
|
||||
|
||||
|
||||
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
|
||||
@router.post("", response_model=User, status_code=status.HTTP_201_CREATED)
|
||||
def create_user(payload: CreateUser):
|
||||
return store.create_user(payload.username, payload.display_name, payload.password)
|
||||
|
||||
|
||||
@router.get("/me", response_model=User)
|
||||
def get_current_user(user: dict = Depends(current_user)):
|
||||
return user
|
||||
@@ -0,0 +1,35 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import Field, StrictStr, field_validator
|
||||
|
||||
from .common import ContractModel
|
||||
|
||||
|
||||
|
||||
class CreateChat(ContractModel):
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: List[StrictStr] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
def validate_title(cls, value):
|
||||
if not value.strip():
|
||||
raise ValueError("blank title")
|
||||
return value
|
||||
|
||||
@field_validator("member_ids")
|
||||
def validate_member_ids(cls, value):
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("duplicate member_ids")
|
||||
return value
|
||||
|
||||
|
||||
class Chat(ContractModel):
|
||||
id: str
|
||||
title: str
|
||||
member_ids: List[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class ChatPage(ContractModel):
|
||||
items: List[Chat]
|
||||
next_cursor: Optional[str]
|
||||
@@ -0,0 +1,17 @@
|
||||
from pydantic import BaseModel, ConfigDict, field_validator
|
||||
|
||||
|
||||
class ContractModel(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
@field_validator('*')
|
||||
def validate_text_encoding(cls, value, info):
|
||||
if isinstance(value, str):
|
||||
value.encode('utf-8')
|
||||
if info.field_name != 'password' and '\x00' in value:
|
||||
raise ValueError('NUL is not supported in PostgreSQL text')
|
||||
return value
|
||||
|
||||
|
||||
class Health(ContractModel):
|
||||
status: str
|
||||
@@ -0,0 +1,39 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from pydantic import Field, StrictStr, field_validator, model_validator
|
||||
from uuid import UUID
|
||||
|
||||
from .common import ContractModel
|
||||
|
||||
|
||||
|
||||
class CreateMessage(ContractModel):
|
||||
text: StrictStr = Field(max_length=4000)
|
||||
attachment_ids: List[StrictStr] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
def validate_ids(cls, value):
|
||||
ids = [str(UUID(item)) for item in value]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError('duplicate attachments')
|
||||
return ids
|
||||
|
||||
@model_validator(mode='after')
|
||||
def validate_content(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('empty message')
|
||||
return self
|
||||
|
||||
|
||||
class Message(ContractModel):
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
attachment_ids: List[str]
|
||||
created_at: str
|
||||
|
||||
|
||||
class MessagePage(ContractModel):
|
||||
items: List[Message]
|
||||
next_cursor: Optional[str]
|
||||
@@ -0,0 +1,34 @@
|
||||
import re
|
||||
|
||||
from pydantic import Field, StrictStr, field_validator
|
||||
|
||||
from .common import ContractModel
|
||||
|
||||
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
|
||||
|
||||
class CreateUser(ContractModel):
|
||||
username: StrictStr
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("username")
|
||||
def validate_username(cls, value):
|
||||
if not USERNAME_RE.fullmatch(value):
|
||||
raise ValueError("invalid username")
|
||||
return value
|
||||
|
||||
@field_validator("display_name")
|
||||
def validate_display_name(cls, value):
|
||||
if not value.strip():
|
||||
raise ValueError("blank display_name")
|
||||
return value
|
||||
|
||||
|
||||
class User(ContractModel):
|
||||
id: str
|
||||
username: str
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
created_at: str
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
import base64
|
||||
import json
|
||||
from datetime import datetime, timezone
|
||||
from typing import Tuple
|
||||
from uuid import UUID
|
||||
|
||||
from sqlalchemy import select, tuple_, update
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from .auth import password_hasher, verify_password, unauthorized
|
||||
from .errors import APIError
|
||||
from .cache import history_cache
|
||||
from .models import User, Chat, ChatMember, Message, IdempotencyRecord, Attachment
|
||||
|
||||
|
||||
def ensure_uuid(value):
|
||||
try:
|
||||
if not isinstance(value, str):
|
||||
raise ValueError()
|
||||
return str(UUID(value))
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise APIError(400, 'invalid_request', 'Неверный UUID') from exc
|
||||
|
||||
|
||||
def serialize(model, *fields):
|
||||
if model is None:
|
||||
return None
|
||||
result = {}
|
||||
for field in fields:
|
||||
value = getattr(model, field)
|
||||
if isinstance(value, datetime):
|
||||
value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
result[field] = value
|
||||
return result
|
||||
|
||||
|
||||
USER_FIELDS = ('id', 'username', 'display_name', 'created_at')
|
||||
CHAT_FIELDS = ('id', 'title', 'created_at')
|
||||
MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at', 'attachment_ids')
|
||||
|
||||
|
||||
def after_condition(model, after):
|
||||
return tuple_(model.created_at, model.id) > (
|
||||
datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1])
|
||||
|
||||
|
||||
def history_query(chat_id, limit, after):
|
||||
query = select(Message).where(Message.chat_id == chat_id)
|
||||
if after:
|
||||
query = query.where(after_condition(Message, after))
|
||||
return query.order_by(Message.created_at, Message.id).limit(limit + 1)
|
||||
|
||||
|
||||
class PostgresStore:
|
||||
def __init__(self, cache=history_cache):
|
||||
self.sessions = None
|
||||
self.cache = cache
|
||||
|
||||
def create_user(self, username, display_name, password):
|
||||
user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password))
|
||||
try:
|
||||
with self.sessions.begin() as session:
|
||||
session.add(user)
|
||||
session.flush()
|
||||
result = serialize(user, *USER_FIELDS)
|
||||
return result
|
||||
except IntegrityError as exc:
|
||||
if getattr(exc.orig, 'sqlstate', None) == '23505':
|
||||
raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc
|
||||
raise
|
||||
|
||||
def get_user(self, user_id):
|
||||
with self.sessions() as session:
|
||||
return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS)
|
||||
|
||||
def login(self, username, password):
|
||||
with self.sessions() as session:
|
||||
user = session.scalar(select(User).where(User.username == username))
|
||||
encoded = user.password_hash if user else None
|
||||
result = serialize(user, *USER_FIELDS)
|
||||
if not verify_password(encoded, password):
|
||||
raise unauthorized()
|
||||
return result
|
||||
|
||||
def create_chat(self, actor_id, title, member_ids):
|
||||
members = [ensure_uuid(member) for member in member_ids]
|
||||
if len(members) != len(set(members)) or actor_id not in members:
|
||||
raise APIError(400, 'invalid_request', 'Неверный список участников')
|
||||
try:
|
||||
with self.sessions.begin() as session:
|
||||
chat = Chat(title=title)
|
||||
session.add(chat)
|
||||
session.flush()
|
||||
session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members])
|
||||
result = serialize(chat, *CHAT_FIELDS)
|
||||
result['member_ids'] = sorted(members)
|
||||
return result
|
||||
except IntegrityError as exc:
|
||||
if getattr(exc.orig, 'sqlstate', None) == '23503':
|
||||
raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def _chat(session, chat_id, actor_id):
|
||||
chat = session.scalar(select(Chat).join(ChatMember).where(
|
||||
Chat.id == chat_id, ChatMember.user_id == actor_id))
|
||||
if chat is None:
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
return chat
|
||||
|
||||
@staticmethod
|
||||
def _members(session, chat):
|
||||
result = serialize(chat, *CHAT_FIELDS)
|
||||
result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where(
|
||||
ChatMember.chat_id == chat.id).order_by(ChatMember.user_id)))
|
||||
return result
|
||||
|
||||
def get_chat_for_user(self, chat_id, actor_id):
|
||||
with self.sessions() as session:
|
||||
return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id))
|
||||
|
||||
def list_chats(self, actor_id, limit, cursor):
|
||||
after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None
|
||||
query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id)
|
||||
if after:
|
||||
query = query.where(after_condition(Chat, after))
|
||||
query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1)
|
||||
with self.sessions() as session:
|
||||
chats = list(session.scalars(query))
|
||||
items = [self._members(session, chat) for chat in chats[:limit]]
|
||||
return self._page(items, len(chats) > limit, 'chats', actor_id)
|
||||
|
||||
def create_message(self, chat_id, actor_id, text, idempotency_key=None, attachment_ids=None):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
attachment_ids = [ensure_uuid(item) for item in (attachment_ids or [])]
|
||||
with self.sessions.begin() as session:
|
||||
self._chat(session, chat_id, actor_id)
|
||||
if attachment_ids:
|
||||
found = list(session.scalars(select(Attachment).where(
|
||||
Attachment.id.in_(attachment_ids), Attachment.chat_id == chat_id)
|
||||
.order_by(Attachment.id).with_for_update()))
|
||||
if len(found) != len(attachment_ids):
|
||||
raise APIError(404, 'not_found', 'Объект не найден')
|
||||
if idempotency_key is not None:
|
||||
# Serialize writers to this chat before looking up the key.
|
||||
# The same row is updated below for history cache invalidation.
|
||||
session.execute(select(Chat.id).where(Chat.id == chat_id).with_for_update())
|
||||
record = session.get(IdempotencyRecord, (actor_id, chat_id, idempotency_key))
|
||||
if record is not None:
|
||||
previous = session.get(Message, record.message_id)
|
||||
if previous.text != text or previous.attachment_ids != attachment_ids:
|
||||
raise APIError(409, 'conflict', 'Ключ уже использован для другого сообщения')
|
||||
return serialize(previous, *MESSAGE_FIELDS)
|
||||
if attachment_ids and any(item.status == 'failed' for item in found):
|
||||
raise APIError(409, 'conflict', 'Обработка вложения завершилась ошибкой')
|
||||
session.execute(update(Chat).where(Chat.id == chat_id).values(
|
||||
history_version=Chat.history_version + 1))
|
||||
message = Message(chat_id=chat_id, sender_id=actor_id, text=text, attachment_ids=attachment_ids)
|
||||
session.add(message)
|
||||
session.flush()
|
||||
if idempotency_key is not None:
|
||||
session.add(IdempotencyRecord(actor_id=actor_id, chat_id=chat_id,
|
||||
key=idempotency_key, message_id=message.id))
|
||||
result = serialize(message, *MESSAGE_FIELDS)
|
||||
return result
|
||||
|
||||
def list_messages(self, chat_id, actor_id, limit, cursor):
|
||||
chat_id = ensure_uuid(chat_id)
|
||||
after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None
|
||||
with self.sessions() as session:
|
||||
version = self._chat(session, chat_id, actor_id).history_version
|
||||
return self.cache.get_or_load(
|
||||
chat_id, version, limit, after,
|
||||
lambda: self._load_messages(chat_id, limit, after),
|
||||
)
|
||||
|
||||
def _load_messages(self, chat_id, limit, after):
|
||||
with self.sessions() as session:
|
||||
messages = list(session.scalars(history_query(chat_id, limit, after)))
|
||||
items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]]
|
||||
return self._page(items, len(messages) > limit, 'messages', chat_id)
|
||||
|
||||
def _page(self, items, has_more, resource, owner_id):
|
||||
cursor = None
|
||||
if has_more:
|
||||
last = items[-1]
|
||||
cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id'])
|
||||
return {'items': items, 'next_cursor': cursor}
|
||||
|
||||
@staticmethod
|
||||
def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str:
|
||||
payload = {
|
||||
"resource": resource,
|
||||
"owner_id": owner_id,
|
||||
"created_at": created_at,
|
||||
"id": item_id,
|
||||
}
|
||||
raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
|
||||
|
||||
@staticmethod
|
||||
def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]:
|
||||
try:
|
||||
padded = cursor + "=" * (-len(cursor) % 4)
|
||||
payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8"))
|
||||
except (ValueError, UnicodeError, json.JSONDecodeError) as exc:
|
||||
raise APIError(400, "invalid_request", "Неверный cursor") from exc
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||
|
||||
if payload.get("resource") != resource or payload.get("owner_id") != owner_id:
|
||||
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||
|
||||
item_id = ensure_uuid(payload.get("id"))
|
||||
created_at = payload.get("created_at")
|
||||
if not isinstance(created_at, str) or not created_at:
|
||||
raise APIError(400, "invalid_request", "Неверный cursor")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00"))
|
||||
if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc:
|
||||
raise ValueError("Expected UTC timestamp")
|
||||
created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
except ValueError as exc:
|
||||
raise APIError(400, "invalid_request", "Неверный cursor") from exc
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
store = PostgresStore()
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Bounded consumers, one unacknowledged message per consumer, hard process timeout."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from .database import create_db_engine
|
||||
from .jobs import GROUP, STREAM, LEASE_SECONDS, TIMEOUT, broker, claim, ensure_group, record_failure
|
||||
|
||||
|
||||
def handle(sessions, client, message_id, fields):
|
||||
try:
|
||||
claimed = claim(sessions, json.loads(fields['job']))
|
||||
except (ValueError, KeyError, TypeError):
|
||||
logging.error('Invalid broker envelope: id=%s', message_id)
|
||||
claimed = None
|
||||
if claimed:
|
||||
job_id, token = claimed
|
||||
try:
|
||||
result = subprocess.run([sys.executable, '-m', 'src.image_processor', job_id, token],
|
||||
timeout=TIMEOUT, check=False)
|
||||
code = result.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
code = 23
|
||||
if code in (91, 92):
|
||||
# Simulate a lost worker before ack: leave the Redis pending entry and lease.
|
||||
return
|
||||
if code:
|
||||
record_failure(sessions, job_id, token,
|
||||
{20: 'invalid_image', 21: 'resource_limit', 23: 'processing_timeout'}
|
||||
.get(code, 'processing_failed'), permanent=code in (20, 21))
|
||||
# Only after a durable ready/failed/retry (or an existing durable lease).
|
||||
with client.pipeline(transaction=True) as pipe:
|
||||
pipe.xack(STREAM, GROUP, message_id)
|
||||
pipe.xdel(STREAM, message_id)
|
||||
pipe.execute()
|
||||
|
||||
|
||||
def consume(sessions):
|
||||
client = broker()
|
||||
consumer = str(uuid4())
|
||||
cursor = '0-0'
|
||||
while True:
|
||||
try:
|
||||
ensure_group(client)
|
||||
pending = client.xautoclaim(STREAM, GROUP, consumer, LEASE_SECONDS * 1000,
|
||||
start_id=cursor, count=1)
|
||||
cursor, messages = pending[0], pending[1]
|
||||
if not messages:
|
||||
batches = client.xreadgroup(GROUP, consumer, {STREAM: '>'}, count=1, block=1000)
|
||||
messages = batches[0][1] if batches else []
|
||||
for message_id, fields in messages:
|
||||
handle(sessions, client, message_id, fields)
|
||||
except Exception as exc:
|
||||
logging.warning('Worker deferred: %s', type(exc).__name__)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
engine = create_db_engine()
|
||||
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
concurrency = min(4, max(1, int(os.getenv('WORKER_CONCURRENCY', '2'))))
|
||||
with ThreadPoolExecutor(max_workers=concurrency) as pool:
|
||||
for task in [pool.submit(consume, sessions) for _ in range(concurrency)]:
|
||||
task.result()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Additional real-container checks: validation, permanent failure, timeout, retention."""
|
||||
import os
|
||||
|
||||
from pipeline import Client, check, compose, remote, state, until
|
||||
|
||||
|
||||
def main():
|
||||
c = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/attachments'
|
||||
c.call('POST', path + '?filename=bad.png', 415, actor=alice, raw=b'not a PNG',
|
||||
headers={'Content-Type': 'image/png'})
|
||||
c.call('POST', path + '?filename=../bad.png', 400, actor=alice, raw=b'bad',
|
||||
headers={'Content-Type': 'image/png'})
|
||||
try:
|
||||
compose('stop', 'worker', 'cleanup')
|
||||
att, data = c.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
before = state(aid)
|
||||
# A queued record cannot be collected even when its staging is old.
|
||||
remote(f'''
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from types import SimpleNamespace
|
||||
from src.cleanup import cleanup
|
||||
client = s3_client()
|
||||
class SingleObject:
|
||||
def __getattr__(self, name):
|
||||
return getattr(client, name)
|
||||
def get_paginator(self, name):
|
||||
obj = dict(client.head_object(Bucket=BUCKET, Key={before['staging']!r}), Key={before['staging']!r})
|
||||
return SimpleNamespace(paginate=lambda **kw: [{{'Contents': [obj]}}])
|
||||
cleanup(sessions, SingleObject(), now=datetime.now(timezone.utc) + timedelta(days=2))
|
||||
s3_client().head_object(Bucket=BUCKET, Key={before['staging']!r})
|
||||
''')
|
||||
# Force a real checksum failure after upload, without changing the public contract.
|
||||
remote(f"s3_client().put_object(Bucket=BUCKET, Key={before['staging']!r}, Body=b'corrupted')\n")
|
||||
key = 'queued-before-failure-' + aid
|
||||
message = c.message(alice, chat, '', key=key, attachments=[aid])['json']
|
||||
compose('start', 'worker')
|
||||
until(lambda: state(aid)['status'] == 'failed')
|
||||
check(state(aid)['attempt'] == 1 and state(aid)['error'] == 'invalid_image', 'permanent error retried')
|
||||
c.call('GET', '/api/v1/attachments/' + aid + '/content', 409, actor=bob)
|
||||
check(c.message(alice, chat, '', key=key, attachments=[aid])['json'] == message,
|
||||
'failed processing broke message idempotency')
|
||||
# Expire this test job's failed retention; cleanup must remove its staging.
|
||||
remote(f'''
|
||||
from datetime import timedelta
|
||||
from src.jobs import now, replay
|
||||
from src.cleanup import cleanup
|
||||
with sessions.begin() as session:
|
||||
a = session.get(Attachment, {aid!r})
|
||||
a.updated_at = now() - timedelta(days=2)
|
||||
cleanup(sessions, s3_client())
|
||||
try:
|
||||
replay(sessions, {before['job_id']!r}, 'expired staging must reject')
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('Replay accepted missing staging')
|
||||
''')
|
||||
check(state(aid)['staging'] is None, 'failed staging retention did not expire')
|
||||
print('PASS: upload validation, active staging retention, permanent failure, message replay, expired staging', flush=True)
|
||||
|
||||
compose('stop', 'worker')
|
||||
att, _ = c.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
until(lambda: state(aid)['published'])
|
||||
# Substitute a real sleeping child for the decoder, keeping the real timeout/kill path.
|
||||
remote(f'''
|
||||
import subprocess, sys
|
||||
import src.worker as worker
|
||||
from unittest.mock import patch
|
||||
real_run = subprocess.run
|
||||
def stalled(*args, **kwargs):
|
||||
return real_run([sys.executable, '-c', 'import time; time.sleep(30)'], timeout=1)
|
||||
with sessions() as session:
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r}))
|
||||
fields = {{'job': json.dumps(envelope(j))}}
|
||||
with patch.object(worker.subprocess, 'run', stalled):
|
||||
worker.handle(sessions, broker(), '0-0', fields)
|
||||
''')
|
||||
check(state(aid)['status'] == 'queued' and state(aid)['failures'] == 1, 'timeout did not schedule retry')
|
||||
remote(f'''
|
||||
with sessions() as session:
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {aid!r}))
|
||||
assert j.last_error == 'processing_timeout'
|
||||
''')
|
||||
compose('start', 'worker')
|
||||
c.wait_ready(bob, att)
|
||||
check(state(aid)['attempt'] == 2, 'timeout recovery did not complete')
|
||||
print('PASS: real subprocess timeout, termination and successful retry', flush=True)
|
||||
finally:
|
||||
compose('start', 'worker', 'cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,204 @@
|
||||
"""Integration/fault checks for the local lab5 Compose project; restores stopped services."""
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, check
|
||||
|
||||
|
||||
def compose(*args, input=None):
|
||||
result = subprocess.run(['docker', 'compose', *args], cwd=ROOT, input=input,
|
||||
text=True, encoding='utf-8', capture_output=True)
|
||||
if result.returncode:
|
||||
raise AssertionError('Compose failed: ' + ' '.join(args[:3]) + '\n' + result.stderr)
|
||||
return result.stdout
|
||||
|
||||
|
||||
PRELUDE = '''
|
||||
import json
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from src.database import create_db_engine
|
||||
from src.models import Attachment, AttachmentJob
|
||||
from src.jobs import broker, envelope, GROUP, STREAM
|
||||
from src.object_storage import BUCKET, s3_client
|
||||
engine = create_db_engine()
|
||||
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
'''
|
||||
|
||||
|
||||
def remote(code, fault=None):
|
||||
options = ['exec', '-T']
|
||||
if fault:
|
||||
options += ['-e', 'JOB_FAULT=' + fault]
|
||||
return compose(*options, 'api', 'python', '-', input=PRELUDE + code).strip()
|
||||
|
||||
|
||||
def state(attachment_id):
|
||||
return json.loads(remote(f'''
|
||||
with sessions() as session:
|
||||
a = session.get(Attachment, {attachment_id!r})
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == a.id))
|
||||
print(json.dumps(dict(status=a.status, error=a.error_code, job_id=j.id,
|
||||
attempt=j.attempt, failures=j.failures, published=bool(j.published_at),
|
||||
staging=a.staging_key, variants=a.variants, audit=j.replay_audit)))
|
||||
'''))
|
||||
|
||||
|
||||
def until(fn, timeout=130):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
result = fn()
|
||||
if result:
|
||||
return result
|
||||
time.sleep(1)
|
||||
raise AssertionError('Timed out waiting for durable state')
|
||||
|
||||
|
||||
def ready(client):
|
||||
def probe():
|
||||
try:
|
||||
return client.call('GET', '/health/ready')['status'] == 200
|
||||
except Exception:
|
||||
return False
|
||||
until(probe, 40)
|
||||
|
||||
|
||||
def inject(attachment_id, fault):
|
||||
# Use a real consumer-group delivery, leaving the message pending at the crash point.
|
||||
return remote(f'''
|
||||
from src.worker import handle
|
||||
client = broker()
|
||||
with sessions() as session:
|
||||
j = session.scalar(select(AttachmentJob).where(AttachmentJob.attachment_id == {attachment_id!r}))
|
||||
target = j.id
|
||||
for _ in range(100):
|
||||
batches = client.xreadgroup(GROUP, 'fault-test', {{STREAM: '>'}}, count=1, block=1000)
|
||||
if batches:
|
||||
mid, fields = batches[0][1][0]
|
||||
handle(sessions, client, mid, fields)
|
||||
if json.loads(fields['job'])['job_id'] == target:
|
||||
print(mid)
|
||||
break
|
||||
else:
|
||||
raise AssertionError('No delivery for target job')
|
||||
''', fault=fault)
|
||||
|
||||
|
||||
def main(outbox_only=False):
|
||||
client = Client(os.getenv('BASE_URL', 'http://127.0.0.1:8080'))
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
try:
|
||||
compose('stop', 'worker', 'dispatcher', 'cleanup')
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
aid = attachment['id']
|
||||
check(attachment['status'] == 'queued', 'upload must return queued')
|
||||
check(not state(aid)['published'], 'outbox was already published')
|
||||
client.message(alice, chat, '', attachments=[aid])
|
||||
client.call('GET', f'/api/v1/attachments/{aid}/content', 409, actor=bob)
|
||||
# Hard crash after durable COMMIT while dispatcher is stopped: no publish yet.
|
||||
compose('kill', '-s', 'SIGKILL', 'api')
|
||||
compose('start', 'api')
|
||||
ready(client)
|
||||
check(state(aid)['status'] == 'queued', 'API restart lost upload')
|
||||
compose('start', 'dispatcher')
|
||||
until(lambda: state(aid)['published'])
|
||||
compose('restart', 'broker')
|
||||
compose('start', 'worker')
|
||||
result = client.wait_ready(bob, attachment)
|
||||
check({v['name'] for v in result['variants']} == {'thumbnail', 'crop', 'optimized'}, 'missing variants')
|
||||
for variant in result['variants']:
|
||||
download = client.call('GET', f'/api/v1/attachments/{aid}/content?variant=' + variant['name'], actor=bob)
|
||||
check(hashlib.sha256(download['bytes']).hexdigest() == variant['sha256'], 'variant checksum mismatch')
|
||||
check(len(download['bytes']) == variant['size_bytes'], 'variant size mismatch')
|
||||
check(download['headers'].get_content_type() == variant['content_type'], 'variant MIME mismatch')
|
||||
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'original changed')
|
||||
print('PASS: outbox COMMIT/publish gap, stopped worker, API and broker restart', flush=True)
|
||||
if outbox_only:
|
||||
return
|
||||
|
||||
for fault in ('after_objects', 'after_commit'):
|
||||
compose('stop', 'worker')
|
||||
att, _ = client.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
until(lambda: state(aid)['published'])
|
||||
pending_id = inject(aid, fault)
|
||||
snapshot = state(aid)
|
||||
check(snapshot['status'] == ('processing' if fault == 'after_objects' else 'ready'), 'wrong crash state')
|
||||
compose('start', 'worker')
|
||||
until(lambda: state(aid)['status'] == 'ready')
|
||||
until(lambda: remote(f"print(int(not broker().xpending_range(STREAM, GROUP, {pending_id!r}, {pending_id!r}, 1)))\n") == '1')
|
||||
if fault == 'after_objects':
|
||||
check(state(aid)['attempt'] == 2, 'expired lease was not retried')
|
||||
else:
|
||||
time.sleep(2)
|
||||
check(state(aid)['attempt'] == 1, 'ready job was reprocessed')
|
||||
# Force duplicate delivery; stable object keys and metadata must remain identical.
|
||||
before = state(aid)
|
||||
remote(f'''
|
||||
with sessions() as session:
|
||||
j = session.get(AttachmentJob, {before['job_id']!r})
|
||||
broker().xadd(STREAM, {{'job': json.dumps(envelope(j))}})
|
||||
''')
|
||||
time.sleep(2)
|
||||
after = state(aid)
|
||||
check(before['variants'] == after['variants'] and before['attempt'] == after['attempt'], 'duplicate changed result')
|
||||
count = remote(f"print(s3_client().list_objects_v2(Bucket=BUCKET, Prefix='variants/{aid}/')['KeyCount'])\n")
|
||||
check(count == '3', 'redelivery produced extra S3 objects')
|
||||
print('PASS: crash ' + fault + ', redelivery and stable variants', flush=True)
|
||||
|
||||
compose('stop', 'worker')
|
||||
att, content = client.attachment(alice, chat)
|
||||
aid = att['id']
|
||||
until(lambda: state(aid)['published'])
|
||||
# Remove only this test's staging object to cause a real temporary S3 error.
|
||||
staging = state(aid)['staging']
|
||||
remote(f"s3_client().delete_object(Bucket=BUCKET, Key={staging!r})\n")
|
||||
compose('start', 'worker')
|
||||
until(lambda: state(aid)['status'] == 'failed', 150)
|
||||
failed = state(aid)
|
||||
check(failed['error'] == 'retry_exhausted' and failed['attempt'] == 5, 'retry budget incorrect')
|
||||
client.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [aid]}, headers={'Idempotency-Key': aid})
|
||||
registry = compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'failed')
|
||||
check(failed['job_id'] in registry, 'failed job not visible')
|
||||
remote(f'''
|
||||
import base64
|
||||
s3_client().put_object(Bucket=BUCKET, Key={staging!r},
|
||||
Body=base64.b64decode({base64.b64encode(content).decode()!r}), ContentType='image/png')
|
||||
''')
|
||||
compose('exec', '-T', 'api', 'python', '-m', 'src.jobs', 'replay', failed['job_id'],
|
||||
'--reason', 'synthetic test: restored staging')
|
||||
client.wait_ready(bob, att)
|
||||
check(len(state(aid)['audit']) == 1 and state(aid)['attempt'] == 6, 'replay lost audit or attempt')
|
||||
remote(f'''
|
||||
from src.jobs import replay
|
||||
try:
|
||||
replay(sessions, {failed['job_id']!r}, 'must reject ready')
|
||||
except ValueError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError('Replayed a ready job')
|
||||
''')
|
||||
print('PASS: bounded retries, failed registry, explicit audited replay, ready replay rejected', flush=True)
|
||||
|
||||
compose('exec', '-T', 'api', 'python', '-m', 'src.cleanup', '--once')
|
||||
check(state(aid)['staging'] is None, 'completed staging was not cleaned')
|
||||
check(client.call('GET', f'/api/v1/attachments/{aid}/content', actor=bob)['bytes'] == content, 'cleanup deleted original')
|
||||
print('PASS: staging cleanup preserves ready objects', flush=True)
|
||||
finally:
|
||||
compose('start', 'broker', 'dispatcher', 'worker', 'cleanup')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import argparse
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--outbox-only', action='store_true', help='Only the COMMIT/publish crash scenario')
|
||||
main(parser.parse_args().outbox_only)
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Executed via Docker exec stdin. Prints only sanitized measurements as JSON."""
|
||||
import argparse
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
sys.path.insert(0, '/app')
|
||||
from PIL import Image
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
from src.database import create_db_engine
|
||||
from src.models import Attachment, AttachmentJob
|
||||
from src.object_storage import BUCKET, s3_client
|
||||
|
||||
BASE = 'http://nginx:8080'
|
||||
|
||||
|
||||
def call(method, path, body=None, token=None, raw=None, headers=None):
|
||||
headers = dict(headers or {})
|
||||
if token:
|
||||
headers['Authorization'] = 'Bearer ' + token
|
||||
if body is not None:
|
||||
raw = json.dumps(body).encode()
|
||||
headers['Content-Type'] = 'application/json'
|
||||
request = urllib.request.Request(BASE + path, data=raw, headers=headers, method=method)
|
||||
try:
|
||||
response = urllib.request.urlopen(request, timeout=60)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
data = response.read()
|
||||
payload = json.loads(data) if response.headers.get_content_type() == 'application/json' else data
|
||||
return response.status, payload, response.headers
|
||||
|
||||
|
||||
def bootstrap():
|
||||
username, password = 'q_' + uuid4().hex[:20], uuid4().hex
|
||||
status, user, _ = call('POST', '/api/v1/users', body={
|
||||
'username': username, 'password': password, 'display_name': 'Synthetic queue experiment'})
|
||||
assert status == 201, ('user', status)
|
||||
basic = base64.b64encode(f'{username}:{password}'.encode()).decode()
|
||||
status, session, _ = call('POST', '/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + basic})
|
||||
assert status == 201, ('session', status)
|
||||
token = session['token']
|
||||
status, chat, _ = call('POST', '/api/v1/chats', token=token,
|
||||
body={'title': 'Synthetic queue experiment', 'member_ids': [user['id']]})
|
||||
assert status == 201, ('chat', status)
|
||||
return user['id'], chat['id'], token
|
||||
|
||||
|
||||
def fixture(width, height):
|
||||
pixels = random.Random(202605).randbytes(width * height * 3)
|
||||
with Image.frombytes('RGB', (width, height), pixels) as picture, io.BytesIO() as output:
|
||||
picture.save(output, format='JPEG', quality=85)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
def percentile(values):
|
||||
return sorted(values)[max(0, math.ceil(len(values) * 0.95) - 1)]
|
||||
|
||||
|
||||
def rss_mib():
|
||||
# Sum process RSS, not cgroup memory.current (which also counts page cache).
|
||||
# Shared pages may be counted twice. Exclude this measurement process.
|
||||
total = 0
|
||||
for path in Path('/proc').glob('[0-9]*/status'):
|
||||
if path.parent.name == str(os.getpid()):
|
||||
continue
|
||||
try:
|
||||
for line in path.read_text().splitlines():
|
||||
if line.startswith('VmRSS:'):
|
||||
total += int(line.split()[1])
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
pass
|
||||
return total / 1024
|
||||
|
||||
|
||||
def rows_for(sessions, owner_id):
|
||||
with sessions() as session:
|
||||
return list(session.execute(select(Attachment, AttachmentJob).join(
|
||||
AttachmentJob, AttachmentJob.attachment_id == Attachment.id)
|
||||
.where(Attachment.owner_id == owner_id)))
|
||||
|
||||
|
||||
def summary(rows):
|
||||
states = {key: 0 for key in ('queued', 'processing', 'ready', 'failed')}
|
||||
ages = []
|
||||
for attachment, _ in rows:
|
||||
states[attachment.status] += 1
|
||||
if attachment.status == 'queued':
|
||||
ages.append((datetime.now(timezone.utc) - attachment.created_at).total_seconds())
|
||||
return {**states, 'oldest_queued_s': max(ages, default=0)}
|
||||
|
||||
|
||||
def staging_keys():
|
||||
return {obj['Key'] for page in s3_client().get_paginator('list_objects_v2').paginate(
|
||||
Bucket=BUCKET, Prefix='staging/') for obj in page.get('Contents', [])}
|
||||
|
||||
|
||||
def upload_batch(args, sessions):
|
||||
owner, chat, token = bootstrap()
|
||||
data = fixture(args.width, args.height)
|
||||
before_staging = staging_keys() if args.mode == 'overload' else set()
|
||||
samples = []
|
||||
started = time.monotonic()
|
||||
|
||||
def upload(_):
|
||||
return call('POST', f'/api/v1/chats/{chat}/attachments?filename=load.jpg', token=token,
|
||||
raw=data, headers={'Content-Type': 'image/jpeg'})
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.clients) as pool:
|
||||
futures = [pool.submit(upload, i) for i in range(args.count)]
|
||||
while True:
|
||||
rows = rows_for(sessions, owner)
|
||||
snapshot = summary(rows)
|
||||
samples.append({'elapsed_s': round(time.monotonic() - started, 3),
|
||||
'rss_mib': round(rss_mib(), 3), **snapshot})
|
||||
if all(f.done() for f in futures):
|
||||
responses = [f.result() for f in futures]
|
||||
accepted = [payload for status, payload, _ in responses if status == 202]
|
||||
if args.mode == 'overload':
|
||||
rows = rows_for(sessions, owner)
|
||||
snapshot = summary(rows)
|
||||
if args.mode == 'overload' or snapshot['ready'] + snapshot['failed'] == len(accepted):
|
||||
break
|
||||
if time.monotonic() - started > 300:
|
||||
raise AssertionError('Batch did not drain in 300 seconds')
|
||||
time.sleep(0.1)
|
||||
elapsed = time.monotonic() - started
|
||||
codes = {str(code): sum(status == code for status, _, _ in responses)
|
||||
for code in sorted({status for status, _, _ in responses})}
|
||||
replicas = len({headers['X-Instance-Id'] for _, _, headers in responses})
|
||||
result = {'width': args.width, 'height': args.height, 'bytes': len(data),
|
||||
'sha256': hashlib.sha256(data).hexdigest(), 'requests': args.count,
|
||||
'clients': args.clients, 'http_statuses': codes, 'api_replicas_seen': replicas,
|
||||
'elapsed_s': elapsed, 'owner_id': owner, 'attachment_ids': [a['id'] for a in accepted]}
|
||||
assert len(rows) == len(accepted), 'Rejected upload created metadata'
|
||||
assert replicas == 2, 'Experiment must exercise both API replicas'
|
||||
if args.mode == 'overload':
|
||||
assert len(accepted) == args.limit, (codes, args.limit)
|
||||
assert codes.get('429') == args.count - args.limit, codes
|
||||
assert all(a.status == 'queued' for a, _ in rows), 'Worker was not stopped'
|
||||
assert staging_keys() - before_staging == {'staging/' + a['id'] for a in accepted}, 'Rejected upload left S3 object'
|
||||
for status, payload, headers in responses:
|
||||
if status == 429:
|
||||
assert payload['error']['code'] == 'rate_limited'
|
||||
assert int(headers['Retry-After']) > 0
|
||||
assert call('GET', '/health/ready')[0] == 200
|
||||
assert call('GET', f"/api/v1/attachments/{accepted[0]['id']}/content", token=token)[0] == 409
|
||||
assert call('POST', f'/api/v1/chats/{chat}/messages', token=token,
|
||||
body={'text': 'Still responsive under overload'},
|
||||
headers={'Idempotency-Key': uuid4().hex})[0] == 201
|
||||
result.update({'limit': args.limit, 'accepted': len(accepted), 'rejected': codes['429'],
|
||||
'health_and_messages_available': True, 'rejected_leave_no_objects': True})
|
||||
else:
|
||||
assert codes == {'202': args.count}, codes
|
||||
assert all(a.status == 'ready' and j.attempt == 1 for a, j in rows), 'Failed/retried job in baseline'
|
||||
waits = [(j.first_started_at - j.created_at).total_seconds() for _, j in rows]
|
||||
latencies = [(a.updated_at - a.created_at).total_seconds() for a, _ in rows]
|
||||
result.update({'throughput_per_s': len(rows) / elapsed,
|
||||
'ready_p95_s': percentile(latencies), 'queue_wait_p95_s': percentile(waits),
|
||||
'queue_lag_max_jobs': max(s['queued'] for s in samples),
|
||||
'queue_oldest_max_s': max(s['oldest_queued_s'] for s in samples),
|
||||
'processing_max': max(s['processing'] for s in samples),
|
||||
'rss_peak_mib': max(s['rss_mib'] for s in samples),
|
||||
'samples': samples})
|
||||
return result
|
||||
|
||||
|
||||
def recover(args, sessions):
|
||||
expected = json.loads(args.expected)
|
||||
started = time.monotonic()
|
||||
while time.monotonic() - started < 180:
|
||||
rows = rows_for(sessions, expected['owner_id'])
|
||||
assert not any(a.status == 'failed' for a, _ in rows), 'Accepted job failed'
|
||||
if len(rows) == expected['accepted'] and all(a.status == 'ready' for a, _ in rows):
|
||||
break
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise AssertionError('Accepted work did not recover')
|
||||
assert {a.id for a, _ in rows} == set(expected['attachment_ids']), 'Lost/replaced accepted IDs'
|
||||
for attachment, _ in rows:
|
||||
keys = [attachment.object_key] + [v['object_key'] for v in attachment.variants]
|
||||
assert len(attachment.variants) == 3
|
||||
for key in keys:
|
||||
s3_client().head_object(Bucket=BUCKET, Key=key)
|
||||
with s3_client().get_object(Bucket=BUCKET, Key=attachment.object_key)['Body'] as body:
|
||||
assert hashlib.sha256(body.read()).hexdigest() == expected['sha256']
|
||||
elapsed = time.monotonic() - started
|
||||
# A new accepted upload proves capacity is released after draining.
|
||||
# One request need not hit both replicas, so perform this explicitly.
|
||||
owner, chat, token = bootstrap()
|
||||
status, _, _ = call('POST', f'/api/v1/chats/{chat}/attachments?filename=after.jpg', token=token,
|
||||
raw=fixture(320, 200), headers={'Content-Type': 'image/jpeg'})
|
||||
assert status == 202, 'Admission did not recover'
|
||||
deadline = time.monotonic() + 60
|
||||
while time.monotonic() < deadline:
|
||||
fresh = rows_for(sessions, owner)
|
||||
if fresh and fresh[0][0].status == 'ready':
|
||||
break
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise AssertionError('New upload did not complete after overload')
|
||||
return {'accepted_ids_preserved': True, 'all_ready': len(rows), 'original_checksums_match': True,
|
||||
'all_variants_exist': True, 'drain_observed_s': elapsed, 'new_upload_status': status}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('mode', choices=['measure', 'overload', 'recover'])
|
||||
parser.add_argument('--width', type=int, default=320)
|
||||
parser.add_argument('--height', type=int, default=200)
|
||||
parser.add_argument('--count', type=int, default=18)
|
||||
parser.add_argument('--clients', type=int, default=6)
|
||||
parser.add_argument('--limit', type=int, default=12)
|
||||
parser.add_argument('--expected')
|
||||
args = parser.parse_args()
|
||||
engine = create_db_engine()
|
||||
try:
|
||||
sessions = sessionmaker(engine, expire_on_commit=False)
|
||||
result = recover(args, sessions) if args.mode == 'recover' else upload_batch(args, sessions)
|
||||
print(json.dumps(result))
|
||||
finally:
|
||||
engine.dispose()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,61 @@
|
||||
"""Run inside the application image: python -m unittest discover -s /checks."""
|
||||
import hashlib
|
||||
import io
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from src.image_processor import variants
|
||||
|
||||
|
||||
class ImageTests(unittest.TestCase):
|
||||
def render(self, image, fmt='PNG', **kwargs):
|
||||
source = io.BytesIO()
|
||||
image.save(source, format=fmt, **kwargs)
|
||||
results = list(variants(source.getvalue()))
|
||||
self.assertEqual([meta['name'] for meta, _ in results], ['thumbnail', 'crop', 'optimized'])
|
||||
decoded = {}
|
||||
for meta, data in results:
|
||||
self.assertEqual(meta['sha256'], hashlib.sha256(data).hexdigest())
|
||||
self.assertEqual(meta['size_bytes'], len(data))
|
||||
with Image.open(io.BytesIO(data)) as item:
|
||||
item.load()
|
||||
self.assertEqual(item.size, (meta['width'], meta['height']))
|
||||
self.assertFalse(item.getexif())
|
||||
self.assertNotIn('icc_profile', item.info)
|
||||
decoded[meta['name']] = item.copy()
|
||||
return decoded
|
||||
|
||||
def test_dimensions_and_floor_rounding(self):
|
||||
images = self.render(Image.new('RGB', (321, 200), 'red'))
|
||||
self.assertEqual(images['thumbnail'].size, (256, 159))
|
||||
self.assertEqual(images['crop'].size, (128, 128))
|
||||
self.assertEqual(images['optimized'].size, (321, 200))
|
||||
|
||||
def test_no_upscale_and_white_alpha(self):
|
||||
images = self.render(Image.new('RGBA', (20, 10), (255, 0, 0, 0)))
|
||||
self.assertEqual(images['thumbnail'].size, (20, 10))
|
||||
self.assertEqual(images['optimized'].size, (20, 10))
|
||||
self.assertEqual(images['thumbnail'].getpixel((5, 5)), (255, 255, 255))
|
||||
|
||||
def test_exif_orientation_and_metadata_removal(self):
|
||||
exif = Image.Exif()
|
||||
exif[274] = 6
|
||||
exif[270] = 'private description'
|
||||
images = self.render(Image.new('RGB', (320, 200), 'blue'), 'JPEG', exif=exif)
|
||||
self.assertEqual(images['thumbnail'].size, (160, 256))
|
||||
self.assertEqual(images['optimized'].size, (200, 320))
|
||||
|
||||
def test_optimized_bound(self):
|
||||
images = self.render(Image.new('RGB', (2000, 1000), 'green'))
|
||||
self.assertEqual(images['optimized'].size, (1280, 640))
|
||||
|
||||
def test_worker_pixel_limit(self):
|
||||
with patch('src.image_processor.MAX_PIXELS', 100):
|
||||
with self.assertRaises(ValueError):
|
||||
self.render(Image.new('RGB', (11, 10)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user