ЛР 5 — БПИ2303 — Лукьянов — на 4 #43

Open
xabenskie wants to merge 1 commits from xabenskie/mtusi2026lab5:solution/bpi2303-lukyanov into main
65 changed files with 4553 additions and 42 deletions
+3
View File
@@ -18,3 +18,6 @@ __pycache__
node_modules
evidence
.DS_Store
.superpowers
.pytest_cache
+4 -24
View File
@@ -1,25 +1,5 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# bootstrap creates .env and private files in secrets/; never commit local values.
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
BROKER_URL_FILE=/run/secrets/broker_url
WORKER_CONCURRENCY=2
JOB_MAX_ATTEMPTS=5
JOB_TIMEOUT_SECONDS=60
STAGING_RETENTION_SECONDS=86400
# Compose configures DATABASE_URL_FILE/CACHE_URL_FILE/S3_*_FILE for the application.
# Broker is separate from the optional profile cache; persistence AOF + appendfsync always.
# Processing limits: one worker, prefetch 1, decoder timeout 20s, lease 90s, 5 attempts.
+24
View File
@@ -0,0 +1,24 @@
name: Application acceptance
on: [push, pull_request]
permissions:
contents: read
jobs:
pipeline:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- run: make up
- run: make check validate-container test test-student test-fresh
- run: make test-jobs
- run: make test-cluster
- uses: actions/upload-artifact@v4
if: always()
with:
name: lab5-evidence
path: evidence/
- if: always()
run: docker compose down
+18
View File
@@ -0,0 +1,18 @@
FROM python:3.13.5-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
WORKDIR /app
COPY requirements.txt requirements-dev.txt requirements.lock ./
RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger
COPY src/ src/
COPY migrations/ migrations/
COPY student-tests/ student-tests/
COPY scripts/ scripts/
COPY tests/ tests/
COPY contracts/ contracts/
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
ARG APP_RELEASE=lab5-v1
ENV APP_RELEASE=${APP_RELEASE}
LABEL org.opencontainers.image.version=${APP_RELEASE}
USER messenger
EXPOSE 8080
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
+35
View File
@@ -28,3 +28,38 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: bootstrap up down test-student test-cluster rolling
bootstrap:
./scripts/bootstrap.sh
up: bootstrap
docker compose build api minio minio-init
docker compose up -d --wait --wait-timeout 120
$(PYTHON) scripts/cluster.py wait
down:
docker compose down
test-student:
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
test-cluster:
$(PYTHON) scripts/verify_cluster.py
rolling:
$(PYTHON) scripts/cluster.py rolling
.PHONY: test-fresh load-evidence verify-persistence verify-failover
test-fresh:
./scripts/check_fresh_migrations.sh
load-evidence:
$(PYTHON) scripts/capture_load.py
verify-persistence:
$(PYTHON) scripts/verify_cluster.py persistence
verify-failover:
$(PYTHON) scripts/verify_cluster.py public-failover
$(PYTHON) scripts/verify_cluster.py failover
.PHONY: validate-container
validate-container:
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
.PHONY: test-jobs
test-jobs:
$(PYTHON) scripts/verify_jobs.py
+86 -18
View File
@@ -1,37 +1,105 @@
# Отчёт — лабораторная 5
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Лукьянов К.Р. БПИ2303
- Целевая оценка: **4**; трек 5 не заявлен.
- Commit/PR: заполняется автором после самостоятельного commit/push.
- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, boto3/botocore, Pillow, pytest; [Redis Streams](https://redis.io/docs/latest/develop/data-types/streams/), [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), [Nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html). Реализация выполнена совместно мной и Codex. Часть кода, тестов и отчёта написана с помощью Codex; результат проверен командами ниже.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Docker с Compose v2, Make, Python 3.9+ на хосте, свободный localhost:8080. Все Python-зависимости и Go для MinIO устанавливаются в сборочных контейнерах. Первая сборка требует доступа к registry, GitHub и пакетным репозиториям. Стенды предыдущих лабораторных следует остановить, сохранив volumes.
```sh
git clone <fork-url>
cd <fork-lab5>
make up
curl -fsS http://localhost:8080/health/ready
make check
make validate-container
make test
make test-student
make test-fresh
make test-jobs
make test-cluster
```
`make up` создаёт локальные secrets, собирает API и MinIO/mc, запускает миграции и S3 bootstrap, затем две API, Nginx, отдельные broker/relay/worker. Версии образов и лимиты закреплены в compose.yaml, Python-зависимости — requirements.lock. MinIO и mc собираются из официальных release-исходников с проверкой commit: готовые образы/архив при выполнении lab4 были недоступны. Go на хосте не нужен. MinIO: RELEASE.2025-04-22T22-12-26Z, mc: RELEASE.2025-04-16T18-13-26Z; исходники и проверки — infra/minio/Dockerfile.
`make down` сохраняет volumes. `docker compose down --volumes` удаляет все данные текущего учебного проекта. `make test-fresh` использует отдельный случайный Compose project и удаляет только его временные volumes. Сценарии отказов запускать последовательно, они меняют состояние контейнеров. Синтетические записи обычных тестов остаются в БД.
Операторский replay без публичного административного endpoint:
```sh
# job_id получить локально из attachment_jobs; токен пользователя не нужен.
docker compose exec -T worker python -m messenger.worker replay \
--job-id <uuid> --reason 'Причина после устранения неисправности'
```
Разрешён только failed с сохранённым staging. Ready replay отклоняется; audit пишется в job_replays. После retention источника replay явно запрещён.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
Клиент → Nginx → API×2 → MinIO staging + PostgreSQL metadata/job/outbox
↓ relay
Redis Streams broker
↓ worker
MinIO originals + thumbnail/crop/WebP → PG ready
```
Из lab4 перенесены API, auth/ACL, PostgreSQL, Redis-кэш профилей, Nginx, общие сессии, keyset и идемпотентность сообщений. Контракты и публичные tests текущей лабы сохранены. Объявленный переход: upload возвращает **202 queued**, content для queued/processing/failed — **409**, ready доступен через GET metadata. Сообщение может ссылаться на queued/processing, failed прикреплять нельзя.
**Граница 202.** Тело ограничено 10 MiB и 40 млн пикселей. Проверяются MIME, структура и имя. Resize и полная декодировка выполняются worker. API сначала PUT в приватный `staging/<uuid>`, затем в одной PG-транзакции создаёт attachment, job и outbox; только после COMMIT возвращает снимок queued. S3 и PG не образуют общей транзакции. Компенсация повторно проверяет metadata под lock, при недоступной БД слепого удаления нет. Неучтённый staging подберёт grace-cleanup.
**Outbox.** Relay выбирает due записи под `FOR UPDATE SKIP LOCKED`, делает XADD и сохраняет dispatched_at. Сбой до XADD оставляет намерение; после XADD до COMMIT допускается дубль. Сообщение соответствует jobs.schema.json: постоянный job_id, attachment_id, schema/type, pipeline_version=1, attempt, created_at, опциональный W3C carrier. Байтов, storage URL и credentials в сообщении нет.
**Устойчивость брокера.** Отдельный Redis 7.4.4 с volume, AOF и appendfsync always, noeviction. Redis-кэш профилей остаётся отдельным сервисом. Consumer group обеспечивает at least once; prefetch 1. XACK и затем XDEL выполняются после COMMIT результата либо устойчивого решения о retry/failed. Безусловного MAXLEN нет. Publisher acceptance не равен consumer ack. Успешный job остаётся в PG для проверки дублей.
**Обработка и варианты.** Worker получает доверенные metadata из PG, сверяет staging SHA-256/размер. Детерминированные ключи оригинала и `(attachment,pipeline=1,variant)` исключают размножение вариантов. EXIF orientation применяется, alpha компонуется на белом, EXIF удаляется. Thumbnail: JPEG quality85, fit 256×256 без увеличения, floor/min1. Crop: центрированный квадрат 128×128 JPEG quality85, допускает увеличение. Optimized: WebP quality80, fit 1280×1280 без увеличения; уменьшение любого маленького файла не обещается. Оригинал побайтно сохраняется. Все варианты записываются до одной транзакции ready/checksums.
**Попытки и блокировки.** PG session advisory lock удерживается через весь эффект. Lease 90s, attempt проверяется перед COMMIT. Relay восстанавливает expired processing только после получения того же lock; живой worker не вытесняется. SQL-ошибка сначала rollback, затем unlock; при неизвестном результате освобождения соединение закрывается, а не возвращается в pool. Дубль ready — no-op, старый attempt не расходует новую попытку. Постоянная ошибка изображения даёт failed, временная — retry. По умолчанию максимум 5 попыток, exponential backoff с jitter, верхняя граница задержки зависит от номера попытки и ограничена конфигурацией алгоритма.
**Ограничение ресурсов.** Worker — 1 CPU/512 MiB, concurrency/prefetch 1. Весь цикл S3+decode+PUT выполняется в отдельной process group с абсолютным deadline 45s; supervisor убивает и дожидается всей группы до освобождения job-lock. Дочерний processor имеет собственный 45s alarm на случай смерти supervisor. Decoder дополнительно ограничен wall time 20s, CPU 15s и RLIMIT_AS 768 MiB; фактический общий container memory limit — 512 MiB. Временный каталог принадлежит supervisor и удаляется после завершения/timeout. Ограничения могут закончить особенно тяжёлый допустимый по пикселям файл контролируемым failed; unlimited RAM не обещается.
**Очистка.** Staging удаляется после ready COMMIT; ключ сохраняется в metadata до успешного DeleteObject, relay повторяет очистку. Для failed источник хранится 24h, затем удаляется и replay запрещается. Grace 300s для неучтённых объектов; lock и свежая проверка PG сохраняют активные uploads, учтённые originals/variants/staging. БД недоступна — удаления нет. Частичные детерминированные варианты failed сохраняются до операторского решения; полного reconciler/retention всех вложений нет.
MinIO приватен, console не опубликована. API/worker получают ограниченный S3 user для originals/staging/variants; root secrets только у MinIO/init. Единственный бизнес-порт — localhost:8080 Nginx. Брокер, API и хранилища не публикуют host ports. Приложение/worker non-root; secrets генерируются локально, не входят в Git/build context.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Прогоны 2026-11-09 на Docker Desktop/macOS Apple Silicon. Ветка `solution/bpi2303-lukyanov`; commits/push выполняет автор.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Проверка | Команда / доказательство | Результат |
| --- | --- | --- |
| Публичный контракт и API | make check, validate-container, test; evidence/check.txt, openapi.txt, smoke.txt | PASS |
| Регрессии, варианты, ACL, лимиты и SQL rollback | make test-student; evidence/student-tests.txt | 27 тестов PASS |
| Пустые volumes и повторные миграции 001–004 | make test-fresh; evidence/fresh-migrations.txt | PASS, 27 тестов |
| COMMIT без publish, worker stop, API/broker restart | make test-jobs; evidence/jobs-faults.json | та же принятая job → ready |
| Устойчивая непрочитанная очередь | evidence/jobs-faults.json | broker restart с volume, затем ready |
| PUT до COMMIT/ack и lease recovery | evidence/jobs-faults.json | S3-эффект есть до kill, та же job завершена со второй попытки |
| Дубль ready | собственный тест + evidence/jobs-faults.json | metadata неизменны, ready replay запрещён |
| Ограниченный retry/failed/replay | evidence/jobs-faults.json | 5 ошибок → retry_exhausted, failed attachment → 409, явный replay → ready |
| Session lock после SQL-ошибки | test_replay_sql_failure_releases_session_lock | независимая PG-сессия получает lock |
| Абсолютный timeout S3 | test_deadline_covers_slow_s3 | drip-feed не удерживает pipeline за deadline |
| Сохранённые restart/failover/rolling | make test-cluster; evidence/*.json | результаты отдельных сценариев в evidence |
В pytest остаются предупреждения botocore об utcnow и диагностический Pillow warning теста pixel-bomb; assertions проходят. Публичные тесты не ослаблялись. RED перед реализацией: публичный lab5 smoke против lab4 ожидал 202 и получил 201 — evidence/lab4-baseline-red.txt.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
`make test-jobs` реально останавливает relay/worker, загружает файл и создаёт сообщение, подтверждает 409 до готовности, перезапускает API и broker, затем запускает обработку. Outbox до восстановления имеет dispatched_at=null; новый job вручную не создаётся. Во втором опыте XADD уже выполнен, worker остановлен, broker перезапускается с volume: непрочитанная задача сохраняется.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
Crash после side effect: BEFORE UPDATE trigger на конкретном attachment задерживает ready COMMIT через pg_sleep(2). После наблюдения PgSleep и существования оригинала в S3 worker получает SIGKILL. Trigger удаляется, lease истекает реально, без подмены часов. Зафиксировано восстановление примерно через **90.804s**, attempt=2, прежний job_id. Затем повтор ready возвращает no-op и прежние metadata.
Для проверки предела повторов удаляется staging синтетической работы. Пять реальных попыток чтения S3 завершаются failed/retry_exhausted. В этом тесте next_at явно переносится на сейчас между попытками, чтобы отдельно проверить счётчик и финальное состояние; измерением реального времени backoff этот опыт не является. После восстановления исходных байтов операторский replay пишет audit, та же job доходит до ready. Failed-вложение нельзя привязать к сообщению.
При code review найден и воспроизведён leak session-lock после SQL-ошибки в replay. Регрессия была RED; после rollback-before-unlock — GREEN. Второй RED→GREEN проверяет медленный HTTP/S3 поток, который не нарушает socket inactivity timeout: теперь общий deadline убивает обработку целиком. Это ограничивает зависший processing, а не просто прекращает ожидание фонового потока.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host, одна PG и один MinIO/broker; отказ диска/узла и backup/HA не покрываются.
- At least once и идемпотентный эффект не означают exactly once. Один pipeline_version=1; конкуренция версий — трек 5, не заявлен.
- Redis AOF always повышает устойчивость на данном volume, но не гарантирует переживание потери самого volume. Не применяется небезопасное trimming непрочитанных jobs.
- Непривязанные учтённые вложения и частичные конечные варианты failed не удаляются автоматически. Failed staging хранится 24h; после удаления источник нельзя восстановить replay.
- API принимает PNG/JPEG в заданных пределах; наиболее тяжёлый файл может получить failed из-за ограничений worker. HTTP/Basic разрешён только в локальном изолированном учебном стенде с синтетическими данными.
- В lab6 добавляются наблюдаемость и alerts, в lab7 — TLS/mTLS и OpenBao. Исследовательские треки 5 не выполнялись.
+173
View File
@@ -0,0 +1,173 @@
name: mtusi-lab5
services:
postgres:
image: postgres:17.5-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 1.0
redis:
image: redis:7.4.4-bookworm
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
volumes: [redis_data:/data]
networks: [data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 128m
cpus: 0.25
migrate:
image: mtusi-lab5-api:lab5
command: [python, -m, messenger.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres: {condition: service_healthy}
restart: 'no'
mem_limit: 256m
cpus: 0.5
api:
image: mtusi-lab5-api:lab5
build: .
scale: 2
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
POOL_MAX_SIZE: '8'
S3_ENDPOINT: http://minio:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
networks: [front, data]
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
minio-init: {condition: service_completed_successfully}
stop_grace_period: 10s
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
interval: 2s
timeout: 5s
retries: 30
mem_limit: 512m
cpus: 1.0
minio:
image: mtusi-lab5-minio:2025-04-22
build:
context: .
dockerfile: infra/minio/Dockerfile
target: minio
secrets: [minio_root_user, minio_root_password]
volumes: [minio_data:/data]
networks: [data]
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"]
interval: 2s
timeout: 3s
retries: 30
stop_grace_period: 10s
mem_limit: 512m
cpus: 1.0
minio-init:
image: mtusi-lab5-mc:2025-04-16
build:
context: .
dockerfile: infra/minio/Dockerfile
target: mc
secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key]
volumes:
- ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro
- ./infra/minio/api-policy.json:/config/api-policy.json:ro
networks: [data]
depends_on:
minio: {condition: service_healthy}
restart: 'no'
mem_limit: 128m
cpus: 0.25
broker:
image: redis:7.4.4-bookworm
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction]
volumes: [broker_data:/data]
networks: [data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 160m
cpus: 0.5
worker: &background
image: mtusi-lab5-api:lab5
command: [python, -m, messenger.worker, worker]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
S3_ENDPOINT: http://minio:9000
S3_REGION: us-east-1
S3_BUCKET: messenger-attachments
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
BROKER_URL: redis://broker:6379/0
JOB_MAX_ATTEMPTS: '5'
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
networks: [data]
depends_on:
broker: {condition: service_healthy}
migrate: {condition: service_completed_successfully}
minio-init: {condition: service_completed_successfully}
stop_grace_period: 45s
restart: unless-stopped
mem_limit: 512m
cpus: 1.0
relay:
<<: *background
command: [python, -m, messenger.worker, relay]
mem_limit: 192m
cpus: 0.25
lb:
image: nginx:1.28.0-alpine
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
networks: [front]
ports: ['127.0.0.1:8080:8080']
depends_on:
api: {condition: service_healthy}
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 1s
timeout: 8s
retries: 30
mem_limit: 128m
cpus: 0.5
networks:
front:
data:
internal: true
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
minio_root_user: {file: ./secrets/minio_root_user}
minio_root_password: {file: ./secrets/minio_root_password}
s3_access_key_id: {file: ./secrets/s3_access_key_id}
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
volumes:
postgres_data:
redis_data:
minio_data:
broker_data:
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"parallel_requests": 8,
"observed_instances": [
"2836440f76e24486ab068d4c71498d15",
"77545b2c0222441193411736edbf54ff"
],
"responses_201": 8,
"messages_created": 1,
"different_body_status": 409
}
+20
View File
@@ -0,0 +1,20 @@
{
"result": "PASS",
"redis": {
"profile_status": 200,
"fallback_seconds": 2.5178,
"anonymous": 401,
"readiness": 200
},
"postgres": {
"authenticated_status": 503,
"response_seconds": 0.006,
"anonymous_status": 401,
"internal_api_liveness": [
200,
200
],
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
},
"same_session_after_recovery": true
}
+70
View File
@@ -0,0 +1,70 @@
./scripts/check_fresh_migrations.sh
Volume mtusi-lab5-migration-check-53406_redis_data Creating
Network mtusi-lab5-migration-check-53406_data Creating
Volume mtusi-lab5-migration-check-53406_postgres_data Creating
Volume mtusi-lab5-migration-check-53406_minio_data Creating
Volume mtusi-lab5-migration-check-53406_redis_data Creating
Network mtusi-lab5-migration-check-53406_data Creating
Volume mtusi-lab5-migration-check-53406_postgres_data Creating
Volume mtusi-lab5-migration-check-53406_minio_data Creating
Volume mtusi-lab5-migration-check-53406_postgres_data Created
Volume mtusi-lab5-migration-check-53406_postgres_data Created
Volume mtusi-lab5-migration-check-53406_redis_data Created
Volume mtusi-lab5-migration-check-53406_redis_data Created
Volume mtusi-lab5-migration-check-53406_minio_data Created
Volume mtusi-lab5-migration-check-53406_minio_data Created
Network mtusi-lab5-migration-check-53406_data Created
Network mtusi-lab5-migration-check-53406_data Created
Container mtusi-lab5-migration-check-53406-redis-1 Creating
Container mtusi-lab5-migration-check-53406-minio-1 Creating
Container mtusi-lab5-migration-check-53406-postgres-1 Creating
Container mtusi-lab5-migration-check-53406-minio-1 Created
Container mtusi-lab5-migration-check-53406-minio-init-1 Creating
Container mtusi-lab5-migration-check-53406-redis-1 Created
Container mtusi-lab5-migration-check-53406-postgres-1 Created
Container mtusi-lab5-migration-check-53406-minio-init-1 Created
Container mtusi-lab5-migration-check-53406-minio-1 Starting
Container mtusi-lab5-migration-check-53406-postgres-1 Starting
Container mtusi-lab5-migration-check-53406-redis-1 Starting
Container mtusi-lab5-migration-check-53406-minio-1 Started
Container mtusi-lab5-migration-check-53406-minio-1 Waiting
Container mtusi-lab5-migration-check-53406-postgres-1 Started
Container mtusi-lab5-migration-check-53406-redis-1 Started
Container mtusi-lab5-migration-check-53406-minio-1 Healthy
Container mtusi-lab5-migration-check-53406-minio-init-1 Starting
Container mtusi-lab5-migration-check-53406-minio-init-1 Started
Container mtusi-lab5-migration-check-53406-redis-1 Waiting
Container mtusi-lab5-migration-check-53406-minio-1 Waiting
Container mtusi-lab5-migration-check-53406-postgres-1 Waiting
Container mtusi-lab5-migration-check-53406-minio-init-1 Waiting
Container mtusi-lab5-migration-check-53406-redis-1 Healthy
Container mtusi-lab5-migration-check-53406-minio-1 Healthy
Container mtusi-lab5-migration-check-53406-postgres-1 Healthy
Container mtusi-lab5-migration-check-53406-minio-init-1 Healthy
Network mtusi-lab5-migration-check-53406_front Creating
Network mtusi-lab5-migration-check-53406_front Creating
Network mtusi-lab5-migration-check-53406_front Created
Network mtusi-lab5-migration-check-53406_front Created
Container mtusi-lab5-migration-check-53406-api-run-80cac000e90f Creating
Container mtusi-lab5-migration-check-53406-api-run-80cac000e90f Created
Applied 001_initial.sql
Applied 002_idempotency.sql
Applied 003_attachments.sql
Applied 004_jobs.sql
Migrations up to date
Migrations up to date
........................... [100%]
=============================== warnings summary ===============================
student-tests/test_attachments.py: 19 warnings
student-tests/test_idempotency.py: 1 warning
student-tests/test_jobs.py: 6 warnings
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
warnings.warn(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
27 passed, 27 warnings in 11.96s
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
+16
View File
@@ -0,0 +1,16 @@
{
"result": "PASS",
"target_container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
"target_instance": "3cd11c3ed6654d15aa331b17da172ba6",
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
"in_flight_observed_in_pg_stat_activity": true,
"in_flight_response": 201,
"new_tcp_after_sigterm": {
"accepted": false,
"error_type": "ConnectionRefusedError"
},
"target_still_running_when_new_tcp_rejected": true,
"exit_code": 0,
"shutdown_seconds_from_sigterm": 2.1039,
"confirmed_message_preserved": true
}
+215
View File
@@ -0,0 +1,215 @@
{
"result": "PASS",
"containers": [
{
"service": "api",
"container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e",
"name": "mtusi-lab5-api-3",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:16:43.516831177Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data",
"mtusi-lab5_front"
],
"http_status": 200,
"instance": "f0356f65fd3349ebb6a12a12cbe8911b",
"release": "lab5-03545ea5"
},
{
"service": "api",
"container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca",
"name": "mtusi-lab5-api-4",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:16:48.637814055Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data",
"mtusi-lab5_front"
],
"http_status": 200,
"instance": "1f3ca0aa757548729c3af44cfe1152d8",
"release": "lab5-03545ea5"
},
{
"service": "broker",
"container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646",
"name": "mtusi-lab5-broker-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.613805961Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 167772160,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_broker_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "lb",
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
"name": "mtusi-lab5-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.692474795Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab5_front"
]
},
{
"service": "migrate",
"container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8",
"name": "mtusi-lab5-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:15:50.728753166Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "minio",
"container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6",
"name": "mtusi-lab5-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.65248217Z",
"image": "mtusi-lab5-minio:2025-04-22",
"image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_minio_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "minio-init",
"container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd",
"name": "mtusi-lab5-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:15:50.664761458Z",
"image": "mtusi-lab5-mc:2025-04-16",
"image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "postgres",
"container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3",
"name": "mtusi-lab5-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:17:03.314404256Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_postgres_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "redis",
"container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3",
"name": "mtusi-lab5-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:17:02.234800422Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_redis_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "relay",
"container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce",
"name": "mtusi-lab5-relay-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:15:28.180030295Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "worker",
"container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5",
"name": "mtusi-lab5-worker-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:15:28.686125545Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
}
]
}
+27
View File
@@ -0,0 +1,27 @@
{
"result": "PASS",
"commit_publish_gap": {
"accepted_status": 202,
"outbox_unpublished": true,
"api_broker_restarted": true,
"same_job_ready": true
},
"broker_persistence": {
"unread_job_survived_restart": true
},
"after_put_before_ack": {
"s3_effect_written": true,
"lease_recovery_seconds": 90.804,
"attempt_after_recovery": 2,
"same_job": true
},
"duplicate": {
"ready_noop": true
},
"retry_replay": {
"failed_after": 5,
"code": "retry_exhausted",
"explicit_replay_ready": true,
"failed_attachment_binding": 409
}
}
+1
View File
@@ -0,0 +1 @@
FAIL: POST /api/v1/chats/2a81dc08-f62f-4897-899d-7c925a255fbf/attachments: expected [202], received 201
+11
View File
@@ -0,0 +1,11 @@
{
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
"name": "mtusi-lab5-api-2",
"instance": "fa72717b0a734be9b12e177fd2ca606d",
"release": "lab5-v1",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"stopped_at_utc": "2026-11-09T15:16:14.991795+00:00",
"stop_seconds": 0.5132,
"exit_code": 0,
"left_stopped": true
}
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"version": "nginx version: nginx/1.28.0",
"image": "nginx:1.28.0-alpine",
"public_server_header": "nginx",
"config_syntax": "PASS",
"observed_instances": [
"2836440f76e24486ab068d4c71498d15",
"77545b2c0222441193411736edbf54ff"
]
}
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/cluster.py exec python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+428
View File
@@ -0,0 +1,428 @@
{
"result": "PASS",
"before": [
{
"service": "api",
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
"name": "mtusi-lab5-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:12:16.797696721Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data",
"mtusi-lab5_front"
],
"http_status": 200,
"instance": "77545b2c0222441193411736edbf54ff",
"release": "lab5-v1"
},
{
"service": "api",
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
"name": "mtusi-lab5-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:12:16.716356512Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data",
"mtusi-lab5_front"
],
"http_status": 200,
"instance": "2836440f76e24486ab068d4c71498d15",
"release": "lab5-v1"
},
{
"service": "broker",
"container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646",
"name": "mtusi-lab5-broker-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:12:27.23476842Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 167772160,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_broker_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "lb",
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
"name": "mtusi-lab5-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:06:24.855529918Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab5_front"
]
},
{
"service": "migrate",
"container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8",
"name": "mtusi-lab5-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:14:23.843447751Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "minio",
"container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6",
"name": "mtusi-lab5-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:06:18.468961513Z",
"image": "mtusi-lab5-minio:2025-04-22",
"image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_minio_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "minio-init",
"container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd",
"name": "mtusi-lab5-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:14:23.912373126Z",
"image": "mtusi-lab5-mc:2025-04-16",
"image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "postgres",
"container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3",
"name": "mtusi-lab5-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:06:18.410650346Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_postgres_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "redis",
"container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3",
"name": "mtusi-lab5-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:06:18.524762596Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_redis_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "relay",
"container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce",
"name": "mtusi-lab5-relay-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:14:20.333630291Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "worker",
"container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5",
"name": "mtusi-lab5-worker-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:14:20.398521Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
}
],
"after": [
{
"service": "api",
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
"name": "mtusi-lab5-api-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.966931253Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data",
"mtusi-lab5_front"
],
"http_status": 200,
"instance": "cda72c7a2f684f4f83c4f124fe97d9a6",
"release": "lab5-v1"
},
{
"service": "api",
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
"name": "mtusi-lab5-api-2",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.899393962Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data",
"mtusi-lab5_front"
],
"http_status": 200,
"instance": "c306444bf8ac4a599b4dc6ac465b038a",
"release": "lab5-v1"
},
{
"service": "broker",
"container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646",
"name": "mtusi-lab5-broker-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.613805961Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 167772160,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_broker_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "lb",
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
"name": "mtusi-lab5-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.692474795Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab5_front"
]
},
{
"service": "migrate",
"container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8",
"name": "mtusi-lab5-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:15:29.872387004Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "minio",
"container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6",
"name": "mtusi-lab5-minio-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.65248217Z",
"image": "mtusi-lab5-minio:2025-04-22",
"image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_minio_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "minio-init",
"container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd",
"name": "mtusi-lab5-minio-init-1",
"status": "exited",
"health": null,
"started_at": "2026-11-09T15:15:30.375772004Z",
"image": "mtusi-lab5-mc:2025-04-16",
"image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "postgres",
"container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3",
"name": "mtusi-lab5-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.672883961Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_postgres_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "redis",
"container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3",
"name": "mtusi-lab5-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.736927712Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab5_redis_data"
],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "relay",
"container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce",
"name": "mtusi-lab5-relay-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:15:28.180030295Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 201326592,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
},
{
"service": "worker",
"container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5",
"name": "mtusi-lab5-worker-1",
"status": "running",
"health": null,
"started_at": "2026-11-09T15:15:28.686125545Z",
"image": "mtusi-lab5-api:lab5",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab5_data"
]
}
],
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
}
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+15
View File
@@ -0,0 +1,15 @@
{
"result": "PASS",
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 133, \"failed_attempts\": 0, \"first_recovery_seconds\": 0.878}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
"stopped": {
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
"name": "mtusi-lab5-api-2",
"instance": "c306444bf8ac4a599b4dc6ac465b038a",
"release": "lab5-v1",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"stopped_at_utc": "2026-11-09T15:15:35.619684+00:00",
"stop_seconds": 0.3357,
"exit_code": 0,
"left_stopped": true
}
}
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 0.878}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+171
View File
@@ -0,0 +1,171 @@
{
"result": "PASS",
"before": [
{
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
"name": "mtusi-lab5-api-1",
"instance": "d23b5da28be949239c1ea5f27a578822",
"release": "lab5-v1",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b"
},
{
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
"name": "mtusi-lab5-api-2",
"instance": "7c84431c2a104eba8f1f3d789d971602",
"release": "lab5-v1",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b"
}
],
"after": [
{
"container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e",
"name": "mtusi-lab5-api-3",
"instance": "f0356f65fd3349ebb6a12a12cbe8911b",
"release": "lab5-03545ea5",
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe"
},
{
"container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca",
"name": "mtusi-lab5-api-4",
"instance": "1f3ca0aa757548729c3af44cfe1152d8",
"release": "lab5-03545ea5",
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe"
}
],
"target_release": "lab5-03545ea5",
"lb_before": {
"service": "lb",
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
"name": "mtusi-lab5-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.692474795Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab5_front"
]
},
"lb_after": {
"service": "lb",
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
"name": "mtusi-lab5-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-09T15:15:27.692474795Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab5_front"
]
},
"lb_not_restarted": true,
"build_seconds": 3.8455,
"migration_output": "Migrations up to date",
"replacements": [
{
"old_container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
"new_container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e",
"old_ip": "172.26.0.3",
"new_ip": "172.26.0.5",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"7c84431c2a104eba8f1f3d789d971602",
"d23b5da28be949239c1ea5f27a578822",
"f0356f65fd3349ebb6a12a12cbe8911b"
],
"observed_instances": [
"7c84431c2a104eba8f1f3d789d971602",
"f0356f65fd3349ebb6a12a12cbe8911b"
],
"old_exit_code": 0,
"seconds": 5.1191
},
{
"old_container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
"new_container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca",
"old_ip": "172.26.0.2",
"new_ip": "172.26.0.3",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"1f3ca0aa757548729c3af44cfe1152d8",
"7c84431c2a104eba8f1f3d789d971602",
"f0356f65fd3349ebb6a12a12cbe8911b"
],
"observed_instances": [
"1f3ca0aa757548729c3af44cfe1152d8",
"f0356f65fd3349ebb6a12a12cbe8911b"
],
"old_exit_code": 0,
"seconds": 6.1668
}
],
"during_replacement": {
"seconds": 11.286,
"concurrency": 4,
"successful_requests": 8541,
"failed_attempts": 0,
"successful_rps": 756.778,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 4.383,
"p95": 7.793,
"p99": 35.123
},
"instances": [
"1f3ca0aa757548729c3af44cfe1152d8",
"7c84431c2a104eba8f1f3d789d971602",
"d23b5da28be949239c1ea5f27a578822",
"f0356f65fd3349ebb6a12a12cbe8911b"
]
},
"total": {
"seconds": 22.0756,
"concurrency": 4,
"successful_requests": 16487,
"failed_attempts": 0,
"successful_rps": 746.843,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 4.601,
"p95": 7.648,
"p99": 26.561
},
"instances": [
"1f3ca0aa757548729c3af44cfe1152d8",
"7c84431c2a104eba8f1f3d789d971602",
"d23b5da28be949239c1ea5f27a578822",
"f0356f65fd3349ebb6a12a12cbe8911b"
]
},
"data": {
"confirmed_unique_messages": 16487,
"history_messages": 16487,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
}
+135
View File
@@ -0,0 +1,135 @@
{
"result": "PASS",
"baseline": {
"seconds": 3.0041,
"concurrency": 4,
"successful_requests": 2367,
"failed_attempts": 0,
"successful_rps": 787.933,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 4.168,
"p95": 6.086,
"p99": 33.522
},
"instances": [
"cda72c7a2f684f4f83c4f124fe97d9a6",
"fa72717b0a734be9b12e177fd2ca606d"
]
},
"phases": [
{
"stopped": {
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
"name": "mtusi-lab5-api-1",
"instance": "cda72c7a2f684f4f83c4f124fe97d9a6",
"release": "lab5-v1",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"stopped_at_utc": "2026-11-09T15:16:01.326356+00:00",
"stop_seconds": 0.4399,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.0034,
"concurrency": 4,
"successful_requests": 4111,
"failed_attempts": 1,
"successful_rps": 513.658,
"error_ratio": 0.000243,
"successful_latency_ms": {
"p50": 6.469,
"p95": 12.349,
"p99": 58.778
},
"instances": [
"cda72c7a2f684f4f83c4f124fe97d9a6",
"fa72717b0a734be9b12e177fd2ca606d"
]
},
"post_stop_write_seconds_from_action_start": 1.4598,
"post_stop_write_seconds_from_stop_completion": 0.029,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.4513,
"seconds_from_stop_completion": 0.0206
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
},
{
"stopped": {
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
"name": "mtusi-lab5-api-2",
"instance": "fa72717b0a734be9b12e177fd2ca606d",
"release": "lab5-v1",
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
"stopped_at_utc": "2026-11-09T15:16:14.991795+00:00",
"stop_seconds": 0.5132,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.0007,
"concurrency": 4,
"successful_requests": 4087,
"failed_attempts": 0,
"successful_rps": 510.829,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 6.497,
"p95": 13.243,
"p99": 57.218
},
"instances": [
"3cd11c3ed6654d15aa331b17da172ba6",
"fa72717b0a734be9b12e177fd2ca606d"
]
},
"post_stop_write_seconds_from_action_start": 1.3033,
"post_stop_write_seconds_from_stop_completion": 0.0159,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.3004,
"seconds_from_stop_completion": 0.013
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
}
],
"total": {
"seconds": 30.9184,
"concurrency": 4,
"successful_requests": 18469,
"failed_attempts": 1,
"successful_rps": 597.347,
"error_ratio": 5.4e-05,
"successful_latency_ms": {
"p50": 5.53,
"p95": 10.788,
"p99": 46.459
},
"instances": [
"3cd11c3ed6654d15aa331b17da172ba6",
"7c84431c2a104eba8f1f3d789d971602",
"cda72c7a2f684f4f83c4f124fe97d9a6",
"fa72717b0a734be9b12e177fd2ca606d"
]
},
"data": {
"confirmed_unique_messages": 18469,
"history_messages": 18469,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"each_original_container_tested": true
}
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
+15
View File
@@ -0,0 +1,15 @@
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
........................... [100%]
=============================== warnings summary ===============================
student-tests/test_attachments.py: 19 warnings
student-tests/test_idempotency.py: 1 warning
student-tests/test_jobs.py: 6 warnings
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
datetime_now = datetime.datetime.utcnow()
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
warnings.warn(
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
27 passed, 27 warnings in 11.58s
+28
View File
@@ -0,0 +1,28 @@
# Official public images/binaries were removed upstream; build fixed release sources.
FROM golang:1.24.2-bookworm AS minio-build
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src
WORKDIR /src
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio .
RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio .
FROM golang:1.24.2-bookworm AS mc-build
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src
WORKDIR /src
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc .
RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc .
FROM python:3.13.5-slim AS minio
LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z"
RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data
COPY --from=minio-build /out/minio /usr/local/bin/minio
COPY infra/minio/server.sh /server.sh
USER 10002
ENTRYPOINT ["/bin/sh", "/server.sh"]
CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"]
FROM debian:bookworm-slim AS mc
LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z"
COPY --from=mc-build /out/mc /usr/local/bin/mc
ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"]
+45
View File
@@ -0,0 +1,45 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": [
"s3:GetBucketLocation"
],
"Resource": [
"arn:aws:s3:::messenger-attachments"
]
},
{
"Effect": "Allow",
"Action": [
"s3:ListBucket"
],
"Resource": [
"arn:aws:s3:::messenger-attachments"
],
"Condition": {
"StringLike": {
"s3:prefix": [
"originals/*",
"staging/*",
"variants/*"
]
}
}
},
{
"Effect": "Allow",
"Action": [
"s3:GetObject",
"s3:PutObject",
"s3:DeleteObject"
],
"Resource": [
"arn:aws:s3:::messenger-attachments/originals/*",
"arn:aws:s3:::messenger-attachments/staging/*",
"arn:aws:s3:::messenger-attachments/variants/*"
]
}
]
}
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null
mc mb --ignore-existing local/messenger-attachments >/dev/null
mc anonymous set none local/messenger-attachments >/dev/null
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
mc admin policy create local messenger-api /config/api-policy.json >/dev/null
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).'
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
exec minio "$@"
+62
View File
@@ -0,0 +1,62 @@
worker_processes 1;
error_log /dev/stderr warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
server_tokens off;
limit_conn_zone $server_name zone=uploads:64k;
# Docker DNS; resolve updates the upstream when replicas get new IPs.
resolver 127.0.0.11 valid=1s ipv6=off;
resolver_timeout 1s;
upstream messenger {
zone messenger 64k;
# All A records of api participate in the default round-robin balancing.
server api:8080 resolve max_fails=1 fail_timeout=2s;
}
server {
listen 8080;
server_name messenger;
location ~ ^/api/v1/chats/[^/]+/attachments$ {
limit_conn uploads 16;
client_max_body_size 11m;
client_body_buffer_size 64k;
client_body_timeout 10s;
proxy_request_buffering on;
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
proxy_next_upstream error timeout http_503;
proxy_next_upstream_tries 2;
}
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 5s;
proxy_send_timeout 5s;
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
# must not be resent by Nginx after an ambiguous response.
proxy_next_upstream error timeout http_503;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 6s;
}
}
}
+34
View File
@@ -0,0 +1,34 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_history_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
CREATE TABLE sessions (
token_hash text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+11
View File
@@ -0,0 +1,11 @@
-- Additive: old lab2 tables and messages remain valid.
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash text NOT NULL,
message_id uuid REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
+23
View File
@@ -0,0 +1,23 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
owner_id uuid NOT NULL REFERENCES users(id),
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL UNIQUE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
);
CREATE INDEX attachments_chat_idx ON attachments(chat_id,id);
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages(id),
attachment_id uuid NOT NULL REFERENCES attachments(id),
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
PRIMARY KEY (message_id,attachment_id),
UNIQUE (message_id,position)
);
+19
View File
@@ -0,0 +1,19 @@
ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed'));
ALTER TABLE attachments ADD COLUMN error_code text;
ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE;
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
CREATE TABLE attachment_jobs (
id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1),
attempt integer NOT NULL DEFAULT 0, replay_count integer NOT NULL DEFAULT 0,
lease_until timestamptz, next_at timestamptz NOT NULL DEFAULT clock_timestamp(),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), traceparent text, tracestate text
);
CREATE TABLE job_outbox (
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), dispatched_at timestamptz
);
CREATE TABLE job_replays (
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), reason text NOT NULL
);
CREATE INDEX jobs_due ON attachment_jobs(next_at);
+51
View File
@@ -0,0 +1,51 @@
annotated-types==0.8.0
anyio==4.9.0
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
attrs==26.1.0
boto3==1.38.27
botocore==1.38.46
certifi==2026.10.23
cffi==2.1.1
charset-normalizer==3.5.2
click==8.5.0
fastapi==0.115.12
h11==0.16.0
httpcore==1.0.9
httpx==0.28.1
idna==3.20
iniconfig==2.3.1
jmespath==1.1.0
jsonschema==4.26.0
jsonschema-path==0.3.4
jsonschema-specifications==2025.9.1
lazy-object-proxy==1.13.0
openapi-schema-validator==0.6.3
openapi-spec-validator==0.7.2
packaging==26.3
pathable==0.4.4
pillow==11.3.0
pluggy==1.7.0
psycopg==3.2.9
psycopg-binary==3.2.9
psycopg-pool==3.2.6
pycparser==3.12
pydantic==2.11.5
pydantic_core==2.33.2
Pygments==2.21.0
pytest==8.4.0
python-dateutil==2.9.0.post0
PyYAML==6.0.3
redis==6.2.0
referencing==0.36.2
requests==2.34.2
rfc3339-validator==0.1.4
rpds-py==2026.9.1
s3transfer==0.13.1
six==1.17.0
sniffio==1.3.1
starlette==0.46.2
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.34.3
+12
View File
@@ -0,0 +1,12 @@
fastapi==0.115.12
uvicorn==0.34.3
psycopg[binary]==3.2.9
psycopg-pool==3.2.6
redis==6.2.0
argon2-cffi==25.1.0
pytest==8.4.0
httpx==0.28.1
pydantic==2.11.5
anyio==4.9.0
boto3==1.38.27
Pillow==11.3.0
+32
View File
@@ -0,0 +1,32 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
python3 - <<'PY'
from pathlib import Path
import secrets
root = Path('secrets')
p = root / 'postgres_password'
if not p.exists():
p.write_text(secrets.token_hex(24))
pw = p.read_text().strip()
for name, value in {
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
'cache_url': 'redis://redis:6379/0',
'minio_root_user': 'lab4-admin',
'minio_root_password': secrets.token_hex(24),
's3_access_key_id': 'messenger-api',
's3_secret_access_key': secrets.token_hex(24),
}.items():
path = root / name
if not path.exists():
path.write_text(value)
# Bind-mounted Docker secrets must be readable by container uid 10001.
# Parent directory remains 0700 on the host.
for path in root.iterdir():
path.chmod(0o444)
if not Path('.env').exists():
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
print('Local secrets ready (values hidden).')
PY
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# A unique disposable Compose project; never touches normal project volumes.
set -eu
cd "$(dirname "$0")/.."
project="mtusi-lab5-migration-check-$$"
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
echo 'Refusing to reuse a pre-existing scratch project' >&2
exit 1
fi
cleanup() {
docker compose -p "$project" down --volumes >/dev/null 2>&1
}
trap cleanup EXIT INT TERM
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
import argparse
import datetime
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
def command(args, timeout=120):
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
def compose(*args, timeout=120):
return command(['docker', 'compose', *args], timeout=timeout)
def inspect(containers):
if not containers:
return []
return json.loads(command(['docker','inspect',*containers]))
def container_ids(service=None, all_containers=False):
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
return compose(*args).split()
def probe(container, host='127.0.0.1', path='/health/live'):
code = '''import json,sys,urllib.request,urllib.error
try:
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
except urllib.error.HTTPError as exc:
r=exc
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
'''
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
def replicas():
items = []
for data in inspect(container_ids('api')):
if data['State']['Running']:
result = probe(data['Id'])
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
return items
def snapshot():
items = []
for data in inspect(container_ids(all_containers=True)):
service = data['Config']['Labels']['com.docker.compose.service']
item = {
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
'published_ports':data['HostConfig']['PortBindings'],
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
'networks':sorted(data['NetworkSettings']['Networks']),
}
if service == 'api' and data['State']['Running']:
result = probe(data['Id'])
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
items.append(item)
return items
def wait_healthy(container, timeout=60):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
data = inspect([container])[0]
if data['State'].get('Health',{}).get('Status') == 'healthy':
return
time.sleep(0.3)
raise RuntimeError('Container did not become healthy')
def wait_cluster(count=2, timeout=60):
deadline = time.monotonic()+timeout
seen = set()
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
alias = response.headers.get('X-Instance-Id')
if alias:
seen.add(alias)
if len(seen) >= count:
return sorted(seen)
except (OSError, urllib.error.URLError):
pass
time.sleep(0.1)
raise RuntimeError('Load balancer did not discover required API instances')
def stop_one(target):
if not re.fullmatch(r'[a-f0-9]{32}', target):
raise ValueError('Invalid target instance alias')
matching = [item for item in replicas() if item['instance']==target]
if len(matching) != 1:
raise RuntimeError('Target must match exactly one running API process')
item = matching[0]
started = time.monotonic()
command(['docker','stop','--time','7',item['container']],timeout=9)
state = inspect([item['container']])[0]['State']
if state['Running'] or state['ExitCode'] != 0:
raise RuntimeError('API did not stop gracefully')
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
return result
def restart():
compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30)
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
return wait_cluster(2,timeout=15)
def front_address(container):
networks = inspect([container])[0]['NetworkSettings']['Networks']
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
def replace_one(container):
before = set(container_ids('api'))
if container not in before or len(before) != 2:
raise RuntimeError('Replacement expects exactly two running API containers')
old_address = front_address(container)
# Start-first ensures a genuinely different IP while the old address is still occupied.
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
expanded = set(container_ids('api'))
added = expanded-before
if len(added) != 1 or not before.issubset(expanded):
raise RuntimeError('Start-first step unexpectedly changed an existing container')
replacement = added.pop()
wait_healthy(replacement)
new_address = front_address(replacement)
if new_address == old_address:
raise RuntimeError('Replacement did not get a different IP')
expected_three = {item['instance'] for item in replicas()}
if set(wait_cluster(3)) != expected_three:
raise RuntimeError('Load balancer did not discover all three live addresses')
command(['docker','stop','--time','7',container],timeout=10)
state = inspect([container])[0]['State']
if state['ExitCode'] != 0:
raise RuntimeError('Old release did not terminate gracefully')
command(['docker','rm',container])
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
after = set(container_ids('api'))
if after != (before-{container}) | {replacement}:
raise RuntimeError('Sequential replacement changed the wrong containers')
expected = {item['instance'] for item in replicas()}
if set(wait_cluster(2)) != expected:
raise RuntimeError('Unexpected final backend aliases')
return {'old_container':container,'new_container':replacement,
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
'instances_during_start_first':sorted(expected_three),
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
parser.add_argument('arguments',nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.action == 'wait':
print(json.dumps({'instances':wait_cluster()}))
elif args.action == 'snapshot':
print(json.dumps(snapshot(),indent=2))
elif args.action == 'stop-one':
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
elif args.action == 'exec':
ids = container_ids('api')
if not ids or not args.arguments:
raise RuntimeError('A live API container and command are required')
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
elif args.action == 'restart':
print(json.dumps({'instances':restart()}))
else:
compose('run','--rm','--no-deps','migrate')
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/cluster.py restart
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
exec python3 scripts/cluster.py stop-one
+443
View File
@@ -0,0 +1,443 @@
#!/usr/bin/env python3
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import uuid
import cluster
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from resilience import history
def save(name, value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:', name)
def percentiles(values):
values = sorted(values)
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
class Traffic:
def __init__(self, concurrency=4):
self.c = Client(timeout=6)
self.alice,self.bob,_,self.chat = self.c.bootstrap()
self.concurrency = concurrency
self.done = threading.Event()
self.lock = threading.Lock()
self.attempts = []
self.confirmed = {}
self.prefix = uuid.uuid4().hex
def worker(self, number):
client = Client(timeout=6)
index = 0
while not self.done.is_set():
text = self.prefix+'-'+str(number)+'-'+str(index)
key = self.prefix+'-'+str(number)+'-'+str(index)
started = time.monotonic()
try:
response = client.message(self.alice,self.chat,text,key=key)
row = response['json']
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
with self.lock:
self.confirmed[row['id']] = row
index += 1
except Failure:
# Only idempotent message POST is retried, with its original body and key.
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
time.sleep(0.02)
with self.lock:
self.attempts.append(attempt)
def start(self):
self.started = time.monotonic()
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
return self
def stop(self):
self.done.set()
for future in self.futures:
future.result(timeout=10)
self.pool.shutdown()
self.ended = time.monotonic()
def summary(self, start=None, end=None):
start = self.started if start is None else start
end = getattr(self,'ended',time.monotonic()) if end is None else end
with self.lock:
attempts = [a for a in self.attempts if start <= a['started'] < end]
success = [a for a in attempts if a['ok']]
duration = end-start
return {'seconds':round(duration,4),'concurrency':self.concurrency,
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
'successful_rps':round(len(success)/duration,3),
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
'successful_latency_ms':percentiles([a['latency'] for a in success]),
'instances':sorted({a['instance'] for a in success})}
def verify_data(self):
rows = history(self.c,self.bob,self.chat)
actual = {row['id']:row for row in rows}
check(len(actual)==len(rows),'duplicated message ids')
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
def concurrent_replay():
cluster.wait_cluster()
c = Client()
alice,bob,_,chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(8)
def send(_):
local = Client()
barrier.wait()
return local.message(alice,chat,'Один результат через разные процессы',key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send,range(8)))
instances = {r['headers']['X-Instance-Id'] for r in results}
check(len(instances)>=2,'concurrent retry did not reach different API processes')
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
body={'text':'Другой body'},headers={'Idempotency-Key':key})
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
'responses_201':8,'messages_created':1,'different_body_status':409})
def verify_persistence():
before = cluster.snapshot()
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
after = cluster.snapshot()
roles = ['api','postgres','redis','minio','broker','relay','worker','lb']
previous = {item['container']:item for item in before}
for item in after:
if item['service'] in roles:
old = previous[item['container']]
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
check(old['volumes']==item['volumes'],'volume identities changed')
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
def public_failover():
cluster.wait_cluster()
try:
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
finally:
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
cluster.wait_cluster()
def reject_new_connection(survivor, address):
code = '''import json,socket,sys
try:
s=socket.create_connection((sys.argv[1],8080),timeout=1)
except OSError as exc:
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
else:
s.close()
print(json.dumps({'accepted':True}))
'''
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
def failover_each():
cluster.wait_cluster()
originals = cluster.replicas()
check(len(originals)==2,'two running replicas required')
traffic = Traffic().start()
phases = []
try:
time.sleep(3)
baseline_end = time.monotonic()
baseline = traffic.summary(end=baseline_end)
for original in originals:
# Alias changes on restart, so resolve the current process of this exact container.
current = next(r for r in cluster.replicas() if r['container']==original['container'])
all_before = cluster.inspect([current['container']])[0]
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
start = time.monotonic()
try:
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
stop_returned = time.monotonic()
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
read_deadline = start+10
while True:
try:
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
read_done = time.monotonic()
break
except Failure:
check(time.monotonic()<read_deadline,'reads did not recover within10s')
time.sleep(0.1)
check(read_done-start<=10,'read recovery exceeded10s')
rejected = reject_new_connection(survivor,address)
check(not rejected['accepted'],'new connection accepted by stopped API')
time.sleep(max(0,8-(time.monotonic()-start)))
end = time.monotonic()
summary = traffic.summary(start,end)
with traffic.lock:
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
recent = [a for a in successful if a['started']>stop_returned]
recovery = min(a['ended'] for a in recent)-start if recent else None
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
'seconds_from_action_start':round(read_done-start,4),
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
finally:
cluster.command(['docker','start',current['container']])
cluster.wait_healthy(current['container'])
cluster.wait_cluster()
time.sleep(2)
finally:
traffic.stop()
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
'total':traffic.summary(),'data':traffic.verify_data(),
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
def rolling_update():
cluster.wait_cluster()
before = cluster.replicas()
check(len(before)==2,'rolling demonstration expects two replicas')
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
target_release = 'lab5-'+uuid.uuid4().hex[:8]
traffic = Traffic().start()
replacements = []
try:
time.sleep(3)
build_start = time.monotonic()
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
build_end = time.monotonic()
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
swap_start = time.monotonic()
for old in before:
step_start = time.monotonic()
result = cluster.replace_one(old['container'])
result['seconds'] = round(time.monotonic()-step_start,4)
replacements.append(result)
swap_end = time.monotonic()
time.sleep(3)
finally:
traffic.stop()
after = cluster.replicas()
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
'total':traffic.summary(),'data':traffic.verify_data(),
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
def database_fixture(container, data):
code = '''import json,sys,psycopg
from psycopg import sql
from messenger.db import setting
p=json.load(sys.stdin)
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
if p['action']=='install':
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
result={'installed':True}
elif p['action']=='remove':
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
result={'removed':True}
else:
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
print(json.dumps(result))
'''
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(response.stdout)
def graceful_shutdown():
cluster.wait_cluster()
nodes = cluster.replicas()
target,survivor = nodes[0],nodes[1]
address = cluster.front_address(target['container'])
c = Client()
alice,bob,_,chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']+'/messages'
marker = 'drain_'+uuid.uuid4().hex
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
database_fixture(survivor['container'],{**fixture,'action':'install'})
code = '''import json,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
try:
response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as exc:
response=exc
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
'''
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
def in_flight():
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(result.stdout)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(in_flight)
try:
deadline = time.monotonic()+3
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
time.sleep(0.05)
signaled = time.monotonic()
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
deadline = signaled+1.5
while True:
rejected = reject_new_connection(survivor['container'],address)
still_running = cluster.inspect([target['container']])[0]['State']['Running']
if not rejected['accepted']:
break
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
time.sleep(0.05)
check(still_running,'connection refusal observed only after process exit, not during drain')
drained = future.result(timeout=8)
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
check(drained['message']['text']==marker,'in-flight body changed')
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
elapsed = time.monotonic()-signaled
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
'confirmed_message_preserved':True}
finally:
pool.shutdown(wait=True)
database_fixture(survivor['container'],{**fixture,'action':'remove'})
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('graceful-shutdown.json',result)
def infrastructure():
state = cluster.snapshot()
for item in state:
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
if item['service']!='lb':
check(not item['published_ports'],'service exposed a port bypassing LB')
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
save('infrastructure.json',{'result':'PASS','containers':state})
def dependency_failures():
c = Client(timeout=6)
actor = c.user('dependencies')
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result = {'result':'PASS'}
try:
cluster.compose('stop','-t','2','redis')
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
c.call('GET','/api/v1/users/me',401)
c.call('GET','/health/ready',schema='Health')
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
finally:
cluster.compose('start','redis')
import urllib.request
import urllib.error
def status(path, token=None):
headers = {'Authorization':'Bearer '+token} if token else {}
request = urllib.request.Request(c.base+path,headers=headers)
started = time.monotonic()
try:
response = urllib.request.urlopen(request,timeout=6)
except urllib.error.HTTPError as error:
response = error
with response:
return response.code, round(time.monotonic()-started,4)
try:
cluster.compose('stop','-t','2','postgres')
code,elapsed = status('/api/v1/users/me',actor['token'])
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
anonymous,_ = status('/api/v1/users/me')
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
check(all(code==200 for code in live),'API process liveness depends on DB')
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
finally:
cluster.compose('start','postgres')
cluster.wait_cluster()
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result['same_session_after_recovery'] = True
save('dependency-failures.json',result)
def verify_nginx():
state = cluster.snapshot()
lb = next(item for item in state if item['service']=='lb')
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
c = Client()
response = c.call('GET','/health/ready',schema='Health')
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
args = parser.parse_args()
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__ == '__main__':
main()
+122
View File
@@ -0,0 +1,122 @@
#!/usr/bin/env python3
"""Explicit real fault injection; emits IDs/results, never session tokens."""
import json
from pathlib import Path
import subprocess
import sys
import time
import uuid
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client,check
def compose(*args):
result=subprocess.run(['docker','compose',*args],cwd=ROOT,capture_output=True,text=True,timeout=120)
if result.returncode: raise RuntimeError(result.stderr)
return result.stdout
def inside(body,**values):
script="import json,io,time\nfrom uuid import UUID\nfrom messenger.db import Database\nfrom messenger.object_store import ObjectStore\nfrom messenger.jobs import work,replay,broker_client,STREAM\nfrom psycopg.types.json import Jsonb\ndb=Database();db.open();store=ObjectStore()\nv="+repr(values)+'\n'+body+'\nprint(json.dumps(result,default=str))\nstore.close();db.close()'
proc=subprocess.run(['docker','compose','exec','-T','api','python','-'],cwd=ROOT,input=script,capture_output=True,text=True,timeout=120)
if proc.returncode: raise RuntimeError(proc.stderr)
return json.loads(proc.stdout)
def state(aid):
return inside("with db.pool.connection() as c:\n result=c.execute('SELECT a.status,a.staging_key,j.id AS job_id,j.attempt,j.lease_until,o.dispatched_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN job_outbox o ON o.job_id=j.id WHERE a.id=%s',(v[\"aid\"],)).fetchone()",aid=aid)
def wait_for(predicate,seconds=30):
end=time.monotonic()+seconds
while time.monotonic()<end:
result=predicate()
if result: return result
time.sleep(.2)
raise AssertionError('Condition timed out')
def ready(c,actor,att):
return c.wait_ready(actor,att)
def main():
c=Client();alice,bob,_,chat=c.bootstrap()
evidence={'result':'INCOMPLETE'}
trigger=False
try:
# Durable acceptance while both relay and worker are absent: real COMMIT/publish gap.
compose('stop','worker','relay')
att,raw=c.attachment(alice,chat)
c.message(alice,chat,'',attachments=[att['id']])
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
before=state(att['id']);check(before['dispatched_at'] is None,'outbox already published')
compose('restart','api','broker')
wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0)
c.call('GET','/api/v1/users/me',actor=alice)
compose('start','relay','worker')
meta=ready(c,bob,att)
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original changed')
evidence['commit_publish_gap']={'accepted_status':202,'outbox_unpublished':True,'api_broker_restarted':True,'same_job_ready':state(att['id'])['job_id']==before['job_id']}
# Publisher has already confirmed acceptance, broker restart preserves unread work.
compose('stop','worker')
att2,_=c.attachment(alice,chat)
wait_for(lambda: state(att2['id'])['dispatched_at'])
compose('restart','broker')
compose('start','worker')
ready(c,bob,att2)
evidence['broker_persistence']={'unread_job_survived_restart':True}
# Delay ready COMMIT in real PostgreSQL AFTER all S3 PUTs, kill before ack.
compose('stop','worker')
target,_=c.attachment(alice,chat)
inside("with db.pool.connection() as c:\n c.execute(\"CREATE OR REPLACE FUNCTION lab5_pause_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.id='\"+v['aid']+\"'::uuid AND NEW.status='ready' THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$\")\n c.execute('CREATE TRIGGER lab5_pause BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab5_pause_ready()')\nresult=True",aid=target['id'])
trigger=True
compose('start','worker')
def at_commit():
return inside("with db.pool.connection() as c:\n result=bool(c.execute(\"SELECT 1 FROM pg_stat_activity WHERE wait_event='PgSleep' AND query LIKE 'UPDATE attachments SET status=%'\").fetchone())")
wait_for(at_commit)
observed=inside("with db.pool.connection() as c:\n row=c.execute('SELECT object_key FROM attachments WHERE id=%s',(v['aid'],)).fetchone()\nobj=store.get(row['object_key']);result={'s3_effect_written':obj.length>0};obj.close()",aid=target['id'])
compose('kill','-s','SIGKILL','worker')
# unless-stopped can restart a killed worker; stop it to retain the fault boundary.
compose('stop','worker')
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER lab5_pause ON attachments');c.execute('DROP FUNCTION lab5_pause_ready()')\nresult=True")
trigger=False
crashed=state(target['id']);check(crashed['attempt']==1,'expected first leased attempt')
print('Waiting for real 90s lease expiry after killed worker.',flush=True)
started=time.monotonic()
wait_for(lambda: state(target['id'])['status']=='queued',110)
compose('start','worker')
ready(c,bob,target)
after=state(target['id'])
evidence['after_put_before_ack']={**observed,'lease_recovery_seconds':round(time.monotonic()-started,3),'attempt_after_recovery':after['attempt'],'same_job':after['job_id']==crashed['job_id']}
# Re-deliver the completed original job. Ready metadata is immutable.
original_meta=c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']
duplicate=inside("result=work(db,store,v['job'],1)",job=after['job_id'])
check(duplicate and c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']==original_meta,'duplicate changed ready result')
evidence['duplicate']={'ready_noop':True}
# Retry exhaustion with real missing staging (transient storage failure), explicit replay.
compose('stop','worker','relay')
failed,failed_raw=c.attachment(alice,chat)
st=state(failed['id'])
inside("store.delete(v['key']);result=True",key=st['staging_key'])
for attempt in range(5):
inside("with db.pool.connection() as c:\n c.execute('UPDATE attachment_jobs SET next_at=clock_timestamp() WHERE id=%s',(v['job'],))\nresult=work(db,store,v['job'])",job=st['job_id'])
failure=c.call('GET','/api/v1/attachments/'+failed['id'],actor=alice,schema='Attachment')['json']
check(failure['status']=='failed' and failure['error_code']=='retry_exhausted','retry did not terminate')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[failed['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
inside("store.put(v['key'],io.BytesIO(bytes.fromhex(v['raw'])),v['length'],'image/png')\nreplay(db,v['job'],'restore synthetic staging and retry')\nresult=True",key=st['staging_key'],raw=failed_raw.hex(),length=len(failed_raw),job=st['job_id'])
compose('start','relay','worker')
ready(c,bob,failed)
evidence['retry_replay']={'failed_after':5,'code':'retry_exhausted','explicit_replay_ready':True,'failed_attachment_binding':409}
evidence['result']='PASS'
(ROOT/'evidence/jobs-faults.json').write_text(json.dumps(evidence,indent=2)+'\n')
print('PASS: jobs-faults.json')
finally:
if trigger:
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER IF EXISTS lab5_pause ON attachments');c.execute('DROP FUNCTION IF EXISTS lab5_pause_ready()')\nresult=True")
compose('start','relay','worker')
if __name__=='__main__': main()
View File
+307
View File
@@ -0,0 +1,307 @@
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
import json
import asyncio
import logging
import os
import re
from contextlib import asynccontextmanager
from uuid import uuid4
import psycopg
from fastapi import Depends, FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from psycopg_pool import PoolTimeout, TooManyRequests
from redis.exceptions import RedisError
from starlette.exceptions import HTTPException
from starlette.concurrency import run_in_threadpool
from .db import Database
from .errors import ApiError, invalid, not_found, unauthorized
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
from .messages import store_message, validate_key, MESSAGE_FIELDS
from .object_store import ObjectStore, unavailable
from . import attachments
from urllib.parse import quote
from .pagination import decode_cursor, page, page_limit, public, stamp
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
LOG = logging.getLogger('messenger')
USER_FIELDS = 'id, username, display_name, created_at'
def error_response(request, error):
return JSONResponse({'error': {'code': error.code, 'message': error.message,
'request_id': request.state.request_id}},
status_code=error.status, headers=error.headers)
def authenticate(request: Request):
token = bearer(request.headers.get('Authorization', ''))
token_hash = digest(token)
with request.app.state.db.pool.connection() as conn:
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
(token_hash,)).fetchone()
if not row:
raise unauthorized()
request.state.token_hash = token_hash
return row['user_id']
def require_member(conn, chat_id, user_id):
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
if not found:
raise not_found()
def chat_public(conn, row):
item = public(row)
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
return item
def create_app():
@asynccontextmanager
async def lifespan(app):
app.state.db = Database()
app.state.storage = ObjectStore()
app.state.upload_slots = asyncio.Semaphore(6)
app.state.db.open()
try:
yield
finally:
app.state.storage.close()
app.state.db.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.state.release = os.environ.get('APP_RELEASE', 'lab5-v1')
@app.middleware('http')
async def request_context(request: Request, call_next):
request.state.request_id = str(uuid4())
try:
path = request.url.path
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
if request.method == 'POST' and json_endpoint:
media = request.headers.get('Content-Type', '').split(';')
unsupported = media[0].strip().lower() != 'application/json'
for parameter in media[1:]:
name, _, value = parameter.strip().partition('=')
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
unsupported = True
if unsupported:
if path != '/api/v1/users':
await run_in_threadpool(authenticate, request)
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
try:
json.loads((await request.body()).decode('utf-8'))
except (UnicodeError, json.JSONDecodeError):
raise invalid() from None
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc)
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
except Exception as exc:
# Log exception class only: credentials, SQL and message content stay out of logs.
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
response.headers['X-Request-Id'] = request.state.request_id
response.headers['X-Instance-Id'] = app.state.instance_id
response.headers['X-App-Release'] = app.state.release
return response
@app.exception_handler(ApiError)
async def api_error(request, exc):
return error_response(request, exc)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error_response(request, invalid())
@app.exception_handler(HTTPException)
async def http_error(request, exc):
return error_response(request, not_found() if exc.status_code == 404 else invalid())
@app.get('/health/live')
def live():
return {'status': 'ok'}
@app.get('/health/ready')
def ready(request: Request):
try:
with request.app.state.db.pool.connection() as conn:
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
conn.execute('SELECT status,variants FROM attachments LIMIT 0')
conn.execute('SELECT 1 FROM job_outbox LIMIT 0')
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
request.app.state.storage.check()
return {'status': 'ok'}
@app.post('/api/v1/users', status_code=201)
def register(body: CreateUser, request: Request):
password_hash = HASHER.hash(body.password)
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute(
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
except psycopg.errors.UniqueViolation:
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
return public(row)
@app.post('/api/v1/auth/sessions', status_code=201)
async def create_session(request: Request):
# Run hashing and synchronous database calls outside the event loop.
username, password = basic_credentials(request.headers.get('Authorization', ''))
if await request.body():
raise invalid()
return await run_in_threadpool(login, request, username, password)
def login(request, username, password):
db = request.app.state.db
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
if not row or not valid:
raise unauthorized(basic=True)
row.pop('password_hash')
token, expires_at = new_session(db.session_ttl)
with db.pool.connection() as conn:
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
(digest(token), row['id'], expires_at))
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
@app.get('/api/v1/users/me')
def me(request: Request, user_id=Depends(authenticate)):
db = request.app.state.db
key = 'user:' + str(user_id)
try:
cached = db.cache.get(key)
if cached:
return json.loads(cached)
except (RedisError, ValueError):
pass
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
if not row:
raise unauthorized()
result = public(row)
try:
db.cache.set(key, json.dumps(result), ex=60)
except RedisError:
pass
return result
@app.delete('/api/v1/auth/sessions/current', status_code=204)
def logout(request: Request, user_id=Depends(authenticate)):
with request.app.state.db.pool.connection() as conn:
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
return Response(status_code=204)
@app.post('/api/v1/chats', status_code=201)
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
if str(user_id) not in body.member_ids:
raise invalid()
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
(uuid4(), body.title)).fetchone()
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
[(row['id'], uuid_value(member)) for member in body.member_ids])
result = chat_public(conn, row)
except psycopg.errors.ForeignKeyViolation:
raise invalid() from None
return result
@app.get('/api/v1/chats')
def list_chats(request: Request, user_id=Depends(authenticate)):
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'chats:' + str(user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
with request.app.state.db.pool.connection() as conn:
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
result = page(scope, rows, limit)
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
return result
@app.get('/api/v1/chats/{chat_id}')
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
return chat_public(conn, row)
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
key = validate_key(request.headers.get('Idempotency-Key', ''))
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids)
return public(row)
@app.get('/api/v1/chats/{chat_id}/messages')
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'messages:' + str(cid)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (m.created_at,m.id) > (%s,%s)' if after else ''
params = (cid, *after, limit+1) if after else (cid, limit+1)
rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+
' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall()
return page(scope, rows, limit)
@app.post('/api/v1/chats/{chat_id}/attachments',status_code=202)
async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)):
cid=uuid_value(chat_id)
def access():
with request.app.state.db.pool.connection() as conn:
require_member(conn,cid,user_id)
await run_in_threadpool(access)
return await attachments.upload(request,user_id,cid,filename)
@app.get('/api/v1/attachments/{attachment_id}')
def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)):
aid=uuid_value(attachment_id)
with request.app.state.db.pool.connection() as conn:
row=attachments.lookup(conn,aid,user_id)
return attachments.metadata(row)
@app.get('/api/v1/attachments/{attachment_id}/content')
def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)):
aid=uuid_value(attachment_id)
with request.app.state.db.pool.connection() as conn:
row=attachments.lookup(conn,aid,user_id)
if variant not in ('original','thumbnail','crop','optimized'):
raise invalid()
if row['status']!='ready':
raise ApiError(409,'conflict','Вложение ещё не готово')
item=row
key=row['object_key']
if variant!='original':
item=next((v for v in row['variants'] if v['name']==variant),None)
if item is None:
raise not_found()
key='variants/'+str(row['id']).replace('-','')+'/v1/'+variant
opened=request.app.state.storage.get(key)
if opened.length!=item['size_bytes']:
opened.close()
raise unavailable()
return attachments.ObjectResponse(opened,media_type=item['content_type'],headers={
'Content-Length':str(item['size_bytes']),
'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')})
return app
+173
View File
@@ -0,0 +1,173 @@
"""Bounded binary upload, image validation and DB/S3 coordination."""
import asyncio
import hashlib
import logging
import tempfile
import threading
import unicodedata
import warnings
from uuid import uuid4
import psycopg
from PIL import Image, PngImagePlugin, UnidentifiedImageError
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
from starlette.responses import StreamingResponse
from .errors import ApiError, invalid, not_found
from .pagination import public
MAX_BYTES=10485760
MAX_PIXELS=40000000
Image.MAX_IMAGE_PIXELS=MAX_PIXELS
PngImagePlugin.MAX_TEXT_CHUNK=1048576
PngImagePlugin.MAX_TEXT_MEMORY=1048576
warnings.filterwarnings('error',category=Image.DecompressionBombWarning)
DECODER=threading.BoundedSemaphore(1)
LOG=logging.getLogger('messenger')
FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,status,error_code,variants'
def object_lock(conn,key,wait=True):
function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone()
return True if wait else row['locked']
def filename_value(value):
if not value or not value.strip() or len(value)>255:
raise invalid()
try:
value.encode('utf-8')
except UnicodeError:
raise invalid() from None
if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value):
raise invalid()
return value
def metadata(row):
result=public({key:value for key,value in row.items() if key not in ('object_key','staging_key')})
return result
def validate_image(file,mime):
if not DECODER.acquire(timeout=3):
raise ApiError(503,'unavailable','Обработка изображений занята')
try:
file.seek(0)
with Image.open(file) as image:
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format)
if actual is None or actual!=mime:
raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME')
if image.width*image.height>MAX_PIXELS:
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей')
image.verify()
file.seek(0)
except (Image.DecompressionBombWarning,Image.DecompressionBombError):
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None
except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError):
raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None
finally:
DECODER.release()
def compensate(db,store,key):
"""A failed/unknown COMMIT is not proof of absence. Recheck under the same lock."""
try:
with db.pool.connection() as conn:
if not object_lock(conn,key,wait=False):
return
exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s',(key,key)).fetchone()
if not exists:
store.delete(key)
except Exception as exc:
# A periodic, grace-protected janitor will retry after dependency recovery.
LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__)
def persist(db,store,user_id,chat_id,filename,mime,file,length,digest):
aid=uuid4()
key='staging/'+aid.hex
try:
with db.pool.connection() as conn:
object_lock(conn,key)
store.put(key,file,length,mime)
row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,staging_key,status) '
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING "+FIELDS,
(aid,chat_id,user_id,filename,mime,length,digest,'originals/'+aid.hex,key)).fetchone()
job=uuid4()
conn.execute('INSERT INTO attachment_jobs(id,attachment_id) VALUES (%s,%s)',(job,aid))
conn.execute('INSERT INTO job_outbox(job_id) VALUES (%s)',(job,))
return metadata(row)
except psycopg.Error:
compensate(db,store,key)
raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None
except BaseException:
compensate(db,store,key)
raise
async def upload(request,user_id,chat_id,filename):
filename=filename_value(filename)
mime=request.headers.get('Content-Type','').split(';')[0].strip().lower()
if mime not in ('image/png','image/jpeg'):
raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg')
declared=request.headers.get('Content-Length')
if declared is not None:
try:
length=int(declared)
if length<0:
raise ValueError
except ValueError:
raise invalid() from None
if length>MAX_BYTES:
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
try:
await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1)
except TimeoutError:
raise ApiError(503,'unavailable','Слишком много загрузок') from None
try:
with tempfile.TemporaryFile(mode='w+b') as file:
total=0
digest=hashlib.sha256()
async def read_body():
nonlocal total
async for chunk in request.stream():
total+=len(chunk)
if total>MAX_BYTES:
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
digest.update(chunk)
await run_in_threadpool(file.write,chunk)
try:
await asyncio.wait_for(read_body(),timeout=30)
except TimeoutError:
raise ApiError(400,'invalid_request','Время загрузки истекло') from None
except ClientDisconnect:
raise invalid() from None
await run_in_threadpool(validate_image,file,mime)
return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage,
user_id,chat_id,filename,mime,file,total,digest.hexdigest())
finally:
request.app.state.upload_slots.release()
def lookup(conn,attachment_id,user_id):
row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+
' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone()
if not row:
raise not_found()
return row
class ObjectResponse(StreamingResponse):
def __init__(self,opened,**kwargs):
self.opened=opened
super().__init__(opened.chunks(),**kwargs)
async def __call__(self,scope,receive,send):
try:
await super().__call__(scope,receive,send)
finally:
self.opened.close()
+72
View File
@@ -0,0 +1,72 @@
"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock."""
import argparse
from datetime import datetime,timezone,timedelta
import json
from pathlib import Path
import time
from .attachments import object_lock
from .db import Database
from .object_store import ObjectStore
def sweep(db,store,grace_seconds=300,apply=False):
if grace_seconds<60:
raise ValueError('Grace must be at least60seconds')
cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds)
result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds,
'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0}
for page in (page for prefix in ("originals/","staging/","variants/") for page in store.pages(prefix)):
for obj in page.get('Contents',[]):
result['listed']+=1
if obj['LastModified']>=cutoff:
result['fresh']+=1
continue
with db.pool.connection() as conn:
if not object_lock(conn,obj['Key'],wait=False):
result['locked']+=1
continue
if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s OR %s LIKE 'variants/'||replace(id::text,'-','')||'/v1/%%'",(obj['Key'],obj['Key'],obj['Key'])).fetchone():
result['tracked']+=1
continue
result['orphans']+=1
if apply:
store.delete(obj['Key'])
result['deleted']+=1
return result
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--grace-seconds',type=int,default=300)
parser.add_argument('--apply',action='store_true')
parser.add_argument('--dry-run',action='store_true')
parser.add_argument('--loop',action='store_true')
parser.add_argument('--interval',type=int,default=60)
args=parser.parse_args()
if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run):
parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run')
db=Database()
store=ObjectStore()
db.open()
try:
while True:
try:
result=sweep(db,store,args.grace_seconds,args.apply)
print(json.dumps(result),flush=True)
except Exception as exc:
print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True)
if not args.loop:
raise
finally:
Path('/tmp/cleanup-heartbeat').touch()
if not args.loop:
break
time.sleep(args.interval)
finally:
store.close()
db.close()
if __name__=='__main__':
main()
+42
View File
@@ -0,0 +1,42 @@
"""Bounded dependencies; PostgreSQL is always the source of truth."""
import os
from pathlib import Path
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from redis import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
def setting(name, default=None):
filename = os.environ.get(name + '_FILE')
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
if value is None:
raise RuntimeError(f'Missing configuration: {name}')
return value
class Database:
def __init__(self):
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
if self.session_ttl < 1:
raise ValueError('SESSION_TTL_SECONDS must be positive')
self.pool = ConnectionPool(
setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
timeout=2, max_waiting=16, open=False,
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
)
self.cache = Redis.from_url(
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
retry=Retry(NoBackoff(), 0),
)
def open(self):
self.pool.open(wait=True, timeout=30)
def close(self):
self.pool.close()
self.cache.close()
+19
View File
@@ -0,0 +1,19 @@
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid():
return ApiError(400, 'invalid_request', 'Неверный запрос')
def unauthorized(basic=False):
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
def not_found():
return ApiError(404, 'not_found', 'Объект не найден')
+222
View File
@@ -0,0 +1,222 @@
"""Transactional outbox, fenced processing, bounded retries and operator replay."""
from contextlib import nullcontext
import signal
import hashlib
import json
import logging
import os
from pathlib import Path
import random
import subprocess
import sys
import tempfile
from uuid import UUID
from psycopg.types.json import Jsonb
from redis import Redis
from .db import setting
from .attachments import object_lock
LOG=logging.getLogger('messenger')
STREAM='attachment-jobs'
GROUP='image-workers'
MAX_ATTEMPTS=int(os.environ.get('JOB_MAX_ATTEMPTS','5'))
LEASE_SECONDS=90
def broker_client():
return Redis.from_url(setting('BROKER_URL','redis://broker:6379/0'),decode_responses=True,
socket_connect_timeout=1,socket_timeout=3,max_connections=4)
def job_lock(conn,job_id):
return conn.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',
('job:'+str(job_id),)).fetchone()['ok']
def unlock(conn,job_id):
conn.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job_id),))
def release_lock(conn,job_id):
# Session locks survive ROLLBACK. A failed transaction must be cleared before unlock.
try:
conn.rollback()
unlock(conn,job_id)
conn.commit()
except BaseException:
conn.close() # never return a session with an uncertain held lock to the pool
raise
def relay_tick(db,broker):
# Recovery uses the same session lock as the processor. A slow live worker is never fenced out.
with db.pool.connection() as conn:
expired=conn.execute("SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
"WHERE a.status='processing' AND j.lease_until<clock_timestamp() LIMIT 50").fetchall()
for row in expired:
with db.pool.connection() as conn:
if not job_lock(conn,row['id']):
continue
try:
conn.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id="
"(SELECT attachment_id FROM attachment_jobs WHERE id=%s AND lease_until<clock_timestamp()) "
"AND status='processing'",(row['id'],))
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(row['id'],))
conn.execute('UPDATE attachment_jobs SET next_at=clock_timestamp(),lease_until=NULL WHERE id=%s',(row['id'],))
conn.commit()
finally:
release_lock(conn,row['id'])
with db.pool.connection() as conn:
rows=conn.execute("SELECT j.*,o.job_id FROM job_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
"JOIN attachments a ON a.id=j.attachment_id WHERE o.dispatched_at IS NULL AND a.status='queued' "
"AND j.next_at<=clock_timestamp() ORDER BY j.next_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED").fetchall()
for row in rows:
payload={'schema_version':1,'type':'attachment.process.v1','job_id':str(row['id']),
'attachment_id':str(row['attachment_id']),'pipeline_version':row['pipeline_version'],
'attempt':row['attempt']+1,'created_at':row['created_at'].isoformat()}
for key in ('traceparent','tracestate'):
if row[key]: payload[key]=row[key]
broker.xadd(STREAM,{'payload':json.dumps(payload)})
conn.execute('UPDATE job_outbox SET dispatched_at=clock_timestamp() WHERE job_id=%s',(row['id'],))
return len(rows)
def process_objects_local(store,row,directory=None):
with (nullcontext(directory) if directory else tempfile.TemporaryDirectory(prefix='image-job-')) as directory:
source=Path(directory)/'input'
opened=store.get(row['staging_key'])
try:
with source.open('wb') as file:
total=0
for chunk in opened.chunks():
total+=len(chunk)
if total>row['size_bytes'] or total>10485760: raise ValueError('invalid_image')
file.write(chunk)
finally:
opened.close()
raw_hash=hashlib.sha256(source.read_bytes()).hexdigest()
if source.stat().st_size!=row['size_bytes'] or raw_hash!=row['sha256']:
raise ValueError('invalid_image')
completed=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),directory],
capture_output=True,timeout=20)
if completed.returncode:
raise ValueError('invalid_image' if completed.returncode==65 else 'processing_limit')
variants=json.loads(completed.stdout)
with source.open('rb') as file:
store.put(row['object_key'],file,row['size_bytes'],row['content_type'])
for name,item in variants.items():
with (Path(directory)/name).open('rb') as file:
store.put('variants/'+row['id'].hex+'/v1/'+name,file,item['size_bytes'],item['content_type'])
return list(variants.values())
def process_objects(store,row):
# A killed child cannot continue S3 writes or retain a stream/decoder in a detached thread.
timeout=float(os.environ.get('PROCESS_TIMEOUT_SECONDS','45'))
if not 1<=timeout<=45: raise ValueError('processing_limit')
with tempfile.TemporaryDirectory(prefix='image-job-') as directory:
process=subprocess.Popen([sys.executable,'-m','messenger.processor',directory],
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
text=True,start_new_session=True)
try:
output,_=process.communicate(json.dumps(row,default=str),timeout=timeout)
except BaseException:
# Terminate processor AND decoder before lock release or another attempt.
try: os.killpg(process.pid,signal.SIGKILL)
except ProcessLookupError: pass
process.communicate()
raise
if process.returncode==65: raise ValueError('invalid_image')
if process.returncode==66: raise ValueError('processing_limit')
if process.returncode: raise RuntimeError('storage_unavailable')
return json.loads(output)
def work(db,store,job_id,delivery_attempt=None):
job_id=UUID(str(job_id))
# Dedicated pooled session holds the lock, with short committed transactions between effects.
with db.pool.connection() as conn:
if not job_lock(conn,job_id):
return False
conn.commit()
try:
row=conn.execute('SELECT a.*,j.attempt,j.pipeline_version,j.next_at<=clock_timestamp() AS due '
'FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s',(job_id,)).fetchone()
conn.commit()
if not row or row['status'] in ('ready','failed'):
return True
if row['status']!='queued' or not row['due']:
return False
if delivery_attempt is not None and delivery_attempt!=row['attempt']+1:
return True
attempt=row['attempt']+1
if attempt>MAX_ATTEMPTS:
conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
conn.commit()
return True
conn.execute("UPDATE attachment_jobs SET attempt=%s,lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s",
(attempt,LEASE_SECONDS,job_id))
conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
conn.commit()
try:
variants=process_objects(store,row)
except Exception as exc:
permanent=isinstance(exc,ValueError)
final=permanent or attempt>=MAX_ATTEMPTS
code=str(exc) if permanent else ('retry_exhausted' if final else 'storage_unavailable')
if code not in ('invalid_image','processing_limit','retry_exhausted','storage_unavailable'):
code='processing_failed'
delay=min(30,2**(attempt-1))*random.uniform(.5,1.5)
conn.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s",
('failed' if final else 'queued',code if final else None,row['id']))
conn.execute("UPDATE attachment_jobs SET next_at=clock_timestamp()+%s*interval '1 second',lease_until=NULL WHERE id=%s",(delay,job_id))
if not final: conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
conn.commit()
LOG.warning('job_attempt outcome=%s attempt=%s', 'failed' if final else 'retry',attempt)
return True
# Same live lock and attempt fence protect publication; no new attempt can publish concurrently.
current=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone()
if current['attempt']!=attempt:
conn.rollback()
return False
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
(Jsonb(variants),row['id']))
conn.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s',(job_id,))
conn.commit()
LOG.info('job_completed attempt=%s',attempt)
clean_staging(db,store,row['id'])
return True
finally:
release_lock(conn,job_id)
def clean_staging(db,store,attachment_id=None):
with db.pool.connection() as conn:
rows=conn.execute("SELECT id,staging_key FROM attachments WHERE staging_key IS NOT NULL "
"AND (status='ready' OR (status='failed' AND updated_at<clock_timestamp()-interval '24 hours')) "
+('AND id=%s ' if attachment_id else '')+'LIMIT 50', (attachment_id,) if attachment_id else ()).fetchall()
for row in rows:
with db.pool.connection() as conn:
if not object_lock(conn,row['staging_key'],wait=False): continue
current=conn.execute("SELECT status FROM attachments WHERE id=%s FOR UPDATE",(row['id'],)).fetchone()
if current['status'] not in ('ready','failed'): continue
store.delete(row['staging_key'])
conn.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s',(row['id'],))
def replay(db,job_id,reason):
if not reason.strip() or len(reason)>200: raise ValueError('Short audit reason required')
with db.pool.connection() as conn:
if not job_lock(conn,job_id): raise ValueError('Job active')
try:
row=conn.execute('SELECT a.* FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE j.id=%s FOR UPDATE OF a',(job_id,)).fetchone()
if not row or row['status']!='failed' or not row['staging_key']:
raise ValueError('Replay requires failed job with retained staging')
conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
conn.execute('UPDATE attachment_jobs SET attempt=0,replay_count=replay_count+1,lease_until=NULL,next_at=clock_timestamp() WHERE id=%s',(job_id,))
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
conn.execute('INSERT INTO job_replays(job_id,reason) VALUES (%s,%s)',(job_id,reason))
conn.commit()
finally:
release_lock(conn,job_id)
+54
View File
@@ -0,0 +1,54 @@
"""One PostgreSQL transaction owns both a message and its retry result."""
import hashlib
import json
import re
from uuid import uuid4
from .errors import ApiError, invalid, not_found
from .models import uuid_value
def validate_key(key):
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
raise invalid()
return key
MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids'
def store_message(conn, user_id, chat_id, text, key, attachment_ids=None):
attachment_ids=[] if attachment_ids is None else attachment_ids
payload={'text':text}
if attachment_ids:
payload['attachment_ids']=attachment_ids
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
params = (user_id, chat_id, key)
# A competing INSERT waits for the other transaction, then the next statement sees its result.
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
(*params, fingerprint))
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
if record['message_id'] is not None and record['active']:
if record['request_hash'] != fingerprint:
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',
(record['message_id'],)).fetchone()
ids=[uuid_value(item) for item in attachment_ids]
if ids:
allowed=conn.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall()
if len(allowed)!=len(ids):
raise not_found()
if any(a['status']=='failed' for a in allowed):
raise ApiError(409,'conflict','Обработка вложения завершилась ошибкой')
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
if ids:
with conn.cursor() as cur:
cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
[(row['id'],aid,i) for i,aid in enumerate(ids)])
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone()
+29
View File
@@ -0,0 +1,29 @@
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
import hashlib
from pathlib import Path
import psycopg
from .db import setting
def main():
directory = Path(__file__).resolve().parents[2] / 'migrations'
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
for path in sorted(directory.glob('*.sql')):
sql = path.read_text()
checksum = hashlib.sha256(sql.encode()).hexdigest()
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
if found:
if found[0] != checksum:
raise RuntimeError('Applied migration checksum changed: ' + path.name)
continue
conn.execute(sql)
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
print('Applied', path.name)
print('Migrations up to date')
if __name__ == '__main__':
main()
+75
View File
@@ -0,0 +1,75 @@
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from .errors import invalid
def uuid_value(value):
try:
# Require the UUID string representation, not integer/coercible values.
if not isinstance(value, str) or len(value) != 36:
raise ValueError
return UUID(value)
except (ValueError, TypeError, AttributeError):
raise invalid() from None
class StrictModel(BaseModel):
model_config = ConfigDict(extra='forbid', strict=True)
@field_validator('*')
@classmethod
def valid_strings(cls, value, info):
if isinstance(value, str):
if not value.strip() and info.field_name != 'text':
raise ValueError('Blank string')
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
if '\x00' in value:
raise ValueError('NUL is not supported')
value.encode('utf-8')
return value
class CreateUser(StrictModel):
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator('username')
@classmethod
def username_ascii(cls, value):
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
raise ValueError('Invalid username')
return value
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator('member_ids')
@classmethod
def members(cls, value):
ids = [uuid_value(item) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('Duplicate member')
return [str(item) for item in ids]
class CreateMessage(StrictModel):
text: str = Field(max_length=4000)
attachment_ids: list[str] = Field(default_factory=list,max_length=10)
@field_validator('attachment_ids')
@classmethod
def attachment_values(cls,value):
ids=[uuid_value(item) for item in value]
if len(ids)!=len(set(ids)):
raise ValueError('Duplicate attachment')
return [str(item) for item in ids]
@model_validator(mode='after')
def nonempty_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('Message must contain text or attachment')
return self
+109
View File
@@ -0,0 +1,109 @@
"""Small bounded S3 client. API credentials never contain root/admin permissions."""
from contextlib import contextmanager
import threading
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from .db import setting
from .errors import ApiError
PREFIX = 'originals/'
def unavailable():
return ApiError(503,'unavailable','Хранилище файлов временно недоступно')
class OpenObject:
def __init__(self,body,release,length):
self.body=body
self.release=release
self.length=length
self.closed=False
self.close_lock=threading.Lock()
def chunks(self):
try:
while True:
data=self.body.read(65536)
if not data:
break
yield data
finally:
self.close()
def close(self):
with self.close_lock:
if self.closed:
return
self.closed=True
try:
self.body.close()
finally:
self.release()
class ObjectStore:
def __init__(self):
self.bucket=setting('S3_BUCKET','messenger-attachments')
self.gate=threading.BoundedSemaphore(8)
self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'),
region_name=setting('S3_REGION','us-east-1'),
aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'),
config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8,
retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'},
request_checksum_calculation='when_required',response_checksum_validation='when_required'))
@contextmanager
def operation(self):
if not self.gate.acquire(timeout=1):
raise unavailable()
try:
yield
except (BotoCoreError,ClientError):
raise unavailable() from None
finally:
self.gate.release()
def check(self):
with self.operation():
self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1)
def put(self,key,body,length,mime):
with self.operation():
body.seek(0)
self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime)
def delete(self,key):
with self.operation():
self.client.delete_object(Bucket=self.bucket,Key=key)
def get(self,key):
if not self.gate.acquire(timeout=1):
raise unavailable()
try:
result=self.client.get_object(Bucket=self.bucket,Key=key)
return OpenObject(result['Body'],self.gate.release,result['ContentLength'])
except (BotoCoreError,ClientError):
self.gate.release()
raise unavailable() from None
except BaseException:
self.gate.release()
raise
def pages(self,prefix=PREFIX):
token=None
while True:
params={'Bucket':self.bucket,'Prefix':prefix,'MaxKeys':1000}
if token:
params['ContinuationToken']=token
with self.operation():
result=self.client.list_objects_v2(**params)
yield result
if not result.get('IsTruncated'):
break
token=result['NextContinuationToken']
def close(self):
self.client.close()
+56
View File
@@ -0,0 +1,56 @@
import base64
import json
import re
from datetime import datetime, timezone
from uuid import UUID
from .errors import invalid
from .models import uuid_value
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def encode_cursor(scope, row):
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
def decode_cursor(scope, value):
if value is None:
return None
try:
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
raise ValueError
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
raise ValueError
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
if ts.tzinfo is None:
raise ValueError
return ts, uuid_value(data[2])
except (ValueError, TypeError, AttributeError, OverflowError):
raise invalid() from None
def page_limit(value):
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
raise invalid()
return int(value)
def public(row):
def value(item):
if isinstance(item,datetime):
return stamp(item)
if isinstance(item,UUID):
return str(item)
if isinstance(item,list):
return [value(part) for part in item]
return item
return {key:value(item) for key,item in row.items()}
def page(scope, rows, limit):
return {'items': [public(row) for row in rows[:limit]],
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
+49
View File
@@ -0,0 +1,49 @@
"""Bounded image decoder subprocess. Original bytes are never re-encoded."""
import hashlib
import json
import resource
import sys
from pathlib import Path
from PIL import Image, ImageOps, PngImagePlugin
def fitted(image, maximum):
scale=min(1,maximum/image.width,maximum/image.height)
size=(max(1,int(image.width*scale)),max(1,int(image.height*scale)))
return image.resize(size,Image.Resampling.LANCZOS)
def process_file(source, directory):
Image.MAX_IMAGE_PIXELS=40000000
PngImagePlugin.MAX_TEXT_CHUNK=1048576
PngImagePlugin.MAX_TEXT_MEMORY=1048576
with Image.open(source) as original:
if original.format not in ('PNG','JPEG') or original.width*original.height>40000000:
raise ValueError('invalid_image')
image=ImageOps.exif_transpose(original)
# Alpha is composited on white; output carries no source metadata.
rgba=image.convert('RGBA')
white=Image.new('RGBA',rgba.size,'white')
white.alpha_composite(rgba)
image=white.convert('RGB')
image.info.clear()
outputs={'thumbnail':(fitted(image,256),'JPEG','image/jpeg'),
'crop':(ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(.5,.5)),'JPEG','image/jpeg'),
'optimized':(fitted(image,1280),'WEBP','image/webp')}
manifest={}
for name,(variant,fmt,mime) in outputs.items():
path=Path(directory)/name
variant.save(path,format=fmt,quality=80 if fmt=='WEBP' else 85,exif=b'')
raw=path.read_bytes()
manifest[name]={'name':name,'content_type':mime,'width':variant.width,'height':variant.height,
'size_bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}
return manifest
if __name__=='__main__':
resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024))
resource.setrlimit(resource.RLIMIT_CPU,(15,15))
try:
print(json.dumps(process_file(sys.argv[1],sys.argv[2])))
except (OSError,ValueError,MemoryError,Image.DecompressionBombError):
sys.exit(65)
+29
View File
@@ -0,0 +1,29 @@
"""Isolated complete S3+decode+publish effect, terminated by parent after 45 seconds."""
import os
import signal
import json
import sys
from uuid import UUID
from .jobs import process_objects_local
from .object_store import ObjectStore
def main():
# Survives SIGKILL of the supervising worker; kill the isolated effect group at deadline.
signal.signal(signal.SIGALRM,lambda *_:os.killpg(os.getpgrp(),signal.SIGKILL))
signal.alarm(45)
row=json.load(sys.stdin)
row['id']=UUID(row['id'])
store=ObjectStore()
try:
print(json.dumps(process_objects_local(store,row,sys.argv[1])))
except ValueError as exc:
sys.exit(66 if str(exc)=='processing_limit' else 65)
except Exception:
# Do not export credentials, storage keys or SDK response details.
sys.exit(75)
finally:
store.close()
if __name__=='__main__': main()
+49
View File
@@ -0,0 +1,49 @@
import base64
import hashlib
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, InvalidHashError
from .errors import unauthorized
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
def digest(token):
return hashlib.sha256(token.encode()).hexdigest()
def basic_credentials(header):
try:
scheme, encoded = header.split(' ', 1)
if scheme.lower() != 'basic':
raise ValueError
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
raise ValueError
return username, password
except (ValueError, UnicodeError):
raise unauthorized(basic=True) from None
def check_password(encoded, password):
try:
return HASHER.verify(encoded, password)
except (VerificationError, InvalidHashError):
return False
def bearer(header):
try:
scheme, token = header.split(' ', 1)
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise ValueError
return token
except ValueError:
raise unauthorized() from None
def new_session(ttl):
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
+69
View File
@@ -0,0 +1,69 @@
"""Separate relay/consumer processes; durable Redis Streams consumer group."""
import argparse
import json
import logging
import signal
import time
from uuid import uuid4
from redis.exceptions import ResponseError
from .cleanup import sweep
from .db import Database
from .jobs import broker_client, relay_tick, work, clean_staging, replay, STREAM, GROUP
from .object_store import ObjectStore
LOG=logging.getLogger('messenger')
STOP=False
def main():
parser=argparse.ArgumentParser()
parser.add_argument('role',choices=['worker','relay','replay'])
parser.add_argument('--job-id')
parser.add_argument('--reason',default='operator retry')
args=parser.parse_args()
logging.basicConfig(level=logging.INFO)
def stop(*_):
global STOP
STOP=True
signal.signal(signal.SIGTERM,stop)
signal.signal(signal.SIGINT,stop)
db=Database();db.open();store=ObjectStore();broker=broker_client()
consumer=uuid4().hex
last_cleanup=0
try:
if args.role=='replay':
replay(db,args.job_id,args.reason)
print('Replay queued with audit')
return
if args.role=='worker':
try: broker.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
except ResponseError as exc:
if 'BUSYGROUP' not in str(exc): raise
while not STOP:
try:
if args.role=='relay':
relay_tick(db,broker)
clean_staging(db,store)
if time.monotonic()-last_cleanup>60:
sweep(db,store,300,True)
last_cleanup=time.monotonic()
time.sleep(.3)
else:
claimed=broker.xautoclaim(STREAM,GROUP,consumer,min_idle_time=90000,start_id='0-0',count=1)
messages=claimed[1]
if not messages:
batches=broker.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
messages=batches[0][1] if batches else []
for message_id,fields in messages:
payload=json.loads(fields['payload'])
if work(db,store,payload['job_id'],payload['attempt']):
broker.xack(STREAM,GROUP,message_id)
broker.xdel(STREAM,message_id)
except Exception as exc:
LOG.warning('background_unavailable role=%s type=%s',args.role,type(exc).__name__)
time.sleep(1)
finally:
broker.close();store.close();db.close()
if __name__=='__main__': main()
+141
View File
@@ -0,0 +1,141 @@
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
import base64
import json
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import pytest
from fastapi.testclient import TestClient
from messenger.app import create_app
PASSWORD = 'Тестовый:пароль🔐' * 8
@pytest.fixture
def client():
with TestClient(create_app()) as c:
yield c
def user(c):
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
r = c.post('/api/v1/users', json=body)
assert r.status_code == 201, r.text
return r.json(), body
def login(c, body):
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
assert r.status_code == 201, r.text
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
def test_auth_unicode_logout_and_cache(client):
u, body = user(client)
h, session = login(client, body)
assert session['user'] == u
assert len(session['token']) >= 43
key = 'user:' + u['id']
client.app.state.db.cache.delete(key)
for _ in range(2):
assert client.get('/api/v1/users/me', headers=h).json() == u
cached = json.loads(client.app.state.db.cache.get(key))
assert cached == u
assert 0 < client.app.state.db.cache.ttl(key) <= 60
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
r = client.get('/api/v1/users/me', headers=h)
assert r.status_code == 401
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
assert r.json()['error']['request_id'] == r.headers['x-request-id']
def test_session_expiry(monkeypatch):
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
with TestClient(create_app()) as client:
_, body = user(client)
h, _ = login(client, body)
assert client.get('/api/v1/users/me', headers=h).status_code == 200
time.sleep(2.1)
assert client.get('/api/v1/users/me', headers=h).status_code == 401
def test_registration_race(client):
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
with ThreadPoolExecutor(max_workers=6) as executor:
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
def test_chat_rollback_after_insert(client):
u, body = user(client)
h, _ = login(client, body)
title = 'rollback_' + uuid.uuid4().hex
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
assert r.status_code == 400
with client.app.state.db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
def test_cursor_access_and_strict_validation(client):
a, ab = user(client)
b, bb = user(client)
ah, _ = login(client, ab)
bh, _ = login(client, bb)
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
path = '/api/v1/chats/' + chats[0]['id']
for text in ['Один 👋', 'Два', 'Три']:
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
p = client.get(path+'/messages?limit=2', headers=ah).json()
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
assert [m['text'] for m in tail['items']] == ['Три']
assert tail['next_cursor'] is None
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
assert r.status_code == 400
assert client.get(path, headers=bh).status_code == 404
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
assert client.post('/api/v1/users',json=payload).status_code == 400
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
def test_session_storage_and_migrations(client):
from messenger.migrate import main as migrate
u, body = user(client)
_, session = login(client, body)
db = client.app.state.db
def snapshot():
with db.pool.connection() as conn:
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
before = snapshot()
migrate()
migrate()
assert snapshot() == before
with db.pool.connection() as conn:
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
assert row['token_hash'] != session['token']
assert len(row['token_hash']) == 64
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
assert stored.startswith('$argon2id$')
assert stored != body['password']
def test_json_media_type_and_utf8(client):
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
raw = json.dumps(body).encode()
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
r = client.post('/api/v1/users',content=raw,headers=headers)
assert r.status_code == 415
assert r.json()['error']['code'] == 'unsupported_media_type'
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
assert r.status_code == 400, encoding
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
assert r.status_code == 401
+187
View File
@@ -0,0 +1,187 @@
"""Real PostgreSQL + private S3 integration; synthetic images only."""
import hashlib
import io
import struct
import uuid
import zlib
from PIL import Image
from psycopg import sql
from test_api import client, user, login
def image_bytes(kind='PNG'):
stream = io.BytesIO()
Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind)
return stream.getvalue()
def actors(c):
a,ab=user(c)
b,bb=user(c)
e,eb=user(c)
ah,_=login(c,ab)
bh,_=login(c,bb)
eh,_=login(c,eb)
chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json()
return ah,bh,eh,chat
def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'):
raw=image_bytes() if raw is None else raw
return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw,
headers={**headers,'Content-Type':mime})
def finish(client,attachment_id):
from messenger.jobs import work
with client.app.state.db.pool.connection() as conn:
job=conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(attachment_id,)).fetchone()['id']
work(client.app.state.db,client.app.state.storage,job)
import time
for _ in range(100):
with client.app.state.db.pool.connection() as conn:
status=conn.execute('SELECT status FROM attachments WHERE id=%s',(attachment_id,)).fetchone()['status']
if status=='ready': return
time.sleep(.1)
raise AssertionError('Job did not become ready: '+status)
def test_original_png_jpeg_and_message(client):
ah,bh,eh,chat=actors(client)
for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]:
raw=image_bytes(kind)
r=upload(client,ah,chat,raw,mime)
assert r.status_code==202
att=r.json()
assert att['status']=='queued' and att['variants']==[] and att['error_code'] is None
assert client.get('/api/v1/attachments/'+att['id']+'/content',headers=bh).status_code==409
finish(client,att['id'])
att=client.get('/api/v1/attachments/'+att['id'],headers=bh).json()
assert att['status']=='ready' and len(att['variants'])==3
assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw)
assert 'object_key' not in att and 'bucket' not in att
path='/api/v1/attachments/'+att['id']
assert client.get(path,headers=bh).json()==att
content=client.get(path+'/content',headers=bh)
assert content.status_code==200 and content.content==raw
assert content.headers['Content-Type']==mime
for suffix in ['', '/content']:
assert client.get(path+suffix,headers=eh).status_code==404
assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==200
response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},
json={'text':'','attachment_ids':[att['id']]})
assert response.status_code==201
assert response.json()['attachment_ids']==[att['id']]
def test_invalid_media_corruption_limits_and_filename(client):
ah,_,_,chat=actors(client)
cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415),
(image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415),
(image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)]
for raw,mime,expected in cases:
response=upload(client,ah,chat,raw,mime)
assert response.status_code==expected
assert response.json()['error']['request_id']==response.headers['X-Request-Id']
# Valid PNG header advertises >40Mp; reject before attempting decompression.
header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0)
chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff)
bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'')
assert upload(client,ah,chat,bomb).status_code==413
for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]:
assert upload(client,ah,chat,filename=filename).status_code==400
def test_foreign_attachment_binding_and_normalized_replay(client):
ah,bh,eh,chat=actors(client)
att=upload(client,ah,chat).json()
other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json()
path='/api/v1/chats/'+other['id']+'/messages'
for aid in [att['id'],str(uuid.uuid4())]:
response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]})
assert response.status_code==404
assert upload(client,eh,chat).status_code==404
h={**ah,'Idempotency-Key':uuid.uuid4().hex}
path='/api/v1/chats/'+chat['id']+'/messages'
first=client.post(path,headers=h,json={'text':'same'})
replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'})
assert first.status_code==replay.status_code==201 and first.json()==replay.json()
conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]})
assert conflict.status_code==409
duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]})
assert duplicates.status_code==400
def test_compensate_s3_when_metadata_insert_fails(client):
ah,_,_,chat=actors(client)
db=client.app.state.db
store=client.app.state.storage
before={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
function='fail_attachment_'+uuid.uuid4().hex
trigger='fail_attachment_'+uuid.uuid4().hex
filename='fault_'+uuid.uuid4().hex+'.png'
with db.pool.connection() as conn:
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename)))
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
try:
response=upload(client,ah,chat,filename=filename)
assert response.status_code==503
with db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0
after={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
assert after==before
finally:
with db.pool.connection() as conn:
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger)))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
def test_compensation_preserves_committed_object(client):
from messenger.attachments import compensate
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
with client.app.state.db.pool.connection() as conn:
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
# A retry after an unknown COMMIT result must never delete an already tracked object.
compensate(client.app.state.db,client.app.state.storage,key)
content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah)
assert content.status_code==200 and content.content==image_bytes()
def test_stream_close_even_if_headers_cannot_be_sent(client):
import asyncio
import pytest
from messenger.attachments import ObjectResponse
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
with client.app.state.db.pool.connection() as conn:
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
store=client.app.state.storage
opened=store.get(key)
response=ObjectResponse(opened,media_type='image/png')
async def fail_send(message):
raise OSError('synthetic disconnected transport')
async def receive():
return {'type':'http.disconnect'}
with pytest.raises(Exception):
asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send))
assert opened.closed
acquired=0
try:
for _ in range(8):
assert store.gate.acquire(blocking=False)
acquired+=1
finally:
for _ in range(acquired):
store.gate.release()
def test_chunked_upload_is_limited_without_content_length(client):
ah,_,_,chat=actors(client)
chunks=iter([b'x'*1048576]*10+[b'x'])
response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png',
content=chunks,headers={**ah,'Content-Type':'image/png'})
assert response.status_code==413
assert response.json()['error']['code']=='payload_too_large'
+119
View File
@@ -0,0 +1,119 @@
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
from concurrent.futures import ThreadPoolExecutor
import uuid
from psycopg import sql
from test_api import client, user, login
def setup_chat(c, extra=None):
u, body = user(c)
headers, _ = login(c, body)
members = [u['id']] + (extra or [])
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
def test_instance_alias_on_success_and_errors(client):
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
ids = [r.headers.get('X-Instance-Id') for r in responses]
assert ids[0] and len(set(ids)) == 1
assert all(r.headers.get('X-Request-Id') for r in responses)
def test_key_required_replay_and_conflict(client):
_, headers, path = setup_chat(client)
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
for key in ['', 'a b', 'bad/', 'x'*129]:
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
h = {**headers, 'Idempotency-Key':'valid:._-123'}
first = client.post(path, headers=h, json={'text':'same'})
assert first.status_code == 201
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
assert replay.status_code == 201 and replay.json() == first.json()
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
assert len(client.get(path,headers=headers).json()['items']) == 1
def test_concurrent_key_produces_one_message(client):
_, headers, path = setup_chat(client)
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
assert [r.status_code for r in results] == [201]*16
assert all(r.json() == results[0].json() for r in results)
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
def test_scope_and_replay_rechecks_authorization(client):
b, bb = user(client)
bh, _ = login(client, bb)
a, ah, path = setup_chat(client, [b['id']])
key = 'same-in-independent-scopes'
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
assert second.status_code == 201 and second.json()['id'] != first['id']
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
other_path = '/api/v1/chats/'+chat['id']+'/messages'
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
assert third.status_code == 201 and third.json()['id'] != first['id']
outsider, ob = user(client)
oh, _ = login(client, ob)
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
def test_expired_key_can_be_reused(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
with client.app.state.db.pool.connection() as conn:
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
assert 86390 <= ttl <= 86400
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
assert second.status_code == 201 and second.json()['id'] != first['id']
assert len(client.get(path,headers=h).json()['items']) == 2
def test_rollback_side_effect_and_key_on_database_error(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
text = 'fault_' + uuid.uuid4().hex
function = 'fault_' + uuid.uuid4().hex
trigger = 'trigger_' + uuid.uuid4().hex
db = client.app.state.db
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
with db.pool.connection() as conn:
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
try:
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert failed.status_code == 500
assert failed.headers['X-Instance-Id']
with db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
finally:
with db.pool.connection() as conn:
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert retried.status_code == 201
assert client.get(path,headers=h).json()['items'] == [retried.json()]
def test_ready_requires_compatible_schema(client):
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
try:
r = client.get('/health/ready')
assert r.status_code == 503
assert r.json()['error']['code'] == 'unavailable'
assert client.get('/health/live').status_code == 200
finally:
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))
+63
View File
@@ -0,0 +1,63 @@
import io
import pytest
from messenger.jobs import work, replay
from test_attachments import actors,upload,image_bytes,finish
from test_api import client
def job_for(client,aid):
with client.app.state.db.pool.connection() as conn:
return conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(aid,)).fetchone()['id']
def test_duplicate_ready_is_noop_and_not_replayable(client):
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
path='/api/v1/attachments/'+att['id']
before=client.get(path,headers=ah).json()
job=job_for(client,att['id'])
for _ in range(3): assert work(client.app.state.db,client.app.state.storage,job,1)
assert client.get(path,headers=ah).json()==before
with pytest.raises(ValueError): replay(client.app.state.db,job,'must reject ready')
with client.app.state.db.pool.connection() as conn:
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
assert row['attempt']==1
assert conn.execute('SELECT staging_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['staging_key'] is None
def test_old_delivery_does_not_consume_attempt(client):
# Direct invocation avoids consumer timing; it exercises the same real DB/S3 code.
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
job=job_for(client,att['id'])
assert work(client.app.state.db,client.app.state.storage,job,999)
with client.app.state.db.pool.connection() as conn:
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
assert row['attempt'] in (0,1) # independent live consumer may already finish this job
finish(client,att['id'])
def test_replay_sql_failure_releases_session_lock(client):
import psycopg
from messenger.db import setting
from psycopg.rows import dict_row
ah,_,_,chat=actors(client)
att=upload(client,ah,chat).json()
finish(client,att['id'])
job=job_for(client,att['id'])
db=client.app.state.db
with db.pool.connection() as conn:
conn.execute("UPDATE attachments SET status='failed',staging_key=%s WHERE id=%s",('staging/'+att['id'],att['id']))
conn.execute("CREATE FUNCTION lab5_fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic audit failure'; END $$")
conn.execute('CREATE TRIGGER lab5_fail_audit BEFORE INSERT ON job_replays FOR EACH ROW EXECUTE FUNCTION lab5_fail_audit()')
try:
with pytest.raises(psycopg.Error): replay(db,job,'test failed transaction unlock')
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as other:
acquired=other.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',('job:'+str(job),)).fetchone()['ok']
if acquired: other.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job),))
assert acquired, 'Aborted replay leaked a session advisory lock into the pool'
finally:
with db.pool.connection() as conn:
conn.execute('DROP TRIGGER lab5_fail_audit ON job_replays')
conn.execute('DROP FUNCTION lab5_fail_audit()')
+72
View File
@@ -0,0 +1,72 @@
"""Real image transformations, no mock storage."""
import hashlib
import io
import json
import subprocess
import sys
from PIL import Image
def test_worker_variants_and_original(tmp_path):
source=tmp_path/'input'
original=Image.new('RGBA',(320,200),(30,90,150,70))
original.save(source,format='PNG')
before=hashlib.sha256(source.read_bytes()).hexdigest()
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
assert result.returncode==0, result.stderr
manifest=json.loads(result.stdout)
assert hashlib.sha256(source.read_bytes()).hexdigest()==before
for kind,fmt,size in [('thumbnail','JPEG',(256,160)),('crop','JPEG',(128,128)),('optimized','WEBP',(320,200))]:
item=manifest[kind]
with Image.open(tmp_path/kind) as image:
assert image.format==fmt and image.size==size
assert not image.getexif()
raw=(tmp_path/kind).read_bytes()
assert item['sha256']==hashlib.sha256(raw).hexdigest()
assert item['size_bytes']==len(raw)
def test_no_upscale_thumbnail_and_exif(tmp_path):
source=tmp_path/'input'
image=Image.new('RGB',(80,40),'red')
exif=Image.Exif();exif[274]=6
image.save(source,format='JPEG',exif=exif)
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
assert result.returncode==0, result.stderr
with Image.open(tmp_path/'thumbnail') as output:
assert output.size==(40,80)
assert not output.getexif()
def test_deadline_covers_slow_s3(monkeypatch):
import contextlib
import http.server
import os
import threading
import time
import uuid
import pytest
from messenger.jobs import process_objects
class Slow(http.server.BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.send_header('Content-Length','100')
self.end_headers()
try:
for _ in range(100):
self.wfile.write(b'x');self.wfile.flush();time.sleep(.05)
except (BrokenPipeError,ConnectionResetError): pass
def log_message(self,*args): pass
server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Slow)
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
monkeypatch.delenv('S3_ENDPOINT_FILE',raising=False)
monkeypatch.setenv('S3_ENDPOINT','http://127.0.0.1:'+str(server.server_port))
monkeypatch.setenv('PROCESS_TIMEOUT_SECONDS','2')
row={'id':uuid.uuid4(),'staging_key':'staging/slow','object_key':'originals/slow',
'size_bytes':100,'sha256':hashlib.sha256(b'x'*100).hexdigest(),'content_type':'image/png'}
started=time.monotonic()
try:
with pytest.raises(subprocess.TimeoutExpired): process_objects(__import__("messenger.object_store",fromlist=["ObjectStore"]).ObjectStore(),row)
assert time.monotonic()-started<4
finally:
server.shutdown();server.server_close()