ЛР 5 — БПИ2303 — Лукьянов — на 4 #43
@@ -18,3 +18,6 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
|
||||
.superpowers
|
||||
.pytest_cache
|
||||
|
||||
+4
-24
@@ -1,25 +1,5 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# bootstrap creates .env and private files in secrets/; never commit local values.
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
# Compose configures DATABASE_URL_FILE/CACHE_URL_FILE/S3_*_FILE for the application.
|
||||
# Broker is separate from the optional profile cache; persistence AOF + appendfsync always.
|
||||
# Processing limits: one worker, prefetch 1, decoder timeout 20s, lease 90s, 5 attempts.
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
name: Application acceptance
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
pipeline:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- run: make up
|
||||
- run: make check validate-container test test-student test-fresh
|
||||
- run: make test-jobs
|
||||
- run: make test-cluster
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
with:
|
||||
name: lab5-evidence
|
||||
path: evidence/
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
FROM python:3.13.5-slim
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
|
||||
WORKDIR /app
|
||||
COPY requirements.txt requirements-dev.txt requirements.lock ./
|
||||
RUN pip install --no-cache-dir -r requirements.lock -r requirements.txt && useradd --uid 10001 --create-home messenger
|
||||
COPY src/ src/
|
||||
COPY migrations/ migrations/
|
||||
COPY student-tests/ student-tests/
|
||||
COPY scripts/ scripts/
|
||||
COPY tests/ tests/
|
||||
COPY contracts/ contracts/
|
||||
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
|
||||
ARG APP_RELEASE=lab5-v1
|
||||
ENV APP_RELEASE=${APP_RELEASE}
|
||||
LABEL org.opencontainers.image.version=${APP_RELEASE}
|
||||
USER messenger
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
|
||||
@@ -28,3 +28,38 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
.PHONY: bootstrap up down test-student test-cluster rolling
|
||||
bootstrap:
|
||||
./scripts/bootstrap.sh
|
||||
up: bootstrap
|
||||
docker compose build api minio minio-init
|
||||
docker compose up -d --wait --wait-timeout 120
|
||||
$(PYTHON) scripts/cluster.py wait
|
||||
down:
|
||||
docker compose down
|
||||
test-student:
|
||||
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
test-cluster:
|
||||
$(PYTHON) scripts/verify_cluster.py
|
||||
rolling:
|
||||
$(PYTHON) scripts/cluster.py rolling
|
||||
|
||||
.PHONY: test-fresh load-evidence verify-persistence verify-failover
|
||||
test-fresh:
|
||||
./scripts/check_fresh_migrations.sh
|
||||
load-evidence:
|
||||
$(PYTHON) scripts/capture_load.py
|
||||
verify-persistence:
|
||||
$(PYTHON) scripts/verify_cluster.py persistence
|
||||
verify-failover:
|
||||
$(PYTHON) scripts/verify_cluster.py public-failover
|
||||
$(PYTHON) scripts/verify_cluster.py failover
|
||||
|
||||
.PHONY: validate-container
|
||||
validate-container:
|
||||
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
|
||||
|
||||
.PHONY: test-jobs
|
||||
test-jobs:
|
||||
$(PYTHON) scripts/verify_jobs.py
|
||||
|
||||
@@ -1,37 +1,105 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Лукьянов К.Р. БПИ2303
|
||||
- Целевая оценка: **4**; трек 5 не заявлен.
|
||||
- Commit/PR: заполняется автором после самостоятельного commit/push.
|
||||
- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, boto3/botocore, Pillow, pytest; [Redis Streams](https://redis.io/docs/latest/develop/data-types/streams/), [Pillow ImageOps](https://pillow.readthedocs.io/en/stable/reference/ImageOps.html), [S3 uploads](https://docs.aws.amazon.com/AmazonS3/latest/userguide/upload-objects.html), [Nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html). Реализация выполнена совместно мной и Codex. Часть кода, тестов и отчёта написана с помощью Codex; результат проверен командами ниже.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Нужны Docker с Compose v2, Make, Python 3.9+ на хосте, свободный localhost:8080. Все Python-зависимости и Go для MinIO устанавливаются в сборочных контейнерах. Первая сборка требует доступа к registry, GitHub и пакетным репозиториям. Стенды предыдущих лабораторных следует остановить, сохранив volumes.
|
||||
|
||||
```sh
|
||||
git clone <fork-url>
|
||||
cd <fork-lab5>
|
||||
make up
|
||||
curl -fsS http://localhost:8080/health/ready
|
||||
make check
|
||||
make validate-container
|
||||
make test
|
||||
make test-student
|
||||
make test-fresh
|
||||
make test-jobs
|
||||
make test-cluster
|
||||
```
|
||||
|
||||
`make up` создаёт локальные secrets, собирает API и MinIO/mc, запускает миграции и S3 bootstrap, затем две API, Nginx, отдельные broker/relay/worker. Версии образов и лимиты закреплены в compose.yaml, Python-зависимости — requirements.lock. MinIO и mc собираются из официальных release-исходников с проверкой commit: готовые образы/архив при выполнении lab4 были недоступны. Go на хосте не нужен. MinIO: RELEASE.2025-04-22T22-12-26Z, mc: RELEASE.2025-04-16T18-13-26Z; исходники и проверки — infra/minio/Dockerfile.
|
||||
|
||||
`make down` сохраняет volumes. `docker compose down --volumes` удаляет все данные текущего учебного проекта. `make test-fresh` использует отдельный случайный Compose project и удаляет только его временные volumes. Сценарии отказов запускать последовательно, они меняют состояние контейнеров. Синтетические записи обычных тестов остаются в БД.
|
||||
|
||||
Операторский replay без публичного административного endpoint:
|
||||
|
||||
```sh
|
||||
# job_id получить локально из attachment_jobs; токен пользователя не нужен.
|
||||
docker compose exec -T worker python -m messenger.worker replay \
|
||||
--job-id <uuid> --reason 'Причина после устранения неисправности'
|
||||
```
|
||||
|
||||
Разрешён только failed с сохранённым staging. Ready replay отклоняется; audit пишется в job_replays. После retention источника replay явно запрещён.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент → Nginx → API×2 → MinIO staging + PostgreSQL metadata/job/outbox
|
||||
↓ relay
|
||||
Redis Streams broker
|
||||
↓ worker
|
||||
MinIO originals + thumbnail/crop/WebP → PG ready
|
||||
```
|
||||
|
||||
Из lab4 перенесены API, auth/ACL, PostgreSQL, Redis-кэш профилей, Nginx, общие сессии, keyset и идемпотентность сообщений. Контракты и публичные tests текущей лабы сохранены. Объявленный переход: upload возвращает **202 queued**, content для queued/processing/failed — **409**, ready доступен через GET metadata. Сообщение может ссылаться на queued/processing, failed прикреплять нельзя.
|
||||
|
||||
**Граница 202.** Тело ограничено 10 MiB и 40 млн пикселей. Проверяются MIME, структура и имя. Resize и полная декодировка выполняются worker. API сначала PUT в приватный `staging/<uuid>`, затем в одной PG-транзакции создаёт attachment, job и outbox; только после COMMIT возвращает снимок queued. S3 и PG не образуют общей транзакции. Компенсация повторно проверяет metadata под lock, при недоступной БД слепого удаления нет. Неучтённый staging подберёт grace-cleanup.
|
||||
|
||||
**Outbox.** Relay выбирает due записи под `FOR UPDATE SKIP LOCKED`, делает XADD и сохраняет dispatched_at. Сбой до XADD оставляет намерение; после XADD до COMMIT допускается дубль. Сообщение соответствует jobs.schema.json: постоянный job_id, attachment_id, schema/type, pipeline_version=1, attempt, created_at, опциональный W3C carrier. Байтов, storage URL и credentials в сообщении нет.
|
||||
|
||||
**Устойчивость брокера.** Отдельный Redis 7.4.4 с volume, AOF и appendfsync always, noeviction. Redis-кэш профилей остаётся отдельным сервисом. Consumer group обеспечивает at least once; prefetch 1. XACK и затем XDEL выполняются после COMMIT результата либо устойчивого решения о retry/failed. Безусловного MAXLEN нет. Publisher acceptance не равен consumer ack. Успешный job остаётся в PG для проверки дублей.
|
||||
|
||||
**Обработка и варианты.** Worker получает доверенные metadata из PG, сверяет staging SHA-256/размер. Детерминированные ключи оригинала и `(attachment,pipeline=1,variant)` исключают размножение вариантов. EXIF orientation применяется, alpha компонуется на белом, EXIF удаляется. Thumbnail: JPEG quality85, fit 256×256 без увеличения, floor/min1. Crop: центрированный квадрат 128×128 JPEG quality85, допускает увеличение. Optimized: WebP quality80, fit 1280×1280 без увеличения; уменьшение любого маленького файла не обещается. Оригинал побайтно сохраняется. Все варианты записываются до одной транзакции ready/checksums.
|
||||
|
||||
**Попытки и блокировки.** PG session advisory lock удерживается через весь эффект. Lease 90s, attempt проверяется перед COMMIT. Relay восстанавливает expired processing только после получения того же lock; живой worker не вытесняется. SQL-ошибка сначала rollback, затем unlock; при неизвестном результате освобождения соединение закрывается, а не возвращается в pool. Дубль ready — no-op, старый attempt не расходует новую попытку. Постоянная ошибка изображения даёт failed, временная — retry. По умолчанию максимум 5 попыток, exponential backoff с jitter, верхняя граница задержки зависит от номера попытки и ограничена конфигурацией алгоритма.
|
||||
|
||||
**Ограничение ресурсов.** Worker — 1 CPU/512 MiB, concurrency/prefetch 1. Весь цикл S3+decode+PUT выполняется в отдельной process group с абсолютным deadline 45s; supervisor убивает и дожидается всей группы до освобождения job-lock. Дочерний processor имеет собственный 45s alarm на случай смерти supervisor. Decoder дополнительно ограничен wall time 20s, CPU 15s и RLIMIT_AS 768 MiB; фактический общий container memory limit — 512 MiB. Временный каталог принадлежит supervisor и удаляется после завершения/timeout. Ограничения могут закончить особенно тяжёлый допустимый по пикселям файл контролируемым failed; unlimited RAM не обещается.
|
||||
|
||||
**Очистка.** Staging удаляется после ready COMMIT; ключ сохраняется в metadata до успешного DeleteObject, relay повторяет очистку. Для failed источник хранится 24h, затем удаляется и replay запрещается. Grace 300s для неучтённых объектов; lock и свежая проверка PG сохраняют активные uploads, учтённые originals/variants/staging. БД недоступна — удаления нет. Частичные детерминированные варианты failed сохраняются до операторского решения; полного reconciler/retention всех вложений нет.
|
||||
|
||||
MinIO приватен, console не опубликована. API/worker получают ограниченный S3 user для originals/staging/variants; root secrets только у MinIO/init. Единственный бизнес-порт — localhost:8080 Nginx. Брокер, API и хранилища не публикуют host ports. Приложение/worker non-root; secrets генерируются локально, не входят в Git/build context.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Прогоны 2026-11-09 на Docker Desktop/macOS Apple Silicon. Ветка `solution/bpi2303-lukyanov`; commits/push выполняет автор.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Проверка | Команда / доказательство | Результат |
|
||||
| --- | --- | --- |
|
||||
| Публичный контракт и API | make check, validate-container, test; evidence/check.txt, openapi.txt, smoke.txt | PASS |
|
||||
| Регрессии, варианты, ACL, лимиты и SQL rollback | make test-student; evidence/student-tests.txt | 27 тестов PASS |
|
||||
| Пустые volumes и повторные миграции 001–004 | make test-fresh; evidence/fresh-migrations.txt | PASS, 27 тестов |
|
||||
| COMMIT без publish, worker stop, API/broker restart | make test-jobs; evidence/jobs-faults.json | та же принятая job → ready |
|
||||
| Устойчивая непрочитанная очередь | evidence/jobs-faults.json | broker restart с volume, затем ready |
|
||||
| PUT до COMMIT/ack и lease recovery | evidence/jobs-faults.json | S3-эффект есть до kill, та же job завершена со второй попытки |
|
||||
| Дубль ready | собственный тест + evidence/jobs-faults.json | metadata неизменны, ready replay запрещён |
|
||||
| Ограниченный retry/failed/replay | evidence/jobs-faults.json | 5 ошибок → retry_exhausted, failed attachment → 409, явный replay → ready |
|
||||
| Session lock после SQL-ошибки | test_replay_sql_failure_releases_session_lock | независимая PG-сессия получает lock |
|
||||
| Абсолютный timeout S3 | test_deadline_covers_slow_s3 | drip-feed не удерживает pipeline за deadline |
|
||||
| Сохранённые restart/failover/rolling | make test-cluster; evidence/*.json | результаты отдельных сценариев в evidence |
|
||||
|
||||
В pytest остаются предупреждения botocore об utcnow и диагностический Pillow warning теста pixel-bomb; assertions проходят. Публичные тесты не ослаблялись. RED перед реализацией: публичный lab5 smoke против lab4 ожидал 202 и получил 201 — evidence/lab4-baseline-red.txt.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
`make test-jobs` реально останавливает relay/worker, загружает файл и создаёт сообщение, подтверждает 409 до готовности, перезапускает API и broker, затем запускает обработку. Outbox до восстановления имеет dispatched_at=null; новый job вручную не создаётся. Во втором опыте XADD уже выполнен, worker остановлен, broker перезапускается с volume: непрочитанная задача сохраняется.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
Crash после side effect: BEFORE UPDATE trigger на конкретном attachment задерживает ready COMMIT через pg_sleep(2). После наблюдения PgSleep и существования оригинала в S3 worker получает SIGKILL. Trigger удаляется, lease истекает реально, без подмены часов. Зафиксировано восстановление примерно через **90.804s**, attempt=2, прежний job_id. Затем повтор ready возвращает no-op и прежние metadata.
|
||||
|
||||
Для проверки предела повторов удаляется staging синтетической работы. Пять реальных попыток чтения S3 завершаются failed/retry_exhausted. В этом тесте next_at явно переносится на сейчас между попытками, чтобы отдельно проверить счётчик и финальное состояние; измерением реального времени backoff этот опыт не является. После восстановления исходных байтов операторский replay пишет audit, та же job доходит до ready. Failed-вложение нельзя привязать к сообщению.
|
||||
|
||||
При code review найден и воспроизведён leak session-lock после SQL-ошибки в replay. Регрессия была RED; после rollback-before-unlock — GREEN. Второй RED→GREEN проверяет медленный HTTP/S3 поток, который не нарушает socket inactivity timeout: теперь общий deadline убивает обработку целиком. Это ограничивает зависший processing, а не просто прекращает ожидание фонового потока.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker-host, одна PG и один MinIO/broker; отказ диска/узла и backup/HA не покрываются.
|
||||
- At least once и идемпотентный эффект не означают exactly once. Один pipeline_version=1; конкуренция версий — трек 5, не заявлен.
|
||||
- Redis AOF always повышает устойчивость на данном volume, но не гарантирует переживание потери самого volume. Не применяется небезопасное trimming непрочитанных jobs.
|
||||
- Непривязанные учтённые вложения и частичные конечные варианты failed не удаляются автоматически. Failed staging хранится 24h; после удаления источник нельзя восстановить replay.
|
||||
- API принимает PNG/JPEG в заданных пределах; наиболее тяжёлый файл может получить failed из-за ограничений worker. HTTP/Basic разрешён только в локальном изолированном учебном стенде с синтетическими данными.
|
||||
- В lab6 добавляются наблюдаемость и alerts, в lab7 — TLS/mTLS и OpenBao. Исследовательские треки 5 не выполнялись.
|
||||
|
||||
+173
@@ -0,0 +1,173 @@
|
||||
name: mtusi-lab5
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.5-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
redis:
|
||||
image: redis:7.4.4-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
|
||||
volumes: [redis_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.25
|
||||
migrate:
|
||||
image: mtusi-lab5-api:lab5
|
||||
command: [python, -m, messenger.migrate]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
restart: 'no'
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
api:
|
||||
image: mtusi-lab5-api:lab5
|
||||
build: .
|
||||
scale: 2
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
POOL_MAX_SIZE: '8'
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [front, data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
minio:
|
||||
image: mtusi-lab5-minio:2025-04-22
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: minio
|
||||
secrets: [minio_root_user, minio_root_password]
|
||||
volumes: [minio_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:9000/minio/health/ready',timeout=2)"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
stop_grace_period: 10s
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
minio-init:
|
||||
image: mtusi-lab5-mc:2025-04-16
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infra/minio/Dockerfile
|
||||
target: mc
|
||||
secrets: [minio_root_user, minio_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
volumes:
|
||||
- ./infra/minio/bootstrap.sh:/config/bootstrap.sh:ro
|
||||
- ./infra/minio/api-policy.json:/config/api-policy.json:ro
|
||||
networks: [data]
|
||||
depends_on:
|
||||
minio: {condition: service_healthy}
|
||||
restart: 'no'
|
||||
mem_limit: 128m
|
||||
cpus: 0.25
|
||||
broker:
|
||||
image: redis:7.4.4-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --appendfsync, always, --maxmemory, 96mb, --maxmemory-policy, noeviction]
|
||||
volumes: [broker_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 160m
|
||||
cpus: 0.5
|
||||
worker: &background
|
||||
image: mtusi-lab5-api:lab5
|
||||
command: [python, -m, messenger.worker, worker]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL: redis://broker:6379/0
|
||||
JOB_MAX_ATTEMPTS: '5'
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
broker: {condition: service_healthy}
|
||||
migrate: {condition: service_completed_successfully}
|
||||
minio-init: {condition: service_completed_successfully}
|
||||
stop_grace_period: 45s
|
||||
restart: unless-stopped
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
relay:
|
||||
<<: *background
|
||||
command: [python, -m, messenger.worker, relay]
|
||||
mem_limit: 192m
|
||||
cpus: 0.25
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine
|
||||
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
|
||||
networks: [front]
|
||||
ports: ['127.0.0.1:8080:8080']
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 1s
|
||||
timeout: 8s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
front:
|
||||
data:
|
||||
internal: true
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
minio_root_user: {file: ./secrets/minio_root_user}
|
||||
minio_root_password: {file: ./secrets/minio_root_password}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
minio_data:
|
||||
broker_data:
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"parallel_requests": 8,
|
||||
"observed_instances": [
|
||||
"2836440f76e24486ab068d4c71498d15",
|
||||
"77545b2c0222441193411736edbf54ff"
|
||||
],
|
||||
"responses_201": 8,
|
||||
"messages_created": 1,
|
||||
"different_body_status": 409
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"redis": {
|
||||
"profile_status": 200,
|
||||
"fallback_seconds": 2.5178,
|
||||
"anonymous": 401,
|
||||
"readiness": 200
|
||||
},
|
||||
"postgres": {
|
||||
"authenticated_status": 503,
|
||||
"response_seconds": 0.006,
|
||||
"anonymous_status": 401,
|
||||
"internal_api_liveness": [
|
||||
200,
|
||||
200
|
||||
],
|
||||
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
|
||||
},
|
||||
"same_session_after_recovery": true
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
./scripts/check_fresh_migrations.sh
|
||||
Volume mtusi-lab5-migration-check-53406_redis_data Creating
|
||||
Network mtusi-lab5-migration-check-53406_data Creating
|
||||
Volume mtusi-lab5-migration-check-53406_postgres_data Creating
|
||||
Volume mtusi-lab5-migration-check-53406_minio_data Creating
|
||||
Volume mtusi-lab5-migration-check-53406_redis_data Creating
|
||||
Network mtusi-lab5-migration-check-53406_data Creating
|
||||
Volume mtusi-lab5-migration-check-53406_postgres_data Creating
|
||||
Volume mtusi-lab5-migration-check-53406_minio_data Creating
|
||||
Volume mtusi-lab5-migration-check-53406_postgres_data Created
|
||||
Volume mtusi-lab5-migration-check-53406_postgres_data Created
|
||||
Volume mtusi-lab5-migration-check-53406_redis_data Created
|
||||
Volume mtusi-lab5-migration-check-53406_redis_data Created
|
||||
Volume mtusi-lab5-migration-check-53406_minio_data Created
|
||||
Volume mtusi-lab5-migration-check-53406_minio_data Created
|
||||
Network mtusi-lab5-migration-check-53406_data Created
|
||||
Network mtusi-lab5-migration-check-53406_data Created
|
||||
Container mtusi-lab5-migration-check-53406-redis-1 Creating
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Creating
|
||||
Container mtusi-lab5-migration-check-53406-postgres-1 Creating
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Created
|
||||
Container mtusi-lab5-migration-check-53406-minio-init-1 Creating
|
||||
Container mtusi-lab5-migration-check-53406-redis-1 Created
|
||||
Container mtusi-lab5-migration-check-53406-postgres-1 Created
|
||||
Container mtusi-lab5-migration-check-53406-minio-init-1 Created
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Starting
|
||||
Container mtusi-lab5-migration-check-53406-postgres-1 Starting
|
||||
Container mtusi-lab5-migration-check-53406-redis-1 Starting
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Started
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Waiting
|
||||
Container mtusi-lab5-migration-check-53406-postgres-1 Started
|
||||
Container mtusi-lab5-migration-check-53406-redis-1 Started
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Healthy
|
||||
Container mtusi-lab5-migration-check-53406-minio-init-1 Starting
|
||||
Container mtusi-lab5-migration-check-53406-minio-init-1 Started
|
||||
Container mtusi-lab5-migration-check-53406-redis-1 Waiting
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Waiting
|
||||
Container mtusi-lab5-migration-check-53406-postgres-1 Waiting
|
||||
Container mtusi-lab5-migration-check-53406-minio-init-1 Waiting
|
||||
Container mtusi-lab5-migration-check-53406-redis-1 Healthy
|
||||
Container mtusi-lab5-migration-check-53406-minio-1 Healthy
|
||||
Container mtusi-lab5-migration-check-53406-postgres-1 Healthy
|
||||
Container mtusi-lab5-migration-check-53406-minio-init-1 Healthy
|
||||
Network mtusi-lab5-migration-check-53406_front Creating
|
||||
Network mtusi-lab5-migration-check-53406_front Creating
|
||||
Network mtusi-lab5-migration-check-53406_front Created
|
||||
Network mtusi-lab5-migration-check-53406_front Created
|
||||
Container mtusi-lab5-migration-check-53406-api-run-80cac000e90f Creating
|
||||
Container mtusi-lab5-migration-check-53406-api-run-80cac000e90f Created
|
||||
Applied 001_initial.sql
|
||||
Applied 002_idempotency.sql
|
||||
Applied 003_attachments.sql
|
||||
Applied 004_jobs.sql
|
||||
Migrations up to date
|
||||
Migrations up to date
|
||||
........................... [100%]
|
||||
=============================== warnings summary ===============================
|
||||
student-tests/test_attachments.py: 19 warnings
|
||||
student-tests/test_idempotency.py: 1 warning
|
||||
student-tests/test_jobs.py: 6 warnings
|
||||
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
|
||||
datetime_now = datetime.datetime.utcnow()
|
||||
|
||||
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
|
||||
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
|
||||
warnings.warn(
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
27 passed, 27 warnings in 11.96s
|
||||
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"target_container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
|
||||
"target_instance": "3cd11c3ed6654d15aa331b17da172ba6",
|
||||
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
|
||||
"in_flight_observed_in_pg_stat_activity": true,
|
||||
"in_flight_response": 201,
|
||||
"new_tcp_after_sigterm": {
|
||||
"accepted": false,
|
||||
"error_type": "ConnectionRefusedError"
|
||||
},
|
||||
"target_still_running_when_new_tcp_rejected": true,
|
||||
"exit_code": 0,
|
||||
"shutdown_seconds_from_sigterm": 2.1039,
|
||||
"confirmed_message_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,215 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"containers": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e",
|
||||
"name": "mtusi-lab5-api-3",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:16:43.516831177Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data",
|
||||
"mtusi-lab5_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "f0356f65fd3349ebb6a12a12cbe8911b",
|
||||
"release": "lab5-03545ea5"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca",
|
||||
"name": "mtusi-lab5-api-4",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:16:48.637814055Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data",
|
||||
"mtusi-lab5_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "1f3ca0aa757548729c3af44cfe1152d8",
|
||||
"release": "lab5-03545ea5"
|
||||
},
|
||||
{
|
||||
"service": "broker",
|
||||
"container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646",
|
||||
"name": "mtusi-lab5-broker-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.613805961Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 167772160,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_broker_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
|
||||
"name": "mtusi-lab5-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.692474795Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8",
|
||||
"name": "mtusi-lab5-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:50.728753166Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6",
|
||||
"name": "mtusi-lab5-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.65248217Z",
|
||||
"image": "mtusi-lab5-minio:2025-04-22",
|
||||
"image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd",
|
||||
"name": "mtusi-lab5-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:50.664761458Z",
|
||||
"image": "mtusi-lab5-mc:2025-04-16",
|
||||
"image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3",
|
||||
"name": "mtusi-lab5-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:17:03.314404256Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3",
|
||||
"name": "mtusi-lab5-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:17:02.234800422Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "relay",
|
||||
"container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce",
|
||||
"name": "mtusi-lab5-relay-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:28.180030295Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5",
|
||||
"name": "mtusi-lab5-worker-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:28.686125545Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"commit_publish_gap": {
|
||||
"accepted_status": 202,
|
||||
"outbox_unpublished": true,
|
||||
"api_broker_restarted": true,
|
||||
"same_job_ready": true
|
||||
},
|
||||
"broker_persistence": {
|
||||
"unread_job_survived_restart": true
|
||||
},
|
||||
"after_put_before_ack": {
|
||||
"s3_effect_written": true,
|
||||
"lease_recovery_seconds": 90.804,
|
||||
"attempt_after_recovery": 2,
|
||||
"same_job": true
|
||||
},
|
||||
"duplicate": {
|
||||
"ready_noop": true
|
||||
},
|
||||
"retry_replay": {
|
||||
"failed_after": 5,
|
||||
"code": "retry_exhausted",
|
||||
"explicit_replay_ready": true,
|
||||
"failed_attachment_binding": 409
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
FAIL: POST /api/v1/chats/2a81dc08-f62f-4897-899d-7c925a255fbf/attachments: expected [202], received 201
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
|
||||
"name": "mtusi-lab5-api-2",
|
||||
"instance": "fa72717b0a734be9b12e177fd2ca606d",
|
||||
"release": "lab5-v1",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"stopped_at_utc": "2026-11-09T15:16:14.991795+00:00",
|
||||
"stop_seconds": 0.5132,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"version": "nginx version: nginx/1.28.0",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"public_server_header": "nginx",
|
||||
"config_syntax": "PASS",
|
||||
"observed_instances": [
|
||||
"2836440f76e24486ab068d4c71498d15",
|
||||
"77545b2c0222441193411736edbf54ff"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,428 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
|
||||
"name": "mtusi-lab5-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:12:16.797696721Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data",
|
||||
"mtusi-lab5_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "77545b2c0222441193411736edbf54ff",
|
||||
"release": "lab5-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
|
||||
"name": "mtusi-lab5-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:12:16.716356512Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data",
|
||||
"mtusi-lab5_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "2836440f76e24486ab068d4c71498d15",
|
||||
"release": "lab5-v1"
|
||||
},
|
||||
{
|
||||
"service": "broker",
|
||||
"container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646",
|
||||
"name": "mtusi-lab5-broker-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:12:27.23476842Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 167772160,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_broker_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
|
||||
"name": "mtusi-lab5-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:06:24.855529918Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8",
|
||||
"name": "mtusi-lab5-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:14:23.843447751Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6",
|
||||
"name": "mtusi-lab5-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:06:18.468961513Z",
|
||||
"image": "mtusi-lab5-minio:2025-04-22",
|
||||
"image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd",
|
||||
"name": "mtusi-lab5-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:14:23.912373126Z",
|
||||
"image": "mtusi-lab5-mc:2025-04-16",
|
||||
"image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3",
|
||||
"name": "mtusi-lab5-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:06:18.410650346Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3",
|
||||
"name": "mtusi-lab5-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:06:18.524762596Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "relay",
|
||||
"container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce",
|
||||
"name": "mtusi-lab5-relay-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:14:20.333630291Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5",
|
||||
"name": "mtusi-lab5-worker-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:14:20.398521Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
|
||||
"name": "mtusi-lab5-api-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.966931253Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data",
|
||||
"mtusi-lab5_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "cda72c7a2f684f4f83c4f124fe97d9a6",
|
||||
"release": "lab5-v1"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
|
||||
"name": "mtusi-lab5-api-2",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.899393962Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data",
|
||||
"mtusi-lab5_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "c306444bf8ac4a599b4dc6ac465b038a",
|
||||
"release": "lab5-v1"
|
||||
},
|
||||
{
|
||||
"service": "broker",
|
||||
"container": "4ae9bb96997fc330297b8b1f7dc802643300d3d5fb2dcc0431fd24e7dd6bd646",
|
||||
"name": "mtusi-lab5-broker-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.613805961Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 167772160,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_broker_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
|
||||
"name": "mtusi-lab5-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.692474795Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "7a5be77365652cc141dc5f96a13977d9ece465f5e11313a80719e83cc23bd8b8",
|
||||
"name": "mtusi-lab5-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:29.872387004Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio",
|
||||
"container": "28139ee8cc9a93c011dbc795a747e711cf532953c0222ecfff055456aad67ab6",
|
||||
"name": "mtusi-lab5-minio-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.65248217Z",
|
||||
"image": "mtusi-lab5-minio:2025-04-22",
|
||||
"image_id": "sha256:72715c518d214a04aaa43f38aaae882a07e6e59d8dd9e6b0dd31a42fd47dfa41",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_minio_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "minio-init",
|
||||
"container": "3ecadb4d965df869dd2e46c9f43b9388ac1fdcc0ef84051e13830e02504c9dfd",
|
||||
"name": "mtusi-lab5-minio-init-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:30.375772004Z",
|
||||
"image": "mtusi-lab5-mc:2025-04-16",
|
||||
"image_id": "sha256:48bb08e6faef719679f13f7bc92e12f5ad550f92066a6c0a553ff7d463d8d5c3",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "74681e7e4e0a21be82e14a032a7180b6a1eca6f5723d17b1fe5aa590087c79f3",
|
||||
"name": "mtusi-lab5-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.672883961Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "d95d99e43cd645266f73e98e513eb233b614e607f7610089fa7beba7fec561f3",
|
||||
"name": "mtusi-lab5-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.736927712Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab5_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "relay",
|
||||
"container": "64d602cba0426d571d9e213f54fc78dff1824a617202eaf85186be25018eefce",
|
||||
"name": "mtusi-lab5-relay-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:28.180030295Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 201326592,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "worker",
|
||||
"container": "63cbd712bb72a3cc09d3dec9c20f439e7780bb5eb6ffd54e91fe79842d8f42e5",
|
||||
"name": "mtusi-lab5-worker-1",
|
||||
"status": "running",
|
||||
"health": null,
|
||||
"started_at": "2026-11-09T15:15:28.686125545Z",
|
||||
"image": "mtusi-lab5-api:lab5",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 133, \"failed_attempts\": 0, \"first_recovery_seconds\": 0.878}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
|
||||
"stopped": {
|
||||
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
|
||||
"name": "mtusi-lab5-api-2",
|
||||
"instance": "c306444bf8ac4a599b4dc6ac465b038a",
|
||||
"release": "lab5-v1",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"stopped_at_utc": "2026-11-09T15:15:35.619684+00:00",
|
||||
"stop_seconds": 0.3357,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 133, "failed_attempts": 0, "first_recovery_seconds": 0.878}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,171 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
|
||||
"name": "mtusi-lab5-api-1",
|
||||
"instance": "d23b5da28be949239c1ea5f27a578822",
|
||||
"release": "lab5-v1",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b"
|
||||
},
|
||||
{
|
||||
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
|
||||
"name": "mtusi-lab5-api-2",
|
||||
"instance": "7c84431c2a104eba8f1f3d789d971602",
|
||||
"release": "lab5-v1",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b"
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e",
|
||||
"name": "mtusi-lab5-api-3",
|
||||
"instance": "f0356f65fd3349ebb6a12a12cbe8911b",
|
||||
"release": "lab5-03545ea5",
|
||||
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe"
|
||||
},
|
||||
{
|
||||
"container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca",
|
||||
"name": "mtusi-lab5-api-4",
|
||||
"instance": "1f3ca0aa757548729c3af44cfe1152d8",
|
||||
"release": "lab5-03545ea5",
|
||||
"image_id": "sha256:d0161806e14c815373596e7ed6ad1dd90c1099f1d6effa1178f6d723272084fe"
|
||||
}
|
||||
],
|
||||
"target_release": "lab5-03545ea5",
|
||||
"lb_before": {
|
||||
"service": "lb",
|
||||
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
|
||||
"name": "mtusi-lab5-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.692474795Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_front"
|
||||
]
|
||||
},
|
||||
"lb_after": {
|
||||
"service": "lb",
|
||||
"container": "1e8af54790b3d81c6ab07b8b48e2190ffd58177f02cfbb71a4e9d70642d28732",
|
||||
"name": "mtusi-lab5-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-09T15:15:27.692474795Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab5_front"
|
||||
]
|
||||
},
|
||||
"lb_not_restarted": true,
|
||||
"build_seconds": 3.8455,
|
||||
"migration_output": "Migrations up to date",
|
||||
"replacements": [
|
||||
{
|
||||
"old_container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
|
||||
"new_container": "04dace371f7c119107574888627a31b46870064efb055240ece1e76f25aa6f0e",
|
||||
"old_ip": "172.26.0.3",
|
||||
"new_ip": "172.26.0.5",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"7c84431c2a104eba8f1f3d789d971602",
|
||||
"d23b5da28be949239c1ea5f27a578822",
|
||||
"f0356f65fd3349ebb6a12a12cbe8911b"
|
||||
],
|
||||
"observed_instances": [
|
||||
"7c84431c2a104eba8f1f3d789d971602",
|
||||
"f0356f65fd3349ebb6a12a12cbe8911b"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 5.1191
|
||||
},
|
||||
{
|
||||
"old_container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
|
||||
"new_container": "ddd943a740c7befc0853727aa039a802ebc7c2ec6e82b61747da7c301964f5ca",
|
||||
"old_ip": "172.26.0.2",
|
||||
"new_ip": "172.26.0.3",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"1f3ca0aa757548729c3af44cfe1152d8",
|
||||
"7c84431c2a104eba8f1f3d789d971602",
|
||||
"f0356f65fd3349ebb6a12a12cbe8911b"
|
||||
],
|
||||
"observed_instances": [
|
||||
"1f3ca0aa757548729c3af44cfe1152d8",
|
||||
"f0356f65fd3349ebb6a12a12cbe8911b"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 6.1668
|
||||
}
|
||||
],
|
||||
"during_replacement": {
|
||||
"seconds": 11.286,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8541,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 756.778,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.383,
|
||||
"p95": 7.793,
|
||||
"p99": 35.123
|
||||
},
|
||||
"instances": [
|
||||
"1f3ca0aa757548729c3af44cfe1152d8",
|
||||
"7c84431c2a104eba8f1f3d789d971602",
|
||||
"d23b5da28be949239c1ea5f27a578822",
|
||||
"f0356f65fd3349ebb6a12a12cbe8911b"
|
||||
]
|
||||
},
|
||||
"total": {
|
||||
"seconds": 22.0756,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 16487,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 746.843,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.601,
|
||||
"p95": 7.648,
|
||||
"p99": 26.561
|
||||
},
|
||||
"instances": [
|
||||
"1f3ca0aa757548729c3af44cfe1152d8",
|
||||
"7c84431c2a104eba8f1f3d789d971602",
|
||||
"d23b5da28be949239c1ea5f27a578822",
|
||||
"f0356f65fd3349ebb6a12a12cbe8911b"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 16487,
|
||||
"history_messages": 16487,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"baseline": {
|
||||
"seconds": 3.0041,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 2367,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 787.933,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.168,
|
||||
"p95": 6.086,
|
||||
"p99": 33.522
|
||||
},
|
||||
"instances": [
|
||||
"cda72c7a2f684f4f83c4f124fe97d9a6",
|
||||
"fa72717b0a734be9b12e177fd2ca606d"
|
||||
]
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"stopped": {
|
||||
"container": "07cfdd91be6804999a83f2e8f9d6fe5ebc8a655db57e6d84c681b8223f653808",
|
||||
"name": "mtusi-lab5-api-1",
|
||||
"instance": "cda72c7a2f684f4f83c4f124fe97d9a6",
|
||||
"release": "lab5-v1",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"stopped_at_utc": "2026-11-09T15:16:01.326356+00:00",
|
||||
"stop_seconds": 0.4399,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.0034,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 4111,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 513.658,
|
||||
"error_ratio": 0.000243,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.469,
|
||||
"p95": 12.349,
|
||||
"p99": 58.778
|
||||
},
|
||||
"instances": [
|
||||
"cda72c7a2f684f4f83c4f124fe97d9a6",
|
||||
"fa72717b0a734be9b12e177fd2ca606d"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.4598,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.029,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.4513,
|
||||
"seconds_from_stop_completion": 0.0206
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
},
|
||||
{
|
||||
"stopped": {
|
||||
"container": "9db348f53667a52f5b86a5f5669261201905a75eb0b74b7871d45257294054e6",
|
||||
"name": "mtusi-lab5-api-2",
|
||||
"instance": "fa72717b0a734be9b12e177fd2ca606d",
|
||||
"release": "lab5-v1",
|
||||
"image_id": "sha256:dace11316f63abedc33626ca68f91be7185a168ce126bb50220c27a70d07087b",
|
||||
"stopped_at_utc": "2026-11-09T15:16:14.991795+00:00",
|
||||
"stop_seconds": 0.5132,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.0007,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 4087,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 510.829,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.497,
|
||||
"p95": 13.243,
|
||||
"p99": 57.218
|
||||
},
|
||||
"instances": [
|
||||
"3cd11c3ed6654d15aa331b17da172ba6",
|
||||
"fa72717b0a734be9b12e177fd2ca606d"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.3033,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.0159,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.3004,
|
||||
"seconds_from_stop_completion": 0.013
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 30.9184,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 18469,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 597.347,
|
||||
"error_ratio": 5.4e-05,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.53,
|
||||
"p95": 10.788,
|
||||
"p99": 46.459
|
||||
},
|
||||
"instances": [
|
||||
"3cd11c3ed6654d15aa331b17da172ba6",
|
||||
"7c84431c2a104eba8f1f3d789d971602",
|
||||
"cda72c7a2f684f4f83c4f124fe97d9a6",
|
||||
"fa72717b0a734be9b12e177fd2ca606d"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 18469,
|
||||
"history_messages": 18469,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"each_original_container_tested": true
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,15 @@
|
||||
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
........................... [100%]
|
||||
=============================== warnings summary ===============================
|
||||
student-tests/test_attachments.py: 19 warnings
|
||||
student-tests/test_idempotency.py: 1 warning
|
||||
student-tests/test_jobs.py: 6 warnings
|
||||
/usr/local/lib/python3.13/site-packages/botocore/auth.py:422: DeprecationWarning: datetime.datetime.utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
|
||||
datetime_now = datetime.datetime.utcnow()
|
||||
|
||||
student-tests/test_attachments.py::test_invalid_media_corruption_limits_and_filename
|
||||
/usr/local/lib/python3.13/site-packages/PIL/Image.py:3452: DecompressionBombWarning: Image size (40004000 pixels) exceeds limit of 40000000 pixels, could be decompression bomb DOS attack.
|
||||
warnings.warn(
|
||||
|
||||
-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html
|
||||
27 passed, 27 warnings in 11.58s
|
||||
@@ -0,0 +1,28 @@
|
||||
# Official public images/binaries were removed upstream; build fixed release sources.
|
||||
FROM golang:1.24.2-bookworm AS minio-build
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
|
||||
RUN git clone --depth 1 --branch RELEASE.2025-04-22T22-12-26Z https://github.com/minio/minio.git /src
|
||||
WORKDIR /src
|
||||
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/minio .
|
||||
RUN test "$(git rev-parse HEAD)" = 0d7408fc9969caf07de6a8c3a84f9fbb10a6739e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/minio/cmd.Version=2025-04-22T22:12:26Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2025-04-22T22-12-26Z -X github.com/minio/minio/cmd.CommitID=0d7408fc9969caf07de6a8c3a84f9fbb10a6739e' -o /out/minio .
|
||||
|
||||
FROM golang:1.24.2-bookworm AS mc-build
|
||||
ENV CGO_ENABLED=0 GOMAXPROCS=2 GOTOOLCHAIN=local
|
||||
RUN git clone --depth 1 --branch RELEASE.2025-04-16T18-13-26Z https://github.com/minio/mc.git /src
|
||||
WORKDIR /src
|
||||
RUN go build -p 2 -trimpath -ldflags='-s -w' -o /out/mc .
|
||||
RUN test "$(git rev-parse HEAD)" = b00526b153a31b36767991a4f5ce2cced435ee8e && go build -p 2 -trimpath -ldflags='-s -w -X github.com/minio/mc/cmd.Version=2025-04-16T18:13:26Z -X github.com/minio/mc/cmd.ReleaseTag=RELEASE.2025-04-16T18-13-26Z -X github.com/minio/mc/cmd.CommitID=b00526b153a31b36767991a4f5ce2cced435ee8e' -o /out/mc .
|
||||
|
||||
FROM python:3.13.5-slim AS minio
|
||||
LABEL org.opencontainers.image.source="https://github.com/minio/minio" org.opencontainers.image.version="RELEASE.2025-04-22T22-12-26Z"
|
||||
RUN useradd --uid 10002 --create-home minio && mkdir /data && chown 10002:10002 /data
|
||||
COPY --from=minio-build /out/minio /usr/local/bin/minio
|
||||
COPY infra/minio/server.sh /server.sh
|
||||
USER 10002
|
||||
ENTRYPOINT ["/bin/sh", "/server.sh"]
|
||||
CMD ["server", "/data", "--address", ":9000", "--console-address", "127.0.0.1:9001"]
|
||||
|
||||
FROM debian:bookworm-slim AS mc
|
||||
LABEL org.opencontainers.image.source="https://github.com/minio/mc" org.opencontainers.image.version="RELEASE.2025-04-16T18-13-26Z"
|
||||
COPY --from=mc-build /out/mc /usr/local/bin/mc
|
||||
ENTRYPOINT ["/bin/sh", "/config/bootstrap.sh"]
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetBucketLocation"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments"
|
||||
]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:ListBucket"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments"
|
||||
],
|
||||
"Condition": {
|
||||
"StringLike": {
|
||||
"s3:prefix": [
|
||||
"originals/*",
|
||||
"staging/*",
|
||||
"variants/*"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": [
|
||||
"s3:GetObject",
|
||||
"s3:PutObject",
|
||||
"s3:DeleteObject"
|
||||
],
|
||||
"Resource": [
|
||||
"arn:aws:s3:::messenger-attachments/originals/*",
|
||||
"arn:aws:s3:::messenger-attachments/staging/*",
|
||||
"arn:aws:s3:::messenger-attachments/variants/*"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null
|
||||
mc mb --ignore-existing local/messenger-attachments >/dev/null
|
||||
mc anonymous set none local/messenger-attachments >/dev/null
|
||||
mc admin user add local "$(cat /run/secrets/s3_access_key_id)" "$(cat /run/secrets/s3_secret_access_key)" >/dev/null
|
||||
mc admin policy create local messenger-api /config/api-policy.json >/dev/null
|
||||
mc admin policy attach local messenger-api --user "$(cat /run/secrets/s3_access_key_id)" >/dev/null
|
||||
printf '%s\n' 'PASS: private bucket and restricted API policy ready (credentials hidden).'
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
export MINIO_ROOT_USER="$(cat /run/secrets/minio_root_user)"
|
||||
export MINIO_ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
|
||||
exec minio "$@"
|
||||
@@ -0,0 +1,62 @@
|
||||
worker_processes 1;
|
||||
error_log /dev/stderr warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
access_log off;
|
||||
server_tokens off;
|
||||
limit_conn_zone $server_name zone=uploads:64k;
|
||||
|
||||
# Docker DNS; resolve updates the upstream when replicas get new IPs.
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
# All A records of api participate in the default round-robin balancing.
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name messenger;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
limit_conn uploads 16;
|
||||
client_max_body_size 11m;
|
||||
client_body_buffer_size 64k;
|
||||
client_body_timeout 10s;
|
||||
proxy_request_buffering on;
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Connection "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_next_upstream error timeout http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_send_timeout 5s;
|
||||
|
||||
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
|
||||
# must not be resent by Nginx after an ambiguous response.
|
||||
proxy_next_upstream error timeout http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 6s;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX chats_history_idx ON chats(created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Additive: old lab2 tables and messages remain valid.
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash text NOT NULL,
|
||||
message_id uuid REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
|
||||
@@ -0,0 +1,23 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png','image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id,owner_id) REFERENCES chat_members(chat_id,user_id)
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments(chat_id,id);
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) <= 4000);
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id,attachment_id),
|
||||
UNIQUE (message_id,position)
|
||||
);
|
||||
@@ -0,0 +1,19 @@
|
||||
ALTER TABLE attachments ADD COLUMN status text NOT NULL DEFAULT 'ready' CHECK (status IN ('queued','processing','ready','failed'));
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE;
|
||||
ALTER TABLE attachments ADD COLUMN variants jsonb NOT NULL DEFAULT '[]';
|
||||
CREATE TABLE attachment_jobs (
|
||||
id uuid PRIMARY KEY, attachment_id uuid NOT NULL UNIQUE REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version=1),
|
||||
attempt integer NOT NULL DEFAULT 0, replay_count integer NOT NULL DEFAULT 0,
|
||||
lease_until timestamptz, next_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), traceparent text, tracestate text
|
||||
);
|
||||
CREATE TABLE job_outbox (
|
||||
job_id uuid PRIMARY KEY REFERENCES attachment_jobs(id), dispatched_at timestamptz
|
||||
);
|
||||
CREATE TABLE job_replays (
|
||||
id bigserial PRIMARY KEY, job_id uuid NOT NULL REFERENCES attachment_jobs(id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(), reason text NOT NULL
|
||||
);
|
||||
CREATE INDEX jobs_due ON attachment_jobs(next_at);
|
||||
@@ -0,0 +1,51 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.9.0
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
attrs==26.1.0
|
||||
boto3==1.38.27
|
||||
botocore==1.38.46
|
||||
certifi==2026.10.23
|
||||
cffi==2.1.1
|
||||
charset-normalizer==3.5.2
|
||||
click==8.5.0
|
||||
fastapi==0.115.12
|
||||
h11==0.16.0
|
||||
httpcore==1.0.9
|
||||
httpx==0.28.1
|
||||
idna==3.20
|
||||
iniconfig==2.3.1
|
||||
jmespath==1.1.0
|
||||
jsonschema==4.26.0
|
||||
jsonschema-path==0.3.4
|
||||
jsonschema-specifications==2025.9.1
|
||||
lazy-object-proxy==1.13.0
|
||||
openapi-schema-validator==0.6.3
|
||||
openapi-spec-validator==0.7.2
|
||||
packaging==26.3
|
||||
pathable==0.4.4
|
||||
pillow==11.3.0
|
||||
pluggy==1.7.0
|
||||
psycopg==3.2.9
|
||||
psycopg-binary==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
pycparser==3.12
|
||||
pydantic==2.11.5
|
||||
pydantic_core==2.33.2
|
||||
Pygments==2.21.0
|
||||
pytest==8.4.0
|
||||
python-dateutil==2.9.0.post0
|
||||
PyYAML==6.0.3
|
||||
redis==6.2.0
|
||||
referencing==0.36.2
|
||||
requests==2.34.2
|
||||
rfc3339-validator==0.1.4
|
||||
rpds-py==2026.9.1
|
||||
s3transfer==0.13.1
|
||||
six==1.17.0
|
||||
sniffio==1.3.1
|
||||
starlette==0.46.2
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.3
|
||||
@@ -0,0 +1,12 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.3
|
||||
psycopg[binary]==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.2.0
|
||||
argon2-cffi==25.1.0
|
||||
pytest==8.4.0
|
||||
httpx==0.28.1
|
||||
pydantic==2.11.5
|
||||
anyio==4.9.0
|
||||
boto3==1.38.27
|
||||
Pillow==11.3.0
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root = Path('secrets')
|
||||
p = root / 'postgres_password'
|
||||
if not p.exists():
|
||||
p.write_text(secrets.token_hex(24))
|
||||
pw = p.read_text().strip()
|
||||
for name, value in {
|
||||
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
'minio_root_user': 'lab4-admin',
|
||||
'minio_root_password': secrets.token_hex(24),
|
||||
's3_access_key_id': 'messenger-api',
|
||||
's3_secret_access_key': secrets.token_hex(24),
|
||||
}.items():
|
||||
path = root / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
# Bind-mounted Docker secrets must be readable by container uid 10001.
|
||||
# Parent directory remains 0700 on the host.
|
||||
for path in root.iterdir():
|
||||
path.chmod(0o444)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
|
||||
print('Local secrets ready (values hidden).')
|
||||
PY
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# A unique disposable Compose project; never touches normal project volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
project="mtusi-lab5-migration-check-$$"
|
||||
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
|
||||
echo 'Refusing to reuse a pre-existing scratch project' >&2
|
||||
exit 1
|
||||
fi
|
||||
cleanup() {
|
||||
docker compose -p "$project" down --volumes >/dev/null 2>&1
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis minio minio-init
|
||||
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
|
||||
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
|
||||
|
||||
def command(args, timeout=120):
|
||||
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120):
|
||||
return command(['docker', 'compose', *args], timeout=timeout)
|
||||
|
||||
|
||||
def inspect(containers):
|
||||
if not containers:
|
||||
return []
|
||||
return json.loads(command(['docker','inspect',*containers]))
|
||||
|
||||
|
||||
def container_ids(service=None, all_containers=False):
|
||||
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
|
||||
return compose(*args).split()
|
||||
|
||||
|
||||
def probe(container, host='127.0.0.1', path='/health/live'):
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
try:
|
||||
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
|
||||
except urllib.error.HTTPError as exc:
|
||||
r=exc
|
||||
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
|
||||
'''
|
||||
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
|
||||
|
||||
|
||||
def replicas():
|
||||
items = []
|
||||
for data in inspect(container_ids('api')):
|
||||
if data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
|
||||
return items
|
||||
|
||||
|
||||
def snapshot():
|
||||
items = []
|
||||
for data in inspect(container_ids(all_containers=True)):
|
||||
service = data['Config']['Labels']['com.docker.compose.service']
|
||||
item = {
|
||||
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
|
||||
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
|
||||
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
|
||||
'published_ports':data['HostConfig']['PortBindings'],
|
||||
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
|
||||
'networks':sorted(data['NetworkSettings']['Networks']),
|
||||
}
|
||||
if service == 'api' and data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def wait_healthy(container, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
while time.monotonic() < deadline:
|
||||
data = inspect([container])[0]
|
||||
if data['State'].get('Health',{}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError('Container did not become healthy')
|
||||
|
||||
|
||||
def wait_cluster(count=2, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
|
||||
alias = response.headers.get('X-Instance-Id')
|
||||
if alias:
|
||||
seen.add(alias)
|
||||
if len(seen) >= count:
|
||||
return sorted(seen)
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError('Load balancer did not discover required API instances')
|
||||
|
||||
|
||||
def stop_one(target):
|
||||
if not re.fullmatch(r'[a-f0-9]{32}', target):
|
||||
raise ValueError('Invalid target instance alias')
|
||||
matching = [item for item in replicas() if item['instance']==target]
|
||||
if len(matching) != 1:
|
||||
raise RuntimeError('Target must match exactly one running API process')
|
||||
item = matching[0]
|
||||
started = time.monotonic()
|
||||
command(['docker','stop','--time','7',item['container']],timeout=9)
|
||||
state = inspect([item['container']])[0]['State']
|
||||
if state['Running'] or state['ExitCode'] != 0:
|
||||
raise RuntimeError('API did not stop gracefully')
|
||||
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
|
||||
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
return result
|
||||
|
||||
|
||||
def restart():
|
||||
compose('restart','api','postgres','redis','minio','broker','relay','worker','lb',timeout=30)
|
||||
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
|
||||
return wait_cluster(2,timeout=15)
|
||||
|
||||
|
||||
def front_address(container):
|
||||
networks = inspect([container])[0]['NetworkSettings']['Networks']
|
||||
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
|
||||
|
||||
|
||||
def replace_one(container):
|
||||
before = set(container_ids('api'))
|
||||
if container not in before or len(before) != 2:
|
||||
raise RuntimeError('Replacement expects exactly two running API containers')
|
||||
old_address = front_address(container)
|
||||
# Start-first ensures a genuinely different IP while the old address is still occupied.
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
|
||||
expanded = set(container_ids('api'))
|
||||
added = expanded-before
|
||||
if len(added) != 1 or not before.issubset(expanded):
|
||||
raise RuntimeError('Start-first step unexpectedly changed an existing container')
|
||||
replacement = added.pop()
|
||||
wait_healthy(replacement)
|
||||
new_address = front_address(replacement)
|
||||
if new_address == old_address:
|
||||
raise RuntimeError('Replacement did not get a different IP')
|
||||
expected_three = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(3)) != expected_three:
|
||||
raise RuntimeError('Load balancer did not discover all three live addresses')
|
||||
command(['docker','stop','--time','7',container],timeout=10)
|
||||
state = inspect([container])[0]['State']
|
||||
if state['ExitCode'] != 0:
|
||||
raise RuntimeError('Old release did not terminate gracefully')
|
||||
command(['docker','rm',container])
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
|
||||
after = set(container_ids('api'))
|
||||
if after != (before-{container}) | {replacement}:
|
||||
raise RuntimeError('Sequential replacement changed the wrong containers')
|
||||
expected = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(2)) != expected:
|
||||
raise RuntimeError('Unexpected final backend aliases')
|
||||
return {'old_container':container,'new_container':replacement,
|
||||
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
|
||||
'instances_during_start_first':sorted(expected_three),
|
||||
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
|
||||
parser.add_argument('arguments',nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'wait':
|
||||
print(json.dumps({'instances':wait_cluster()}))
|
||||
elif args.action == 'snapshot':
|
||||
print(json.dumps(snapshot(),indent=2))
|
||||
elif args.action == 'stop-one':
|
||||
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
|
||||
elif args.action == 'exec':
|
||||
ids = container_ids('api')
|
||||
if not ids or not args.arguments:
|
||||
raise RuntimeError('A live API container and command are required')
|
||||
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
|
||||
elif args.action == 'restart':
|
||||
print(json.dumps({'instances':restart()}))
|
||||
else:
|
||||
compose('run','--rm','--no-deps','migrate')
|
||||
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/cluster.py restart
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
exec python3 scripts/cluster.py stop-one
|
||||
@@ -0,0 +1,443 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
import cluster
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
|
||||
|
||||
def save(name, value):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS:', name)
|
||||
|
||||
|
||||
def percentiles(values):
|
||||
values = sorted(values)
|
||||
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
|
||||
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, concurrency=4):
|
||||
self.c = Client(timeout=6)
|
||||
self.alice,self.bob,_,self.chat = self.c.bootstrap()
|
||||
self.concurrency = concurrency
|
||||
self.done = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts = []
|
||||
self.confirmed = {}
|
||||
self.prefix = uuid.uuid4().hex
|
||||
|
||||
def worker(self, number):
|
||||
client = Client(timeout=6)
|
||||
index = 0
|
||||
while not self.done.is_set():
|
||||
text = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
key = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = client.message(self.alice,self.chat,text,key=key)
|
||||
row = response['json']
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
|
||||
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
|
||||
with self.lock:
|
||||
self.confirmed[row['id']] = row
|
||||
index += 1
|
||||
except Failure:
|
||||
# Only idempotent message POST is retried, with its original body and key.
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
|
||||
time.sleep(0.02)
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
|
||||
def start(self):
|
||||
self.started = time.monotonic()
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
||||
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.done.set()
|
||||
for future in self.futures:
|
||||
future.result(timeout=10)
|
||||
self.pool.shutdown()
|
||||
self.ended = time.monotonic()
|
||||
|
||||
def summary(self, start=None, end=None):
|
||||
start = self.started if start is None else start
|
||||
end = getattr(self,'ended',time.monotonic()) if end is None else end
|
||||
with self.lock:
|
||||
attempts = [a for a in self.attempts if start <= a['started'] < end]
|
||||
success = [a for a in attempts if a['ok']]
|
||||
duration = end-start
|
||||
return {'seconds':round(duration,4),'concurrency':self.concurrency,
|
||||
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
|
||||
'successful_rps':round(len(success)/duration,3),
|
||||
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
|
||||
'successful_latency_ms':percentiles([a['latency'] for a in success]),
|
||||
'instances':sorted({a['instance'] for a in success})}
|
||||
|
||||
def verify_data(self):
|
||||
rows = history(self.c,self.bob,self.chat)
|
||||
actual = {row['id']:row for row in rows}
|
||||
check(len(actual)==len(rows),'duplicated message ids')
|
||||
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
|
||||
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
|
||||
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
|
||||
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
|
||||
|
||||
|
||||
def concurrent_replay():
|
||||
cluster.wait_cluster()
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(8)
|
||||
def send(_):
|
||||
local = Client()
|
||||
barrier.wait()
|
||||
return local.message(alice,chat,'Один результат через разные процессы',key=key)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send,range(8)))
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(instances)>=2,'concurrent retry did not reach different API processes')
|
||||
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
||||
body={'text':'Другой body'},headers={'Idempotency-Key':key})
|
||||
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
|
||||
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
|
||||
'responses_201':8,'messages_created':1,'different_body_status':409})
|
||||
|
||||
|
||||
def verify_persistence():
|
||||
before = cluster.snapshot()
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
|
||||
after = cluster.snapshot()
|
||||
roles = ['api','postgres','redis','minio','broker','relay','worker','lb']
|
||||
previous = {item['container']:item for item in before}
|
||||
for item in after:
|
||||
if item['service'] in roles:
|
||||
old = previous[item['container']]
|
||||
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
|
||||
check(old['volumes']==item['volumes'],'volume identities changed')
|
||||
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
|
||||
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
|
||||
|
||||
|
||||
def public_failover():
|
||||
cluster.wait_cluster()
|
||||
try:
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
|
||||
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
|
||||
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
|
||||
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
|
||||
finally:
|
||||
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
|
||||
cluster.wait_cluster()
|
||||
|
||||
|
||||
def reject_new_connection(survivor, address):
|
||||
code = '''import json,socket,sys
|
||||
try:
|
||||
s=socket.create_connection((sys.argv[1],8080),timeout=1)
|
||||
except OSError as exc:
|
||||
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
|
||||
else:
|
||||
s.close()
|
||||
print(json.dumps({'accepted':True}))
|
||||
'''
|
||||
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
|
||||
|
||||
|
||||
def failover_each():
|
||||
cluster.wait_cluster()
|
||||
originals = cluster.replicas()
|
||||
check(len(originals)==2,'two running replicas required')
|
||||
traffic = Traffic().start()
|
||||
phases = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
baseline_end = time.monotonic()
|
||||
baseline = traffic.summary(end=baseline_end)
|
||||
for original in originals:
|
||||
# Alias changes on restart, so resolve the current process of this exact container.
|
||||
current = next(r for r in cluster.replicas() if r['container']==original['container'])
|
||||
all_before = cluster.inspect([current['container']])[0]
|
||||
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
|
||||
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
|
||||
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
|
||||
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
|
||||
start = time.monotonic()
|
||||
try:
|
||||
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
|
||||
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
|
||||
stop_returned = time.monotonic()
|
||||
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
|
||||
read_deadline = start+10
|
||||
while True:
|
||||
try:
|
||||
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
|
||||
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
|
||||
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
|
||||
read_done = time.monotonic()
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<read_deadline,'reads did not recover within10s')
|
||||
time.sleep(0.1)
|
||||
check(read_done-start<=10,'read recovery exceeded10s')
|
||||
rejected = reject_new_connection(survivor,address)
|
||||
check(not rejected['accepted'],'new connection accepted by stopped API')
|
||||
time.sleep(max(0,8-(time.monotonic()-start)))
|
||||
end = time.monotonic()
|
||||
summary = traffic.summary(start,end)
|
||||
with traffic.lock:
|
||||
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
|
||||
recent = [a for a in successful if a['started']>stop_returned]
|
||||
recovery = min(a['ended'] for a in recent)-start if recent else None
|
||||
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
|
||||
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
|
||||
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
|
||||
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
|
||||
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
|
||||
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
|
||||
'seconds_from_action_start':round(read_done-start,4),
|
||||
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
|
||||
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
|
||||
finally:
|
||||
cluster.command(['docker','start',current['container']])
|
||||
cluster.wait_healthy(current['container'])
|
||||
cluster.wait_cluster()
|
||||
time.sleep(2)
|
||||
finally:
|
||||
traffic.stop()
|
||||
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
|
||||
|
||||
|
||||
def rolling_update():
|
||||
cluster.wait_cluster()
|
||||
before = cluster.replicas()
|
||||
check(len(before)==2,'rolling demonstration expects two replicas')
|
||||
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
target_release = 'lab5-'+uuid.uuid4().hex[:8]
|
||||
traffic = Traffic().start()
|
||||
replacements = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
build_start = time.monotonic()
|
||||
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
|
||||
build_end = time.monotonic()
|
||||
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
|
||||
swap_start = time.monotonic()
|
||||
for old in before:
|
||||
step_start = time.monotonic()
|
||||
result = cluster.replace_one(old['container'])
|
||||
result['seconds'] = round(time.monotonic()-step_start,4)
|
||||
replacements.append(result)
|
||||
swap_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
finally:
|
||||
traffic.stop()
|
||||
after = cluster.replicas()
|
||||
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
|
||||
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
|
||||
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
|
||||
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
|
||||
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
|
||||
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
|
||||
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
|
||||
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
|
||||
|
||||
|
||||
def database_fixture(container, data):
|
||||
code = '''import json,sys,psycopg
|
||||
from psycopg import sql
|
||||
from messenger.db import setting
|
||||
p=json.load(sys.stdin)
|
||||
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
|
||||
if p['action']=='install':
|
||||
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
|
||||
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
|
||||
result={'installed':True}
|
||||
elif p['action']=='remove':
|
||||
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
|
||||
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
|
||||
result={'removed':True}
|
||||
else:
|
||||
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
|
||||
print(json.dumps(result))
|
||||
'''
|
||||
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(response.stdout)
|
||||
|
||||
|
||||
def graceful_shutdown():
|
||||
cluster.wait_cluster()
|
||||
nodes = cluster.replicas()
|
||||
target,survivor = nodes[0],nodes[1]
|
||||
address = cluster.front_address(target['container'])
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
marker = 'drain_'+uuid.uuid4().hex
|
||||
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
|
||||
database_fixture(survivor['container'],{**fixture,'action':'install'})
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
p=json.load(sys.stdin)
|
||||
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
|
||||
try:
|
||||
response=urllib.request.urlopen(r,timeout=6)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response=exc
|
||||
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
|
||||
'''
|
||||
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
|
||||
def in_flight():
|
||||
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(result.stdout)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(in_flight)
|
||||
try:
|
||||
deadline = time.monotonic()+3
|
||||
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
|
||||
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
|
||||
time.sleep(0.05)
|
||||
signaled = time.monotonic()
|
||||
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
|
||||
deadline = signaled+1.5
|
||||
while True:
|
||||
rejected = reject_new_connection(survivor['container'],address)
|
||||
still_running = cluster.inspect([target['container']])[0]['State']['Running']
|
||||
if not rejected['accepted']:
|
||||
break
|
||||
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
|
||||
time.sleep(0.05)
|
||||
check(still_running,'connection refusal observed only after process exit, not during drain')
|
||||
drained = future.result(timeout=8)
|
||||
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
|
||||
check(drained['message']['text']==marker,'in-flight body changed')
|
||||
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
|
||||
elapsed = time.monotonic()-signaled
|
||||
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
|
||||
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
|
||||
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
|
||||
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
|
||||
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
|
||||
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
|
||||
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
|
||||
'confirmed_message_preserved':True}
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
database_fixture(survivor['container'],{**fixture,'action':'remove'})
|
||||
cluster.command(['docker','start',target['container']])
|
||||
cluster.wait_healthy(target['container'])
|
||||
cluster.wait_cluster()
|
||||
save('graceful-shutdown.json',result)
|
||||
|
||||
|
||||
def infrastructure():
|
||||
state = cluster.snapshot()
|
||||
for item in state:
|
||||
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
|
||||
if item['service']!='lb':
|
||||
check(not item['published_ports'],'service exposed a port bypassing LB')
|
||||
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
|
||||
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
|
||||
save('infrastructure.json',{'result':'PASS','containers':state})
|
||||
|
||||
|
||||
def dependency_failures():
|
||||
c = Client(timeout=6)
|
||||
actor = c.user('dependencies')
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result = {'result':'PASS'}
|
||||
try:
|
||||
cluster.compose('stop','-t','2','redis')
|
||||
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
|
||||
c.call('GET','/api/v1/users/me',401)
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
|
||||
finally:
|
||||
cluster.compose('start','redis')
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
def status(path, token=None):
|
||||
headers = {'Authorization':'Bearer '+token} if token else {}
|
||||
request = urllib.request.Request(c.base+path,headers=headers)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = urllib.request.urlopen(request,timeout=6)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
return response.code, round(time.monotonic()-started,4)
|
||||
try:
|
||||
cluster.compose('stop','-t','2','postgres')
|
||||
code,elapsed = status('/api/v1/users/me',actor['token'])
|
||||
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
|
||||
anonymous,_ = status('/api/v1/users/me')
|
||||
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
|
||||
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
|
||||
check(all(code==200 for code in live),'API process liveness depends on DB')
|
||||
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
|
||||
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
|
||||
finally:
|
||||
cluster.compose('start','postgres')
|
||||
cluster.wait_cluster()
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result['same_session_after_recovery'] = True
|
||||
save('dependency-failures.json',result)
|
||||
|
||||
|
||||
def verify_nginx():
|
||||
state = cluster.snapshot()
|
||||
lb = next(item for item in state if item['service']=='lb')
|
||||
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
|
||||
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
|
||||
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
|
||||
c = Client()
|
||||
response = c.call('GET','/health/ready',schema='Health')
|
||||
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
|
||||
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
|
||||
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
|
||||
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
|
||||
args = parser.parse_args()
|
||||
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
|
||||
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
|
||||
if args.mode=='all':
|
||||
for function in modes.values():
|
||||
function()
|
||||
else:
|
||||
modes[args.mode]()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit real fault injection; emits IDs/results, never session tokens."""
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client,check
|
||||
|
||||
|
||||
def compose(*args):
|
||||
result=subprocess.run(['docker','compose',*args],cwd=ROOT,capture_output=True,text=True,timeout=120)
|
||||
if result.returncode: raise RuntimeError(result.stderr)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def inside(body,**values):
|
||||
script="import json,io,time\nfrom uuid import UUID\nfrom messenger.db import Database\nfrom messenger.object_store import ObjectStore\nfrom messenger.jobs import work,replay,broker_client,STREAM\nfrom psycopg.types.json import Jsonb\ndb=Database();db.open();store=ObjectStore()\nv="+repr(values)+'\n'+body+'\nprint(json.dumps(result,default=str))\nstore.close();db.close()'
|
||||
proc=subprocess.run(['docker','compose','exec','-T','api','python','-'],cwd=ROOT,input=script,capture_output=True,text=True,timeout=120)
|
||||
if proc.returncode: raise RuntimeError(proc.stderr)
|
||||
return json.loads(proc.stdout)
|
||||
|
||||
|
||||
def state(aid):
|
||||
return inside("with db.pool.connection() as c:\n result=c.execute('SELECT a.status,a.staging_key,j.id AS job_id,j.attempt,j.lease_until,o.dispatched_at FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id JOIN job_outbox o ON o.job_id=j.id WHERE a.id=%s',(v[\"aid\"],)).fetchone()",aid=aid)
|
||||
|
||||
|
||||
def wait_for(predicate,seconds=30):
|
||||
end=time.monotonic()+seconds
|
||||
while time.monotonic()<end:
|
||||
result=predicate()
|
||||
if result: return result
|
||||
time.sleep(.2)
|
||||
raise AssertionError('Condition timed out')
|
||||
|
||||
|
||||
def ready(c,actor,att):
|
||||
return c.wait_ready(actor,att)
|
||||
|
||||
|
||||
def main():
|
||||
c=Client();alice,bob,_,chat=c.bootstrap()
|
||||
evidence={'result':'INCOMPLETE'}
|
||||
trigger=False
|
||||
try:
|
||||
# Durable acceptance while both relay and worker are absent: real COMMIT/publish gap.
|
||||
compose('stop','worker','relay')
|
||||
att,raw=c.attachment(alice,chat)
|
||||
c.message(alice,chat,'',attachments=[att['id']])
|
||||
c.call('GET','/api/v1/attachments/'+att['id']+'/content',409,actor=bob)
|
||||
before=state(att['id']);check(before['dispatched_at'] is None,'outbox already published')
|
||||
compose('restart','api','broker')
|
||||
wait_for(lambda: subprocess.run(['python3','scripts/cluster.py','wait'],cwd=ROOT,capture_output=True).returncode==0)
|
||||
c.call('GET','/api/v1/users/me',actor=alice)
|
||||
compose('start','relay','worker')
|
||||
meta=ready(c,bob,att)
|
||||
check(c.call('GET','/api/v1/attachments/'+att['id']+'/content',actor=bob)['bytes']==raw,'original changed')
|
||||
evidence['commit_publish_gap']={'accepted_status':202,'outbox_unpublished':True,'api_broker_restarted':True,'same_job_ready':state(att['id'])['job_id']==before['job_id']}
|
||||
# Publisher has already confirmed acceptance, broker restart preserves unread work.
|
||||
compose('stop','worker')
|
||||
att2,_=c.attachment(alice,chat)
|
||||
wait_for(lambda: state(att2['id'])['dispatched_at'])
|
||||
compose('restart','broker')
|
||||
compose('start','worker')
|
||||
ready(c,bob,att2)
|
||||
evidence['broker_persistence']={'unread_job_survived_restart':True}
|
||||
# Delay ready COMMIT in real PostgreSQL AFTER all S3 PUTs, kill before ack.
|
||||
compose('stop','worker')
|
||||
target,_=c.attachment(alice,chat)
|
||||
inside("with db.pool.connection() as c:\n c.execute(\"CREATE OR REPLACE FUNCTION lab5_pause_ready() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.id='\"+v['aid']+\"'::uuid AND NEW.status='ready' THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$\")\n c.execute('CREATE TRIGGER lab5_pause BEFORE UPDATE ON attachments FOR EACH ROW EXECUTE FUNCTION lab5_pause_ready()')\nresult=True",aid=target['id'])
|
||||
trigger=True
|
||||
compose('start','worker')
|
||||
def at_commit():
|
||||
return inside("with db.pool.connection() as c:\n result=bool(c.execute(\"SELECT 1 FROM pg_stat_activity WHERE wait_event='PgSleep' AND query LIKE 'UPDATE attachments SET status=%'\").fetchone())")
|
||||
wait_for(at_commit)
|
||||
observed=inside("with db.pool.connection() as c:\n row=c.execute('SELECT object_key FROM attachments WHERE id=%s',(v['aid'],)).fetchone()\nobj=store.get(row['object_key']);result={'s3_effect_written':obj.length>0};obj.close()",aid=target['id'])
|
||||
compose('kill','-s','SIGKILL','worker')
|
||||
# unless-stopped can restart a killed worker; stop it to retain the fault boundary.
|
||||
compose('stop','worker')
|
||||
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER lab5_pause ON attachments');c.execute('DROP FUNCTION lab5_pause_ready()')\nresult=True")
|
||||
trigger=False
|
||||
crashed=state(target['id']);check(crashed['attempt']==1,'expected first leased attempt')
|
||||
print('Waiting for real 90s lease expiry after killed worker.',flush=True)
|
||||
started=time.monotonic()
|
||||
wait_for(lambda: state(target['id'])['status']=='queued',110)
|
||||
compose('start','worker')
|
||||
ready(c,bob,target)
|
||||
after=state(target['id'])
|
||||
evidence['after_put_before_ack']={**observed,'lease_recovery_seconds':round(time.monotonic()-started,3),'attempt_after_recovery':after['attempt'],'same_job':after['job_id']==crashed['job_id']}
|
||||
# Re-deliver the completed original job. Ready metadata is immutable.
|
||||
original_meta=c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']
|
||||
duplicate=inside("result=work(db,store,v['job'],1)",job=after['job_id'])
|
||||
check(duplicate and c.call('GET','/api/v1/attachments/'+target['id'],actor=alice,schema='Attachment')['json']==original_meta,'duplicate changed ready result')
|
||||
evidence['duplicate']={'ready_noop':True}
|
||||
# Retry exhaustion with real missing staging (transient storage failure), explicit replay.
|
||||
compose('stop','worker','relay')
|
||||
failed,failed_raw=c.attachment(alice,chat)
|
||||
st=state(failed['id'])
|
||||
inside("store.delete(v['key']);result=True",key=st['staging_key'])
|
||||
for attempt in range(5):
|
||||
inside("with db.pool.connection() as c:\n c.execute('UPDATE attachment_jobs SET next_at=clock_timestamp() WHERE id=%s',(v['job'],))\nresult=work(db,store,v['job'])",job=st['job_id'])
|
||||
failure=c.call('GET','/api/v1/attachments/'+failed['id'],actor=alice,schema='Attachment')['json']
|
||||
check(failure['status']=='failed' and failure['error_code']=='retry_exhausted','retry did not terminate')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,body={'text':'','attachment_ids':[failed['id']]},headers={'Idempotency-Key':uuid.uuid4().hex})
|
||||
inside("store.put(v['key'],io.BytesIO(bytes.fromhex(v['raw'])),v['length'],'image/png')\nreplay(db,v['job'],'restore synthetic staging and retry')\nresult=True",key=st['staging_key'],raw=failed_raw.hex(),length=len(failed_raw),job=st['job_id'])
|
||||
compose('start','relay','worker')
|
||||
ready(c,bob,failed)
|
||||
evidence['retry_replay']={'failed_after':5,'code':'retry_exhausted','explicit_replay_ready':True,'failed_attachment_binding':409}
|
||||
evidence['result']='PASS'
|
||||
(ROOT/'evidence/jobs-faults.json').write_text(json.dumps(evidence,indent=2)+'\n')
|
||||
print('PASS: jobs-faults.json')
|
||||
finally:
|
||||
if trigger:
|
||||
inside("with db.pool.connection() as c:\n c.execute('DROP TRIGGER IF EXISTS lab5_pause ON attachments');c.execute('DROP FUNCTION IF EXISTS lab5_pause_ready()')\nresult=True")
|
||||
compose('start','relay','worker')
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,307 @@
|
||||
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
|
||||
import json
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
from redis.exceptions import RedisError
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .db import Database
|
||||
from .errors import ApiError, invalid, not_found, unauthorized
|
||||
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
|
||||
from .messages import store_message, validate_key, MESSAGE_FIELDS
|
||||
from .object_store import ObjectStore, unavailable
|
||||
from . import attachments
|
||||
from urllib.parse import quote
|
||||
from .pagination import decode_cursor, page, page_limit, public, stamp
|
||||
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
|
||||
|
||||
LOG = logging.getLogger('messenger')
|
||||
USER_FIELDS = 'id, username, display_name, created_at'
|
||||
|
||||
|
||||
def error_response(request, error):
|
||||
return JSONResponse({'error': {'code': error.code, 'message': error.message,
|
||||
'request_id': request.state.request_id}},
|
||||
status_code=error.status, headers=error.headers)
|
||||
|
||||
|
||||
def authenticate(request: Request):
|
||||
token = bearer(request.headers.get('Authorization', ''))
|
||||
token_hash = digest(token)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
|
||||
(token_hash,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
request.state.token_hash = token_hash
|
||||
return row['user_id']
|
||||
|
||||
|
||||
def require_member(conn, chat_id, user_id):
|
||||
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
|
||||
if not found:
|
||||
raise not_found()
|
||||
|
||||
|
||||
def chat_public(conn, row):
|
||||
item = public(row)
|
||||
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
|
||||
return item
|
||||
|
||||
|
||||
def create_app():
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
app.state.db = Database()
|
||||
app.state.storage = ObjectStore()
|
||||
app.state.upload_slots = asyncio.Semaphore(6)
|
||||
app.state.db.open()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
app.state.storage.close()
|
||||
app.state.db.close()
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.state.release = os.environ.get('APP_RELEASE', 'lab5-v1')
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_context(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
path = request.url.path
|
||||
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
|
||||
if request.method == 'POST' and json_endpoint:
|
||||
media = request.headers.get('Content-Type', '').split(';')
|
||||
unsupported = media[0].strip().lower() != 'application/json'
|
||||
for parameter in media[1:]:
|
||||
name, _, value = parameter.strip().partition('=')
|
||||
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
|
||||
unsupported = True
|
||||
if unsupported:
|
||||
if path != '/api/v1/users':
|
||||
await run_in_threadpool(authenticate, request)
|
||||
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
|
||||
try:
|
||||
json.loads((await request.body()).decode('utf-8'))
|
||||
except (UnicodeError, json.JSONDecodeError):
|
||||
raise invalid() from None
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = error_response(request, exc)
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
|
||||
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
|
||||
except Exception as exc:
|
||||
# Log exception class only: credentials, SQL and message content stay out of logs.
|
||||
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
|
||||
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = app.state.instance_id
|
||||
response.headers['X-App-Release'] = app.state.release
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error(request, exc):
|
||||
return error_response(request, exc)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return error_response(request, invalid())
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return error_response(request, not_found() if exc.status_code == 404 else invalid())
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.get('/health/ready')
|
||||
def ready(request: Request):
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
|
||||
conn.execute('SELECT status,variants FROM attachments LIMIT 0')
|
||||
conn.execute('SELECT 1 FROM job_outbox LIMIT 0')
|
||||
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
|
||||
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
|
||||
request.app.state.storage.check()
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.post('/api/v1/users', status_code=201)
|
||||
def register(body: CreateUser, request: Request):
|
||||
password_hash = HASHER.hash(body.password)
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
|
||||
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
|
||||
return public(row)
|
||||
|
||||
@app.post('/api/v1/auth/sessions', status_code=201)
|
||||
async def create_session(request: Request):
|
||||
# Run hashing and synchronous database calls outside the event loop.
|
||||
username, password = basic_credentials(request.headers.get('Authorization', ''))
|
||||
if await request.body():
|
||||
raise invalid()
|
||||
return await run_in_threadpool(login, request, username, password)
|
||||
|
||||
def login(request, username, password):
|
||||
db = request.app.state.db
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
|
||||
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
|
||||
if not row or not valid:
|
||||
raise unauthorized(basic=True)
|
||||
row.pop('password_hash')
|
||||
token, expires_at = new_session(db.session_ttl)
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
|
||||
(digest(token), row['id'], expires_at))
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
|
||||
|
||||
@app.get('/api/v1/users/me')
|
||||
def me(request: Request, user_id=Depends(authenticate)):
|
||||
db = request.app.state.db
|
||||
key = 'user:' + str(user_id)
|
||||
try:
|
||||
cached = db.cache.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (RedisError, ValueError):
|
||||
pass
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
result = public(row)
|
||||
try:
|
||||
db.cache.set(key, json.dumps(result), ex=60)
|
||||
except RedisError:
|
||||
pass
|
||||
return result
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
def logout(request: Request, user_id=Depends(authenticate)):
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.post('/api/v1/chats', status_code=201)
|
||||
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
|
||||
if str(user_id) not in body.member_ids:
|
||||
raise invalid()
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
|
||||
(uuid4(), body.title)).fetchone()
|
||||
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
|
||||
[(row['id'], uuid_value(member)) for member in body.member_ids])
|
||||
result = chat_public(conn, row)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise invalid() from None
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats')
|
||||
def list_chats(request: Request, user_id=Depends(authenticate)):
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'chats:' + str(user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
|
||||
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
|
||||
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
|
||||
result = page(scope, rows, limit)
|
||||
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}')
|
||||
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
|
||||
return chat_public(conn, row)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
|
||||
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
key = validate_key(request.headers.get('Idempotency-Key', ''))
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = store_message(conn, user_id, cid, body.text, key, body.attachment_ids)
|
||||
return public(row)
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}/messages')
|
||||
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'messages:' + str(cid)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (m.created_at,m.id) > (%s,%s)' if after else ''
|
||||
params = (cid, *after, limit+1) if after else (cid, limit+1)
|
||||
rows = conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.chat_id=%s'+where+
|
||||
' ORDER BY m.created_at,m.id LIMIT %s', params).fetchall()
|
||||
return page(scope, rows, limit)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments',status_code=202)
|
||||
async def upload_attachment(chat_id: str,request: Request,filename: str='',user_id=Depends(authenticate)):
|
||||
cid=uuid_value(chat_id)
|
||||
def access():
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn,cid,user_id)
|
||||
await run_in_threadpool(access)
|
||||
return await attachments.upload(request,user_id,cid,filename)
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
def attachment_metadata(attachment_id: str,request: Request,user_id=Depends(authenticate)):
|
||||
aid=uuid_value(attachment_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row=attachments.lookup(conn,aid,user_id)
|
||||
return attachments.metadata(row)
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
def attachment_content(attachment_id: str,request: Request,variant: str='original',user_id=Depends(authenticate)):
|
||||
aid=uuid_value(attachment_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row=attachments.lookup(conn,aid,user_id)
|
||||
if variant not in ('original','thumbnail','crop','optimized'):
|
||||
raise invalid()
|
||||
if row['status']!='ready':
|
||||
raise ApiError(409,'conflict','Вложение ещё не готово')
|
||||
item=row
|
||||
key=row['object_key']
|
||||
if variant!='original':
|
||||
item=next((v for v in row['variants'] if v['name']==variant),None)
|
||||
if item is None:
|
||||
raise not_found()
|
||||
key='variants/'+str(row['id']).replace('-','')+'/v1/'+variant
|
||||
opened=request.app.state.storage.get(key)
|
||||
if opened.length!=item['size_bytes']:
|
||||
opened.close()
|
||||
raise unavailable()
|
||||
return attachments.ObjectResponse(opened,media_type=item['content_type'],headers={
|
||||
'Content-Length':str(item['size_bytes']),
|
||||
'Content-Disposition':"inline; filename*=UTF-8''"+quote(row['filename'],safe='')})
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,173 @@
|
||||
"""Bounded binary upload, image validation and DB/S3 coordination."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import tempfile
|
||||
import threading
|
||||
import unicodedata
|
||||
import warnings
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from PIL import Image, PngImagePlugin, UnidentifiedImageError
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.requests import ClientDisconnect
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .pagination import public
|
||||
|
||||
MAX_BYTES=10485760
|
||||
MAX_PIXELS=40000000
|
||||
Image.MAX_IMAGE_PIXELS=MAX_PIXELS
|
||||
PngImagePlugin.MAX_TEXT_CHUNK=1048576
|
||||
PngImagePlugin.MAX_TEXT_MEMORY=1048576
|
||||
warnings.filterwarnings('error',category=Image.DecompressionBombWarning)
|
||||
DECODER=threading.BoundedSemaphore(1)
|
||||
LOG=logging.getLogger('messenger')
|
||||
FIELDS='id,chat_id,owner_id,filename,content_type,size_bytes,sha256,created_at,updated_at,status,error_code,variants'
|
||||
|
||||
|
||||
def object_lock(conn,key,wait=True):
|
||||
function='pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
|
||||
row=conn.execute('SELECT '+function+'(hashtextextended(%s,0)) AS locked',(key,)).fetchone()
|
||||
return True if wait else row['locked']
|
||||
|
||||
|
||||
def filename_value(value):
|
||||
if not value or not value.strip() or len(value)>255:
|
||||
raise invalid()
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeError:
|
||||
raise invalid() from None
|
||||
if any(c in '/\\' or unicodedata.category(c)=='Cc' for c in value):
|
||||
raise invalid()
|
||||
return value
|
||||
|
||||
|
||||
def metadata(row):
|
||||
result=public({key:value for key,value in row.items() if key not in ('object_key','staging_key')})
|
||||
return result
|
||||
|
||||
|
||||
def validate_image(file,mime):
|
||||
if not DECODER.acquire(timeout=3):
|
||||
raise ApiError(503,'unavailable','Обработка изображений занята')
|
||||
try:
|
||||
file.seek(0)
|
||||
with Image.open(file) as image:
|
||||
actual={'PNG':'image/png','JPEG':'image/jpeg'}.get(image.format)
|
||||
if actual is None or actual!=mime:
|
||||
raise ApiError(415,'unsupported_media_type','Ожидается PNG или JPEG с правильным MIME')
|
||||
if image.width*image.height>MAX_PIXELS:
|
||||
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей')
|
||||
image.verify()
|
||||
file.seek(0)
|
||||
except (Image.DecompressionBombWarning,Image.DecompressionBombError):
|
||||
raise ApiError(413,'payload_too_large','Изображение превышает лимит пикселей') from None
|
||||
except (UnidentifiedImageError,OSError,SyntaxError,ValueError,EOFError):
|
||||
raise ApiError(415,'unsupported_media_type','Повреждённое или неподдерживаемое изображение') from None
|
||||
finally:
|
||||
DECODER.release()
|
||||
|
||||
|
||||
def compensate(db,store,key):
|
||||
"""A failed/unknown COMMIT is not proof of absence. Recheck under the same lock."""
|
||||
try:
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,key,wait=False):
|
||||
return
|
||||
exists=conn.execute('SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s',(key,key)).fetchone()
|
||||
if not exists:
|
||||
store.delete(key)
|
||||
except Exception as exc:
|
||||
# A periodic, grace-protected janitor will retry after dependency recovery.
|
||||
LOG.warning('attachment_compensation_deferred type=%s',type(exc).__name__)
|
||||
|
||||
|
||||
def persist(db,store,user_id,chat_id,filename,mime,file,length,digest):
|
||||
aid=uuid4()
|
||||
key='staging/'+aid.hex
|
||||
try:
|
||||
with db.pool.connection() as conn:
|
||||
object_lock(conn,key)
|
||||
store.put(key,file,length,mime)
|
||||
row=conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,content_type,size_bytes,sha256,object_key,staging_key,status) '
|
||||
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued') RETURNING "+FIELDS,
|
||||
(aid,chat_id,user_id,filename,mime,length,digest,'originals/'+aid.hex,key)).fetchone()
|
||||
job=uuid4()
|
||||
conn.execute('INSERT INTO attachment_jobs(id,attachment_id) VALUES (%s,%s)',(job,aid))
|
||||
conn.execute('INSERT INTO job_outbox(job_id) VALUES (%s)',(job,))
|
||||
return metadata(row)
|
||||
except psycopg.Error:
|
||||
compensate(db,store,key)
|
||||
raise ApiError(503,'unavailable','Не удалось сохранить вложение') from None
|
||||
except BaseException:
|
||||
compensate(db,store,key)
|
||||
raise
|
||||
|
||||
|
||||
async def upload(request,user_id,chat_id,filename):
|
||||
filename=filename_value(filename)
|
||||
mime=request.headers.get('Content-Type','').split(';')[0].strip().lower()
|
||||
if mime not in ('image/png','image/jpeg'):
|
||||
raise ApiError(415,'unsupported_media_type','Ожидается image/png или image/jpeg')
|
||||
declared=request.headers.get('Content-Length')
|
||||
if declared is not None:
|
||||
try:
|
||||
length=int(declared)
|
||||
if length<0:
|
||||
raise ValueError
|
||||
except ValueError:
|
||||
raise invalid() from None
|
||||
if length>MAX_BYTES:
|
||||
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
|
||||
try:
|
||||
await asyncio.wait_for(request.app.state.upload_slots.acquire(),timeout=1)
|
||||
except TimeoutError:
|
||||
raise ApiError(503,'unavailable','Слишком много загрузок') from None
|
||||
try:
|
||||
with tempfile.TemporaryFile(mode='w+b') as file:
|
||||
total=0
|
||||
digest=hashlib.sha256()
|
||||
async def read_body():
|
||||
nonlocal total
|
||||
async for chunk in request.stream():
|
||||
total+=len(chunk)
|
||||
if total>MAX_BYTES:
|
||||
raise ApiError(413,'payload_too_large','Файл превышает 10 MiB')
|
||||
digest.update(chunk)
|
||||
await run_in_threadpool(file.write,chunk)
|
||||
try:
|
||||
await asyncio.wait_for(read_body(),timeout=30)
|
||||
except TimeoutError:
|
||||
raise ApiError(400,'invalid_request','Время загрузки истекло') from None
|
||||
except ClientDisconnect:
|
||||
raise invalid() from None
|
||||
await run_in_threadpool(validate_image,file,mime)
|
||||
return await run_in_threadpool(persist,request.app.state.db,request.app.state.storage,
|
||||
user_id,chat_id,filename,mime,file,total,digest.hexdigest())
|
||||
finally:
|
||||
request.app.state.upload_slots.release()
|
||||
|
||||
|
||||
def lookup(conn,attachment_id,user_id):
|
||||
row=conn.execute('SELECT '+', '.join('a.'+field for field in (FIELDS+',object_key').split(','))+
|
||||
' FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
|
||||
'WHERE a.id=%s AND m.user_id=%s',(attachment_id,user_id)).fetchone()
|
||||
if not row:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
class ObjectResponse(StreamingResponse):
|
||||
def __init__(self,opened,**kwargs):
|
||||
self.opened=opened
|
||||
super().__init__(opened.chunks(),**kwargs)
|
||||
|
||||
async def __call__(self,scope,receive,send):
|
||||
try:
|
||||
await super().__call__(scope,receive,send)
|
||||
finally:
|
||||
self.opened.close()
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Grace-protected orphan cleanup; absence is verified in the primary DB under lock."""
|
||||
import argparse
|
||||
from datetime import datetime,timezone,timedelta
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
|
||||
from .attachments import object_lock
|
||||
from .db import Database
|
||||
from .object_store import ObjectStore
|
||||
|
||||
|
||||
def sweep(db,store,grace_seconds=300,apply=False):
|
||||
if grace_seconds<60:
|
||||
raise ValueError('Grace must be at least60seconds')
|
||||
cutoff=datetime.now(timezone.utc)-timedelta(seconds=grace_seconds)
|
||||
result={'mode':'apply' if apply else 'dry-run','grace_seconds':grace_seconds,
|
||||
'listed':0,'fresh':0,'tracked':0,'locked':0,'orphans':0,'deleted':0}
|
||||
for page in (page for prefix in ("originals/","staging/","variants/") for page in store.pages(prefix)):
|
||||
for obj in page.get('Contents',[]):
|
||||
result['listed']+=1
|
||||
if obj['LastModified']>=cutoff:
|
||||
result['fresh']+=1
|
||||
continue
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,obj['Key'],wait=False):
|
||||
result['locked']+=1
|
||||
continue
|
||||
if conn.execute("SELECT 1 FROM attachments WHERE object_key=%s OR staging_key=%s OR %s LIKE 'variants/'||replace(id::text,'-','')||'/v1/%%'",(obj['Key'],obj['Key'],obj['Key'])).fetchone():
|
||||
result['tracked']+=1
|
||||
continue
|
||||
result['orphans']+=1
|
||||
if apply:
|
||||
store.delete(obj['Key'])
|
||||
result['deleted']+=1
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('--grace-seconds',type=int,default=300)
|
||||
parser.add_argument('--apply',action='store_true')
|
||||
parser.add_argument('--dry-run',action='store_true')
|
||||
parser.add_argument('--loop',action='store_true')
|
||||
parser.add_argument('--interval',type=int,default=60)
|
||||
args=parser.parse_args()
|
||||
if args.grace_seconds<60 or args.interval<1 or (args.apply and args.dry_run):
|
||||
parser.error('Use grace>=60, interval>=1 and either --apply or --dry-run')
|
||||
db=Database()
|
||||
store=ObjectStore()
|
||||
db.open()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
result=sweep(db,store,args.grace_seconds,args.apply)
|
||||
print(json.dumps(result),flush=True)
|
||||
except Exception as exc:
|
||||
print(json.dumps({'error':type(exc).__name__,'deleted_unknown':False}),flush=True)
|
||||
if not args.loop:
|
||||
raise
|
||||
finally:
|
||||
Path('/tmp/cleanup-heartbeat').touch()
|
||||
if not args.loop:
|
||||
break
|
||||
time.sleep(args.interval)
|
||||
finally:
|
||||
store.close()
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
main()
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Bounded dependencies; PostgreSQL is always the source of truth."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
|
||||
def setting(name, default=None):
|
||||
filename = os.environ.get(name + '_FILE')
|
||||
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
|
||||
if value is None:
|
||||
raise RuntimeError(f'Missing configuration: {name}')
|
||||
return value
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
|
||||
if self.session_ttl < 1:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||
self.pool = ConnectionPool(
|
||||
setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
|
||||
timeout=2, max_waiting=16, open=False,
|
||||
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
|
||||
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
|
||||
)
|
||||
self.cache = Redis.from_url(
|
||||
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=30)
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
self.cache.close()
|
||||
@@ -0,0 +1,19 @@
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid():
|
||||
return ApiError(400, 'invalid_request', 'Неверный запрос')
|
||||
|
||||
|
||||
def unauthorized(basic=False):
|
||||
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
|
||||
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
|
||||
|
||||
|
||||
def not_found():
|
||||
return ApiError(404, 'not_found', 'Объект не найден')
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Transactional outbox, fenced processing, bounded retries and operator replay."""
|
||||
from contextlib import nullcontext
|
||||
import signal
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
import random
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from uuid import UUID
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
from redis import Redis
|
||||
from .db import setting
|
||||
from .attachments import object_lock
|
||||
|
||||
LOG=logging.getLogger('messenger')
|
||||
STREAM='attachment-jobs'
|
||||
GROUP='image-workers'
|
||||
MAX_ATTEMPTS=int(os.environ.get('JOB_MAX_ATTEMPTS','5'))
|
||||
LEASE_SECONDS=90
|
||||
|
||||
|
||||
def broker_client():
|
||||
return Redis.from_url(setting('BROKER_URL','redis://broker:6379/0'),decode_responses=True,
|
||||
socket_connect_timeout=1,socket_timeout=3,max_connections=4)
|
||||
|
||||
|
||||
def job_lock(conn,job_id):
|
||||
return conn.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',
|
||||
('job:'+str(job_id),)).fetchone()['ok']
|
||||
|
||||
|
||||
def unlock(conn,job_id):
|
||||
conn.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job_id),))
|
||||
|
||||
|
||||
def release_lock(conn,job_id):
|
||||
# Session locks survive ROLLBACK. A failed transaction must be cleared before unlock.
|
||||
try:
|
||||
conn.rollback()
|
||||
unlock(conn,job_id)
|
||||
conn.commit()
|
||||
except BaseException:
|
||||
conn.close() # never return a session with an uncertain held lock to the pool
|
||||
raise
|
||||
|
||||
|
||||
def relay_tick(db,broker):
|
||||
# Recovery uses the same session lock as the processor. A slow live worker is never fenced out.
|
||||
with db.pool.connection() as conn:
|
||||
expired=conn.execute("SELECT j.id FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id "
|
||||
"WHERE a.status='processing' AND j.lease_until<clock_timestamp() LIMIT 50").fetchall()
|
||||
for row in expired:
|
||||
with db.pool.connection() as conn:
|
||||
if not job_lock(conn,row['id']):
|
||||
continue
|
||||
try:
|
||||
conn.execute("UPDATE attachments SET status='queued',updated_at=clock_timestamp() WHERE id="
|
||||
"(SELECT attachment_id FROM attachment_jobs WHERE id=%s AND lease_until<clock_timestamp()) "
|
||||
"AND status='processing'",(row['id'],))
|
||||
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(row['id'],))
|
||||
conn.execute('UPDATE attachment_jobs SET next_at=clock_timestamp(),lease_until=NULL WHERE id=%s',(row['id'],))
|
||||
conn.commit()
|
||||
finally:
|
||||
release_lock(conn,row['id'])
|
||||
with db.pool.connection() as conn:
|
||||
rows=conn.execute("SELECT j.*,o.job_id FROM job_outbox o JOIN attachment_jobs j ON j.id=o.job_id "
|
||||
"JOIN attachments a ON a.id=j.attachment_id WHERE o.dispatched_at IS NULL AND a.status='queued' "
|
||||
"AND j.next_at<=clock_timestamp() ORDER BY j.next_at LIMIT 20 FOR UPDATE OF o SKIP LOCKED").fetchall()
|
||||
for row in rows:
|
||||
payload={'schema_version':1,'type':'attachment.process.v1','job_id':str(row['id']),
|
||||
'attachment_id':str(row['attachment_id']),'pipeline_version':row['pipeline_version'],
|
||||
'attempt':row['attempt']+1,'created_at':row['created_at'].isoformat()}
|
||||
for key in ('traceparent','tracestate'):
|
||||
if row[key]: payload[key]=row[key]
|
||||
broker.xadd(STREAM,{'payload':json.dumps(payload)})
|
||||
conn.execute('UPDATE job_outbox SET dispatched_at=clock_timestamp() WHERE job_id=%s',(row['id'],))
|
||||
return len(rows)
|
||||
|
||||
|
||||
def process_objects_local(store,row,directory=None):
|
||||
with (nullcontext(directory) if directory else tempfile.TemporaryDirectory(prefix='image-job-')) as directory:
|
||||
source=Path(directory)/'input'
|
||||
opened=store.get(row['staging_key'])
|
||||
try:
|
||||
with source.open('wb') as file:
|
||||
total=0
|
||||
for chunk in opened.chunks():
|
||||
total+=len(chunk)
|
||||
if total>row['size_bytes'] or total>10485760: raise ValueError('invalid_image')
|
||||
file.write(chunk)
|
||||
finally:
|
||||
opened.close()
|
||||
raw_hash=hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
if source.stat().st_size!=row['size_bytes'] or raw_hash!=row['sha256']:
|
||||
raise ValueError('invalid_image')
|
||||
completed=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),directory],
|
||||
capture_output=True,timeout=20)
|
||||
if completed.returncode:
|
||||
raise ValueError('invalid_image' if completed.returncode==65 else 'processing_limit')
|
||||
variants=json.loads(completed.stdout)
|
||||
with source.open('rb') as file:
|
||||
store.put(row['object_key'],file,row['size_bytes'],row['content_type'])
|
||||
for name,item in variants.items():
|
||||
with (Path(directory)/name).open('rb') as file:
|
||||
store.put('variants/'+row['id'].hex+'/v1/'+name,file,item['size_bytes'],item['content_type'])
|
||||
return list(variants.values())
|
||||
|
||||
|
||||
def process_objects(store,row):
|
||||
# A killed child cannot continue S3 writes or retain a stream/decoder in a detached thread.
|
||||
timeout=float(os.environ.get('PROCESS_TIMEOUT_SECONDS','45'))
|
||||
if not 1<=timeout<=45: raise ValueError('processing_limit')
|
||||
with tempfile.TemporaryDirectory(prefix='image-job-') as directory:
|
||||
process=subprocess.Popen([sys.executable,'-m','messenger.processor',directory],
|
||||
stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.PIPE,
|
||||
text=True,start_new_session=True)
|
||||
try:
|
||||
output,_=process.communicate(json.dumps(row,default=str),timeout=timeout)
|
||||
except BaseException:
|
||||
# Terminate processor AND decoder before lock release or another attempt.
|
||||
try: os.killpg(process.pid,signal.SIGKILL)
|
||||
except ProcessLookupError: pass
|
||||
process.communicate()
|
||||
raise
|
||||
if process.returncode==65: raise ValueError('invalid_image')
|
||||
if process.returncode==66: raise ValueError('processing_limit')
|
||||
if process.returncode: raise RuntimeError('storage_unavailable')
|
||||
return json.loads(output)
|
||||
|
||||
|
||||
def work(db,store,job_id,delivery_attempt=None):
|
||||
job_id=UUID(str(job_id))
|
||||
# Dedicated pooled session holds the lock, with short committed transactions between effects.
|
||||
with db.pool.connection() as conn:
|
||||
if not job_lock(conn,job_id):
|
||||
return False
|
||||
conn.commit()
|
||||
try:
|
||||
row=conn.execute('SELECT a.*,j.attempt,j.pipeline_version,j.next_at<=clock_timestamp() AS due '
|
||||
'FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id WHERE j.id=%s',(job_id,)).fetchone()
|
||||
conn.commit()
|
||||
if not row or row['status'] in ('ready','failed'):
|
||||
return True
|
||||
if row['status']!='queued' or not row['due']:
|
||||
return False
|
||||
if delivery_attempt is not None and delivery_attempt!=row['attempt']+1:
|
||||
return True
|
||||
attempt=row['attempt']+1
|
||||
if attempt>MAX_ATTEMPTS:
|
||||
conn.execute("UPDATE attachments SET status='failed',error_code='retry_exhausted',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
||||
conn.commit()
|
||||
return True
|
||||
conn.execute("UPDATE attachment_jobs SET attempt=%s,lease_until=clock_timestamp()+%s*interval '1 second' WHERE id=%s",
|
||||
(attempt,LEASE_SECONDS,job_id))
|
||||
conn.execute("UPDATE attachments SET status='processing',updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
||||
conn.commit()
|
||||
try:
|
||||
variants=process_objects(store,row)
|
||||
except Exception as exc:
|
||||
permanent=isinstance(exc,ValueError)
|
||||
final=permanent or attempt>=MAX_ATTEMPTS
|
||||
code=str(exc) if permanent else ('retry_exhausted' if final else 'storage_unavailable')
|
||||
if code not in ('invalid_image','processing_limit','retry_exhausted','storage_unavailable'):
|
||||
code='processing_failed'
|
||||
delay=min(30,2**(attempt-1))*random.uniform(.5,1.5)
|
||||
conn.execute("UPDATE attachments SET status=%s,error_code=%s,updated_at=clock_timestamp() WHERE id=%s",
|
||||
('failed' if final else 'queued',code if final else None,row['id']))
|
||||
conn.execute("UPDATE attachment_jobs SET next_at=clock_timestamp()+%s*interval '1 second',lease_until=NULL WHERE id=%s",(delay,job_id))
|
||||
if not final: conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
|
||||
conn.commit()
|
||||
LOG.warning('job_attempt outcome=%s attempt=%s', 'failed' if final else 'retry',attempt)
|
||||
return True
|
||||
# Same live lock and attempt fence protect publication; no new attempt can publish concurrently.
|
||||
current=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s FOR UPDATE',(job_id,)).fetchone()
|
||||
if current['attempt']!=attempt:
|
||||
conn.rollback()
|
||||
return False
|
||||
conn.execute("UPDATE attachments SET status='ready',variants=%s,error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",
|
||||
(Jsonb(variants),row['id']))
|
||||
conn.execute('UPDATE attachment_jobs SET lease_until=NULL WHERE id=%s',(job_id,))
|
||||
conn.commit()
|
||||
LOG.info('job_completed attempt=%s',attempt)
|
||||
clean_staging(db,store,row['id'])
|
||||
return True
|
||||
finally:
|
||||
release_lock(conn,job_id)
|
||||
|
||||
|
||||
def clean_staging(db,store,attachment_id=None):
|
||||
with db.pool.connection() as conn:
|
||||
rows=conn.execute("SELECT id,staging_key FROM attachments WHERE staging_key IS NOT NULL "
|
||||
"AND (status='ready' OR (status='failed' AND updated_at<clock_timestamp()-interval '24 hours')) "
|
||||
+('AND id=%s ' if attachment_id else '')+'LIMIT 50', (attachment_id,) if attachment_id else ()).fetchall()
|
||||
for row in rows:
|
||||
with db.pool.connection() as conn:
|
||||
if not object_lock(conn,row['staging_key'],wait=False): continue
|
||||
current=conn.execute("SELECT status FROM attachments WHERE id=%s FOR UPDATE",(row['id'],)).fetchone()
|
||||
if current['status'] not in ('ready','failed'): continue
|
||||
store.delete(row['staging_key'])
|
||||
conn.execute('UPDATE attachments SET staging_key=NULL WHERE id=%s',(row['id'],))
|
||||
|
||||
|
||||
def replay(db,job_id,reason):
|
||||
if not reason.strip() or len(reason)>200: raise ValueError('Short audit reason required')
|
||||
with db.pool.connection() as conn:
|
||||
if not job_lock(conn,job_id): raise ValueError('Job active')
|
||||
try:
|
||||
row=conn.execute('SELECT a.* FROM attachments a JOIN attachment_jobs j ON j.attachment_id=a.id WHERE j.id=%s FOR UPDATE OF a',(job_id,)).fetchone()
|
||||
if not row or row['status']!='failed' or not row['staging_key']:
|
||||
raise ValueError('Replay requires failed job with retained staging')
|
||||
conn.execute("UPDATE attachments SET status='queued',error_code=NULL,updated_at=clock_timestamp() WHERE id=%s",(row['id'],))
|
||||
conn.execute('UPDATE attachment_jobs SET attempt=0,replay_count=replay_count+1,lease_until=NULL,next_at=clock_timestamp() WHERE id=%s',(job_id,))
|
||||
conn.execute('UPDATE job_outbox SET dispatched_at=NULL WHERE job_id=%s',(job_id,))
|
||||
conn.execute('INSERT INTO job_replays(job_id,reason) VALUES (%s,%s)',(job_id,reason))
|
||||
conn.commit()
|
||||
finally:
|
||||
release_lock(conn,job_id)
|
||||
@@ -0,0 +1,54 @@
|
||||
"""One PostgreSQL transaction owns both a message and its retry result."""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from uuid import uuid4
|
||||
|
||||
from .errors import ApiError, invalid, not_found
|
||||
from .models import uuid_value
|
||||
|
||||
|
||||
def validate_key(key):
|
||||
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
raise invalid()
|
||||
return key
|
||||
|
||||
|
||||
MESSAGE_FIELDS = 'm.id,m.chat_id,m.sender_id,m.text,m.created_at,ARRAY(SELECT attachment_id FROM message_attachments WHERE message_id=m.id ORDER BY position) AS attachment_ids'
|
||||
|
||||
|
||||
def store_message(conn, user_id, chat_id, text, key, attachment_ids=None):
|
||||
attachment_ids=[] if attachment_ids is None else attachment_ids
|
||||
payload={'text':text}
|
||||
if attachment_ids:
|
||||
payload['attachment_ids']=attachment_ids
|
||||
canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||||
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
||||
params = (user_id, chat_id, key)
|
||||
# A competing INSERT waits for the other transaction, then the next statement sees its result.
|
||||
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
|
||||
(*params, fingerprint))
|
||||
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
|
||||
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
|
||||
if record['message_id'] is not None and record['active']:
|
||||
if record['request_hash'] != fingerprint:
|
||||
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
|
||||
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',
|
||||
(record['message_id'],)).fetchone()
|
||||
ids=[uuid_value(item) for item in attachment_ids]
|
||||
if ids:
|
||||
allowed=conn.execute('SELECT id,status FROM attachments WHERE chat_id=%s AND id=ANY(%s)',(chat_id,ids)).fetchall()
|
||||
if len(allowed)!=len(ids):
|
||||
raise not_found()
|
||||
if any(a['status']=='failed' for a in allowed):
|
||||
raise ApiError(409,'conflict','Обработка вложения завершилась ошибкой')
|
||||
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
|
||||
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
|
||||
if ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
|
||||
[(row['id'],aid,i) for i,aid in enumerate(ids)])
|
||||
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
|
||||
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
|
||||
return conn.execute('SELECT '+MESSAGE_FIELDS+' FROM messages m WHERE m.id=%s',(row['id'],)).fetchone()
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from .db import setting
|
||||
|
||||
|
||||
def main():
|
||||
directory = Path(__file__).resolve().parents[2] / 'migrations'
|
||||
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
|
||||
for path in sorted(directory.glob('*.sql')):
|
||||
sql = path.read_text()
|
||||
checksum = hashlib.sha256(sql.encode()).hexdigest()
|
||||
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
|
||||
if found:
|
||||
if found[0] != checksum:
|
||||
raise RuntimeError('Applied migration checksum changed: ' + path.name)
|
||||
continue
|
||||
conn.execute(sql)
|
||||
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
|
||||
print('Applied', path.name)
|
||||
print('Migrations up to date')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,75 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from .errors import invalid
|
||||
|
||||
|
||||
def uuid_value(value):
|
||||
try:
|
||||
# Require the UUID string representation, not integer/coercible values.
|
||||
if not isinstance(value, str) or len(value) != 36:
|
||||
raise ValueError
|
||||
return UUID(value)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid', strict=True)
|
||||
|
||||
@field_validator('*')
|
||||
@classmethod
|
||||
def valid_strings(cls, value, info):
|
||||
if isinstance(value, str):
|
||||
if not value.strip() and info.field_name != 'text':
|
||||
raise ValueError('Blank string')
|
||||
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
|
||||
if '\x00' in value:
|
||||
raise ValueError('NUL is not supported')
|
||||
value.encode('utf-8')
|
||||
return value
|
||||
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('username')
|
||||
@classmethod
|
||||
def username_ascii(cls, value):
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
|
||||
raise ValueError('Invalid username')
|
||||
return value
|
||||
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def members(cls, value):
|
||||
ids = [uuid_value(item) for item in value]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError('Duplicate member')
|
||||
return [str(item) for item in ids]
|
||||
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: str = Field(max_length=4000)
|
||||
attachment_ids: list[str] = Field(default_factory=list,max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
@classmethod
|
||||
def attachment_values(cls,value):
|
||||
ids=[uuid_value(item) for item in value]
|
||||
if len(ids)!=len(set(ids)):
|
||||
raise ValueError('Duplicate attachment')
|
||||
return [str(item) for item in ids]
|
||||
|
||||
@model_validator(mode='after')
|
||||
def nonempty_message(self):
|
||||
if not self.text.strip() and not self.attachment_ids:
|
||||
raise ValueError('Message must contain text or attachment')
|
||||
return self
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Small bounded S3 client. API credentials never contain root/admin permissions."""
|
||||
from contextlib import contextmanager
|
||||
import threading
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
from .db import setting
|
||||
from .errors import ApiError
|
||||
|
||||
PREFIX = 'originals/'
|
||||
|
||||
|
||||
def unavailable():
|
||||
return ApiError(503,'unavailable','Хранилище файлов временно недоступно')
|
||||
|
||||
|
||||
class OpenObject:
|
||||
def __init__(self,body,release,length):
|
||||
self.body=body
|
||||
self.release=release
|
||||
self.length=length
|
||||
self.closed=False
|
||||
self.close_lock=threading.Lock()
|
||||
|
||||
def chunks(self):
|
||||
try:
|
||||
while True:
|
||||
data=self.body.read(65536)
|
||||
if not data:
|
||||
break
|
||||
yield data
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
def close(self):
|
||||
with self.close_lock:
|
||||
if self.closed:
|
||||
return
|
||||
self.closed=True
|
||||
try:
|
||||
self.body.close()
|
||||
finally:
|
||||
self.release()
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self):
|
||||
self.bucket=setting('S3_BUCKET','messenger-attachments')
|
||||
self.gate=threading.BoundedSemaphore(8)
|
||||
self.client=boto3.client('s3',endpoint_url=setting('S3_ENDPOINT','http://minio:9000'),
|
||||
region_name=setting('S3_REGION','us-east-1'),
|
||||
aws_access_key_id=setting('S3_ACCESS_KEY_ID'),aws_secret_access_key=setting('S3_SECRET_ACCESS_KEY'),
|
||||
config=Config(signature_version='s3v4',connect_timeout=1,read_timeout=3,max_pool_connections=8,
|
||||
retries={'total_max_attempts':1,'mode':'standard'},s3={'addressing_style':'path'},
|
||||
request_checksum_calculation='when_required',response_checksum_validation='when_required'))
|
||||
|
||||
@contextmanager
|
||||
def operation(self):
|
||||
if not self.gate.acquire(timeout=1):
|
||||
raise unavailable()
|
||||
try:
|
||||
yield
|
||||
except (BotoCoreError,ClientError):
|
||||
raise unavailable() from None
|
||||
finally:
|
||||
self.gate.release()
|
||||
|
||||
def check(self):
|
||||
with self.operation():
|
||||
self.client.list_objects_v2(Bucket=self.bucket,Prefix=PREFIX,MaxKeys=1)
|
||||
|
||||
def put(self,key,body,length,mime):
|
||||
with self.operation():
|
||||
body.seek(0)
|
||||
self.client.put_object(Bucket=self.bucket,Key=key,Body=body,ContentLength=length,ContentType=mime)
|
||||
|
||||
def delete(self,key):
|
||||
with self.operation():
|
||||
self.client.delete_object(Bucket=self.bucket,Key=key)
|
||||
|
||||
def get(self,key):
|
||||
if not self.gate.acquire(timeout=1):
|
||||
raise unavailable()
|
||||
try:
|
||||
result=self.client.get_object(Bucket=self.bucket,Key=key)
|
||||
return OpenObject(result['Body'],self.gate.release,result['ContentLength'])
|
||||
except (BotoCoreError,ClientError):
|
||||
self.gate.release()
|
||||
raise unavailable() from None
|
||||
except BaseException:
|
||||
self.gate.release()
|
||||
raise
|
||||
|
||||
def pages(self,prefix=PREFIX):
|
||||
token=None
|
||||
while True:
|
||||
params={'Bucket':self.bucket,'Prefix':prefix,'MaxKeys':1000}
|
||||
if token:
|
||||
params['ContinuationToken']=token
|
||||
with self.operation():
|
||||
result=self.client.list_objects_v2(**params)
|
||||
yield result
|
||||
if not result.get('IsTruncated'):
|
||||
break
|
||||
token=result['NextContinuationToken']
|
||||
|
||||
def close(self):
|
||||
self.client.close()
|
||||
@@ -0,0 +1,56 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
from .errors import invalid
|
||||
from .models import uuid_value
|
||||
|
||||
|
||||
def stamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def encode_cursor(scope, row):
|
||||
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(scope, value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
|
||||
raise ValueError
|
||||
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
|
||||
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
|
||||
raise ValueError
|
||||
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
|
||||
if ts.tzinfo is None:
|
||||
raise ValueError
|
||||
return ts, uuid_value(data[2])
|
||||
except (ValueError, TypeError, AttributeError, OverflowError):
|
||||
raise invalid() from None
|
||||
|
||||
|
||||
def page_limit(value):
|
||||
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
|
||||
raise invalid()
|
||||
return int(value)
|
||||
|
||||
|
||||
def public(row):
|
||||
def value(item):
|
||||
if isinstance(item,datetime):
|
||||
return stamp(item)
|
||||
if isinstance(item,UUID):
|
||||
return str(item)
|
||||
if isinstance(item,list):
|
||||
return [value(part) for part in item]
|
||||
return item
|
||||
return {key:value(item) for key,item in row.items()}
|
||||
|
||||
|
||||
def page(scope, rows, limit):
|
||||
return {'items': [public(row) for row in rows[:limit]],
|
||||
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Bounded image decoder subprocess. Original bytes are never re-encoded."""
|
||||
import hashlib
|
||||
import json
|
||||
import resource
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from PIL import Image, ImageOps, PngImagePlugin
|
||||
|
||||
|
||||
def fitted(image, maximum):
|
||||
scale=min(1,maximum/image.width,maximum/image.height)
|
||||
size=(max(1,int(image.width*scale)),max(1,int(image.height*scale)))
|
||||
return image.resize(size,Image.Resampling.LANCZOS)
|
||||
|
||||
|
||||
def process_file(source, directory):
|
||||
Image.MAX_IMAGE_PIXELS=40000000
|
||||
PngImagePlugin.MAX_TEXT_CHUNK=1048576
|
||||
PngImagePlugin.MAX_TEXT_MEMORY=1048576
|
||||
with Image.open(source) as original:
|
||||
if original.format not in ('PNG','JPEG') or original.width*original.height>40000000:
|
||||
raise ValueError('invalid_image')
|
||||
image=ImageOps.exif_transpose(original)
|
||||
# Alpha is composited on white; output carries no source metadata.
|
||||
rgba=image.convert('RGBA')
|
||||
white=Image.new('RGBA',rgba.size,'white')
|
||||
white.alpha_composite(rgba)
|
||||
image=white.convert('RGB')
|
||||
image.info.clear()
|
||||
outputs={'thumbnail':(fitted(image,256),'JPEG','image/jpeg'),
|
||||
'crop':(ImageOps.fit(image,(128,128),Image.Resampling.LANCZOS,centering=(.5,.5)),'JPEG','image/jpeg'),
|
||||
'optimized':(fitted(image,1280),'WEBP','image/webp')}
|
||||
manifest={}
|
||||
for name,(variant,fmt,mime) in outputs.items():
|
||||
path=Path(directory)/name
|
||||
variant.save(path,format=fmt,quality=80 if fmt=='WEBP' else 85,exif=b'')
|
||||
raw=path.read_bytes()
|
||||
manifest[name]={'name':name,'content_type':mime,'width':variant.width,'height':variant.height,
|
||||
'size_bytes':len(raw),'sha256':hashlib.sha256(raw).hexdigest()}
|
||||
return manifest
|
||||
|
||||
|
||||
if __name__=='__main__':
|
||||
resource.setrlimit(resource.RLIMIT_AS,(768*1024*1024,768*1024*1024))
|
||||
resource.setrlimit(resource.RLIMIT_CPU,(15,15))
|
||||
try:
|
||||
print(json.dumps(process_file(sys.argv[1],sys.argv[2])))
|
||||
except (OSError,ValueError,MemoryError,Image.DecompressionBombError):
|
||||
sys.exit(65)
|
||||
@@ -0,0 +1,29 @@
|
||||
"""Isolated complete S3+decode+publish effect, terminated by parent after 45 seconds."""
|
||||
import os
|
||||
import signal
|
||||
import json
|
||||
import sys
|
||||
from uuid import UUID
|
||||
from .jobs import process_objects_local
|
||||
from .object_store import ObjectStore
|
||||
|
||||
|
||||
def main():
|
||||
# Survives SIGKILL of the supervising worker; kill the isolated effect group at deadline.
|
||||
signal.signal(signal.SIGALRM,lambda *_:os.killpg(os.getpgrp(),signal.SIGKILL))
|
||||
signal.alarm(45)
|
||||
row=json.load(sys.stdin)
|
||||
row['id']=UUID(row['id'])
|
||||
store=ObjectStore()
|
||||
try:
|
||||
print(json.dumps(process_objects_local(store,row,sys.argv[1])))
|
||||
except ValueError as exc:
|
||||
sys.exit(66 if str(exc)=='processing_limit' else 65)
|
||||
except Exception:
|
||||
# Do not export credentials, storage keys or SDK response details.
|
||||
sys.exit(75)
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,49 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, InvalidHashError
|
||||
from .errors import unauthorized
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
|
||||
|
||||
|
||||
def digest(token):
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
|
||||
def basic_credentials(header):
|
||||
try:
|
||||
scheme, encoded = header.split(' ', 1)
|
||||
if scheme.lower() != 'basic':
|
||||
raise ValueError
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
return username, password
|
||||
except (ValueError, UnicodeError):
|
||||
raise unauthorized(basic=True) from None
|
||||
|
||||
|
||||
def check_password(encoded, password):
|
||||
try:
|
||||
return HASHER.verify(encoded, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
def bearer(header):
|
||||
try:
|
||||
scheme, token = header.split(' ', 1)
|
||||
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise ValueError
|
||||
return token
|
||||
except ValueError:
|
||||
raise unauthorized() from None
|
||||
|
||||
|
||||
def new_session(ttl):
|
||||
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Separate relay/consumer processes; durable Redis Streams consumer group."""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
from uuid import uuid4
|
||||
from redis.exceptions import ResponseError
|
||||
from .cleanup import sweep
|
||||
from .db import Database
|
||||
from .jobs import broker_client, relay_tick, work, clean_staging, replay, STREAM, GROUP
|
||||
from .object_store import ObjectStore
|
||||
|
||||
LOG=logging.getLogger('messenger')
|
||||
STOP=False
|
||||
|
||||
|
||||
def main():
|
||||
parser=argparse.ArgumentParser()
|
||||
parser.add_argument('role',choices=['worker','relay','replay'])
|
||||
parser.add_argument('--job-id')
|
||||
parser.add_argument('--reason',default='operator retry')
|
||||
args=parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
def stop(*_):
|
||||
global STOP
|
||||
STOP=True
|
||||
signal.signal(signal.SIGTERM,stop)
|
||||
signal.signal(signal.SIGINT,stop)
|
||||
db=Database();db.open();store=ObjectStore();broker=broker_client()
|
||||
consumer=uuid4().hex
|
||||
last_cleanup=0
|
||||
try:
|
||||
if args.role=='replay':
|
||||
replay(db,args.job_id,args.reason)
|
||||
print('Replay queued with audit')
|
||||
return
|
||||
if args.role=='worker':
|
||||
try: broker.xgroup_create(STREAM,GROUP,id='0',mkstream=True)
|
||||
except ResponseError as exc:
|
||||
if 'BUSYGROUP' not in str(exc): raise
|
||||
while not STOP:
|
||||
try:
|
||||
if args.role=='relay':
|
||||
relay_tick(db,broker)
|
||||
clean_staging(db,store)
|
||||
if time.monotonic()-last_cleanup>60:
|
||||
sweep(db,store,300,True)
|
||||
last_cleanup=time.monotonic()
|
||||
time.sleep(.3)
|
||||
else:
|
||||
claimed=broker.xautoclaim(STREAM,GROUP,consumer,min_idle_time=90000,start_id='0-0',count=1)
|
||||
messages=claimed[1]
|
||||
if not messages:
|
||||
batches=broker.xreadgroup(GROUP,consumer,{STREAM:'>'},count=1,block=1000)
|
||||
messages=batches[0][1] if batches else []
|
||||
for message_id,fields in messages:
|
||||
payload=json.loads(fields['payload'])
|
||||
if work(db,store,payload['job_id'],payload['attempt']):
|
||||
broker.xack(STREAM,GROUP,message_id)
|
||||
broker.xdel(STREAM,message_id)
|
||||
except Exception as exc:
|
||||
LOG.warning('background_unavailable role=%s type=%s',args.role,type(exc).__name__)
|
||||
time.sleep(1)
|
||||
finally:
|
||||
broker.close();store.close();db.close()
|
||||
|
||||
|
||||
if __name__=='__main__': main()
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from messenger.app import create_app
|
||||
|
||||
PASSWORD = 'Тестовый:пароль🔐' * 8
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
with TestClient(create_app()) as c:
|
||||
yield c
|
||||
|
||||
|
||||
def user(c):
|
||||
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
|
||||
r = c.post('/api/v1/users', json=body)
|
||||
assert r.status_code == 201, r.text
|
||||
return r.json(), body
|
||||
|
||||
|
||||
def login(c, body):
|
||||
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
|
||||
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
|
||||
assert r.status_code == 201, r.text
|
||||
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
|
||||
|
||||
|
||||
def test_auth_unicode_logout_and_cache(client):
|
||||
u, body = user(client)
|
||||
h, session = login(client, body)
|
||||
assert session['user'] == u
|
||||
assert len(session['token']) >= 43
|
||||
key = 'user:' + u['id']
|
||||
client.app.state.db.cache.delete(key)
|
||||
for _ in range(2):
|
||||
assert client.get('/api/v1/users/me', headers=h).json() == u
|
||||
cached = json.loads(client.app.state.db.cache.get(key))
|
||||
assert cached == u
|
||||
assert 0 < client.app.state.db.cache.ttl(key) <= 60
|
||||
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
|
||||
r = client.get('/api/v1/users/me', headers=h)
|
||||
assert r.status_code == 401
|
||||
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
|
||||
assert r.json()['error']['request_id'] == r.headers['x-request-id']
|
||||
|
||||
|
||||
def test_session_expiry(monkeypatch):
|
||||
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
|
||||
with TestClient(create_app()) as client:
|
||||
_, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 200
|
||||
time.sleep(2.1)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 401
|
||||
|
||||
|
||||
def test_registration_race(client):
|
||||
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
|
||||
with ThreadPoolExecutor(max_workers=6) as executor:
|
||||
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
|
||||
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
|
||||
|
||||
|
||||
def test_chat_rollback_after_insert(client):
|
||||
u, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
title = 'rollback_' + uuid.uuid4().hex
|
||||
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
|
||||
assert r.status_code == 400
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
|
||||
|
||||
|
||||
def test_cursor_access_and_strict_validation(client):
|
||||
a, ab = user(client)
|
||||
b, bb = user(client)
|
||||
ah, _ = login(client, ab)
|
||||
bh, _ = login(client, bb)
|
||||
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
|
||||
path = '/api/v1/chats/' + chats[0]['id']
|
||||
for text in ['Один 👋', 'Два', 'Три']:
|
||||
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
|
||||
p = client.get(path+'/messages?limit=2', headers=ah).json()
|
||||
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
|
||||
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
|
||||
assert [m['text'] for m in tail['items']] == ['Три']
|
||||
assert tail['next_cursor'] is None
|
||||
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
|
||||
assert r.status_code == 400
|
||||
assert client.get(path, headers=bh).status_code == 404
|
||||
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
|
||||
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
|
||||
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
|
||||
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
|
||||
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
|
||||
assert client.post('/api/v1/users',json=payload).status_code == 400
|
||||
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
|
||||
|
||||
|
||||
def test_session_storage_and_migrations(client):
|
||||
from messenger.migrate import main as migrate
|
||||
u, body = user(client)
|
||||
_, session = login(client, body)
|
||||
db = client.app.state.db
|
||||
def snapshot():
|
||||
with db.pool.connection() as conn:
|
||||
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
|
||||
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
|
||||
before = snapshot()
|
||||
migrate()
|
||||
migrate()
|
||||
assert snapshot() == before
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
|
||||
assert row['token_hash'] != session['token']
|
||||
assert len(row['token_hash']) == 64
|
||||
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
|
||||
assert stored.startswith('$argon2id$')
|
||||
assert stored != body['password']
|
||||
|
||||
|
||||
def test_json_media_type_and_utf8(client):
|
||||
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
|
||||
raw = json.dumps(body).encode()
|
||||
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
|
||||
r = client.post('/api/v1/users',content=raw,headers=headers)
|
||||
assert r.status_code == 415
|
||||
assert r.json()['error']['code'] == 'unsupported_media_type'
|
||||
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
|
||||
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
|
||||
assert r.status_code == 400, encoding
|
||||
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
|
||||
assert r.status_code == 401
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Real PostgreSQL + private S3 integration; synthetic images only."""
|
||||
import hashlib
|
||||
import io
|
||||
import struct
|
||||
import uuid
|
||||
import zlib
|
||||
from PIL import Image
|
||||
from psycopg import sql
|
||||
from test_api import client, user, login
|
||||
|
||||
|
||||
def image_bytes(kind='PNG'):
|
||||
stream = io.BytesIO()
|
||||
Image.new('RGB',(320,200),(45,110,190)).save(stream,format=kind)
|
||||
return stream.getvalue()
|
||||
|
||||
|
||||
def actors(c):
|
||||
a,ab=user(c)
|
||||
b,bb=user(c)
|
||||
e,eb=user(c)
|
||||
ah,_=login(c,ab)
|
||||
bh,_=login(c,bb)
|
||||
eh,_=login(c,eb)
|
||||
chat=c.post('/api/v1/chats',headers=ah,json={'title':'Файлы','member_ids':[a['id'],b['id']]}).json()
|
||||
return ah,bh,eh,chat
|
||||
|
||||
|
||||
def upload(c,headers,chat,raw=None,mime='image/png',filename='Фото.png'):
|
||||
raw=image_bytes() if raw is None else raw
|
||||
return c.post('/api/v1/chats/'+chat['id']+'/attachments',params={'filename':filename},content=raw,
|
||||
headers={**headers,'Content-Type':mime})
|
||||
|
||||
|
||||
def finish(client,attachment_id):
|
||||
from messenger.jobs import work
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
job=conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(attachment_id,)).fetchone()['id']
|
||||
work(client.app.state.db,client.app.state.storage,job)
|
||||
import time
|
||||
for _ in range(100):
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
status=conn.execute('SELECT status FROM attachments WHERE id=%s',(attachment_id,)).fetchone()['status']
|
||||
if status=='ready': return
|
||||
time.sleep(.1)
|
||||
raise AssertionError('Job did not become ready: '+status)
|
||||
|
||||
|
||||
def test_original_png_jpeg_and_message(client):
|
||||
ah,bh,eh,chat=actors(client)
|
||||
for kind,mime in [('PNG','image/png'),('JPEG','image/jpeg')]:
|
||||
raw=image_bytes(kind)
|
||||
r=upload(client,ah,chat,raw,mime)
|
||||
assert r.status_code==202
|
||||
att=r.json()
|
||||
assert att['status']=='queued' and att['variants']==[] and att['error_code'] is None
|
||||
assert client.get('/api/v1/attachments/'+att['id']+'/content',headers=bh).status_code==409
|
||||
finish(client,att['id'])
|
||||
att=client.get('/api/v1/attachments/'+att['id'],headers=bh).json()
|
||||
assert att['status']=='ready' and len(att['variants'])==3
|
||||
assert att['sha256']==hashlib.sha256(raw).hexdigest() and att['size_bytes']==len(raw)
|
||||
assert 'object_key' not in att and 'bucket' not in att
|
||||
path='/api/v1/attachments/'+att['id']
|
||||
assert client.get(path,headers=bh).json()==att
|
||||
content=client.get(path+'/content',headers=bh)
|
||||
assert content.status_code==200 and content.content==raw
|
||||
assert content.headers['Content-Type']==mime
|
||||
for suffix in ['', '/content']:
|
||||
assert client.get(path+suffix,headers=eh).status_code==404
|
||||
assert client.get(path+'/content?variant=thumbnail',headers=bh).status_code==200
|
||||
response=client.post('/api/v1/chats/'+chat['id']+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},
|
||||
json={'text':'','attachment_ids':[att['id']]})
|
||||
assert response.status_code==201
|
||||
assert response.json()['attachment_ids']==[att['id']]
|
||||
|
||||
|
||||
def test_invalid_media_corruption_limits_and_filename(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
cases=[(b'not an image','image/png',415),(image_bytes(),'image/jpeg',415),
|
||||
(image_bytes('JPEG'),'image/png',415),(image_bytes()[:40],'image/png',415),
|
||||
(image_bytes(),'text/plain',415),(b'x'*(10485760+1),'image/png',413)]
|
||||
for raw,mime,expected in cases:
|
||||
response=upload(client,ah,chat,raw,mime)
|
||||
assert response.status_code==expected
|
||||
assert response.json()['error']['request_id']==response.headers['X-Request-Id']
|
||||
# Valid PNG header advertises >40Mp; reject before attempting decompression.
|
||||
header=struct.pack('!IIBBBBB',10001,4000,8,2,0,0,0)
|
||||
chunk=lambda tag,data: struct.pack('!I',len(data))+tag+data+struct.pack('!I',zlib.crc32(tag+data)&0xffffffff)
|
||||
bomb=b'\x89PNG\r\n\x1a\n'+chunk(b'IHDR',header)+chunk(b'IDAT',zlib.compress(b''))+chunk(b'IEND',b'')
|
||||
assert upload(client,ah,chat,bomb).status_code==413
|
||||
for filename in ['../x.png','a\\b.png','bad\x00.png','bad\x1f.png','bad\x7f.png',' ','x'*256]:
|
||||
assert upload(client,ah,chat,filename=filename).status_code==400
|
||||
|
||||
|
||||
def test_foreign_attachment_binding_and_normalized_replay(client):
|
||||
ah,bh,eh,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
other=client.post('/api/v1/chats',headers=bh,json={'title':'Другой чат','member_ids':[chat['member_ids'][0],chat['member_ids'][1]]}).json()
|
||||
path='/api/v1/chats/'+other['id']+'/messages'
|
||||
for aid in [att['id'],str(uuid.uuid4())]:
|
||||
response=client.post(path,headers={**bh,'Idempotency-Key':uuid.uuid4().hex},json={'text':'test','attachment_ids':[aid]})
|
||||
assert response.status_code==404
|
||||
assert upload(client,eh,chat).status_code==404
|
||||
h={**ah,'Idempotency-Key':uuid.uuid4().hex}
|
||||
path='/api/v1/chats/'+chat['id']+'/messages'
|
||||
first=client.post(path,headers=h,json={'text':'same'})
|
||||
replay=client.post(path,headers=h,json={'attachment_ids':[],'text':'same'})
|
||||
assert first.status_code==replay.status_code==201 and first.json()==replay.json()
|
||||
conflict=client.post(path,headers=h,json={'text':'same','attachment_ids':[att['id']]})
|
||||
assert conflict.status_code==409
|
||||
duplicates=client.post(path,headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json={'text':'','attachment_ids':[att['id'],att['id']]})
|
||||
assert duplicates.status_code==400
|
||||
|
||||
|
||||
def test_compensate_s3_when_metadata_insert_fails(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
db=client.app.state.db
|
||||
store=client.app.state.storage
|
||||
before={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
|
||||
function='fail_attachment_'+uuid.uuid4().hex
|
||||
trigger='fail_attachment_'+uuid.uuid4().hex
|
||||
filename='fault_'+uuid.uuid4().hex+'.png'
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.filename={} THEN RAISE EXCEPTION 'synthetic metadata failure'; END IF; RETURN NEW; END $$").format(sql.Identifier(function),sql.Literal(filename)))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON attachments FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
|
||||
try:
|
||||
response=upload(client,ah,chat,filename=filename)
|
||||
assert response.status_code==503
|
||||
with db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM attachments WHERE filename=%s',(filename,)).fetchone()['n']==0
|
||||
after={item['Key'] for page in store.pages(prefix="staging/") for item in page.get('Contents',[])}
|
||||
assert after==before
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON attachments').format(sql.Identifier(trigger)))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
|
||||
|
||||
|
||||
def test_compensation_preserves_committed_object(client):
|
||||
from messenger.attachments import compensate
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
|
||||
# A retry after an unknown COMMIT result must never delete an already tracked object.
|
||||
compensate(client.app.state.db,client.app.state.storage,key)
|
||||
content=client.get('/api/v1/attachments/'+att['id']+'/content',headers=ah)
|
||||
assert content.status_code==200 and content.content==image_bytes()
|
||||
|
||||
|
||||
def test_stream_close_even_if_headers_cannot_be_sent(client):
|
||||
import asyncio
|
||||
import pytest
|
||||
from messenger.attachments import ObjectResponse
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
key=conn.execute('SELECT object_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['object_key']
|
||||
store=client.app.state.storage
|
||||
opened=store.get(key)
|
||||
response=ObjectResponse(opened,media_type='image/png')
|
||||
async def fail_send(message):
|
||||
raise OSError('synthetic disconnected transport')
|
||||
async def receive():
|
||||
return {'type':'http.disconnect'}
|
||||
with pytest.raises(Exception):
|
||||
asyncio.run(response({'type':'http','asgi':{'spec_version':'2.4'}},receive,fail_send))
|
||||
assert opened.closed
|
||||
acquired=0
|
||||
try:
|
||||
for _ in range(8):
|
||||
assert store.gate.acquire(blocking=False)
|
||||
acquired+=1
|
||||
finally:
|
||||
for _ in range(acquired):
|
||||
store.gate.release()
|
||||
|
||||
|
||||
def test_chunked_upload_is_limited_without_content_length(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
chunks=iter([b'x'*1048576]*10+[b'x'])
|
||||
response=client.post('/api/v1/chats/'+chat['id']+'/attachments?filename=chunked.png',
|
||||
content=chunks,headers={**ah,'Content-Type':'image/png'})
|
||||
assert response.status_code==413
|
||||
assert response.json()['error']['code']=='payload_too_large'
|
||||
@@ -0,0 +1,119 @@
|
||||
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
from psycopg import sql
|
||||
from test_api import client, user, login
|
||||
|
||||
|
||||
def setup_chat(c, extra=None):
|
||||
u, body = user(c)
|
||||
headers, _ = login(c, body)
|
||||
members = [u['id']] + (extra or [])
|
||||
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
|
||||
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
|
||||
|
||||
|
||||
def test_instance_alias_on_success_and_errors(client):
|
||||
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
|
||||
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
|
||||
ids = [r.headers.get('X-Instance-Id') for r in responses]
|
||||
assert ids[0] and len(set(ids)) == 1
|
||||
assert all(r.headers.get('X-Request-Id') for r in responses)
|
||||
|
||||
|
||||
def test_key_required_replay_and_conflict(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
|
||||
for key in ['', 'a b', 'bad/', 'x'*129]:
|
||||
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
|
||||
h = {**headers, 'Idempotency-Key':'valid:._-123'}
|
||||
first = client.post(path, headers=h, json={'text':'same'})
|
||||
assert first.status_code == 201
|
||||
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
|
||||
assert replay.status_code == 201 and replay.json() == first.json()
|
||||
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
|
||||
assert len(client.get(path,headers=headers).json()['items']) == 1
|
||||
|
||||
|
||||
def test_concurrent_key_produces_one_message(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
|
||||
assert [r.status_code for r in results] == [201]*16
|
||||
assert all(r.json() == results[0].json() for r in results)
|
||||
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
|
||||
|
||||
|
||||
def test_scope_and_replay_rechecks_authorization(client):
|
||||
b, bb = user(client)
|
||||
bh, _ = login(client, bb)
|
||||
a, ah, path = setup_chat(client, [b['id']])
|
||||
key = 'same-in-independent-scopes'
|
||||
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
|
||||
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
|
||||
other_path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
|
||||
assert third.status_code == 201 and third.json()['id'] != first['id']
|
||||
outsider, ob = user(client)
|
||||
oh, _ = login(client, ob)
|
||||
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
|
||||
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
|
||||
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
|
||||
|
||||
|
||||
def test_expired_key_can_be_reused(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
|
||||
assert 86390 <= ttl <= 86400
|
||||
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
|
||||
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
assert len(client.get(path,headers=h).json()['items']) == 2
|
||||
|
||||
|
||||
def test_rollback_side_effect_and_key_on_database_error(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'fault_' + uuid.uuid4().hex
|
||||
function = 'fault_' + uuid.uuid4().hex
|
||||
trigger = 'trigger_' + uuid.uuid4().hex
|
||||
db = client.app.state.db
|
||||
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
|
||||
try:
|
||||
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert failed.status_code == 500
|
||||
assert failed.headers['X-Instance-Id']
|
||||
with db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
|
||||
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert retried.status_code == 201
|
||||
assert client.get(path,headers=h).json()['items'] == [retried.json()]
|
||||
|
||||
|
||||
def test_ready_requires_compatible_schema(client):
|
||||
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
|
||||
try:
|
||||
r = client.get('/health/ready')
|
||||
assert r.status_code == 503
|
||||
assert r.json()['error']['code'] == 'unavailable'
|
||||
assert client.get('/health/live').status_code == 200
|
||||
finally:
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))
|
||||
@@ -0,0 +1,63 @@
|
||||
import io
|
||||
import pytest
|
||||
from messenger.jobs import work, replay
|
||||
from test_attachments import actors,upload,image_bytes,finish
|
||||
from test_api import client
|
||||
|
||||
|
||||
def job_for(client,aid):
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
return conn.execute('SELECT id FROM attachment_jobs WHERE attachment_id=%s',(aid,)).fetchone()['id']
|
||||
|
||||
|
||||
def test_duplicate_ready_is_noop_and_not_replayable(client):
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
path='/api/v1/attachments/'+att['id']
|
||||
before=client.get(path,headers=ah).json()
|
||||
job=job_for(client,att['id'])
|
||||
for _ in range(3): assert work(client.app.state.db,client.app.state.storage,job,1)
|
||||
assert client.get(path,headers=ah).json()==before
|
||||
with pytest.raises(ValueError): replay(client.app.state.db,job,'must reject ready')
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
|
||||
assert row['attempt']==1
|
||||
assert conn.execute('SELECT staging_key FROM attachments WHERE id=%s',(att['id'],)).fetchone()['staging_key'] is None
|
||||
|
||||
|
||||
def test_old_delivery_does_not_consume_attempt(client):
|
||||
# Direct invocation avoids consumer timing; it exercises the same real DB/S3 code.
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
job=job_for(client,att['id'])
|
||||
assert work(client.app.state.db,client.app.state.storage,job,999)
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
row=conn.execute('SELECT attempt FROM attachment_jobs WHERE id=%s',(job,)).fetchone()
|
||||
assert row['attempt'] in (0,1) # independent live consumer may already finish this job
|
||||
finish(client,att['id'])
|
||||
|
||||
|
||||
def test_replay_sql_failure_releases_session_lock(client):
|
||||
import psycopg
|
||||
from messenger.db import setting
|
||||
from psycopg.rows import dict_row
|
||||
ah,_,_,chat=actors(client)
|
||||
att=upload(client,ah,chat).json()
|
||||
finish(client,att['id'])
|
||||
job=job_for(client,att['id'])
|
||||
db=client.app.state.db
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute("UPDATE attachments SET status='failed',staging_key=%s WHERE id=%s",('staging/'+att['id'],att['id']))
|
||||
conn.execute("CREATE FUNCTION lab5_fail_audit() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'synthetic audit failure'; END $$")
|
||||
conn.execute('CREATE TRIGGER lab5_fail_audit BEFORE INSERT ON job_replays FOR EACH ROW EXECUTE FUNCTION lab5_fail_audit()')
|
||||
try:
|
||||
with pytest.raises(psycopg.Error): replay(db,job,'test failed transaction unlock')
|
||||
with psycopg.connect(setting('DATABASE_URL'),row_factory=dict_row) as other:
|
||||
acquired=other.execute('SELECT pg_try_advisory_lock(hashtextextended(%s,0)) AS ok',('job:'+str(job),)).fetchone()['ok']
|
||||
if acquired: other.execute('SELECT pg_advisory_unlock(hashtextextended(%s,0))',('job:'+str(job),))
|
||||
assert acquired, 'Aborted replay leaked a session advisory lock into the pool'
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute('DROP TRIGGER lab5_fail_audit ON job_replays')
|
||||
conn.execute('DROP FUNCTION lab5_fail_audit()')
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Real image transformations, no mock storage."""
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from PIL import Image
|
||||
|
||||
|
||||
def test_worker_variants_and_original(tmp_path):
|
||||
source=tmp_path/'input'
|
||||
original=Image.new('RGBA',(320,200),(30,90,150,70))
|
||||
original.save(source,format='PNG')
|
||||
before=hashlib.sha256(source.read_bytes()).hexdigest()
|
||||
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
|
||||
assert result.returncode==0, result.stderr
|
||||
manifest=json.loads(result.stdout)
|
||||
assert hashlib.sha256(source.read_bytes()).hexdigest()==before
|
||||
for kind,fmt,size in [('thumbnail','JPEG',(256,160)),('crop','JPEG',(128,128)),('optimized','WEBP',(320,200))]:
|
||||
item=manifest[kind]
|
||||
with Image.open(tmp_path/kind) as image:
|
||||
assert image.format==fmt and image.size==size
|
||||
assert not image.getexif()
|
||||
raw=(tmp_path/kind).read_bytes()
|
||||
assert item['sha256']==hashlib.sha256(raw).hexdigest()
|
||||
assert item['size_bytes']==len(raw)
|
||||
|
||||
|
||||
def test_no_upscale_thumbnail_and_exif(tmp_path):
|
||||
source=tmp_path/'input'
|
||||
image=Image.new('RGB',(80,40),'red')
|
||||
exif=Image.Exif();exif[274]=6
|
||||
image.save(source,format='JPEG',exif=exif)
|
||||
result=subprocess.run([sys.executable,'-m','messenger.pipeline',str(source),str(tmp_path)],capture_output=True)
|
||||
assert result.returncode==0, result.stderr
|
||||
with Image.open(tmp_path/'thumbnail') as output:
|
||||
assert output.size==(40,80)
|
||||
assert not output.getexif()
|
||||
|
||||
|
||||
def test_deadline_covers_slow_s3(monkeypatch):
|
||||
import contextlib
|
||||
import http.server
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import pytest
|
||||
from messenger.jobs import process_objects
|
||||
class Slow(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Length','100')
|
||||
self.end_headers()
|
||||
try:
|
||||
for _ in range(100):
|
||||
self.wfile.write(b'x');self.wfile.flush();time.sleep(.05)
|
||||
except (BrokenPipeError,ConnectionResetError): pass
|
||||
def log_message(self,*args): pass
|
||||
server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Slow)
|
||||
thread=threading.Thread(target=server.serve_forever,daemon=True);thread.start()
|
||||
monkeypatch.delenv('S3_ENDPOINT_FILE',raising=False)
|
||||
monkeypatch.setenv('S3_ENDPOINT','http://127.0.0.1:'+str(server.server_port))
|
||||
monkeypatch.setenv('PROCESS_TIMEOUT_SECONDS','2')
|
||||
row={'id':uuid.uuid4(),'staging_key':'staging/slow','object_key':'originals/slow',
|
||||
'size_bytes':100,'sha256':hashlib.sha256(b'x'*100).hexdigest(),'content_type':'image/png'}
|
||||
started=time.monotonic()
|
||||
try:
|
||||
with pytest.raises(subprocess.TimeoutExpired): process_objects(__import__("messenger.object_store",fromlist=["ObjectStore"]).ObjectStore(),row)
|
||||
assert time.monotonic()-started<4
|
||||
finally:
|
||||
server.shutdown();server.server_close()
|
||||
Reference in New Issue
Block a user