forked from mtusi2026/mtusi2026lab5
Compare commits
11
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
77a50af326 | ||
|
|
cf6e92957c | ||
|
|
e453ee2312 | ||
|
|
a3fa7713fa | ||
|
|
962adbe03b | ||
|
|
1d37fd0dab | ||
|
|
aef6eb1d4c | ||
|
|
bc32a50967 | ||
|
|
655f264104 | ||
|
|
a633949919 | ||
|
|
5112be83cc |
+75
-2
@@ -5,21 +5,94 @@ APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
# The files are created by `python scripts/init_secrets.py` and mounted by compose as secrets.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
# valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`.
|
||||
SESSION_STORE=valkey
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Number of interchangeable API replicas behind HAProxy (compose deploy.replicas).
|
||||
API_REPLICAS=2
|
||||
# Idempotency-Key results are kept at least this long (contract: >= 24 h).
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
# Write path of POST message: function = one SQL call (migration 0003), multi = lab3 baseline path.
|
||||
MESSAGE_WRITE_PATH=function
|
||||
# Lab 4: private S3 bucket (SeaweedFS service `s3`). The API gets ITS OWN key pair (Read+Write on
|
||||
# this bucket only) as Docker secrets; admin/reconciler pairs go only to s3-init/reconciler.
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
# Lab 5: private staging bucket for originals accepted before 202 (API: Write; worker: Read+Write).
|
||||
S3_STAGING_BUCKET=messenger-staging
|
||||
# Contract limits (may be lowered, never raised above 10485760 / 40000000).
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# Per replica: bodies buffered at once (RAM bound), wait for a slot, body deadline, image checks at once.
|
||||
UPLOAD_CONCURRENCY=4
|
||||
UPLOAD_QUEUE_TIMEOUT_SECONDS=5
|
||||
UPLOAD_READ_TIMEOUT_SECONDS=30
|
||||
IMAGE_VERIFY_TIMEOUT_SECONDS=10
|
||||
IMAGE_VERIFY_CONCURRENCY=1
|
||||
JPEG_MAX_MEMORY_MB=256
|
||||
S3_TIMEOUT_SECONDS=3
|
||||
# Janitor: 'pending' intents older than this are failed uploads (object + row removed).
|
||||
PENDING_GRACE_SECONDS=900
|
||||
JANITOR_INTERVAL_SECONDS=60
|
||||
# Lab 5: broker credentials are files too (relay and worker get their OWN RabbitMQ user/URL;
|
||||
# secrets/broker_url_relay|worker are mounted as /run/secrets/broker_url). The API has none.
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
# Jobs processed at once by one worker = RabbitMQ prefetch; image child processes at once.
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_REPLICAS=1
|
||||
# Managed attempts (claims) per job, then failed/retry_exhausted; backoff = full jitter.
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_RETRY_BASE_SECONDS=2
|
||||
JOB_RETRY_MAX_SECONDS=60
|
||||
# One attempt: hard time limit (child process killed) and the lease renewed by its heartbeat.
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=30
|
||||
# Address-space limit of the image child process (RLIMIT_AS).
|
||||
IMAGE_MAX_MEMORY_MB=768
|
||||
# Staging copy of a FAILED attachment is kept this long for a replay (ready: deleted at once).
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
# Objects of attempts that never published are deleted this long after the attempt ended.
|
||||
ORPHAN_GRACE_SECONDS=600
|
||||
# Queued job without a claim for this long gets a new outbox row (lost-message guard).
|
||||
QUEUED_REPUBLISH_SECONDS=300
|
||||
# Pipeline version of new uploads (track 2: versions 1, 2, 3 in src/messenger/pipeline.py).
|
||||
PIPELINE_VERSION=1
|
||||
# Failure experiments only, per service (see src/messenger/config.py FAULT_POINTS); keep empty.
|
||||
API_FAULT_POINTS=
|
||||
RELAY_FAULT_POINTS=
|
||||
WORKER_FAULT_POINTS=
|
||||
# Additional settings of this implementation (defaults shown; all optional):
|
||||
LOG_LEVEL=info
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_TIMEOUT_SECONDS=3
|
||||
CACHE_TIMEOUT_SECONDS=0.5
|
||||
PASSWORD_HASH_CONCURRENCY=2
|
||||
DOCS_ENABLED=true
|
||||
# Graceful stop of a replica: readiness 503 + drain, then up to SHUTDOWN for in-flight requests.
|
||||
DRAIN_SECONDS=3
|
||||
SHUTDOWN_TIMEOUT_SECONDS=8
|
||||
# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica.
|
||||
API_CPUS=1.0
|
||||
API_MEMORY=512m
|
||||
S3_CPUS=1.0
|
||||
S3_MEMORY=1g
|
||||
PG_CPUS=2.0
|
||||
PG_MEMORY=1g
|
||||
VALKEY_CPUS=0.5
|
||||
LB_CPUS=1.0
|
||||
BROKER_CPUS=1.0
|
||||
BROKER_MEMORY=512m
|
||||
WORKER_CPUS=2.0
|
||||
WORKER_MEMORY=1536m
|
||||
RELAY_CPUS=0.5
|
||||
PG_MAX_CONNECTIONS=100
|
||||
# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing)
|
||||
APP_PUBLISH_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
# LF everywhere: identical files on Windows/Linux and inside `git archive | docker build -`.
|
||||
* text=auto eol=lf
|
||||
*.png binary
|
||||
# cmd.exe expects CRLF in batch files.
|
||||
*.cmd text eol=crlf
|
||||
@@ -0,0 +1,40 @@
|
||||
name: Application (lab5)
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
app:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 40
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- name: Synthetic secrets and env (CI only, never committed)
|
||||
run: python scripts/init_secrets.py && cp .env.example .env
|
||||
- name: Build, migrate once, bootstrap both S3 buckets, start broker, 2 API replicas, LB, relay, worker
|
||||
run: docker compose --env-file .env up -d --build --wait
|
||||
- name: Contract and public smoke
|
||||
run: make check test PYTHON=python
|
||||
- name: Persistence across restart of LB, API replicas, relay, worker, PostgreSQL, Valkey, S3, RabbitMQ
|
||||
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python
|
||||
- name: Failover of the replica that served the test, then restore it
|
||||
run: |
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh PYTHON=python
|
||||
sh scripts/restore-api.sh && python scripts/lbctl.py wait-up 2
|
||||
- name: Defense scenario - worker stopped, 202, 409, restart API and broker, ready
|
||||
run: python -m pip install -q pillow==12.3.0 && python scripts/demo_pipeline.py
|
||||
- name: Own tests (unit + stand; every failure is restored in finally)
|
||||
run: |
|
||||
python -m pip install --require-hashes --no-deps -r requirements.txt
|
||||
python -m pip install -r requirements-test.txt
|
||||
make student-test PYTHON=python
|
||||
- name: Track 2 invariants (read-only audit of PostgreSQL and both buckets)
|
||||
run: docker compose run --rm --no-deps auditor --verify-checksums
|
||||
- name: Logs
|
||||
if: always()
|
||||
run: docker compose logs --tail 100 api lb s3 s3-init broker relay worker
|
||||
- name: Remove CI stand and its synthetic data
|
||||
if: always()
|
||||
run: docker compose down -v
|
||||
+67
@@ -0,0 +1,67 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# Messenger, lab 5 (API replicas behind HAProxy, PostgreSQL + Valkey + S3 + RabbitMQ). Built by
|
||||
# compose.yaml (services api, migrate, s3-init, relay, worker, auditor, jobctl: one image, different
|
||||
# commands; only the worker imports Pillow).
|
||||
#
|
||||
# Python is interpreted, so there is no separate compile stage; the image
|
||||
# contains only the interpreter, locked dependencies and the source code.
|
||||
# Base image is pinned by version AND digest (python 3.13.16, Debian 12 bookworm).
|
||||
FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 1. Dependencies first (layer cache survives source edits). Every package,
|
||||
# including transitive ones, is pinned with sha256 hashes in requirements.txt;
|
||||
# --no-deps guarantees nothing outside the lock file is installed.
|
||||
COPY requirements.txt ./
|
||||
RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt
|
||||
|
||||
# 2. JPEG integrity check: libjpeg-turbo's djpeg (strict mode, see src/messenger/images.py).
|
||||
# Version pinned to the Debian 12 package; apt lists are removed in the same layer.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 3. Unprivileged runtime user. Code stays owned by root, so the process
|
||||
# cannot modify its own code.
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
|
||||
|
||||
COPY src/ ./src/
|
||||
# Versioned SQL migrations, applied by `python -m messenger.migrate` (compose service migrate).
|
||||
COPY migrations/ ./migrations/
|
||||
# Course contract, served read-only at /openapi.json for Swagger UI (/docs).
|
||||
COPY contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
# Non-secret defaults; compose.yaml passes the rest. Passwords come only as files
|
||||
# (DATABASE_URL_FILE / CACHE_URL_FILE -> Docker secrets), never as ENV or COPY.
|
||||
ENV PYTHONPATH=/app/src \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
LOG_LEVEL=info \
|
||||
SHUTDOWN_TIMEOUT_SECONDS=8 \
|
||||
DOCS_ENABLED=true \
|
||||
SESSION_TTL_SECONDS=3600 \
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
|
||||
# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out
|
||||
# image distinguishable in `docker inspect` and in the "starting" log line of each replica.
|
||||
ARG BUILD_REVISION=dev
|
||||
LABEL org.opencontainers.image.revision=$BUILD_REVISION
|
||||
ENV APP_REVISION=$BUILD_REVISION
|
||||
|
||||
USER app:app
|
||||
EXPOSE 8080
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["python", "-m", "messenger.healthcheck"]
|
||||
|
||||
# Exec form: the Python server is PID 1 and receives SIGTERM directly (no shell in between).
|
||||
CMD ["python", "-m", "messenger"]
|
||||
@@ -28,3 +28,81 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
# ---------------------------------------------------------------------------
|
||||
# Student solution (lab 5, Python). On Windows: PowerShell/Git Bash with GNU make, public
|
||||
# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash).
|
||||
.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series bench-memory worker-stop worker-start jobs failed audit track2
|
||||
|
||||
secrets:
|
||||
$(PYTHON) scripts/init_secrets.py
|
||||
|
||||
# Build, apply migrations ONCE (service migrate), start broker, API replicas, LB, relay and worker.
|
||||
up: secrets
|
||||
docker compose up -d --build --wait
|
||||
|
||||
# Safe stop: containers stop, named volumes (data, queue) stay.
|
||||
down:
|
||||
docker compose down
|
||||
|
||||
logs:
|
||||
docker compose logs --tail 50 api lb relay worker
|
||||
|
||||
ps:
|
||||
docker compose ps -a
|
||||
|
||||
migrate:
|
||||
docker compose run --rm migrate
|
||||
|
||||
# HAProxy view of the replicas (UP/DOWN) mapped to containers and X-Instance-Id.
|
||||
lb-status:
|
||||
$(PYTHON) scripts/lbctl.py status
|
||||
$(PYTHON) scripts/lbctl.py replicas
|
||||
|
||||
# Start replicas stopped by scripts/stop-one.sh again and wait until the LB has them all UP.
|
||||
restore:
|
||||
sh scripts/restore-api.sh
|
||||
$(PYTHON) scripts/lbctl.py wait-up $${API_REPLICAS:-2}
|
||||
|
||||
# Replace every API replica one by one with a freshly built image (no downtime).
|
||||
rolling-update:
|
||||
sh scripts/rolling-update.sh
|
||||
|
||||
# Lab 3: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/).
|
||||
failover-series:
|
||||
$(PYTHON) bench/failover_series.py --output evidence/failover
|
||||
|
||||
# Lab 5 demo: stop the worker (uploads still get 202, content answers 409), start it again.
|
||||
worker-stop:
|
||||
docker compose stop worker
|
||||
|
||||
worker-start:
|
||||
docker compose start worker
|
||||
|
||||
# Lab 5 operator view: jobs by state + outbox backlog; failed registry (replay: see REPORT.md).
|
||||
jobs:
|
||||
docker compose run --rm --no-deps jobctl stats
|
||||
|
||||
failed:
|
||||
docker compose run --rm --no-deps jobctl list-failed
|
||||
|
||||
# Track 2: read-only check of the pipeline invariants over PostgreSQL and both buckets.
|
||||
audit:
|
||||
docker compose run --rm --no-deps auditor
|
||||
|
||||
# Track 2: automatic fault-injection matrix (duplicate, reorder, crashes, stale versions).
|
||||
track2:
|
||||
$(PYTHON) bench/pipeline_faults.py --output evidence/track2
|
||||
|
||||
# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv).
|
||||
student-test:
|
||||
$(PYTHON) -m pytest student-tests -v
|
||||
|
||||
unit-test:
|
||||
$(PYTHON) -m pytest student-tests -m "not stack" -q
|
||||
|
||||
explain:
|
||||
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
|
||||
|
||||
# Lab 4: peak RAM of the replicas under 16 simultaneous uploads (evidence/memory/).
|
||||
bench-memory:
|
||||
$(PYTHON) bench/upload_memory.py --output evidence/memory
|
||||
|
||||
@@ -1,37 +1,191 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Боданюк Алексей, БВТ2302
|
||||
- Целевая оценка; трек 5: **5**, трек 2 «Доказанная идемпотентность pipeline»
|
||||
- Commit/PR: ветка `solution/bvt2302-bodanyuk`; PR «ЛР 5 — БВТ2302 — Боданюк — на 5» (ссылка после публикации)
|
||||
- Источники, библиотеки, ИИ: Python 3.13, Starlette, uvicorn, psycopg 3, redis-py, argon2-cffi, aiobotocore (из ЛР 2–4); новое: **aio-pika 9.6.2** (AMQP), **Pillow 12.3.0** (варианты, только в worker). Документация: [RabbitMQ reliability](https://www.rabbitmq.com/docs/reliability), [confirms](https://www.rabbitmq.com/docs/confirms), [quorum queues](https://www.rabbitmq.com/docs/quorum-queues), AWS «Exponential Backoff And Jitter». Код, тесты и документы написаны с помощью ИИ-ассистента (Claude); архитектурные решения разобраны и объяснимы автором (см. [docs/DEFENSE.md](docs/DEFENSE.md)).
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Версии (всё по тегу **и** digest, см. [compose.yaml](compose.yaml)): Python 3.13.16-slim-bookworm, PostgreSQL 18.6, Valkey 8.1.3, SeaweedFS 4.46, **RabbitMQ 4.3.7-alpine**, HAProxy 3.2.26. Python-пакеты — `requirements.txt` с sha256-хешами (`pip install --require-hashes --no-deps`).
|
||||
|
||||
Prerequisites: Docker с Compose v2, Python 3.12+, Git; на Windows — [docs/WINDOWS.md](docs/WINDOWS.md).
|
||||
|
||||
```sh
|
||||
git clone <fork> mtusi2026lab5 && cd mtusi2026lab5 && git checkout solution/bvt2302-bodanyuk
|
||||
python3 scripts/init_secrets.py # ./secrets: пароли БД/Valkey, 4 пары S3, пользователи RabbitMQ (хеши), URL брокера
|
||||
cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait # migrate и s3-init — один раз, exit 0; остальные healthy
|
||||
make check test # Windows: $env:PYTHONUTF8="1"; make ... PYTHON=python
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh && make restore
|
||||
python3 -m venv .venv && .venv/bin/pip install --require-hashes --no-deps -r requirements.txt && .venv/bin/pip install -r requirements-test.txt
|
||||
.venv/bin/python scripts/demo_pipeline.py # сценарий защиты, шаги 2-3
|
||||
.venv/bin/python -m pytest student-tests -v
|
||||
.venv/bin/python bench/pipeline_faults.py --output evidence/track2
|
||||
make audit # инварианты, read-only
|
||||
```
|
||||
|
||||
Всё это одной командой на чистом clone: `python scripts/collect_evidence.py` (вывод — в `evidence/`). Действий оператора нет. Безопасная остановка: `docker compose down` (volumes остаются). Намеренное удаление учебных данных: `docker compose down -v` и удалить `./secrets`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
client ─▶ lb (HAProxy) ─▶ api ×2 ──▶ postgres (метаданные, jobs, outbox, попытки, аудит) [data]
|
||||
├──▶ valkey (сессии) [data]
|
||||
└──▶ s3: messenger-staging (Write) / messenger-attachments (Read) [storage]
|
||||
relay ──▶ postgres (outbox, reaper) ──▶ broker (RabbitMQ, quorum queue + DLQ) [data, queue]
|
||||
worker ◀── broker; ──▶ postgres; ──▶ s3 (Read/Write оба bucket) [data, storage, queue]
|
||||
```
|
||||
|
||||
- **Источник истины — PostgreSQL**, включая работу: состояние, попытки, lease, время повтора, реестр failed. Брокер доставляет только `job_id`; ack — после COMMIT результата. Подробно — [docs/PIPELINE.md](docs/PIPELINE.md); схема — [docs/DATA_MODEL.md](docs/DATA_MODEL.md).
|
||||
- **Транзакционные границы.** Upload: `pending` → PUT staging → **одна транзакция** `queued + jobs + outbox` → 202. Результат: **одна транзакция** `attachment_variants ×3 + attachments ready` с проверкой `lease_token` → ack.
|
||||
- **API не связан с брокером** (нет сети `queue`, нет учётных данных). Недоступность worker/брокера не делает resize синхронным и не ломает upload.
|
||||
- **Перенесено из lab4 без изменений** (коммит `5112be8`): API, сессии, идемпотентность сообщений, HAProxy с активной readiness, лимиты загрузки, PNG-валидатор и `djpeg -strict`, janitor `pending`, graceful shutdown, тесты.
|
||||
- **Изменено:** upload 201 → 202; GET метаданных видит `queued/processing/failed`; `content` → 409 до ready, варианты `thumbnail/crop/optimized`; к сообщению можно прикрепить `queued/processing` (`post_message_v3`), `failed` → 409; S3 разделён на два bucket и четыре identity; restart-хук перезапускает и broker/relay/worker.
|
||||
- **Убрано:** reconciler lab4 (трек прошлой лабы, не обязательный минимум). В lab5 мусор находится по журналу `job_attempts.planned_keys` без права List; аудит `messenger.audit` проверяет оба bucket read-only-ключом. Удаление файлов reconciler попало в коммит `a633949`, а не в `bc32a50`, где об этом сказано, — ошибка при разбиении коммитов, историю не переписывал.
|
||||
- **Ограничения доступа.** S3: api — Write staging + Read final; worker — Read/Write оба; auditor — Read+List; admin — только `s3-init`. RabbitMQ: `relay` — только write в exchange `messenger.jobs`, `worker` — только read очереди, configure ни у кого; `guest` нет; management UI и Prometheus выключены. Проверяется `s3-init` при каждом старте и тестом `test_compose_policy_limits_ports_secrets_images`.
|
||||
|
||||
### Соответствие `.env.example`
|
||||
|
||||
| Переменная | Значение | Где |
|
||||
| --- | --- | --- |
|
||||
| `BROKER_URL_FILE` | `/run/secrets/broker_url` | relay и worker, у каждого свой пользователь RabbitMQ |
|
||||
| `S3_STAGING_BUCKET` | `messenger-staging` | api, worker, s3-init, auditor |
|
||||
| `WORKER_CONCURRENCY` | 2 | prefetch и число дочерних процессов worker |
|
||||
| `JOB_MAX_ATTEMPTS` | 5 | claim-ов на задание (пишется в задание при создании) |
|
||||
| `JOB_TIMEOUT_SECONDS` | 60 | потолок одной попытки (kill процесса, конец продления lease) |
|
||||
| `JOB_LEASE_SECONDS` | 30 | lease, heartbeat каждые 10 с |
|
||||
| `JOB_RETRY_BASE_SECONDS` / `_MAX_` | 2 / 60 | full jitter |
|
||||
| `STAGING_RETENTION_SECONDS` | 86400 | хранение staging у `failed` для replay (у `ready` удаляется сразу) |
|
||||
| `ORPHAN_GRACE_SECONDS` | 600 | удаление объектов неуспешных попыток |
|
||||
| `IMAGE_MAX_MEMORY_MB` | 768 | RLIMIT_AS дочернего процесса |
|
||||
| `PIPELINE_VERSION` | 1 | версия заданий новых загрузок (1, 2, 3 в `pipeline.py`) |
|
||||
| `API_/RELAY_/WORKER_FAULT_POINTS` | пусто | внедрение отказов только для экспериментов |
|
||||
| `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | 10485760, 40000000 | API (до чтения тела) и worker (до скачивания/декодирования) |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Все файлы — из прогона `scripts/collect_evidence.py` на **чистом clone** (новые secrets, пустые volumes), см. раздел «Финальный прогон».
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | `make check validate test` | PASS ×3: 202 → ready, JPEG 256×160 | [make-check-validate-test.txt](evidence/make-check-validate-test.txt) |
|
||||
| **3:** минимум lab4 сохранён, 201 → 202 | `pytest student-tests` (test_api, test_attachments, test_idempotency, test_balancing, test_sessions, test_failures) | 377 passed; 415 на все кейсы рецензий (PNG фильтр 5/0xff, IDAT не zlib, JPEG 27 байт, SOS не из SOF) | [student-tests.txt](evidence/student-tests.txt) |
|
||||
| **3:** 202 только после устойчивого приёма; остановка API не теряет | test_consistency (`after_s3_put`, `after_enqueue_commit`), demo | окно «после staging» — невидимый `pending`, убран janitor; после COMMIT — ready | student-tests.txt |
|
||||
| **3:** отдельные broker и worker, durable, ack после эффекта | compose, `rabbitmqctl list_queues`; `test_worker_crash_after_commit…` | quorum queue durable; redelivery → `noop_finished` | [clean-clone.txt](evidence/clean-clone.txt), student-tests.txt |
|
||||
| **3:** worker недоступен → API не делает resize | `test_worker_stopped…`, demo | 202 queued, content 409, сообщения 201 | [demo-pipeline.txt](evidence/demo-pipeline.txt) |
|
||||
| **3:** состояния queued/processing/ready/failed, thumbnail по контракту | test_pipeline_flow (`check_variants`) | реальные размеры через Pillow, sha256 = метаданные, оригинал побайтно | student-tests.txt |
|
||||
| **3:** ограниченный retry, failed + error_code; повтор не плодит варианты | `test_retry_exhausted…`, `test_duplicate…` | 5 попыток → `retry_exhausted`; дубли → 1 набор | student-tests.txt |
|
||||
| **3:** демонстрация stop worker → 202 → сообщение → restart API (+broker) → ready | `scripts/demo_pipeline.py` | очередь `messages=1` до и после рестарта брокера, ready, одна попытка | demo-pipeline.txt |
|
||||
| Persistence / failover (публичные) | `make test-persistence`, `make test-failover` | PASS, состояния контейнеров до/после | [test-persistence.txt](evidence/test-persistence.txt), [test-failover.txt](evidence/test-failover.txt) |
|
||||
| **4:** crop 128 и WebP 1280, параметры | unit test_pipeline + check_variants | геометрия, EXIF, alpha, без метаданных — [PIPELINE.md](docs/PIPELINE.md#варианты) | student-tests.txt |
|
||||
| **4:** лимиты bytes/пикселей/времени/памяти worker | `test_memory_and_time_limits_of_the_worker`, `test_exactly_40_megapixels…` | 40 Мпикс при 200 МБ → `image_too_large`; 3 с → `processing_timeout`; 40 Мпикс в 768 МБ → ready | student-tests.txt |
|
||||
| **4:** outbox; crash между COMMIT и publish | `test_broker_down…`, `test_relay_down…`, `test_crash_after_publish…` | строка outbox ждёт, публикуется без ручных действий | student-tests.txt |
|
||||
| **4:** backoff с jitter, постоянные/временные | `test_transient_errors…`, `test_permanent_error…` | задержки в `[0, base·2^(n−1)]`, invalid_image — 1 попытка | student-tests.txt |
|
||||
| **4:** DLQ/реестр failed, безопасный replay | `jobctl list-failed/replay`, `test_bad_message…` | replay → ready; replay готового → `already_ready`; плохое сообщение → DLQ | student-tests.txt |
|
||||
| **4:** crash после записи объекта до ack | `test_worker_crash_after_s3_write…`, `…after_commit…` | один опубликованный набор, мусор попытки 1 удалён | student-tests.txt |
|
||||
| **4:** lease зависшего processing, prefetch, очистка staging | `test_hung_worker…` (docker pause), `test_concurrency…`, `test_queued_staging…`, `test_cleanup_racing…` | попытки `[abandoned, succeeded]`; ≤2 processing; staging `queued` не удаляется, CHECK отвергает | student-tests.txt |
|
||||
| **5, трек 2** | `bench/pipeline_faults.py`, `make audit` | 12/12 сценариев, 0 нарушений, аудит ok | [evidence/track2/](evidence/track2/), [track2-runs.txt](evidence/track2-runs.txt) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Трек 2: доказанная идемпотентность versioned pipeline
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
**Гипотеза.** При at-least-once доставке, дублях и переупорядочивании сообщений, crash worker до/после побочного эффекта и устаревших заданиях старой версии pipeline: (H1) ни одна принятая 202 работа не теряется; (H2) каждый эффект `(attachment, version, variant)` опубликован ровно один раз; (H3) опубликованная версия только растёт, `ready` не откатывается; (H4) мусор прерванных попыток удаляется, опубликованное не удаляется никогда.
|
||||
|
||||
**Механизм.** Ключ эффекта — PK `attachment_variants`; ключи объектов включают токен попытки; fencing `lease_token` при каждом COMMIT; монотонный `published_version`; claim по состоянию задания в БД; уборка только по `planned_keys` неуспешных попыток под блокировкой.
|
||||
|
||||
**Методика** ([bench/pipeline_faults.py](bench/pipeline_faults.py)). Для каждого сценария пересоздаётся один сервис с `FAULT_POINTS` (реальные пути кода, реальный `os._exit`, реальный `docker pause`), N загрузок через LB, затем читаются факты из PostgreSQL и S3 (List auditor-ключом), а в конце аудит `I1–I7` с проверкой SHA-256 всех опубликованных объектов по всему стенду.
|
||||
|
||||
| Сценарий | Внедрение |
|
||||
| --- | --- |
|
||||
| baseline | — |
|
||||
| дубль | relay публикует каждое сообщение дважды |
|
||||
| дубль + reorder | дважды и в обратном порядке пачки |
|
||||
| crash relay после confirm до отметки | `os._exit` relay |
|
||||
| crash worker до побочного эффекта | `os._exit` после claim |
|
||||
| crash worker после S3 до COMMIT | `os._exit` после PUT объектов |
|
||||
| crash worker после COMMIT до ack | `os._exit` после COMMIT |
|
||||
| временные ошибки | 2 неудачные попытки |
|
||||
| 2 worker + дубль | конкурирующие потребители |
|
||||
| устаревшая версия | reprocess v2 (медленный) и v3 одновременно; затем вручную опубликованный дубль задания v1 |
|
||||
| reorder версий в очереди | v2 и v3 в outbox, relay публикует в обратном порядке, 1 worker |
|
||||
| зависший worker | 2 worker, `docker pause` владельца lease |
|
||||
|
||||
**Результаты** (чистый clone, 2026-10-31 14:48–14:57 UTC, 5 загрузок на «плоский» сценарий; [summary.txt](evidence/track2/summary.txt), подробно — [summary.json](evidence/track2/summary.json)):
|
||||
|
||||
| Сценарий | Загрузок | ready | claim-ов | Нарушений | Последовательность событий задания |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| baseline | 5 | 5 | 5 | 0 | created › claimed › succeeded |
|
||||
| дубль publish | 5 | 5 | 5 | 0 | то же: второе сообщение — `noop_finished` |
|
||||
| дубль + обратный порядок | 5 | 5 | 5 | 0 | то же |
|
||||
| crash relay после confirm | 5 | 5 | 5 | 0 | created › claimed › fault_injected › succeeded |
|
||||
| crash worker после claim | 5 | 5 | 14 | 0 | claimed › fault_injected › lease_expired › claimed › succeeded (*) |
|
||||
| crash worker после S3, до COMMIT | 5 | 5 | 10 | 0 | claimed › fault_injected › lease_expired › claimed › succeeded |
|
||||
| crash worker после COMMIT, до ack | 5 | 5 | 5 | 0 | claimed › succeeded › fault_injected (redelivery = no-op) |
|
||||
| 2 временные ошибки | 5 | 5 | 15 | 0 | 3 claim-а на загрузку, ровно как задано |
|
||||
| 2 worker + дубль | 5 | 5 | 5 | 0 | один claim при конкурирующих потребителях |
|
||||
| устаревшая версия | 1 | — | — | 0 | v1 succeeded, v2 (медленная) **superseded** при COMMIT, v3 succeeded; статус всё время `ready`; ручной дубль задания v1 — no-op |
|
||||
| reorder версий в очереди | 1 | — | 2 | 0 | v3 опубликована первой, v2 **superseded при claim** без работы |
|
||||
| зависший worker (`docker pause`) | 1 | — | 2 | 0 | claimed › lease_expired › claimed › succeeded; ожившая попытка fenced |
|
||||
| **аудит всего стенда после матрицы** | | | | **0** | `ok=True`, 522 опубликованных объекта, SHA-256 сверены, `final=523` (+sentinel) |
|
||||
|
||||
(*) claim-ов больше, чем 2 на загрузку: `os._exit` при одном задании убивает **весь процесс**, а с ним и соседнюю попытку другого задания (prefetch 2). Ей тоже приходится ждать истечения lease. Это и есть «crash до побочного эффекта»: лишние claim-ы ничего не публикуют, и ни в одном сценарии нет второго набора вариантов или объектов (столбец «Нарушений» — проверки H2/H4 по БД и S3).
|
||||
|
||||
**Вывод.** H1–H4 подтверждены на всех 12 сценариях: 0 нарушений по сценарным проверкам и по аудиту I1–I7. Время до ready при отказах определяется lease (6 с в эксперименте) + интервалом reaper (5 с): 15–22 с против ~2 с без отказов.
|
||||
|
||||
**Ограничения эксперимента.** Один прогон матрицы на 5 загрузок (плюс прогоны тех же сценариев в `student-tests` и отладочный прогон на 3 загрузки — тоже 0 нарушений). Это доказательство на конечном наборе чередований, а не формальная верификация. Отказы внедряются в заранее выбранных точках кода, а не в случайные моменты. Один Docker-хост.
|
||||
|
||||
**Почему это не exactly-once.** Доставка остаётся at-least-once: в сценариях дубля в очереди физически два сообщения, а у зависшего worker вычисление выполнялось дважды (попытки `abandoned` и `succeeded`). Exactly-once — только у **видимого эффекта**, за счёт ключа эффекта, fencing и монотонной версии. Чего нет: (1) защиты от двух независимых **сред** с разными БД; (2) гарантии при потере диска одного Docker-хоста (quorum queue и PostgreSQL на одном узле); (3) удаления опубликованных вариантов старых версий — они остаются (rollback-кандидаты), удаляются только неопубликованные.
|
||||
|
||||
### Отказы
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| worker остановлен, upload | 202, content 409, сообщения работают | так и есть | после старта worker → ready, 0 потерь |
|
||||
| restart API + broker с volumes | сообщение остаётся в quorum queue | `messages=1` до и после | 0 потерь |
|
||||
| broker остановлен | upload 202, outbox ждёт | `published=False`, текст работает | relay публикует после возврата брокера |
|
||||
| relay остановлен | outbox ждёт | так и есть | ready после старта |
|
||||
| crash relay после confirm | дубль в очереди | один claim, один набор | 0 дублей эффекта |
|
||||
| crash worker после claim / S3 / COMMIT | lease → reaper → попытка 2; либо no-op | `[abandoned, succeeded]`; мусор удалён | lease 6 с в тесте (30 с по умолчанию) + ≤5 с reaper |
|
||||
| docker pause worker с lease | второй worker публикует, первый fenced | так и есть | ≤ lease + 5 с + AMQP heartbeat |
|
||||
| 5× временная ошибка | `failed/retry_exhausted` → replay → ready | так и есть | replay оператора |
|
||||
| уборка vs replay (пауза в уборке) | staging не удалён | ready, staging удалён только после ready | — |
|
||||
| S3 остановлен/заморожен (lab4) | readiness 503, upload 503 < 8 с, текст работает | так и есть | файлы целы |
|
||||
|
||||
**Дефекты, найденные в ходе работы и исправленные:**
|
||||
1. После рестарта брокера worker оставался без consumer, а healthcheck показывал healthy; relay падал на `ChannelInvalidStateError`. Причина: aio-pika не выставляет `is_closed` при закрытии соединения сервером. Исправлено (`1d37fd0`): `close_callbacks`, heartbeat-файл только при живом consumer.
|
||||
2. Внедрённый crash worker срабатывал на каждой попытке, а не один раз на задание. Исправлено там же.
|
||||
3. Аудит ошибочно требовал объект staging у `pending` (PUT мог не пройти). Исправлен инвариант I5.
|
||||
|
||||
### Окружение и лимиты
|
||||
|
||||
Ноутбук: AMD Ryzen 5 3400G (4 ядра / 8 потоков), 29,9 ГБ RAM, Windows 10 Enterprise 19045, Docker Desktop 29.8.0 (VM: 8 CPU, 15,7 ГБ). Лимиты контейнеров (фактические, `docker inspect` в clean-clone.txt): api 1 CPU / 512m ×2, worker 2 CPU / 1536m (без swap), broker 1 CPU / 512m, relay 0,5 / 256m, postgres 2 / 1g, s3 1 / 1g, valkey 0,5 / 256m, lb 1 / 128m, migrate/s3-init 0,5 / 256m, auditor 1 / 256m, jobctl 0,5 / 128m. Нагрузочных выводов о производительности в этой лабе не делается (трек 2 — про корректность); `tests/load.py` приложен как контроль регрессии lab3.
|
||||
|
||||
## Финальный прогон
|
||||
|
||||
`python scripts/collect_evidence.py` в **новой папке** (`git clone` ветки, commit `cf6e929`): новые secrets, пустые volumes (стенд разработки удалён `down -v`), образы приложения собраны заново (слои базовых образов из кэша Docker). Все шаги `exit 0`:
|
||||
|
||||
| Файл | Что внутри |
|
||||
| --- | --- |
|
||||
| [clean-clone.txt](evidence/clean-clone.txt) | init_secrets, `compose config`, `up --build --wait`, migrate (0001–0005), s3-init (вердикты прав), пользователи и права RabbitMQ, очереди, **лимиты `docker inspect`**, версии |
|
||||
| [make-check-validate-test.txt](evidence/make-check-validate-test.txt) | `make check` PASS, `make validate` PASS, `make test` PASS lab5 |
|
||||
| [test-persistence.txt](evidence/test-persistence.txt) | PASS; состояния контейнеров до/после (новые pid у lb, api, relay, worker, postgres, valkey, s3, broker) |
|
||||
| [test-failover.txt](evidence/test-failover.txt) | PASS дважды, остановлены обе реплики по очереди |
|
||||
| [demo-pipeline.txt](evidence/demo-pipeline.txt) | сценарий защиты шаги 2–3, очередь `messages=1` до и после рестарта брокера, ready, JPEG 256×160 |
|
||||
| [student-tests.txt](evidence/student-tests.txt) | **377 passed, 4 skipped** (skipped — unit-тесты `djpeg`, которого нет на Windows-хосте; те же случаи проходят против стенда), 25 мин |
|
||||
| [track2-runs.txt](evidence/track2-runs.txt), [track2/](evidence/track2/) | матрица трека 2, аудит `--verify-checksums`, `jobctl stats` (outbox пуст), `tests/load.py` |
|
||||
| [load.json](evidence/load.json) | контроль регрессии lab3: 183 успешных RPS, 0 ошибок, p95 44 мс (4 клиента, 30 с; не сравнивается с lab3 — другой стенд) |
|
||||
|
||||
Свап: у api и worker `memswap = memory` (без swap). У инфраструктурных контейнеров остался умолчательный swap Docker (= 2 × memory) — видно в `docker inspect`, для выводов этой лабы не важно.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker-хост: quorum queue на одном узле (без репликации), PostgreSQL один. «Устойчиво» = переживает рестарт процессов/контейнеров с volumes.
|
||||
- SeaweedFS подтверждает PUT без явного fsync каждой записи; потеря питания хоста — вне модели угроз.
|
||||
- Опубликованные варианты старых версий не удаляются; квоты на число версий нет.
|
||||
- Колонки карантина lab4 (`lost_at`) сохранены, но ничем не выставляются (reconciler удалён).
|
||||
- Объект, записанный замороженной попыткой **после** её уборки (пауза дольше `ORPHAN_GRACE_SECONDS`), станет мусором; аудит покажет его как `leaked_object`. Это не потеря: опубликованное не задевается.
|
||||
- Следующие шаги: трассировка (`traceparent` в сообщении уже предусмотрен схемой) — lab6.
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and
|
||||
measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL)
|
||||
and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case
|
||||
for detection, only the active health check can find it).
|
||||
|
||||
python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes
|
||||
|
||||
Timeline of one round (all times from the moment the stop command is issued):
|
||||
warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy
|
||||
has every replica UP (rejoin time) -> 6 s steady state.
|
||||
Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed
|
||||
request (+s), the first moment after which the stopped replica never answered again
|
||||
(= removal from rotation), time until the restarted replica is UP in HAProxy and serves
|
||||
traffic again, plus the history check (no confirmed message lost, no duplicate).
|
||||
Output: <output>/series.json (summary), <output>/timeline-<mode>-<replica>.csv (per second),
|
||||
<output>/series.md (table for REPORT.md). No tokens are written.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import datetime as dt
|
||||
import json
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
sys.path.insert(0, str(ROOT / "bench"))
|
||||
import lbctl # noqa: E402
|
||||
from loadgen import Load # noqa: E402
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout
|
||||
|
||||
|
||||
def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict:
|
||||
load = Load(workers).start()
|
||||
time.sleep(6)
|
||||
t_stop = load.now()
|
||||
if mode == "pause":
|
||||
docker("pause", replica["id"])
|
||||
else:
|
||||
docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"])
|
||||
t_stopped = load.now()
|
||||
time.sleep(12)
|
||||
t_start = load.now()
|
||||
docker("unpause" if mode == "pause" else "start", replica["id"])
|
||||
lbctl.wait_up(replicas, timeout=90)
|
||||
t_up = load.now()
|
||||
# The restarted container keeps its hostname: wait until it serves requests again.
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events):
|
||||
break
|
||||
time.sleep(0.1)
|
||||
t_serves = load.now()
|
||||
time.sleep(6)
|
||||
load.stop()
|
||||
t_end = load.now()
|
||||
|
||||
failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)]
|
||||
served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start]
|
||||
out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0
|
||||
rows = load.per_second()
|
||||
with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
|
||||
writer.writeheader()
|
||||
for row in rows:
|
||||
row = dict(row)
|
||||
row["instances"] = row["instances"].replace(replica["hostname"], "TARGET")
|
||||
writer.writerow(row)
|
||||
state = json.loads(docker("inspect", replica["id"]))[0]["State"]
|
||||
return {
|
||||
"mode": mode, "replica": replica["name"], "instance_id": replica["hostname"],
|
||||
"stop_command_seconds": round(t_stopped - t_stop, 2),
|
||||
"before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end),
|
||||
"failed_after_stop": len(failed_after_stop),
|
||||
"failed_statuses": sorted({e[2] for e in failed_after_stop}),
|
||||
"last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3),
|
||||
"target_last_served_after_stop_s": round(out_of_rotation, 3),
|
||||
"rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2),
|
||||
"client_retries_same_key": load.retries,
|
||||
"history": load.verify_history(),
|
||||
"restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--output", default="evidence/failover")
|
||||
parser.add_argument("--modes", default="stop,kill,pause")
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"])
|
||||
assert len(replicas) >= 2, "need at least two running replicas"
|
||||
lbctl.wait_up(len(replicas))
|
||||
results = []
|
||||
for mode in args.modes.split(","):
|
||||
for replica in replicas: # each replica in turn; the previous one is already restored
|
||||
print(f"== {mode} {replica['name']} ...", flush=True)
|
||||
result = one_round(replica, mode, len(replicas), args.workers, out)
|
||||
results.append(result)
|
||||
print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s",
|
||||
"target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True)
|
||||
meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
||||
"commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True,
|
||||
text=True).stdout.strip(),
|
||||
"host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}",
|
||||
"workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message "
|
||||
"(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s",
|
||||
"lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; "
|
||||
"retry-on conn-failure + redispatch",
|
||||
"api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"}
|
||||
(out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n",
|
||||
encoding="utf-8")
|
||||
lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | "
|
||||
"Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |",
|
||||
"| --- | --- | --- | --- | --- | --- | --- | --- | --- |"]
|
||||
for r in results:
|
||||
h = r["history"]
|
||||
lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} "
|
||||
f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | "
|
||||
f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | "
|
||||
f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | "
|
||||
f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |")
|
||||
(out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "",
|
||||
*lines, ""]), encoding="utf-8")
|
||||
print("\n".join(lines))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4).
|
||||
|
||||
Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page.
|
||||
An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body,
|
||||
exactly as a correct client must do; the history is checked afterwards for losses and
|
||||
duplicates. Every request is recorded as (t, kind, status, instance, latency).
|
||||
This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that).
|
||||
|
||||
python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
|
||||
|
||||
class Http:
|
||||
"""Minimal stdlib client with keep-alive per thread (http.client under urllib is not)."""
|
||||
|
||||
def __init__(self, base_url: str, timeout: float):
|
||||
import http.client
|
||||
from urllib.parse import urlsplit
|
||||
parts = urlsplit(base_url)
|
||||
self.host, self.port = parts.hostname, parts.port or 80
|
||||
self.timeout = timeout
|
||||
self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout)
|
||||
self.conn = self.factory()
|
||||
|
||||
def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None):
|
||||
try:
|
||||
self.conn.request(method, path, body=body, headers=headers or {})
|
||||
response = self.conn.getresponse()
|
||||
data = response.read()
|
||||
if response.getheader("connection", "").lower() == "close":
|
||||
self.conn.close()
|
||||
self.conn = self.factory()
|
||||
return response.status, dict(response.getheaders()), data
|
||||
except Exception:
|
||||
self.conn.close()
|
||||
self.conn = self.factory()
|
||||
raise
|
||||
|
||||
|
||||
def call(method: str, path: str, body=None, headers=None, timeout=10.0):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
h = {"Content-Type": "application/json", **(headers or {})}
|
||||
req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as r:
|
||||
return r.status, json.loads(r.read() or b"null")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, json.loads(e.read() or b"null")
|
||||
|
||||
|
||||
def make_user(label: str) -> dict:
|
||||
username = f"{label}_{uuid.uuid4().hex[:12]}"
|
||||
password = "Load-password-" + uuid.uuid4().hex
|
||||
status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password})
|
||||
assert status == 201, (status, user)
|
||||
cred = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
assert status == 201, (status, session)
|
||||
return {"id": user["id"], "token": session["token"]}
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0):
|
||||
self.workers = workers
|
||||
self.timeout = timeout
|
||||
self.pause = pause
|
||||
self.alice, self.bob = make_user("load"), make_user("load")
|
||||
status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]},
|
||||
headers={"Authorization": "Bearer " + self.alice["token"]})
|
||||
assert status == 201, chat
|
||||
self.chat = chat
|
||||
self.stop_event = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events: list[tuple[float, str, int, str, float]] = []
|
||||
self.confirmed: dict[str, dict] = {}
|
||||
self.retries = 0
|
||||
self.t0 = time.monotonic()
|
||||
self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)]
|
||||
|
||||
def now(self) -> float:
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def _record(self, kind: str, status: int, instance: str, latency: float) -> None:
|
||||
with self.lock:
|
||||
self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2)))
|
||||
|
||||
def _worker(self, index: int) -> None:
|
||||
http = Http(BASE_URL, self.timeout)
|
||||
auth = {"Authorization": "Bearer " + self.alice["token"]}
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||
n, key = 0, uuid.uuid4().hex
|
||||
text = f"w{index}-0-{key[:6]}"
|
||||
while not self.stop_event.is_set():
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(),
|
||||
{**auth, "Idempotency-Key": key, "Content-Type": "application/json"})
|
||||
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
|
||||
except Exception:
|
||||
status, instance, data = 0, "-", b""
|
||||
self._record("POST", status, instance, time.monotonic() - started)
|
||||
if status == 201:
|
||||
msg = json.loads(data)
|
||||
with self.lock:
|
||||
self.confirmed[msg["id"]] = msg
|
||||
n += 1
|
||||
key = uuid.uuid4().hex
|
||||
text = f"w{index}-{n}-{key[:6]}"
|
||||
elif status in (0, 502, 503, 504, 500):
|
||||
with self.lock:
|
||||
self.retries += 1 # same key + same text on the next iteration
|
||||
started = time.monotonic()
|
||||
try:
|
||||
status, headers, _ = http.request("GET", path + "?limit=5", None, auth)
|
||||
instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-"
|
||||
except Exception:
|
||||
status, instance = 0, "-"
|
||||
self._record("GET", status, instance, time.monotonic() - started)
|
||||
if self.pause:
|
||||
time.sleep(self.pause)
|
||||
|
||||
def start(self) -> "Load":
|
||||
self.t0 = time.monotonic()
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
return self
|
||||
|
||||
def stop(self) -> None:
|
||||
self.stop_event.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=15)
|
||||
|
||||
def verify_history(self) -> dict:
|
||||
"""Walk the whole history with the second user: no confirmed message lost, no duplicate."""
|
||||
items, cursor = [], None
|
||||
while True:
|
||||
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
|
||||
status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}",
|
||||
headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30)
|
||||
assert status == 200, page
|
||||
items += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
ids = [m["id"] for m in items]
|
||||
texts = [m["text"] for m in items]
|
||||
lost = [i for i in self.confirmed if i not in set(ids)]
|
||||
changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m]
|
||||
return {"history_messages": len(items), "confirmed_messages": len(self.confirmed),
|
||||
"lost_confirmed": len(lost), "altered_confirmed": len(changed),
|
||||
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
|
||||
"unconfirmed_but_stored": len(set(ids) - set(self.confirmed))}
|
||||
|
||||
# ------------------------------------------------------------------ summaries
|
||||
def window(self, start: float, end: float) -> dict:
|
||||
ev = [e for e in self.events if start <= e[0] < end]
|
||||
ok = [e for e in ev if e[2] in (200, 201)]
|
||||
lat = sorted(e[4] for e in ok)
|
||||
|
||||
def pct(p: float):
|
||||
return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None
|
||||
span = max(end - start, 1e-9)
|
||||
return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok),
|
||||
"ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None,
|
||||
"statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)),
|
||||
"p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)}
|
||||
|
||||
def per_second(self) -> list[dict]:
|
||||
buckets: dict[int, list] = defaultdict(list)
|
||||
for e in self.events:
|
||||
buckets[int(e[0])].append(e)
|
||||
rows = []
|
||||
for second in sorted(buckets):
|
||||
ev = buckets[second]
|
||||
rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)),
|
||||
"failed": sum(1 for e in ev if e[2] not in (200, 201)),
|
||||
"instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))})
|
||||
return rows
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--duration", type=float, default=20)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
load = Load(args.workers).start()
|
||||
time.sleep(args.duration)
|
||||
load.stop()
|
||||
print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(),
|
||||
"client_retries_same_key": load.retries}, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Track 2 experiment: proven idempotency of the versioned pipeline under injected faults.
|
||||
|
||||
Hypothesis. With at-least-once delivery, duplicated and reordered messages, crashes of the worker
|
||||
before/after its side effects and stale jobs of an older pipeline version:
|
||||
H1 no accepted (202) work is lost - every upload ends ready (or failed with a code, never stuck);
|
||||
H2 every effect (attachment, pipeline_version, variant) is published exactly once - one set of
|
||||
variant rows written by ONE successful attempt, one set of objects in the bucket;
|
||||
H3 the published version only grows - an older job never overwrites a newer result, `ready`
|
||||
never goes back to queued/processing;
|
||||
H4 objects of interrupted attempts are garbage that the cleanup removes; nothing published is
|
||||
ever deleted.
|
||||
|
||||
Method. For every scenario the affected service (relay/worker) is recreated with FAULT_POINTS of
|
||||
the scenario (the real code paths, real crashes = os._exit of the process, real `docker pause`),
|
||||
N uploads go through the LB, then the scenario-specific facts are read from PostgreSQL and S3 and
|
||||
the read-only invariant audit (messenger.audit, I1-I7, with SHA-256 of every published object)
|
||||
runs over the WHOLE stand. The normal configuration is restored after each scenario.
|
||||
|
||||
python bench/pipeline_faults.py --output evidence/track2 [--uploads 5]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
from conftest import (Api, attachment_row, compose, events_of, jobctl, psql, s3_keys, service_with_env, # noqa: E402
|
||||
wait_healthy)
|
||||
|
||||
FAST = {"JOB_LEASE_SECONDS": "6", "ORPHAN_GRACE_SECONDS": "3", "CLEANUP_INTERVAL_SECONDS": "1"}
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def wait(condition, timeout: float, what: str):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
value = condition()
|
||||
if value:
|
||||
return value
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError(what)
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def facts(attachment_id: str, objects: set[str]) -> dict:
|
||||
row = attachment_row(attachment_id)
|
||||
out = psql("SELECT json_build_object("
|
||||
"'claims', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = '{0}'), "
|
||||
"'succeeded_attempts', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
||||
" WHERE j.attachment_id = '{0}' AND a.outcome = 'succeeded'), "
|
||||
"'variant_rows', (SELECT count(*) FROM attachment_variants WHERE attachment_id = '{0}'), "
|
||||
"'versions', (SELECT coalesce(json_agg(DISTINCT pipeline_version), '[]') FROM attachment_variants "
|
||||
" WHERE attachment_id = '{0}'), "
|
||||
"'job_states', (SELECT coalesce(json_agg(state || ':v' || pipeline_version ORDER BY created_at), '[]') "
|
||||
" FROM jobs WHERE attachment_id = '{0}'), "
|
||||
"'uncleaned_failed_attempts', (SELECT count(*) FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
||||
" WHERE j.attachment_id = '{0}' AND a.outcome <> 'succeeded' AND a.objects_cleaned_at IS NULL))"
|
||||
.format(attachment_id))
|
||||
f = json.loads(out)
|
||||
f.update(status=row["status"], published_version=row["published_version"], staging_state=row["staging_state"],
|
||||
objects=len([k for k in objects if k.startswith(f"a/{attachment_id}/")]))
|
||||
f["events"] = [e["event"] for e in events_of(attachment_id)]
|
||||
return f
|
||||
|
||||
|
||||
def audit() -> dict:
|
||||
r = compose("run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums", check=False, timeout=600)
|
||||
report = json.loads(r.stdout)
|
||||
return {"ok": report["ok"], "violations": report["violations"], "notes": len(report["notes"]),
|
||||
"objects": report["objects"]}
|
||||
|
||||
|
||||
class Run:
|
||||
def __init__(self, api: Api, uploads: int):
|
||||
self.api = api
|
||||
self.n = uploads
|
||||
self.alice, self.bob = api.user("t2a"), api.user("t2b")
|
||||
self.chat = api.chat(self.alice, self.bob)
|
||||
|
||||
def upload_many(self, n: int | None = None) -> list[dict]:
|
||||
return [self.api.upload(self.alice, self.chat, imagegen.png(60 + i, 40, noise=True)).json()
|
||||
for i in range(n or self.n)]
|
||||
|
||||
def finish(self, atts: list[dict], timeout: float = 180) -> tuple[list[dict], list[float]]:
|
||||
started = time.monotonic()
|
||||
took = []
|
||||
for a in atts:
|
||||
self.api.wait_status(self.bob, a["id"], timeout=timeout)
|
||||
took.append(round(time.monotonic() - started, 2))
|
||||
return atts, took
|
||||
|
||||
|
||||
def expect_one_effect(f: dict, version: int = 1) -> list[str]:
|
||||
"""H1 + H2 for a plain upload of pipeline `version`."""
|
||||
problems = []
|
||||
if f["status"] != "ready":
|
||||
problems.append(f"status {f['status']}")
|
||||
if f["succeeded_attempts"] != 1 or f["variant_rows"] != 3 or f["published_version"] != version:
|
||||
problems.append(f"effect {f['succeeded_attempts']} attempts/{f['variant_rows']} rows/v{f['published_version']}")
|
||||
if f["objects"] != 4: # original + thumbnail + crop + optimized, cleanup done
|
||||
problems.append(f"{f['objects']} objects in the bucket")
|
||||
return problems
|
||||
|
||||
|
||||
def plain_scenario(run: Run, name: str, service: str, env: dict, *, scale: int | None = None,
|
||||
cleanup_wait: bool = True) -> dict:
|
||||
with service_with_env(service, scale=scale, **env):
|
||||
atts, took = run.finish(run.upload_many())
|
||||
if cleanup_wait: # garbage of interrupted attempts: cleaned after ORPHAN_GRACE_SECONDS
|
||||
try:
|
||||
wait(lambda: all(json.loads(psql(
|
||||
"SELECT json_build_object('n', count(*)) FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
||||
f"WHERE j.attachment_id = '{a['id']}' AND a.outcome <> 'succeeded' "
|
||||
"AND a.objects_cleaned_at IS NULL"))["n"] == 0 for a in atts), 60, "cleanup")
|
||||
except AssertionError:
|
||||
pass # reported below as uncleaned attempts
|
||||
objects = s3_keys("a/")
|
||||
per = [facts(a["id"], objects) for a in atts]
|
||||
problems = {a["id"]: p for a, f in zip(atts, per) if (p := expect_one_effect(f))}
|
||||
return {"scenario": name, "faults": {service: env}, "uploads": len(atts),
|
||||
"ready": sum(f["status"] == "ready" for f in per), "claims_total": sum(f["claims"] for f in per),
|
||||
"max_claims": max(f["claims"] for f in per), "time_to_ready_s": {"p50": statistics.median(took),
|
||||
"max": max(took)},
|
||||
"event_sequences": sorted({" > ".join(f["events"]) for f in per}), "problems": problems}
|
||||
|
||||
|
||||
def versions_scenario(run: Run) -> dict:
|
||||
"""Stale job after a newer version: v2 (slow) and v3 run at the same time; v3 publishes first,
|
||||
v2 finishes later and must end superseded without touching the published v3 result."""
|
||||
att = run.upload_many(1)[0]
|
||||
run.api.wait_status(run.bob, att["id"], "ready")
|
||||
observed = set()
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="slow=10,slow_pipeline=2"):
|
||||
jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "track2", "--reason", "new encoder")
|
||||
wait(lambda: "processing:v2" in facts(att["id"], set())["job_states"], 60, "v2 not claimed")
|
||||
jobctl("reprocess", att["id"], "--pipeline", "3", "--operator", "track2", "--reason", "newer encoder")
|
||||
deadline = time.monotonic() + 120
|
||||
while time.monotonic() < deadline:
|
||||
meta = run.api.attachment(run.bob, att["id"])
|
||||
observed.add(meta["status"])
|
||||
states = facts(att["id"], set())["job_states"]
|
||||
if not any(s.startswith(("queued", "processing")) for s in states):
|
||||
break
|
||||
time.sleep(0.3)
|
||||
final = run.api.attachment(run.bob, att["id"])
|
||||
f = facts(att["id"], s3_keys("a/"))
|
||||
thumb_v3 = psql(f"SELECT encode(sha256, 'hex') FROM attachment_variants WHERE attachment_id = '{att['id']}' "
|
||||
"AND pipeline_version = 3 AND name = 'thumbnail'")
|
||||
problems = []
|
||||
if f["job_states"] != ["succeeded:v1", "superseded:v2", "succeeded:v3"]:
|
||||
problems.append(f"job states {f['job_states']}")
|
||||
if f["published_version"] != 3 or observed != {"ready"}:
|
||||
problems.append(f"published v{f['published_version']}, statuses seen {sorted(observed)}")
|
||||
if next(v["sha256"] for v in final["variants"] if v["name"] == "thumbnail") != thumb_v3:
|
||||
problems.append("published thumbnail is not the v3 one")
|
||||
# Stale duplicate of the FIRST job, published by hand with the relay's broker user: a no-op.
|
||||
job_v1 = psql(f"SELECT id FROM jobs WHERE attachment_id = '{att['id']}' AND pipeline_version = 1")
|
||||
script = ("import asyncio, aio_pika, datetime as d\n"
|
||||
"from messenger import broker\n"
|
||||
"async def m():\n"
|
||||
" c = await aio_pika.connect(open('/run/secrets/broker_url').read().strip())\n"
|
||||
" ch = await c.channel(publisher_confirms=True)\n"
|
||||
" ex = await ch.get_exchange(broker.EXCHANGE, ensure=False)\n"
|
||||
f" body = broker.encode('{job_v1}', '{att['id']}', 1, 1, d.datetime.now(d.timezone.utc))\n"
|
||||
" await ex.publish(aio_pika.Message(body), routing_key=broker.ROUTING_KEY)\n"
|
||||
" await c.close()\n"
|
||||
"asyncio.run(m())\n")
|
||||
compose("exec", "-T", "relay", "python", "-c", script)
|
||||
time.sleep(3)
|
||||
after = run.api.attachment(run.bob, att["id"])
|
||||
if after != final:
|
||||
problems.append("stale v1 message changed the attachment")
|
||||
return {"scenario": "stale_version_and_reorder", "uploads": 1, "statuses_seen_during_reprocess": sorted(observed),
|
||||
"job_states": f["job_states"], "published_version": f["published_version"],
|
||||
"variant_rows_by_version": f["variant_rows"], "objects": f["objects"],
|
||||
"events": f["events"], "problems": problems}
|
||||
|
||||
|
||||
def queued_reorder_scenario(run: Run) -> dict:
|
||||
"""Both newer versions wait in the outbox; the relay publishes them in REVERSE order and one worker
|
||||
takes them one by one: v3 is published first, v2 is then superseded at its claim (no work)."""
|
||||
att = run.upload_many(1)[0]
|
||||
run.api.wait_status(run.bob, att["id"], "ready")
|
||||
# The worker stays stopped until it is recreated with concurrency 1 (no normal start in between).
|
||||
# Worker and relay stay stopped until each is recreated with the scenario's settings: no normal
|
||||
# start in between that would consume/publish the jobs in the ordinary order.
|
||||
compose("stop", "worker", "relay")
|
||||
try:
|
||||
jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "track2", "--reason", "v2")
|
||||
jobctl("reprocess", att["id"], "--pipeline", "3", "--operator", "track2", "--reason", "v3")
|
||||
with service_with_env("relay", RELAY_FAULT_POINTS="reverse_order"):
|
||||
wait(lambda: psql("SELECT count(*) FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = "
|
||||
f"'{att['id']}' AND j.pipeline_version > 1 AND o.published_at IS NOT NULL") == "2",
|
||||
30, "both reprocess jobs not published") # in one relay batch: v3 first, then v2
|
||||
with service_with_env("worker", WORKER_CONCURRENCY="1"):
|
||||
wait(lambda: not any(s.startswith(("queued", "processing"))
|
||||
for s in facts(att["id"], set())["job_states"]), 120, "reprocess jobs did not finish")
|
||||
finally:
|
||||
compose("start", "worker", "relay")
|
||||
wait_healthy("worker", "relay")
|
||||
f = facts(att["id"], s3_keys("a/"))
|
||||
problems = []
|
||||
if f["job_states"] != ["succeeded:v1", "superseded:v2", "succeeded:v3"] or f["published_version"] != 3:
|
||||
problems.append(f"{f['job_states']} published v{f['published_version']}")
|
||||
if f["claims"] != 2: # v1 and v3 only: v2 was superseded before any work
|
||||
problems.append(f"{f['claims']} claims")
|
||||
return {"scenario": "reordered_versions_in_queue", "job_states": f["job_states"], "claims": f["claims"],
|
||||
"published_version": f["published_version"], "events": f["events"], "problems": problems}
|
||||
|
||||
|
||||
def hung_worker_scenario(run: Run) -> dict:
|
||||
with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST):
|
||||
att = run.upload_many(1)[0]
|
||||
owner = wait(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' "
|
||||
"AND state = 'processing'"), 30, "not claimed")
|
||||
ids = compose("ps", "-q", "worker").stdout.split()
|
||||
names = dict(line.split() for line in subprocess.run(
|
||||
["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True, text=True,
|
||||
check=True).stdout.splitlines())
|
||||
subprocess.run(["docker", "pause", names[owner]], check=True)
|
||||
try:
|
||||
run.api.wait_status(run.bob, att["id"], "ready", timeout=150)
|
||||
published = psql(f"SELECT string_agg(object_key, ',' ORDER BY object_key) FROM attachment_variants "
|
||||
f"WHERE attachment_id = '{att['id']}'")
|
||||
finally:
|
||||
subprocess.run(["docker", "unpause", names[owner]], check=True)
|
||||
time.sleep(15)
|
||||
again = psql(f"SELECT string_agg(object_key, ',' ORDER BY object_key) FROM attachment_variants "
|
||||
f"WHERE attachment_id = '{att['id']}'")
|
||||
wait(lambda: json.loads(psql("SELECT json_build_object('n', count(*)) FROM job_attempts a JOIN jobs j "
|
||||
f"ON j.id = a.job_id WHERE j.attachment_id = '{att['id']}' AND a.outcome <> "
|
||||
"'succeeded' AND a.objects_cleaned_at IS NULL"))["n"] == 0, 60, "cleanup")
|
||||
f = facts(att["id"], s3_keys("a/"))
|
||||
problems = expect_one_effect(f)
|
||||
if published != again:
|
||||
problems.append("the resumed (fenced) attempt changed the published result")
|
||||
return {"scenario": "hung_worker_docker_pause", "frozen": owner, "claims": f["claims"], "events": f["events"],
|
||||
"problems": problems}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--output", default="evidence/track2")
|
||||
p.add_argument("--uploads", type=int, default=5)
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
api = Api()
|
||||
run = Run(api, args.uploads)
|
||||
started = now()
|
||||
results = []
|
||||
plan = [
|
||||
("baseline_no_faults", "worker", {}),
|
||||
("duplicate_publish", "relay", {"RELAY_FAULT_POINTS": "duplicate_publish"}),
|
||||
("duplicate_and_reverse_order", "relay", {"RELAY_FAULT_POINTS": "duplicate_publish,reverse_order"}),
|
||||
("relay_crash_after_confirm_before_mark", "relay", {"RELAY_FAULT_POINTS": "crash_after_publish"}),
|
||||
("worker_crash_before_side_effect", "worker", {"WORKER_FAULT_POINTS": "crash_after_claim", **FAST}),
|
||||
("worker_crash_after_s3_before_commit", "worker", {"WORKER_FAULT_POINTS": "crash_after_s3_write", **FAST}),
|
||||
("worker_crash_after_commit_before_ack", "worker", {"WORKER_FAULT_POINTS": "crash_after_commit", **FAST}),
|
||||
("transient_errors_then_success", "worker", {"WORKER_FAULT_POINTS": "transient_error=2",
|
||||
"JOB_RETRY_BASE_SECONDS": "0.5", **FAST}),
|
||||
]
|
||||
for name, service, env in plan:
|
||||
print(f"[{now()}] {name}", flush=True)
|
||||
results.append(plain_scenario(run, name, service, env))
|
||||
print(f"[{now()}] two_workers_duplicate_publish", flush=True)
|
||||
with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish"):
|
||||
results.append(plain_scenario(run, "two_workers_duplicate_publish", "worker", {}, scale=2))
|
||||
for scenario in (versions_scenario, queued_reorder_scenario, hung_worker_scenario):
|
||||
print(f"[{now()}] {scenario.__name__}", flush=True)
|
||||
results.append(scenario(run))
|
||||
wait_healthy("worker", "relay")
|
||||
report = audit()
|
||||
summary = {
|
||||
"experiment": "track 2: idempotency of the versioned pipeline under injected faults",
|
||||
"started": started, "finished": now(),
|
||||
"host": {"os": platform.platform(), "python": platform.python_version()},
|
||||
"uploads_per_plain_scenario": args.uploads,
|
||||
"scenarios": results,
|
||||
"audit_after_all": report,
|
||||
"violations": sum(len(r["problems"]) for r in results) + len(report["violations"]),
|
||||
}
|
||||
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
lines = [f"{'scenario':45} {'uploads':>7} {'ready':>5} {'claims':>6} problems"]
|
||||
for r in results:
|
||||
lines.append(f"{r['scenario']:45} {r.get('uploads', 1):>7} {r.get('ready', '-'):>5} "
|
||||
f"{r.get('claims_total', r.get('claims', '-')):>6} {len(r['problems'])}")
|
||||
lines.append(f"audit after all scenarios: ok={report['ok']} violations={len(report['violations'])} "
|
||||
f"objects={report['objects']}")
|
||||
(out / "summary.txt").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
||||
print("\n".join(lines))
|
||||
return 0 if summary["violations"] == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB.
|
||||
|
||||
python bench/rolling_under_load.py --output evidence/rolling-update
|
||||
|
||||
Writes <output>/summary.json (errors, RPS, history check, revisions before/after),
|
||||
<output>/timeline.csv (per second ok/failed/instances) and <output>/script-output.txt.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
sys.path.insert(0, str(ROOT / "bench"))
|
||||
import lbctl # noqa: E402
|
||||
from loadgen import Load # noqa: E402
|
||||
|
||||
|
||||
def revisions() -> dict[str, str]:
|
||||
result = {}
|
||||
for r in lbctl.replicas():
|
||||
if r["state"] == "running":
|
||||
label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}',
|
||||
r["id"]], capture_output=True, text=True).stdout.strip()
|
||||
result[r["name"]] = f"{r['hostname']} revision={label}"
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
parser.add_argument("--output", default="evidence/rolling-update")
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
args = parser.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
before = revisions()
|
||||
load = Load(args.workers).start()
|
||||
time.sleep(5)
|
||||
t_begin = load.now()
|
||||
bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe")
|
||||
if os.name == "nt" else "sh")
|
||||
env = {**os.environ, "PYTHON": sys.executable}
|
||||
script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
|
||||
capture_output=True, text=True, encoding="utf-8", timeout=900)
|
||||
t_end = load.now()
|
||||
time.sleep(5)
|
||||
load.stop()
|
||||
(out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n"
|
||||
+ script.stdout + script.stderr, encoding="utf-8")
|
||||
with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f:
|
||||
writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"])
|
||||
writer.writeheader()
|
||||
writer.writerows(load.per_second())
|
||||
summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1),
|
||||
"before": before, "after": revisions(),
|
||||
"during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()),
|
||||
"client_retries_same_key": load.retries, "history": load.verify_history(),
|
||||
"workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"}
|
||||
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")},
|
||||
ensure_ascii=False, indent=2))
|
||||
print(json.dumps(summary["during_update"], ensure_ascii=False))
|
||||
return 0 if script.returncode == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 4 experiment: peak memory of the API replicas under >= 10 simultaneous uploads.
|
||||
|
||||
Hypothesis: the memory of one replica is bounded by UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES (bodies
|
||||
waiting for a slot stay in TCP buffers, not in the process) + one image check at a time
|
||||
(IMAGE_VERIFY_CONCURRENCY), independent of how many clients upload at once.
|
||||
|
||||
Method (repeatable, the stand must be up):
|
||||
* for every scenario the api replicas are RECREATED with the scenario's settings, so the cgroup
|
||||
counter memory.peak starts from an idle process (it never decreases during a container's life);
|
||||
* idle memory.current of each replica is read; then N clients upload at the same moment through
|
||||
the LB (a barrier releases them together), each a different file;
|
||||
* during the run memory.current of every replica is sampled every 50 ms inside the container;
|
||||
afterwards memory.peak (cgroup v2: all processes of the container incl. djpeg) and VmHWM of
|
||||
PID 1 (the Python server alone) are read;
|
||||
* the normal configuration is restored at the end.
|
||||
|
||||
python bench/upload_memory.py --output evidence/memory
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import httpx2 as httpx
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
import imagegen # noqa: E402
|
||||
|
||||
BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080")
|
||||
PASSWORD = "Bench-password-Ω-12345"
|
||||
MiB = 1024 * 1024
|
||||
|
||||
|
||||
def sh(*args: str, env: dict | None = None, timeout: float = 300) -> str:
|
||||
return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=True, timeout=timeout,
|
||||
env={**os.environ, **(env or {})}).stdout
|
||||
|
||||
|
||||
def replicas() -> list[str]:
|
||||
return sh("docker", "compose", "ps", "-q", "api").split()
|
||||
|
||||
|
||||
def cg(cid: str, name: str) -> int:
|
||||
return int(sh("docker", "exec", cid, "cat", f"/sys/fs/cgroup/{name}").split()[0])
|
||||
|
||||
|
||||
def memory_stat(cid: str, key: str) -> int:
|
||||
for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat").splitlines():
|
||||
name, value = line.split()
|
||||
if name == key:
|
||||
return int(value)
|
||||
return 0
|
||||
|
||||
|
||||
def vmhwm(cid: str) -> int:
|
||||
for line in sh("docker", "exec", cid, "cat", "/proc/1/status").splitlines():
|
||||
if line.startswith("VmHWM:"):
|
||||
return int(line.split()[1]) * 1024
|
||||
return 0
|
||||
|
||||
|
||||
def container_events() -> dict[str, str]:
|
||||
"""restart count + OOM-kill events per container (cgroup memory.events oom_kill counter)."""
|
||||
out = {}
|
||||
for cid in sh("docker", "compose", "ps", "-q").split():
|
||||
name, restarts = sh("docker", "inspect", "-f", "{{.Name}} {{.RestartCount}}", cid).split()
|
||||
events = dict(line.split() for line in sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.events").splitlines())
|
||||
out[name.lstrip("/")] = f"restarts={restarts} oom_kill={events.get('oom_kill', '?')}"
|
||||
return out
|
||||
|
||||
|
||||
def recreate(env: dict) -> None:
|
||||
sh("docker", "compose", "up", "-d", "--force-recreate", "--no-deps", "--wait", "api", env=env, timeout=300)
|
||||
sh(sys.executable, "scripts/lbctl.py", "wait-up", str(len(replicas())), "--timeout", "90") # LB sees them UP
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
"""memory.current of one container every 50 ms, read by a shell loop inside it."""
|
||||
|
||||
def __init__(self, cid: str):
|
||||
super().__init__(daemon=True)
|
||||
self.cid = cid
|
||||
self.values: list[int] = []
|
||||
self.proc = subprocess.Popen(["docker", "exec", cid, "sh", "-c",
|
||||
"while :; do cat /sys/fs/cgroup/memory.current; sleep 0.05; done"],
|
||||
stdout=subprocess.PIPE, text=True)
|
||||
|
||||
def run(self) -> None:
|
||||
for line in self.proc.stdout:
|
||||
if line.strip().isdigit():
|
||||
self.values.append(int(line))
|
||||
|
||||
def stop(self) -> None:
|
||||
self.proc.kill()
|
||||
|
||||
|
||||
def actor(client: httpx.Client) -> tuple[str, str]:
|
||||
name = "mem_" + uuid.uuid4().hex[:10]
|
||||
client.post("/api/v1/users", json={"username": name, "display_name": "bench", "password": PASSWORD}).raise_for_status()
|
||||
import base64
|
||||
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
|
||||
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}).json()["token"]
|
||||
me = client.get("/api/v1/users/me", headers={"Authorization": "Bearer " + token}).json()
|
||||
chat = client.post("/api/v1/chats", headers={"Authorization": "Bearer " + token},
|
||||
json={"title": "mem", "member_ids": [me["id"]]}).json()
|
||||
return token, chat["id"]
|
||||
|
||||
|
||||
def big_jpeg(width: int, height: int, seed: int) -> bytes:
|
||||
"""Large progressive JPEG (smooth content keeps it < 10 MiB): the costliest file to check."""
|
||||
image = Image.linear_gradient("L").resize((width, height)).convert("RGB")
|
||||
image.putpixel((seed % width, 0), (seed % 255, 0, 0))
|
||||
out = io.BytesIO()
|
||||
image.save(out, "JPEG", quality=90, progressive=True, subsampling=0)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def run_scenario(name: str, env: dict, files: list[tuple[bytes, str]]) -> dict:
|
||||
recreate(env)
|
||||
ids = replicas()
|
||||
idle = {cid: cg(cid, "memory.current") for cid in ids}
|
||||
client = httpx.Client(base_url=BASE, timeout=120)
|
||||
token, chat_id = actor(client)
|
||||
barrier = threading.Barrier(len(files))
|
||||
results: list[tuple[int, float, str]] = [None] * len(files) # type: ignore[list-item]
|
||||
|
||||
def upload(i: int, data: bytes, ctype: str) -> None:
|
||||
c = httpx.Client(base_url=BASE, timeout=120)
|
||||
barrier.wait()
|
||||
started = time.perf_counter()
|
||||
r = c.post(f"/api/v1/chats/{chat_id}/attachments?filename=f{i}", content=data,
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": ctype})
|
||||
reason = r.json()["error"]["message"] if r.status_code != 201 else ""
|
||||
results[i] = (r.status_code, time.perf_counter() - started, r.headers.get("x-instance-id", "-"), reason)
|
||||
c.close()
|
||||
|
||||
s3_id = sh("docker", "compose", "ps", "-q", "s3").strip()
|
||||
restarts_before = container_events()
|
||||
samplers = [Sampler(cid) for cid in ids] + [Sampler(s3_id)]
|
||||
for s in samplers:
|
||||
s.start()
|
||||
time.sleep(0.5)
|
||||
threads = [threading.Thread(target=upload, args=(i, d, t)) for i, (d, t) in enumerate(files)]
|
||||
wall = time.perf_counter()
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
wall = time.perf_counter() - wall
|
||||
time.sleep(0.5)
|
||||
for s in samplers:
|
||||
s.stop()
|
||||
per_replica = {}
|
||||
for cid, s in zip(ids, samplers[:-1]):
|
||||
hostname = sh("docker", "inspect", "-f", "{{.Config.Hostname}}", cid).strip()
|
||||
per_replica[hostname] = {
|
||||
"idle_mib": round(idle[cid] / MiB, 1),
|
||||
"peak_mib": round(cg(cid, "memory.peak") / MiB, 1),
|
||||
"sampled_max_mib": round(max(s.values or [0]) / MiB, 1),
|
||||
"python_vmhwm_mib": round(vmhwm(cid) / MiB, 1),
|
||||
"uploads_served": sum(1 for r in results if r[2] == hostname),
|
||||
"limit_mib": round(cg(cid, "memory.max") / MiB, 1),
|
||||
}
|
||||
per_replica[hostname]["peak_minus_idle_mib"] = round(per_replica[hostname]["peak_mib"]
|
||||
- per_replica[hostname]["idle_mib"], 1)
|
||||
lat = sorted(r[1] for r in results if r[0] == 201)
|
||||
codes: dict[str, int] = {}
|
||||
reasons: dict[str, int] = {}
|
||||
for r in results:
|
||||
codes[str(r[0])] = codes.get(str(r[0]), 0) + 1
|
||||
if r[3]:
|
||||
reasons[r[3]] = reasons.get(r[3], 0) + 1
|
||||
summary = {
|
||||
"scenario": name, "settings": env, "clients": len(files),
|
||||
"s3_sampled_max_mib": round(max(samplers[-1].values or [0]) / MiB, 1),
|
||||
"s3_anon_after_mib": round(memory_stat(s3_id, "anon") / MiB, 1),
|
||||
"s3_limit_mib": round(cg(s3_id, "memory.max") / MiB, 1),
|
||||
# OOM kills / restarts DURING the scenario (any container of the stand): must be empty
|
||||
"incidents": {k: v for k, v in container_events().items() if v != restarts_before.get(k)},
|
||||
"file_mib_each": round(statistics.mean(len(d) for d, _ in files) / MiB, 2),
|
||||
"types": sorted({t for _, t in files}), "status_codes": codes, "error_reasons": reasons, "wall_seconds": round(wall, 2),
|
||||
"latency_s": {"p50": round(lat[len(lat) // 2], 2), "max": round(lat[-1], 2)} if lat else None,
|
||||
"replicas": per_replica,
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False), flush=True)
|
||||
return summary
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--clients", type=int, default=16)
|
||||
p.add_argument("--output", default="evidence/memory")
|
||||
args = p.parse_args()
|
||||
out = ROOT / args.output
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
print("generating files ...", flush=True)
|
||||
pngs = [(imagegen.png_of_size(int(9.5 * MiB)), "image/png") for _ in range(2 * args.clients)]
|
||||
jpegs = [(big_jpeg(8000, 5000, i), "image/jpeg") for i in range(args.clients)]
|
||||
base = {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "1"}
|
||||
n, n2 = args.clients, 2 * args.clients
|
||||
scenarios = [
|
||||
(f"png-9.4MiB x{n}, slots=4 (default)", base, pngs[:n]),
|
||||
(f"png-9.4MiB x{n2}, slots=4 (default)", base, pngs),
|
||||
(f"png-9.4MiB x{n2}, slots=16 (bound raised)", {"UPLOAD_CONCURRENCY": "16", "IMAGE_VERIFY_CONCURRENCY": "1"}, pngs),
|
||||
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=1 (default)", base, jpegs),
|
||||
(f"jpeg-40Mpx-progressive-4:4:4 x{n}, verify=4", {"UPLOAD_CONCURRENCY": "4", "IMAGE_VERIFY_CONCURRENCY": "4"}, jpegs),
|
||||
]
|
||||
results = []
|
||||
sh("docker", "compose", "restart", "s3") # S3 memory counters start from an idle process
|
||||
sh("docker", "compose", "up", "-d", "--wait", "s3")
|
||||
try:
|
||||
for name, env, files in scenarios:
|
||||
results.append(run_scenario(name, env, files))
|
||||
finally:
|
||||
recreate({}) # back to the compose defaults
|
||||
meta = {
|
||||
"date": time.strftime("%Y-%m-%d %H:%M:%S"), "host": platform.platform(), "cpus": os.cpu_count(),
|
||||
"commit": sh("git", "rev-parse", "--short", "HEAD").strip(),
|
||||
"api_limits": sh("docker", "inspect", "-f", "{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}",
|
||||
replicas()[0]).strip(),
|
||||
"jpeg_file_mib": round(len(jpegs[0][0]) / MiB, 2), "png_file_mib": round(len(pngs[0][0]) / MiB, 2),
|
||||
}
|
||||
(out / "summary.json").write_text(json.dumps({"meta": meta, "scenarios": results}, ensure_ascii=False, indent=1),
|
||||
encoding="utf-8")
|
||||
print(json.dumps(meta, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+521
@@ -0,0 +1,521 @@
|
||||
# Lab 5 stand: client -> HAProxy (lb) -> API x N (scalable service `api`)
|
||||
# -> PostgreSQL (metadata, jobs, outbox: source of truth) + Valkey (sessions)
|
||||
# + SeaweedFS S3 (staging bucket and final bucket, both private)
|
||||
# relay : outbox in PostgreSQL -> RabbitMQ (publisher confirms) [service `relay`]
|
||||
# broker : RabbitMQ 4.3, durable quorum queue + dead-letter queue [service `broker`]
|
||||
# worker : RabbitMQ -> variants -> final bucket + PostgreSQL, then ack [service `worker`]
|
||||
# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md):
|
||||
# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored)
|
||||
# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...)
|
||||
# docker compose --env-file .env up -d --build --wait
|
||||
# Order: postgres/valkey/s3/broker healthy -> migrate + s3-init (ONE run each, exit 0)
|
||||
# -> api healthy -> lb; relay and worker start after migrate/s3-init.
|
||||
# Images are pinned by version AND digest. Passwords/keys are passed only as secret files, and
|
||||
# every service gets only the secrets it needs. Only the LB publishes ports (loopback only).
|
||||
# Networks: `edge` (lb <-> api); internal (no route outside): `data` (postgres/valkey),
|
||||
# `storage` (s3), `queue` (broker). The API is NOT on `queue`: it never talks to the broker.
|
||||
name: mtusi-lab5
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: ${SESSION_STORE:-valkey}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
DB_POOL_MIN: ${DB_POOL_MIN:-1}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
|
||||
CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5}
|
||||
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
# Graceful stop: 3 s drain (readiness 503, still serving) + up to 8 s for in-flight requests.
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8}
|
||||
DOCS_ENABLED: ${DOCS_ENABLED:-true}
|
||||
MESSAGE_WRITE_PATH: ${MESSAGE_WRITE_PATH:-function}
|
||||
# S3: the API's own key pair (Write staging + Read final, see infra/seaweedfs/start.sh).
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-4}
|
||||
UPLOAD_QUEUE_TIMEOUT_SECONDS: ${UPLOAD_QUEUE_TIMEOUT_SECONDS:-5}
|
||||
UPLOAD_READ_TIMEOUT_SECONDS: ${UPLOAD_READ_TIMEOUT_SECONDS:-30}
|
||||
IMAGE_VERIFY_TIMEOUT_SECONDS: ${IMAGE_VERIFY_TIMEOUT_SECONDS:-10}
|
||||
IMAGE_VERIFY_CONCURRENCY: ${IMAGE_VERIFY_CONCURRENCY:-1}
|
||||
JPEG_MAX_MEMORY_MB: ${JPEG_MAX_MEMORY_MB:-256}
|
||||
PENDING_GRACE_SECONDS: ${PENDING_GRACE_SECONDS:-900}
|
||||
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60}
|
||||
# Lab 5: pipeline version and attempt budget of the jobs created by uploads.
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
# Failure experiments only (student-tests, bench/); empty = no injected failures.
|
||||
FAULT_POINTS: ${API_FAULT_POINTS:-}
|
||||
|
||||
# Settings of the pipeline shared by relay and worker (docs/PIPELINE.md).
|
||||
x-pipeline-env: &pipeline-env
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
DB_POOL_MAX: ${PIPELINE_DB_POOL_MAX:-6}
|
||||
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
|
||||
JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2}
|
||||
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60}
|
||||
QUEUED_REPUBLISH_SECONDS: ${QUEUED_REPUBLISH_SECONDS:-300}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
# pg_stat_statements: per-statement counters for the experiments.
|
||||
command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -q"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"}
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
# The password goes into a config include readable only by the valkey user,
|
||||
# so it never appears in the process arguments. docker-entrypoint.sh drops root.
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- >-
|
||||
umask 077 &&
|
||||
printf 'requirepass %s\n' "$$(cat /run/secrets/valkey_password)" > /tmp/requirepass.conf &&
|
||||
chown valkey /tmp/requirepass.conf &&
|
||||
exec docker-entrypoint.sh valkey-server /etc/valkey/valkey.conf
|
||||
secrets: [valkey_password]
|
||||
volumes:
|
||||
- ./infra/valkey/valkey.conf:/etc/valkey/valkey.conf:ro
|
||||
- valkeydata:/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m}
|
||||
|
||||
# S3-compatible object storage (SeaweedFS: master + volume + filer + S3 gateway in one process).
|
||||
# Bytes live ONLY here, on the named volume s3data: staging bucket (originals accepted before
|
||||
# 202) and final bucket (published originals and variants). Identities: infra/seaweedfs/start.sh.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62
|
||||
entrypoint: ["/bin/sh", "/etc/seaweedfs/start.sh"]
|
||||
environment:
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
# Go heap target below the container limit: the GC returns memory before the cgroup OOM.
|
||||
GOMEMLIMIT: ${S3_GOMEMLIMIT:-600MiB}
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_api_access_key_id
|
||||
- s3_api_secret_access_key
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
- s3_auditor_access_key_id
|
||||
- s3_auditor_secret_access_key
|
||||
volumes:
|
||||
- ./infra/seaweedfs/start.sh:/etc/seaweedfs/start.sh:ro
|
||||
- s3data:/data
|
||||
healthcheck:
|
||||
# Unauthenticated health endpoint of the S3 gateway (200 once it serves requests).
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8333/healthz"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
networks: [storage]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${S3_CPUS:-1.0}", memory: "${S3_MEMORY:-1g}"}
|
||||
|
||||
# Lab 5: RabbitMQ. Topology (exchange, quorum queue, DLX/DLQ) and users come from definition
|
||||
# files imported at boot (infra/rabbitmq, secrets/rabbitmq_users.json). Messages of the quorum
|
||||
# queue live on the named volume rabbitmqdata. No management UI, no host port.
|
||||
broker:
|
||||
image: rabbitmq:4.3.7-alpine@sha256:b1f997f74103d08fc1f20a13890dc44fe6cfa51d528cf521b480ada988d3ffa5
|
||||
hostname: broker # node name rabbit@broker: stable across container re-creation
|
||||
secrets:
|
||||
- source: rabbitmq_users
|
||||
target: /etc/rabbitmq/definitions.d/users.json
|
||||
volumes:
|
||||
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/20-lab5.conf:ro
|
||||
- ./infra/rabbitmq/enabled_plugins:/etc/rabbitmq/enabled_plugins:ro
|
||||
- ./infra/rabbitmq/topology.json:/etc/rabbitmq/definitions.d/topology.json:ro
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
# The node runs and the AMQP listener accepts connections.
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q ping && rabbitmq-diagnostics -q check_port_connectivity"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
start_period: 20s
|
||||
stop_grace_period: 30s
|
||||
restart: unless-stopped
|
||||
networks: [queue]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${BROKER_CPUS:-1.0}", memory: "${BROKER_MEMORY:-512m}"}
|
||||
|
||||
# One-shot schema migration: runs ONCE per `up`, before any API replica starts
|
||||
# (api depends_on service_completed_successfully). Replicas never migrate themselves.
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256m}
|
||||
|
||||
# One-shot, idempotent S3 bootstrap: both buckets, sentinel object, verification of the access
|
||||
# model (private buckets, least-privilege keys). The ONLY holder of the admin key pair besides s3.
|
||||
s3-init:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.s3_bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_access_key
|
||||
S3_WORKER_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key_id
|
||||
S3_WORKER_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_access_key
|
||||
S3_AUDITOR_ACCESS_KEY_ID_FILE: /run/secrets/s3_auditor_access_key_id
|
||||
S3_AUDITOR_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_auditor_secret_access_key
|
||||
secrets:
|
||||
- s3_admin_access_key_id
|
||||
- s3_admin_secret_access_key
|
||||
- s3_api_access_key_id
|
||||
- s3_api_secret_access_key
|
||||
- s3_worker_access_key_id
|
||||
- s3_worker_secret_access_key
|
||||
- s3_auditor_access_key_id
|
||||
- s3_auditor_secret_access_key
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully # same image: built once by `migrate`
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
networks: [storage]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256m}
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
BUILD_REVISION: ${BUILD_REVISION:-dev}
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
environment: *api-env
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- source: s3_api_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_api_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
# No `ports` and no container_name: replicas are reachable only through `lb`,
|
||||
# and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname.
|
||||
expose: ["8080"]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-m", "messenger.healthcheck"]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s)
|
||||
restart: unless-stopped
|
||||
networks: [edge, data, storage]
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
# 512m: UPLOAD_CONCURRENCY (4) x 10 MiB bodies + one djpeg (<= JPEG_MAX_MEMORY_MB) + runtime;
|
||||
# measured in lab 4 (bench/upload_memory.py). No resizing happens in the API.
|
||||
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512m}"}
|
||||
# No swap on top of the limit (Docker's default would allow as much swap again): the RAM bound
|
||||
# of the experiments is a real bound, and running out of it is visible, not hidden in swap.
|
||||
memswap_limit: ${API_MEMORY:-512m}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
# The ONLY public entry point, on the loopback interfaces only. Both IPv4 and IPv6:
|
||||
# on Windows `localhost` resolves to ::1 first, and a refused ::1 connection costs ~2 s
|
||||
# per request before the fallback to 127.0.0.1 (measured in lab 3: 2.0 s vs 0.007 s).
|
||||
- "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080"
|
||||
- "[::1]:${APP_PUBLISH_PORT:-8080}:8080"
|
||||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas)
|
||||
healthcheck:
|
||||
# Config is valid and the process answers; whether any API is ready is the LB's own job.
|
||||
test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && wget -q -O /dev/null http://127.0.0.1:8404/"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
networks: [edge]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m}
|
||||
|
||||
# Lab 5: outbox relay + lease reaper (src/messenger/relay.py). One process; a second one would
|
||||
# be safe (SKIP LOCKED) but is not needed. Database + broker only, no S3 keys.
|
||||
relay:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.relay"]
|
||||
environment:
|
||||
<<: *pipeline-env
|
||||
RELAY_POLL_SECONDS: ${RELAY_POLL_SECONDS:-0.5}
|
||||
FAULT_POINTS: ${RELAY_FAULT_POINTS:-}
|
||||
secrets:
|
||||
- database_url
|
||||
- source: broker_url_relay
|
||||
target: broker_url
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
networks: [data, queue]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${RELAY_CPUS:-0.5}", memory: 256m}
|
||||
|
||||
# Lab 5: image worker (src/messenger/worker.py). Scalable (`--scale worker=2`): competing
|
||||
# consumers of one queue; the database lease decides who works on a job.
|
||||
worker:
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
command: ["python", "-m", "messenger.worker"]
|
||||
environment:
|
||||
<<: *pipeline-env
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_SHUTDOWN_SECONDS: ${WORKER_SHUTDOWN_SECONDS:-20}
|
||||
IMAGE_MAX_MEMORY_MB: ${IMAGE_MAX_MEMORY_MB:-768}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-600}
|
||||
CLEANUP_INTERVAL_SECONDS: ${CLEANUP_INTERVAL_SECONDS:-30}
|
||||
FAULT_POINTS: ${WORKER_FAULT_POINTS:-}
|
||||
secrets:
|
||||
- database_url
|
||||
- source: broker_url_worker
|
||||
target: broker_url
|
||||
- source: s3_worker_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_worker_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-m", "messenger.healthcheck", "--file", "/tmp/heartbeat", "--max-age", "30"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
stop_grace_period: 30s # > WORKER_SHUTDOWN_SECONDS (20 s) + release
|
||||
restart: unless-stopped
|
||||
networks: [data, storage, queue]
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
# WORKER_CONCURRENCY (2) child processes x IMAGE_MAX_MEMORY_MB (768) address space + worker.
|
||||
limits: {cpus: "${WORKER_CPUS:-2.0}", memory: "${WORKER_MEMORY:-1536m}"}
|
||||
memswap_limit: ${WORKER_MEMORY:-1536m}
|
||||
|
||||
# Track 2: read-only checker of the pipeline invariants (database in read-only transactions,
|
||||
# S3 auditor key: Read + List, no Write). On demand: docker compose run --rm auditor
|
||||
auditor:
|
||||
profiles: ["tools"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
entrypoint: ["python", "-m", "messenger.audit"]
|
||||
command: []
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
DB_TIMEOUT_SECONDS: ${AUDIT_DB_TIMEOUT_SECONDS:-30}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_STAGING_BUCKET: ${S3_STAGING_BUCKET:-messenger-staging}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_TIMEOUT_SECONDS: ${S3_TIMEOUT_SECONDS:-3}
|
||||
secrets:
|
||||
- database_url
|
||||
- source: s3_auditor_access_key_id
|
||||
target: s3_access_key_id
|
||||
- source: s3_auditor_secret_access_key
|
||||
target: s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [data, storage]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 256m}
|
||||
|
||||
# Operator CLI (replay of failed jobs, reprocess with a newer pipeline, failed registry):
|
||||
# docker compose run --rm jobctl list-failed
|
||||
# docker compose run --rm jobctl replay <attachment_id> --operator alice --reason "..."
|
||||
jobctl:
|
||||
profiles: ["tools"]
|
||||
image: ${API_IMAGE:-mtusi-messenger:lab5}
|
||||
entrypoint: ["python", "-m", "messenger.jobctl"]
|
||||
command: []
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
secrets: [database_url]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
networks: [data]
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128m}
|
||||
|
||||
networks:
|
||||
edge:
|
||||
data:
|
||||
internal: true # no route outside: stores talk only to the services above
|
||||
storage:
|
||||
internal: true # S3 API reachable only by api, s3-init, worker, auditor
|
||||
queue:
|
||||
internal: true # broker reachable only by relay and worker
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
valkeydata:
|
||||
s3data:
|
||||
rabbitmqdata:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
valkey_password:
|
||||
file: ./secrets/valkey_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_admin_access_key_id:
|
||||
file: ./secrets/s3_admin_access_key_id
|
||||
s3_admin_secret_access_key:
|
||||
file: ./secrets/s3_admin_secret_access_key
|
||||
s3_api_access_key_id:
|
||||
file: ./secrets/s3_api_access_key_id
|
||||
s3_api_secret_access_key:
|
||||
file: ./secrets/s3_api_secret_access_key
|
||||
s3_worker_access_key_id:
|
||||
file: ./secrets/s3_worker_access_key_id
|
||||
s3_worker_secret_access_key:
|
||||
file: ./secrets/s3_worker_secret_access_key
|
||||
s3_auditor_access_key_id:
|
||||
file: ./secrets/s3_auditor_access_key_id
|
||||
s3_auditor_secret_access_key:
|
||||
file: ./secrets/s3_auditor_secret_access_key
|
||||
rabbitmq_users:
|
||||
file: ./secrets/rabbitmq_users.json
|
||||
broker_url_relay:
|
||||
file: ./secrets/broker_url_relay
|
||||
broker_url_worker:
|
||||
file: ./secrets/broker_url_worker
|
||||
@@ -0,0 +1,57 @@
|
||||
# Модель данных lab5
|
||||
|
||||
Схема: [0001_init.sql](../migrations/0001_init.sql) (lab2), [0002_idempotency_keys.sql](../migrations/0002_idempotency_keys.sql) и [0003_post_message_function.sql](../migrations/0003_post_message_function.sql) (lab3), [0004_attachments.sql](../migrations/0004_attachments.sql) (lab4), [0005_pipeline.sql](../migrations/0005_pipeline.sql) (lab5: задания, outbox, попытки, варианты). Миграции применяет один раз отдельный сервис `migrate` до старта реплик API; миграции lab3–lab5 только добавляют объекты (0004 ослабляет `CHECK` длины текста, 0005 — `CHECK` статуса и ключа объекта), поэтому совместимы с работающей предыдущей версией. Байты вложений живут **только** в двух приватных bucket S3 (SeaweedFS): `messenger-staging` (оригинал до обработки) и `messenger-attachments` (опубликованный оригинал и варианты); в PostgreSQL — метаданные, задания и их история. Порядок записи и окна отказа — [PIPELINE.md](PIPELINE.md). Источник истины для пользователей, чатов, участников и сообщений — PostgreSQL 18. Для сессий источник истины — хранилище, выбранное через `SESSION_STORE`. По умолчанию это Valkey с AOF, его можно переключить на таблицу `sessions` в PostgreSQL.
|
||||
|
||||
## Таблицы и ограничения
|
||||
|
||||
| Таблица | Ключ | Ограничения | Зачем |
|
||||
| --- | --- | --- | --- |
|
||||
| `users` | `id uuid` | `UNIQUE (username)`, `CHECK` формата username, длина `display_name` | уникальность имени держит БД, а не код: гонку регистрации разрешает уникальный индекс |
|
||||
| `chats` | `id uuid` | FK `created_by → users`, длина `title` | `last_message_at` — «часы» чата для упорядочивания сообщений |
|
||||
| `chat_members` | `(chat_id, user_id)` | FK на `chats` и `users`, `UNIQUE (chat_id, position)` | `position` хранит порядок `member_ids`; `chat_created_at` — копия времени чата для keyset-списка чатов |
|
||||
| `messages` | `id uuid` | FK `(chat_id, sender_id) → chat_members`, длина `text` | отправитель обязан быть участником — это гарантирует сама БД |
|
||||
| `sessions` | `token_hash bytea` | FK `user_id → users`, длина хеша 32 байта | хранится SHA-256 токена; сам токен не хранится нигде |
|
||||
| `idempotency_keys` | `(user_id, chat_id, key)` | FK на `users`, `chats`; `message_id → messages` (DEFERRABLE, `UNIQUE`); `CHECK` формата ключа и длины SHA-256 | общая для всех реплик идемпотентность отправки: ключ → отпечаток тела → сообщение. Хранится ≥ `IDEMPOTENCY_TTL_SECONDS` (24 ч), очистка — пачками раз в 10 мин под advisory-замком |
|
||||
| `attachments` (lab4, lab5) | `id uuid` | FK `(chat_id, owner_id) → chat_members`; `UNIQUE object_key`, `UNIQUE staging_key` (`COLLATE "C"`); `CHECK` статуса `pending/queued/processing/ready/failed`, формата ключей, типа, размера, пикселей ≤ 40 млн; **`attachments_active_has_source`**: у `queued/processing` есть опубликованный оригинал или staging `present`; `ready` ⇒ `object_key` и `published_version`; `failed` ⇔ `error_code` + `failed_at` | метаданные оригинала. `staging_key` (`s/<128 бит>`) — копия в staging, `staging_state none/present/deleting/deleted`; `object_key` — опубликованный оригинал (`a/<id>/<token>/original`, у строк lab4 — `o/<hex>`). `published_version` — версия pipeline, чьи варианты видны, только растёт |
|
||||
| `jobs` (lab5) | `id uuid` = job_id | FK на `attachments`; `CHECK` состояния, вида (`upload/replay/reprocess/migrate`), `attempt ≤ max_attempts`, lease есть ⇔ `processing`, `finished_at` ⇔ конечное; частичный `UNIQUE (attachment_id, pipeline_version) WHERE state IN ('queued','processing')` | одна логическая работа «версия V для вложения A». `available_at` — backoff, `lease_token`/`lease_until`/`lease_owner` — аренда текущей попытки (fencing). Индексы: истекающие lease, queued, failed |
|
||||
| `job_attempts` (lab5) | `token uuid` | FK на `jobs`; `UNIQUE (job_id, attempt)`; `CHECK` исхода `running/succeeded/retry/failed/abandoned/superseded`; опубликованную попытку нельзя «почистить» | один claim worker. `planned_keys` — ключи всех объектов попытки, записаны ДО записи в S3: уборке не нужен List. Частичный индекс — кандидаты на уборку |
|
||||
| `attachment_variants` (lab5) | `(attachment_id, pipeline_version, name)` | FK на `attachments`, `job_attempts`; `UNIQUE object_key`, `CHECK` имени, типа, размера, длины SHA-256 | опубликованные варианты; первичный ключ — ключ эффекта контракта, дубль физически невозможен |
|
||||
| `outbox` (lab5) | `id bigint` identity | FK на `jobs`; `attempt ≥ 1` | намерение опубликовать сообщение; пишется в одной транзакции с изменением задания. Частичный индекс `WHERE published_at IS NULL` — работа relay |
|
||||
| `job_events` (lab5) | `id bigint` identity | FK на `jobs` | аудит: создание, claim, retry с задержкой, lease_expired, replay/reprocess (оператор и причина), результат, внедрённые отказы. Без секретов и имён файлов. View `failed_jobs` — реестр failed |
|
||||
| `message_attachments` (lab4) | `(message_id, position)` | FK на `messages` и `attachments`, `UNIQUE (message_id, attachment_id)`, индекс по `attachment_id` | целостность ссылок (вложение, на которое ссылается сообщение, удалить нельзя) и обратный поиск «используется ли вложение» |
|
||||
| `schema_migrations` | `version` | checksum файла | какие миграции применены; изменённую задним числом миграцию мигратор не применит |
|
||||
|
||||
**Почему `bytea` для текста.** Контракт разрешает символ U+0000, а тип `text` в PostgreSQL не может содержать байт NUL (`DataError`). Поэтому `display_name`, `title` и `text` хранятся как байты UTF-8. Длину в code points проверяет API, а `CHECK` в байтах — страховка (1 code point ≤ 4 байта).
|
||||
|
||||
## Индексы и запросы
|
||||
|
||||
- `messages_chat_keyset (chat_id, created_at, id)`. Первая страница — `WHERE chat_id=$1 ORDER BY created_at, id LIMIT n`, следующая — `WHERE chat_id=$1 AND (created_at, id) > ($2, $3) ORDER BY ... LIMIT n`. Сравнение строк `(created_at, id) > (...)` попадает в `Index Cond`: поиск начинается прямо с позиции cursor и читает только `n+1` строк. Запросов два, а не один с `($2 IS NULL OR ...)`: иначе при generic plan условие cursor уходит в `Filter`, и поздняя страница сканирует всю историю чата до неё.
|
||||
- `chat_members_user_keyset (user_id, chat_created_at, chat_id)` — список чатов пользователя тем же keyset-способом.
|
||||
- `users_username_key` — вход по username и уникальность.
|
||||
- `sessions_expires_at` — очистка истёкших сессий.
|
||||
|
||||
## Транзакционные границы
|
||||
|
||||
| Операция | Транзакция | Что гарантирует |
|
||||
| --- | --- | --- |
|
||||
| Регистрация | один `INSERT` | `UniqueViolation` → 409; ровно одна строка при гонке |
|
||||
| Создание чата | `INSERT chats` → `INSERT chat_members` (все участники одним запросом) → `COMMIT` | неизвестный участник вызывает FK-ошибку **после** вставки чата; откат удаляет и чат, и частичное членство |
|
||||
| Сообщение (lab3, с ключом) | проверка членства → `INSERT idempotency_keys … ON CONFLICT DO UPDATE … WHERE ключ старше 24 ч` → если вставлен: `UPDATE chats … RETURNING` → `INSERT messages`; если нет: прочитать сохранённое сообщение и сравнить отпечаток → `COMMIT`. По умолчанию всё это — одна SQL-функция `post_message` в autocommit (одно обращение к БД) | ключ, сообщение и «результат» коммитятся вместе или не коммитятся вовсе: «висящего» ключа нет. Конкурентный повтор ждёт на уникальном индексе завершения первой транзакции и затем видит её результат (READ COMMITTED, новый снимок на каждый оператор). Другое тело → 409, тот же — тот же Message |
|
||||
| Сообщение с `attachment_ids` (lab4, lab5) | функция `post_message_v3` (0005): проверка членства → проверка вложений (этого чата, не `pending`; `failed`/карантин → 409; `FOR SHARE`) → только если всё хорошо — захват ключа → `UPDATE chats` → `INSERT messages (…, attachment_ids)` → `INSERT message_attachments` | чужое/неизвестное/незавершённое вложение — 404, в карантине — 409, ключ при этом не «сгорает». Порядок `attachment_ids` хранится массивом в строке сообщения: страница истории остаётся одним диапазонным сканом индекса |
|
||||
| Загрузка вложения (lab5) | `INSERT attachments … 'pending'` (autocommit) → S3 PUT в staging (Content-MD5) → **одна транзакция**: `UPDATE … SET status='queued' WHERE status='pending'` + `INSERT jobs` + `INSERT outbox` + `INSERT job_events` → 202 | 202 только когда оригинал в staging и задание с outbox зафиксированы. Сбой до COMMIT оставляет невидимый `pending`, его убирает janitor. Компенсации в обработчике нет |
|
||||
| Claim (worker) | `SELECT jobs … FOR UPDATE` → решение (no-op / superseded / работа) → `SELECT attachments … FOR UPDATE` → `UPDATE jobs` (processing, attempt+1, lease, token) + `INSERT job_attempts` + `UPDATE attachments` (processing) + событие | дубль доставки видит конечное состояние или чужой lease и ничего не делает |
|
||||
| Результат (worker) | `SELECT jobs … FOR UPDATE` (проверка токена) → `SELECT attachments … FOR UPDATE` → если версия не новее опубликованной: `superseded`; иначе `INSERT attachment_variants` ×3 + `UPDATE attachments` (ready, published_version, object_key) + `UPDATE jobs/job_attempts` + событие → потом ack | результат публикуется атомарно и только владельцем текущей аренды; версия только растёт |
|
||||
| Retry / failed (worker), lease (relay) | под блокировкой задания: попытка `retry`/`failed`/`abandoned`; задание `queued` с `available_at` + новая строка outbox, или `failed` (реестр) | повтор идёт тем же путём outbox; бесконечного немого retry нет |
|
||||
| Replay / reprocess (jobctl), уборка staging | `SELECT attachments … FOR UPDATE` (одна и та же блокировка) → проверки → новое задание + outbox, либо `staging_state='deleting'` | уборка и replay сериализованы строкой вложения |
|
||||
| Порядок блокировок | везде, где нужны обе строки: сначала `jobs`, потом `attachments` | нет взаимных ожиданий в разном порядке |
|
||||
| Сообщение (lab2-часть) | проверка членства → `UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + 1 мкс) RETURNING` → `INSERT messages` → `COMMIT` | строка чата заблокирована до конца транзакции, поэтому писатели одного чата идут по очереди. Время сообщений строго растёт **в порядке коммитов**: страница по cursor не пропустит сообщение, закоммиченное «задним числом» |
|
||||
| Вход | `SELECT` пользователя, проверка Argon2id, запись сессии | неверный пароль ничего не пишет |
|
||||
| Logout | `DELETE` сессии (или `DEL` в Valkey) | повтор с тем же токеном — 401 |
|
||||
|
||||
Чтение истории не требует snapshot между HTTP-запросами (так в контракте): новое сообщение появится на следующей странице. Список чатов сортируется по `chats.created_at = clock_timestamp()` без блокировки. Два чата, создаваемые одновременно, могут закоммититься не в порядке времени. Тогда у обхода, который уже начался, такой чат может оказаться «до» cursor; новый обход его увидит. Чаты создаются редко, поэтому мы не сериализуем их глобальным замком; это ограничение описано в REPORT.md.
|
||||
|
||||
## Сессии
|
||||
|
||||
- Токен: `secrets.token_urlsafe(32)` — 256 бит случайности, в ответе 43 символа. В хранилище лежит только SHA-256 токена. Для случайного 256-битного значения быстрого хеша достаточно, медленный KDF нужен только для низкоэнтропийных паролей.
|
||||
- `SESSION_STORE=valkey` (по умолчанию): ключ `session:<sha256-hex>`, значение `{user_id, expires_at}`. Срок жизни задаётся абсолютным `PXAT expires_at`, поэтому рестарт его не продлевает. Включён `appendonly yes` + `appendfsync always`: подтверждённые вход и logout сохраняются до ответа клиенту. Valkey — **единственное** хранилище сессий, это не кэш над PostgreSQL. Поэтому нет второй копии, которая могла бы «оживить» отозванный токен после отказа. Если Valkey недоступен, проверить сессию нельзя: защищённые запросы и logout получают 503, анонимный доступ не открывается.
|
||||
- `SESSION_STORE=postgres`: таблица `sessions`, проверка `expires_at > now()` в запросе.
|
||||
@@ -0,0 +1,48 @@
|
||||
# Ответы на вопросы защиты ЛР 5
|
||||
|
||||
Подробности и схемы: [PIPELINE.md](PIPELINE.md). Тесты: [test_pipeline_flow.py](../student-tests/test_pipeline_flow.py), [test_consistency.py](../student-tests/test_consistency.py), трек 2: [bench/pipeline_faults.py](../bench/pipeline_faults.py).
|
||||
|
||||
## Чем publisher confirm отличается от consumer ack?
|
||||
|
||||
- **Publisher confirm** — ответ брокера **отправителю** (у нас relay): «сообщение принято и записано в quorum-очередь на диск». Без confirm relay не знает, дошло ли сообщение, поэтому строку outbox он помечает `published_at` только после confirm. `mandatory=True`: если сообщение некуда положить, брокер вернёт его, и это будет ошибка, а не тихая потеря.
|
||||
- **Consumer ack** — ответ **потребителя** (worker) брокеру: «можно удалить сообщение, работа сделана». Worker шлёт ack только после COMMIT результата в PostgreSQL. Если упасть до ack, брокер отдаст сообщение снова.
|
||||
- Это две разные границы. Confirm говорит «намерение надёжно лежит в очереди», ack — «эффект надёжно сохранён». Между ними могут пройти часы (worker остановлен), и сообщение всё это время лежит в quorum log на volume `rabbitmqdata`. Тест: worker остановлен → upload 202 → relay публикует (confirm) → рестарт брокера → сообщение на месте → worker стартует → ack после ready.
|
||||
|
||||
## Что именно уже сохранено в момент 202?
|
||||
|
||||
1. **Оригинал** — полностью и уже проверенный (лимиты, PNG/JPEG-валидаторы, 413/415 решаются до записи), в приватном bucket `messenger-staging`. S3 подтвердил PUT с Content-MD5.
|
||||
2. **Метаданные + задание + строка outbox** — одной транзакцией PostgreSQL: вложение `queued`, `jobs` (`queued`, attempt 0), `outbox`.
|
||||
|
||||
Чего ещё нет: варианта, опубликованного оригинала, сообщения в брокере (его отправит relay). После 202 можно убить API — ничего не пропадёт. Если брокер лежит, upload всё равно 202: API с брокером вообще не связан. Сбой **до** COMMIT задания (окно «после staging») 202 не даёт: клиент получает 503, строка `pending` невидима, janitor убирает staging и строку (test_consistency).
|
||||
|
||||
## Почему повторная доставка нормальна и где ключ идемпотентности job?
|
||||
|
||||
- Гарантия at-least-once неизбежна: relay может упасть после confirm, но до отметки в outbox; worker может упасть после COMMIT, но до ack; соединение рвётся. Сообщение придёт ещё раз — это не ошибка, а цена того, что мы его не теряем.
|
||||
- **Ключ идемпотентности задания — `jobs.id` (job_id) в PostgreSQL.** Каждая доставка начинается с `claim` под `SELECT … FOR UPDATE` строки задания: `succeeded` → ack без работы; `processing` с живым lease → ack (его судьбу решит reaper); backoff не истёк → ack. Попадание в работу возможно только из `queued`.
|
||||
- **Ключ эффекта — `(attachment_id, pipeline_version, variant)`**: первичный ключ `attachment_variants`. Объекты пишутся под ключами своей попытки (`a/<id>/<token>/thumbnail`); опубликовано ровно то, на что ссылается один COMMIT. Дубль не создаёт второго набора, а мусор прерванных попыток удаляет уборка.
|
||||
- Это не exactly-once доставка: работа может выполниться дважды (зависший worker досчитает). Exactly-once только у видимого результата.
|
||||
|
||||
## Как отличить зависший worker от долго работающего и не запустить два неконтролируемых дубля?
|
||||
|
||||
- **Lease с heartbeat.** Claim даёт попытке `lease_until = now + 30 s` и случайный `lease_token`. Живой worker продлевает lease каждые 10 с (`UPDATE … WHERE lease_token = мой`). Долго работающий продлевает, зависший — нет.
|
||||
- **Потолок времени.** Даже живой heartbeat не продлевает lease дольше `JOB_TIMEOUT_SECONDS` (60 с); дочерний процесс с Pillow в этот момент убивается.
|
||||
- **Reaper** (в relay, раз в 5 с) возвращает задания с истёкшим lease в `queued` через outbox. Попытка помечается `abandoned`, токен обнуляется.
|
||||
- **Fencing токеном.** Второй исполнитель может появиться, но опубликовать результат может только владелец текущего токена: COMMIT результата, retry, heartbeat — все с условием `lease_token = мой`. Старая попытка, если оживёт, получит `fenced` и ничего не изменит; её объекты лежат под её токеном, их удалит уборка.
|
||||
- **Сторона брокера.** AMQP heartbeat 10 с: соединение замороженного клиента закрывается, его неподтверждённые сообщения возвращаются в очередь. `consumer_timeout` 5 мин — для подключённого, но не отвечающего ack.
|
||||
- Тест: два worker, `docker pause` того, кто держит lease → reaper → второй worker публикует → `unpause` → опубликованный результат не меняется, попытки `[abandoned, succeeded]`.
|
||||
|
||||
## Частые уточняющие вопросы
|
||||
|
||||
**Почему RabbitMQ, а не Valkey Streams?** Термины README (confirm, ack, prefetch, DLX) — это его родные механизмы; quorum queue пишет на диск до confirm. Брокер отделён от хранилища сессий, отказ одного не задевает другое. Retry, lease и реестр failed всё равно в PostgreSQL, поэтому сложной логики брокера (TTL-очередей) нет.
|
||||
|
||||
**Зачем outbox, а не «COMMIT, потом publish в API»?** Между COMMIT и publish процесс может умереть — задание есть, сообщения нет. Outbox пишется в той же транзакции, relay дошлёт. Проверено остановкой брокера и relay.
|
||||
|
||||
**Почему staging в отдельном bucket?** У ключей разные права: API может только **писать** в staging и **читать** конечный bucket; worker читает/пишет оба; никто, кроме auditor, не может List. Это проверяет `s3-init` при каждом старте (`api: read staging → refused`, `api: write final → refused`).
|
||||
|
||||
**Может ли уборка удалить исходник принятой работы?** Нет, и на трёх уровнях: (1) кандидаты — только `ready` или `failed` старше retention; (2) решение повторяется под блокировкой строки в момент пометки `deleting`, которую берёт и replay; (3) CHECK `attachments_active_has_source` в БД запрещает `queued/processing` без источника. Тесты: retention 0 при `queued` (relay остановлен) — staging цел, а ручной UPDATE отвергнут БД; гонка «уборка прочитала кандидатов → replay → уборка продолжила» воспроизведена паузой `pause_cleanup`.
|
||||
|
||||
**Replay = повтор задания?** Нет. `jobctl replay` разрешён только для `failed` с живым источником. Он создаёт **новое** задание `replay_of=<старое>` и пишет оператора и причину в аудит. Для `ready` — отказ `already_ready`, ничего не создаётся.
|
||||
|
||||
**Чем постоянная ошибка отличается от временной?** Постоянная не исправится повтором с тем же входом: `invalid_image`, `image_too_large`, `source_missing`, `source_corrupt` → сразу `failed`. Временная — S3/БД недоступны, таймаут, процесс убит → повтор с backoff (full jitter), не более 5 claim-ов, затем `retry_exhausted`.
|
||||
|
||||
**Почему 415 в API, а не в worker?** Контракт требует 415 на upload. Преподаватель ловил «202 → failed/invalid_image» в чужих PR. Проверки lab4 (свой PNG-валидатор с проверкой фильтра строк и zlib-потока, `djpeg -strict`) остались синхронными; тесты включают все случаи из рецензий.
|
||||
@@ -0,0 +1,128 @@
|
||||
# Pipeline обработки вложений (lab5)
|
||||
|
||||
Схема: [migrations/0005_pipeline.sql](../migrations/0005_pipeline.sql). Код: API — [attachments.py](../src/messenger/attachments.py), переходы заданий — [jobs.py](../src/messenger/jobs.py), relay — [relay.py](../src/messenger/relay.py), worker — [worker.py](../src/messenger/worker.py), картинки — [imaging.py](../src/messenger/imaging.py), версии — [pipeline.py](../src/messenger/pipeline.py), оператор — [jobctl.py](../src/messenger/jobctl.py), аудит — [audit.py](../src/messenger/audit.py).
|
||||
|
||||
## Кто что делает
|
||||
|
||||
```text
|
||||
клиент ──POST──▶ LB ──▶ API ──(1) INSERT attachments 'pending' ──▶ PostgreSQL
|
||||
│ (2) PUT оригинал в messenger-staging ──▶ S3 (staging)
|
||||
│ (3) ОДНА транзакция: 'queued' + jobs + outbox ──▶ PostgreSQL
|
||||
└─ 202 queued
|
||||
relay: outbox ─(4) publish + publisher confirm─▶ RabbitMQ (quorum queue) ─(5) deliver─▶ worker
|
||||
worker: (6) claim: lease + token попытки ─▶ (7) читает источник, сверяет SHA-256 ─▶ (8) варианты
|
||||
в дочернем процессе ─▶ (9) PUT a/<id>/<token>/* в messenger-attachments ─▶
|
||||
(10) ОДНА транзакция: варианты + attachment 'ready' (с проверкой токена) ─▶ (11) ack ─▶
|
||||
(12) staging 'deleting' → S3 DELETE → 'deleted'
|
||||
```
|
||||
|
||||
- **Источник истины — PostgreSQL**: состояние задания, попытки, lease, время следующей попытки (`available_at`), реестр failed, аудит (`job_events`). Брокер только доставляет `job_id`.
|
||||
- **API не знает брокер** (нет даже сети `queue`). Брокер упал — upload всё равно 202, задания копятся в outbox.
|
||||
- В сообщении только поля [jobs.schema.json](../contracts/jobs.schema.json): `schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at`. Никаких байтов, URL, ключей S3, имени файла. Worker берёт `content_type`, размер, SHA-256 и ключ источника из БД.
|
||||
|
||||
## Что сохранено в момент 202
|
||||
|
||||
1. Полный, **проверенный** оригинал лежит в приватном bucket `messenger-staging`: S3 подтвердил PUT и Content-MD5. Проверки 413/415 делаются до этого, синхронно, теми же валидаторами, что в lab4 (PNG — свой потоковый, JPEG — `djpeg -strict`).
|
||||
2. Строка `attachments` в статусе `queued`, строка `jobs` (`queued`, attempt 0) и строка `outbox` — **в одной транзакции**, закоммичены (fsync PostgreSQL).
|
||||
|
||||
Значит, после 202 можно убить API, relay, broker и worker: работа восстановится из этих двух записей. Честная граница: staging — это уже S3, а не «временная папка». На одном Docker-хосте «устойчиво» значит «переживает рестарт процессов и контейнеров с volumes», а не потерю диска.
|
||||
|
||||
## Состояния
|
||||
|
||||
Вложение (`attachments.status`):
|
||||
|
||||
| Статус | Видно клиенту | Как попадает | Как выходит |
|
||||
| --- | --- | --- | --- |
|
||||
| `pending` | нет (404) | INSERT до PUT в staging | → `queued` (COMMIT задания) или удаляется janitor через `PENDING_GRACE_SECONDS` |
|
||||
| `queued` | да, content 409 | COMMIT задания; retry; истёк lease; replay | → `processing` при claim |
|
||||
| `processing` | да, content 409 | claim worker | → `ready` / `queued` (retry, lease) / `failed` |
|
||||
| `ready` | да, варианты | COMMIT результата | никогда не возвращается назад. Reprocess новой версии не меняет статус |
|
||||
| `failed` | да, `error_code`, content 409 | постоянная ошибка или попытки кончились | → `queued` только через `jobctl replay` |
|
||||
|
||||
Задание (`jobs.state`): `queued → processing → succeeded | failed | superseded`, `processing → queued` (retry, lease, graceful stop). `superseded` — старая версия pipeline, когда опубликована более новая.
|
||||
|
||||
Ограничения в самой БД (CHECK) не дают коду ошибиться:
|
||||
- `attachments_active_has_source`: у `queued/processing` есть источник — опубликованный оригинал или staging в состоянии `present`. **Уборка физически не может пометить staging принятой работы**;
|
||||
- `jobs_lease`: lease есть ровно тогда, когда `processing`;
|
||||
- `jobs_one_active`: одно активное задание на `(attachment, pipeline_version)`;
|
||||
- PK `attachment_variants (attachment_id, pipeline_version, name)` — ключ эффекта из контракта.
|
||||
|
||||
## Окна отказа (contracts/attachment-states.md)
|
||||
|
||||
| Окно | Что остаётся | Кто восстанавливает | Проверка |
|
||||
| --- | --- | --- | --- |
|
||||
| после staging, до записи задания | `pending` + объект staging, задания нет, 503 клиенту | janitor API: объект, потом строка (после grace) | `API_FAULT_POINTS=after_s3_put`, test_consistency |
|
||||
| после COMMIT, до publish | строка outbox без `published_at` | relay при следующем проходе; никто не создаёт задание вручную | broker/relay остановлены, test_pipeline_flow |
|
||||
| после publish, до записи confirm | сообщение в очереди + неотмеченная строка outbox | relay публикует ещё раз → дубль → no-op | `RELAY_FAULT_POINTS=crash_after_publish` |
|
||||
| после записи S3, до COMMIT результата | объекты под ключами попытки 1, не опубликованы | lease истекает → reaper → попытка 2 со своими ключами; уборка удаляет ключи попытки 1 | `WORKER_FAULT_POINTS=crash_after_s3_write` |
|
||||
| после COMMIT результата, до ack | `ready` | redelivery → claim видит `succeeded` → ack без работы | `WORKER_FAULT_POINTS=crash_after_commit` |
|
||||
| worker умер/завис в processing | lease без продления | reaper (в relay, раз в 5 с) → `queued` + outbox; commit старой попытки отвергается токеном | `crash_after_claim`, `docker pause` |
|
||||
|
||||
## Publisher confirm ≠ consumer ack
|
||||
|
||||
- **Publisher confirm** (relay ← broker): «брокер принял сообщение и записал его в quorum log». Это граница «намерение передано в очередь»; после неё relay ставит `published_at`.
|
||||
- **Consumer ack** (worker → broker): «результат работы устойчиво зафиксирован». Worker шлёт ack только **после COMMIT** результата (или после COMMIT решения «retry/failed/no-op»).
|
||||
- Между ними может пройти сколько угодно времени и случиться любой сбой. Confirm без ack = сообщение ждёт в очереди; потеря ack = повторная доставка.
|
||||
|
||||
## Идемпотентность и повторная доставка
|
||||
|
||||
Доставка at-least-once: дубль может появиться из relay (crash до отметки), из брокера (нет ack из-за падения worker, разрыв соединения, consumer_timeout), от оператора. Поэтому каждая доставка начинается с `claim` под блокировкой строки `jobs`:
|
||||
|
||||
| Состояние задания | Решение | ack |
|
||||
| --- | --- | --- |
|
||||
| `succeeded/failed/superseded` | no-op (`noop_finished`) | да |
|
||||
| `processing` (lease держит другой) | no-op (`noop_leased`), его смерть обработает reaper | да |
|
||||
| `queued`, `available_at` в будущем | no-op (`noop_not_due`), outbox опубликует вовремя | да |
|
||||
| `queued`, опубликована версия ≥ своей | `superseded` | да |
|
||||
| `queued`, пора | claim: attempt+1, новый `lease_token`, строка `job_attempts` с заранее известными ключами | после результата |
|
||||
|
||||
Ключ идемпотентности задания — `job_id` (строка в БД), ключ эффекта — `(attachment_id, pipeline_version, variant)`. Объекты пишутся под ключами `a/<attachment>/<token>/<name>`: у каждой попытки свои. Опубликовано ровно то, на что ссылается COMMIT результата. **Fencing**: COMMIT проходит, только если `jobs.lease_token` всё ещё равен токену попытки. Reaper обнуляет токен, поэтому «проснувшаяся» старая попытка ничего не опубликует.
|
||||
|
||||
Это не exactly-once **доставка**. Сообщение может прийти несколько раз, а работа выполниться дважды (например, зависший worker досчитает варианты). Exactly-once — только **видимый эффект**: одна опубликованная запись на ключ эффекта, монотонная версия, мусор попыток убирается.
|
||||
|
||||
## Зависший vs долго работающий worker
|
||||
|
||||
- Долго работающий продлевает lease каждые `JOB_LEASE_SECONDS/3` (heartbeat в БД, `WHERE lease_token = мой`).
|
||||
- Зависший (`docker pause`, deadlock, потеря сети к БД) не продлевает; через `JOB_LEASE_SECONDS` reaper возвращает задание.
|
||||
- Worker, у которого жив heartbeat, но зависла обработка, ограничен `JOB_TIMEOUT_SECONDS`: дочерний процесс убивается, heartbeat перестаёт продлевать lease после этого срока.
|
||||
- Если heartbeat узнал, что lease потерян, попытка отменяется сама. Даже если не отменилась — её COMMIT отвергнет fencing.
|
||||
- Брокер со своей стороны: AMQP heartbeat 10 с закрывает соединение замороженного клиента и возвращает его неподтверждённые сообщения; `consumer_timeout` 5 мин — для подключённого, но не отвечающего ack.
|
||||
|
||||
## Retry, failed, replay
|
||||
|
||||
- `JOB_MAX_ATTEMPTS=5` claim-ов на задание. Backoff — full jitter: `uniform(0, min(JOB_RETRY_MAX_SECONDS, JOB_RETRY_BASE_SECONDS·2^(n−1)))`. Задержка хранится в `jobs.available_at` и строке outbox — повтор проходит тем же путём outbox → broker, без TTL-очередей.
|
||||
- Постоянные ошибки (сразу `failed`): `invalid_image`, `image_too_large`, `source_missing`, `source_corrupt`. Временные: `storage_unavailable`, `processing_timeout`, `processing_killed`, `processing_failed`. Когда попытки кончились — `retry_exhausted`.
|
||||
- Реестр failed — view `failed_jobs` (`jobctl list-failed`). DLQ RabbitMQ (`messenger.attachment-jobs.dlq`) — для «ядовитых» сообщений: не по схеме или превысили `x-delivery-limit=20`.
|
||||
- Страховка от потерянного сообщения: `queued`-задание без claim и без публикаций дольше `QUEUED_REPUBLISH_SECONDS` получает новую строку outbox.
|
||||
- `jobctl replay <id>`: только для `failed` с живым источником, создаёт **новое** задание `kind=replay, replay_of=<failed>`, пишет оператора и причину в `job_events`. Для `ready` — отказ `already_ready`, ничего не создаётся.
|
||||
|
||||
## Уборка (worker), всегда под блокировкой строки
|
||||
|
||||
- staging `ready` — сразу после COMMIT результата; staging `failed` — через `STAGING_RETENTION_SECONDS`. Порядок: под `UPDATE … WHERE <условие заново>` пометить `deleting` → S3 DELETE → `deleted`. Replay берёт ту же блокировку: либо replay первым (строка уже `queued`, уборка её пропустит), либо уборка первой (replay видит `deleting` и отказывает `source_missing`).
|
||||
- Объекты попыток, которые не опубликовали результат (`retry/failed/abandoned/superseded`): удаляются по `planned_keys` через `ORPHAN_GRACE_SECONDS`. Перед этим проверяется, что токен не текущий и ни один ключ не опубликован. Новые результаты (replay, другая попытка) лежат под другими ключами — устаревший проход не может их удалить.
|
||||
- `pending` без задания — janitor API (как в lab4).
|
||||
- Опубликованные варианты старых версий pipeline сохраняются (ограничение, см. REPORT).
|
||||
|
||||
## Версии pipeline (трек 2)
|
||||
|
||||
Версия фиксируется в задании при создании (`PIPELINE_VERSION` для upload, `jobctl reprocess --pipeline N` для готового вложения). Версии 1/2/3 отличаются параметрами кодеков, геометрия — по контракту. Публикация: только если `N > published_version`; иначе `superseded`. Reprocess неготового вложения запрещён: его единственный источник — staging текущего задания.
|
||||
|
||||
## Лимиты worker
|
||||
|
||||
| Что | Чем |
|
||||
| --- | --- |
|
||||
| параллельность | prefetch = `WORKER_CONCURRENCY` (2) + семафор на дочерние процессы |
|
||||
| bytes | размер из БД ≤ `MAX_UPLOAD_BYTES` проверяется до скачивания; длина и SHA-256 скачанного = БД |
|
||||
| пиксели | `width×height` из заголовка ≤ `MAX_IMAGE_PIXELS` до декодирования; JPEG декодируется с DCT-масштабом (`draft`) |
|
||||
| память | `RLIMIT_AS = IMAGE_MAX_MEMORY_MB` (768) у дочернего процесса; контейнер 1536m без swap |
|
||||
| время | `JOB_TIMEOUT_SECONDS` (60): дочерний процесс убивается, lease перестаёт продлеваться |
|
||||
|
||||
## Варианты
|
||||
|
||||
| Вариант | Формат | Геометрия | Параметры v1 |
|
||||
| --- | --- | --- | --- |
|
||||
| thumbnail | JPEG baseline 4:2:0 | вписан в 256×256, без увеличения, округление вниз, ≥1 px | q=85, optimize |
|
||||
| crop | JPEG | центральный квадрат → 128×128 (маленькие увеличиваются) | q=85 |
|
||||
| optimized | WebP lossy | вписан в 1280×1280, без увеличения, alpha сохраняется | q=80, method=4 |
|
||||
|
||||
EXIF orientation применяется к пикселям; в варианты не копируются EXIF, XMP, ICC, комментарии. Пиксели считаются sRGB. Alpha при ресайзе — premultiplied (без цветных ореолов), для JPEG композится на белый. 16-битный серый масштабируется (`/257`), а не обрезается. Оригинал публикуется побайтно без изменений. Уменьшение размера маленьких файлов не обещается.
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: Боданюк Алексей, БВТ2302
|
||||
Целевая оценка (3/4/5): 5
|
||||
Трек 5, если заявлен: трек 2 «Доказанная идемпотентность pipeline» (versioned pipeline + автоматический fault injection)
|
||||
|
||||
Что реализовано и что перенесено: реализация lab4 перенесена отдельным коммитом без изменений (`5112be8`). Добавлено:
|
||||
- upload 201 → 202: оригинал в приватном staging-bucket + одна транзакция `queued` + задание + outbox;
|
||||
- RabbitMQ 4.3.7 (quorum queue, publisher confirms, DLX/DLQ, пользователи с минимальными правами, без management UI);
|
||||
- relay (transactional outbox, reaper истёкших lease);
|
||||
- отдельный worker: lease с heartbeat и fencing по токену попытки, варианты thumbnail/crop/WebP в дочернем процессе с лимитами памяти, времени и пикселей, ack только после COMMIT;
|
||||
- retry с full jitter, реестр failed и `jobctl replay/reprocess`;
|
||||
- уборка staging и мусора попыток под блокировкой строки;
|
||||
- read-only аудит инвариантов.
|
||||
|
||||
Reconciler lab4 убран, причина — в REPORT. Подробно — REPORT.md, docs/PIPELINE.md, docs/DEFENSE.md.
|
||||
|
||||
Команды запуска из чистого clone: REPORT.md → «Запуск из чистого clone»: `python3 scripts/init_secrets.py`, `cp .env.example .env`, `docker compose --env-file .env up -d --build --wait`. Windows — docs/WINDOWS.md.
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md: REPORT.md → «Доказательства критериев»; вывод прогона из чистого clone — в evidence/: make check/validate/test, persistence, failover, сценарий защиты, свои тесты, матрица трека 2, аудит.
|
||||
|
||||
Ограничения / известные ошибки: REPORT.md → «Ограничения и следующие шаги»: один Docker-хост, опубликованные варианты старых версий не удаляются, fsync SeaweedFS.
|
||||
|
||||
Источники и использование ИИ: REPORT.md, шапка. Код, тесты и отчёт написаны с помощью Claude Code; все прогоны реально выполнены.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
🤖 Generated with [Claude Code](https://claude.com/claude-code)
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
# Запуск на ноутбуке с Windows и сценарий защиты ЛР 5
|
||||
|
||||
Команды — в **PowerShell** из папки проекта (где лежит `compose.yaml`). Переменные `$env:...` действуют до закрытия окна.
|
||||
|
||||
## 1. Что установить один раз
|
||||
|
||||
| Программа | Зачем | Важно |
|
||||
| --- | --- | --- |
|
||||
| [Git for Windows](https://git-scm.com/download/win) | clone и Git Bash | путь по умолчанию `C:\Program Files\Git`: оттуда `scripts\*.cmd` запускают `.sh`-хуки |
|
||||
| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | PostgreSQL, Valkey, SeaweedFS (S3), RabbitMQ, HAProxy, API, relay, worker | backend WSL 2; в Settings → Resources ≥ 4 CPU и ≥ 8 ГБ RAM (сумма лимитов контейнеров ~6,5 ГБ) |
|
||||
| [Python 3.12+](https://www.python.org/downloads/windows/) | secrets, публичные и свои тесты, сценарии | галочка **Add python.exe to PATH** |
|
||||
| GNU make (необязательно) | `make check/test/...` | `winget install ezwinports.make`, затем **новое** окно; в каждом окне `$env:Path += ";C:\Program Files\Git\usr\bin"` (рецептам нужен `sh`) |
|
||||
|
||||
## 2. Развернуть проект (заранее, нужен интернет)
|
||||
|
||||
```powershell
|
||||
git clone https://gitea.privcloud.tech/wfa/mtusi2026lab5 mtusi2026lab5
|
||||
cd mtusi2026lab5
|
||||
git checkout solution/bvt2302-bodanyuk
|
||||
python scripts\init_secrets.py # пароли БД/Valkey, 4 пары ключей S3, пользователи RabbitMQ -> .\secrets
|
||||
Copy-Item .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
```
|
||||
|
||||
Первый запуск качает образы (~1,2 ГБ) и собирает образ приложения — 3–10 минут. Команда завершается, когда postgres, valkey, s3, broker, обе реплики api, lb, relay и worker в `healthy`, а `migrate` и `s3-init` вышли с кодом 0. Проверки прав S3: `docker compose logs s3-init`.
|
||||
|
||||
Окружение для своих тестов (один раз):
|
||||
|
||||
```powershell
|
||||
python -m venv .venv
|
||||
.venv\Scripts\python -m pip install --require-hashes --no-deps -r requirements.txt
|
||||
.venv\Scripts\python -m pip install -r requirements-test.txt -r requirements-dev.txt
|
||||
```
|
||||
|
||||
Стенды ЛР 3/4 занимают порт 8080 — остановить: `docker compose -p mtusi-lab4 stop` (данные сохранятся).
|
||||
|
||||
## 3. Перед защитой
|
||||
|
||||
```powershell
|
||||
$env:PYTHONUTF8 = "1"; $env:Path += ";C:\Program Files\Git\usr\bin"
|
||||
docker compose up -d --wait
|
||||
make check test PYTHON=python # 202 -> ready, настоящий JPEG 256x160
|
||||
make jobs # задания по состояниям, хвост outbox
|
||||
```
|
||||
|
||||
## 4. Сценарий защиты README — по шагам
|
||||
|
||||
**Шаг 1. `make test`**: API возвращает 202, GET доходит до ready, оригинал и настоящий JPEG thumbnail проверяются (см. раздел 3).
|
||||
|
||||
**Шаги 2–3. Worker остановлен → 202 → content 409 → рестарт API и брокера с volumes → worker → ready.** Скрипт печатает каждый шаг, ID вложения и сообщения, код ответа; токены не печатает:
|
||||
|
||||
```powershell
|
||||
.venv\Scripts\python scripts\demo_pipeline.py
|
||||
```
|
||||
|
||||
Что он делает:
|
||||
1. `docker compose stop worker`;
|
||||
2. upload → **202 queued**; `GET /content` и `?variant=thumbnail` → **409**; обычное сообщение → 201; сообщение с этим вложением → 201;
|
||||
3. показывает строку outbox (`published_at` заполнен — relay получил confirm) и глубину очереди `messenger.attachment-jobs` (1);
|
||||
4. `docker compose restart api broker` (volumes не трогаются), ждёт готовности; очередь снова показывает 1, вложение по-прежнему `queued`;
|
||||
5. `docker compose start worker` → **ready**, скачивает thumbnail (JPEG 256×160), сверяет SHA-256 и размеры;
|
||||
6. печатает журнал задания (`jobctl show`): `created → claimed → succeeded`, одна попытка.
|
||||
|
||||
То же автоматически: `.venv\Scripts\python -m pytest student-tests\test_pipeline_flow.py -k worker_stopped -v`.
|
||||
|
||||
**Шаг 4. Уровень 4 вживую.**
|
||||
|
||||
```powershell
|
||||
# crash после записи в S3 до ack/COMMIT; разрыв COMMIT/publish; publish/confirm:
|
||||
.venv\Scripts\python -m pytest student-tests\test_pipeline_flow.py -k "crash or broker_down or relay" -v
|
||||
# retry с backoff, retry_exhausted -> реестр failed -> replay; replay готового -> отказ:
|
||||
.venv\Scripts\python -m pytest student-tests\test_pipeline_flow.py -k "retry or permanent" -v
|
||||
make failed # реестр failed
|
||||
docker compose run --rm --no-deps jobctl replay <attachment_id> --operator alice --reason "причина устранена"
|
||||
docker compose run --rm --no-deps jobctl show <attachment_id> # задания, попытки, аудит
|
||||
# зависший worker (docker pause) теряет lease, второй worker публикует, первый не может:
|
||||
.venv\Scripts\python -m pytest student-tests\test_pipeline_flow.py -k hung -v -s
|
||||
```
|
||||
|
||||
**Трек 2 вживую.**
|
||||
|
||||
```powershell
|
||||
make audit # инварианты I1-I7 по всей БД и обоим bucket (read-only)
|
||||
.venv\Scripts\python bench\pipeline_faults.py --output evidence\track2-demo --uploads 3 # ~15 мин
|
||||
```
|
||||
|
||||
Готовые результаты: [evidence/track2/summary.txt](../evidence/track2/summary.txt).
|
||||
|
||||
**Публичные проверки сохранности и failover.**
|
||||
|
||||
```powershell
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.cmd PYTHON=python # restart всего стенда с volumes
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.cmd PYTHON=python
|
||||
make restore PYTHON=python
|
||||
```
|
||||
|
||||
**Все свои тесты** (~40 мин): `.venv\Scripts\python -m pytest student-tests -v`.
|
||||
|
||||
## 5. Если что-то пошло не так
|
||||
|
||||
| Симптом | Что сделать |
|
||||
| --- | --- |
|
||||
| `error during connect … dockerDesktopLinuxEngine` | Docker Desktop не запущен — запустить и подождать минуту |
|
||||
| `port is already allocated` на 8080/8404 | стенд lab3/lab4 (`docker compose -p mtusi-lab4 stop`) или в `.env` `APP_PUBLISH_PORT=8081`, `LB_STATS_PORT=8405` |
|
||||
| `secrets/... not found` | не выполнен `python scripts\init_secrets.py` |
|
||||
| `s3-init` exit 1 | `docker compose logs s3-init`: какая проверка доступа не прошла |
|
||||
| broker долго `starting` | первый старт RabbitMQ ~20 с; `docker compose logs broker` |
|
||||
| вложение висит в `queued` | `make jobs`: если `outbox_due > 0` — смотреть `docker compose logs relay`; если очередь не пуста — `docker compose logs worker` |
|
||||
| `UnicodeDecodeError` в `tests\*.py` | не задан `$env:PYTHONUTF8 = "1"` |
|
||||
| локальный `.env` устарел | сравнить с `.env.example`: значения `.env` перекрывают умолчания compose |
|
||||
| начать с чистого листа | `docker compose down -v` (удаляет учебные данные: БД, bucket, очередь) → `up -d --build --wait` |
|
||||
|
||||
Остановить после защиты: `docker compose down` (данные сохранятся).
|
||||
@@ -0,0 +1,392 @@
|
||||
# Clean clone: secrets, config, build, migrations from zero, start, versions
|
||||
# 2026-10-31T14:18:15Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT
|
||||
|
||||
$ git status --short --branch
|
||||
## solution/bvt2302-bodanyuk...origin/solution/bvt2302-bodanyuk
|
||||
[exit 0, 0.1 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe --version
|
||||
Python 3.14.6
|
||||
[exit 0, 0.0 s]
|
||||
|
||||
$ docker version --format client {{.Client.Version}} / server {{.Server.Version}}
|
||||
client 29.8.0 / server 29.8.0
|
||||
[exit 0, 0.1 s]
|
||||
|
||||
$ docker compose version
|
||||
Docker Compose version v5.5.1
|
||||
[exit 0, 0.1 s]
|
||||
|
||||
$ make --version
|
||||
GNU Make 4.4.1
|
||||
Built for Windows32
|
||||
Copyright (C) 1988-2023 Free Software Foundation, Inc.
|
||||
License GPLv3+: GNU GPL version 3 or later <https://gnu.org/licenses/gpl.html>
|
||||
This is free software: you are free to change and redistribute it.
|
||||
There is NO WARRANTY, to the extent permitted by law.
|
||||
[exit 0, 0.0 s]
|
||||
|
||||
$ python scripts/init_secrets.py
|
||||
created secrets/postgres_password
|
||||
created secrets/valkey_password
|
||||
created secrets/database_url
|
||||
created secrets/cache_url
|
||||
created secrets/s3_admin_access_key_id
|
||||
created secrets/s3_admin_secret_access_key
|
||||
created secrets/s3_api_access_key_id
|
||||
created secrets/s3_api_secret_access_key
|
||||
created secrets/s3_worker_access_key_id
|
||||
created secrets/s3_worker_secret_access_key
|
||||
created secrets/s3_auditor_access_key_id
|
||||
created secrets/s3_auditor_secret_access_key
|
||||
created secrets/rabbitmq_relay_password
|
||||
created secrets/rabbitmq_worker_password
|
||||
created secrets/rabbitmq_users.json
|
||||
created secrets/broker_url_relay
|
||||
created secrets/broker_url_worker
|
||||
[exit 0, 0.1 s]
|
||||
|
||||
$ cp .env.example .env
|
||||
|
||||
$ docker compose --env-file .env config --quiet
|
||||
|
||||
[exit 0, 0.2 s]
|
||||
|
||||
$ docker compose --env-file .env up -d --build --wait
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 1.32kB 0.0s done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [migrate internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 2.96kB done
|
||||
#2 DONE 0.1s
|
||||
|
||||
#3 [migrate] resolve image config for docker-image://docker.io/docker/dockerfile:1
|
||||
#3 DONE 6.2s
|
||||
|
||||
#4 [migrate] docker-image://docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e
|
||||
#4 resolve docker.io/docker/dockerfile:1@sha256:4edf897a3ffa55b89f906fc8cc78afdb3f1834cc9c7083565e611a8a7d5fe99e 0.0s done
|
||||
#4 CACHED
|
||||
|
||||
#5 [api internal] load metadata for docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
|
||||
#5 DONE 2.1s
|
||||
|
||||
#6 [migrate internal] load .dockerignore
|
||||
#6 transferring context: 196B done
|
||||
#6 DONE 0.1s
|
||||
|
||||
#7 [migrate 1/9] FROM docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
|
||||
#7 resolve docker.io/library/python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 0.1s done
|
||||
#7 DONE 0.1s
|
||||
|
||||
#8 [api internal] load build context
|
||||
#8 transferring context: 525.52kB 0.1s done
|
||||
#8 transferring context: 525.52kB 0.1s done
|
||||
#8 DONE 0.2s
|
||||
|
||||
#9 [api 2/9] WORKDIR /app
|
||||
#9 CACHED
|
||||
|
||||
#10 [api 3/9] COPY requirements.txt ./
|
||||
#10 CACHED
|
||||
|
||||
#11 [api 4/9] RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt
|
||||
#11 CACHED
|
||||
|
||||
#12 [api 5/9] RUN apt-get update && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 && rm -rf /var/lib/apt/lists/*
|
||||
#12 CACHED
|
||||
|
||||
#13 [api 6/9] RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
|
||||
#13 CACHED
|
||||
|
||||
#14 [api 7/9] COPY src/ ./src/
|
||||
#14 DONE 0.1s
|
||||
|
||||
#15 [api 8/9] COPY migrations/ ./migrations/
|
||||
#15 DONE 0.1s
|
||||
|
||||
#16 [migrate 9/9] COPY contracts/openapi.json ./contracts/openapi.json
|
||||
#16 DONE 0.1s
|
||||
|
||||
#17 [api] exporting to image
|
||||
#17 exporting layers
|
||||
#17 exporting layers 0.3s done
|
||||
#17 exporting manifest sha256:d5ce436c70dc889fae295b9c569550143a49023a8c8c276e83ec5b3e3e156d4d 0.0s done
|
||||
#17 exporting config sha256:bd8b08e1de339150644db0527530d2ee10f7fdac39fb495916259bb92f8157ea 0.1s done
|
||||
#17 exporting attestation manifest sha256:88b54cc6ae37ccc6d7bed5b2f499e6d2ad7b184c25c8f8da067db4010c4f341f 0.1s done
|
||||
#17 exporting manifest list sha256:7689cb4ef3a9f3649524415f9528e0c84610759d6536e522dc5ea0d4305e3878
|
||||
#17 exporting manifest list sha256:7689cb4ef3a9f3649524415f9528e0c84610759d6536e522dc5ea0d4305e3878 0.0s done
|
||||
#17 naming to docker.io/library/mtusi-messenger:lab5 0.0s done
|
||||
#17 unpacking to docker.io/library/mtusi-messenger:lab5
|
||||
#17 unpacking to docker.io/library/mtusi-messenger:lab5 0.2s done
|
||||
#17 DONE 0.8s
|
||||
|
||||
#18 [migrate] exporting to image
|
||||
#18 exporting layers 0.3s done
|
||||
#18 exporting manifest sha256:9b9171737d004d0514b7adf53f37158be51e19440ab86d15192ef7d71bcf1ec0 0.1s done
|
||||
#18 exporting config sha256:2b03ff83f38876a18738b5fcf9676e1c840ca6162cdee4dd6252e1cab242448e 0.1s done
|
||||
#18 exporting attestation manifest sha256:b62c3a0d59a966c45c4de062852cb7c68895fe85e1e669dce3bfe885f907ac72 0.1s done
|
||||
#18 exporting manifest list sha256:a3c335533065814c753479d7820efef885c03a58eb62edbf2e21d0f267be982a 0.0s done
|
||||
#18 naming to docker.io/library/mtusi-messenger:lab5 done
|
||||
#18 unpacking to docker.io/library/mtusi-messenger:lab5 0.2s done
|
||||
#18 DONE 0.9s
|
||||
|
||||
#19 [api] resolving provenance for metadata file
|
||||
#19 DONE 0.0s
|
||||
|
||||
#20 [migrate] resolving provenance for metadata file
|
||||
#20 DONE 0.0s
|
||||
Image mtusi-messenger:lab5 Building
|
||||
Image mtusi-messenger:lab5 Building
|
||||
Image mtusi-messenger:lab5 Built
|
||||
Image mtusi-messenger:lab5 Built
|
||||
Network mtusi-lab5_data Creating
|
||||
Network mtusi-lab5_edge Creating
|
||||
Network mtusi-lab5_data Creating
|
||||
Network mtusi-lab5_queue Creating
|
||||
Network mtusi-lab5_queue Creating
|
||||
Volume mtusi-lab5_pgdata Creating
|
||||
Volume mtusi-lab5_pgdata Creating
|
||||
Volume mtusi-lab5_s3data Creating
|
||||
Volume mtusi-lab5_s3data Creating
|
||||
Volume mtusi-lab5_rabbitmqdata Creating
|
||||
Volume mtusi-lab5_rabbitmqdata Creating
|
||||
Volume mtusi-lab5_valkeydata Creating
|
||||
Volume mtusi-lab5_valkeydata Creating
|
||||
Network mtusi-lab5_edge Creating
|
||||
Network mtusi-lab5_storage Creating
|
||||
Network mtusi-lab5_storage Creating
|
||||
Volume mtusi-lab5_valkeydata Created
|
||||
Volume mtusi-lab5_valkeydata Created
|
||||
Volume mtusi-lab5_pgdata Created
|
||||
Volume mtusi-lab5_pgdata Created
|
||||
Volume mtusi-lab5_s3data Created
|
||||
Volume mtusi-lab5_s3data Created
|
||||
Volume mtusi-lab5_rabbitmqdata Created
|
||||
Volume mtusi-lab5_rabbitmqdata Created
|
||||
Network mtusi-lab5_data Created
|
||||
Network mtusi-lab5_data Created
|
||||
Container mtusi-lab5-valkey-1 Creating
|
||||
Container mtusi-lab5-postgres-1 Creating
|
||||
Network mtusi-lab5_storage Created
|
||||
Network mtusi-lab5_storage Created
|
||||
Container mtusi-lab5-s3-1 Creating
|
||||
Network mtusi-lab5_queue Created
|
||||
Network mtusi-lab5_queue Created
|
||||
Container mtusi-lab5-broker-1 Creating
|
||||
Network mtusi-lab5_edge Created
|
||||
Network mtusi-lab5_edge Created
|
||||
Container mtusi-lab5-postgres-1 Created
|
||||
Container mtusi-lab5-migrate-1 Creating
|
||||
Container mtusi-lab5-valkey-1 Created
|
||||
Container mtusi-lab5-broker-1 Created
|
||||
Container mtusi-lab5-s3-1 Created
|
||||
Container mtusi-lab5-migrate-1 Created
|
||||
Container mtusi-lab5-s3-init-1 Creating
|
||||
Container mtusi-lab5-relay-1 Creating
|
||||
Container mtusi-lab5-s3-init-1 Created
|
||||
Container mtusi-lab5-api-2 Creating
|
||||
Container mtusi-lab5-api-1 Creating
|
||||
Container mtusi-lab5-worker-1 Creating
|
||||
Container mtusi-lab5-relay-1 Created
|
||||
Container mtusi-lab5-worker-1 Created
|
||||
Container mtusi-lab5-api-1 Created
|
||||
Container mtusi-lab5-api-2 Created
|
||||
Container mtusi-lab5-lb-1 Creating
|
||||
Container mtusi-lab5-lb-1 Created
|
||||
Container mtusi-lab5-s3-1 Starting
|
||||
Container mtusi-lab5-valkey-1 Starting
|
||||
Container mtusi-lab5-postgres-1 Starting
|
||||
Container mtusi-lab5-broker-1 Starting
|
||||
Container mtusi-lab5-s3-1 Started
|
||||
Container mtusi-lab5-valkey-1 Started
|
||||
Container mtusi-lab5-postgres-1 Started
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-broker-1 Started
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Starting
|
||||
Container mtusi-lab5-migrate-1 Started
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-broker-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-s3-init-1 Starting
|
||||
Container mtusi-lab5-s3-init-1 Started
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-valkey-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-s3-init-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-broker-1 Waiting
|
||||
Container mtusi-lab5-s3-init-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-valkey-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-broker-1 Healthy
|
||||
Container mtusi-lab5-broker-1 Healthy
|
||||
Container mtusi-lab5-relay-1 Starting
|
||||
Container mtusi-lab5-relay-1 Started
|
||||
Container mtusi-lab5-s3-init-1 Exited
|
||||
Container mtusi-lab5-worker-1 Starting
|
||||
Container mtusi-lab5-s3-init-1 Exited
|
||||
Container mtusi-lab5-api-2 Starting
|
||||
Container mtusi-lab5-worker-1 Started
|
||||
Container mtusi-lab5-api-2 Started
|
||||
Container mtusi-lab5-api-1 Starting
|
||||
Container mtusi-lab5-api-1 Started
|
||||
Container mtusi-lab5-api-2 Waiting
|
||||
Container mtusi-lab5-api-1 Waiting
|
||||
Container mtusi-lab5-api-2 Healthy
|
||||
Container mtusi-lab5-api-1 Healthy
|
||||
Container mtusi-lab5-lb-1 Starting
|
||||
Container mtusi-lab5-lb-1 Started
|
||||
Container mtusi-lab5-valkey-1 Waiting
|
||||
Container mtusi-lab5-postgres-1 Waiting
|
||||
Container mtusi-lab5-s3-init-1 Waiting
|
||||
Container mtusi-lab5-migrate-1 Waiting
|
||||
Container mtusi-lab5-api-2 Waiting
|
||||
Container mtusi-lab5-api-1 Waiting
|
||||
Container mtusi-lab5-s3-1 Waiting
|
||||
Container mtusi-lab5-relay-1 Waiting
|
||||
Container mtusi-lab5-broker-1 Waiting
|
||||
Container mtusi-lab5-lb-1 Waiting
|
||||
Container mtusi-lab5-worker-1 Waiting
|
||||
Container mtusi-lab5-valkey-1 Healthy
|
||||
Container mtusi-lab5-broker-1 Healthy
|
||||
Container mtusi-lab5-worker-1 Healthy
|
||||
Container mtusi-lab5-relay-1 Healthy
|
||||
Container mtusi-lab5-postgres-1 Healthy
|
||||
Container mtusi-lab5-migrate-1 Exited
|
||||
Container mtusi-lab5-s3-1 Healthy
|
||||
Container mtusi-lab5-s3-init-1 Exited
|
||||
Container mtusi-lab5-api-2 Healthy
|
||||
Container mtusi-lab5-api-1 Healthy
|
||||
Container mtusi-lab5-lb-1 Healthy
|
||||
[exit 0, 41.2 s]
|
||||
|
||||
$ docker compose ps -a --format table {{.Service}} {{.Image}} {{.Status}}
|
||||
SERVICE IMAGE STATUS
|
||||
api mtusi-messenger:lab5 Up 11 seconds (healthy)
|
||||
api mtusi-messenger:lab5 Up 12 seconds (healthy)
|
||||
broker rabbitmq:4.3.7-alpine@sha256:b1f997f74103d08fc1f20a13890dc44fe6cfa51d528cf521b480ada988d3ffa5 Up 26 seconds (healthy)
|
||||
lb haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 Up 5 seconds (healthy)
|
||||
migrate mtusi-messenger:lab5 Exited (0) 19 seconds ago
|
||||
postgres postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c Up 26 seconds (healthy)
|
||||
relay mtusi-messenger:lab5 Up 15 seconds (healthy)
|
||||
s3 chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62 Up 27 seconds (healthy)
|
||||
s3-init mtusi-messenger:lab5 Exited (0) 12 seconds ago
|
||||
valkey valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 Up 27 seconds (healthy)
|
||||
worker mtusi-messenger:lab5 Up 12 seconds (healthy)
|
||||
[exit 0, 0.3 s]
|
||||
|
||||
$ docker compose logs --no-log-prefix migrate
|
||||
migrate: 0001_init: applied
|
||||
migrate: 0002_idempotency_keys: applied
|
||||
migrate: 0003_post_message_function: applied
|
||||
migrate: 0004_attachments: applied
|
||||
migrate: 0005_pipeline: applied
|
||||
migrate: done, 5 migration file(s) checked
|
||||
[exit 0, 0.3 s]
|
||||
|
||||
$ docker compose logs --no-log-prefix s3-init
|
||||
s3-bootstrap: bucket messenger-attachments: created
|
||||
s3-bootstrap: bucket messenger-staging: created
|
||||
s3-bootstrap: sentinel health/sentinel: written
|
||||
s3-bootstrap: anonymous GET object of messenger-attachments: HTTP 403 -> refused
|
||||
s3-bootstrap: anonymous LIST of messenger-attachments: HTTP 403 -> refused
|
||||
s3-bootstrap: anonymous GET object of messenger-staging: HTTP 403 -> refused
|
||||
s3-bootstrap: anonymous LIST of messenger-staging: HTTP 403 -> refused
|
||||
s3-bootstrap: api: write staging: allowed -> OK
|
||||
s3-bootstrap: api: delete staging (janitor): allowed -> OK
|
||||
s3-bootstrap: api: read staging: refused (403) -> OK
|
||||
s3-bootstrap: api: read final: allowed -> OK
|
||||
s3-bootstrap: api: write final: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: api: list final: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: api: list staging: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: api: create bucket: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: api: read another bucket: refused (403) -> OK
|
||||
s3-bootstrap: worker: read staging: allowed -> OK
|
||||
s3-bootstrap: worker: write final: allowed -> OK
|
||||
s3-bootstrap: worker: delete final: allowed -> OK
|
||||
s3-bootstrap: worker: write staging: allowed -> OK
|
||||
s3-bootstrap: worker: delete staging: allowed -> OK
|
||||
s3-bootstrap: worker: list final: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: worker: create bucket: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: worker: read another bucket: refused (403) -> OK
|
||||
s3-bootstrap: auditor: list final: allowed -> OK
|
||||
s3-bootstrap: auditor: list staging: allowed -> OK
|
||||
s3-bootstrap: auditor: read final: allowed -> OK
|
||||
s3-bootstrap: auditor: write final: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: auditor: delete staging: refused (AccessDenied) -> OK
|
||||
s3-bootstrap: api: buckets visible in ListBuckets: [] -> OK
|
||||
s3-bootstrap: done: private buckets, least-privilege keys verified
|
||||
[exit 0, 0.3 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py replicas
|
||||
mtusi-lab5-api-1 X-Instance-Id=d08185c0e3f3 running health=healthy pid=83074 started=2026-10-31T14:18:45.487 ip=172.25.0.7,172.26.0.5,172.28.0.3
|
||||
mtusi-lab5-api-2 X-Instance-Id=85c7cde066aa running health=healthy pid=82998 started=2026-10-31T14:18:45.132 ip=172.25.0.6,172.26.0.4,172.28.0.2
|
||||
[exit 0, 0.5 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
|
||||
api1 172.28.0.3 UP mtusi-lab5-api-1 X-Instance-Id=d08185c0e3f3
|
||||
api2 172.28.0.2 UP mtusi-lab5-api-2 X-Instance-Id=85c7cde066aa
|
||||
(other slots: MAINT, no container)
|
||||
[exit 0, 0.7 s]
|
||||
|
||||
$ docker compose exec -T broker rabbitmqctl -q list_users
|
||||
user tags
|
||||
relay []
|
||||
worker []
|
||||
[exit 0, 2.0 s]
|
||||
|
||||
$ docker compose exec -T broker rabbitmqctl -q list_queues name type durable messages consumers arguments
|
||||
name type durable messages consumers arguments
|
||||
messenger.attachment-jobs quorum true 0 1 [{"x-dead-letter-exchange","messenger.jobs.dlx"},{"x-delivery-limit",20},{"x-queue-type","quorum"}]
|
||||
messenger.attachment-jobs.dlq quorum true 0 0 [{"x-queue-type","quorum"}]
|
||||
[exit 0, 2.0 s]
|
||||
|
||||
$ docker compose exec -T broker rabbitmqctl -q list_permissions
|
||||
user configure write read
|
||||
relay ^$ ^messenger\.jobs$ ^$
|
||||
worker ^$ ^$ ^messenger\.attachment-jobs$
|
||||
[exit 0, 3.9 s]
|
||||
|
||||
$ docker inspect (limits)
|
||||
/mtusi-lab5-api-1 cpus=1000000000 memory=536870912 memswap=536870912 image=mtusi-messenger:lab5
|
||||
/mtusi-lab5-api-2 cpus=1000000000 memory=536870912 memswap=536870912 image=mtusi-messenger:lab5
|
||||
/mtusi-lab5-broker-1 cpus=1000000000 memory=536870912 memswap=1073741824 image=rabbitmq:4.3.7-alpine@sha256:b1f997f74103d08fc1f20a13890dc44fe6cfa51d528cf521b480ada988d3ffa5
|
||||
/mtusi-lab5-lb-1 cpus=1000000000 memory=134217728 memswap=268435456 image=haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122
|
||||
/mtusi-lab5-postgres-1 cpus=2000000000 memory=1073741824 memswap=2147483648 image=postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c
|
||||
/mtusi-lab5-relay-1 cpus=500000000 memory=268435456 memswap=536870912 image=mtusi-messenger:lab5
|
||||
/mtusi-lab5-s3-1 cpus=1000000000 memory=1073741824 memswap=2147483648 image=chrislusf/seaweedfs:4.46@sha256:08d516132314207d10c8e37cbffc1f32b147d870169688734cc61c6231625b62
|
||||
/mtusi-lab5-valkey-1 cpus=500000000 memory=268435456 memswap=536870912 image=valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
|
||||
/mtusi-lab5-worker-1 cpus=2000000000 memory=1610612736 memswap=1610612736 image=mtusi-messenger:lab5
|
||||
[exit 0, 0.1 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe -m venv .venv
|
||||
|
||||
[exit 0, 7.0 s]
|
||||
|
||||
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean\.venv\Scripts\python.exe -m pip install -q --require-hashes --no-deps -r requirements.txt
|
||||
|
||||
[notice] A new release of pip is available: 26.1.2 -> 26.2.2
|
||||
[notice] To update, run: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean\.venv\Scripts\python.exe -m pip install --upgrade pip
|
||||
[exit 0, 28.9 s]
|
||||
|
||||
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean\.venv\Scripts\python.exe -m pip install -q -r requirements-test.txt -r requirements-dev.txt
|
||||
|
||||
[notice] A new release of pip is available: 26.1.2 -> 26.2.2
|
||||
[notice] To update, run: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean\.venv\Scripts\python.exe -m pip install --upgrade pip
|
||||
[exit 0, 16.0 s]
|
||||
@@ -0,0 +1,60 @@
|
||||
# Defense steps 2-3: worker stopped, upload 202, content 409, restart API and broker with volumes, worker started, ready + thumbnail
|
||||
# 2026-10-31T14:22:23Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT
|
||||
|
||||
Containers BEFORE:
|
||||
mtusi-lab5-api-1 host=d08185c0e3f3 running exit=0 pid=94963 started=2026-10-31T14:21:47.426
|
||||
mtusi-lab5-api-2 host=85c7cde066aa running exit=0 pid=96896 started=2026-10-31T14:22:15.716
|
||||
mtusi-lab5-broker-1 host=broker running exit=0 pid=89550 started=2026-10-31T14:20:17.480
|
||||
mtusi-lab5-lb-1 host=a22e1c19fc90 running exit=0 pid=92390 started=2026-10-31T14:21:12.759
|
||||
mtusi-lab5-migrate-1 host=fa0a8a1c9191 exited exit=0 pid=0 started=2026-10-31T14:21:14.217
|
||||
mtusi-lab5-postgres-1 host=7961919f90c9 running exit=0 pid=89433 started=2026-10-31T14:20:16.605
|
||||
mtusi-lab5-relay-1 host=9b6249446ece running exit=0 pid=92854 started=2026-10-31T14:21:16.976
|
||||
mtusi-lab5-s3-1 host=bdee3ea030a4 running exit=0 pid=90402 started=2026-10-31T14:20:36.421
|
||||
mtusi-lab5-s3-init-1 host=2c85c6921c88 exited exit=0 pid=0 started=2026-10-31T14:21:17.299
|
||||
mtusi-lab5-valkey-1 host=d53b8a88e5b5 running exit=0 pid=89376 started=2026-10-31T14:20:16.424
|
||||
mtusi-lab5-worker-1 host=ee8c11f59aa8 running exit=0 pid=93271 started=2026-10-31T14:21:22.639
|
||||
|
||||
$ python scripts/demo_pipeline.py
|
||||
|
||||
[17:22:24] 1. docker compose stop worker
|
||||
worker exited
|
||||
|
||||
[17:22:26] 2. upload while the worker is stopped
|
||||
POST /chats/{id}/attachments -> 202 [X-Instance-Id 85c7cde066aa] {"id":"e9d4a7b9-a5da-4c05-a12d-abb95c9a1ff5","chat_id":"d60cc7c6-9ce0-4a81-90ca-3fb3e0e4abbb","owner_id":"7ef6ca59-758c-4753-8037-d00537802aba","filename":"sample.png","content_type":"image/png","size_bytes":111817,"sha256":"99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75","status":
|
||||
attachment_id = e9d4a7b9-a5da-4c05-a12d-abb95c9a1ff5 status = queued
|
||||
GET /attachments/{id}/content -> 409 [X-Instance-Id d08185c0e3f3] {"error":{"code":"conflict","message":"Вложение ещё не готово","request_id":"3c6fd35c-0ce9-4f9f-8551-8b9a4eebb88c"}}
|
||||
GET /attachments/{id}/content?variant=thumbnail -> 409 [X-Instance-Id 85c7cde066aa] {"error":{"code":"conflict","message":"Вложение ещё не готово","request_id":"d017985f-e358-4a88-9459-70e377707782"}}
|
||||
POST /messages (text only) -> 201 [X-Instance-Id d08185c0e3f3] {"id":"fda54822-f633-4f82-946c-fcee50e0e124","chat_id":"d60cc7c6-9ce0-4a81-90ca-3fb3e0e4abbb","sender_id":"41252a17-4106-481a-8a43-a5187721c7b9","text":"обычное сообщение работает","created_at":"2026-10-31T14:22:26.348105Z","attachment_ids":[]}
|
||||
POST /messages attachment_ids=[queued id] -> 201 [X-Instance-Id 85c7cde066aa] {"id":"279910fd-0f7c-4123-bd16-ae3fe1e52833","chat_id":"d60cc7c6-9ce0-4a81-90ca-3fb3e0e4abbb","sender_id":"7ef6ca59-758c-4753-8037-d00537802aba","text":"фото","created_at":"2026-10-31T14:22:26.360988Z","attachment_ids":["e9d4a7b9-a5da-4c05-a12d-abb95c9a1ff5"]}
|
||||
job + outbox in PostgreSQL: c59c75a5-67d1-48e6-95cd-b6af894c4e4b state=queued attempt=0 | outbox published_at=2026-10-31 14:22:26.626499+00
|
||||
RabbitMQ queue messenger.attachment-jobs: messages=1 consumers=0
|
||||
|
||||
[17:22:38] 3. docker compose restart api broker (volumes preserved)
|
||||
RabbitMQ queue after the restart: messages=1 consumers=0
|
||||
GET /attachments/{id} -> 200 [X-Instance-Id 85c7cde066aa] {"id":"e9d4a7b9-a5da-4c05-a12d-abb95c9a1ff5","chat_id":"d60cc7c6-9ce0-4a81-90ca-3fb3e0e4abbb","owner_id":"7ef6ca59-758c-4753-8037-d00537802aba","filename":"sample.png","content_type":"image/png","size_bytes":111817,"sha256":"99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75","status":
|
||||
|
||||
[17:23:00] 4. docker compose start worker
|
||||
GET /attachments/{id} until ready -> 200 [X-Instance-Id 85c7cde066aa] {"id":"e9d4a7b9-a5da-4c05-a12d-abb95c9a1ff5","chat_id":"d60cc7c6-9ce0-4a81-90ca-3fb3e0e4abbb","owner_id":"7ef6ca59-758c-4753-8037-d00537802aba","filename":"sample.png","content_type":"image/png","size_bytes":111817,"sha256":"99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75","status":
|
||||
GET /content?variant=thumbnail -> 200 [X-Instance-Id d08185c0e3f3] 4679 bytes, Content-Type image/jpeg
|
||||
thumbnail: real JPEG 256x160, sha256 matches metadata: True
|
||||
original unchanged byte for byte: True
|
||||
message 279910fd-0f7c-4123-bd16-ae3fe1e52833 still references it: {e9d4a7b9-a5da-4c05-a12d-abb95c9a1ff5}
|
||||
|
||||
[17:23:13] 5. job history (jobctl show)
|
||||
2026-10-31 14:22:26.316695+00:00 created attempt=None actor=api:85c7cde066aa
|
||||
2026-10-31 14:23:12.130790+00:00 claimed attempt=1 actor=worker:ee8c11f59aa8
|
||||
2026-10-31 14:23:12.981082+00:00 succeeded attempt=1 actor=worker:ee8c11f59aa8
|
||||
[exit 0, 53.7 s]
|
||||
|
||||
Containers AFTER (api-*, broker restarted; worker stopped and started):
|
||||
mtusi-lab5-api-1 host=d08185c0e3f3 running exit=0 pid=99058 started=2026-10-31T14:22:43.321
|
||||
mtusi-lab5-api-2 host=85c7cde066aa running exit=0 pid=98980 started=2026-10-31T14:22:42.925
|
||||
mtusi-lab5-broker-1 host=broker running exit=0 pid=98657 started=2026-10-31T14:22:40.329
|
||||
mtusi-lab5-lb-1 host=a22e1c19fc90 running exit=0 pid=92390 started=2026-10-31T14:21:12.759
|
||||
mtusi-lab5-migrate-1 host=fa0a8a1c9191 exited exit=0 pid=0 started=2026-10-31T14:23:01.939
|
||||
mtusi-lab5-postgres-1 host=7961919f90c9 running exit=0 pid=89433 started=2026-10-31T14:20:16.605
|
||||
mtusi-lab5-relay-1 host=9b6249446ece running exit=0 pid=92854 started=2026-10-31T14:21:16.976
|
||||
mtusi-lab5-s3-1 host=bdee3ea030a4 running exit=0 pid=90402 started=2026-10-31T14:20:36.421
|
||||
mtusi-lab5-s3-init-1 host=2c85c6921c88 exited exit=0 pid=0 started=2026-10-31T14:23:04.208
|
||||
mtusi-lab5-valkey-1 host=d53b8a88e5b5 running exit=0 pid=89376 started=2026-10-31T14:20:16.424
|
||||
mtusi-lab5-worker-1 host=ee8c11f59aa8 running exit=0 pid=1259 started=2026-10-31T14:23:09.985
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 5,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.066,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 5515,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 183.429,
|
||||
"successful_latency_ms": {
|
||||
"p50": 17.96,
|
||||
"p95": 43.491,
|
||||
"p99": 58.653
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
# make check, make validate (venv), make test
|
||||
# 2026-10-31T14:19:59Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT
|
||||
|
||||
$ make check
|
||||
python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
[exit 0, 1.2 s]
|
||||
|
||||
$ make validate PYTHON=.venv/Scripts/python.exe
|
||||
.venv/Scripts/python.exe scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
[exit 0, 2.3 s]
|
||||
|
||||
$ make test
|
||||
python tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
[exit 0, 3.3 s]
|
||||
@@ -0,0 +1,394 @@
|
||||
# Own tests: unit + stand (pytest -v); failures restored in finally
|
||||
# 2026-10-31T14:23:18Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT
|
||||
|
||||
$ python -m pytest student-tests -v
|
||||
============================= test session starts =============================
|
||||
platform win32 -- Python 3.14.6, pytest-9.1.1, pluggy-1.7.0 -- C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean\.venv\Scripts\python.exe
|
||||
rootdir: C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean
|
||||
plugins: anyio-4.15.1
|
||||
collecting ... collected 381 items
|
||||
|
||||
student-tests/test_api.py::test_register_never_returns_password_and_login_returns_bearer PASSED [ 0%]
|
||||
student-tests/test_api.py::test_duplicate_username_is_409 PASSED [ 0%]
|
||||
student-tests/test_api.py::test_bad_password_on_registration_is_400[short-11ch] PASSED [ 0%]
|
||||
student-tests/test_api.py::test_bad_password_on_registration_is_400[ ] PASSED [ 1%]
|
||||
student-tests/test_api.py::test_bad_password_on_registration_is_400[ \t \n \t \n \t \n ] PASSED [ 1%]
|
||||
student-tests/test_api.py::test_bad_password_on_registration_is_400[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 1%]
|
||||
student-tests/test_api.py::test_edge_passwords_register_and_login[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 1%]
|
||||
student-tests/test_api.py::test_edge_passwords_register_and_login[with:colons:inside] PASSED [ 2%]
|
||||
student-tests/test_api.py::test_edge_passwords_register_and_login[Twelve chars] PASSED [ 2%]
|
||||
student-tests/test_api.py::test_wrong_password_and_unknown_user_give_the_same_401 PASSED [ 2%]
|
||||
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic \xe9] PASSED [ 2%]
|
||||
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic !!!] PASSED [ 3%]
|
||||
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic bm9fY29sb24=] PASSED [ 3%]
|
||||
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic //46eA==] PASSED [ 3%]
|
||||
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 3%]
|
||||
student-tests/test_api.py::test_malformed_basic_is_401_not_500[] PASSED [ 4%]
|
||||
student-tests/test_api.py::test_logout_revokes_only_this_token PASSED [ 4%]
|
||||
student-tests/test_api.py::test_x_user_id_basic_and_garbage_do_not_authenticate PASSED [ 4%]
|
||||
student-tests/test_api.py::test_malformed_json_is_400[{] PASSED [ 4%]
|
||||
student-tests/test_api.py::test_malformed_json_is_400[] PASSED [ 5%]
|
||||
student-tests/test_api.py::test_malformed_json_is_400[[]] PASSED [ 5%]
|
||||
student-tests/test_api.py::test_malformed_json_is_400[null] PASSED [ 5%]
|
||||
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}] PASSED [ 6%]
|
||||
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}] PASSED [ 6%]
|
||||
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "\ud800", "password": "Twelve-chars"}] PASSED [ 6%]
|
||||
student-tests/test_api.py::test_invalid_registration_fields_are_400[body0] PASSED [ 6%]
|
||||
student-tests/test_api.py::test_invalid_registration_fields_are_400[body1] PASSED [ 7%]
|
||||
student-tests/test_api.py::test_invalid_registration_fields_are_400[body2] PASSED [ 7%]
|
||||
student-tests/test_api.py::test_invalid_registration_fields_are_400[body3] PASSED [ 7%]
|
||||
student-tests/test_api.py::test_invalid_registration_fields_are_400[body4] PASSED [ 7%]
|
||||
student-tests/test_api.py::test_invalid_registration_fields_are_400[body5] PASSED [ 8%]
|
||||
student-tests/test_api.py::test_unicode_limits_are_code_points_and_nul_is_stored PASSED [ 8%]
|
||||
student-tests/test_api.py::test_oversized_body_and_wrong_content_type PASSED [ 8%]
|
||||
student-tests/test_api.py::test_chat_validation PASSED [ 8%]
|
||||
student-tests/test_api.py::test_foreign_chat_is_indistinguishable_from_missing PASSED [ 9%]
|
||||
student-tests/test_api.py::test_chat_list_order_pagination_and_isolation PASSED [ 9%]
|
||||
student-tests/test_api.py::test_history_pages_without_loss_or_duplicates PASSED [ 9%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=0] PASSED [ 9%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=101] PASSED [ 10%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=abc] PASSED [ 10%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=1&limit=2] PASSED [ 10%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=] PASSED [ 11%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=garbage] PASSED [ 11%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDF8Mg] PASSED [ 11%]
|
||||
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDk5OTk5OTk5OTk5OTk5OTk5OTk5fHk=] PASSED [ 11%]
|
||||
student-tests/test_api.py::test_cursor_bound_to_chat PASSED [ 12%]
|
||||
student-tests/test_api.py::test_read_after_write PASSED [ 12%]
|
||||
student-tests/test_api.py::test_malformed_cursor_values_are_400_on_both_lists PASSED [ 12%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[\xff] PASSED [ 12%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"first","TEXT":"second"}] PASSED [ 13%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"a","text":"b"}] PASSED [ 13%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":123}] PASSED [ 13%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}}] PASSED [ 13%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"valid"}]] PASSED [ 14%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x"} {}] PASSED [ 14%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}] PASSED [ 14%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":null}] PASSED [ 14%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":"x","attachment_ids":["nope"]}] PASSED [ 15%]
|
||||
student-tests/test_api.py::test_bad_message_body_is_400_and_nothing_is_stored[{"text":" \n\t "}] PASSED [ 15%]
|
||||
student-tests/test_api.py::test_ff_byte_body_on_registration_is_contract_error PASSED [ 15%]
|
||||
student-tests/test_api.py::test_text_is_stored_verbatim_and_length_counts_raw_text PASSED [ 16%]
|
||||
student-tests/test_api.py::test_unknown_attachment_ids_are_404 PASSED [ 16%]
|
||||
student-tests/test_api.py::test_member_ids_are_compared_as_uuids_not_strings PASSED [ 16%]
|
||||
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9] PASSED [ 16%]
|
||||
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer .] PASSED [ 17%]
|
||||
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer a b] PASSED [ 17%]
|
||||
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 17%]
|
||||
student-tests/test_api.py::test_non_ascii_or_garbage_bearer_is_401_not_500[bearer\taaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 17%]
|
||||
student-tests/test_api.py::test_login_with_a_body_is_rejected PASSED [ 18%]
|
||||
student-tests/test_api.py::test_password_range_is_12_to_128_code_points PASSED [ 18%]
|
||||
student-tests/test_api.py::test_every_api_response_carries_instance_and_request_id PASSED [ 18%]
|
||||
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\xa0] PASSED [ 18%]
|
||||
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[\u2003\u2003] PASSED [ 19%]
|
||||
student-tests/test_api.py::test_unicode_whitespace_only_strings_are_400[ \xa0\t\u3000] PASSED [ 19%]
|
||||
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":"x","password":1e1000}] PASSED [ 19%]
|
||||
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"ff_probe","display_name":"x","password":"12345678901\xff"}] PASSED [ 19%]
|
||||
student-tests/test_api.py::test_odd_registration_bodies_are_400_and_the_api_survives[{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}] PASSED [ 20%]
|
||||
student-tests/test_api.py::test_more_than_100_members_is_400 PASSED [ 20%]
|
||||
student-tests/test_api.py::test_cursor_of_one_list_is_rejected_by_the_other PASSED [ 20%]
|
||||
student-tests/test_attachments.py::test_png_and_jpeg_roundtrip_byte_for_byte PASSED [ 20%]
|
||||
student-tests/test_attachments.py::test_metadata_reveals_no_storage_details PASSED [ 21%]
|
||||
student-tests/test_attachments.py::test_original_is_not_stored_on_api_containers PASSED [ 21%]
|
||||
student-tests/test_attachments.py::test_outsider_and_unknown_get_404 PASSED [ 21%]
|
||||
student-tests/test_attachments.py::test_attachment_ids_are_checked_per_chat PASSED [ 22%]
|
||||
student-tests/test_attachments.py::test_message_with_attachments_and_history PASSED [ 22%]
|
||||
student-tests/test_attachments.py::test_idempotent_replay_with_attachments PASSED [ 22%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"attachment_ids":["%(a)s"]}] PASSED [ 22%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":null,"attachment_ids":["%(a)s"]}] PASSED [ 23%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":1,"attachment_ids":["%(a)s"]}] PASSED [ 23%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":null}] PASSED [ 23%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":"%(a)s"}] PASSED [ 23%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":{}}] PASSED [ 24%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s","%] PASSED [ 24%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["%(a)s",1]] PASSED [ 24%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":["not-a-uui] PASSED [ 24%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"","attachment_ids":[]}] PASSED [ 25%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":" "}] PASSED [ 25%]
|
||||
student-tests/test_attachments.py::test_message_body_validation[{"text":"x","attachment_ids":[],"sender_] PASSED [ 25%]
|
||||
student-tests/test_attachments.py::test_at_most_ten_attachments PASSED [ 25%]
|
||||
student-tests/test_attachments.py::test_size_limit_through_the_load_balancer PASSED [ 26%]
|
||||
student-tests/test_attachments.py::test_pixel_limit_before_decoding PASSED [ 26%]
|
||||
student-tests/test_attachments.py::test_exactly_40_megapixels_is_accepted PASSED [ 26%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[png: first 33 bytes of the fixture] PASSED [ 27%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[png: truncated] PASSED [ 27%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[png: corrupt IDAT, valid CRC] PASSED [ 27%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[png: fewer rows than IHDR] PASSED [ 27%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[png: flipped byte] PASSED [ 28%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[png: bytes after IEND] PASSED [ 28%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[jpeg: no scan data after SOS] PASSED [ 28%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[jpeg: truncated] PASSED [ 28%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[review: png row filter 5] PASSED [ 29%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[review: png row filter 0xff] PASSED [ 29%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[review: png IDAT not-a-zlib-stream] PASSED [ 29%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[review: jpeg 27 bytes, no tables/scan] PASSED [ 29%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[review: jpeg SOS component not in SOF] PASSED [ 30%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[mime: png bytes declared jpeg] PASSED [ 30%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[mime: jpeg bytes declared png] PASSED [ 30%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[mime: gif declared png] PASSED [ 30%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[mime: html declared png] PASSED [ 31%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[type: text/plain] PASSED [ 31%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[type: octet-stream] PASSED [ 31%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[type: multipart] PASSED [ 32%]
|
||||
student-tests/test_attachments.py::test_bad_content_is_415[type: missing] PASSED [ 32%]
|
||||
student-tests/test_attachments.py::test_empty_body_is_400 PASSED [ 32%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[missing] PASSED [ 32%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=] PASSED [ 33%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=%20%20%20] PASSED [ 33%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=.] PASSED [ 33%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=..] PASSED [ 33%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%2Fb.png] PASSED [ 34%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%5Cb.png] PASSED [ 34%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a%00b.png] PASSED [ 34%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=c1%C2%85.png] PASSED [ 34%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=tab%09.png] PASSED [ 35%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=photo%E2%80%AEgnp.exe] PASSED [ 35%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=%FF%FE.png] PASSED [ 35%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=a.png&filename=b.png] PASSED [ 35%]
|
||||
student-tests/test_attachments.py::test_bad_filename_is_400[filename=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 36%]
|
||||
student-tests/test_attachments.py::test_unicode_filename_is_kept_and_safely_exposed PASSED [ 36%]
|
||||
student-tests/test_attachments.py::test_variant_parameter PASSED [ 36%]
|
||||
student-tests/test_balancing.py::test_one_session_is_served_by_every_real_replica PASSED [ 37%]
|
||||
student-tests/test_balancing.py::test_write_on_one_replica_is_read_on_the_other PASSED [ 37%]
|
||||
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-pg_hba] PASSED [ 37%]
|
||||
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[0-network] PASSED [ 37%]
|
||||
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-pg_hba] PASSED [ 38%]
|
||||
student-tests/test_balancing.py::test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready[1-network] PASSED [ 38%]
|
||||
student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[0] PASSED [ 38%]
|
||||
student-tests/test_balancing.py::test_graceful_stop_of_each_replica_loses_no_request[1] PASSED [ 38%]
|
||||
student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[0] PASSED [ 39%]
|
||||
student-tests/test_balancing.py::test_killed_replica_bounded_errors_recovery_and_no_duplicates[1] PASSED [ 39%]
|
||||
student-tests/test_balancing.py::test_graceful_shutdown_order_in_logs PASSED [ 39%]
|
||||
student-tests/test_balancing.py::test_full_restart_lb_rediscovers_all_replicas PASSED [ 39%]
|
||||
student-tests/test_balancing.py::test_scale_up_and_down_is_picked_up_without_errors PASSED [ 40%]
|
||||
student-tests/test_balancing.py::test_compose_policy_limits_ports_secrets_images PASSED [ 40%]
|
||||
student-tests/test_balancing.py::test_running_containers_have_limits_and_no_bypass_ports PASSED [ 40%]
|
||||
student-tests/test_balancing.py::test_migrations_ran_once_by_the_migrate_service PASSED [ 40%]
|
||||
student-tests/test_consistency.py::test_interrupted_and_stalled_uploads_leave_nothing PASSED [ 41%]
|
||||
student-tests/test_consistency.py::test_crash_after_staging_before_job_is_cleaned_by_janitor PASSED [ 41%]
|
||||
student-tests/test_consistency.py::test_lost_answer_after_job_commit_still_becomes_ready PASSED [ 41%]
|
||||
student-tests/test_consistency.py::test_fresh_pending_upload_is_not_touched_by_janitor PASSED [ 41%]
|
||||
student-tests/test_db.py::test_registration_race_exactly_one_201_and_seven_409 PASSED [ 42%]
|
||||
student-tests/test_db.py::test_create_chat_rolls_back_after_the_chat_row_was_inserted PASSED [ 42%]
|
||||
student-tests/test_db.py::test_migrations_rerun_is_a_noop PASSED [ 42%]
|
||||
student-tests/test_db.py::test_100_messages_from_10_clients_keep_order_and_cursor_walk PASSED [ 43%]
|
||||
student-tests/test_db.py::test_secrets_are_not_stored_in_plain_text PASSED [ 43%]
|
||||
student-tests/test_explain.py::test_late_page_uses_index_range_not_filter PASSED [ 43%]
|
||||
student-tests/test_failures.py::test_valkey_failure[stop] PASSED [ 43%]
|
||||
student-tests/test_failures.py::test_valkey_failure[pause] PASSED [ 44%]
|
||||
student-tests/test_failures.py::test_postgres_failure[stop] PASSED [ 44%]
|
||||
student-tests/test_failures.py::test_postgres_failure[pause] PASSED [ 44%]
|
||||
student-tests/test_failures.py::test_locked_table_gives_bounded_503 PASSED [ 44%]
|
||||
student-tests/test_failures.py::test_logout_during_valkey_outage_is_never_lost PASSED [ 45%]
|
||||
student-tests/test_idempotency.py::test_replay_returns_the_same_201_and_message_from_every_replica PASSED [ 45%]
|
||||
student-tests/test_idempotency.py::test_other_body_with_the_same_key_is_409_and_keeps_the_original PASSED [ 45%]
|
||||
student-tests/test_idempotency.py::test_key_order_and_missing_attachment_ids_are_the_same_body PASSED [ 45%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[] PASSED [ 46%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 46%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[\u043a\u043b\u044e\u0447] PASSED [ 46%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a b] PASSED [ 46%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a/b] PASSED [ 47%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a,b] PASSED [ 47%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[a\tb] PASSED [ 47%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_400[*] PASSED [ 48%]
|
||||
student-tests/test_idempotency.py::test_repeated_key_header_is_400 PASSED [ 48%]
|
||||
student-tests/test_idempotency.py::test_longest_key_and_every_allowed_character PASSED [ 48%]
|
||||
student-tests/test_idempotency.py::test_scope_is_user_plus_chat_plus_key PASSED [ 48%]
|
||||
student-tests/test_idempotency.py::test_outsider_and_revoked_session_never_see_the_stored_result PASSED [ 49%]
|
||||
student-tests/test_idempotency.py::test_concurrent_same_key_through_different_replicas_creates_exactly_one PASSED [ 49%]
|
||||
student-tests/test_idempotency.py::test_concurrent_same_key_different_bodies_one_wins PASSED [ 49%]
|
||||
student-tests/test_idempotency.py::test_failure_after_the_key_insert_rolls_everything_back PASSED [ 49%]
|
||||
student-tests/test_idempotency.py::test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message PASSED [ 50%]
|
||||
student-tests/test_idempotency.py::test_lock_timeout_is_a_transient_503_that_is_not_recorded PASSED [ 50%]
|
||||
student-tests/test_idempotency.py::test_replay_after_the_serving_replica_is_stopped PASSED [ 50%]
|
||||
student-tests/test_pipeline_flow.py::test_upload_is_202_queued_then_ready_with_contract_variants PASSED [ 50%]
|
||||
student-tests/test_pipeline_flow.py::test_queued_attachment_is_attachable_failed_is_not PASSED [ 51%]
|
||||
student-tests/test_pipeline_flow.py::test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready PASSED [ 51%]
|
||||
student-tests/test_pipeline_flow.py::test_broker_down_commit_to_publish_gap_is_closed_by_the_outbox PASSED [ 51%]
|
||||
student-tests/test_pipeline_flow.py::test_relay_down_job_waits_in_outbox_not_lost PASSED [ 51%]
|
||||
student-tests/test_pipeline_flow.py::test_crash_after_publish_before_confirm_is_recorded_gives_a_harmless_duplicate PASSED [ 52%]
|
||||
student-tests/test_pipeline_flow.py::test_duplicate_and_reordered_deliveries_are_noops PASSED [ 52%]
|
||||
student-tests/test_pipeline_flow.py::test_worker_crash_before_side_effect_lease_expires_and_job_completes PASSED [ 52%]
|
||||
student-tests/test_pipeline_flow.py::test_worker_crash_after_s3_write_before_commit_republishes_without_duplicates PASSED [ 53%]
|
||||
student-tests/test_pipeline_flow.py::test_worker_crash_after_commit_before_ack_redelivery_is_noop PASSED [ 53%]
|
||||
student-tests/test_pipeline_flow.py::test_transient_errors_are_retried_with_jittered_backoff PASSED [ 53%]
|
||||
student-tests/test_pipeline_flow.py::test_retry_exhausted_failed_registry_then_replay PASSED [ 53%]
|
||||
student-tests/test_pipeline_flow.py::test_permanent_error_fails_at_once_without_retries PASSED [ 54%]
|
||||
student-tests/test_pipeline_flow.py::test_memory_and_time_limits_of_the_worker PASSED [ 54%]
|
||||
student-tests/test_pipeline_flow.py::test_bad_message_is_dead_lettered_not_processed PASSED [ 54%]
|
||||
student-tests/test_pipeline_flow.py::test_hung_worker_loses_its_lease_and_cannot_publish PASSED [ 54%]
|
||||
student-tests/test_pipeline_flow.py::test_graceful_worker_stop_finishes_the_running_attempt PASSED [ 55%]
|
||||
student-tests/test_pipeline_flow.py::test_concurrency_is_bounded_by_prefetch PASSED [ 55%]
|
||||
student-tests/test_pipeline_flow.py::test_queued_staging_is_never_deleted_even_after_retention PASSED [ 55%]
|
||||
student-tests/test_pipeline_flow.py::test_retention_deletes_failed_staging_then_replay_is_refused PASSED [ 55%]
|
||||
student-tests/test_pipeline_flow.py::test_cleanup_racing_a_replay_never_deletes_the_replayed_source PASSED [ 56%]
|
||||
student-tests/test_pipeline_flow.py::test_reprocess_of_a_not_ready_upload_is_refused PASSED [ 56%]
|
||||
student-tests/test_pipeline_flow.py::test_zz_invariants_hold_after_all_scenarios PASSED [ 56%]
|
||||
student-tests/test_sessions.py::test_short_ttl_expires[valkey] PASSED [ 56%]
|
||||
student-tests/test_sessions.py::test_short_ttl_expires[postgres] PASSED [ 57%]
|
||||
student-tests/test_sessions.py::test_restart_does_not_extend_ttl PASSED [ 57%]
|
||||
student-tests/test_sessions.py::test_postgres_session_store PASSED [ 57%]
|
||||
student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[stop] PASSED [ 58%]
|
||||
student-tests/test_storage_failures.py::test_s3_outage_is_503_and_text_keeps_working[pause] PASSED [ 58%]
|
||||
student-tests/test_storage_failures.py::test_upload_survives_failover_of_the_serving_replica PASSED [ 58%]
|
||||
student-tests/test_storage_failures.py::test_bucket_is_private_outside_the_api PASSED [ 58%]
|
||||
student-tests/test_storage_failures.py::test_s3_restart_keeps_objects PASSED [ 59%]
|
||||
student-tests/unit/test_images.py::test_public_fixture_is_accepted PASSED [ 59%]
|
||||
student-tests/unit/test_images.py::test_valid_png_kinds[rgba] PASSED [ 59%]
|
||||
student-tests/unit/test_images.py::test_valid_png_kinds[palette] PASSED [ 59%]
|
||||
student-tests/unit/test_images.py::test_valid_png_kinds[1bit] PASSED [ 60%]
|
||||
student-tests/unit/test_images.py::test_valid_png_kinds[gray] PASSED [ 60%]
|
||||
student-tests/unit/test_images.py::test_valid_png_kinds[noise] PASSED [ 60%]
|
||||
student-tests/unit/test_images.py::test_valid_png_kinds[1x1] PASSED [ 60%]
|
||||
student-tests/unit/test_images.py::test_interlaced_png_is_accepted PASSED [ 61%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[first 33 bytes of the fixture] PASSED [ 61%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[truncated (no IEND)] PASSED [ 61%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[truncated in IDAT] PASSED [ 61%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[flipped byte -> CRC error] PASSED [ 62%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[garbage IDAT with valid CRC] PASSED [ 62%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[fewer rows than IHDR] PASSED [ 62%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[bytes after IEND (polyglot)] PASSED [ 62%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[text] PASSED [ 63%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[gif] PASSED [ 63%]
|
||||
student-tests/unit/test_images.py::test_broken_png_is_415[two bytes of a signature] PASSED [ 63%]
|
||||
student-tests/unit/test_images.py::test_pixel_limit_is_checked_from_the_header PASSED [ 64%]
|
||||
student-tests/unit/test_images.py::test_png_verification_is_streaming_and_fast PASSED [ 64%]
|
||||
student-tests/unit/test_images.py::test_jpeg_header_is_parsed_incrementally PASSED [ 64%]
|
||||
student-tests/unit/test_images.py::test_valid_jpeg[base] SKIPPED (dj...) [ 64%]
|
||||
student-tests/unit/test_images.py::test_valid_jpeg[prog] SKIPPED (dj...) [ 65%]
|
||||
student-tests/unit/test_images.py::test_broken_jpeg_is_415[no scan data after SOS-] SKIPPED [ 65%]
|
||||
student-tests/unit/test_images.py::test_broken_jpeg_is_415[truncated-] SKIPPED [ 65%]
|
||||
student-tests/unit/test_images.py::test_good_filenames[photo.png] PASSED [ 65%]
|
||||
student-tests/unit/test_images.py::test_good_filenames[\u0444\u043e\u0442\u043e \u043e\u0442\u043f\u0443\u0441\u043a\u0430.jpg] PASSED [ 66%]
|
||||
student-tests/unit/test_images.py::test_good_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 66%]
|
||||
student-tests/unit/test_images.py::test_good_filenames[emoji \U0001f468\u200d\U0001f469\u200d\U0001f467.png] PASSED [ 66%]
|
||||
student-tests/unit/test_images.py::test_good_filenames[ x ] PASSED [ 66%]
|
||||
student-tests/unit/test_images.py::test_good_filenames[..x] PASSED [ 67%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[] PASSED [ 67%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[ ] PASSED [ 67%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[\u3000] PASSED [ 67%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[.] PASSED [ 68%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[..] PASSED [ 68%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[a/b.png] PASSED [ 68%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[a\\b.png] PASSED [ 69%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[a\x00b] PASSED [ 69%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[c1\x85.png] PASSED [ 69%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[tab\t.png] PASSED [ 69%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[del\x7f.png] PASSED [ 70%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[x\u2028y] PASSED [ 70%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[photo\u202egnp.exe] PASSED [ 70%]
|
||||
student-tests/unit/test_images.py::test_bad_filenames[aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 70%]
|
||||
student-tests/unit/test_images.py::test_filename_must_be_single_and_present PASSED [ 71%]
|
||||
student-tests/unit/test_images.py::test_query_must_be_valid_utf8 PASSED [ 71%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size0-256-expected0] PASSED [ 71%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size1-256-expected1] PASSED [ 71%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size2-256-expected2] PASSED [ 72%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size3-256-expected3] PASSED [ 72%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size4-256-expected4] PASSED [ 72%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size5-256-expected5] PASSED [ 72%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size6-256-expected6] PASSED [ 73%]
|
||||
student-tests/unit/test_pipeline.py::test_fit_rounds_down_without_upscaling[size7-1280-expected7] PASSED [ 73%]
|
||||
student-tests/unit/test_pipeline.py::test_center_square PASSED [ 73%]
|
||||
student-tests/unit/test_pipeline.py::test_public_fixture_variants_match_the_contract PASSED [ 74%]
|
||||
student-tests/unit/test_pipeline.py::test_small_image_is_not_upscaled_but_crop_is PASSED [ 74%]
|
||||
student-tests/unit/test_pipeline.py::test_big_image_is_bounded_by_1280 PASSED [ 74%]
|
||||
student-tests/unit/test_pipeline.py::test_exif_orientation_is_applied_and_dropped PASSED [ 74%]
|
||||
student-tests/unit/test_pipeline.py::test_alpha_is_composited_on_white_for_jpeg_and_kept_for_webp PASSED [ 75%]
|
||||
student-tests/unit/test_pipeline.py::test_metadata_is_not_copied PASSED [ 75%]
|
||||
student-tests/unit/test_pipeline.py::test_png_modes_are_processed[RGBA] PASSED [ 75%]
|
||||
student-tests/unit/test_pipeline.py::test_png_modes_are_processed[P] PASSED [ 75%]
|
||||
student-tests/unit/test_pipeline.py::test_png_modes_are_processed[1] PASSED [ 76%]
|
||||
student-tests/unit/test_pipeline.py::test_png_modes_are_processed[L] PASSED [ 76%]
|
||||
student-tests/unit/test_pipeline.py::test_png_modes_are_processed[LA] PASSED [ 76%]
|
||||
student-tests/unit/test_pipeline.py::test_png_modes_are_processed[RGB] PASSED [ 76%]
|
||||
student-tests/unit/test_pipeline.py::test_16_bit_grey_is_scaled_not_clipped PASSED [ 77%]
|
||||
student-tests/unit/test_pipeline.py::test_pipeline_versions_differ_in_bytes_not_in_geometry PASSED [ 77%]
|
||||
student-tests/unit/test_pipeline.py::test_pixel_limit_is_checked_before_decoding PASSED [ 77%]
|
||||
student-tests/unit/test_pipeline.py::test_child_process_protocol_round_trip PASSED [ 77%]
|
||||
student-tests/unit/test_pipeline.py::test_encoded_message_matches_jobs_schema_and_carries_ids_only PASSED [ 78%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change0] PASSED [ 78%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change1] PASSED [ 78%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change2] PASSED [ 79%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change3] PASSED [ 79%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change4] PASSED [ 79%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change5] PASSED [ 79%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change6] PASSED [ 80%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change7] PASSED [ 80%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change8] PASSED [ 80%]
|
||||
student-tests/unit/test_pipeline.py::test_invalid_messages_are_rejected[change9] PASSED [ 80%]
|
||||
student-tests/unit/test_pipeline.py::test_garbage_is_rejected PASSED [ 81%]
|
||||
student-tests/unit/test_pipeline.py::test_backoff_is_full_jitter_and_capped PASSED [ 81%]
|
||||
student-tests/unit/test_pipeline.py::test_pipeline_settings PASSED [ 81%]
|
||||
student-tests/unit/test_unit.py::test_valid_passwords[Twelve-chars] PASSED [ 81%]
|
||||
student-tests/unit/test_unit.py::test_valid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 82%]
|
||||
student-tests/unit/test_unit.py::test_valid_passwords[\u043f\u0430\u0440\u043e\u043b\u044c:\u0441:\u0434\u0432\u043e\u0435\u0442\u043e\u0447\u0438\u044f\u043c\u0438] PASSED [ 82%]
|
||||
student-tests/unit/test_unit.py::test_valid_passwords[\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600] PASSED [ 82%]
|
||||
student-tests/unit/test_unit.py::test_valid_passwords[a b c d e f g] PASSED [ 82%]
|
||||
student-tests/unit/test_unit.py::test_invalid_passwords[short-11ch] PASSED [ 83%]
|
||||
student-tests/unit/test_unit.py::test_invalid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED [ 83%]
|
||||
student-tests/unit/test_unit.py::test_invalid_passwords[ ] PASSED [ 83%]
|
||||
student-tests/unit/test_unit.py::test_invalid_passwords[\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000] PASSED [ 83%]
|
||||
student-tests/unit/test_unit.py::test_invalid_passwords[\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800] PASSED [ 84%]
|
||||
student-tests/unit/test_unit.py::test_invalid_passwords[None] PASSED [ 84%]
|
||||
student-tests/unit/test_unit.py::test_invalid_passwords[123456789012] PASSED [ 84%]
|
||||
student-tests/unit/test_unit.py::test_text_may_contain_nul_but_not_lone_surrogates PASSED [ 85%]
|
||||
student-tests/unit/test_unit.py::test_cursor_roundtrip_and_binding PASSED [ 85%]
|
||||
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|253402300800000000|11111111-1111-4111-8111-111111111111] PASSED [ 85%]
|
||||
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|9999999999999999999|11111111-1111-4111-8111-111111111111] PASSED [ 85%]
|
||||
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|-1|11111111-1111-4111-8111-111111111111] PASSED [ 86%]
|
||||
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|not-a-uuid] PASSED [ 86%]
|
||||
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-11111111111G] PASSED [ 86%]
|
||||
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v2|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-111111111111] PASSED [ 86%]
|
||||
student-tests/unit/test_unit.py::test_time_helpers_are_exact PASSED [ 87%]
|
||||
student-tests/unit/test_unit.py::test_basic_is_parsed_and_password_may_contain_colons PASSED [ 87%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[None] PASSED [ 87%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[] PASSED [ 87%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic] PASSED [ 88%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic \xe9] PASSED [ 88%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic !!!!] PASSED [ 88%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic bm8tY29sb24=] PASSED [ 88%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic //46cHc=] PASSED [ 89%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic OnBhc3N3b3Jk] PASSED [ 89%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic YWxpY2U6] PASSED [ 89%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic 0JDQu9C40YHQsDpwdw==] PASSED [ 90%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 90%]
|
||||
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA] PASSED [ 90%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[None] PASSED [ 90%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[] PASSED [ 91%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer] PASSED [ 91%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer short] PASSED [ 91%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!] PASSED [ 91%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Basic YWxpY2U6cGFzc3dvcmQtMTIz] PASSED [ 92%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED [ 92%]
|
||||
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED [ 92%]
|
||||
student-tests/unit/test_unit.py::test_bearer_ok PASSED [ 92%]
|
||||
student-tests/unit/test_unit.py::test_argon2id_hash_and_verify PASSED [ 93%]
|
||||
student-tests/unit/test_unit.py::test_s3_settings_are_validated_and_contract_limits_cannot_be_raised PASSED [ 93%]
|
||||
student-tests/unit/test_unit.py::test_settings_read_secret_files_and_contract_names PASSED [ 93%]
|
||||
student-tests/unit/test_unit.py::test_valid_idempotency_keys[a] PASSED [ 93%]
|
||||
student-tests/unit/test_unit.py::test_valid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 94%]
|
||||
student-tests/unit/test_unit.py::test_valid_idempotency_keys[A-Za-z0-9._:-] PASSED [ 94%]
|
||||
student-tests/unit/test_unit.py::test_valid_idempotency_keys[2026-10-16T10:00:00.1] PASSED [ 94%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[None] PASSED [ 95%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[] PASSED [ 95%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[kkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkkk] PASSED [ 95%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\u043a\u043b\u044e\u0447] PASSED [ 95%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a b] PASSED [ 96%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a/b] PASSED [ 96%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a,b] PASSED [ 96%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\tb] PASSED [ 96%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[a\x00b] PASSED [ 97%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[*] PASSED [ 97%]
|
||||
student-tests/unit/test_unit.py::test_invalid_idempotency_keys[\xe9] PASSED [ 97%]
|
||||
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"a","text":"b"}] PASSED [ 97%]
|
||||
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"a":{"x":1,"x":2}}] PASSED [ 98%]
|
||||
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[\xff] PASSED [ 98%]
|
||||
student-tests/unit/test_unit.py::test_duplicate_keys_and_garbage_json_are_rejected[{"text":"x"} {}] PASSED [ 98%]
|
||||
student-tests/unit/test_unit.py::test_attachment_ids_absent_equals_empty_and_is_validated PASSED [ 98%]
|
||||
student-tests/unit/test_unit.py::test_request_fingerprint_ignores_key_order_and_default_attachments PASSED [ 99%]
|
||||
student-tests/unit/test_unit.py::test_instance_id_defaults_to_hostname_and_is_validated PASSED [ 99%]
|
||||
student-tests/unit/test_unit.py::test_idempotency_ttl_cannot_be_below_24_hours PASSED [ 99%]
|
||||
student-tests/unit/test_unit.py::test_message_write_path_setting PASSED [100%]
|
||||
|
||||
================= 377 passed, 4 skipped in 1515.54s (0:25:15) =================
|
||||
[exit 0, 1516.9 s]
|
||||
@@ -0,0 +1,74 @@
|
||||
# make test-failover with scripts/stop-one: which container is really stopped, restore, repeat
|
||||
# 2026-10-31T14:21:25Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT
|
||||
|
||||
--- run 1; running API replicas BEFORE:
|
||||
mtusi-lab5-api-1 host=d08185c0e3f3 running exit=0 pid=91728 started=2026-10-31T14:21:03.235
|
||||
mtusi-lab5-api-2 host=85c7cde066aa running exit=0 pid=91671 started=2026-10-31T14:21:02.858
|
||||
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
|
||||
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 126, "failed_attempts": 0, "first_recovery_seconds": 6.326}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
[exit 0, 19.4 s]
|
||||
|
||||
AFTER (the stopped replica, left stopped by the hook):
|
||||
mtusi-lab5-api-1 host=d08185c0e3f3 exited exit=0 pid=0 started=2026-10-31T14:21:03.235
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
|
||||
api2 172.28.0.2 UP mtusi-lab5-api-2 X-Instance-Id=85c7cde066aa
|
||||
(other slots: MAINT, no container)
|
||||
[exit 0, 0.9 s]
|
||||
|
||||
$ sh scripts/restore-api.sh
|
||||
mtusi-lab5-api-1 Up 6 seconds (healthy)
|
||||
mtusi-lab5-api-2 Up 50 seconds (healthy)
|
||||
Container mtusi-lab5-api-2 Running
|
||||
Container mtusi-lab5-api-1 Starting
|
||||
Container mtusi-lab5-api-1 Started
|
||||
Container mtusi-lab5-api-2 Waiting
|
||||
Container mtusi-lab5-api-1 Waiting
|
||||
Container mtusi-lab5-api-2 Healthy
|
||||
Container mtusi-lab5-api-1 Healthy
|
||||
[exit 0, 6.9 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
|
||||
2 API replica(s) UP in HAProxy after 0.6 s: ['85c7cde066aa', 'd08185c0e3f3']
|
||||
[exit 0, 1.4 s]
|
||||
|
||||
--- run 2; running API replicas BEFORE:
|
||||
mtusi-lab5-api-1 host=d08185c0e3f3 running exit=0 pid=94963 started=2026-10-31T14:21:47.426
|
||||
mtusi-lab5-api-2 host=85c7cde066aa running exit=0 pid=91671 started=2026-10-31T14:21:02.858
|
||||
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.cmd
|
||||
python tests/resilience.py --mode failover --action-script "scripts/stop-one.cmd" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 6.074}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
[exit 0, 18.0 s]
|
||||
|
||||
AFTER (the stopped replica, left stopped by the hook):
|
||||
mtusi-lab5-api-2 host=85c7cde066aa exited exit=0 pid=0 started=2026-10-31T14:21:02.858
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py status
|
||||
api1 172.28.0.3 UP mtusi-lab5-api-1 X-Instance-Id=d08185c0e3f3
|
||||
(other slots: MAINT, no container)
|
||||
[exit 0, 0.7 s]
|
||||
|
||||
$ sh scripts/restore-api.sh
|
||||
mtusi-lab5-api-1 Up 34 seconds (healthy)
|
||||
mtusi-lab5-api-2 Up 6 seconds (healthy)
|
||||
Container mtusi-lab5-api-1 Running
|
||||
Container mtusi-lab5-api-2 Starting
|
||||
Container mtusi-lab5-api-2 Started
|
||||
Container mtusi-lab5-api-2 Waiting
|
||||
Container mtusi-lab5-api-1 Waiting
|
||||
Container mtusi-lab5-api-2 Healthy
|
||||
Container mtusi-lab5-api-1 Healthy
|
||||
[exit 0, 7.4 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
|
||||
2 API replica(s) UP in HAProxy after 0.6 s: ['85c7cde066aa', 'd08185c0e3f3']
|
||||
[exit 0, 1.6 s]
|
||||
|
||||
Replicas stopped by the public test across runs: ['mtusi-lab5-api-1', 'mtusi-lab5-api-2']
|
||||
@@ -0,0 +1,38 @@
|
||||
# make test-persistence: restart of LB, API replicas, relay, worker, PostgreSQL, Valkey, S3, RabbitMQ
|
||||
# 2026-10-31T14:20:06Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT
|
||||
|
||||
Containers BEFORE:
|
||||
mtusi-lab5-api-1 host=d08185c0e3f3 running exit=0 pid=83074 started=2026-10-31T14:18:45.487
|
||||
mtusi-lab5-api-2 host=85c7cde066aa running exit=0 pid=82998 started=2026-10-31T14:18:45.132
|
||||
mtusi-lab5-broker-1 host=broker running exit=0 pid=82136 started=2026-10-31T14:18:30.831
|
||||
mtusi-lab5-lb-1 host=a22e1c19fc90 running exit=0 pid=83486 started=2026-10-31T14:18:51.489
|
||||
mtusi-lab5-migrate-1 host=fa0a8a1c9191 exited exit=0 pid=0 started=2026-10-31T14:18:36.261
|
||||
mtusi-lab5-postgres-1 host=7961919f90c9 running exit=0 pid=82047 started=2026-10-31T14:18:30.501
|
||||
mtusi-lab5-relay-1 host=9b6249446ece running exit=0 pid=82799 started=2026-10-31T14:18:41.997
|
||||
mtusi-lab5-s3-1 host=bdee3ea030a4 running exit=0 pid=81901 started=2026-10-31T14:18:29.927
|
||||
mtusi-lab5-s3-init-1 host=2c85c6921c88 exited exit=0 pid=0 started=2026-10-31T14:18:38.517
|
||||
mtusi-lab5-valkey-1 host=d53b8a88e5b5 running exit=0 pid=81977 started=2026-10-31T14:18:30.236
|
||||
mtusi-lab5-worker-1 host=ee8c11f59aa8 running exit=0 pid=82939 started=2026-10-31T14:18:44.768
|
||||
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.cmd
|
||||
python tests/resilience.py --mode persistence --action-script "scripts/restart.cmd" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
[exit 0, 76.0 s]
|
||||
|
||||
Containers AFTER (new pid/started for lb, api-*, relay, worker, postgres, valkey, s3, broker; volumes kept):
|
||||
mtusi-lab5-api-1 host=d08185c0e3f3 running exit=0 pid=91728 started=2026-10-31T14:21:03.235
|
||||
mtusi-lab5-api-2 host=85c7cde066aa running exit=0 pid=91671 started=2026-10-31T14:21:02.858
|
||||
mtusi-lab5-broker-1 host=broker running exit=0 pid=89550 started=2026-10-31T14:20:17.480
|
||||
mtusi-lab5-lb-1 host=a22e1c19fc90 running exit=0 pid=92390 started=2026-10-31T14:21:12.759
|
||||
mtusi-lab5-migrate-1 host=fa0a8a1c9191 exited exit=0 pid=0 started=2026-10-31T14:21:14.217
|
||||
mtusi-lab5-postgres-1 host=7961919f90c9 running exit=0 pid=89433 started=2026-10-31T14:20:16.605
|
||||
mtusi-lab5-relay-1 host=9b6249446ece running exit=0 pid=92854 started=2026-10-31T14:21:16.976
|
||||
mtusi-lab5-s3-1 host=bdee3ea030a4 running exit=0 pid=90402 started=2026-10-31T14:20:36.421
|
||||
mtusi-lab5-s3-init-1 host=2c85c6921c88 exited exit=0 pid=0 started=2026-10-31T14:21:17.299
|
||||
mtusi-lab5-valkey-1 host=d53b8a88e5b5 running exit=0 pid=89376 started=2026-10-31T14:20:16.424
|
||||
mtusi-lab5-worker-1 host=ee8c11f59aa8 running exit=0 pid=93271 started=2026-10-31T14:21:22.639
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
|
||||
2 API replica(s) UP in HAProxy after 0.6 s: ['85c7cde066aa', 'd08185c0e3f3']
|
||||
[exit 0, 1.5 s]
|
||||
@@ -0,0 +1,112 @@
|
||||
# Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung worker) + read-only audit of the invariants
|
||||
# 2026-10-31T14:48:35Z, commit cf6e929 docs: PIPELINE, DEFENSE, DATA_MODEL, WINDOWS для lab5; черновик REPORT
|
||||
|
||||
$ C:\Users\wfa22\Desktop\Высоконагруженные приложения\lab5-clean\.venv\Scripts\python.exe bench/pipeline_faults.py --output evidence/track2 --uploads 5
|
||||
[2026-10-31T14:48:37Z] baseline_no_faults
|
||||
[2026-10-31T14:48:54Z] duplicate_publish
|
||||
[2026-10-31T14:49:25Z] duplicate_and_reverse_order
|
||||
[2026-10-31T14:49:56Z] relay_crash_after_confirm_before_mark
|
||||
[2026-10-31T14:50:27Z] worker_crash_before_side_effect
|
||||
[2026-10-31T14:51:19Z] worker_crash_after_s3_before_commit
|
||||
[2026-10-31T14:52:05Z] worker_crash_after_commit_before_ack
|
||||
[2026-10-31T14:53:10Z] transient_errors_then_success
|
||||
[2026-10-31T14:53:42Z] two_workers_duplicate_publish
|
||||
[2026-10-31T14:54:22Z] versions_scenario
|
||||
[2026-10-31T14:55:08Z] queued_reorder_scenario
|
||||
[2026-10-31T14:56:08Z] hung_worker_scenario
|
||||
bcd68c4d813f0d14a490156098130cab5e63ae21352e9fc620b8ac85f48df995
|
||||
bcd68c4d813f0d14a490156098130cab5e63ae21352e9fc620b8ac85f48df995
|
||||
scenario uploads ready claims problems
|
||||
baseline_no_faults 5 5 5 0
|
||||
duplicate_publish 5 5 5 0
|
||||
duplicate_and_reverse_order 5 5 5 0
|
||||
relay_crash_after_confirm_before_mark 5 5 5 0
|
||||
worker_crash_before_side_effect 5 5 14 0
|
||||
worker_crash_after_s3_before_commit 5 5 10 0
|
||||
worker_crash_after_commit_before_ack 5 5 5 0
|
||||
transient_errors_then_success 5 5 15 0
|
||||
two_workers_duplicate_publish 5 5 5 0
|
||||
stale_version_and_reorder 1 - - 0
|
||||
reordered_versions_in_queue 1 - 2 0
|
||||
hung_worker_docker_pause 1 - 2 0
|
||||
audit after all scenarios: ok=True violations=0 objects={'final': 523, 'staging': 2, 'published': 522, 'checksums_verified': True}
|
||||
[exit 0, 515.1 s]
|
||||
|
||||
$ docker compose run --rm auditor --verify-checksums
|
||||
{
|
||||
"ok": true,
|
||||
"violations": [],
|
||||
"notes": [],
|
||||
"attachments": {
|
||||
"failed": 4,
|
||||
"pending": 2,
|
||||
"ready": 129
|
||||
},
|
||||
"jobs": {
|
||||
"superseded": 2,
|
||||
"succeeded": 131,
|
||||
"failed": 6
|
||||
},
|
||||
"objects": {
|
||||
"final": 523,
|
||||
"staging": 2,
|
||||
"published": 522,
|
||||
"checksums_verified": true
|
||||
}
|
||||
}
|
||||
Container mtusi-lab5-auditor-run-e2fdeabd6d0f Creating
|
||||
Container mtusi-lab5-auditor-run-e2fdeabd6d0f Created
|
||||
[exit 0, 7.2 s]
|
||||
|
||||
$ docker compose run --rm jobctl stats
|
||||
{
|
||||
"jobs": {
|
||||
"failed": 6,
|
||||
"succeeded": 131,
|
||||
"superseded": 2
|
||||
},
|
||||
"outbox_unpublished": 0,
|
||||
"outbox_due": 0
|
||||
}
|
||||
Container mtusi-lab5-jobctl-run-6f68d2c5bb69 Creating
|
||||
Container mtusi-lab5-jobctl-run-6f68d2c5bb69 Created
|
||||
[exit 0, 5.0 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/lbctl.py wait-up 2
|
||||
2 API replica(s) UP in HAProxy after 0.7 s: ['1fb1ea74a42b', '492b7b24bd1f']
|
||||
[exit 0, 1.7 s]
|
||||
|
||||
$ C:\Users\wfa22\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
|
||||
{
|
||||
"lab": 5,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.066,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 5515,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 183.429,
|
||||
"successful_latency_ms": {
|
||||
"p50": 17.96,
|
||||
"p95": 43.491,
|
||||
"p99": 58.653
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
[exit 0, 31.5 s]
|
||||
|
||||
$ docker compose ps -a --format table {{.Service}} {{.Status}}
|
||||
SERVICE STATUS
|
||||
api Up 10 minutes (healthy)
|
||||
api Up 11 minutes (healthy)
|
||||
broker Up 23 minutes (healthy)
|
||||
lb Up 30 minutes (healthy)
|
||||
migrate Exited (0) About a minute ago
|
||||
postgres Up 11 minutes (healthy)
|
||||
relay Up 2 minutes (healthy)
|
||||
s3 Up 9 minutes (healthy)
|
||||
s3-init Exited (0) About a minute ago
|
||||
valkey Up 13 minutes (healthy)
|
||||
worker Up About a minute (healthy)
|
||||
[exit 0, 0.4 s]
|
||||
@@ -0,0 +1,277 @@
|
||||
{
|
||||
"experiment": "track 2: idempotency of the versioned pipeline under injected faults",
|
||||
"started": "2026-10-31T14:48:37Z",
|
||||
"finished": "2026-10-31T14:57:10Z",
|
||||
"host": {
|
||||
"os": "Windows-10-10.0.19045-SP0",
|
||||
"python": "3.14.6"
|
||||
},
|
||||
"uploads_per_plain_scenario": 5,
|
||||
"scenarios": [
|
||||
{
|
||||
"scenario": "baseline_no_faults",
|
||||
"faults": {
|
||||
"worker": {}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 5,
|
||||
"max_claims": 1,
|
||||
"time_to_ready_s": {
|
||||
"p50": 2.21,
|
||||
"max": 3.15
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "duplicate_publish",
|
||||
"faults": {
|
||||
"relay": {
|
||||
"RELAY_FAULT_POINTS": "duplicate_publish"
|
||||
}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 5,
|
||||
"max_claims": 1,
|
||||
"time_to_ready_s": {
|
||||
"p50": 1.89,
|
||||
"max": 2.52
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "duplicate_and_reverse_order",
|
||||
"faults": {
|
||||
"relay": {
|
||||
"RELAY_FAULT_POINTS": "duplicate_publish,reverse_order"
|
||||
}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 5,
|
||||
"max_claims": 1,
|
||||
"time_to_ready_s": {
|
||||
"p50": 1.89,
|
||||
"max": 2.52
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "relay_crash_after_confirm_before_mark",
|
||||
"faults": {
|
||||
"relay": {
|
||||
"RELAY_FAULT_POINTS": "crash_after_publish"
|
||||
}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 5,
|
||||
"max_claims": 1,
|
||||
"time_to_ready_s": {
|
||||
"p50": 1.89,
|
||||
"max": 2.83
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > fault_injected > succeeded",
|
||||
"created > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "worker_crash_before_side_effect",
|
||||
"faults": {
|
||||
"worker": {
|
||||
"WORKER_FAULT_POINTS": "crash_after_claim",
|
||||
"JOB_LEASE_SECONDS": "6",
|
||||
"ORPHAN_GRACE_SECONDS": "3",
|
||||
"CLEANUP_INTERVAL_SECONDS": "1"
|
||||
}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 14,
|
||||
"max_claims": 3,
|
||||
"time_to_ready_s": {
|
||||
"p50": 21.66,
|
||||
"max": 21.68
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > fault_injected > lease_expired > claimed > lease_expired > claimed > succeeded",
|
||||
"created > claimed > fault_injected > lease_expired > claimed > succeeded",
|
||||
"created > claimed > lease_expired > claimed > fault_injected > lease_expired > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "worker_crash_after_s3_before_commit",
|
||||
"faults": {
|
||||
"worker": {
|
||||
"WORKER_FAULT_POINTS": "crash_after_s3_write",
|
||||
"JOB_LEASE_SECONDS": "6",
|
||||
"ORPHAN_GRACE_SECONDS": "3",
|
||||
"CLEANUP_INTERVAL_SECONDS": "1"
|
||||
}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 10,
|
||||
"max_claims": 2,
|
||||
"time_to_ready_s": {
|
||||
"p50": 15.49,
|
||||
"max": 16.13
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > fault_injected > lease_expired > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "worker_crash_after_commit_before_ack",
|
||||
"faults": {
|
||||
"worker": {
|
||||
"WORKER_FAULT_POINTS": "crash_after_commit",
|
||||
"JOB_LEASE_SECONDS": "6",
|
||||
"ORPHAN_GRACE_SECONDS": "3",
|
||||
"CLEANUP_INTERVAL_SECONDS": "1"
|
||||
}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 5,
|
||||
"max_claims": 1,
|
||||
"time_to_ready_s": {
|
||||
"p50": 5.3,
|
||||
"max": 8.4
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > succeeded > fault_injected"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "transient_errors_then_success",
|
||||
"faults": {
|
||||
"worker": {
|
||||
"WORKER_FAULT_POINTS": "transient_error=2",
|
||||
"JOB_RETRY_BASE_SECONDS": "0.5",
|
||||
"JOB_LEASE_SECONDS": "6",
|
||||
"ORPHAN_GRACE_SECONDS": "3",
|
||||
"CLEANUP_INTERVAL_SECONDS": "1"
|
||||
}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 15,
|
||||
"max_claims": 3,
|
||||
"time_to_ready_s": {
|
||||
"p50": 4.11,
|
||||
"max": 4.13
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > retry_scheduled > claimed > retry_scheduled > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "two_workers_duplicate_publish",
|
||||
"faults": {
|
||||
"worker": {}
|
||||
},
|
||||
"uploads": 5,
|
||||
"ready": 5,
|
||||
"claims_total": 5,
|
||||
"max_claims": 1,
|
||||
"time_to_ready_s": {
|
||||
"p50": 1.58,
|
||||
"max": 2.21
|
||||
},
|
||||
"event_sequences": [
|
||||
"created > claimed > succeeded"
|
||||
],
|
||||
"problems": {}
|
||||
},
|
||||
{
|
||||
"scenario": "stale_version_and_reorder",
|
||||
"uploads": 1,
|
||||
"statuses_seen_during_reprocess": [
|
||||
"ready"
|
||||
],
|
||||
"job_states": [
|
||||
"succeeded:v1",
|
||||
"superseded:v2",
|
||||
"succeeded:v3"
|
||||
],
|
||||
"published_version": 3,
|
||||
"variant_rows_by_version": 6,
|
||||
"objects": 10,
|
||||
"events": [
|
||||
"created",
|
||||
"claimed",
|
||||
"succeeded",
|
||||
"created",
|
||||
"claimed",
|
||||
"created",
|
||||
"claimed",
|
||||
"succeeded",
|
||||
"superseded"
|
||||
],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"scenario": "reordered_versions_in_queue",
|
||||
"job_states": [
|
||||
"succeeded:v1",
|
||||
"superseded:v2",
|
||||
"succeeded:v3"
|
||||
],
|
||||
"claims": 2,
|
||||
"published_version": 3,
|
||||
"events": [
|
||||
"created",
|
||||
"claimed",
|
||||
"succeeded",
|
||||
"created",
|
||||
"created",
|
||||
"claimed",
|
||||
"succeeded",
|
||||
"superseded"
|
||||
],
|
||||
"problems": []
|
||||
},
|
||||
{
|
||||
"scenario": "hung_worker_docker_pause",
|
||||
"frozen": "bcd68c4d813f",
|
||||
"claims": 2,
|
||||
"events": [
|
||||
"created",
|
||||
"claimed",
|
||||
"lease_expired",
|
||||
"claimed",
|
||||
"succeeded"
|
||||
],
|
||||
"problems": []
|
||||
}
|
||||
],
|
||||
"audit_after_all": {
|
||||
"ok": true,
|
||||
"violations": [],
|
||||
"notes": 0,
|
||||
"objects": {
|
||||
"final": 523,
|
||||
"staging": 2,
|
||||
"published": 522,
|
||||
"checksums_verified": true
|
||||
}
|
||||
},
|
||||
"violations": 0
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
scenario uploads ready claims problems
|
||||
baseline_no_faults 5 5 5 0
|
||||
duplicate_publish 5 5 5 0
|
||||
duplicate_and_reverse_order 5 5 5 0
|
||||
relay_crash_after_confirm_before_mark 5 5 5 0
|
||||
worker_crash_before_side_effect 5 5 14 0
|
||||
worker_crash_after_s3_before_commit 5 5 10 0
|
||||
worker_crash_after_commit_before_ack 5 5 5 0
|
||||
transient_errors_then_success 5 5 15 0
|
||||
two_workers_duplicate_publish 5 5 5 0
|
||||
stale_version_and_reorder 1 - - 0
|
||||
reordered_versions_in_queue 1 - 2 0
|
||||
hung_worker_docker_pause 1 - 2 0
|
||||
audit after all scenarios: ok=True violations=0 objects={'final': 523, 'staging': 2, 'published': 522, 'checksums_verified': True}
|
||||
@@ -0,0 +1,116 @@
|
||||
# HAProxy 3.2 LTS — the single public entry point of the lab 4 stand (compose service `lb`).
|
||||
#
|
||||
# Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the
|
||||
# compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and
|
||||
# re-resolves every second, so scale up/down, a restarted container with a new IP and a
|
||||
# full stack restart are picked up without restarting the LB. A stopped container drops
|
||||
# out of DNS and its slot goes to MAINT.
|
||||
#
|
||||
# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey + S3 reachable, not
|
||||
# draining) every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide
|
||||
# anything on their own, and there are NO fallback servers without checks: a replica that
|
||||
# answers /health/live but not /health/ready never gets traffic while a fully ready one exists.
|
||||
#
|
||||
# Degraded mode (lab 4): S3 is a SHARED dependency. If it is down, /health/ready is 503 on
|
||||
# every replica and backend `api` has no server. Instead of answering 503 to everything, the
|
||||
# frontend then switches to backend `api_core`: the same replicas, checked by
|
||||
# /health/ready/core (PostgreSQL + Valkey, not draining). Chats and text messages keep working,
|
||||
# every attachment request is answered 503 by the API itself (fast, S3_TIMEOUT_SECONDS).
|
||||
# A replica that alone lost S3 (e.g. its storage network) is DOWN in `api` while the others are
|
||||
# UP, so it gets no traffic at all.
|
||||
# Worst case detection of a failed replica (a check answered at once counts at once, a check
|
||||
# that hangs counts only after `timeout check`):
|
||||
# fast explicit 503 (drain after SIGTERM): inter 1 s + fastinter 0.5 s ~= 1.5 s;
|
||||
# slow ready (PostgreSQL/Valkey unreachable: the answer takes ~1 s = timeout check)
|
||||
# or frozen process: inter 1 s + 1 s + fastinter 0.5 s + 1 s ~= 3.5 s.
|
||||
# Requests that could not even be SENT (connection refused/reset before the request)
|
||||
# are retried on another replica (retry-on conn-failure + redispatch) - safe for POST,
|
||||
# because the API has not seen them. Nothing that may have reached the API is retried by
|
||||
# the LB: an uncertain POST is retried by the CLIENT with the same Idempotency-Key.
|
||||
# No sticky sessions: sessions live in Valkey, every replica serves every user.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
# Runtime API for `show servers state` / `show stat` (scripts and tests, no network port).
|
||||
stats socket /var/lib/haproxy/admin.sock mode 600 level admin
|
||||
hard-stop-after 15s
|
||||
|
||||
resolvers docker
|
||||
nameserver dns1 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold other 1s
|
||||
hold refused 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold obsolete 1s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
option http-keep-alive
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s # > DB_TIMEOUT_SECONDS (3 s): the API answers 503 itself first
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 10s
|
||||
timeout queue 3s
|
||||
timeout check 1s
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option redispatch 1
|
||||
# Errors generated by the LB itself, in the contract error format with the same request id.
|
||||
# They carry no X-Instance-Id: no API replica produced them. 502/504 = the outcome of the
|
||||
# request is unknown (a replica died mid-request / did not answer in time): retry with the
|
||||
# same Idempotency-Key. 503 = no ready replica at all.
|
||||
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика оборвала ответ; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
|
||||
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет готовых API-реплик\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
|
||||
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика не ответила вовремя; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]"
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
# One request id end to end: keep a valid client X-Request-Id, otherwise create one; the
|
||||
# API echoes it, and errors generated by the LB itself carry the same value.
|
||||
http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ }
|
||||
http-request set-var(txn.request_id) req.hdr(x-request-id)
|
||||
log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\""
|
||||
use_backend api if { nbsrv(api) gt 0 }
|
||||
default_backend api_core
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
# Uploads (<= 10 MiB body + verification + S3 PUT): the API's own budget is
|
||||
# UPLOAD_READ_TIMEOUT 30 s + IMAGE_VERIFY 10 s + S3; the LB must not cut it at 10 s (a 504
|
||||
# after a successful upload would make the client upload a duplicate).
|
||||
acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$
|
||||
http-request set-timeout server 60s if is_upload METH_POST
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200
|
||||
server-template api 8 api:8080
|
||||
|
||||
# Degraded mode only (see the header): used when NO replica is fully ready.
|
||||
backend api_core
|
||||
balance roundrobin
|
||||
acl is_upload path_reg ^/api/v1/chats/[^/]+/attachments$
|
||||
http-request set-timeout server 60s if is_upload METH_POST
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready/core ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200
|
||||
server-template core 8 api:8080
|
||||
|
||||
# Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now.
|
||||
frontend stats
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /
|
||||
stats refresh 2s
|
||||
stats show-legends
|
||||
@@ -0,0 +1 @@
|
||||
[].
|
||||
@@ -0,0 +1,24 @@
|
||||
# RabbitMQ 4.3 of the lab 5 stand (compose service `broker`), mounted as conf.d/20-lab5.conf.
|
||||
# Only AMQP 0-9-1 on 5672 inside the internal network `queue`: no management UI, no Prometheus
|
||||
# listener (enabled_plugins is empty), nothing published on the host.
|
||||
listeners.tcp.default = 5672
|
||||
|
||||
# Users, permissions and topology are imported from files at every boot (idempotent):
|
||||
# /etc/rabbitmq/definitions.d/topology.json exchanges, quorum queues, bindings (this repo)
|
||||
# /etc/rabbitmq/definitions.d/users.json users relay/worker + permissions (Docker secret,
|
||||
# generated by scripts/init_secrets.py)
|
||||
# With imported definitions no default `guest` user is created.
|
||||
definitions.import_backend = local_filesystem
|
||||
definitions.local.path = /etc/rabbitmq/definitions.d
|
||||
definitions.skip_if_unchanged = false
|
||||
|
||||
# A delivery that stays unacked longer than this closes the consumer's channel and the message
|
||||
# is returned to the queue (a worker that hangs while connected). Far above JOB_TIMEOUT_SECONDS.
|
||||
consumer_timeout = 300000
|
||||
|
||||
# The container has a 512 MiB limit; flow control starts well before the cgroup OOM.
|
||||
vm_memory_high_watermark.relative = 0.5
|
||||
disk_free_limit.absolute = 200MB
|
||||
|
||||
log.console = true
|
||||
log.console.level = info
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"vhosts": [{"name": "/"}],
|
||||
"exchanges": [
|
||||
{"name": "messenger.jobs", "vhost": "/", "type": "direct", "durable": true, "auto_delete": false,
|
||||
"internal": false, "arguments": {}},
|
||||
{"name": "messenger.jobs.dlx", "vhost": "/", "type": "fanout", "durable": true, "auto_delete": false,
|
||||
"internal": false, "arguments": {}}
|
||||
],
|
||||
"queues": [
|
||||
{"name": "messenger.attachment-jobs", "vhost": "/", "durable": true, "auto_delete": false,
|
||||
"arguments": {"x-queue-type": "quorum", "x-delivery-limit": 20, "x-dead-letter-exchange": "messenger.jobs.dlx"}},
|
||||
{"name": "messenger.attachment-jobs.dlq", "vhost": "/", "durable": true, "auto_delete": false,
|
||||
"arguments": {"x-queue-type": "quorum"}}
|
||||
],
|
||||
"bindings": [
|
||||
{"source": "messenger.jobs", "vhost": "/", "destination": "messenger.attachment-jobs", "destination_type": "queue",
|
||||
"routing_key": "attachment.process.v1", "arguments": {}},
|
||||
{"source": "messenger.jobs.dlx", "vhost": "/", "destination": "messenger.attachment-jobs.dlq",
|
||||
"destination_type": "queue", "routing_key": "", "arguments": {}}
|
||||
]
|
||||
}
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/bin/sh
|
||||
# Entry point of the compose service `s3` (SeaweedFS 4.46, one container: master + volume +
|
||||
# filer + S3 gateway). Renders the S3 identities from Docker secrets, then execs `weed`.
|
||||
#
|
||||
# Two private buckets (lab 5): S3_STAGING_BUCKET holds originals accepted by the API before 202,
|
||||
# S3_BUCKET holds the published originals and variants written by the worker.
|
||||
# Identities (SeaweedFS s3.json, actions scoped to a bucket):
|
||||
# bootstrap-admin Admin (create buckets) -> only s3-init has these keys
|
||||
# messenger-api Write staging, Read final -> API replicas (no List, no Admin)
|
||||
# messenger-worker Read + Write staging, Read + Write final -> worker (no List, no Admin)
|
||||
# messenger-auditor Read + List both buckets (no Write) -> invariant checker (tools profile)
|
||||
# There is NO anonymous identity, so every unsigned request is refused: the buckets are private.
|
||||
#
|
||||
# Exposure: master (9333), volume (8080), filer (8888) with their web UIs and gRPC listen on
|
||||
# 127.0.0.1 INSIDE the container only (-ip.bind); only the S3 API port 8333 listens on the
|
||||
# container network `storage` (internal, not published on the host). Telemetry is off.
|
||||
set -eu
|
||||
umask 077
|
||||
secret() { tr -d '\r\n' < "/run/secrets/$1"; }
|
||||
bucket="${S3_BUCKET:?S3_BUCKET is required}"
|
||||
staging="${S3_STAGING_BUCKET:?S3_STAGING_BUCKET is required}"
|
||||
cat > /tmp/s3.json <<EOF
|
||||
{
|
||||
"identities": [
|
||||
{"name": "bootstrap-admin",
|
||||
"credentials": [{"accessKey": "$(secret s3_admin_access_key_id)", "secretKey": "$(secret s3_admin_secret_access_key)"}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "messenger-api",
|
||||
"credentials": [{"accessKey": "$(secret s3_api_access_key_id)", "secretKey": "$(secret s3_api_secret_access_key)"}],
|
||||
"actions": ["Write:${staging}", "Read:${bucket}"]},
|
||||
{"name": "messenger-worker",
|
||||
"credentials": [{"accessKey": "$(secret s3_worker_access_key_id)", "secretKey": "$(secret s3_worker_secret_access_key)"}],
|
||||
"actions": ["Read:${staging}", "Write:${staging}", "Read:${bucket}", "Write:${bucket}"]},
|
||||
{"name": "messenger-auditor",
|
||||
"credentials": [{"accessKey": "$(secret s3_auditor_access_key_id)", "secretKey": "$(secret s3_auditor_secret_access_key)"}],
|
||||
"actions": ["Read:${staging}", "List:${staging}", "Read:${bucket}", "List:${bucket}"]}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
# Memory: by default SeaweedFS buffers every in-flight upload without a limit and the Go runtime
|
||||
# keeps freed heap; measured ~1 GB under 16 parallel 9.4 MiB PUTs (lab 4). The upload limits make
|
||||
# excess PUTs wait inside SeaweedFS instead of growing the heap, and GOMEMLIMIT (set in
|
||||
# compose.yaml below the container limit) makes the Go GC return memory before the cgroup OOM.
|
||||
exec weed server \
|
||||
-dir=/data \
|
||||
-ip=127.0.0.1 -ip.bind=127.0.0.1 \
|
||||
-master.telemetry=false \
|
||||
-master.volumeSizeLimitMB=1024 -volume.max=0 \
|
||||
-filer -filer.disableDirListing -filer.concurrentUploadLimitMB=128 \
|
||||
-volume.concurrentUploadLimitMB=128 \
|
||||
-s3 -s3.ip.bind=0.0.0.0 -s3.port=8333 -s3.config=/tmp/s3.json -s3.concurrentUploadLimitMB=128 \
|
||||
-s3.iam=false -s3.autoCreateBucket=false -s3.port.iceberg=0 -s3.port.lance=0
|
||||
@@ -0,0 +1,17 @@
|
||||
# Valkey is the single store of sessions (SESSION_STORE=valkey), so it must be durable.
|
||||
# AOF with fsync on every write: a confirmed login or logout survives a crash/restart,
|
||||
# a logout can never be lost and "resurrect" a token. Sessions are small and rare
|
||||
# writes, so the fsync cost is acceptable here (see REPORT.md).
|
||||
appendonly yes
|
||||
appendfsync always
|
||||
appenddirname "appendonlydir"
|
||||
# RDB snapshots are not needed on top of AOF.
|
||||
save ""
|
||||
dir /data
|
||||
port 6379
|
||||
protected-mode yes
|
||||
# Sessions are never evicted silently: if memory is full, writes fail and login returns 503.
|
||||
maxmemory 128mb
|
||||
maxmemory-policy noeviction
|
||||
# Password, generated by scripts/init_secrets.py and written here at container start.
|
||||
include /tmp/requirepass.conf
|
||||
@@ -0,0 +1,65 @@
|
||||
-- 0001: initial schema of the messenger (lab 2). Design rationale: docs/DATA_MODEL.md
|
||||
-- Applied once by `python -m messenger.migrate`; the version is recorded in schema_migrations.
|
||||
|
||||
-- User-supplied free text (display_name, title, message text) is stored as UTF-8 in BYTEA:
|
||||
-- the contract allows U+0000, and PostgreSQL TEXT cannot contain NUL bytes.
|
||||
-- Lengths are limited in code points by the API; the CHECKs below are byte-level safety nets
|
||||
-- (one code point is at most 4 bytes in UTF-8).
|
||||
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL,
|
||||
display_name bytea NOT NULL,
|
||||
password_hash text NOT NULL, -- Argon2id PHC string, never the password
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT users_username_key UNIQUE (username),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (octet_length(display_name) BETWEEN 1 AND 400)
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title bytea NOT NULL,
|
||||
created_by uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
-- Per-chat clock: created_at of the newest message. Updating this row serialises writers
|
||||
-- of one chat, so message timestamps strictly increase in commit order (see DATA_MODEL.md).
|
||||
last_message_at timestamptz,
|
||||
CONSTRAINT chats_title_len CHECK (octet_length(title) BETWEEN 1 AND 400)
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
position smallint NOT NULL, -- order of member_ids sent by the creator
|
||||
chat_created_at timestamptz NOT NULL, -- copy of chats.created_at for keyset listing
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
|
||||
);
|
||||
-- GET /chats of one user: (user_id, created_at, id) range scan, no sort.
|
||||
CREATE INDEX chat_members_user_keyset ON chat_members (user_id, chat_created_at, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text bytea NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
-- The sender must be a member of this chat: enforced by the database, not only by the API.
|
||||
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
|
||||
REFERENCES chat_members (chat_id, user_id),
|
||||
CONSTRAINT messages_text_len CHECK (octet_length(text) BETWEEN 1 AND 16000)
|
||||
);
|
||||
-- History page: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
|
||||
-- becomes an index range scan that starts exactly at the cursor.
|
||||
CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id);
|
||||
|
||||
-- Sessions for SESSION_STORE=postgres (the default store is Valkey; see docs/DATA_MODEL.md).
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY, -- SHA-256 of the token; the token is not stored
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32)
|
||||
);
|
||||
CREATE INDEX sessions_expires_at ON sessions (expires_at);
|
||||
@@ -0,0 +1,25 @@
|
||||
-- 0002 (lab 3): shared idempotency of POST /chats/{chat_id}/messages.
|
||||
-- Additive only (new table), so the lab 2 API keeps working while this runs: it is safe
|
||||
-- to apply before a rolling update of the API replicas (see REPORT.md, "Миграции").
|
||||
--
|
||||
-- Scope of a key (contracts/README.md): current user + HTTP method + chat path + key.
|
||||
-- The method (POST) and the path template are fixed for this table, so the scope
|
||||
-- is (user_id, chat_id, key). The key row, the message and the result are written in
|
||||
-- ONE transaction (repo.post_message): there is no "pending" state that a crash could
|
||||
-- leave behind, and the stored result is the message row itself.
|
||||
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
key text NOT NULL,
|
||||
request_hash bytea NOT NULL, -- SHA-256 of the canonical JSON body
|
||||
-- Inserted before the message in the same transaction, hence checked at COMMIT.
|
||||
message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (user_id, chat_id, key),
|
||||
CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT idempotency_keys_message_key UNIQUE (message_id)
|
||||
);
|
||||
-- Housekeeping of keys older than IDEMPOTENCY_TTL_SECONDS (>= 24 h).
|
||||
CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,50 @@
|
||||
-- 0003 (lab 3, track 1 improvement): the whole idempotent "send message" as ONE statement.
|
||||
-- Additive (a new function), so replicas of the previous version keep working while it is
|
||||
-- applied, and the previous write path (MESSAGE_WRITE_PATH=multi) stays usable for A/B runs.
|
||||
--
|
||||
-- Same semantics and the same SQL as repo.post_message (multi-statement path), but executed
|
||||
-- inside the server: the API sends one call instead of ~11 round trips in two transactions
|
||||
-- (membership check, BEGIN, key claim, chat clock, message, COMMIT, pool checks). Called in
|
||||
-- autocommit mode, so the function call IS the transaction: key row, chat clock and message
|
||||
-- commit together or not at all. VOLATILE + READ COMMITTED: every statement inside takes a
|
||||
-- fresh snapshot, so after waiting on a concurrent claim of the same key the stored result
|
||||
-- of the winner is visible, exactly as in the multi-statement path.
|
||||
--
|
||||
-- o_status: created | replayed | conflict | not_found | retry (key vanished in between: cleanup)
|
||||
CREATE FUNCTION post_message(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
|
||||
p_text bytea, p_ttl_seconds integer)
|
||||
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz)
|
||||
LANGUAGE plpgsql VOLATILE AS $$
|
||||
DECLARE
|
||||
v_created timestamptz;
|
||||
BEGIN
|
||||
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, p_message)
|
||||
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
|
||||
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
|
||||
IF FOUND THEN
|
||||
-- Per-chat clock under the chat row lock (see migrations/0001 and docs/DATA_MODEL.md).
|
||||
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
|
||||
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES (p_message, p_chat, p_user, p_text, v_created);
|
||||
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
RETURN QUERY
|
||||
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT 'retry'::text, NULL::uuid, NULL::bytea, NULL::timestamptz;
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
@@ -0,0 +1,145 @@
|
||||
-- 0004 (lab 4): attachments. Metadata live here, the bytes in the private S3 bucket.
|
||||
-- Additive (new tables, a relaxed CHECK, a NEW function post_message_v2), so replicas of the
|
||||
-- lab 3 version keep working while it is applied (they never send attachment_ids).
|
||||
--
|
||||
-- Consistency between the two stores (docs/CONSISTENCY.md):
|
||||
-- 1. INSERT the row as status='pending' (intent: the server-generated object_key is known
|
||||
-- BEFORE anything is written to S3) and COMMIT;
|
||||
-- 2. PUT the object into S3;
|
||||
-- 3. UPDATE status='ready' WHERE status='pending'; only then 201.
|
||||
-- Invariant: an object never exists without a row that names its key (row first, object
|
||||
-- second; on cleanup object first, row second). A crash/outage between the steps leaves a
|
||||
-- 'pending' row; the janitor deletes its object (idempotent) and then the row once it is older
|
||||
-- than PENDING_GRACE_SECONDS. The reconciler (python -m messenger.reconcile) checks the
|
||||
-- invariant over the whole bucket.
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL, -- display name only, never a path or a key
|
||||
content_type text NOT NULL, -- detected from the bytes, not from the header
|
||||
size_bytes integer NOT NULL,
|
||||
sha256 bytea NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
-- Random server-generated key ("o/" + 128 random bits in hex), unrelated to id/filename.
|
||||
-- COLLATE "C": byte order = S3 ListObjectsV2 order, the reconciler merge-joins both lists.
|
||||
object_key text COLLATE "C" NOT NULL,
|
||||
status text NOT NULL DEFAULT 'pending',
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
-- Set by the reconciler when the object of a READY row is missing or has the wrong size:
|
||||
-- the row is quarantined (content -> 404, cannot be attached), never deleted, and the mark
|
||||
-- is removed again if the object comes back (restore from backup).
|
||||
lost_at timestamptz,
|
||||
lost_reason text,
|
||||
-- The uploader must be a member of the chat (enforced by the database too).
|
||||
CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id),
|
||||
CONSTRAINT attachments_object_key_key UNIQUE (object_key),
|
||||
CONSTRAINT attachments_object_key_format CHECK (object_key ~ '^o/[0-9a-f]{32}$'),
|
||||
CONSTRAINT attachments_status CHECK (status IN ('pending', 'ready')),
|
||||
CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
CONSTRAINT attachments_sha256_len CHECK (octet_length(sha256) = 32),
|
||||
CONSTRAINT attachments_pixels CHECK (width >= 1 AND height >= 1 AND width::bigint * height <= 40000000),
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
CONSTRAINT attachments_lost CHECK ((lost_at IS NULL) = (lost_reason IS NULL) AND (lost_at IS NULL OR status = 'ready'))
|
||||
);
|
||||
-- Janitor: stale intents, oldest first; tiny partial index (only rows of unfinished uploads).
|
||||
CREATE INDEX attachments_pending_created ON attachments (created_at) WHERE status = 'pending';
|
||||
|
||||
-- attachment_ids of a message, twice on purpose (both written in the message's transaction):
|
||||
-- * messages.attachment_ids (ordered array): the history page reads it from the same row, so the
|
||||
-- keyset range scan of messages stays ONE index scan, without a per-row sub-query;
|
||||
-- * message_attachments: foreign keys (a referenced attachment can never be deleted) and the
|
||||
-- reverse lookup "is this attachment used?" (janitor/reconciler) by an index on attachment_id.
|
||||
ALTER TABLE messages ADD COLUMN attachment_ids uuid[] NOT NULL DEFAULT '{}';
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_attachment_ids_len CHECK (cardinality(attachment_ids) <= 10);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id),
|
||||
position smallint NOT NULL,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
PRIMARY KEY (message_id, position),
|
||||
CONSTRAINT message_attachments_unique UNIQUE (message_id, attachment_id),
|
||||
CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9)
|
||||
);
|
||||
-- "Is this attachment used by any message?" (janitor/reconciler never touch used ones).
|
||||
CREATE INDEX message_attachments_attachment ON message_attachments (attachment_id);
|
||||
|
||||
-- Lab 4: text may be "" when the message has attachments (the API enforces "text or files").
|
||||
-- DROP CONSTRAINT takes a short exclusive lock on messages and the new CHECK scans the table
|
||||
-- once, inside this migration's transaction (seconds for a lab-sized table; for a big table
|
||||
-- this would be ADD ... NOT VALID + VALIDATE in a separate migration).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_len;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (octet_length(text) <= 16000);
|
||||
|
||||
-- post_message_v2: post_message (0003) + attachment_ids, still ONE statement and ONE transaction.
|
||||
-- The attachments are checked BEFORE the key is claimed and the key is claimed only if they are
|
||||
-- fine, so a failed check never leaves a key behind. A replay (same key, same body) returns the
|
||||
-- stored message even if an attachment was quarantined since: the result of a key is final.
|
||||
--
|
||||
-- o_status: created | replayed | conflict | not_found | retry | attachment_not_found | attachment_conflict
|
||||
CREATE FUNCTION post_message_v2(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
|
||||
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
|
||||
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
|
||||
LANGUAGE plpgsql VOLATILE AS $$
|
||||
DECLARE
|
||||
v_created timestamptz;
|
||||
v_found integer;
|
||||
v_usable integer;
|
||||
v_check text := 'ok';
|
||||
BEGIN
|
||||
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF cardinality(p_attachments) > 0 THEN
|
||||
-- Visible = ready attachment of THIS chat (pending, foreign or unknown -> 404).
|
||||
-- Usable = visible and not quarantined (lost -> 409, like a failed attachment).
|
||||
-- FOR SHARE: a concurrent quarantine waits until this message is committed.
|
||||
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL) INTO v_found, v_usable
|
||||
FROM (SELECT a.lost_at FROM attachments a
|
||||
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status = 'ready'
|
||||
FOR SHARE) a;
|
||||
IF v_found <> cardinality(p_attachments) THEN
|
||||
v_check := 'attachment_not_found';
|
||||
ELSIF v_usable <> v_found THEN
|
||||
v_check := 'attachment_conflict';
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
IF v_check = 'ok' THEN
|
||||
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, p_message)
|
||||
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
|
||||
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
|
||||
IF FOUND THEN
|
||||
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
|
||||
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
|
||||
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
|
||||
INSERT INTO message_attachments (message_id, position, attachment_id)
|
||||
SELECT p_message, (a.ord - 1)::smallint, a.id
|
||||
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
|
||||
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
RETURN QUERY
|
||||
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
|
||||
m.attachment_ids
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
|
||||
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
@@ -0,0 +1,260 @@
|
||||
-- 0005 (lab 5): asynchronous processing of attachments. Details: docs/PIPELINE.md.
|
||||
--
|
||||
-- Upload (API): INSERT attachment 'pending' (intent: the staging key is known before the PUT)
|
||||
-- -> PUT original into the PRIVATE STAGING bucket
|
||||
-- -> ONE transaction: attachment 'queued' + jobs row + outbox row -> 202.
|
||||
-- Relay: outbox row -> publish to RabbitMQ with publisher confirm -> outbox.published_at.
|
||||
-- Worker: claim job (lease + attempt token) -> variants into the final bucket under keys of
|
||||
-- THIS attempt -> ONE transaction: variants + attachment 'ready' (fenced by the
|
||||
-- lease token, never downgrading published_version) -> ack -> staging deleted.
|
||||
--
|
||||
-- PostgreSQL is the source of truth for the work: state, attempts, lease, backoff (available_at)
|
||||
-- and the failed registry. The broker only delivers job ids (at least once).
|
||||
|
||||
-- ------------------------------------------------------------------ attachments
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status
|
||||
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
|
||||
-- object_key = key of the ORIGINAL in the final bucket. NULL until the worker publishes it.
|
||||
-- Keys of lab 4 ('o/<hex>') stay valid; lab 5 writes 'a/<attachment>/<attempt token>/original'.
|
||||
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_format;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_format
|
||||
CHECK (object_key ~ '^(o/[0-9a-f]{32}|a/[0-9a-f-]{36}/[0-9a-f]{32}/original)$');
|
||||
ALTER TABLE attachments
|
||||
-- Key of the original in the staging bucket ('s/<128 random bits>'), written by the API.
|
||||
ADD COLUMN staging_key text COLLATE "C",
|
||||
-- none (lab 4 row) | present | deleting (marked, S3 DELETE pending) | deleted
|
||||
ADD COLUMN staging_state text NOT NULL DEFAULT 'none',
|
||||
-- pipeline version whose variants are published (GET shows them); only ever grows.
|
||||
ADD COLUMN published_version integer,
|
||||
ADD COLUMN error_code text,
|
||||
ADD COLUMN failed_at timestamptz,
|
||||
ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key),
|
||||
ADD CONSTRAINT attachments_staging_key_format CHECK (staging_key ~ '^s/[0-9a-f]{32}$'),
|
||||
ADD CONSTRAINT attachments_staging_state CHECK (staging_state IN ('none', 'present', 'deleting', 'deleted')),
|
||||
ADD CONSTRAINT attachments_staging_has_key CHECK (staging_state = 'none' OR staging_key IS NOT NULL),
|
||||
ADD CONSTRAINT attachments_published_version CHECK (published_version >= 1),
|
||||
ADD CONSTRAINT attachments_error_code CHECK (error_code ~ '^[a-z_]{1,64}$');
|
||||
|
||||
-- Lab 4 rows: a 'pending' intent of lab 4 never got its object confirmed and has no staging copy;
|
||||
-- it becomes a failed upload (invisible: its id was never returned). A lab 4 'ready' row has its
|
||||
-- original in the final bucket but no thumbnail yet: it is queued for pipeline 1 (below).
|
||||
UPDATE attachments SET status = 'failed', error_code = 'upload_incomplete', failed_at = clock_timestamp()
|
||||
WHERE status = 'pending';
|
||||
UPDATE attachments SET status = 'queued' WHERE status = 'ready';
|
||||
|
||||
-- Structural guarantees (the database refuses a buggy cleanup or worker):
|
||||
-- * work that is queued/processing always has a source: the published original or the staging
|
||||
-- copy - the cleanup can never mark the staging of an accepted upload as deleting;
|
||||
-- * ready = published original + published pipeline version; failed = error code.
|
||||
ALTER TABLE attachments
|
||||
ADD CONSTRAINT attachments_pending_staging CHECK (status <> 'pending' OR staging_state = 'present'),
|
||||
ADD CONSTRAINT attachments_active_has_source
|
||||
CHECK (status NOT IN ('queued', 'processing') OR object_key IS NOT NULL OR staging_state = 'present'),
|
||||
ADD CONSTRAINT attachments_ready_published
|
||||
CHECK (status <> 'ready' OR (object_key IS NOT NULL AND published_version IS NOT NULL)),
|
||||
ADD CONSTRAINT attachments_failed_code
|
||||
CHECK ((status = 'failed') = (error_code IS NOT NULL AND failed_at IS NOT NULL));
|
||||
-- Janitor of the API: the lab 4 partial index on pending rows stays as is.
|
||||
-- Cleanup of the worker: staging copies that may be deleted (ready: at once; failed: after retention).
|
||||
CREATE INDEX attachments_staging_cleanup ON attachments (updated_at)
|
||||
WHERE staging_state IN ('present', 'deleting') AND status IN ('ready', 'failed');
|
||||
|
||||
-- ------------------------------------------------------------------ jobs
|
||||
-- One row = one logical piece of work: "build pipeline version V for attachment A".
|
||||
-- job_id never changes on redelivery; `attempt` counts claims by workers (managed attempts).
|
||||
CREATE TABLE jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL,
|
||||
kind text NOT NULL, -- upload | replay | reprocess | migrate
|
||||
replay_of uuid REFERENCES jobs (id),
|
||||
state text NOT NULL DEFAULT 'queued',
|
||||
attempt integer NOT NULL DEFAULT 0,
|
||||
max_attempts integer NOT NULL,
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(), -- backoff: not before
|
||||
lease_token uuid, -- token of the attempt that holds the job
|
||||
lease_owner text,
|
||||
lease_until timestamptz,
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
finished_at timestamptz,
|
||||
CONSTRAINT jobs_pipeline_version CHECK (pipeline_version >= 1),
|
||||
CONSTRAINT jobs_kind CHECK (kind IN ('upload', 'replay', 'reprocess', 'migrate')),
|
||||
CONSTRAINT jobs_replay CHECK ((kind = 'replay') = (replay_of IS NOT NULL)),
|
||||
CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed', 'superseded')),
|
||||
CONSTRAINT jobs_attempts CHECK (attempt >= 0 AND max_attempts BETWEEN 1 AND 100 AND attempt <= max_attempts),
|
||||
-- A lease exists exactly while the job is processing.
|
||||
CONSTRAINT jobs_lease CHECK ((state = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)),
|
||||
CONSTRAINT jobs_finished CHECK ((state IN ('succeeded', 'failed', 'superseded')) = (finished_at IS NOT NULL)),
|
||||
CONSTRAINT jobs_failed_code CHECK (state <> 'failed' OR error_code IS NOT NULL),
|
||||
CONSTRAINT jobs_error_code CHECK (error_code ~ '^[a-z_]{1,64}$')
|
||||
);
|
||||
-- At most ONE active job per effect set (attachment, pipeline version): a second upload job,
|
||||
-- replay or reprocess of the same version cannot exist while one is queued/processing.
|
||||
CREATE UNIQUE INDEX jobs_one_active ON jobs (attachment_id, pipeline_version) WHERE state IN ('queued', 'processing');
|
||||
CREATE INDEX jobs_attachment ON jobs (attachment_id, created_at);
|
||||
CREATE INDEX jobs_lease_expiry ON jobs (lease_until) WHERE state = 'processing'; -- reaper
|
||||
CREATE INDEX jobs_queued ON jobs (updated_at) WHERE state = 'queued'; -- lost-message guard
|
||||
CREATE INDEX jobs_failed ON jobs (finished_at) WHERE state = 'failed'; -- failed registry
|
||||
|
||||
-- One row per claim of a job by a worker. The token is part of every object key the attempt
|
||||
-- writes, so the keys of an attempt are known without listing the bucket (planned_keys), and an
|
||||
-- attempt that is no longer the lease holder can never publish (fencing by token).
|
||||
CREATE TABLE job_attempts (
|
||||
token uuid PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs (id),
|
||||
attempt integer NOT NULL,
|
||||
worker text NOT NULL,
|
||||
source text NOT NULL, -- staging | original
|
||||
planned_keys text[] NOT NULL,
|
||||
outcome text NOT NULL DEFAULT 'running',
|
||||
error_code text,
|
||||
started_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
finished_at timestamptz,
|
||||
objects_cleaned_at timestamptz,
|
||||
CONSTRAINT job_attempts_number UNIQUE (job_id, attempt),
|
||||
CONSTRAINT job_attempts_source CHECK (source IN ('staging', 'original')),
|
||||
CONSTRAINT job_attempts_outcome
|
||||
CHECK (outcome IN ('running', 'succeeded', 'retry', 'failed', 'abandoned', 'superseded')),
|
||||
CONSTRAINT job_attempts_finished CHECK ((outcome = 'running') = (finished_at IS NULL)),
|
||||
-- Objects of a successful attempt are published: they are never cleaned.
|
||||
CONSTRAINT job_attempts_cleaned CHECK (objects_cleaned_at IS NULL OR outcome NOT IN ('running', 'succeeded'))
|
||||
);
|
||||
CREATE INDEX job_attempts_cleanup ON job_attempts (finished_at)
|
||||
WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') AND objects_cleaned_at IS NULL;
|
||||
|
||||
-- Published variants. The primary key is the idempotency key of the EFFECT from the contract:
|
||||
-- (attachment_id, pipeline_version, variant) exists at most once, whatever the deliveries.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL,
|
||||
object_key text COLLATE "C" NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
size_bytes integer NOT NULL,
|
||||
width integer NOT NULL,
|
||||
height integer NOT NULL,
|
||||
sha256 bytea NOT NULL,
|
||||
attempt_token uuid NOT NULL REFERENCES job_attempts (token),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name),
|
||||
CONSTRAINT attachment_variants_key UNIQUE (object_key),
|
||||
CONSTRAINT attachment_variants_key_format CHECK (object_key ~ '^a/[0-9a-f-]{36}/[0-9a-f]{32}/(thumbnail|crop|optimized)$'),
|
||||
CONSTRAINT attachment_variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/webp')),
|
||||
CONSTRAINT attachment_variants_size CHECK (size_bytes >= 1 AND width >= 1 AND height >= 1),
|
||||
CONSTRAINT attachment_variants_sha256 CHECK (octet_length(sha256) = 32)
|
||||
);
|
||||
|
||||
-- Transactional outbox: written in the SAME transaction as the job (upload, retry, replay,
|
||||
-- reprocess, lease expiry). The relay publishes due rows and marks them; a crash after the
|
||||
-- publish but before the mark publishes again (a duplicate the worker ignores).
|
||||
CREATE TABLE outbox (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs (id),
|
||||
attempt integer NOT NULL, -- the attempt this delivery is meant for
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
publish_failures integer NOT NULL DEFAULT 0,
|
||||
CONSTRAINT outbox_attempt CHECK (attempt >= 1)
|
||||
);
|
||||
CREATE INDEX outbox_due ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
CREATE INDEX outbox_job ON outbox (job_id);
|
||||
CREATE INDEX outbox_published ON outbox (published_at) WHERE published_at IS NOT NULL; -- purge
|
||||
|
||||
-- Audit of every transition (no secrets, no file names): replay/reprocess requests with the
|
||||
-- operator and reason, claims, retries with their delay, lease expiries, results.
|
||||
CREATE TABLE job_events (
|
||||
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs (id),
|
||||
at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
event text NOT NULL,
|
||||
attempt integer,
|
||||
actor text NOT NULL,
|
||||
error_code text,
|
||||
detail jsonb NOT NULL DEFAULT '{}'
|
||||
);
|
||||
CREATE INDEX job_events_job ON job_events (job_id, id);
|
||||
|
||||
-- Failed registry for the operator (jobctl list-failed).
|
||||
CREATE VIEW failed_jobs AS
|
||||
SELECT j.id AS job_id, j.attachment_id, j.pipeline_version, j.kind, j.attempt, j.max_attempts, j.error_code,
|
||||
j.finished_at, a.status AS attachment_status, a.staging_state
|
||||
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.state = 'failed';
|
||||
|
||||
-- Lab 4 'ready' rows (now queued): a job for pipeline 1 from their published original.
|
||||
WITH queued AS (
|
||||
SELECT a.id, gen_random_uuid() AS job_id FROM attachments a WHERE a.status = 'queued'
|
||||
), created AS (
|
||||
INSERT INTO jobs (id, attachment_id, pipeline_version, kind, max_attempts)
|
||||
SELECT q.job_id, q.id, 1, 'migrate', 5 FROM queued q RETURNING id
|
||||
)
|
||||
INSERT INTO outbox (job_id, attempt) SELECT c.id, 1 FROM created c;
|
||||
|
||||
-- post_message_v3 = post_message_v2 (0004) with the lab 5 attachment states: queued, processing
|
||||
-- and ready attachments of this chat may be attached; failed (or quarantined) -> 409; pending,
|
||||
-- foreign or unknown -> 404. Everything else (key claim, chat clock, one transaction) unchanged.
|
||||
CREATE FUNCTION post_message_v3(p_user uuid, p_chat uuid, p_key text, p_hash bytea, p_message uuid,
|
||||
p_text bytea, p_ttl_seconds integer, p_attachments uuid[])
|
||||
RETURNS TABLE (o_status text, o_id uuid, o_text bytea, o_created_at timestamptz, o_attachments uuid[])
|
||||
LANGUAGE plpgsql VOLATILE AS $$
|
||||
DECLARE
|
||||
v_created timestamptz;
|
||||
v_found integer;
|
||||
v_usable integer;
|
||||
v_check text := 'ok';
|
||||
BEGIN
|
||||
PERFORM 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT 'not_found'::text, NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF cardinality(p_attachments) > 0 THEN
|
||||
-- FOR SHARE: a concurrent transition to failed/quarantine waits until this message commits.
|
||||
SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') INTO v_found, v_usable
|
||||
FROM (SELECT a.lost_at, a.status FROM attachments a
|
||||
WHERE a.id = ANY (p_attachments) AND a.chat_id = p_chat AND a.status <> 'pending'
|
||||
FOR SHARE) a;
|
||||
IF v_found <> cardinality(p_attachments) THEN
|
||||
v_check := 'attachment_not_found';
|
||||
ELSIF v_usable <> v_found THEN
|
||||
v_check := 'attachment_conflict';
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
IF v_check = 'ok' THEN
|
||||
INSERT INTO idempotency_keys AS k (user_id, chat_id, key, request_hash, message_id)
|
||||
VALUES (p_user, p_chat, p_key, p_hash, p_message)
|
||||
ON CONFLICT (user_id, chat_id, key) DO UPDATE
|
||||
SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp()
|
||||
WHERE k.created_at < clock_timestamp() - make_interval(secs => p_ttl_seconds);
|
||||
IF FOUND THEN
|
||||
UPDATE chats c SET last_message_at = greatest(clock_timestamp(), c.last_message_at + interval '1 microsecond')
|
||||
WHERE c.id = p_chat RETURNING c.last_message_at INTO v_created;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids)
|
||||
VALUES (p_message, p_chat, p_user, p_text, v_created, p_attachments);
|
||||
INSERT INTO message_attachments (message_id, position, attachment_id)
|
||||
SELECT p_message, (a.ord - 1)::smallint, a.id
|
||||
FROM unnest(p_attachments) WITH ORDINALITY AS a(id, ord);
|
||||
RETURN QUERY SELECT 'created'::text, p_message, p_text, v_created, p_attachments;
|
||||
RETURN;
|
||||
END IF;
|
||||
END IF;
|
||||
|
||||
RETURN QUERY
|
||||
SELECT CASE WHEN k.request_hash = p_hash THEN 'replayed' ELSE 'conflict' END, m.id, m.text, m.created_at,
|
||||
m.attachment_ids
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.key = p_key;
|
||||
IF NOT FOUND THEN
|
||||
RETURN QUERY SELECT CASE WHEN v_check = 'ok' THEN 'retry' ELSE v_check END,
|
||||
NULL::uuid, NULL::bytea, NULL::timestamptz, NULL::uuid[];
|
||||
END IF;
|
||||
END
|
||||
$$;
|
||||
@@ -0,0 +1,6 @@
|
||||
# Student tests (in-process, via Starlette TestClient). Runtime deps come from requirements.txt.
|
||||
pytest==9.1.1
|
||||
httpx2==2.13.1
|
||||
jsonschema==4.26.0
|
||||
# Lab 4 tests: generate PNG/JPEG fixtures (valid, corrupt, oversized) on the fly.
|
||||
pillow==12.3.0
|
||||
@@ -0,0 +1,24 @@
|
||||
# Direct runtime dependencies. The pinned lock with hashes is requirements.txt:
|
||||
# docker run --rm -v "$PWD:/w" -w /w python:3.13.16-slim-bookworm \
|
||||
# sh -c "pip install -q pip-tools==7.6.1 && pip-compile --generate-hashes --strip-extras --no-emit-index-url -o requirements.txt requirements.in"
|
||||
starlette==1.7.0
|
||||
uvicorn==0.54.0
|
||||
# Fast C parser/event loop used by uvicorn in the container (Linux only).
|
||||
httptools==0.9.0
|
||||
uvloop==0.23.0 ; sys_platform != "win32"
|
||||
# PostgreSQL driver (libpq bundled in the binary wheel) and its async connection pool.
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
# Valkey client (Valkey speaks the Redis protocol).
|
||||
redis==8.1.0
|
||||
# Argon2id password KDF.
|
||||
argon2-cffi==25.1.0
|
||||
# Lab 4: S3 client (async botocore over aiohttp; requests are cancellable, unlike libpq cancel).
|
||||
aiobotocore==3.9.2
|
||||
# Lab 5: AMQP 0-9-1 client for RabbitMQ (publisher confirms, consumer acks, prefetch); last 9.x line.
|
||||
aio-pika==9.6.2
|
||||
# Lab 5: image variants in the worker only (thumbnail/crop JPEG, WebP). Same version as the tests.
|
||||
pillow==12.3.0
|
||||
# Uploads are still checked without Pillow: PNG by src/messenger/images.py, JPEG by libjpeg-turbo
|
||||
# `djpeg -strict` (Debian package libjpeg-turbo-progs, see Dockerfile).
|
||||
+1460
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run every check of REPORT.md on the current checkout and save the real output in evidence/.
|
||||
|
||||
python scripts/collect_evidence.py # ~90 min; the stand must NOT be running yet
|
||||
|
||||
Intended for a CLEAN CLONE (fresh volumes): it creates secrets and .env, builds and starts the
|
||||
stand, then runs make check/validate/test/test-persistence/test-failover, the lab 5 defense
|
||||
scenario (scripts/demo_pipeline.py), the own tests, the track 2 fault-injection matrix
|
||||
(bench/pipeline_faults.py) and the invariant audit, and records container states before/after
|
||||
every hook and the real resource limits (docker inspect). Prints no secrets
|
||||
(hook output is shown by the public tests as suppressed; our scripts print names and states only).
|
||||
On Windows the hooks are the scripts/*.cmd wrappers (Git Bash) and make needs Git's usr/bin.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / "evidence"
|
||||
WINDOWS = os.name == "nt"
|
||||
HOOK = ".cmd" if WINDOWS else ".sh"
|
||||
PYTHON = sys.executable
|
||||
VENV_PY = str(ROOT / ".venv" / ("Scripts/python.exe" if WINDOWS else "bin/python"))
|
||||
# Makefile recipes use $(PYTHON) unquoted: pass interpreters WITHOUT spaces in the path.
|
||||
MAKE_PY = "python" if WINDOWS else "python3"
|
||||
MAKE_VENV = ".venv/Scripts/python.exe" if WINDOWS else ".venv/bin/python"
|
||||
ENV = {**os.environ, "PYTHONUTF8": "1"}
|
||||
if WINDOWS:
|
||||
ENV["PATH"] = ENV["PATH"] + os.pathsep + r"C:\Program Files\Git\usr\bin"
|
||||
|
||||
|
||||
def now() -> str:
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def sh(cmd: list[str], timeout: float = 1800, env: dict | None = None) -> tuple[int, str]:
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", errors="replace",
|
||||
timeout=timeout, env={**ENV, **(env or {})})
|
||||
took = time.monotonic() - started
|
||||
return result.returncode, (result.stdout + result.stderr).rstrip() + f"\n[exit {result.returncode}, {took:.1f} s]\n"
|
||||
|
||||
|
||||
class Log:
|
||||
def __init__(self, name: str, title: str):
|
||||
self.path = EVIDENCE / name
|
||||
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||
commit = subprocess.run(["git", "log", "-1", "--format=%h %s"], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8").stdout.strip()
|
||||
self.text = f"# {title}\n# {now()}, commit {commit}\n"
|
||||
|
||||
def run(self, cmd: list[str], shown: str | None = None, **kwargs) -> int:
|
||||
code, out = sh(cmd, **kwargs)
|
||||
self.text += f"\n$ {shown or ' '.join(cmd)}\n{out}"
|
||||
self.save()
|
||||
print(f" {self.path.name}: {shown or ' '.join(cmd[:4])} -> exit {code}", flush=True)
|
||||
return code
|
||||
|
||||
def note(self, text: str) -> None:
|
||||
self.text += f"\n{text}\n"
|
||||
self.save()
|
||||
|
||||
def save(self) -> None:
|
||||
self.path.write_text(self.text, encoding="utf-8")
|
||||
|
||||
|
||||
def states() -> list[str]:
|
||||
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
|
||||
rows = []
|
||||
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True).stdout or "[]"):
|
||||
st = item["State"]
|
||||
rows.append(f"{item['Name'].lstrip('/'):24} host={item['Config']['Hostname']:13} {st['Status']:8} "
|
||||
f"exit={st['ExitCode']:<3} pid={st['Pid']:<6} started={st['StartedAt'][:23]}")
|
||||
return sorted(rows)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
make = shutil.which("make", path=ENV["PATH"]) or "make"
|
||||
# 1. Clean clone -> secrets -> build -> start
|
||||
log = Log("clean-clone.txt", "Clean clone: secrets, config, build, migrations from zero, start, versions")
|
||||
log.run(["git", "status", "--short", "--branch"])
|
||||
log.run([PYTHON, "--version"]); log.run(["docker", "version", "--format", "client {{.Client.Version}} / server {{.Server.Version}}"])
|
||||
log.run(["docker", "compose", "version"]); log.run([make, "--version"], shown="make --version")
|
||||
log.run([PYTHON, "scripts/init_secrets.py"], shown="python scripts/init_secrets.py")
|
||||
if not (ROOT / ".env").exists():
|
||||
shutil.copy(ROOT / ".env.example", ROOT / ".env")
|
||||
log.note("$ cp .env.example .env")
|
||||
log.run(["docker", "compose", "--env-file", ".env", "config", "--quiet"])
|
||||
log.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait"], timeout=1800)
|
||||
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Image}}\t{{.Status}}"])
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "migrate"])
|
||||
log.run(["docker", "compose", "logs", "--no-log-prefix", "s3-init"]) # bucket + access-model verdicts
|
||||
log.run([PYTHON, "scripts/lbctl.py", "replicas"]); log.run([PYTHON, "scripts/lbctl.py", "status"])
|
||||
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"])
|
||||
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "type",
|
||||
"durable", "messages", "consumers", "arguments"])
|
||||
log.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_permissions"])
|
||||
ids = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
|
||||
log.run(["docker", "inspect", "-f", "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} "
|
||||
"memswap={{.HostConfig.MemorySwap}} image={{.Config.Image}}", *ids], shown="docker inspect (limits)")
|
||||
# venv for validate / own tests
|
||||
if not Path(VENV_PY).exists():
|
||||
log.run([PYTHON, "-m", "venv", ".venv"])
|
||||
log.run([VENV_PY, "-m", "pip", "install", "-q", "--require-hashes", "--no-deps", "-r", "requirements.txt"], timeout=1800)
|
||||
log.run([VENV_PY, "-m", "pip", "install", "-q", "-r", "requirements-test.txt", "-r", "requirements-dev.txt"], timeout=1800)
|
||||
|
||||
# 2. make check / validate / test
|
||||
log = Log("make-check-validate-test.txt", "make check, make validate (venv), make test")
|
||||
log.run([make, "check", f"PYTHON={MAKE_PY}"], shown="make check")
|
||||
log.run([make, "validate", f"PYTHON={MAKE_VENV}"], shown=f"make validate PYTHON={MAKE_VENV}")
|
||||
log.run([make, "test", f"PYTHON={MAKE_PY}"], shown="make test")
|
||||
|
||||
# 3. persistence with states before/after
|
||||
log = Log("test-persistence.txt", "make test-persistence: restart of LB, API replicas, relay, worker, PostgreSQL, "
|
||||
"Valkey, S3, RabbitMQ")
|
||||
log.note("Containers BEFORE:\n" + "\n".join(states()))
|
||||
log.run([make, "test-persistence", f"ACTION_SCRIPT=scripts/restart{HOOK}", f"PYTHON={MAKE_PY}"],
|
||||
shown=f"make test-persistence ACTION_SCRIPT=scripts/restart{HOOK}")
|
||||
log.note("Containers AFTER (new pid/started for lb, api-*, relay, worker, postgres, valkey, s3, broker; volumes "
|
||||
"kept):\n" + "\n".join(states()))
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
|
||||
# 4. failover: until BOTH replicas have been the stopped one (each run is a real public test)
|
||||
log = Log("test-failover.txt", "make test-failover with scripts/stop-one: which container is really stopped, restore, repeat")
|
||||
stopped: set[str] = set()
|
||||
import urllib.request
|
||||
for attempt in range(1, 7):
|
||||
# The public test stops the replica that served its first POST; its request sequence is
|
||||
# fixed, so with round robin it hits the same replica every time. Shift the rotation by
|
||||
# (attempt - 1) extra requests so that the other replica becomes the target as well.
|
||||
for _ in range(attempt - 1):
|
||||
urllib.request.urlopen("http://127.0.0.1:8080/health/live", timeout=5).read()
|
||||
before = {r for r in states() if "-api-" in r and " running " in r}
|
||||
log.note(f"--- run {attempt}; running API replicas BEFORE:\n" + "\n".join(sorted(before)))
|
||||
log.run([make, "test-failover", f"ACTION_SCRIPT=scripts/stop-one{HOOK}", f"PYTHON={MAKE_PY}"],
|
||||
shown=f"make test-failover ACTION_SCRIPT=scripts/stop-one{HOOK}")
|
||||
after = states()
|
||||
gone = [r for r in after if "-api-" in r and " exited " in r]
|
||||
log.note("AFTER (the stopped replica, left stopped by the hook):\n" + "\n".join(gone))
|
||||
log.run([PYTHON, "scripts/lbctl.py", "status"])
|
||||
stopped.update(r.split()[0] for r in gone)
|
||||
log.run(["sh" if not WINDOWS else r"C:\Program Files\Git\bin\bash.exe", "scripts/restore-api.sh"],
|
||||
shown="sh scripts/restore-api.sh")
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
if len(stopped) >= 2:
|
||||
break
|
||||
log.note(f"Replicas stopped by the public test across runs: {sorted(stopped)}")
|
||||
|
||||
# 5. lab 5 defense scenario: worker stopped -> 202 -> 409 -> restart api+broker -> worker -> ready
|
||||
log = Log("demo-pipeline.txt", "Defense steps 2-3: worker stopped, upload 202, content 409, restart API and broker "
|
||||
"with volumes, worker started, ready + thumbnail")
|
||||
log.note("Containers BEFORE:\n" + "\n".join(states()))
|
||||
log.run([VENV_PY, "scripts/demo_pipeline.py"], shown="python scripts/demo_pipeline.py", timeout=900)
|
||||
log.note("Containers AFTER (api-*, broker restarted; worker stopped and started):\n" + "\n".join(states()))
|
||||
|
||||
# 6. own tests
|
||||
log = Log("student-tests.txt", "Own tests: unit + stand (pytest -v); failures restored in finally")
|
||||
log.run([VENV_PY, "-m", "pytest", "student-tests", "-v", "-p", "no:cacheprovider"], shown="python -m pytest student-tests -v",
|
||||
timeout=5400)
|
||||
|
||||
# 7. track 2: fault-injection matrix + audit; lab 3 load script
|
||||
log = Log("track2-runs.txt", "Track 2: bench/pipeline_faults.py (duplicate, reorder, crashes, stale versions, hung "
|
||||
"worker) + read-only audit of the invariants")
|
||||
log.run([VENV_PY, "bench/pipeline_faults.py", "--output", "evidence/track2", "--uploads", "5"], timeout=5400)
|
||||
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "auditor", "--verify-checksums"],
|
||||
shown="docker compose run --rm auditor --verify-checksums", timeout=900)
|
||||
log.run(["docker", "compose", "run", "--rm", "--no-deps", "-T", "jobctl", "stats"], shown="docker compose run --rm jobctl stats")
|
||||
log.run([PYTHON, "scripts/lbctl.py", "wait-up", "2"])
|
||||
log.run([PYTHON, "tests/load.py", "--duration", "30", "--concurrency", "4", "--output", "evidence/load.json"])
|
||||
log.run(["docker", "compose", "ps", "-a", "--format", "table {{.Service}}\t{{.Status}}"])
|
||||
print("done:", EVIDENCE)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab 4 defense scenario (kept in lab 5; upload is now 202 queued, the download waits for ready):
|
||||
upload the fixture, send a message with attachment_ids,
|
||||
download as the second member and compare SHA-256; the outsider gets 404; an unauthenticated
|
||||
request straight to S3 (from inside the storage network) is refused, the signed one works.
|
||||
|
||||
python scripts/demo_attachments.py [--base-url http://127.0.0.1:8080]
|
||||
|
||||
Prints requests and answers; never prints tokens or keys. Creates synthetic users.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from email.message import Message
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FIXTURE = ROOT / "tests" / "fixtures" / "sample.png"
|
||||
PASSWORD = "Demo-password-Ω-12345"
|
||||
|
||||
|
||||
class Client:
|
||||
def __init__(self, base: str):
|
||||
self.base = base.rstrip("/")
|
||||
|
||||
def call(self, method: str, path: str, *, token: str | None = None, body=None, raw: bytes | None = None,
|
||||
headers: dict | None = None) -> tuple[int, Message, bytes]:
|
||||
h = dict(headers or {})
|
||||
if token:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
return r.status, r.headers, r.read() # case-insensitive HTTPMessage
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.headers, e.read()
|
||||
|
||||
|
||||
def show(label: str, status: int, headers: Message, payload: bytes, *, binary: bool = False) -> None:
|
||||
inst = headers.get("X-Instance-Id", "-")
|
||||
if binary:
|
||||
text = f"{len(payload)} bytes, Content-Type {headers.get('Content-Type')}"
|
||||
else:
|
||||
text = payload.decode("utf-8", "replace")[:300]
|
||||
print(f" {label:58} -> {status} [X-Instance-Id {inst}] {text}")
|
||||
|
||||
|
||||
def user(c: Client, label: str) -> dict:
|
||||
name = f"{label}_{uuid.uuid4().hex[:8]}"
|
||||
status, _, body = c.call("POST", "/api/v1/users", body={"username": name, "display_name": label.title(),
|
||||
"password": PASSWORD})
|
||||
assert status == 201, body
|
||||
basic = base64.b64encode(f"{name}:{PASSWORD}".encode()).decode()
|
||||
status, _, body = c.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
session = json.loads(body)
|
||||
return {"id": session["user"]["id"], "token": session["token"], "name": name}
|
||||
|
||||
|
||||
def wait_ready(c: Client, actor: dict, attachment_id: str, timeout: float = 60):
|
||||
import time
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{attachment_id}", token=actor["token"])
|
||||
if st != 200 or json.loads(b)["status"] in ("ready", "failed") or time.monotonic() > deadline:
|
||||
return st, h, b
|
||||
time.sleep(0.3)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url)
|
||||
content = FIXTURE.read_bytes()
|
||||
print(f"fixture {FIXTURE.name}: {len(content)} bytes, sha256 {hashlib.sha256(content).hexdigest()}")
|
||||
alice, bob, eve = user(c, "alice"), user(c, "bob"), user(c, "eve")
|
||||
print(f"users: alice, bob (members), eve (outsider)")
|
||||
status, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
|
||||
print("\n1. upload -> message -> download by the second member")
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
||||
raw=content, headers={"Content-Type": "image/png"})
|
||||
show("POST /chats/{id}/attachments (alice, raw PNG)", st, h, b)
|
||||
att = json.loads(b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /chats/{id}/messages text='' attachment_ids=[id]", st, h, b)
|
||||
st, h, b = wait_ready(c, bob, att["id"])
|
||||
show("GET /attachments/{id} (bob) until ready", st, h, b)
|
||||
st, h, got = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
show("GET /attachments/{id}/content (bob)", st, h, got, binary=True)
|
||||
same = hashlib.sha256(got).hexdigest() == hashlib.sha256(content).hexdigest() == att["sha256"]
|
||||
print(f" SHA-256 of the download == fixture == metadata: {same}")
|
||||
|
||||
print("\n2. outsider and direct S3 access")
|
||||
for suffix in ("", "/content"):
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", token=eve["token"])
|
||||
show(f"GET /attachments/{{id}}{suffix} (eve, not a member)", st, h, b)
|
||||
key = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-c", f"SELECT object_key FROM attachments WHERE id = '{att['id']}'"],
|
||||
cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
|
||||
print(f" object key in PostgreSQL: {key} (random; not in any API response: "
|
||||
f"{key not in json.dumps(att)})")
|
||||
probe = (
|
||||
"import urllib.request as u, urllib.error as e, asyncio\n"
|
||||
"def st(url):\n"
|
||||
" try: return u.urlopen(url, timeout=3).status\n"
|
||||
" except e.HTTPError as x: return x.code\n"
|
||||
f"print('anonymous GET http://s3:8333/messenger-attachments/{key} ->', st('http://s3:8333/messenger-attachments/{key}'))\n"
|
||||
"print('anonymous LIST http://s3:8333/messenger-attachments?list-type=2 ->', st('http://s3:8333/messenger-attachments?list-type=2'))\n"
|
||||
"from messenger.config import Settings\nfrom messenger.storage import Storage\n"
|
||||
"async def m():\n s = Storage(Settings.from_env()); await s.open()\n"
|
||||
f" print('signed HEAD with the API key pair ->', (await s.head('{key}')).size, 'bytes'); await s.close()\n"
|
||||
"asyncio.run(m())\n")
|
||||
out = subprocess.run(["docker", "compose", "exec", "-T", "api", "python", "-c", probe], cwd=ROOT,
|
||||
capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
print(" (inside network storage) " + line)
|
||||
print(" S3 is not published on the host at all: only the LB has ports (127.0.0.1:8080, 8404).")
|
||||
return 0 if same else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,132 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Lab 5 defense scenario, steps 2-3 (README): worker stopped -> upload 202 -> content 409 while the
|
||||
rest of the API works -> a message with the attachment is created -> API and broker restarted with
|
||||
their volumes -> worker started -> the same attachment becomes ready, thumbnail downloadable.
|
||||
|
||||
python scripts/demo_pipeline.py [--base-url http://127.0.0.1:8080]
|
||||
|
||||
Prints every step with HTTP codes, IDs, the outbox row and the queue depth; never prints tokens or
|
||||
keys. Creates synthetic users. Always starts the worker again at the end.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
from demo_attachments import FIXTURE, ROOT, Client, show, user, wait_ready # noqa: E402
|
||||
|
||||
|
||||
def compose(*args: str, check: bool = True) -> str:
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
|
||||
check=check, timeout=300).stdout
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-c", sql).strip()
|
||||
|
||||
|
||||
def queue_depth() -> str:
|
||||
"""messages/consumers of the jobs queue; rabbitmqctl refreshes these counters every ~5 s."""
|
||||
time.sleep(6)
|
||||
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages", "consumers")
|
||||
line = next(l for l in out.splitlines() if l.startswith("messenger.attachment-jobs\t"))
|
||||
_, messages, consumers = line.split("\t")
|
||||
return f"messages={messages} consumers={consumers}"
|
||||
|
||||
|
||||
def wait_api(c: Client, timeout: float = 120) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if c.call("GET", "/health/ready")[0] == 200:
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(1)
|
||||
raise SystemExit("API did not become ready")
|
||||
|
||||
|
||||
def step(text: str) -> None:
|
||||
print(f"\n[{time.strftime('%H:%M:%S')}] {text}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://127.0.0.1:8080")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url)
|
||||
content = FIXTURE.read_bytes()
|
||||
alice, bob = user(c, "alice"), user(c, "bob")
|
||||
_, _, body = c.call("POST", "/api/v1/chats", token=alice["token"],
|
||||
body={"title": "Демо lab5", "member_ids": [alice["id"], bob["id"]]})
|
||||
chat = json.loads(body)
|
||||
try:
|
||||
step("1. docker compose stop worker")
|
||||
compose("stop", "worker")
|
||||
print(" " + compose("ps", "-a", "--format", "{{.Service}} {{.State}}", "worker").strip())
|
||||
|
||||
step("2. upload while the worker is stopped")
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=sample.png", token=alice["token"],
|
||||
raw=content, headers={"Content-Type": "image/png"})
|
||||
show("POST /chats/{id}/attachments", st, h, b)
|
||||
att = json.loads(b)
|
||||
print(f" attachment_id = {att['id']} status = {att['status']}")
|
||||
for query in ("", "?variant=thumbnail"):
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}/content{query}", token=bob["token"])
|
||||
show(f"GET /attachments/{{id}}/content{query}", st, h, b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=bob["token"],
|
||||
body={"text": "обычное сообщение работает"}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /messages (text only)", st, h, b)
|
||||
st, h, b = c.call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"],
|
||||
body={"text": "фото", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
show("POST /messages attachment_ids=[queued id]", st, h, b)
|
||||
message_id = json.loads(b)["id"]
|
||||
time.sleep(2)
|
||||
print(" job + outbox in PostgreSQL: " + psql(
|
||||
"SELECT j.id || ' state=' || j.state || ' attempt=' || j.attempt || ' | outbox published_at=' || "
|
||||
f"coalesce(o.published_at::text, 'NULL') FROM jobs j JOIN outbox o ON o.job_id = j.id "
|
||||
f"WHERE j.attachment_id = '{att['id']}'"))
|
||||
print(" RabbitMQ queue messenger.attachment-jobs: " + queue_depth())
|
||||
|
||||
step("3. docker compose restart api broker (volumes preserved)")
|
||||
compose("restart", "api", "broker")
|
||||
wait_api(c)
|
||||
time.sleep(3)
|
||||
print(" RabbitMQ queue after the restart: " + queue_depth())
|
||||
st, h, b = c.call("GET", f"/api/v1/attachments/{att['id']}", token=bob["token"])
|
||||
show("GET /attachments/{id}", st, h, b)
|
||||
finally:
|
||||
step("4. docker compose start worker")
|
||||
compose("start", "worker")
|
||||
st, h, b = wait_ready(c, bob, att["id"], timeout=90)
|
||||
show("GET /attachments/{id} until ready", st, h, b)
|
||||
ready = json.loads(b)
|
||||
thumb = next(v for v in ready["variants"] if v["name"] == "thumbnail")
|
||||
st, h, data = c.call("GET", f"/api/v1/attachments/{att['id']}/content?variant=thumbnail", token=bob["token"])
|
||||
show("GET /content?variant=thumbnail", st, h, data, binary=True)
|
||||
from PIL import Image
|
||||
size = Image.open(io.BytesIO(data)).size
|
||||
ok = (st == 200 and hashlib.sha256(data).hexdigest() == thumb["sha256"] and size == (256, 160)
|
||||
and ready["status"] == "ready")
|
||||
print(f" thumbnail: real JPEG {size[0]}x{size[1]}, sha256 matches metadata: {ok}")
|
||||
st, h, orig = c.call("GET", f"/api/v1/attachments/{att['id']}/content", token=bob["token"])
|
||||
print(f" original unchanged byte for byte: {orig == content}")
|
||||
print(f" message {message_id} still references it: "
|
||||
f"{psql(f'SELECT attachment_ids FROM messages WHERE id = {chr(39)}{message_id}{chr(39)}')}")
|
||||
step("5. job history (jobctl show)")
|
||||
shown = json.loads(compose("run", "--rm", "--no-deps", "-T", "jobctl", "show", att["id"]))
|
||||
for e in shown["events"]:
|
||||
print(f" {e['at']} {e['event']:16} attempt={e['attempt']} actor={e['actor']}")
|
||||
return 0 if ok and orig == content else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4).
|
||||
|
||||
python scripts/explain_history.py --output evidence/explain-history.txt
|
||||
|
||||
Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in
|
||||
30 other chats, then explains the EXACT statements of src/messenger/repo.py:
|
||||
1. first page (HISTORY_FIRST)
|
||||
2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement
|
||||
reused by the driver would get it
|
||||
3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan
|
||||
4. the rejected OFFSET pagination for the same late page
|
||||
Runs psql inside the postgres container; no database port is published.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL)
|
||||
|
||||
CHAT = "00000000-0000-4000-8000-00000000e002"
|
||||
USER = "00000000-0000-4000-8000-00000000e001"
|
||||
TARGET = 20_000
|
||||
LATE = 19_950 # cursor position: the 50 newest messages remain
|
||||
|
||||
SEED = f"""
|
||||
\\set ON_ERROR_STOP on
|
||||
BEGIN;
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
|
||||
INSERT INTO chats (id, title, created_by, created_at)
|
||||
SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'),
|
||||
'{USER}', timestamptz '2026-01-01 00:00:00+00'
|
||||
FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats
|
||||
ON CONFLICT DO NOTHING;
|
||||
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
|
||||
SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'),
|
||||
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
|
||||
FROM generate_series(1::int, {TARGET}::int) AS g
|
||||
WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}');
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'),
|
||||
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
|
||||
FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g
|
||||
WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}'
|
||||
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
|
||||
COMMIT;
|
||||
ANALYZE messages;
|
||||
"""
|
||||
|
||||
|
||||
def pg(sql_param: str) -> str:
|
||||
"""Turn psycopg %s placeholders into $1..$n for a PREPARE statement."""
|
||||
parts = sql_param.split("%s")
|
||||
return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts))
|
||||
|
||||
|
||||
EXPLAIN = f"""
|
||||
\\set ON_ERROR_STOP on
|
||||
\\pset pager off
|
||||
SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages;
|
||||
SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}'
|
||||
ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset
|
||||
SET plan_cache_mode = force_generic_plan;
|
||||
PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)};
|
||||
PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)};
|
||||
PREPARE or_page(uuid, timestamptz, uuid, int) AS
|
||||
SELECT id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4;
|
||||
PREPARE offset_page(uuid, int, int) AS
|
||||
SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3;
|
||||
\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51);
|
||||
\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51);
|
||||
\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51);
|
||||
\\echo '=== 4. REJECTED: OFFSET pagination, same late page'
|
||||
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51);
|
||||
"""
|
||||
|
||||
|
||||
def psql(script: str) -> str:
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"],
|
||||
cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True)
|
||||
return result.stdout
|
||||
|
||||
|
||||
def run() -> str:
|
||||
psql(SEED)
|
||||
return psql(EXPLAIN)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
parser.add_argument("--output", default=None)
|
||||
args = parser.parse_args()
|
||||
text = run()
|
||||
print(text)
|
||||
if args.output:
|
||||
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create local synthetic credentials for the stand in ./secrets (ignored by Git).
|
||||
|
||||
python scripts/init_secrets.py
|
||||
|
||||
Idempotent: existing files are kept, so re-running never changes passwords of a
|
||||
database that already has data. Prints only file names, never values.
|
||||
Delete ./secrets together with the volumes (`docker compose down -v`) to start over.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
|
||||
|
||||
def write(name: str, value: str) -> None:
|
||||
path = SECRETS / name
|
||||
if path.exists():
|
||||
print(f"keep secrets/{name}")
|
||||
return
|
||||
path.write_text(value + "\n", encoding="utf-8", newline="\n")
|
||||
# Readable by the container users (postgres, valkey, seaweedfs, rabbitmq, app uid 10001)
|
||||
# through the compose bind mount. Local synthetic lab credentials only.
|
||||
os.chmod(path, 0o644)
|
||||
print(f"created secrets/{name}")
|
||||
|
||||
|
||||
def read(name: str) -> str:
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
def rabbit_hash(password: str) -> str:
|
||||
"""RabbitMQ `rabbit_password_hashing_sha256`: base64(salt + sha256(salt + password)), 4-byte salt.
|
||||
The broker stores and compares only this hash; the plain password is in the client's URL."""
|
||||
salt = secrets.token_bytes(4)
|
||||
return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
|
||||
write("valkey_password", secrets.token_urlsafe(24))
|
||||
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
|
||||
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
|
||||
# S3 key pairs (SeaweedFS identities, infra/seaweedfs/start.sh): admin = bucket bootstrap only;
|
||||
# api = Write staging + Read final; worker = Read/Write both; auditor = Read + List (no Write).
|
||||
for who in ("admin", "api", "worker", "auditor"):
|
||||
write(f"s3_{who}_access_key_id", "AK" + secrets.token_hex(9).upper())
|
||||
write(f"s3_{who}_secret_access_key", secrets.token_hex(20))
|
||||
# Lab 5: RabbitMQ users. relay may only publish to the jobs exchange, worker may only consume
|
||||
# the jobs queue; neither may declare/delete anything (configure "^$").
|
||||
write("rabbitmq_relay_password", secrets.token_urlsafe(24))
|
||||
write("rabbitmq_worker_password", secrets.token_urlsafe(24))
|
||||
users = {
|
||||
"users": [{"name": who, "password_hash": rabbit_hash(read(f"rabbitmq_{who}_password")),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} for who in ("relay", "worker")],
|
||||
"permissions": [
|
||||
{"user": "relay", "vhost": "/", "configure": "^$", "write": "^messenger\\.jobs$", "read": "^$"},
|
||||
{"user": "worker", "vhost": "/", "configure": "^$", "write": "^$", "read": "^messenger\\.attachment-jobs$"},
|
||||
],
|
||||
}
|
||||
write("rabbitmq_users.json", json.dumps(users, indent=1))
|
||||
for who in ("relay", "worker"):
|
||||
# AMQP heartbeat 10 s: a frozen client loses its connection after ~2 missed heartbeats, and
|
||||
# the broker returns its unacked messages to the queue for the other consumers.
|
||||
write(f"broker_url_{who}", f"amqp://{who}:{read(f'rabbitmq_{who}_password')}@broker:5672/?heartbeat=10")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,125 @@
|
||||
#!/usr/bin/env python3
|
||||
"""What the load balancer thinks right now, mapped to real containers.
|
||||
|
||||
python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id
|
||||
python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy
|
||||
python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state
|
||||
|
||||
Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`,
|
||||
so no admin port is published. Container hostname = X-Instance-Id of that replica.
|
||||
Prints no secrets. Used by student-tests and bench scripts as a library as well.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping.
|
||||
OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"}
|
||||
ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it)
|
||||
|
||||
|
||||
def run(*args: str, timeout: float = 60) -> str:
|
||||
result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8",
|
||||
timeout=timeout)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}")
|
||||
return result.stdout
|
||||
|
||||
|
||||
def replicas(all_states: bool = True) -> list[dict]:
|
||||
"""API containers of this compose project (running and stopped)."""
|
||||
ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split()
|
||||
if not ids:
|
||||
return []
|
||||
data = json.loads(run("docker", "inspect", *ids))
|
||||
result = []
|
||||
for item in data:
|
||||
networks = item["NetworkSettings"]["Networks"] or {}
|
||||
result.append({
|
||||
"id": item["Id"][:12],
|
||||
"name": item["Name"].lstrip("/"),
|
||||
"hostname": item["Config"]["Hostname"],
|
||||
"state": item["State"]["Status"],
|
||||
"health": (item["State"].get("Health") or {}).get("Status", "-"),
|
||||
"exit_code": item["State"]["ExitCode"],
|
||||
"started_at": item["State"]["StartedAt"],
|
||||
"pid": item["State"]["Pid"],
|
||||
"ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")),
|
||||
# compose network name without the project prefix -> IP, e.g. {"edge": ..., "data": ...}
|
||||
"networks": {name.split("_", 1)[-1]: n["IPAddress"] for name, n in networks.items() if n.get("IPAddress")},
|
||||
})
|
||||
return sorted(result, key=lambda r: r["name"])
|
||||
|
||||
|
||||
def lb_servers() -> list[dict]:
|
||||
raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c",
|
||||
'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock')
|
||||
by_ip = {ip: r for r in replicas() for ip in r["ips"]}
|
||||
servers = []
|
||||
for line in raw.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) < 7 or not parts[0].isdigit():
|
||||
continue
|
||||
name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6])
|
||||
if admin & ADMIN_RMAINT or addr == "-":
|
||||
state = "MAINT"
|
||||
else:
|
||||
state = OP_STATE.get(op, str(op))
|
||||
container = by_ip.get(addr)
|
||||
servers.append({"slot": name, "addr": addr, "state": state,
|
||||
"container": container["name"] if container else None,
|
||||
"instance_id": container["hostname"] if container else None})
|
||||
return servers
|
||||
|
||||
|
||||
def up_instances() -> set[str]:
|
||||
return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"}
|
||||
|
||||
|
||||
def wait_up(count: int, timeout: float = 60) -> float:
|
||||
"""Seconds until exactly `count` servers are UP (raises TimeoutError otherwise)."""
|
||||
started = time.monotonic()
|
||||
while True:
|
||||
try:
|
||||
if len(up_instances()) == count:
|
||||
return time.monotonic() - started
|
||||
except RuntimeError:
|
||||
pass # lb restarting
|
||||
if time.monotonic() - started > timeout:
|
||||
raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s")
|
||||
time.sleep(0.25)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("status")
|
||||
sub.add_parser("replicas")
|
||||
wait = sub.add_parser("wait-up")
|
||||
wait.add_argument("count", type=int)
|
||||
wait.add_argument("--timeout", type=float, default=60)
|
||||
args = parser.parse_args()
|
||||
if args.command == "status":
|
||||
for s in lb_servers():
|
||||
if s["state"] != "MAINT" or s["container"]:
|
||||
print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}")
|
||||
print(f"(other slots: MAINT, no container)")
|
||||
elif args.command == "replicas":
|
||||
for r in replicas():
|
||||
print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} "
|
||||
f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}")
|
||||
else:
|
||||
took = wait_up(args.count, args.timeout)
|
||||
print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
|
||||
rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash.
|
||||
rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh"
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+21
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
|
||||
# Restarts EVERY process of the stand: LB, all API replicas, relay, worker(s), PostgreSQL, Valkey,
|
||||
# S3 (SeaweedFS) and the broker (RabbitMQ). Volumes, schema, data and queued messages are
|
||||
# preserved: no `down -v`, no FLUSH, no purge, no re-seed.
|
||||
# Output is suppressed by the test; run it by hand to see the steps.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# 1. Entry point, API and the pipeline processes first: nothing is served or processed while the
|
||||
# stores restart underneath.
|
||||
docker compose stop lb api relay worker
|
||||
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync, SeaweedFS volume/filer
|
||||
# flush, RabbitMQ quorum log), then start again. Bytes live on s3data, messages on rabbitmqdata.
|
||||
docker compose restart postgres valkey s3 broker
|
||||
# 3. API replicas (wait for healthy stores via depends_on), the LB, then relay and worker. HAProxy
|
||||
# resolves the new replica addresses through Docker DNS by itself - no stale backend list.
|
||||
docker compose start api
|
||||
docker compose start lb
|
||||
docker compose start relay worker
|
||||
echo "restart.sh: lb, api (all replicas), relay, worker, postgres, valkey, s3 and broker restarted with volumes preserved"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker
|
||||
# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s);
|
||||
# `python scripts/lbctl.py wait-up 2` waits for that explicitly.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-recreate --no-deps --wait api
|
||||
docker compose ps api --format '{{.Name}} {{.Status}}'
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
# Rolling update of the API without downtime (Compose has no built-in one for a scaled service).
|
||||
# 1. build the new image from the working tree;
|
||||
# 2. run migrations ONCE with the new image (they must be additive / compatible with the
|
||||
# running old version: expand first, contract in a later release);
|
||||
# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until
|
||||
# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully
|
||||
# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it.
|
||||
# At every moment at least N replicas serve traffic. Stops on the first error (set -e);
|
||||
# the old replicas that were not replaced yet keep running.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
PY="${PYTHON:-$(command -v python3 || command -v python)}"
|
||||
# New image = new revision label (git commit + UTC time), visible in docker inspect and logs.
|
||||
BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}"
|
||||
export BUILD_REVISION
|
||||
|
||||
replicas="$(docker compose ps -q api | wc -l | tr -d ' ')"
|
||||
[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; }
|
||||
old="$(docker compose ps -q api)"
|
||||
echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION"
|
||||
|
||||
docker compose build api
|
||||
docker compose run --rm --no-deps migrate
|
||||
|
||||
for id in $old; do
|
||||
name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')"
|
||||
docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api
|
||||
"$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60
|
||||
docker stop --time 15 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
"$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60
|
||||
echo "rolling-update: replaced $name"
|
||||
done
|
||||
for id in $(docker compose ps -q api); do
|
||||
docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id"
|
||||
done
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
|
||||
rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash;
|
||||
rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment.
|
||||
rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh"
|
||||
exit /b %ERRORLEVEL%
|
||||
Executable
+35
@@ -0,0 +1,35 @@
|
||||
#!/bin/sh
|
||||
# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`.
|
||||
# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname,
|
||||
# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are
|
||||
# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy);
|
||||
# restore it explicitly afterwards: scripts/restore-api.sh.
|
||||
# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s)
|
||||
# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process
|
||||
# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
mode="${STOP_MODE:-stop}"
|
||||
|
||||
target=""
|
||||
# Only RUNNING containers of THIS compose project's service `api` are candidates.
|
||||
for id in $(docker compose ps -q api); do
|
||||
if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
|
||||
target="$id"
|
||||
fi
|
||||
done
|
||||
if [ -z "$target" ]; then
|
||||
echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2
|
||||
exit 3
|
||||
fi
|
||||
name="$(docker inspect -f '{{.Name}}' "$target")"
|
||||
case "$mode" in
|
||||
stop) docker stop --time 15 "$target" >/dev/null ;;
|
||||
kill) docker stop --time 0 "$target" >/dev/null ;;
|
||||
*) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;;
|
||||
esac
|
||||
echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")"
|
||||
@@ -0,0 +1,3 @@
|
||||
"""In-memory messenger for lab 1 (MTUSI, «Высоконагруженные приложения»)."""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
@@ -0,0 +1,88 @@
|
||||
"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker).
|
||||
|
||||
Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy):
|
||||
1. readiness flips to 503 (`app.state.draining = True`), requests are still served;
|
||||
2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has
|
||||
marked this replica DOWN and sends nothing new here;
|
||||
3. uvicorn closes the listening socket; in-flight requests get up to
|
||||
SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish;
|
||||
4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0.
|
||||
DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL.
|
||||
A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
from types import FrameType
|
||||
|
||||
import uvicorn
|
||||
|
||||
from . import __version__, logging_setup
|
||||
from .app import create_app
|
||||
from .config import ConfigError, Settings
|
||||
|
||||
log = logging.getLogger("messenger")
|
||||
|
||||
|
||||
class GracefulServer(uvicorn.Server):
|
||||
"""uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop."""
|
||||
|
||||
def __init__(self, config: uvicorn.Config, drain_seconds: float):
|
||||
super().__init__(config)
|
||||
self.drain_seconds = drain_seconds
|
||||
self.drain_timer: threading.Timer | None = None
|
||||
|
||||
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
|
||||
self.config.app.state.draining = True # config.app is our Starlette instance
|
||||
if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit:
|
||||
log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds,
|
||||
"timeout_s": self.config.timeout_graceful_shutdown}})
|
||||
# uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe.
|
||||
self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame))
|
||||
self.drain_timer.daemon = True
|
||||
self.drain_timer.start()
|
||||
return
|
||||
if self.drain_timer is not None:
|
||||
self.drain_timer.cancel() # second signal: stop draining now
|
||||
self._stop(sig, frame)
|
||||
|
||||
def _stop(self, sig: int, frame: FrameType | None) -> None:
|
||||
if not self.should_exit:
|
||||
log.info("shutdown_requested", extra={"fields": {"signal": sig}})
|
||||
super().handle_exit(sig, frame)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
settings = Settings.from_env()
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
logging_setup.configure(settings.log_level)
|
||||
app = create_app(settings)
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
log_config=None,
|
||||
access_log=False,
|
||||
server_header=False,
|
||||
proxy_headers=False,
|
||||
lifespan="on",
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout,
|
||||
)
|
||||
server = GracefulServer(config, settings.drain_seconds)
|
||||
log.info("starting", extra={"fields": {
|
||||
"version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port,
|
||||
"python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id,
|
||||
}})
|
||||
server.run()
|
||||
log.info("stopped")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,609 @@
|
||||
"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
from starlette.applications import Starlette
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import JSONResponse, Response, StreamingResponse
|
||||
from starlette.routing import Route
|
||||
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
|
||||
|
||||
from . import attachments, authhdr, cursor, docs, passwords, repo, validation
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .errors import BASIC_CHALLENGE, ApiError, Conflict, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
|
||||
from .sessions import SessionStore, build_store
|
||||
from .storage import ObjectMissing, Storage
|
||||
from .timefmt import rfc3339
|
||||
|
||||
log = logging.getLogger("messenger")
|
||||
access_log = logging.getLogger("messenger.access")
|
||||
|
||||
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Responses
|
||||
# --------------------------------------------------------------------------
|
||||
def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse:
|
||||
body = {"error": {"code": code, "message": message, "request_id": request_id}}
|
||||
return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})})
|
||||
|
||||
|
||||
async def api_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
assert isinstance(exc, ApiError)
|
||||
return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
"""Unknown routes and wrong methods also answer in the contract error format."""
|
||||
assert isinstance(exc, HTTPException)
|
||||
if exc.status_code == 404:
|
||||
return error_response(request.state.request_id, 404, "not_found", "Объект не найден")
|
||||
if exc.status_code == 405:
|
||||
return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers)
|
||||
return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
|
||||
# --------------------------------------------------------------------------
|
||||
class RequestContextMiddleware:
|
||||
"""Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503
|
||||
included), writes one access-log line, turns crashes into 500."""
|
||||
|
||||
def __init__(self, app: ASGIApp, instance_id: str):
|
||||
self.app = app
|
||||
self.instance_id = instance_id.encode("ascii")
|
||||
|
||||
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
||||
if scope["type"] != "http":
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1")
|
||||
request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
|
||||
scope.setdefault("state", {})["request_id"] = request_id
|
||||
status = 500
|
||||
started = False
|
||||
begin = time.perf_counter()
|
||||
|
||||
async def send_with_id(message: AsgiMessage) -> None:
|
||||
nonlocal status, started
|
||||
if message["type"] == "http.response.start":
|
||||
started = True
|
||||
status = message["status"]
|
||||
headers = [(k, v) for k, v in message.get("headers", [])
|
||||
if k.lower() not in (b"x-request-id", b"x-instance-id")]
|
||||
headers.append((b"x-request-id", request_id.encode("latin-1")))
|
||||
headers.append((b"x-instance-id", self.instance_id))
|
||||
message = {**message, "headers": headers}
|
||||
await send(message)
|
||||
|
||||
try:
|
||||
await self.app(scope, receive, send_with_id)
|
||||
except asyncio.CancelledError:
|
||||
# uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires:
|
||||
# answer 503 (retryable) instead of a generic 500, then let the cancellation proceed.
|
||||
if not started:
|
||||
status = 503
|
||||
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
|
||||
await response(scope, receive, send_with_id)
|
||||
raise
|
||||
except Exception:
|
||||
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
|
||||
if not started:
|
||||
status = 500
|
||||
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
await response(scope, receive, send_with_id)
|
||||
finally:
|
||||
access_log.info("request", extra={"fields": {
|
||||
"request_id": request_id,
|
||||
"method": scope["method"],
|
||||
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
|
||||
"status": status,
|
||||
"instance_id": self.instance_id.decode("ascii"),
|
||||
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
|
||||
}})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Request helpers
|
||||
# --------------------------------------------------------------------------
|
||||
def db_of(request: Request) -> Database:
|
||||
return request.app.state.db
|
||||
|
||||
|
||||
def sessions_of(request: Request) -> SessionStore:
|
||||
return request.app.state.sessions
|
||||
|
||||
|
||||
def storage_of(request: Request) -> Storage:
|
||||
return request.app.state.storage
|
||||
|
||||
|
||||
def staging_of(request: Request) -> Storage:
|
||||
return request.app.state.staging
|
||||
|
||||
|
||||
async def authenticate(request: Request) -> str:
|
||||
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
|
||||
|
||||
Store failures are 503 (Unavailable), never 401 and never anonymous access.
|
||||
"""
|
||||
token = authhdr.parse_bearer(request.headers.get("authorization"))
|
||||
user_id = await sessions_of(request).user_id_for(token)
|
||||
if user_id is None:
|
||||
raise Unauthorized("Сессия недействительна, истекла или отозвана")
|
||||
return user_id
|
||||
|
||||
|
||||
def path_uuid(request: Request, name: str) -> str:
|
||||
return validation.parse_uuid(request.path_params[name], name)
|
||||
|
||||
|
||||
async def read_json(request: Request) -> dict[str, Any]:
|
||||
content_type = request.headers.get("content-type", "")
|
||||
media_type, _, params = content_type.partition(";")
|
||||
if media_type.strip().lower() != "application/json":
|
||||
raise InvalidRequest("Content-Type должен быть application/json")
|
||||
charset = params.strip().lower()
|
||||
if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}:
|
||||
raise InvalidRequest("Поддерживается только кодировка UTF-8")
|
||||
limit: int = request.app.state.settings.max_body_bytes
|
||||
declared = request.headers.get("content-length")
|
||||
if declared is not None and declared.isdigit() and int(declared) > limit:
|
||||
raise PayloadTooLarge()
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > limit:
|
||||
raise PayloadTooLarge()
|
||||
chunks.append(chunk)
|
||||
return validation.parse_json_object(b"".join(chunks))
|
||||
|
||||
|
||||
def page_params(request: Request) -> tuple[int, str | None]:
|
||||
query = request.query_params.multi_items()
|
||||
limit = validation.parse_limit(validation.single_query_param(query, "limit"))
|
||||
raw_cursor = validation.single_query_param(query, "cursor")
|
||||
if raw_cursor is not None and raw_cursor == "":
|
||||
raise InvalidRequest("cursor не может быть пустым")
|
||||
return limit, raw_cursor
|
||||
|
||||
|
||||
async def reject_body(request: Request) -> None:
|
||||
"""POST /auth/sessions takes credentials only from the Basic header; any body is 400."""
|
||||
if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0":
|
||||
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
|
||||
async for chunk in request.stream():
|
||||
if chunk:
|
||||
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
|
||||
|
||||
|
||||
def idempotency_key(request: Request) -> str:
|
||||
values = request.headers.getlist("idempotency-key")
|
||||
return validation.idempotency_key(values[0] if len(values) == 1 else None)
|
||||
|
||||
|
||||
def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes:
|
||||
"""SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter."""
|
||||
canonical = json.dumps({"attachment_ids": attachment_ids, "text": text},
|
||||
ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||
|
||||
|
||||
def ok(body: Any, status: int = 200) -> JSONResponse:
|
||||
return JSONResponse(body, status_code=status)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------
|
||||
# Handlers
|
||||
# --------------------------------------------------------------------------
|
||||
async def live(request: Request) -> JSONResponse:
|
||||
return ok({"status": "ok"})
|
||||
|
||||
|
||||
async def ready(request: Request) -> JSONResponse:
|
||||
"""200 only if the WHOLE mandatory synchronous path can be served: PostgreSQL, Valkey
|
||||
(SESSION_STORE=valkey) and, since lab 4, the S3 bucket (upload/download of attachments)
|
||||
answer. 503 while draining after SIGTERM. HAProxy routes to such replicas first."""
|
||||
if request.app.state.draining:
|
||||
raise Unavailable("Сервер завершает работу")
|
||||
db_ok, store_ok, s3_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping(),
|
||||
storage_of(request).ping())
|
||||
if not db_ok:
|
||||
raise Unavailable("PostgreSQL недоступен")
|
||||
if not store_ok:
|
||||
raise Unavailable("Хранилище сессий недоступно")
|
||||
if not s3_ok:
|
||||
raise Unavailable("Хранилище файлов (S3) недоступно")
|
||||
return ok({"status": "ok"})
|
||||
|
||||
|
||||
async def ready_core(request: Request) -> JSONResponse:
|
||||
"""Not in the contract: readiness WITHOUT S3, for the load balancer's degraded mode only.
|
||||
If S3 is down for every replica, /health/ready is 503 everywhere; HAProxy then routes to
|
||||
replicas that are at least core-ready, so chats and messages keep working while every
|
||||
attachment request answers 503 itself (see infra/haproxy/haproxy.cfg)."""
|
||||
if request.app.state.draining:
|
||||
raise Unavailable("Сервер завершает работу")
|
||||
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
|
||||
if not (db_ok and store_ok):
|
||||
raise Unavailable("PostgreSQL или хранилище сессий недоступно")
|
||||
return ok({"status": "ok"})
|
||||
|
||||
|
||||
async def register_user(request: Request) -> JSONResponse:
|
||||
data = await read_json(request)
|
||||
validation.check_fields(data, {"username", "display_name", "password"})
|
||||
username = validation.username_field(data)
|
||||
display_name = validation.text_field(data, "display_name", 100)
|
||||
password = validation.password_field(data)
|
||||
password_hash = await passwords.hash_password(password)
|
||||
user = await repo.create_user(db_of(request), username, display_name, password_hash)
|
||||
log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
|
||||
return ok(user.to_json(), 201)
|
||||
|
||||
|
||||
async def create_session(request: Request) -> JSONResponse:
|
||||
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
|
||||
username, password = authhdr.parse_basic(request.headers.get("authorization"))
|
||||
await reject_body(request)
|
||||
found = await repo.get_credentials(db_of(request), username)
|
||||
user, stored_hash = found if found else (None, None)
|
||||
if not await passwords.verify_password(stored_hash, password) or user is None:
|
||||
log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}})
|
||||
raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
|
||||
ttl = request.app.state.settings.session_ttl_seconds
|
||||
token, expires_at = await sessions_of(request).create(user.id, ttl)
|
||||
log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
|
||||
return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201)
|
||||
|
||||
|
||||
async def revoke_session(request: Request) -> Response:
|
||||
token = authhdr.parse_bearer(request.headers.get("authorization"))
|
||||
if not await sessions_of(request).revoke(token):
|
||||
raise Unauthorized("Сессия недействительна, истекла или отозвана")
|
||||
log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}})
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
async def me(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
user = await repo.get_user(db_of(request), user_id)
|
||||
if user is None:
|
||||
raise Unauthorized("Сессия недействительна, истекла или отозвана")
|
||||
return ok(user.to_json())
|
||||
|
||||
|
||||
async def create_chat(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
data = await read_json(request)
|
||||
validation.check_fields(data, {"title", "member_ids"})
|
||||
title = validation.text_field(data, "title", 100)
|
||||
member_ids = validation.member_ids_field(data)
|
||||
chat = await repo.create_chat(db_of(request), user_id, title, member_ids)
|
||||
log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}})
|
||||
return ok(chat.to_json(), 201)
|
||||
|
||||
|
||||
async def list_chats(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
limit, raw_cursor = page_params(request)
|
||||
after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None
|
||||
items, has_more = await repo.list_chats(db_of(request), user_id, after, limit)
|
||||
next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None
|
||||
return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor})
|
||||
|
||||
|
||||
async def get_chat(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
chat_id = path_uuid(request, "chat_id")
|
||||
return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json())
|
||||
|
||||
|
||||
def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]:
|
||||
"""`text` is required and a string (also with attachments); `attachment_ids`, if present, an
|
||||
array of 0-10 unique UUIDs. "" or whitespace-only text is allowed only with attachments."""
|
||||
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
|
||||
attachment_ids = validation.attachment_ids_field(data)
|
||||
text = validation.text_field(data, "text", 4000, min_len=0 if attachment_ids else 1,
|
||||
allow_blank=bool(attachment_ids))
|
||||
return text, attachment_ids
|
||||
|
||||
|
||||
async def create_message(request: Request) -> JSONResponse:
|
||||
"""Order of answers for both write paths: 401 -> 400 bad chat id -> 404 outsider -> 400 key/body
|
||||
-> 409 other body with the same key -> 201 (new or replayed). sender_id comes from the session."""
|
||||
user_id = await authenticate(request)
|
||||
chat_id = path_uuid(request, "chat_id")
|
||||
db = db_of(request)
|
||||
settings = request.app.state.settings
|
||||
if settings.message_write_path == "multi":
|
||||
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at
|
||||
key = idempotency_key(request)
|
||||
text, attachment_ids = parse_message(await read_json(request))
|
||||
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
|
||||
settings.idempotency_ttl_seconds, attachment_ids)
|
||||
else:
|
||||
try:
|
||||
key = idempotency_key(request)
|
||||
text, attachment_ids = parse_message(await read_json(request))
|
||||
except ApiError:
|
||||
# Rare error path only: an outsider must still get 404, not a hint about the body.
|
||||
await repo.ensure_member(db, user_id, chat_id)
|
||||
raise
|
||||
# Membership check, key claim, chat clock and message: one SQL call, one transaction.
|
||||
result = await repo.post_message_single_call(db, user_id, chat_id, key,
|
||||
request_fingerprint(text, attachment_ids), text,
|
||||
settings.idempotency_ttl_seconds, attachment_ids)
|
||||
if result.replayed:
|
||||
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
|
||||
"message_id": result.message.id}})
|
||||
return ok(result.message.to_json(), 201)
|
||||
|
||||
|
||||
async def list_messages(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
chat_id = path_uuid(request, "chat_id")
|
||||
db = db_of(request)
|
||||
await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak
|
||||
limit, raw_cursor = page_params(request)
|
||||
after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None
|
||||
items, has_more = await repo.list_messages(db, chat_id, after, limit)
|
||||
next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None
|
||||
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ attachments (lab 4, lab 5)
|
||||
def fault(request: Request, point: str) -> None:
|
||||
"""Failure injection for the experiments (FAULT_POINTS of the api service, compose variable
|
||||
API_FAULT_POINTS; empty in normal runs)."""
|
||||
if point in request.app.state.settings.fault_points:
|
||||
log.warning("fault_injected", extra={"fields": {"request_id": request.state.request_id, "point": point}})
|
||||
raise Unavailable("Внедрённый отказ: " + point)
|
||||
|
||||
|
||||
async def upload_attachment(request: Request) -> JSONResponse:
|
||||
"""POST /chats/{chat_id}/attachments?filename=...: raw PNG/JPEG -> 202 queued (attachments.py)."""
|
||||
settings: Settings = request.app.state.settings
|
||||
reader = attachments.BodyReader(request)
|
||||
slots: asyncio.Semaphore = request.app.state.upload_slots
|
||||
acquired = False
|
||||
try:
|
||||
try:
|
||||
user_id = await authenticate(request)
|
||||
chat_id = path_uuid(request, "chat_id")
|
||||
db = db_of(request)
|
||||
await repo.ensure_member(db, user_id, chat_id) # outsiders: 404 before anything about the body
|
||||
filename = validation.filename_param(validation.query_items(request.scope["query_string"]))
|
||||
declared = attachments.declared_type(request)
|
||||
length = attachments.declared_length(request, settings.max_upload_bytes)
|
||||
try:
|
||||
await asyncio.wait_for(slots.acquire(), settings.upload_queue_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise attachments.Busy("Слишком много одновременных загрузок, повторите позже") from None
|
||||
acquired = True
|
||||
upload = await attachments.read_image(reader, settings, declared, request.app.state.verify_slots, length)
|
||||
except ApiError:
|
||||
# Early rejection: drop (not store) a bounded rest of the body so the client reads our error.
|
||||
await reader.discard(settings.max_upload_bytes + (1 << 20), timeout=5.0)
|
||||
raise
|
||||
# The slot is held until the buffer is no longer needed (after the S3 PUT), so at most
|
||||
# UPLOAD_CONCURRENCY bodies of this replica exist in memory at any moment.
|
||||
return await store_upload(request, db, chat_id, user_id, filename, upload)
|
||||
finally:
|
||||
if acquired:
|
||||
slots.release()
|
||||
|
||||
|
||||
async def store_upload(request: Request, db: Database, chat_id: str, user_id: str, filename: str,
|
||||
upload: attachments.Upload) -> JSONResponse:
|
||||
"""intent (pending) -> staging PUT -> queued + job + outbox -> 202 (docs/PIPELINE.md).
|
||||
No resize here: the worker does it; a stopped worker only delays `ready`."""
|
||||
settings: Settings = request.app.state.settings
|
||||
attachment_id = uuid.uuid4()
|
||||
staging_key = attachments.new_staging_key()
|
||||
await attachments.create_intent(db, attachment_id, chat_id, user_id, filename, upload.info, len(upload.data),
|
||||
upload.sha256, staging_key)
|
||||
await staging_of(request).put(staging_key, upload.data, upload.info.content_type,
|
||||
{"attachment-id": str(attachment_id), "sha256": upload.sha256.hex()})
|
||||
fault(request, "after_s3_put")
|
||||
attachment = await attachments.enqueue(db, attachment_id, settings)
|
||||
if attachment is None:
|
||||
raise Unavailable("Загрузка не завершена, повторите запрос")
|
||||
fault(request, "after_enqueue_commit")
|
||||
log.info("attachment_queued", extra={"fields": {"request_id": request.state.request_id,
|
||||
"attachment_id": attachment.id, "chat_id": chat_id,
|
||||
"size": attachment.size_bytes, "type": attachment.content_type,
|
||||
"pipeline_version": settings.pipeline_version}})
|
||||
return ok(attachment.to_json(), 202)
|
||||
|
||||
|
||||
async def get_attachment(request: Request) -> JSONResponse:
|
||||
user_id = await authenticate(request)
|
||||
attachment_id = path_uuid(request, "attachment_id")
|
||||
return ok((await attachments.get_visible(db_of(request), user_id, attachment_id)).to_json())
|
||||
|
||||
|
||||
def content_disposition(filename: str) -> str:
|
||||
"""RFC 6266/5987: ASCII fallback + UTF-8 name; the stored name itself is never interpreted."""
|
||||
from urllib.parse import quote
|
||||
fallback = "".join(c if 32 <= ord(c) < 127 and c not in '"\\%;' else "_" for c in filename)
|
||||
return f"attachment; filename=\"{fallback}\"; filename*=UTF-8''{quote(filename, safe='')}"
|
||||
|
||||
|
||||
async def download_attachment(request: Request) -> Response:
|
||||
"""Original or variant bytes through the authorized API: no redirect, no public S3 URL.
|
||||
queued/processing/failed -> 409 (not ready); a variant a ready attachment lacks -> 404.
|
||||
Streamed from S3 in 64 KiB chunks (memory per download does not depend on the file size)."""
|
||||
user_id = await authenticate(request)
|
||||
attachment_id = path_uuid(request, "attachment_id")
|
||||
query = validation.query_items(request.scope["query_string"])
|
||||
try:
|
||||
variant = validation.variant_param(query)
|
||||
except ApiError:
|
||||
await attachments.get_visible(db_of(request), user_id, attachment_id) # 404 for outsiders first
|
||||
raise
|
||||
attachment = await attachments.get_visible(db_of(request), user_id, attachment_id)
|
||||
if attachment.status != "ready":
|
||||
raise Conflict("Обработка вложения не удалась" if attachment.status == "failed" else "Вложение ещё не готово")
|
||||
if attachment.lost:
|
||||
raise NotFound("Оригинал вложения утрачен")
|
||||
if variant == "original":
|
||||
key, size_expected, media_type, etag = (attachment.object_key, attachment.size_bytes,
|
||||
attachment.content_type, attachment.sha256)
|
||||
name = attachment.filename
|
||||
else:
|
||||
found = attachment.variant(variant)
|
||||
if found is None:
|
||||
raise NotFound("Такого варианта у вложения нет")
|
||||
key, size_expected, media_type, etag = found.object_key, found.size_bytes, found.content_type, found.sha256
|
||||
stem = attachment.filename.rsplit(".", 1)[0] or "image"
|
||||
name = stem + "-" + variant + (".webp" if media_type == "image/webp" else ".jpg")
|
||||
try:
|
||||
size, chunks = await storage_of(request).open_stream(key)
|
||||
except ObjectMissing:
|
||||
log.error("attachment_object_missing", extra={"fields": {"request_id": request.state.request_id,
|
||||
"attachment_id": attachment.id, "variant": variant}})
|
||||
raise Unavailable("Файл вложения временно недоступен") from None
|
||||
if size != size_expected:
|
||||
log.error("attachment_size_mismatch", extra={"fields": {"request_id": request.state.request_id,
|
||||
"attachment_id": attachment.id, "variant": variant}})
|
||||
await chunks.aclose()
|
||||
raise Unavailable("Файл вложения временно недоступен")
|
||||
headers = {
|
||||
"Content-Length": str(size),
|
||||
"Content-Disposition": content_disposition(name),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Cache-Control": "private, no-store",
|
||||
"ETag": '"' + etag + '"',
|
||||
}
|
||||
return StreamingResponse(chunks, media_type=media_type, headers=headers)
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> Starlette:
|
||||
settings = settings or Settings.from_env()
|
||||
db = Database(settings)
|
||||
sessions = build_store(settings, db)
|
||||
storage = Storage(settings) # final bucket: Read
|
||||
staging = Storage(settings, bucket=settings.s3_staging_bucket) # staging bucket: Write (+ janitor DELETE)
|
||||
|
||||
async def purge_loop() -> None:
|
||||
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
|
||||
while True:
|
||||
await asyncio.sleep(600)
|
||||
try:
|
||||
deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds)
|
||||
if deleted:
|
||||
log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}})
|
||||
except ApiError:
|
||||
pass # database unavailable right now; next round
|
||||
|
||||
async def janitor_loop(every: float) -> None:
|
||||
"""Cleanup of intents of failed uploads older than PENDING_GRACE_SECONDS (attachments.py).
|
||||
Every replica runs it; the advisory lock lets one round run at a time."""
|
||||
while True:
|
||||
await asyncio.sleep(every * (0.5 + random.random()))
|
||||
try:
|
||||
removed = await attachments.clean_stale_pending(db, staging, settings.pending_grace_seconds)
|
||||
if removed:
|
||||
log.info("stale_uploads_removed", extra={"fields": {"count": removed}})
|
||||
except ApiError:
|
||||
pass # PostgreSQL or S3 unavailable right now; the rows stay pending for the next round
|
||||
|
||||
stats_log = logging.getLogger("messenger.stats")
|
||||
|
||||
async def stats_loop(every: float) -> None:
|
||||
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
|
||||
cpu = time.process_time()
|
||||
db.pool.pop_stats()
|
||||
while True:
|
||||
await asyncio.sleep(every)
|
||||
stats = db.pool.pop_stats()
|
||||
now_cpu = time.process_time()
|
||||
stats_log.info("pool_stats", extra={"fields": {
|
||||
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
|
||||
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
|
||||
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
|
||||
cpu = now_cpu
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def lifespan(app: Starlette):
|
||||
passwords.configure(settings.hash_concurrency)
|
||||
await db.open()
|
||||
await storage.open()
|
||||
await staging.open()
|
||||
log.info("dependencies_configured", extra={"fields": {
|
||||
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
|
||||
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
|
||||
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
|
||||
"message_write_path": settings.message_write_path, "s3_bucket": settings.s3_bucket,
|
||||
"s3_staging_bucket": settings.s3_staging_bucket, "pipeline_version": settings.pipeline_version,
|
||||
"upload_concurrency": settings.upload_concurrency, "verify_concurrency": settings.image_verify_concurrency,
|
||||
"pending_grace_s": settings.pending_grace_seconds,
|
||||
"fault_points": sorted(settings.fault_points)}})
|
||||
tasks = [asyncio.create_task(purge_loop())]
|
||||
if settings.janitor_interval_seconds > 0:
|
||||
tasks.append(asyncio.create_task(janitor_loop(settings.janitor_interval_seconds)))
|
||||
if settings.stats_log_seconds > 0:
|
||||
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await sessions.close()
|
||||
await staging.close()
|
||||
await storage.close()
|
||||
await db.close()
|
||||
|
||||
routes = [
|
||||
Route("/health/live", live, methods=["GET"]),
|
||||
Route("/health/ready", ready, methods=["GET"]),
|
||||
Route("/health/ready/core", ready_core, methods=["GET"]),
|
||||
Route("/api/v1/users", register_user, methods=["POST"]),
|
||||
Route("/api/v1/users/me", me, methods=["GET"]),
|
||||
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
|
||||
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
|
||||
Route("/api/v1/chats", create_chat, methods=["POST"]),
|
||||
Route("/api/v1/chats", list_chats, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
|
||||
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
|
||||
Route("/api/v1/chats/{chat_id}/attachments", upload_attachment, methods=["POST"]),
|
||||
Route("/api/v1/attachments/{attachment_id}", get_attachment, methods=["GET"]),
|
||||
Route("/api/v1/attachments/{attachment_id}/content", download_attachment, methods=["GET"]),
|
||||
]
|
||||
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
|
||||
routes += [
|
||||
Route("/docs", docs.swagger_page, methods=["GET"]),
|
||||
Route("/openapi.json", docs.openapi_json, methods=["GET"]),
|
||||
]
|
||||
app = Starlette(
|
||||
routes=routes,
|
||||
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
|
||||
lifespan=lifespan,
|
||||
)
|
||||
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id)
|
||||
app.state.settings = settings
|
||||
app.state.db = db
|
||||
app.state.sessions = sessions
|
||||
app.state.storage = storage
|
||||
app.state.staging = staging
|
||||
app.state.upload_slots = asyncio.Semaphore(settings.upload_concurrency)
|
||||
app.state.verify_slots = asyncio.Semaphore(settings.image_verify_concurrency)
|
||||
app.state.draining = False
|
||||
return app
|
||||
@@ -0,0 +1,339 @@
|
||||
"""Attachments: metadata in PostgreSQL, bytes in private S3 buckets (lab 4), processed
|
||||
asynchronously by the worker (lab 5, docs/PIPELINE.md).
|
||||
|
||||
Upload (POST /chats/{id}/attachments, raw PNG/JPEG body), in this order:
|
||||
|
||||
auth -> membership (404) -> filename / Content-Type / Content-Length (400/415/413)
|
||||
-> upload slot (at most UPLOAD_CONCURRENCY bodies per replica, else 503 + Retry-After)
|
||||
-> body in chunks: size limit, read deadline, format + pixel count from the first bytes
|
||||
-> full integrity check (images.verify) -> SHA-256 / size known (415 is decided HERE)
|
||||
-> INSERT row status='pending' with a fresh random staging key, COMMIT (intent)
|
||||
-> PUT the original into the STAGING bucket (Content-MD5)
|
||||
-> ONE transaction: status 'pending' -> 'queued' + jobs row + outbox row, COMMIT
|
||||
-> 202 queued.
|
||||
|
||||
What is durable at 202: the complete, verified original in the staging bucket (S3 confirmed the
|
||||
PUT and its MD5), the metadata row, the job and the outbox row that makes the relay publish it.
|
||||
A crash of the API after the answer loses nothing; a broker that is down only delays the publish.
|
||||
|
||||
Failures between the steps (docs/PIPELINE.md, failure windows):
|
||||
* before the intent COMMIT: nothing was written;
|
||||
* after the intent, before/while PUT, or between PUT and the job COMMIT (FAULT_POINTS=after_s3_put):
|
||||
a 'pending' row (+ maybe the staging object), invisible to everybody (404). The client got 503
|
||||
and may retry (a new attachment). The janitor deletes the staging object - idempotent - and then
|
||||
the row once it is older than PENDING_GRACE_SECONDS, deciding under the row lock;
|
||||
* after the job COMMIT, answer lost (FAULT_POINTS=after_enqueue_commit): a correct queued
|
||||
attachment the client does not know; the worker processes it as usual.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import logging
|
||||
import secrets
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors as pg_errors
|
||||
from starlette.requests import Request
|
||||
|
||||
from . import images, jobs
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .errors import ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unavailable, UnsupportedMediaType
|
||||
from .storage import STAGING_PREFIX, Storage
|
||||
from .timefmt import rfc3339
|
||||
|
||||
log = logging.getLogger("messenger.attachments")
|
||||
|
||||
JANITOR_LOCK_ID = 2026_04 # advisory lock: one janitor round at a time across all replicas
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Variant:
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
object_key: str
|
||||
|
||||
def to_json(self) -> dict:
|
||||
return {"name": self.name, "content_type": self.content_type, "size_bytes": self.size_bytes,
|
||||
"width": self.width, "height": self.height, "sha256": self.sha256}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Attachment:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
object_key: str | None # published original in the final bucket (None until the worker publishes it)
|
||||
status: str
|
||||
created_at: dt.datetime
|
||||
updated_at: dt.datetime
|
||||
lost: bool = False
|
||||
error_code: str | None = None
|
||||
published_version: int | None = None
|
||||
variants: tuple[Variant, ...] = ()
|
||||
|
||||
def to_json(self) -> dict:
|
||||
"""Public representation: no object keys, no buckets, no job internals."""
|
||||
return {"id": self.id, "chat_id": self.chat_id, "owner_id": self.owner_id, "filename": self.filename,
|
||||
"content_type": self.content_type, "size_bytes": self.size_bytes, "sha256": self.sha256,
|
||||
"status": self.status, "variants": [v.to_json() for v in self.variants] if self.status == "ready" else [],
|
||||
"error_code": self.error_code if self.status == "failed" else None,
|
||||
"created_at": rfc3339(self.created_at), "updated_at": rfc3339(self.updated_at)}
|
||||
|
||||
def variant(self, name: str) -> Variant | None:
|
||||
return next((v for v in self.variants if v.name == name), None)
|
||||
|
||||
|
||||
COLUMNS = ("a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, a.object_key, "
|
||||
"a.status, a.created_at, a.updated_at, a.lost_at IS NOT NULL, a.error_code, a.published_version")
|
||||
|
||||
|
||||
def _attachment(row, variants: tuple[Variant, ...] = ()) -> Attachment:
|
||||
return Attachment(str(row[0]), str(row[1]), str(row[2]), row[3], row[4], row[5], bytes(row[6]).hex(), row[7],
|
||||
row[8], row[9], row[10], row[11], row[12], row[13], variants)
|
||||
|
||||
|
||||
def new_staging_key() -> str:
|
||||
"""Server-generated, 128 random bits, unrelated to the attachment id and the filename."""
|
||||
return STAGING_PREFIX + secrets.token_hex(16)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ SQL
|
||||
async def create_intent(db: Database, attachment_id: uuid.UUID, chat_id: str, owner_id: str, filename: str,
|
||||
info: images.ImageInfo, size: int, sha256: bytes, staging_key: str) -> None:
|
||||
async def work(conn: psycopg.AsyncConnection) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, "
|
||||
"width, height, staging_key, staging_state, status) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'present', 'pending')",
|
||||
(attachment_id, chat_id, owner_id, filename, info.content_type, size, sha256, info.width, info.height,
|
||||
staging_key))
|
||||
try:
|
||||
await db.run(work, transaction=False)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise NotFound("Чат не найден") from None
|
||||
|
||||
|
||||
async def enqueue(db: Database, attachment_id: uuid.UUID, settings: Settings) -> Attachment | None:
|
||||
"""pending -> queued + job + outbox row in ONE transaction (the durable handoff before 202).
|
||||
None if the row is no longer pending (the janitor removed a stale intent)."""
|
||||
async def work(conn: psycopg.AsyncConnection) -> Attachment | None:
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachments a SET status = 'queued', updated_at = clock_timestamp() "
|
||||
f"WHERE a.id = %s AND a.status = 'pending' RETURNING {COLUMNS}", (attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
await jobs.create_job(conn, attachment_id, settings.pipeline_version, "upload", settings.job_max_attempts,
|
||||
"api:" + settings.instance_id)
|
||||
return _attachment(row)
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
async def get_visible(db: Database, user_id: str, attachment_id: str) -> Attachment:
|
||||
"""An attachment of a chat the user is a member of, in any state but the invisible intent
|
||||
'pending'; anything else is 404 (unknown, foreign chat, unfinished upload) - existence of
|
||||
foreign objects is never revealed. A ready attachment comes with its published variants."""
|
||||
async def work(conn: psycopg.AsyncConnection) -> Attachment:
|
||||
cur = await conn.execute(
|
||||
f"SELECT {COLUMNS} FROM attachments a "
|
||||
"JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = %s "
|
||||
"WHERE a.id = %s AND a.status <> 'pending'", (user_id, attachment_id))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
raise NotFound("Вложение не найдено")
|
||||
variants: tuple[Variant, ...] = ()
|
||||
if row[8] == "ready":
|
||||
cur = await conn.execute(
|
||||
"SELECT name, content_type, size_bytes, width, height, sha256, object_key FROM attachment_variants "
|
||||
"WHERE attachment_id = %s AND pipeline_version = %s "
|
||||
"ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)", (row[0], row[13]))
|
||||
variants = tuple(Variant(r[0], r[1], r[2], r[3], r[4], bytes(r[5]).hex(), r[6]) for r in await cur.fetchall())
|
||||
return _attachment(row, variants)
|
||||
# REPEATABLE READ is not needed: the variant rows of a published version never change.
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
async def clean_stale_pending(db: Database, staging: Storage, grace_seconds: int, batch: int = 20) -> int:
|
||||
"""Janitor round: remove intents of uploads that never got their job (no 202 was sent).
|
||||
|
||||
Under FOR UPDATE: an upload that is (impossibly) still running would block on the row in
|
||||
enqueue() and then find it gone (-> 503). Order: staging object first (idempotent), row second;
|
||||
a crash in between leaves the pending row and the next round repeats. Only status='pending'
|
||||
rows older than the grace period are touched: queued/processing/ready/failed never.
|
||||
"""
|
||||
async def work(conn: psycopg.AsyncConnection) -> int:
|
||||
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (JANITOR_LOCK_ID,))
|
||||
if not (await cur.fetchone())[0]:
|
||||
return 0
|
||||
cur = await conn.execute(
|
||||
"SELECT id, staging_key FROM attachments WHERE status = 'pending' "
|
||||
"AND created_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"ORDER BY created_at LIMIT %s FOR UPDATE SKIP LOCKED", (grace_seconds, batch))
|
||||
rows = await cur.fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
await asyncio.gather(*(staging.delete(key) for _, key in rows))
|
||||
await conn.execute("DELETE FROM attachments WHERE id = ANY(%s) AND status = 'pending'",
|
||||
([r[0] for r in rows],))
|
||||
return len(rows)
|
||||
return await db.run(work, deadline=staging.timeout + 5)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ request body
|
||||
class Busy(Unavailable):
|
||||
"""All upload slots of this replica are taken: retry shortly (no body was read)."""
|
||||
|
||||
@property
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {"Retry-After": "1"}
|
||||
|
||||
|
||||
@dataclass
|
||||
class Upload:
|
||||
data: bytearray
|
||||
info: images.ImageInfo
|
||||
sha256: bytes
|
||||
|
||||
|
||||
class BodyReader:
|
||||
"""Reads the raw ASGI body ourselves (not request.stream()), so we always know how much of it
|
||||
is consumed and can discard the rest after an early rejection."""
|
||||
|
||||
def __init__(self, request: Request):
|
||||
self.receive = request.receive
|
||||
self.done = False
|
||||
|
||||
async def chunk(self, timeout: float) -> bytes | None:
|
||||
if self.done:
|
||||
return None
|
||||
message = await asyncio.wait_for(self.receive(), timeout)
|
||||
if message["type"] == "http.disconnect":
|
||||
self.done = True
|
||||
raise InvalidRequest("Клиент прервал загрузку")
|
||||
if not message.get("more_body", False):
|
||||
self.done = True
|
||||
return message.get("body", b"")
|
||||
|
||||
async def discard(self, limit: int, timeout: float) -> None:
|
||||
"""After an early 4xx: read and DROP up to `limit` more bytes, so an ordinary client that
|
||||
writes the whole body before reading the answer still gets our JSON error instead of a
|
||||
connection reset. Nothing is kept in memory; beyond the limit the connection is dropped."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + timeout
|
||||
seen = 0
|
||||
try:
|
||||
while not self.done and seen <= limit:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
return
|
||||
seen += len(await self.chunk(remaining) or b"")
|
||||
except (ApiError, asyncio.TimeoutError):
|
||||
return
|
||||
|
||||
|
||||
def declared_type(request: Request) -> str:
|
||||
"""Content-Type of the upload must be image/png or image/jpeg (parameters ignored)."""
|
||||
media_type = request.headers.get("content-type", "").partition(";")[0].strip().lower()
|
||||
if media_type not in (images.PNG, images.JPEG):
|
||||
raise UnsupportedMediaType("Content-Type должен быть image/png или image/jpeg")
|
||||
return media_type
|
||||
|
||||
|
||||
def declared_length(request: Request, limit: int) -> int | None:
|
||||
raw = request.headers.get("content-length")
|
||||
if raw is None:
|
||||
return None # chunked transfer: the limit is enforced while reading
|
||||
if not raw.isdigit():
|
||||
raise InvalidRequest("Неверный Content-Length")
|
||||
length = int(raw)
|
||||
if length > limit:
|
||||
raise PayloadTooLarge(f"Файл больше {limit} байт")
|
||||
if length == 0:
|
||||
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
|
||||
return length
|
||||
|
||||
|
||||
async def read_image(reader: BodyReader, settings: Settings, declared: str, verify_slots: asyncio.Semaphore,
|
||||
length: int | None) -> Upload:
|
||||
"""Body -> verified image. Memory: ONE buffer of at most MAX_UPLOAD_BYTES per upload - exactly
|
||||
Content-Length bytes allocated once when the length is declared (no growth, no final copy);
|
||||
it is passed as is to the checks and to the S3 PUT. The format and the pixel count are known
|
||||
after the first bytes, so a picture that is too big or not a PNG/JPEG is rejected without
|
||||
reading/decoding the rest."""
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + settings.upload_read_timeout_seconds
|
||||
limit = settings.max_upload_bytes
|
||||
buffer = bytearray(length) if length else bytearray()
|
||||
view = memoryview(buffer) if length else None
|
||||
filled = 0
|
||||
digest = hashlib.sha256()
|
||||
info: images.ImageInfo | None = None
|
||||
while True:
|
||||
remaining = deadline - loop.time()
|
||||
if remaining <= 0:
|
||||
raise InvalidRequest("Тело запроса не получено вовремя")
|
||||
try:
|
||||
chunk = await reader.chunk(remaining)
|
||||
except asyncio.TimeoutError:
|
||||
raise InvalidRequest("Тело запроса не получено вовремя") from None
|
||||
if chunk is None:
|
||||
break
|
||||
if filled + len(chunk) > (length or limit):
|
||||
raise PayloadTooLarge(f"Файл больше {limit} байт")
|
||||
if view is not None:
|
||||
view[filled:filled + len(chunk)] = chunk
|
||||
else:
|
||||
buffer += chunk
|
||||
filled += len(chunk)
|
||||
digest.update(chunk)
|
||||
if info is None and filled:
|
||||
info = _sniff(view[:filled] if view is not None else buffer, settings, declared)
|
||||
if view is not None:
|
||||
view.release()
|
||||
if not filled:
|
||||
raise InvalidRequest("Тело запроса пустое: нужны байты изображения")
|
||||
if length and filled != length:
|
||||
raise InvalidRequest("Тело запроса короче Content-Length")
|
||||
if info is None:
|
||||
raise UnsupportedMediaType("Файл не является корректным изображением PNG или JPEG")
|
||||
try:
|
||||
async with verify_slots:
|
||||
await images.verify(buffer, info, timeout=settings.image_verify_timeout_seconds,
|
||||
jpeg_max_memory_mb=settings.jpeg_max_memory_mb)
|
||||
except images.ImageError as exc:
|
||||
raise _image_error(exc) from None
|
||||
except asyncio.TimeoutError:
|
||||
raise UnsupportedMediaType("Изображение не удалось проверить за отведённое время") from None
|
||||
except (images.VerifierUnavailable, MemoryError):
|
||||
log.warning("image_verifier_unavailable", extra={"fields": {"type": info.content_type, "size": filled}})
|
||||
raise Busy("Не хватило ресурсов для проверки изображения, повторите позже") from None
|
||||
return Upload(buffer, info, digest.digest())
|
||||
|
||||
|
||||
def _sniff(buffer: bytearray, settings: Settings, declared: str) -> images.ImageInfo | None:
|
||||
try:
|
||||
info = images.sniff(buffer, settings.max_image_pixels)
|
||||
except images.ImageError as exc:
|
||||
raise _image_error(exc) from None
|
||||
if info is not None and info.content_type != declared:
|
||||
# "MIME spoofing": the header says PNG, the bytes are JPEG (or vice versa).
|
||||
raise UnsupportedMediaType("Содержимое не совпадает с Content-Type")
|
||||
return info
|
||||
|
||||
|
||||
def _image_error(exc: images.ImageError) -> ApiError:
|
||||
return PayloadTooLarge(exc.message) if exc.status == 413 else UnsupportedMediaType(exc.message)
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
"""Track 2: read-only checker of the pipeline invariants: `python -m messenger.audit`
|
||||
(compose service `auditor`, profile tools; `make audit`).
|
||||
|
||||
Read-only by construction: every database transaction is `READ ONLY` (PostgreSQL refuses any
|
||||
write), and the S3 key pair of the auditor has Read + List only. Output: one JSON report; exit 1
|
||||
if any invariant is violated.
|
||||
|
||||
Invariants (docs/PIPELINE.md):
|
||||
I1 no lost work - a queued/processing attachment has an active job; an active job has an
|
||||
outbox row (published or due) - nothing waits for a message that nobody sends.
|
||||
I2 one effect - per (attachment, pipeline_version) either no variant rows or exactly the
|
||||
three variants, all written by ONE successful attempt of a job of that version.
|
||||
I3 published data - every published object (original, variants) exists with the stored size
|
||||
(--verify-checksums: and the stored SHA-256).
|
||||
I4 monotonic - published_version = the highest version with a succeeded job; a ready
|
||||
attachment is never queued/processing again (published_version stays set).
|
||||
I5 source kept - an active job whose original is not published still has its staging copy
|
||||
(state present AND the object exists in the staging bucket).
|
||||
I6 no stray data - every object in the buckets is referenced: published, staging of a live row,
|
||||
or planned by an attempt whose objects were not cleaned yet. An object of an
|
||||
already cleaned attempt is reported as `leaked_object` (written by a stalled
|
||||
attempt after its cleanup: garbage, never a loss).
|
||||
I7 attempts - a succeeded job has exactly one succeeded attempt; at most one running attempt
|
||||
per job, and it is the lease holder.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import sys
|
||||
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .pipeline import VARIANT_NAMES
|
||||
from .storage import ObjectMissing, SENTINEL_KEY, Storage
|
||||
|
||||
|
||||
async def rows(db: Database, sql: str, params=()) -> list[tuple]:
|
||||
async def work(conn):
|
||||
await conn.execute("SET TRANSACTION READ ONLY")
|
||||
cur = await conn.execute(sql, params)
|
||||
return await cur.fetchall()
|
||||
return await db.run(work, deadline=db.deadline)
|
||||
|
||||
|
||||
async def list_all(storage: Storage, prefix: str) -> dict[str, int]:
|
||||
found, after = {}, None
|
||||
while True:
|
||||
page, more = await storage.list_page(prefix, after, 1000)
|
||||
for item in page:
|
||||
found[item.key] = item.size
|
||||
if not more or not page:
|
||||
return found
|
||||
after = page[-1].key
|
||||
|
||||
|
||||
async def audit(settings: Settings, verify: bool) -> dict:
|
||||
db = Database(settings, name="messenger-audit")
|
||||
final = Storage(settings)
|
||||
staging = Storage(settings, bucket=settings.s3_staging_bucket)
|
||||
await db.open()
|
||||
await final.open()
|
||||
await staging.open()
|
||||
violations: list[dict] = []
|
||||
notes: list[dict] = []
|
||||
|
||||
def bad(kind: str, **detail) -> None:
|
||||
violations.append({"invariant": kind, **{k: str(v) for k, v in detail.items()}})
|
||||
|
||||
try:
|
||||
# I1: queued/processing attachment without an active job; active job without outbox row.
|
||||
for (aid,) in await rows(db, "SELECT a.id FROM attachments a WHERE a.status IN ('queued', 'processing') "
|
||||
"AND NOT EXISTS (SELECT 1 FROM jobs j WHERE j.attachment_id = a.id "
|
||||
"AND j.state IN ('queued', 'processing'))"):
|
||||
bad("I1_no_active_job", attachment_id=aid)
|
||||
for (jid,) in await rows(db, "SELECT j.id FROM jobs j WHERE j.state = 'queued' AND NOT EXISTS "
|
||||
"(SELECT 1 FROM outbox o WHERE o.job_id = j.id)"):
|
||||
bad("I1_queued_without_outbox", job_id=jid)
|
||||
|
||||
# I2: exactly three variants per published version, one successful attempt of that version.
|
||||
for aid, version, names, tokens in await rows(
|
||||
db, "SELECT attachment_id, pipeline_version, array_agg(name ORDER BY name), "
|
||||
"array_agg(DISTINCT attempt_token) FROM attachment_variants GROUP BY 1, 2"):
|
||||
if sorted(names) != sorted(VARIANT_NAMES):
|
||||
bad("I2_variant_set", attachment_id=aid, version=version, names=names)
|
||||
if len(tokens) != 1:
|
||||
bad("I2_mixed_attempts", attachment_id=aid, version=version, attempts=len(tokens))
|
||||
for aid, version, outcome, jversion in await rows(
|
||||
db, "SELECT v.attachment_id, v.pipeline_version, ja.outcome, j.pipeline_version "
|
||||
"FROM attachment_variants v JOIN job_attempts ja ON ja.token = v.attempt_token "
|
||||
"JOIN jobs j ON j.id = ja.job_id WHERE ja.outcome <> 'succeeded' OR j.pipeline_version "
|
||||
"<> v.pipeline_version OR j.attachment_id <> v.attachment_id"):
|
||||
bad("I2_variant_from_unsuccessful_attempt", attachment_id=aid, version=version, outcome=outcome)
|
||||
|
||||
# I4: published_version = highest succeeded version; ready <=> published.
|
||||
for aid, status, published, best in await rows(
|
||||
db, "SELECT a.id, a.status, a.published_version, (SELECT max(j.pipeline_version) FROM jobs j "
|
||||
"WHERE j.attachment_id = a.id AND j.state = 'succeeded') FROM attachments a "
|
||||
"WHERE a.published_version IS DISTINCT FROM (SELECT max(j.pipeline_version) FROM jobs j "
|
||||
"WHERE j.attachment_id = a.id AND j.state = 'succeeded') "
|
||||
"OR (a.published_version IS NOT NULL AND a.status <> 'ready')"):
|
||||
bad("I4_published_version", attachment_id=aid, status=status, published=published, best_succeeded=best)
|
||||
|
||||
# I7: attempts.
|
||||
for (jid,) in await rows(db, "SELECT j.id FROM jobs j WHERE j.state = 'succeeded' AND (SELECT count(*) "
|
||||
"FROM job_attempts a WHERE a.job_id = j.id AND a.outcome = 'succeeded') <> 1"):
|
||||
bad("I7_succeeded_attempts", job_id=jid)
|
||||
for jid, token in await rows(db, "SELECT a.job_id, a.token FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
||||
"WHERE a.outcome = 'running' AND j.lease_token IS DISTINCT FROM a.token"):
|
||||
bad("I7_running_not_holder", job_id=jid, token=token)
|
||||
|
||||
# I3 + I5 + I6: storage.
|
||||
published = {k: (s, h) for k, s, h in await rows(
|
||||
db, "SELECT object_key, size_bytes, sha256 FROM attachment_variants UNION ALL "
|
||||
"SELECT object_key, size_bytes, sha256 FROM attachments WHERE object_key IS NOT NULL")}
|
||||
staging_live = {k: (s, st, status) for k, s, st, status in await rows(
|
||||
db, "SELECT staging_key, size_bytes, staging_state, status FROM attachments WHERE staging_key IS NOT NULL")}
|
||||
planned_open, planned_cleaned = set(), set()
|
||||
for keys, cleaned in await rows(db, "SELECT planned_keys, objects_cleaned_at IS NOT NULL FROM job_attempts"):
|
||||
(planned_cleaned if cleaned else planned_open).update(keys)
|
||||
final_objects = await list_all(final, "")
|
||||
staging_objects = await list_all(staging, "")
|
||||
for key, (size, sha) in published.items():
|
||||
if key not in final_objects:
|
||||
bad("I3_published_missing", key=key)
|
||||
elif final_objects[key] != size:
|
||||
bad("I3_published_size", key=key, stored=size, actual=final_objects[key])
|
||||
elif verify:
|
||||
data = await final.read_all(key, limit=settings.max_upload_bytes)
|
||||
if hashlib.sha256(data).digest() != bytes(sha):
|
||||
bad("I3_published_checksum", key=key)
|
||||
for key, (size, state, status) in staging_live.items():
|
||||
# 'pending' may lack its object (the PUT failed: the janitor removes the row); accepted
|
||||
# work (queued/processing) must have it.
|
||||
active = status in ("queued", "processing")
|
||||
if state == "present" and key not in staging_objects and active:
|
||||
bad("I5_staging_missing", key=key, status=status)
|
||||
for (aid,) in await rows(db, "SELECT a.id FROM attachments a WHERE a.status IN ('queued', 'processing') "
|
||||
"AND a.object_key IS NULL AND a.staging_state <> 'present'"):
|
||||
bad("I5_active_without_source", attachment_id=aid)
|
||||
for key in final_objects:
|
||||
if key == SENTINEL_KEY or key in published or key in planned_open:
|
||||
continue
|
||||
if key in planned_cleaned:
|
||||
notes.append({"kind": "leaked_object", "key": key})
|
||||
else:
|
||||
bad("I6_unknown_final_object", key=key)
|
||||
for key in staging_objects:
|
||||
if key == SENTINEL_KEY:
|
||||
continue
|
||||
row = staging_live.get(key)
|
||||
if row is None:
|
||||
bad("I6_unknown_staging_object", key=key)
|
||||
elif row[1] == "deleted":
|
||||
notes.append({"kind": "staging_marked_deleted_still_present", "key": key})
|
||||
counts = dict(await rows(db, "SELECT status, count(*) FROM attachments GROUP BY status"))
|
||||
jobs = dict(await rows(db, "SELECT state, count(*) FROM jobs GROUP BY state"))
|
||||
return {"ok": not violations, "violations": violations, "notes": notes,
|
||||
"attachments": counts, "jobs": jobs,
|
||||
"objects": {"final": len(final_objects), "staging": len(staging_objects),
|
||||
"published": len(published), "checksums_verified": verify}}
|
||||
finally:
|
||||
await staging.close()
|
||||
await final.close()
|
||||
await db.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
p = argparse.ArgumentParser(prog="audit", description="Read-only check of the pipeline invariants")
|
||||
p.add_argument("--verify-checksums", action="store_true", help="read every published object and compare SHA-256")
|
||||
args = p.parse_args(argv)
|
||||
try:
|
||||
settings = Settings.from_env()
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
report = asyncio.run(audit(settings, args.verify_checksums))
|
||||
except (Unavailable, ObjectMissing) as exc:
|
||||
print(json.dumps({"ok": False, "error": "store_unavailable", "detail": type(exc).__name__}))
|
||||
return 1
|
||||
print(json.dumps(report, ensure_ascii=False, indent=1, default=str))
|
||||
return 0 if report["ok"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Parsing of the Authorization header. Any malformed value is 401, never 500.
|
||||
|
||||
Basic (RFC 7617) is accepted only by POST /auth/sessions; everything else needs
|
||||
Bearer. Basic is an encoding, not encryption: plain HTTP is acceptable only for the
|
||||
local synthetic lab stand (lab 7 adds TLS).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import re
|
||||
|
||||
from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, Unauthorized
|
||||
from .validation import USERNAME_RE
|
||||
|
||||
B64_RE = re.compile(r"[A-Za-z0-9+/]+={0,2}")
|
||||
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") # token_urlsafe(32) gives 43 characters
|
||||
|
||||
|
||||
def _split(header: str | None) -> tuple[str, str]:
|
||||
if not header:
|
||||
return "", ""
|
||||
scheme, _, value = header.strip().partition(" ")
|
||||
return scheme.lower(), value.strip()
|
||||
|
||||
|
||||
def parse_basic(header: str | None) -> tuple[str, str]:
|
||||
"""(username, password) from `Basic base64(username:password)`; the password may contain ':'."""
|
||||
invalid = Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
|
||||
scheme, value = _split(header)
|
||||
if scheme != "basic" or B64_RE.fullmatch(value) is None or len(value) > 1024:
|
||||
raise invalid
|
||||
try:
|
||||
decoded = base64.b64decode(value, validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise invalid from None
|
||||
username, sep, password = decoded.partition(":") # split at the FIRST colon
|
||||
if not sep or USERNAME_RE.fullmatch(username) is None or not password:
|
||||
raise invalid
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(header: str | None) -> str:
|
||||
scheme, value = _split(header)
|
||||
if scheme != "bearer" or TOKEN_RE.fullmatch(value) is None:
|
||||
# Missing header, Basic on a business endpoint, garbage token: one answer.
|
||||
raise Unauthorized("Требуется действительная Bearer-сессия", challenge=BEARER_CHALLENGE)
|
||||
return value
|
||||
@@ -0,0 +1,86 @@
|
||||
"""RabbitMQ (AMQP 0-9-1) for the attachment jobs: topology names and the message format.
|
||||
|
||||
Topology (declared by the broker itself from its definitions file at boot, infra/rabbitmq and
|
||||
scripts/init_secrets.py; the relay and the worker have NO configure permission and declare nothing):
|
||||
|
||||
exchange messenger.jobs direct, durable
|
||||
queue messenger.attachment-jobs quorum (Raft log on disk), bound with routing key
|
||||
attachment.process.v1; x-delivery-limit and dead-lettering to
|
||||
exchange messenger.jobs.dlx fanout -> queue messenger.attachment-jobs.dlq (quorum)
|
||||
|
||||
Publisher side (relay): persistent message, `mandatory` (an unroutable message is returned, not
|
||||
silently dropped), channel in confirm mode - `publish()` returns only after the broker has
|
||||
written the message to the quorum log (publisher confirm). Consumer side (worker): manual ack
|
||||
after the durable effect, prefetch = WORKER_CONCURRENCY.
|
||||
|
||||
The message carries IDs and service fields only (contracts/jobs.schema.json): no bytes, no URL,
|
||||
no S3 key, no filename, no credentials. The worker takes every trusted detail from PostgreSQL.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
|
||||
EXCHANGE = "messenger.jobs"
|
||||
QUEUE = "messenger.attachment-jobs"
|
||||
DLX = "messenger.jobs.dlx"
|
||||
DLQ = "messenger.attachment-jobs.dlq"
|
||||
ROUTING_KEY = "attachment.process.v1"
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
MAX_MESSAGE_BYTES = 4096
|
||||
|
||||
REQUIRED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
||||
OPTIONAL = {"traceparent", "tracestate"}
|
||||
TRACEPARENT_RE = re.compile(r"00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}")
|
||||
DATETIME_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z")
|
||||
|
||||
|
||||
class BadMessage(ValueError):
|
||||
"""Not a valid job message (schema v1): dead-lettered, never processed."""
|
||||
|
||||
|
||||
def rfc3339(value: dt.datetime) -> str:
|
||||
return value.astimezone(dt.timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def encode(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: dt.datetime) -> bytes:
|
||||
body = {"schema_version": 1, "job_id": job_id, "type": JOB_TYPE, "attachment_id": attachment_id,
|
||||
"pipeline_version": pipeline_version, "attempt": attempt, "created_at": rfc3339(created_at)}
|
||||
return json.dumps(body, separators=(",", ":")).encode()
|
||||
|
||||
|
||||
def decode(raw: bytes) -> dict:
|
||||
"""Strict check against contracts/jobs.schema.json (additionalProperties: false)."""
|
||||
if len(raw) > MAX_MESSAGE_BYTES:
|
||||
raise BadMessage("too large")
|
||||
try:
|
||||
body = json.loads(raw)
|
||||
except (ValueError, UnicodeError):
|
||||
raise BadMessage("not JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise BadMessage("not an object")
|
||||
keys = set(body)
|
||||
if not REQUIRED <= keys or keys - REQUIRED - OPTIONAL:
|
||||
raise BadMessage("wrong fields")
|
||||
if body["schema_version"] != 1 or isinstance(body["schema_version"], bool) or body["type"] != JOB_TYPE:
|
||||
raise BadMessage("unsupported schema/type")
|
||||
for name in ("job_id", "attachment_id"):
|
||||
if not isinstance(body[name], str):
|
||||
raise BadMessage(name)
|
||||
try:
|
||||
uuid.UUID(body[name])
|
||||
except ValueError:
|
||||
raise BadMessage(name) from None
|
||||
for name in ("pipeline_version", "attempt"):
|
||||
value = body[name]
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||
raise BadMessage(name)
|
||||
if not isinstance(body["created_at"], str) or not DATETIME_RE.fullmatch(body["created_at"]):
|
||||
raise BadMessage("created_at")
|
||||
if "traceparent" in body and not (isinstance(body["traceparent"], str) and TRACEPARENT_RE.fullmatch(body["traceparent"])):
|
||||
raise BadMessage("traceparent")
|
||||
if "tracestate" in body and not (isinstance(body["tracestate"], str) and len(body["tracestate"]) <= 512):
|
||||
raise BadMessage("tracestate")
|
||||
return body
|
||||
@@ -0,0 +1,343 @@
|
||||
"""Runtime configuration read from environment variables (12-factor style).
|
||||
|
||||
Non-secret settings have safe defaults. Connection strings with passwords are
|
||||
read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in
|
||||
/run/secrets), so they never appear in `docker inspect`, compose files or logs.
|
||||
Invalid values stop the process at startup with a clear message.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ConfigError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
|
||||
|
||||
|
||||
def _instance_id() -> str:
|
||||
"""X-Instance-Id of this process: INSTANCE_ID or the container hostname.
|
||||
|
||||
Docker sets the hostname of a container to its short container ID, so the value
|
||||
is unique per replica, stable for the life of the process and maps 1:1 to a real
|
||||
container (`docker inspect`), which scripts/stop-one.sh relies on.
|
||||
"""
|
||||
value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname()
|
||||
if INSTANCE_ID_RE.fullmatch(value) is None:
|
||||
raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}")
|
||||
return value
|
||||
|
||||
|
||||
def _int(name: str, default: int, low: int, high: int) -> int:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
return default
|
||||
try:
|
||||
value = int(raw.strip())
|
||||
except ValueError:
|
||||
raise ConfigError(f"{name} must be an integer, got {raw!r}") from None
|
||||
if not low <= value <= high:
|
||||
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
|
||||
return value
|
||||
|
||||
|
||||
def _float(name: str, default: float, low: float, high: float) -> float:
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw.strip() == "":
|
||||
return default
|
||||
try:
|
||||
value = float(raw.strip())
|
||||
except ValueError:
|
||||
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
|
||||
if not low <= value <= high:
|
||||
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
|
||||
return value
|
||||
|
||||
|
||||
def _bool(name: str, default: bool) -> bool:
|
||||
raw = os.environ.get(name, "").strip().lower()
|
||||
if raw == "":
|
||||
return default
|
||||
if raw not in {"true", "false", "1", "0"}:
|
||||
raise ConfigError(f"{name} must be true/false, got {raw!r}")
|
||||
return raw in {"true", "1"}
|
||||
|
||||
|
||||
def _choice(name: str, default: str, allowed: set[str]) -> str:
|
||||
value = os.environ.get(name, "").strip().lower() or default
|
||||
if value not in allowed:
|
||||
raise ConfigError(f"{name} must be one of {sorted(allowed)}, got {value!r}")
|
||||
return value
|
||||
|
||||
|
||||
def _secret(name: str, required: bool) -> str:
|
||||
"""Value of <NAME>_FILE (preferred, Docker secret) or <NAME> (local runs/tests)."""
|
||||
path = os.environ.get(f"{name}_FILE", "").strip()
|
||||
if path:
|
||||
try:
|
||||
value = Path(path).read_text(encoding="utf-8").strip()
|
||||
except OSError as exc:
|
||||
# The message names the file, never its content.
|
||||
raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None
|
||||
else:
|
||||
value = os.environ.get(name, "").strip()
|
||||
if required and not value:
|
||||
raise ConfigError(f"{name}_FILE (or {name}) must be set")
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
host: str
|
||||
port: int
|
||||
env: str
|
||||
log_level: str
|
||||
shutdown_timeout: float
|
||||
max_body_bytes: int
|
||||
docs_enabled: bool = True
|
||||
database_url: str = ""
|
||||
session_store: str = "postgres"
|
||||
cache_url: str = ""
|
||||
session_ttl_seconds: int = 3600
|
||||
db_pool_min: int = 1
|
||||
db_pool_max: int = 10
|
||||
db_timeout_seconds: float = 3.0
|
||||
cache_timeout_seconds: float = 0.5
|
||||
hash_concurrency: int = 2
|
||||
instance_id: str = "local"
|
||||
drain_seconds: float = 0.0
|
||||
idempotency_ttl_seconds: int = 86400
|
||||
stats_log_seconds: float = 0.0
|
||||
message_write_path: str = "function"
|
||||
# Lab 4: private S3 bucket and attachment limits (see storage.py, attachments.py).
|
||||
s3_endpoint: str = "http://s3:8333"
|
||||
s3_region: str = "us-east-1"
|
||||
s3_bucket: str = "messenger-attachments"
|
||||
s3_access_key_id: str = ""
|
||||
s3_secret_access_key: str = ""
|
||||
s3_timeout_seconds: float = 3.0
|
||||
s3_connect_timeout_seconds: float = 1.0
|
||||
s3_max_connections: int = 20
|
||||
max_upload_bytes: int = 10 * 1024 * 1024
|
||||
max_image_pixels: int = 40_000_000
|
||||
upload_concurrency: int = 4
|
||||
upload_queue_timeout_seconds: float = 5.0
|
||||
upload_read_timeout_seconds: float = 30.0
|
||||
image_verify_timeout_seconds: float = 10.0
|
||||
image_verify_concurrency: int = 1
|
||||
jpeg_max_memory_mb: int = 256
|
||||
pending_grace_seconds: int = 900
|
||||
janitor_interval_seconds: float = 60.0
|
||||
# Lab 5: staging bucket (API writes, worker reads) and the asynchronous pipeline (jobs_settings()).
|
||||
s3_staging_bucket: str = "messenger-staging"
|
||||
broker_url: str = ""
|
||||
pipeline_version: int = 1
|
||||
job_max_attempts: int = 5
|
||||
job_timeout_seconds: float = 60.0
|
||||
job_lease_seconds: float = 30.0
|
||||
job_retry_base_seconds: float = 2.0
|
||||
job_retry_max_seconds: float = 60.0
|
||||
worker_concurrency: int = 2
|
||||
worker_shutdown_seconds: float = 20.0
|
||||
image_max_memory_mb: int = 768
|
||||
staging_retention_seconds: int = 86400
|
||||
orphan_grace_seconds: int = 600
|
||||
cleanup_interval_seconds: float = 30.0
|
||||
relay_poll_seconds: float = 0.5
|
||||
relay_batch: int = 50
|
||||
queued_republish_seconds: float = 300.0
|
||||
fault_points: frozenset[str] = frozenset()
|
||||
fault_values: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def fault_value(self, point: str, default: int = 0) -> int:
|
||||
return self.fault_values.get(point, default)
|
||||
|
||||
@property
|
||||
def db_statement_timeout_ms(self) -> int:
|
||||
# The server stops a statement a bit before the client-side deadline fires.
|
||||
return max(100, int(self.db_timeout_seconds * 1000) - 500)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, *, s3_credentials: bool = True) -> "Settings":
|
||||
"""s3_credentials=False: processes without S3 access (relay, jobctl)."""
|
||||
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
|
||||
if log_level not in {"debug", "info", "warning", "error"}:
|
||||
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
|
||||
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres"
|
||||
if session_store == "redis":
|
||||
session_store = "valkey" # same protocol; the course allows Redis OR Valkey
|
||||
if session_store not in {"postgres", "valkey"}:
|
||||
raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}")
|
||||
db_pool_min = _int("DB_POOL_MIN", 1, 0, 50)
|
||||
db_pool_max = _int("DB_POOL_MAX", 10, 1, 100)
|
||||
if db_pool_min > db_pool_max:
|
||||
raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX")
|
||||
return cls(
|
||||
host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0",
|
||||
port=_int("APP_PORT", 8080, 1, 65535),
|
||||
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
|
||||
log_level=log_level,
|
||||
# DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period.
|
||||
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0),
|
||||
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
|
||||
docs_enabled=_bool("DOCS_ENABLED", True),
|
||||
database_url=_secret("DATABASE_URL", required=True),
|
||||
session_store=session_store,
|
||||
cache_url=_secret("CACHE_URL", required=session_store == "valkey"),
|
||||
# Contract name. Minimum 1 s so the expiry test can use 2 s.
|
||||
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
|
||||
db_pool_min=db_pool_min,
|
||||
db_pool_max=db_pool_max,
|
||||
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
|
||||
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
|
||||
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
|
||||
instance_id=_instance_id(),
|
||||
# After SIGTERM: readiness is 503 but requests are still served for this long,
|
||||
# so the load balancer stops routing here before the listener closes.
|
||||
drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0),
|
||||
# Contract: a replay within 24 h returns the original result.
|
||||
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600),
|
||||
# Experiments (bench/track1): log DB pool counters and process CPU every N s; 0 = off.
|
||||
stats_log_seconds=_float("STATS_LOG_SECONDS", 0.0, 0.0, 60.0),
|
||||
# Track 1: `function` = one SQL call per send (migration 0003), `multi` = the previous
|
||||
# multi-statement transaction; both have identical semantics (A/B on one image).
|
||||
message_write_path=_choice("MESSAGE_WRITE_PATH", "function", {"function", "multi"}),
|
||||
**s3_settings(credentials_required=s3_credentials),
|
||||
**jobs_settings(broker_required=False),
|
||||
)
|
||||
|
||||
|
||||
# Contract limits (contracts/README.md): configurable DOWN for experiments, never above them.
|
||||
CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
CONTRACT_MAX_IMAGE_PIXELS = 40_000_000
|
||||
# Failure points for the experiments only (FAULT_POINTS of ONE service, compose variables
|
||||
# API_/RELAY_/WORKER_FAULT_POINTS); empty in normal runs. `name` or `name=<integer>`.
|
||||
FAULT_POINTS = {
|
||||
# API, upload: between the staging PUT and the job COMMIT / after the job COMMIT, before 202.
|
||||
"after_s3_put", "after_enqueue_commit",
|
||||
# Relay: crash after the broker confirmed a publish but before outbox.published_at is
|
||||
# committed; publish every message twice; publish a batch in reverse order.
|
||||
"crash_after_publish", "duplicate_publish", "reverse_order",
|
||||
# Worker: crash after the claim (before any side effect), after the S3 writes (before the
|
||||
# result COMMIT), after the result COMMIT (before the ack); transient failure of the first N
|
||||
# attempts; permanent failure; slow processing (N s, lease kept alive) of every job or only of
|
||||
# pipeline version `slow_pipeline`; hang: stop the heartbeat and sleep N s inside processing.
|
||||
"crash_after_claim", "crash_after_s3_write", "crash_after_commit", "transient_error", "permanent_error",
|
||||
"slow", "slow_pipeline", "hang",
|
||||
# Worker cleanup: pause N s after reading the candidates, before locking/deleting them.
|
||||
"pause_cleanup",
|
||||
}
|
||||
|
||||
|
||||
def _faults() -> tuple[frozenset[str], dict[str, int]]:
|
||||
names, values = set(), {}
|
||||
for item in os.environ.get("FAULT_POINTS", "").split(","):
|
||||
name, _, raw = item.strip().partition("=")
|
||||
if not name:
|
||||
continue
|
||||
if name not in FAULT_POINTS:
|
||||
raise ConfigError(f"FAULT_POINTS may contain only {sorted(FAULT_POINTS)}")
|
||||
if raw:
|
||||
if not raw.isdigit():
|
||||
raise ConfigError(f"FAULT_POINTS value of {name} must be an integer")
|
||||
values[name] = int(raw)
|
||||
names.add(name)
|
||||
return frozenset(names), values
|
||||
|
||||
|
||||
def _bucket(name: str, default: str) -> str:
|
||||
bucket = os.environ.get(name, default).strip() or default
|
||||
if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", bucket):
|
||||
raise ConfigError(f"{name} is not a valid bucket name: {bucket!r}")
|
||||
return bucket
|
||||
|
||||
|
||||
def s3_settings(*, credentials_required: bool = True) -> dict:
|
||||
"""S3 part of Settings; also used by the bootstrap CLI with its own keys."""
|
||||
faults, fault_values = _faults()
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").strip() or "http://s3:8333"
|
||||
if not re.fullmatch(r"https?://[A-Za-z0-9.\-]+(:[0-9]{1,5})?/?", endpoint):
|
||||
raise ConfigError(f"S3_ENDPOINT must be http(s)://host[:port], got {endpoint!r}")
|
||||
bucket = _bucket("S3_BUCKET", "messenger-attachments")
|
||||
staging = _bucket("S3_STAGING_BUCKET", "messenger-staging")
|
||||
if staging == bucket:
|
||||
raise ConfigError("S3_STAGING_BUCKET must differ from S3_BUCKET")
|
||||
return dict(
|
||||
s3_endpoint=endpoint,
|
||||
s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1",
|
||||
s3_bucket=bucket,
|
||||
s3_staging_bucket=staging,
|
||||
s3_access_key_id=_secret("S3_ACCESS_KEY_ID", required=credentials_required),
|
||||
s3_secret_access_key=_secret("S3_SECRET_ACCESS_KEY", required=credentials_required),
|
||||
s3_timeout_seconds=_float("S3_TIMEOUT_SECONDS", 3.0, 0.2, 30.0),
|
||||
s3_connect_timeout_seconds=_float("S3_CONNECT_TIMEOUT_SECONDS", 1.0, 0.1, 10.0),
|
||||
s3_max_connections=_int("S3_MAX_CONNECTIONS", 20, 1, 200),
|
||||
max_upload_bytes=_int("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES, 1024, CONTRACT_MAX_UPLOAD_BYTES),
|
||||
max_image_pixels=_int("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS, 1, CONTRACT_MAX_IMAGE_PIXELS),
|
||||
# At most this many request bodies of one replica are buffered/checked at once (RAM bound:
|
||||
# UPLOAD_CONCURRENCY x MAX_UPLOAD_BYTES); further uploads wait, then get 503.
|
||||
upload_concurrency=_int("UPLOAD_CONCURRENCY", 4, 1, 64),
|
||||
upload_queue_timeout_seconds=_float("UPLOAD_QUEUE_TIMEOUT_SECONDS", 5.0, 0.0, 60.0),
|
||||
upload_read_timeout_seconds=_float("UPLOAD_READ_TIMEOUT_SECONDS", 30.0, 1.0, 300.0),
|
||||
image_verify_timeout_seconds=_float("IMAGE_VERIFY_TIMEOUT_SECONDS", 10.0, 1.0, 120.0),
|
||||
# Full checks running at once per replica: a strict JPEG decode of a 40 Mpx progressive
|
||||
# file needs up to ~120-240 MB of coefficient buffers (capped by JPEG_MAX_MEMORY_MB).
|
||||
image_verify_concurrency=_int("IMAGE_VERIFY_CONCURRENCY", 1, 1, 16),
|
||||
jpeg_max_memory_mb=_int("JPEG_MAX_MEMORY_MB", 256, 16, 2048),
|
||||
# An intent row still `pending` after this long belongs to a failed upload: the janitor
|
||||
# deletes its object and the row. Far above the longest possible upload (~45 s).
|
||||
pending_grace_seconds=_int("PENDING_GRACE_SECONDS", 900, 1, 7 * 24 * 3600),
|
||||
janitor_interval_seconds=_float("JANITOR_INTERVAL_SECONDS", 60.0, 0.0, 3600.0),
|
||||
fault_points=faults,
|
||||
fault_values=fault_values,
|
||||
)
|
||||
|
||||
|
||||
def jobs_settings(*, broker_required: bool) -> dict:
|
||||
"""Lab 5 pipeline: retries, lease, worker limits, cleanup, relay (docs/PIPELINE.md)."""
|
||||
from .pipeline import PIPELINES # local import: pipeline.py imports nothing from here
|
||||
timeout = _float("JOB_TIMEOUT_SECONDS", 60.0, 1.0, 600.0)
|
||||
lease = _float("JOB_LEASE_SECONDS", 30.0, 2.0, 600.0)
|
||||
base = _float("JOB_RETRY_BASE_SECONDS", 2.0, 0.0, 600.0)
|
||||
cap = _float("JOB_RETRY_MAX_SECONDS", 60.0, 0.0, 3600.0)
|
||||
if base > cap:
|
||||
raise ConfigError("JOB_RETRY_BASE_SECONDS must not exceed JOB_RETRY_MAX_SECONDS")
|
||||
orphan_grace = _int("ORPHAN_GRACE_SECONDS", 600, 1, 7 * 24 * 3600)
|
||||
version = _int("PIPELINE_VERSION", 1, 1, 1000)
|
||||
if version not in PIPELINES:
|
||||
raise ConfigError(f"PIPELINE_VERSION must be one of {sorted(PIPELINES)}")
|
||||
return dict(
|
||||
broker_url=_secret("BROKER_URL", required=broker_required),
|
||||
# Pipeline version of the jobs created by new uploads (track 2: versioned pipeline).
|
||||
pipeline_version=version,
|
||||
# Managed attempts (claims) per job before failed/retry_exhausted.
|
||||
job_max_attempts=_int("JOB_MAX_ATTEMPTS", 5, 1, 100),
|
||||
# Hard limit of ONE attempt: the image child process is killed, the lease is no longer renewed.
|
||||
job_timeout_seconds=timeout,
|
||||
# Lease of a claimed job, renewed by the heartbeat every lease/3 while the attempt runs.
|
||||
job_lease_seconds=lease,
|
||||
# Backoff of transient failures: full jitter, uniform(0, min(max, base * 2^(attempt-1))).
|
||||
job_retry_base_seconds=base,
|
||||
job_retry_max_seconds=cap,
|
||||
# Jobs processed at once by ONE worker process = RabbitMQ prefetch (basic.qos).
|
||||
worker_concurrency=_int("WORKER_CONCURRENCY", 2, 1, 32),
|
||||
worker_shutdown_seconds=_float("WORKER_SHUTDOWN_SECONDS", 20.0, 0.0, 120.0),
|
||||
# Address-space limit (RLIMIT_AS) of the child process that decodes/encodes ONE image.
|
||||
image_max_memory_mb=_int("IMAGE_MAX_MEMORY_MB", 768, 128, 8192),
|
||||
# Staging copy of a FAILED attachment is kept this long for a replay, then deleted.
|
||||
staging_retention_seconds=_int("STAGING_RETENTION_SECONDS", 86400, 0, 30 * 24 * 3600),
|
||||
# Objects of attempts that did not publish are deleted once the attempt ended this long ago.
|
||||
orphan_grace_seconds=orphan_grace,
|
||||
cleanup_interval_seconds=_float("CLEANUP_INTERVAL_SECONDS", 30.0, 0.0, 3600.0),
|
||||
relay_poll_seconds=_float("RELAY_POLL_SECONDS", 0.5, 0.05, 60.0),
|
||||
relay_batch=_int("RELAY_BATCH", 50, 1, 1000),
|
||||
# A queued job whose message has not led to a claim for this long is published again
|
||||
# (message lost in the broker, dead-lettered, purged): no accepted job waits forever.
|
||||
queued_republish_seconds=_float("QUEUED_REPUBLISH_SECONDS", 300.0, 5.0, 86400.0),
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Opaque keyset cursors.
|
||||
|
||||
A cursor encodes the sort key `(created_at, id)` of the last item on the page
|
||||
plus the resource it belongs to, e.g. `v1|m|<chat_id>|<ts_us>|<message_id>`,
|
||||
packed in URL-safe base64. The client treats it as an opaque string; the server
|
||||
rejects cursors of another chat/user (400) and any non-canonical encoding.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import re
|
||||
|
||||
from .errors import InvalidRequest
|
||||
from .validation import UUID_RE
|
||||
|
||||
B64_RE = re.compile(r"[A-Za-z0-9_-]{1,512}")
|
||||
TS_RE = re.compile(r"[0-9]{1,19}")
|
||||
|
||||
# Latest instant both Python datetime and PostgreSQL accept: 9999-12-31T23:59:59.999999Z.
|
||||
MAX_TS_US = 253402300799999999
|
||||
|
||||
MESSAGES = "m"
|
||||
CHATS = "c"
|
||||
|
||||
SortKey = tuple[int, str]
|
||||
|
||||
|
||||
def encode(scope: str, owner_id: str, key: SortKey) -> str:
|
||||
ts, item_id = key
|
||||
raw = f"v1|{scope}|{owner_id}|{ts}|{item_id}".encode("ascii")
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
|
||||
|
||||
|
||||
def decode(cursor: str, scope: str, owner_id: str) -> SortKey:
|
||||
"""Return the sort key stored in the cursor or raise InvalidRequest."""
|
||||
bad = InvalidRequest("Неверный cursor")
|
||||
if B64_RE.fullmatch(cursor) is None:
|
||||
raise bad
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("ascii")
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
raise bad from None
|
||||
parts = raw.split("|")
|
||||
if len(parts) != 5 or parts[0] != "v1":
|
||||
raise bad
|
||||
_, got_scope, got_owner, ts, item_id = parts
|
||||
if TS_RE.fullmatch(ts) is None or UUID_RE.fullmatch(item_id) is None or item_id != item_id.lower():
|
||||
raise bad
|
||||
key = (int(ts), item_id)
|
||||
if key[0] > MAX_TS_US:
|
||||
raise bad # out-of-range time must be 400, not an overflow (500) in datetime/SQL
|
||||
if encode(got_scope, got_owner, key) != cursor:
|
||||
raise bad # only the canonical form produced by the server is accepted
|
||||
if got_scope != scope or got_owner != owner_id:
|
||||
raise InvalidRequest("cursor относится к другому ресурсу")
|
||||
return key
|
||||
@@ -0,0 +1,131 @@
|
||||
"""PostgreSQL access: a bounded connection pool and a hard deadline for every unit of work.
|
||||
|
||||
Three independent limits keep a request from hanging when the database misbehaves:
|
||||
|
||||
* pool `timeout` — waiting for a free connection (pool exhausted);
|
||||
* server `statement_timeout` / `lock_timeout` — PostgreSQL itself cancels a slow
|
||||
statement or a statement stuck behind a lock;
|
||||
* client deadline — covers what the server cannot report, e.g. a
|
||||
paused/frozen PostgreSQL container or a dead network.
|
||||
|
||||
The client deadline needs care. Every unit of work runs as its own task; at the deadline
|
||||
the REQUEST stops waiting and gets 503 at once, but the task is NOT cancelled: cancelling a
|
||||
psycopg query makes psycopg send a cancel request, and libpq starts that connection
|
||||
(PQcancelStart) synchronously in the event-loop thread, resolving the host name first.
|
||||
When the API has lost its network to PostgreSQL (lab 3 review scenario: `docker network
|
||||
disconnect`), that blocked the WHOLE replica, /health/live included, for ~1 minute.
|
||||
The abandoned task instead ends by itself, bounded by:
|
||||
|
||||
* TCP keepalives + `tcp_user_timeout` (= the deadline): a dead peer or a vanished network
|
||||
breaks the socket within seconds, the connection is discarded by the pool;
|
||||
* `statement_timeout` / `lock_timeout` on a live but slow server;
|
||||
* a frozen (paused) server keeps the connection until it resumes; meanwhile new work waits
|
||||
at most the pool timeout and gets 503.
|
||||
|
||||
Any failure becomes 503 `unavailable` — never 401, never a hang.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TypeVar
|
||||
|
||||
import psycopg
|
||||
from psycopg_pool import AsyncConnectionPool, PoolTimeout
|
||||
|
||||
from .config import Settings
|
||||
from .errors import Unavailable
|
||||
|
||||
log = logging.getLogger("messenger.db")
|
||||
|
||||
T = TypeVar("T")
|
||||
Work = Callable[[psycopg.AsyncConnection], Awaitable[T]]
|
||||
|
||||
# Errors that mean "the database is unavailable right now", not "the request is wrong".
|
||||
# QueryCanceled (statement_timeout), LockNotAvailable (lock_timeout) and AdminShutdown
|
||||
# (server restart) are subclasses of OperationalError.
|
||||
UNAVAILABLE_ERRORS = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TimeoutError)
|
||||
|
||||
|
||||
def _forget(task: asyncio.Task) -> None:
|
||||
"""Done-callback for abandoned tasks: retrieve the result so nothing is logged as 'never retrieved'."""
|
||||
if not task.cancelled():
|
||||
task.exception()
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, settings: Settings, name: str = "messenger-api"):
|
||||
self.deadline = settings.db_timeout_seconds
|
||||
options = (
|
||||
f"-c statement_timeout={settings.db_statement_timeout_ms} "
|
||||
f"-c lock_timeout={max(100, settings.db_statement_timeout_ms // 2)} "
|
||||
"-c idle_in_transaction_session_timeout=10000"
|
||||
)
|
||||
self.pool = AsyncConnectionPool(
|
||||
conninfo=settings.database_url,
|
||||
min_size=settings.db_pool_min,
|
||||
max_size=settings.db_pool_max,
|
||||
timeout=settings.db_timeout_seconds, # max wait for a free connection
|
||||
open=False,
|
||||
name=name,
|
||||
check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart
|
||||
kwargs={
|
||||
"options": options,
|
||||
"connect_timeout": max(1, int(settings.db_timeout_seconds)),
|
||||
"application_name": name, # visible in pg_stat_activity: api / relay / worker
|
||||
# Autocommit connections: a single statement (post_message function, ping) is its
|
||||
# own transaction with ONE round trip; multi-statement work uses an explicit
|
||||
# conn.transaction() (BEGIN ... COMMIT), see run().
|
||||
"autocommit": True,
|
||||
# Detect a dead peer / lost network without any cancel request (see module doc).
|
||||
"keepalives": 1,
|
||||
"keepalives_idle": 5,
|
||||
"keepalives_interval": 1,
|
||||
"keepalives_count": 3,
|
||||
"tcp_user_timeout": int(settings.db_timeout_seconds * 1000),
|
||||
},
|
||||
)
|
||||
|
||||
async def open(self) -> None:
|
||||
# wait=False: the API starts (liveness 200) even if PostgreSQL is not up yet;
|
||||
# readiness stays 503 until the database answers.
|
||||
await self.pool.open(wait=False)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.pool.close(timeout=2)
|
||||
|
||||
async def run(self, work: Work[T], deadline: float | None = None, *, transaction: bool = True) -> T:
|
||||
"""Run `work(conn)` in one transaction (COMMIT on success, ROLLBACK on any exception).
|
||||
transaction=False: `work` issues exactly ONE statement, which autocommit makes atomic.
|
||||
|
||||
Domain errors raised by `work` (NotFound, Conflict, ...) propagate unchanged;
|
||||
database unavailability and the deadline become Unavailable (503).
|
||||
"""
|
||||
async def unit() -> T:
|
||||
async with self.pool.connection() as conn:
|
||||
if not transaction:
|
||||
return await work(conn)
|
||||
async with conn.transaction():
|
||||
return await work(conn)
|
||||
|
||||
task = asyncio.ensure_future(unit())
|
||||
try:
|
||||
return await asyncio.wait_for(asyncio.shield(task), deadline or self.deadline)
|
||||
except UNAVAILABLE_ERRORS as exc:
|
||||
log.warning("database_unavailable", extra={"fields": {"error": type(exc).__name__}})
|
||||
raise Unavailable("База данных временно недоступна") from None
|
||||
finally:
|
||||
if not task.done():
|
||||
# Deadline or cancelled request: stop waiting now. The task is left to finish
|
||||
# on its own (no task.cancel(): that would trigger psycopg's blocking cancel).
|
||||
task.add_done_callback(_forget)
|
||||
|
||||
async def ping(self, timeout: float = 1.0) -> bool:
|
||||
async def select_one(conn: psycopg.AsyncConnection) -> None:
|
||||
await conn.execute("SELECT 1")
|
||||
try:
|
||||
await self.run(select_one, deadline=timeout, transaction=False)
|
||||
return True
|
||||
except Unavailable:
|
||||
return False
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Swagger UI for manual experiments: http://127.0.0.1:8080/docs
|
||||
|
||||
The page shows the course contract contracts/openapi.json as is (it is not
|
||||
generated from our code). Only `servers` is replaced with "/" so that the
|
||||
"Try it out" requests go to the same host the page was opened from.
|
||||
Swagger UI itself (JS/CSS) is loaded by the browser from the jsDelivr CDN.
|
||||
Disable with DOCS_ENABLED=false.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from starlette.requests import Request
|
||||
from starlette.responses import HTMLResponse, JSONResponse
|
||||
|
||||
SPEC_PATH = Path(__file__).resolve().parents[2] / "contracts" / "openapi.json"
|
||||
SWAGGER_UI = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.33.0"
|
||||
|
||||
PAGE = f"""<!doctype html>
|
||||
<html lang="ru">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>Messenger lab1 — Swagger UI</title>
|
||||
<link rel="stylesheet" href="{SWAGGER_UI}/swagger-ui.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="swagger-ui"></div>
|
||||
<script src="{SWAGGER_UI}/swagger-ui-bundle.js"></script>
|
||||
<script>
|
||||
window.ui = SwaggerUIBundle({{
|
||||
url: "/openapi.json",
|
||||
dom_id: "#swagger-ui",
|
||||
tryItOutEnabled: true, // fields are editable right away
|
||||
persistAuthorization: true, // X-User-Id from "Authorize" survives page reload
|
||||
displayRequestDuration: true,
|
||||
}});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _spec() -> dict:
|
||||
spec = json.loads(SPEC_PATH.read_text(encoding="utf-8"))
|
||||
spec["servers"] = [{"url": "/", "description": "этот сервер"}]
|
||||
return spec
|
||||
|
||||
|
||||
async def swagger_page(request: Request) -> HTMLResponse:
|
||||
return HTMLResponse(PAGE)
|
||||
|
||||
|
||||
async def openapi_json(request: Request) -> JSONResponse:
|
||||
return JSONResponse(_spec())
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Domain errors mapped 1:1 to the contract error codes (contracts/README.md)."""
|
||||
from __future__ import annotations
|
||||
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
status = 500
|
||||
code = "internal_error"
|
||||
default_message = "Внутренняя ошибка сервера"
|
||||
|
||||
def __init__(self, message: str | None = None):
|
||||
self.message = message or self.default_message
|
||||
super().__init__(self.message)
|
||||
|
||||
@property
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {}
|
||||
|
||||
|
||||
class InvalidRequest(ApiError):
|
||||
status = 400
|
||||
code = "invalid_request"
|
||||
default_message = "Неверный запрос"
|
||||
|
||||
|
||||
class Unauthorized(ApiError):
|
||||
"""401 with the challenge of the scheme the endpoint expects (RFC 7235)."""
|
||||
|
||||
status = 401
|
||||
code = "unauthorized"
|
||||
default_message = "Требуется действительная сессия"
|
||||
|
||||
def __init__(self, message: str | None = None, challenge: str = BEARER_CHALLENGE):
|
||||
super().__init__(message)
|
||||
self.challenge = challenge
|
||||
|
||||
@property
|
||||
def headers(self) -> dict[str, str]:
|
||||
return {"WWW-Authenticate": self.challenge}
|
||||
|
||||
|
||||
class NotFound(ApiError):
|
||||
status = 404
|
||||
code = "not_found"
|
||||
default_message = "Объект не найден"
|
||||
|
||||
|
||||
class Conflict(ApiError):
|
||||
status = 409
|
||||
code = "conflict"
|
||||
default_message = "Конфликт"
|
||||
|
||||
|
||||
class PayloadTooLarge(ApiError):
|
||||
status = 413
|
||||
code = "payload_too_large"
|
||||
default_message = "Тело запроса слишком большое"
|
||||
|
||||
|
||||
class UnsupportedMediaType(ApiError):
|
||||
status = 415
|
||||
code = "unsupported_media_type"
|
||||
default_message = "Неподдерживаемый тип содержимого"
|
||||
|
||||
|
||||
class Unavailable(ApiError):
|
||||
status = 503
|
||||
code = "unavailable"
|
||||
default_message = "Сервис временно недоступен"
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`.
|
||||
|
||||
(no arguments) API: GET /health/ready on the local port
|
||||
--file /tmp/heartbeat --max-age 30 relay/worker: the main loop wrote its heartbeat recently
|
||||
(it writes it only while its database/broker work)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
if len(argv) == 4 and argv[0] == "--file" and argv[2] == "--max-age":
|
||||
try:
|
||||
with open(argv[1], encoding="ascii") as f:
|
||||
return 0 if time.time() - float(f.read().strip()) <= float(argv[3]) else 1
|
||||
except (OSError, ValueError):
|
||||
return 1
|
||||
port = os.environ.get("APP_PORT", "8080").strip() or "8080"
|
||||
try:
|
||||
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
|
||||
return 0 if response.status == 200 else 1
|
||||
except Exception:
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
@@ -0,0 +1,328 @@
|
||||
"""Content checks of uploaded images (lab 4): what the bytes ARE, not what the client says.
|
||||
|
||||
Two stages, both with bounded memory and both BEFORE anything is stored:
|
||||
|
||||
1. `sniff(head)` — runs on the first bytes while the body is still arriving. Recognises
|
||||
PNG/JPEG by the signature and reads the dimensions from the header (PNG IHDR, JPEG SOFn).
|
||||
A foreign format is rejected (415) and a picture with more than MAX_IMAGE_PIXELS pixels
|
||||
(413) before the rest of the body is read and long before any pixel is decoded.
|
||||
2. `verify(data, info)` — full integrity check of the complete file, run in a worker thread:
|
||||
* PNG: our own streaming validator. Every chunk CRC is checked, the IDAT zlib stream is
|
||||
inflated in 1 MiB steps and DISCARDED (no raster in memory), its size must be exactly
|
||||
height x (1 + row bytes) (Adam7 per pass), every row filter byte must be 0..4, IEND
|
||||
must close the file. A truncated file, a broken IDAT or extra/missing pixel data fail.
|
||||
Pillow is not used for PNG: it would allocate the whole raster (40 Mpx RGBA = 160 MB).
|
||||
* JPEG: libjpeg-turbo `djpeg -strict` decodes the whole entropy-coded data at 1/8 scale
|
||||
(DCT scaling: the output raster is 64 times smaller) in a child process; any decoder
|
||||
warning is fatal, so a truncated file or a file without scan data after SOS fails.
|
||||
|
||||
Every problem of the CONTENT is an ImageError carrying the HTTP status (415 or 413);
|
||||
nothing here may turn user input into a 500.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import struct
|
||||
import zlib
|
||||
from dataclasses import dataclass
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SOI = b"\xff\xd8\xff"
|
||||
|
||||
PNG = "image/png"
|
||||
JPEG = "image/jpeg"
|
||||
|
||||
# Bits per pixel channel count by PNG colour type, and the bit depths allowed for it.
|
||||
PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
|
||||
# Adam7 passes: (x start, y start, x step, y step).
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
# JPEG start-of-frame markers that carry the frame size (not DHT C4, JPG C8, DAC CC).
|
||||
JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
|
||||
# Standalone markers without a length field.
|
||||
JPEG_STANDALONE = {0x01, *range(0xD0, 0xD8)}
|
||||
|
||||
DJPEG = shutil.which("djpeg") or "djpeg" # libjpeg-turbo-progs, installed in the Dockerfile
|
||||
INPUT_STEP = 1 << 16 # compressed bytes handed to zlib at once
|
||||
INFLATE_STEP = 1 << 20 # bytes of raw PNG data produced per zlib call
|
||||
MAX_DIMENSION = 2**31 - 1
|
||||
|
||||
|
||||
class ImageError(Exception):
|
||||
"""The uploaded bytes are not an acceptable image. `status`: 415 (format/corrupt) or 413."""
|
||||
|
||||
def __init__(self, status: int, message: str):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.message = message
|
||||
|
||||
|
||||
class VerifierUnavailable(Exception):
|
||||
"""The checker itself could not run to the end (resources): the API answers 503, retryable."""
|
||||
|
||||
|
||||
def unsupported(message: str ="Файл не является корректным изображением PNG или JPEG") -> ImageError:
|
||||
return ImageError(415, message)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ImageInfo:
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
|
||||
|
||||
def detect(head: bytes) -> str | None:
|
||||
"""Format by signature; None while fewer bytes than a signature have arrived."""
|
||||
if head[:8] == PNG_SIGNATURE:
|
||||
return PNG
|
||||
if head[:3] == JPEG_SOI:
|
||||
return JPEG
|
||||
probe = min(len(head), 8)
|
||||
if PNG_SIGNATURE.startswith(head[:probe]) or JPEG_SOI.startswith(head[:min(probe, 3)]):
|
||||
return None
|
||||
raise unsupported()
|
||||
|
||||
|
||||
def sniff(head: bytes, max_pixels: int) -> ImageInfo | None:
|
||||
"""Format and size from the beginning of the file, or None if more bytes are needed.
|
||||
|
||||
Raises ImageError: 415 for a foreign/broken header, 413 for too many pixels."""
|
||||
kind = detect(head)
|
||||
if kind is None:
|
||||
return None
|
||||
info = _png_header(head) if kind == PNG else _jpeg_header(head)
|
||||
if info is None:
|
||||
return None
|
||||
if info.width * info.height > max_pixels:
|
||||
raise ImageError(413, f"Изображение больше {max_pixels} пикселей")
|
||||
return info
|
||||
|
||||
|
||||
def _png_header(head: bytes) -> ImageInfo | None:
|
||||
if len(head) < 33:
|
||||
return None
|
||||
length, ctype = struct.unpack(">I4s", head[8:16])
|
||||
if ctype != b"IHDR" or length != 13:
|
||||
raise unsupported()
|
||||
if zlib.crc32(head[12:29]) != struct.unpack(">I", head[29:33])[0]:
|
||||
raise unsupported()
|
||||
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", head[16:29])
|
||||
if not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION):
|
||||
raise unsupported()
|
||||
if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1:
|
||||
raise unsupported()
|
||||
return ImageInfo(PNG, width, height)
|
||||
|
||||
|
||||
def _jpeg_header(head: bytes) -> ImageInfo | None:
|
||||
"""Walk the marker segments up to the first SOFn. None = the SOF has not arrived yet."""
|
||||
i = 2
|
||||
n = len(head)
|
||||
while True:
|
||||
if i >= n:
|
||||
return None
|
||||
if head[i] != 0xFF:
|
||||
raise unsupported()
|
||||
while i < n and head[i] == 0xFF: # fill bytes
|
||||
i += 1
|
||||
if i >= n:
|
||||
return None
|
||||
marker = head[i]
|
||||
i += 1
|
||||
if marker in JPEG_STANDALONE:
|
||||
continue
|
||||
if marker in (0xD8, 0xD9, 0xDA, 0x00):
|
||||
# Second SOI, EOI or scan data before any frame header: not a usable JPEG.
|
||||
raise unsupported()
|
||||
if i + 2 > n:
|
||||
return None
|
||||
length = struct.unpack(">H", head[i:i + 2])[0]
|
||||
if length < 2:
|
||||
raise unsupported()
|
||||
if marker in JPEG_SOF:
|
||||
if i + 7 > n:
|
||||
return None
|
||||
if length < 8:
|
||||
raise unsupported()
|
||||
height, width = struct.unpack(">HH", head[i + 3:i + 7])
|
||||
if width == 0 or height == 0: # height 0 = DNL-defined height: not supported
|
||||
raise unsupported()
|
||||
return ImageInfo(JPEG, width, height)
|
||||
i += length
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ full verification
|
||||
async def verify(data: bytes | bytearray | memoryview, info: ImageInfo, *, timeout: float = 10.0,
|
||||
jpeg_max_memory_mb: int = 256) -> None:
|
||||
"""Complete integrity check of the whole file. PNG: our validator in a worker thread (zlib
|
||||
releases the GIL); JPEG: a strict libjpeg decoder in a child process."""
|
||||
if info.content_type == PNG:
|
||||
await asyncio.wait_for(asyncio.to_thread(verify_png, data), timeout)
|
||||
else:
|
||||
await verify_jpeg(data, timeout, jpeg_max_memory_mb)
|
||||
|
||||
|
||||
def _png_segments(width: int, height: int, depth: int, color: int, interlace: int) -> list[tuple[int, int]]:
|
||||
"""(byte count, row stride) of every pass of the raw (filtered) image data."""
|
||||
bpp = PNG_CHANNELS[color] * depth
|
||||
passes = ADAM7 if interlace else ((0, 0, 1, 1),)
|
||||
segments = []
|
||||
for xs, ys, dx, dy in passes:
|
||||
pw = (width - xs + dx - 1) // dx if width > xs else 0
|
||||
ph = (height - ys + dy - 1) // dy if height > ys else 0
|
||||
if pw and ph:
|
||||
stride = 1 + (pw * bpp + 7) // 8
|
||||
segments.append((ph * stride, stride))
|
||||
return segments
|
||||
|
||||
|
||||
class _RawChecker:
|
||||
"""Counts inflated bytes against the expected size and checks the filter byte of every row
|
||||
using C-speed slicing (`piece[first::stride]`), so even 40 000 000 one-pixel rows are fast."""
|
||||
|
||||
VALID_FILTERS = bytes(range(5))
|
||||
|
||||
def __init__(self, segments: list[tuple[int, int]]):
|
||||
self.segments = segments
|
||||
self.index = 0 # current pass
|
||||
self.offset = 0 # bytes already seen inside the current pass
|
||||
self.expected = sum(size for size, _ in segments)
|
||||
self.total = 0
|
||||
|
||||
def feed(self, piece: bytes) -> None:
|
||||
self.total += len(piece)
|
||||
if self.total > self.expected:
|
||||
raise unsupported()
|
||||
pos = 0
|
||||
while pos < len(piece):
|
||||
size, stride = self.segments[self.index]
|
||||
take = min(size - self.offset, len(piece) - pos)
|
||||
part = piece[pos:pos + take]
|
||||
first = (-self.offset) % stride
|
||||
if part[first::stride].translate(None, self.VALID_FILTERS):
|
||||
raise unsupported()
|
||||
self.offset += take
|
||||
pos += take
|
||||
if self.offset == size:
|
||||
self.index += 1
|
||||
self.offset = 0
|
||||
|
||||
|
||||
def verify_png(data: bytes | bytearray | memoryview) -> None:
|
||||
view = memoryview(data)
|
||||
if bytes(view[:8]) != PNG_SIGNATURE:
|
||||
raise unsupported()
|
||||
pos = 8
|
||||
n = len(view)
|
||||
checker: _RawChecker | None = None
|
||||
inflater = zlib.decompressobj()
|
||||
color = None
|
||||
seen_plte = seen_idat = idat_done = False
|
||||
first = True
|
||||
while True:
|
||||
if pos + 12 > n:
|
||||
raise unsupported() # truncated: no room for another chunk header + CRC
|
||||
length, ctype = struct.unpack(">I4s", view[pos:pos + 8])
|
||||
if length > n - pos - 12:
|
||||
raise unsupported()
|
||||
body = view[pos + 8:pos + 8 + length]
|
||||
crc = struct.unpack(">I", view[pos + 8 + length:pos + 12 + length])[0]
|
||||
if zlib.crc32(body, zlib.crc32(ctype)) != crc:
|
||||
raise unsupported()
|
||||
pos += 12 + length
|
||||
if not all(65 <= c <= 90 or 97 <= c <= 122 for c in ctype):
|
||||
raise unsupported()
|
||||
if first:
|
||||
if ctype != b"IHDR" or length != 13:
|
||||
raise unsupported()
|
||||
width, height, depth, color, compression, filtering, interlace = struct.unpack(">IIBBBBB", body)
|
||||
if (not (1 <= width <= MAX_DIMENSION and 1 <= height <= MAX_DIMENSION) or color not in PNG_DEPTHS
|
||||
or depth not in PNG_DEPTHS[color] or compression or filtering or interlace > 1):
|
||||
raise unsupported()
|
||||
checker = _RawChecker(_png_segments(width, height, depth, color, interlace))
|
||||
first = False
|
||||
continue
|
||||
assert checker is not None
|
||||
if ctype == b"IHDR":
|
||||
raise unsupported()
|
||||
if ctype == b"PLTE":
|
||||
if seen_plte or seen_idat or color in (0, 4) or length % 3 or not 3 <= length <= 768:
|
||||
raise unsupported()
|
||||
seen_plte = True
|
||||
elif ctype == b"IDAT":
|
||||
if idat_done:
|
||||
raise unsupported() # IDAT chunks must be consecutive
|
||||
if color == 3 and not seen_plte:
|
||||
raise unsupported()
|
||||
seen_idat = True
|
||||
_inflate(inflater, body, checker)
|
||||
elif ctype == b"IEND":
|
||||
if length != 0 or not seen_idat:
|
||||
raise unsupported()
|
||||
if pos != n:
|
||||
raise unsupported() # bytes after IEND (polyglot files) are not accepted
|
||||
break
|
||||
elif not 97 <= ctype[0] <= 122:
|
||||
raise unsupported() # unknown CRITICAL chunk: a decoder could not show the image
|
||||
if ctype != b"IDAT" and seen_idat:
|
||||
idat_done = True
|
||||
# The zlib stream must end exactly with the last pixel row, without trailing data.
|
||||
if not inflater.eof or inflater.unused_data or checker.total != checker.expected:
|
||||
raise unsupported()
|
||||
|
||||
|
||||
def _inflate(inflater, body: memoryview, checker: _RawChecker) -> None:
|
||||
if inflater.eof:
|
||||
if len(body):
|
||||
raise unsupported() # data after the end of the zlib stream
|
||||
return
|
||||
# Input is fed in small slices: zlib's `unconsumed_tail` is a COPY of the unread input, so
|
||||
# feeding the whole 10 MiB IDAT at once would copy the rest of it after every output step.
|
||||
try:
|
||||
for start in range(0, len(body), INPUT_STEP):
|
||||
pending: bytes | memoryview = body[start:start + INPUT_STEP]
|
||||
while pending:
|
||||
if inflater.eof:
|
||||
raise unsupported() # data after the end of the zlib stream
|
||||
checker.feed(inflater.decompress(pending, INFLATE_STEP))
|
||||
pending = inflater.unconsumed_tail
|
||||
if inflater.eof and inflater.unused_data:
|
||||
raise unsupported()
|
||||
except zlib.error:
|
||||
raise unsupported() from None
|
||||
|
||||
|
||||
async def verify_jpeg(data: bytes | bytearray | memoryview, timeout: float, max_memory_mb: int) -> None:
|
||||
"""Decode the WHOLE file with the reference decoder (libjpeg-turbo `djpeg`) in strict mode.
|
||||
|
||||
`-strict` turns every libjpeg warning into a failure: "premature end of data segment"
|
||||
(scan data missing or cut before EOI), "premature end of JPEG file", corrupt Huffman data.
|
||||
Pillow cannot be used for this: it ignores libjpeg warnings and returns a grey picture for
|
||||
a JPEG whose scan data is missing. `-scale 1/8` decodes every block but writes a 64x
|
||||
smaller raster to /dev/null; `-maxmemory` caps libjpeg's working memory (coefficient
|
||||
buffers of progressive files); the separate process is killed at the deadline.
|
||||
"""
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
DJPEG, "-strict", "-scale", "1/8", "-maxmemory", str(max_memory_mb * 1024), "-outfile", os.devnull,
|
||||
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.DEVNULL, stderr=asyncio.subprocess.PIPE)
|
||||
stderr = b""
|
||||
try:
|
||||
_, stderr = await asyncio.wait_for(process.communicate(data), timeout)
|
||||
except (asyncio.TimeoutError, asyncio.CancelledError):
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
await process.wait() # djpeg stopped reading: the file is already known to be broken
|
||||
if process.returncode is not None and process.returncode < 0:
|
||||
# Killed by a signal (the cgroup OOM killer under memory pressure): a problem of the
|
||||
# SERVER, not of the file - never report it as 415.
|
||||
raise VerifierUnavailable()
|
||||
if process.returncode != 0:
|
||||
if b"Insufficient memory" in (stderr or b""):
|
||||
# A valid but extremely memory-hungry file (progressive, 4:4:4, ~40 Mpx): a limit, not corruption.
|
||||
raise ImageError(413, "Изображение слишком велико для обработки")
|
||||
raise unsupported()
|
||||
@@ -0,0 +1,208 @@
|
||||
"""Variants of ONE image, built in a separate child process with hard limits (lab 5 worker).
|
||||
|
||||
parent (worker): await render(data, content_type, version, ...) -> list[Rendered]
|
||||
child: python -m messenger.imaging --type image/png --pipeline 1 ...
|
||||
stdin: original bytes; stdout: one JSON header line + variant bytes
|
||||
|
||||
Why a child process per image:
|
||||
* memory: RLIMIT_AS (IMAGE_MAX_MEMORY_MB) caps the address space of the decoder; a picture that
|
||||
needs more fails with MemoryError -> `image_too_large` (permanent), the worker keeps running;
|
||||
* time: the parent kills the child at JOB_TIMEOUT_SECONDS (C code of Pillow cannot be
|
||||
interrupted inside a thread) -> `processing_timeout` (transient, bounded by JOB_MAX_ATTEMPTS);
|
||||
* pixels: the size is read from the header and checked against MAX_IMAGE_PIXELS BEFORE the
|
||||
pixels are decoded (`Image.open` is lazy).
|
||||
A crash of the decoder (signal) is a transient `processing_killed`: the next attempt may run
|
||||
with less memory pressure; JOB_MAX_ATTEMPTS bounds it.
|
||||
|
||||
Geometry and encoder settings: pipeline.py. Alpha is handled in premultiplied form (RGBa) while
|
||||
resizing, so transparent pixels do not bleed their colour into the edges; for JPEG the result is
|
||||
then composited onto white.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .pipeline import CONTENT_TYPES, CROP_SIDE, OPTIMIZED_BOX, PIPELINES, THUMBNAIL_BOX, center_square, fit
|
||||
|
||||
EXIT_INVALID = 3 # the bytes are not a decodable PNG/JPEG
|
||||
EXIT_TOO_LARGE = 4 # pixel limit or memory limit
|
||||
HEADER_LIMIT = 4096
|
||||
|
||||
|
||||
class ProcessingError(Exception):
|
||||
"""Outcome of an attempt that did not produce variants. `permanent`: another attempt with the
|
||||
same input cannot succeed (job -> failed at once); otherwise retry with backoff."""
|
||||
|
||||
def __init__(self, code: str, permanent: bool):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.permanent = permanent
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Rendered:
|
||||
name: str
|
||||
content_type: str
|
||||
width: int
|
||||
height: int
|
||||
data: bytes
|
||||
|
||||
@property
|
||||
def sha256(self) -> bytes:
|
||||
return hashlib.sha256(self.data).digest()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ parent side
|
||||
async def render(data: bytes, content_type: str, pipeline_version: int, *, max_pixels: int, timeout: float,
|
||||
memory_mb: int) -> list[Rendered]:
|
||||
process = await asyncio.create_subprocess_exec(
|
||||
sys.executable, "-m", "messenger.imaging", "--type", content_type, "--pipeline", str(pipeline_version),
|
||||
"--max-pixels", str(max_pixels), "--memory-mb", str(memory_mb),
|
||||
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE)
|
||||
try:
|
||||
out, _ = await asyncio.wait_for(process.communicate(data), timeout)
|
||||
except asyncio.TimeoutError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise ProcessingError("processing_timeout", permanent=False) from None
|
||||
except asyncio.CancelledError:
|
||||
process.kill()
|
||||
await process.wait()
|
||||
raise
|
||||
code = process.returncode
|
||||
if code == EXIT_INVALID:
|
||||
raise ProcessingError("invalid_image", permanent=True)
|
||||
if code == EXIT_TOO_LARGE:
|
||||
raise ProcessingError("image_too_large", permanent=True)
|
||||
if code is None or code < 0:
|
||||
raise ProcessingError("processing_killed", permanent=False) # OOM killer or another signal
|
||||
if code != 0:
|
||||
raise ProcessingError("processing_failed", permanent=False)
|
||||
return parse_output(out)
|
||||
|
||||
|
||||
def parse_output(out: bytes) -> list[Rendered]:
|
||||
newline = out.find(b"\n", 0, HEADER_LIMIT)
|
||||
if newline < 0:
|
||||
raise ProcessingError("processing_failed", permanent=False)
|
||||
header = json.loads(out[:newline])
|
||||
pos = newline + 1
|
||||
result = []
|
||||
for item in header["variants"]:
|
||||
size = int(item["size"])
|
||||
result.append(Rendered(item["name"], item["content_type"], int(item["width"]), int(item["height"]),
|
||||
out[pos:pos + size]))
|
||||
pos += size
|
||||
if pos != len(out) or [r.name for r in result] != list(CONTENT_TYPES):
|
||||
raise ProcessingError("processing_failed", permanent=False)
|
||||
return result
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ child side
|
||||
def build(data: bytes, content_type: str, pipeline_version: int, max_pixels: int) -> list[Rendered]:
|
||||
"""Decode once, build every variant. Raises the Pillow/OS errors of a bad file."""
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
pipeline = PIPELINES[pipeline_version]
|
||||
warnings.simplefilter("ignore", Image.DecompressionBombWarning) # our own pixel limit below
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
expected = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type]
|
||||
img = Image.open(io.BytesIO(data), formats=[expected]) # header only: nothing decoded yet
|
||||
width, height = img.size
|
||||
if width * height > max_pixels:
|
||||
raise TooLarge()
|
||||
if expected == "JPEG":
|
||||
# DCT scaling: decode a big JPEG directly at 1/2..1/8 of its size, never below what the
|
||||
# largest variant needs (the box stays square, so EXIF rotation does not matter).
|
||||
img.draft("RGB", (OPTIMIZED_BOX, OPTIMIZED_BOX))
|
||||
img.load()
|
||||
img = ImageOps.exif_transpose(img) # orientation goes into the pixels, the tag is dropped
|
||||
img = _normalize(img)
|
||||
alpha = img.mode == "RGBA"
|
||||
base = img.convert("RGBa") if alpha else img # premultiplied alpha for resampling
|
||||
del img
|
||||
|
||||
def finish(im, *, flatten: bool):
|
||||
if alpha:
|
||||
im = im.convert("RGBA")
|
||||
if flatten:
|
||||
white = Image.new("RGB", im.size, (255, 255, 255))
|
||||
white.paste(im, mask=im.getchannel("A"))
|
||||
im = white
|
||||
im.info = {} # no EXIF/XMP/ICC/comments of the source in a variant
|
||||
return im
|
||||
|
||||
def resized(im, size):
|
||||
return im if im.size == size else im.resize(size, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
|
||||
thumbnail = finish(resized(base, fit(*base.size, THUMBNAIL_BOX)), flatten=True)
|
||||
crop = finish(resized(base.crop(center_square(*base.size)), (CROP_SIDE, CROP_SIDE)), flatten=True)
|
||||
optimized = finish(resized(base, fit(*base.size, OPTIMIZED_BOX)), flatten=False)
|
||||
out = []
|
||||
for name, im in (("thumbnail", thumbnail), ("crop", crop), ("optimized", optimized)):
|
||||
buf = io.BytesIO()
|
||||
if CONTENT_TYPES[name] == "image/jpeg":
|
||||
im.save(buf, "JPEG", quality=pipeline.jpeg_quality, optimize=True, subsampling=2, progressive=False)
|
||||
else:
|
||||
im.save(buf, "WEBP", quality=pipeline.webp_quality, method=pipeline.webp_method, exif=b"",
|
||||
icc_profile="", xmp=b"")
|
||||
out.append(Rendered(name, CONTENT_TYPES[name], im.width, im.height, buf.getvalue()))
|
||||
return out
|
||||
|
||||
|
||||
class TooLarge(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def _normalize(img):
|
||||
"""Any PNG/JPEG mode -> RGB, or RGBA when the image has transparency."""
|
||||
if img.mode.startswith("I;16") or img.mode == "I":
|
||||
# 16-bit grey: scale 0..65535 to 0..255 (a plain convert("L") would clip to white).
|
||||
img = img.convert("I").point(lambda v: v * (1 / 257)).convert("L")
|
||||
has_alpha = img.mode in ("RGBA", "LA", "PA", "La", "RGBa") or "transparency" in img.info
|
||||
return img.convert("RGBA" if has_alpha else "RGB") if img.mode != ("RGBA" if has_alpha else "RGB") else img
|
||||
|
||||
|
||||
def child_main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--type", required=True, choices=["image/png", "image/jpeg"])
|
||||
parser.add_argument("--pipeline", type=int, required=True, choices=sorted(PIPELINES))
|
||||
parser.add_argument("--max-pixels", type=int, required=True)
|
||||
parser.add_argument("--memory-mb", type=int, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
import resource
|
||||
limit = args.memory_mb * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
||||
except (ImportError, ValueError, OSError):
|
||||
pass # not Linux (unit tests on Windows): the container memory limit still applies
|
||||
data = sys.stdin.buffer.read()
|
||||
try:
|
||||
variants = build(data, args.type, args.pipeline, args.max_pixels)
|
||||
except (TooLarge, MemoryError):
|
||||
return EXIT_TOO_LARGE
|
||||
except Exception as exc: # noqa: BLE001 - every decoder error of the CONTENT is "invalid image"
|
||||
from PIL import Image
|
||||
if isinstance(exc, Image.DecompressionBombError):
|
||||
return EXIT_TOO_LARGE
|
||||
if isinstance(exc, (OSError, SyntaxError, ValueError, EOFError, KeyError, IndexError, TypeError)):
|
||||
return EXIT_INVALID
|
||||
raise
|
||||
header = {"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width, "height": v.height,
|
||||
"size": len(v.data)} for v in variants]}
|
||||
sys.stdout.buffer.write(json.dumps(header).encode() + b"\n")
|
||||
for v in variants:
|
||||
sys.stdout.buffer.write(v.data)
|
||||
sys.stdout.buffer.flush()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(child_main())
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Operator CLI for attachment jobs: `python -m messenger.jobctl ...` (compose service `jobctl`).
|
||||
|
||||
list-failed [--limit N] failed registry (view failed_jobs), newest first
|
||||
show <attachment_id> attachment state, its jobs, attempts and audit events
|
||||
stats jobs by state, outbox backlog
|
||||
replay <attachment_id> --operator NAME --reason TEXT
|
||||
failed -> queued: a NEW job (kind=replay, replay_of = the failed job) with a fresh attempt
|
||||
budget. Refused (exit 3, nothing changed) unless the attachment is failed AND its source
|
||||
still exists (staging copy present or original published). "Repeat" never means
|
||||
"run a finished job again": a ready attachment is refused (already_ready).
|
||||
reprocess <attachment_id> --pipeline N --operator NAME --reason TEXT
|
||||
Track 2: build a NEWER pipeline version of a READY attachment from its published original.
|
||||
Refused unless ready, N known, N > published version and no job of version >= N active.
|
||||
A not-yet-ready upload is refused: its only source is the staging copy that belongs to the
|
||||
running job, and its result would race with the first publication.
|
||||
|
||||
Every decision is taken under the attachments row lock, the same lock the cleanup takes before it
|
||||
marks a staging copy `deleting`: either the replay sees the copy and the cleanup then skips the
|
||||
row (no longer failed), or the cleanup marked it first and the replay refuses (source_missing).
|
||||
Output: one JSON document on stdout, without secrets or file names.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
from . import jobs
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .pipeline import PIPELINES
|
||||
|
||||
OPERATOR_RE = re.compile(r"[A-Za-z0-9._-]{1,64}")
|
||||
EXIT_REFUSED = 3
|
||||
|
||||
|
||||
class Refused(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def operator(value: str) -> str:
|
||||
if not OPERATOR_RE.fullmatch(value):
|
||||
raise argparse.ArgumentTypeError("operator must match [A-Za-z0-9._-]{1,64}")
|
||||
return value
|
||||
|
||||
|
||||
def reason(value: str) -> str:
|
||||
value = value.strip()
|
||||
if not 1 <= len(value) <= 200 or any(ord(ch) < 32 for ch in value):
|
||||
raise argparse.ArgumentTypeError("reason: 1-200 printable characters")
|
||||
return value
|
||||
|
||||
|
||||
def attachment_id(value: str) -> str:
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError("not a UUID") from None
|
||||
|
||||
|
||||
async def lock_attachment(conn, attachment: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT status, staging_state, object_key IS NOT NULL, published_version FROM attachments "
|
||||
"WHERE id = %s FOR UPDATE", (attachment,))
|
||||
row = await cur.fetchone()
|
||||
if row is None or row[0] == "pending":
|
||||
raise Refused("not_found")
|
||||
return row
|
||||
|
||||
|
||||
async def replay(conn, args, settings: Settings) -> dict:
|
||||
status, staging_state, has_original, _ = await lock_attachment(conn, args.attachment_id)
|
||||
if status == "ready":
|
||||
raise Refused("already_ready")
|
||||
if status != "failed":
|
||||
raise Refused("not_failed:" + status)
|
||||
if not has_original and staging_state != "present":
|
||||
raise Refused("source_missing") # staging removed by the retention cleanup
|
||||
cur = await conn.execute(
|
||||
"SELECT id, pipeline_version FROM jobs WHERE attachment_id = %s AND state = 'failed' "
|
||||
"ORDER BY finished_at DESC LIMIT 1", (args.attachment_id,))
|
||||
last = await cur.fetchone()
|
||||
if last is None:
|
||||
raise Refused("no_failed_job")
|
||||
cur = await conn.execute("SELECT 1 FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
|
||||
(args.attachment_id,))
|
||||
if await cur.fetchone():
|
||||
raise Refused("active_job")
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', error_code = NULL, failed_at = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = %s", (args.attachment_id,))
|
||||
actor = "jobctl:" + args.operator
|
||||
await jobs.event(conn, last[0], "replay_requested", actor, reason=args.reason)
|
||||
job_id = await jobs.create_job(conn, args.attachment_id, last[1], "replay", settings.job_max_attempts, actor,
|
||||
replay_of=last[0], reason=args.reason)
|
||||
return {"result": "queued", "job_id": str(job_id), "replay_of": str(last[0]), "pipeline_version": last[1]}
|
||||
|
||||
|
||||
async def reprocess(conn, args, settings: Settings) -> dict:
|
||||
status, _, _, published = await lock_attachment(conn, args.attachment_id)
|
||||
if args.pipeline not in PIPELINES:
|
||||
raise Refused("unknown_pipeline")
|
||||
if status != "ready":
|
||||
raise Refused("not_ready:" + status)
|
||||
if args.pipeline <= published:
|
||||
raise Refused("not_newer")
|
||||
cur = await conn.execute(
|
||||
"SELECT max(pipeline_version) FROM jobs WHERE attachment_id = %s AND state IN ('queued', 'processing')",
|
||||
(args.attachment_id,))
|
||||
(active,) = await cur.fetchone()
|
||||
if active is not None and active >= args.pipeline:
|
||||
raise Refused("newer_or_same_active")
|
||||
job_id = await jobs.create_job(conn, args.attachment_id, args.pipeline, "reprocess", settings.job_max_attempts,
|
||||
"jobctl:" + args.operator, reason=args.reason)
|
||||
return {"result": "queued", "job_id": str(job_id), "pipeline_version": args.pipeline,
|
||||
"published_version": published}
|
||||
|
||||
|
||||
async def list_failed(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute(
|
||||
"SELECT job_id, attachment_id, pipeline_version, kind, attempt, max_attempts, error_code, finished_at, "
|
||||
"attachment_status, staging_state FROM failed_jobs ORDER BY finished_at DESC LIMIT %s", (args.limit,))
|
||||
cols = [d.name for d in cur.description]
|
||||
return {"failed": [dict(zip(cols, r)) for r in await cur.fetchall()]}
|
||||
|
||||
|
||||
async def show(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute(
|
||||
"SELECT status, error_code, published_version, staging_state, object_key IS NOT NULL AS original_published, "
|
||||
"updated_at FROM attachments WHERE id = %s", (args.attachment_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
raise Refused("not_found")
|
||||
attachment = dict(zip([d.name for d in cur.description], row))
|
||||
|
||||
async def rows(sql: str) -> list[dict]:
|
||||
c = await conn.execute(sql, (args.attachment_id,))
|
||||
names = [d.name for d in c.description]
|
||||
return [dict(zip(names, r)) for r in await c.fetchall()]
|
||||
|
||||
return {
|
||||
"attachment": attachment,
|
||||
"jobs": await rows("SELECT id, pipeline_version, kind, state, attempt, max_attempts, error_code, replay_of, "
|
||||
"created_at, finished_at FROM jobs WHERE attachment_id = %s ORDER BY created_at"),
|
||||
"attempts": await rows("SELECT a.job_id, a.attempt, a.token, a.worker, a.source, a.outcome, a.error_code, "
|
||||
"a.started_at, a.finished_at, a.objects_cleaned_at FROM job_attempts a "
|
||||
"JOIN jobs j ON j.id = a.job_id WHERE j.attachment_id = %s ORDER BY a.started_at"),
|
||||
"variants": await rows("SELECT pipeline_version, name, content_type, size_bytes, width, height, "
|
||||
"encode(sha256, 'hex') AS sha256 FROM attachment_variants WHERE attachment_id = %s "
|
||||
"ORDER BY pipeline_version, name"),
|
||||
"events": await rows("SELECT e.job_id, e.at, e.event, e.attempt, e.actor, e.error_code, e.detail "
|
||||
"FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = %s "
|
||||
"ORDER BY e.id"),
|
||||
}
|
||||
|
||||
|
||||
async def stats(conn, args, settings: Settings) -> dict:
|
||||
cur = await conn.execute("SELECT state, count(*) FROM jobs GROUP BY state ORDER BY state")
|
||||
by_state = {r[0]: r[1] for r in await cur.fetchall()}
|
||||
cur = await conn.execute("SELECT count(*), count(*) FILTER (WHERE available_at <= clock_timestamp()) "
|
||||
"FROM outbox WHERE published_at IS NULL")
|
||||
pending, due = await cur.fetchone()
|
||||
return {"jobs": by_state, "outbox_unpublished": pending, "outbox_due": due}
|
||||
|
||||
|
||||
COMMANDS = {"replay": replay, "reprocess": reprocess, "list-failed": list_failed, "show": show, "stats": stats}
|
||||
|
||||
|
||||
def arguments(argv: list[str] | None) -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(prog="jobctl", description="Attachment jobs: failed registry, replay, reprocess")
|
||||
sub = p.add_subparsers(dest="command", required=True)
|
||||
for name in ("replay", "reprocess"):
|
||||
s = sub.add_parser(name)
|
||||
s.add_argument("attachment_id", type=attachment_id)
|
||||
if name == "reprocess":
|
||||
s.add_argument("--pipeline", type=int, required=True)
|
||||
s.add_argument("--operator", type=operator, required=True)
|
||||
s.add_argument("--reason", type=reason, required=True)
|
||||
s = sub.add_parser("list-failed")
|
||||
s.add_argument("--limit", type=int, default=50)
|
||||
s = sub.add_parser("show")
|
||||
s.add_argument("attachment_id", type=attachment_id)
|
||||
sub.add_parser("stats")
|
||||
return p.parse_args(argv)
|
||||
|
||||
|
||||
async def run(args: argparse.Namespace) -> int:
|
||||
settings = Settings.from_env(s3_credentials=False)
|
||||
db = Database(settings, name="messenger-jobctl")
|
||||
await db.open()
|
||||
try:
|
||||
writes = args.command in ("replay", "reprocess")
|
||||
result = await db.run(lambda c: COMMANDS[args.command](c, args, settings), deadline=10)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=1, default=str))
|
||||
return 0 if not writes or result.get("result") == "queued" else 1
|
||||
except Refused as exc:
|
||||
print(json.dumps({"result": "refused", "reason": str(exc)}))
|
||||
return EXIT_REFUSED
|
||||
except Unavailable:
|
||||
print(json.dumps({"result": "error", "reason": "database_unavailable"}))
|
||||
return 1
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = arguments(argv)
|
||||
try:
|
||||
return asyncio.run(run(args))
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,398 @@
|
||||
"""State machine of attachment jobs in PostgreSQL (lab 5). Every transition is ONE transaction.
|
||||
|
||||
Lock order, everywhere both rows are locked: the jobs row FIRST, then the attachments row
|
||||
(claim, complete, retry, lease expiry). Replay/reprocess/cleanup lock only the attachments row
|
||||
and insert NEW job rows. So two transitions never wait on each other in opposite order.
|
||||
|
||||
Fencing: a claim gives the attempt a fresh random `lease_token`. Every later write of that attempt
|
||||
(heartbeat, result, retry) is conditional on `jobs.lease_token = <its token>` under the jobs row
|
||||
lock. When the lease expires, the reaper clears the token; from then on the old attempt - even if
|
||||
its process is still alive (paused, slow) - can neither publish nor schedule anything, and the
|
||||
objects it wrote under its own keys are garbage for the cleanup.
|
||||
|
||||
Monotonic publication: a result of pipeline version V is published only if V is greater than the
|
||||
published version of the attachment; otherwise the job ends as `superseded` and nothing visible
|
||||
changes. `ready` never goes back to queued/processing.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
import psycopg
|
||||
|
||||
from .pipeline import VARIANT_NAMES
|
||||
from .storage import attempt_key
|
||||
|
||||
ACTIVE = ("queued", "processing")
|
||||
|
||||
|
||||
def backoff_delay(attempt: int, base: float, cap: float, rnd=random.random) -> float:
|
||||
"""Full jitter (AWS Architecture Blog, "Exponential Backoff And Jitter"): uniform in
|
||||
[0, min(cap, base * 2^(attempt-1))]. Spreads the retries of many jobs that failed together."""
|
||||
return rnd() * min(cap, base * (2 ** max(0, attempt - 1)))
|
||||
|
||||
|
||||
async def event(conn: psycopg.AsyncConnection, job_id, name: str, actor: str, *, attempt: int | None = None,
|
||||
error_code: str | None = None, **detail) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO job_events (job_id, event, attempt, actor, error_code, detail) VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(job_id, name, attempt, actor, error_code, json.dumps(detail, default=str)))
|
||||
|
||||
|
||||
async def create_job(conn: psycopg.AsyncConnection, attachment_id, pipeline_version: int, kind: str,
|
||||
max_attempts: int, actor: str, *, replay_of=None, **detail) -> uuid.UUID:
|
||||
"""jobs row + its first outbox row + audit, in the CALLER's transaction."""
|
||||
job_id = uuid.uuid4()
|
||||
await conn.execute(
|
||||
"INSERT INTO jobs (id, attachment_id, pipeline_version, kind, replay_of, max_attempts) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s)", (job_id, attachment_id, pipeline_version, kind, replay_of, max_attempts))
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, 1)", (job_id,))
|
||||
await event(conn, job_id, "created", actor, kind=kind, pipeline_version=pipeline_version,
|
||||
replay_of=str(replay_of) if replay_of else None, **detail)
|
||||
return job_id
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ relay
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Due:
|
||||
outbox_id: int
|
||||
job_id: str
|
||||
attachment_id: str
|
||||
pipeline_version: int
|
||||
attempt: int
|
||||
created_at: dt.datetime
|
||||
job_state: str
|
||||
|
||||
|
||||
async def due_outbox(conn: psycopg.AsyncConnection, batch: int) -> list[Due]:
|
||||
"""Due, unpublished outbox rows, locked for this relay pass (SKIP LOCKED: several relays
|
||||
never publish the same row at the same time - a crash may still publish it twice)."""
|
||||
cur = await conn.execute(
|
||||
"SELECT o.id, o.job_id, j.attachment_id, j.pipeline_version, o.attempt, j.created_at, j.state "
|
||||
"FROM outbox o JOIN jobs j ON j.id = o.job_id "
|
||||
"WHERE o.published_at IS NULL AND o.available_at <= clock_timestamp() "
|
||||
"ORDER BY o.available_at, o.id LIMIT %s FOR UPDATE OF o SKIP LOCKED", (batch,))
|
||||
return [Due(r[0], str(r[1]), str(r[2]), r[3], r[4], r[5], r[6]) for r in await cur.fetchall()]
|
||||
|
||||
|
||||
async def mark_published(conn: psycopg.AsyncConnection, ids: list[int]) -> None:
|
||||
if ids:
|
||||
await conn.execute("UPDATE outbox SET published_at = clock_timestamp() WHERE id = ANY(%s)", (ids,))
|
||||
|
||||
|
||||
async def count_publish_failure(conn: psycopg.AsyncConnection, ids: list[int]) -> None:
|
||||
if ids:
|
||||
await conn.execute("UPDATE outbox SET publish_failures = publish_failures + 1 WHERE id = ANY(%s)", (ids,))
|
||||
|
||||
|
||||
async def expire_leases(conn: psycopg.AsyncConnection, actor: str, limit: int = 50) -> int:
|
||||
"""Reaper: processing jobs whose lease ran out (the worker died, hung or lost its database).
|
||||
The attempt is abandoned; the job goes back to queued with a new outbox row, or to failed
|
||||
`retry_exhausted` when it used all its attempts."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id, attempt, max_attempts, lease_token, attachment_id FROM jobs "
|
||||
"WHERE state = 'processing' AND lease_until < clock_timestamp() "
|
||||
"ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED", (limit,))
|
||||
rows = await cur.fetchall()
|
||||
for job_id, attempt, max_attempts, token, attachment_id in rows:
|
||||
await conn.execute(
|
||||
"UPDATE job_attempts SET outcome = 'abandoned', error_code = 'lease_expired', finished_at = clock_timestamp() "
|
||||
"WHERE token = %s AND outcome = 'running'", (token,))
|
||||
if attempt >= max_attempts:
|
||||
await _fail(conn, job_id, attachment_id, "retry_exhausted", actor, attempt, cause="lease_expired")
|
||||
else:
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
|
||||
"available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (job_id,))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() "
|
||||
"WHERE id = %s AND status = 'processing'", (attachment_id,))
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1))
|
||||
await event(conn, job_id, "lease_expired", actor, attempt=attempt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def republish_stale(conn: psycopg.AsyncConnection, older_than: float, actor: str, limit: int = 50) -> int:
|
||||
"""Lost-message guard: a queued, due job that has had no unpublished outbox row and no
|
||||
publish for `older_than` seconds gets a new outbox row (its message was lost by the broker,
|
||||
dead-lettered after too many redeliveries, or the queue was purged)."""
|
||||
cur = await conn.execute(
|
||||
"SELECT j.id, j.attempt FROM jobs j "
|
||||
"WHERE j.state = 'queued' AND j.available_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"AND j.updated_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.id AND (o.published_at IS NULL "
|
||||
" OR o.published_at > clock_timestamp() - make_interval(secs => %s))) "
|
||||
"ORDER BY j.updated_at LIMIT %s FOR UPDATE OF j SKIP LOCKED", (older_than, older_than, older_than, limit))
|
||||
rows = await cur.fetchall()
|
||||
for job_id, attempt in rows:
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (job_id, attempt + 1))
|
||||
await event(conn, job_id, "republished", actor, attempt=attempt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def purge_outbox(conn: psycopg.AsyncConnection, keep_seconds: int = 7 * 86400) -> int:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM outbox WHERE id IN (SELECT id FROM outbox WHERE published_at < clock_timestamp() "
|
||||
"- make_interval(secs => %s) LIMIT 1000)", (keep_seconds,))
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ worker
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Claim:
|
||||
job_id: str
|
||||
attachment_id: str
|
||||
pipeline_version: int
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
token: str
|
||||
source: str # staging | original
|
||||
source_key: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: bytes
|
||||
write_original: bool # the original is not published yet: this attempt publishes it too
|
||||
|
||||
def key(self, name: str) -> str:
|
||||
return attempt_key(self.attachment_id, self.token, name)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Noop:
|
||||
reason: str # finished | leased | not_due | superseded | unknown_job
|
||||
|
||||
|
||||
async def claim(conn: psycopg.AsyncConnection, job_id: str, worker: str, lease_seconds: float) -> Claim | Noop:
|
||||
cur = await conn.execute(
|
||||
"SELECT state, attempt, max_attempts, available_at > clock_timestamp(), attachment_id, pipeline_version "
|
||||
"FROM jobs WHERE id = %s FOR UPDATE", (job_id,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return Noop("unknown_job")
|
||||
state, attempt, max_attempts, not_due, attachment_id, version = row
|
||||
if state not in ACTIVE:
|
||||
return Noop("finished") # duplicate/redelivery of a finished job: ack, nothing else
|
||||
if state == "processing":
|
||||
return Noop("leased") # another attempt holds the lease; the reaper handles its death
|
||||
if not_due:
|
||||
return Noop("not_due") # backoff not over: the outbox publishes it at available_at
|
||||
cur = await conn.execute(
|
||||
"SELECT status, published_version, object_key, staging_key, staging_state, content_type, size_bytes, sha256 "
|
||||
"FROM attachments WHERE id = %s FOR UPDATE", (attachment_id,))
|
||||
status, published, object_key, staging_key, staging_state, content_type, size, sha = await cur.fetchone()
|
||||
actor = "worker:" + worker
|
||||
if published is not None and published >= version:
|
||||
# A newer (or the same) version is already published: this job can only be stale.
|
||||
await _finish(conn, job_id, "superseded", None)
|
||||
await event(conn, job_id, "superseded", actor, attempt=attempt, published_version=published)
|
||||
return Noop("superseded")
|
||||
if object_key is not None:
|
||||
source, source_key = "original", object_key
|
||||
elif staging_state == "present":
|
||||
source, source_key = "staging", staging_key
|
||||
else:
|
||||
# Cannot happen while the CHECK attachments_active_has_source holds; fail visibly anyway.
|
||||
await _fail(conn, job_id, attachment_id, "source_missing", actor, attempt)
|
||||
return Noop("finished")
|
||||
token = uuid.uuid4().hex
|
||||
attempt += 1
|
||||
write_original = object_key is None
|
||||
names = (("original",) if write_original else ()) + VARIANT_NAMES
|
||||
planned = [attempt_key(str(attachment_id), token, n) for n in names]
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, "
|
||||
"lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s",
|
||||
(attempt, token, worker, lease_seconds, job_id))
|
||||
await conn.execute(
|
||||
"INSERT INTO job_attempts (token, job_id, attempt, worker, source, planned_keys) VALUES (%s, %s, %s, %s, %s, %s)",
|
||||
(token, job_id, attempt, worker, source, planned))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() WHERE id = %s AND status = 'queued'",
|
||||
(attachment_id,))
|
||||
await event(conn, job_id, "claimed", actor, attempt=attempt, source=source, token=token)
|
||||
return Claim(str(job_id), str(attachment_id), version, attempt, max_attempts, token, source, source_key,
|
||||
content_type, size, bytes(sha), write_original)
|
||||
|
||||
|
||||
async def heartbeat(conn: psycopg.AsyncConnection, claim: Claim, lease_seconds: float) -> bool:
|
||||
"""Extend the lease; False = the lease was lost (expired and taken over): stop the attempt."""
|
||||
cur = await conn.execute(
|
||||
"UPDATE jobs SET lease_until = clock_timestamp() + make_interval(secs => %s) "
|
||||
"WHERE id = %s AND state = 'processing' AND lease_token = %s", (lease_seconds, claim.job_id, claim.token))
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
async def _holds_lease(conn: psycopg.AsyncConnection, claim: Claim) -> bool:
|
||||
cur = await conn.execute("SELECT state = 'processing' AND lease_token = %s FROM jobs WHERE id = %s FOR UPDATE",
|
||||
(claim.token, claim.job_id))
|
||||
row = await cur.fetchone()
|
||||
return bool(row and row[0])
|
||||
|
||||
|
||||
async def complete(conn: psycopg.AsyncConnection, claim: Claim, variants, worker: str) -> str:
|
||||
"""Publish the result of an attempt atomically: variant rows + attachment ready + job succeeded.
|
||||
Returns succeeded | superseded | fenced (this attempt lost its lease: nothing written)."""
|
||||
actor = "worker:" + worker
|
||||
if not await _holds_lease(conn, claim):
|
||||
return "fenced"
|
||||
cur = await conn.execute("SELECT published_version FROM attachments WHERE id = %s FOR UPDATE", (claim.attachment_id,))
|
||||
(published,) = await cur.fetchone()
|
||||
if published is not None and published >= claim.pipeline_version:
|
||||
await _end_attempt(conn, claim, "superseded", None)
|
||||
await _finish(conn, claim.job_id, "superseded", None)
|
||||
await event(conn, claim.job_id, "superseded", actor, attempt=claim.attempt, published_version=published)
|
||||
return "superseded"
|
||||
for v in variants:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type, "
|
||||
"size_bytes, width, height, sha256, attempt_token) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s)",
|
||||
(claim.attachment_id, claim.pipeline_version, v.name, claim.key(v.name), v.content_type, len(v.data),
|
||||
v.width, v.height, v.sha256, claim.token))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', published_version = %s, error_code = NULL, failed_at = NULL, "
|
||||
"object_key = COALESCE(object_key, %s), updated_at = clock_timestamp() WHERE id = %s",
|
||||
(claim.pipeline_version, claim.key("original") if claim.write_original else None, claim.attachment_id))
|
||||
await _end_attempt(conn, claim, "succeeded", None)
|
||||
await _finish(conn, claim.job_id, "succeeded", None)
|
||||
await event(conn, claim.job_id, "succeeded", actor, attempt=claim.attempt, pipeline_version=claim.pipeline_version)
|
||||
return "succeeded"
|
||||
|
||||
|
||||
async def retry_or_fail(conn: psycopg.AsyncConnection, claim: Claim, code: str, permanent: bool, worker: str,
|
||||
base: float, cap: float) -> tuple[str, float]:
|
||||
"""After a failed attempt: permanent error or no attempts left -> failed (failed registry);
|
||||
otherwise queued again after a jittered backoff. Returns (outcome, delay seconds)."""
|
||||
actor = "worker:" + worker
|
||||
if not await _holds_lease(conn, claim):
|
||||
return "fenced", 0.0
|
||||
if permanent or claim.attempt >= claim.max_attempts:
|
||||
final = code if permanent else "retry_exhausted"
|
||||
await _end_attempt(conn, claim, "failed", code)
|
||||
await _fail(conn, claim.job_id, claim.attachment_id, final, actor, claim.attempt, cause=code)
|
||||
return "failed", 0.0
|
||||
delay = backoff_delay(claim.attempt, base, cap)
|
||||
await _end_attempt(conn, claim, "retry", code)
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, error_code = %s, "
|
||||
"available_at = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp() WHERE id = %s",
|
||||
(code, delay, claim.job_id))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'",
|
||||
(claim.attachment_id,))
|
||||
await conn.execute(
|
||||
"INSERT INTO outbox (job_id, attempt, available_at) "
|
||||
"VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s))", (claim.job_id, claim.attempt + 1, delay))
|
||||
await event(conn, claim.job_id, "retry_scheduled", actor, attempt=claim.attempt, error_code=code,
|
||||
delay_s=round(delay, 3))
|
||||
return "retry", delay
|
||||
|
||||
|
||||
async def release(conn: psycopg.AsyncConnection, claim: Claim, worker: str) -> str:
|
||||
"""Graceful stop of a worker with an attempt still running: give the job back at once
|
||||
(without waiting for the lease to expire). The claim already counted as an attempt."""
|
||||
if not await _holds_lease(conn, claim):
|
||||
return "fenced"
|
||||
await _end_attempt(conn, claim, "abandoned", "worker_shutdown")
|
||||
if claim.attempt >= claim.max_attempts:
|
||||
await _fail(conn, claim.job_id, claim.attachment_id, "retry_exhausted", "worker:" + worker, claim.attempt,
|
||||
cause="worker_shutdown")
|
||||
return "failed"
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = 'queued', lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
|
||||
"available_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (claim.job_id,))
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() WHERE id = %s AND status = 'processing'",
|
||||
(claim.attachment_id,))
|
||||
await conn.execute("INSERT INTO outbox (job_id, attempt) VALUES (%s, %s)", (claim.job_id, claim.attempt + 1))
|
||||
await event(conn, claim.job_id, "released", "worker:" + worker, attempt=claim.attempt)
|
||||
return "released"
|
||||
|
||||
|
||||
async def _end_attempt(conn: psycopg.AsyncConnection, claim: Claim, outcome: str, code: str | None) -> None:
|
||||
await conn.execute(
|
||||
"UPDATE job_attempts SET outcome = %s, error_code = %s, finished_at = clock_timestamp() "
|
||||
"WHERE token = %s AND outcome = 'running'", (outcome, code, claim.token))
|
||||
|
||||
|
||||
async def _finish(conn: psycopg.AsyncConnection, job_id, state: str, code: str | None) -> None:
|
||||
await conn.execute(
|
||||
"UPDATE jobs SET state = %s, error_code = %s, lease_token = NULL, lease_owner = NULL, lease_until = NULL, "
|
||||
"finished_at = clock_timestamp(), updated_at = clock_timestamp() WHERE id = %s", (state, code, job_id))
|
||||
|
||||
|
||||
async def _fail(conn: psycopg.AsyncConnection, job_id, attachment_id, code: str, actor: str, attempt: int,
|
||||
**detail) -> None:
|
||||
"""Job -> failed (registry). The attachment becomes failed only if it is not ready: a failed
|
||||
reprocess of a newer version leaves the published result in place."""
|
||||
await _finish(conn, job_id, "failed", code)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = %s, failed_at = clock_timestamp(), "
|
||||
"updated_at = clock_timestamp() WHERE id = %s AND status IN ('queued', 'processing')", (code, attachment_id))
|
||||
await event(conn, job_id, "failed", actor, attempt=attempt, error_code=code, **detail)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cleanup (worker)
|
||||
async def staging_to_delete(conn: psycopg.AsyncConnection, retention_seconds: int, limit: int = 50) -> list[str]:
|
||||
"""Candidates only (no lock): ready attachments (staging no longer needed) and failed ones
|
||||
older than the retention. The decision is re-checked under the row lock in mark_staging_deleting."""
|
||||
cur = await conn.execute(
|
||||
"SELECT id FROM attachments WHERE staging_state IN ('present', 'deleting') AND (status = 'ready' OR "
|
||||
"(status = 'failed' AND failed_at < clock_timestamp() - make_interval(secs => %s))) "
|
||||
"ORDER BY updated_at LIMIT %s", (retention_seconds, limit))
|
||||
return [str(r[0]) for r in await cur.fetchall()]
|
||||
|
||||
|
||||
async def mark_staging_deleting(conn: psycopg.AsyncConnection, attachment_id: str, retention_seconds: int) -> str | None:
|
||||
"""Under the attachment row lock, decide AGAIN from the committed state and mark the staging copy
|
||||
`deleting` BEFORE the S3 DELETE. A replay that committed in between made the row queued, so it
|
||||
no longer matches; a replay that comes later sees `deleting` and refuses. Queued/processing
|
||||
rows never match (and the CHECK attachments_active_has_source forbids it anyway)."""
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachments SET staging_state = 'deleting' " # internal state: updated_at stays (public field)
|
||||
"WHERE id = %s AND staging_state IN ('present', 'deleting') AND (status = 'ready' OR (status = 'failed' "
|
||||
"AND failed_at < clock_timestamp() - make_interval(secs => %s) AND NOT EXISTS (SELECT 1 FROM jobs j "
|
||||
"WHERE j.attachment_id = attachments.id AND j.state IN ('queued', 'processing')))) RETURNING staging_key",
|
||||
(attachment_id, retention_seconds))
|
||||
row = await cur.fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
|
||||
async def mark_staging_deleted(conn: psycopg.AsyncConnection, attachment_id: str) -> None:
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET staging_state = 'deleted' WHERE id = %s AND staging_state = 'deleting'", (attachment_id,))
|
||||
|
||||
|
||||
async def attempts_to_clean(conn: psycopg.AsyncConnection, grace_seconds: int, limit: int = 50) -> list[str]:
|
||||
cur = await conn.execute(
|
||||
"SELECT token FROM job_attempts WHERE outcome IN ('retry', 'failed', 'abandoned', 'superseded') "
|
||||
"AND objects_cleaned_at IS NULL AND finished_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"ORDER BY finished_at LIMIT %s", (grace_seconds, limit))
|
||||
return [r[0].hex for r in await cur.fetchall()]
|
||||
|
||||
|
||||
async def lock_attempt_for_cleanup(conn: psycopg.AsyncConnection, token: str) -> list[str] | None:
|
||||
"""Keys of an attempt that can never publish: it ended without success, it is not the lease
|
||||
holder of its job (tokens are never reused) and nothing published references its keys."""
|
||||
cur = await conn.execute(
|
||||
"SELECT ja.planned_keys FROM job_attempts ja JOIN jobs j ON j.id = ja.job_id "
|
||||
"WHERE ja.token = %s AND ja.objects_cleaned_at IS NULL "
|
||||
"AND ja.outcome IN ('retry', 'failed', 'abandoned', 'superseded') "
|
||||
"AND j.lease_token IS DISTINCT FROM ja.token FOR UPDATE OF ja", (token,))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
keys = list(row[0])
|
||||
cur = await conn.execute(
|
||||
"SELECT EXISTS (SELECT 1 FROM attachment_variants WHERE object_key = ANY(%s)) "
|
||||
"OR EXISTS (SELECT 1 FROM attachments WHERE object_key = ANY(%s))", (keys, keys))
|
||||
if (await cur.fetchone())[0]:
|
||||
return None # published: never delete (would be a bug elsewhere; logged by the caller)
|
||||
return keys
|
||||
|
||||
|
||||
async def mark_attempt_cleaned(conn: psycopg.AsyncConnection, token: str) -> None:
|
||||
await conn.execute("UPDATE job_attempts SET objects_cleaned_at = clock_timestamp() WHERE token = %s", (token,))
|
||||
@@ -0,0 +1,36 @@
|
||||
"""JSON-lines logging to stdout. Logs never contain message texts or request bodies."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
entry = {
|
||||
"ts": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
|
||||
"level": record.levelname.lower(),
|
||||
"logger": record.name,
|
||||
"event": record.getMessage(),
|
||||
}
|
||||
fields = getattr(record, "fields", None)
|
||||
if fields:
|
||||
entry.update(fields)
|
||||
if record.exc_info:
|
||||
# Stack traces go only to the server log, never to the HTTP client.
|
||||
entry["exception"] = self.formatException(record.exc_info)
|
||||
return json.dumps(entry, ensure_ascii=False)
|
||||
|
||||
|
||||
def configure(level: str) -> None:
|
||||
handler = logging.StreamHandler(sys.stdout)
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger()
|
||||
root.handlers[:] = [handler]
|
||||
root.setLevel(level.upper())
|
||||
# uvicorn's own access log is disabled; our middleware writes one line per request.
|
||||
for name in ("uvicorn", "uvicorn.error"):
|
||||
logging.getLogger(name).handlers[:] = []
|
||||
logging.getLogger(name).propagate = True
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Versioned SQL migrations: `python -m messenger.migrate` (compose service `migrate`).
|
||||
|
||||
* Files `migrations/NNNN_name.sql` are applied in name order, each in its own
|
||||
transaction together with its row in `schema_migrations` (DDL is transactional
|
||||
in PostgreSQL, so a failed migration leaves nothing behind).
|
||||
* Re-running is safe: applied versions are skipped, so data is never duplicated.
|
||||
* The SHA-256 of every applied file is stored; if an applied file was edited later,
|
||||
the run stops instead of silently diverging from the real schema.
|
||||
* `pg_advisory_lock` serialises concurrent runs (two API replicas starting at once).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
|
||||
from .config import ConfigError, _secret
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"
|
||||
LOCK_ID = 2026_02 # arbitrary application-wide advisory lock id
|
||||
|
||||
|
||||
def say(text: str) -> None:
|
||||
print(f"migrate: {text}", flush=True)
|
||||
|
||||
|
||||
def connect(url: str, wait_seconds: float = 60) -> psycopg.Connection:
|
||||
deadline = time.monotonic() + wait_seconds
|
||||
while True:
|
||||
try:
|
||||
return psycopg.connect(url, autocommit=True, connect_timeout=3, application_name="messenger-migrate")
|
||||
except psycopg.OperationalError as exc:
|
||||
if time.monotonic() > deadline:
|
||||
raise SystemExit(f"migrate: database is not reachable: {type(exc).__name__}") from None
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
url = _secret("DATABASE_URL", required=True)
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
|
||||
with connect(url) as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY, checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now())")
|
||||
applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall())
|
||||
for path in files:
|
||||
version, sql = path.stem, path.read_bytes()
|
||||
checksum = hashlib.sha256(sql).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
say(f"{version}: file changed after it was applied; create a new migration instead")
|
||||
return 1
|
||||
say(f"{version}: already applied, skipped")
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(sql.decode("utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (version, checksum))
|
||||
say(f"{version}: applied")
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
say(f"done, {len(files)} migration file(s) checked")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Password hashing with Argon2id (memory-hard password KDF; also mandatory in lab 7).
|
||||
|
||||
* Parameters follow the OWASP minimum for Argon2id: m=19 MiB, t=2, p=1. Unlike bcrypt
|
||||
there is no 72-byte limit, so any 12-128 code point UTF-8 password is hashed whole.
|
||||
* Hashing takes tens of milliseconds of CPU and 19 MiB of RAM. It runs in a worker
|
||||
thread (argon2-cffi releases the GIL), so the event loop keeps serving other
|
||||
requests, and a semaphore caps parallel hashes (memory/CPU DoS protection).
|
||||
* An unknown username is checked against a dummy hash, so "no such user" and
|
||||
"wrong password" take the same time and return the same 401.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
|
||||
|
||||
_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1, hash_len=32, salt_len=16)
|
||||
_semaphore: asyncio.Semaphore | None = None
|
||||
_dummy_hash: str | None = None
|
||||
|
||||
|
||||
def configure(concurrency: int) -> None:
|
||||
global _semaphore
|
||||
_semaphore = asyncio.Semaphore(concurrency)
|
||||
|
||||
|
||||
async def _run(func, *args):
|
||||
if _semaphore is None:
|
||||
configure(2)
|
||||
async with _semaphore:
|
||||
return await asyncio.to_thread(func, *args)
|
||||
|
||||
|
||||
async def hash_password(password: str) -> str:
|
||||
return await _run(_hasher.hash, password)
|
||||
|
||||
|
||||
def _verify(stored_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(stored_hash, password)
|
||||
except (VerifyMismatchError, VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
async def verify_password(stored_hash: str | None, password: str) -> bool:
|
||||
global _dummy_hash
|
||||
if stored_hash is None:
|
||||
if _dummy_hash is None:
|
||||
_dummy_hash = await hash_password("dummy-password-for-timing")
|
||||
await _run(_verify, _dummy_hash, password)
|
||||
return False
|
||||
return await _run(_verify, stored_hash, password)
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Versioned image pipeline (lab 5): WHAT the worker builds for one attachment. Pure definitions,
|
||||
no I/O; the pixels are produced by imaging.py in a child process.
|
||||
|
||||
Every pipeline version builds the same three variants with the geometry of the contract; versions
|
||||
differ only in encoder parameters, so the published version is visible in the bytes (sha256):
|
||||
|
||||
thumbnail JPEG, fits into 256x256, keeps the aspect ratio, never upscales, floor rounding
|
||||
(minimum 1 px), EXIF orientation applied, alpha composited onto white;
|
||||
crop JPEG, centred square 128x128 (small images ARE upscaled, the contract allows it);
|
||||
optimized WebP, fits into 1280x1280, never upscales, alpha kept.
|
||||
|
||||
Metadata: nothing of the source is copied into a variant - no EXIF (orientation is applied to
|
||||
the pixels instead), no XMP, no ICC profile, no comments (the pixels are treated as sRGB).
|
||||
The original is published byte for byte and keeps everything.
|
||||
|
||||
Track 2: the version of a job is fixed when the job is created (PIPELINE_VERSION for uploads,
|
||||
`jobctl reprocess --pipeline N` later). A result is published only if its version is greater
|
||||
than the published one, so an old job can never overwrite a newer result.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
THUMBNAIL_BOX = 256
|
||||
CROP_SIDE = 128
|
||||
OPTIMIZED_BOX = 1280
|
||||
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
|
||||
CONTENT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pipeline:
|
||||
version: int
|
||||
jpeg_quality: int # thumbnail and crop: baseline JPEG, 4:2:0, optimized Huffman tables
|
||||
webp_quality: int # optimized: lossy WebP
|
||||
webp_method: int # 0 (fast) .. 6 (slow, smaller)
|
||||
|
||||
|
||||
PIPELINES = {
|
||||
1: Pipeline(1, jpeg_quality=85, webp_quality=80, webp_method=4),
|
||||
# Versions 2 and 3 exist for the track 2 experiments (reprocess, reorder, stale jobs):
|
||||
# different encoder settings -> different bytes, identical geometry.
|
||||
2: Pipeline(2, jpeg_quality=80, webp_quality=75, webp_method=4),
|
||||
3: Pipeline(3, jpeg_quality=90, webp_quality=85, webp_method=5),
|
||||
}
|
||||
|
||||
|
||||
def fit(width: int, height: int, box: int) -> tuple[int, int]:
|
||||
"""Size inside box x box keeping the aspect ratio, never upscaling, rounded DOWN, >= 1 px.
|
||||
Integer arithmetic: 320x200 -> 256x160 exactly, 1x1000 -> 1x256 (0.256 -> 1 px minimum)."""
|
||||
if width <= box and height <= box:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return box, max(1, height * box // width)
|
||||
return max(1, width * box // height), box
|
||||
|
||||
|
||||
def center_square(width: int, height: int) -> tuple[int, int, int, int]:
|
||||
"""Crop box (left, top, right, bottom) of the centred square with side min(width, height)."""
|
||||
side = min(width, height)
|
||||
left = (width - side) // 2
|
||||
top = (height - side) // 2
|
||||
return left, top, left + side, top + side
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Outbox relay: `python -m messenger.relay` (compose service `relay`, one process).
|
||||
|
||||
loop: SELECT due outbox rows FOR UPDATE SKIP LOCKED
|
||||
-> publish each job message to RabbitMQ, wait for the publisher CONFIRM
|
||||
-> UPDATE outbox SET published_at -> COMMIT
|
||||
every few seconds: reaper (expired leases -> queued/failed), lost-message guard.
|
||||
|
||||
Why an outbox: the API commits the job and the outbox row in ONE PostgreSQL transaction and never
|
||||
talks to the broker. A crash between COMMIT and publish leaves the row unpublished; the relay
|
||||
publishes it when it (or the broker) is back - nobody re-creates a job by hand. While the broker
|
||||
is down, uploads still get 202 and the rows wait in the outbox.
|
||||
|
||||
A crash after the confirm but before the COMMIT of published_at publishes the row once more after
|
||||
the restart: the queue then holds a duplicate. That is at-least-once delivery; the worker turns
|
||||
the duplicate into a no-op (jobs.claim). The relay never deletes or rewrites jobs.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
import aio_pika
|
||||
from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractExchange
|
||||
|
||||
from . import broker, jobs, logging_setup, service
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
|
||||
log = logging.getLogger("messenger.relay")
|
||||
ACTOR = "relay"
|
||||
MAINTENANCE_SECONDS = 5.0
|
||||
PURGE_SECONDS = 600.0
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.db = Database(settings, name="messenger-relay")
|
||||
self.connection: AbstractConnection | None = None
|
||||
self.channel: AbstractChannel | None = None
|
||||
self.exchange: AbstractExchange | None = None
|
||||
self.lost: asyncio.Event | None = None
|
||||
|
||||
def connected(self) -> bool:
|
||||
return (self.connection is not None and self.lost is not None and not self.lost.is_set()
|
||||
and self.channel is not None and not self.channel.is_closed)
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.connection = await aio_pika.connect(
|
||||
self.settings.broker_url, timeout=5,
|
||||
client_properties={"connection_name": "relay@" + self.settings.instance_id})
|
||||
self.lost = service.watch_connection(self.connection)
|
||||
# Confirm mode: every publish() waits for basic.ack of the broker (message in the quorum log).
|
||||
self.channel = await self.connection.channel(publisher_confirms=True, on_return_raises=True)
|
||||
# No declare: the topology belongs to the broker definitions; the relay may only write.
|
||||
self.exchange = await self.channel.get_exchange(broker.EXCHANGE, ensure=False)
|
||||
log.info("broker_connected", extra={"fields": {"exchange": broker.EXCHANGE}})
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
if self.connection is not None:
|
||||
try:
|
||||
await self.connection.close()
|
||||
except Exception: # noqa: BLE001 - already broken
|
||||
pass
|
||||
self.connection = self.channel = self.exchange = self.lost = None
|
||||
|
||||
async def publish_due(self) -> int:
|
||||
settings = self.settings
|
||||
exchange = self.exchange
|
||||
|
||||
async def work(conn) -> list[jobs.Due]:
|
||||
rows = await jobs.due_outbox(conn, settings.relay_batch)
|
||||
if "reverse_order" in settings.fault_points:
|
||||
rows.reverse()
|
||||
# Rows of jobs that are no longer queued (finished meanwhile) need no message.
|
||||
sendable = [r for r in rows if r.job_state == "queued"]
|
||||
copies = 2 if "duplicate_publish" in settings.fault_points else 1
|
||||
for row in sendable:
|
||||
message = aio_pika.Message(
|
||||
broker.encode(row.job_id, row.attachment_id, row.pipeline_version, row.attempt, row.created_at),
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT, content_type="application/json",
|
||||
message_id=f"{row.job_id}:{row.outbox_id}", type=broker.JOB_TYPE)
|
||||
for _ in range(copies):
|
||||
await exchange.publish(message, routing_key=broker.ROUTING_KEY, mandatory=True, timeout=5)
|
||||
for row in sendable:
|
||||
# Confirmed by the broker, outbox not marked yet: the window "after publish, before
|
||||
# the confirm is recorded". The crash rolls this transaction back.
|
||||
await service.crash_once(settings, self.db, "crash_after_publish", row.job_id, ACTOR,
|
||||
outbox_id=row.outbox_id)
|
||||
await jobs.mark_published(conn, [r.outbox_id for r in rows])
|
||||
return sendable
|
||||
|
||||
sent = await self.db.run(work, deadline=10 + 5 * settings.relay_batch)
|
||||
for row in sent:
|
||||
log.info("job_published", extra={"fields": {"job_id": row.job_id, "attempt": row.attempt,
|
||||
"outbox_id": row.outbox_id}})
|
||||
return len(sent)
|
||||
|
||||
async def maintenance(self) -> None:
|
||||
expired = await self.db.run(lambda c: jobs.expire_leases(c, ACTOR))
|
||||
republished = await self.db.run(lambda c: jobs.republish_stale(c, self.settings.queued_republish_seconds, ACTOR))
|
||||
if expired or republished:
|
||||
log.info("maintenance", extra={"fields": {"leases_expired": expired, "republished": republished}})
|
||||
|
||||
async def run(self) -> int:
|
||||
stop = service.stop_event()
|
||||
await self.db.open()
|
||||
next_maintenance = next_purge = 0.0
|
||||
failures = 0
|
||||
try:
|
||||
while not stop.is_set():
|
||||
healthy = True
|
||||
now = time.monotonic()
|
||||
try:
|
||||
if now >= next_maintenance:
|
||||
await self.maintenance()
|
||||
next_maintenance = now + MAINTENANCE_SECONDS
|
||||
if now >= next_purge:
|
||||
purged = await self.db.run(jobs.purge_outbox)
|
||||
next_purge = now + PURGE_SECONDS
|
||||
if purged:
|
||||
log.info("outbox_purged", extra={"fields": {"rows": purged}})
|
||||
except Unavailable:
|
||||
healthy = False
|
||||
try:
|
||||
if not self.connected():
|
||||
await self.disconnect()
|
||||
await self.connect()
|
||||
sent = await self.publish_due()
|
||||
failures = 0
|
||||
except Unavailable:
|
||||
healthy, sent = False, 0
|
||||
except Exception as exc: # noqa: BLE001 - every broker/channel error, never a crash
|
||||
# Broker down/unreachable, channel closed, message returned/nacked: nothing marked,
|
||||
# the rows stay in the outbox (transaction rolled back) and are published later.
|
||||
failures += 1
|
||||
healthy, sent = False, 0
|
||||
log.warning("publish_failed", extra={"fields": {"error": type(exc).__name__, "failures": failures}})
|
||||
await self.disconnect()
|
||||
if healthy:
|
||||
service.beat()
|
||||
if not sent:
|
||||
await service.sleep_or_stop(stop, min(5.0, self.settings.relay_poll_seconds * (2 ** min(failures, 3))))
|
||||
finally:
|
||||
await self.disconnect()
|
||||
await self.db.close()
|
||||
log.info("relay_stopped")
|
||||
return 0
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
settings = Settings.from_env(s3_credentials=False)
|
||||
if not settings.broker_url:
|
||||
raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set")
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
logging_setup.configure(settings.log_level)
|
||||
log.info("relay_starting", extra={"fields": {"instance_id": settings.instance_id,
|
||||
"fault_points": sorted(settings.fault_points)}})
|
||||
return asyncio.run(Relay(settings).run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,373 @@
|
||||
"""SQL for users, chats and messages. All statements are parameterised (%s placeholders):
|
||||
values travel separately from the SQL text, so input can never change the query.
|
||||
|
||||
Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is
|
||||
encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors as pg_errors
|
||||
|
||||
from .cursor import SortKey
|
||||
from .db import Database
|
||||
from .errors import Conflict, InvalidRequest, NotFound, Unavailable
|
||||
from .timefmt import from_us, rfc3339, to_us
|
||||
|
||||
|
||||
def _b(text: str) -> bytes:
|
||||
return text.encode("utf-8")
|
||||
|
||||
|
||||
def _s(raw: bytes | memoryview) -> str:
|
||||
return bytes(raw).decode("utf-8")
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: dt.datetime
|
||||
|
||||
def to_json(self) -> dict:
|
||||
return {"id": self.id, "username": self.username, "display_name": self.display_name,
|
||||
"created_at": rfc3339(self.created_at)}
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: tuple[str, ...]
|
||||
created_at: dt.datetime
|
||||
|
||||
@property
|
||||
def key(self) -> SortKey:
|
||||
return (to_us(self.created_at), self.id)
|
||||
|
||||
def to_json(self) -> dict:
|
||||
return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids),
|
||||
"created_at": rfc3339(self.created_at)}
|
||||
|
||||
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: dt.datetime
|
||||
attachment_ids: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def key(self) -> SortKey:
|
||||
return (to_us(self.created_at), self.id)
|
||||
|
||||
def to_json(self) -> dict:
|
||||
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
|
||||
"created_at": rfc3339(self.created_at), "attachment_ids": list(self.attachment_ids)}
|
||||
|
||||
|
||||
def _user(row) -> User:
|
||||
return User(str(row[0]), row[1], _s(row[2]), row[3])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ users
|
||||
async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User:
|
||||
async def work(conn: psycopg.AsyncConnection) -> User:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) "
|
||||
"RETURNING id, username, display_name, created_at",
|
||||
(uuid.uuid4(), username, _b(display_name), password_hash))
|
||||
return _user(await cur.fetchone())
|
||||
try:
|
||||
return await db.run(work)
|
||||
except pg_errors.UniqueViolation:
|
||||
# Two concurrent registrations: the unique index lets exactly one INSERT commit.
|
||||
raise Conflict("Имя пользователя уже занято") from None
|
||||
|
||||
|
||||
async def get_user(db: Database, user_id: str) -> User | None:
|
||||
async def work(conn: psycopg.AsyncConnection) -> User | None:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
|
||||
row = await cur.fetchone()
|
||||
return _user(row) if row else None
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
async def get_credentials(db: Database, username: str) -> tuple[User, str] | None:
|
||||
async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,))
|
||||
row = await cur.fetchone()
|
||||
return (_user(row), row[4]) if row else None
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ chats
|
||||
async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat:
|
||||
"""Atomic: the chat row and all memberships commit together or not at all.
|
||||
|
||||
The chat is inserted FIRST; an unknown member then fails the membership INSERT
|
||||
with a foreign-key violation and the whole transaction (chat row included) is
|
||||
rolled back. student-tests check that nothing is left behind.
|
||||
"""
|
||||
if creator_id not in member_ids:
|
||||
raise InvalidRequest("member_ids должен включать текущего пользователя")
|
||||
chat_id = uuid.uuid4()
|
||||
|
||||
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at",
|
||||
(chat_id, _b(title), creator_id))
|
||||
(created_at,) = await cur.fetchone()
|
||||
await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) "
|
||||
"SELECT %s, m.user_id, m.ord - 1, %s "
|
||||
"FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
|
||||
(chat_id, created_at, member_ids))
|
||||
return created_at
|
||||
try:
|
||||
created_at = await db.run(work)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise InvalidRequest("member_ids содержит несуществующих пользователей") from None
|
||||
return Chat(str(chat_id), title, tuple(member_ids), created_at)
|
||||
|
||||
|
||||
async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]:
|
||||
if not rows:
|
||||
return []
|
||||
ids = [r[0] for r in rows]
|
||||
cur = await conn.execute(
|
||||
"SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,))
|
||||
members: dict[uuid.UUID, list[str]] = {i: [] for i in ids}
|
||||
for chat_id, user_id in await cur.fetchall():
|
||||
members[chat_id].append(str(user_id))
|
||||
return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows]
|
||||
|
||||
|
||||
async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat:
|
||||
"""The chat if the user is a member; a missing chat and a foreign chat are both 404."""
|
||||
async def work(conn: psycopg.AsyncConnection) -> Chat:
|
||||
cur = await conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chats c "
|
||||
"JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
|
||||
(user_id, chat_id))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
raise NotFound("Чат не найден")
|
||||
return (await _load_chats(conn, [row]))[0]
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None:
|
||||
cur = await conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
|
||||
if await cur.fetchone() is None:
|
||||
raise NotFound("Чат не найден")
|
||||
|
||||
|
||||
async def ensure_member(db: Database, user_id: str, chat_id: str) -> None:
|
||||
async def work(conn: psycopg.AsyncConnection) -> None:
|
||||
await _check_member(conn, user_id, chat_id)
|
||||
await db.run(work)
|
||||
|
||||
|
||||
async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]:
|
||||
async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]:
|
||||
if after is None:
|
||||
cur = await conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
|
||||
"WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
|
||||
(user_id, limit + 1))
|
||||
else:
|
||||
cur = await conn.execute(
|
||||
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
|
||||
"WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) "
|
||||
"ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
|
||||
(user_id, from_us(after[0]), after[1], limit + 1))
|
||||
rows = await cur.fetchall()
|
||||
return await _load_chats(conn, rows[:limit]), len(rows) > limit
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ messages
|
||||
@dataclass(slots=True, frozen=True)
|
||||
class PostResult:
|
||||
message: Message
|
||||
replayed: bool # True: the key was already used with this body, nothing new was written
|
||||
|
||||
|
||||
async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str,
|
||||
text: str, attachment_ids: list[str]) -> dt.datetime:
|
||||
"""Insert with a per-chat strictly increasing timestamp.
|
||||
|
||||
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
|
||||
chat are serialised: a message with a smaller created_at can never commit after
|
||||
a message with a bigger one, and a reader paging by cursor cannot skip it.
|
||||
"""
|
||||
cur = await conn.execute(
|
||||
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
|
||||
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
|
||||
(chat_id,))
|
||||
(created_at,) = await cur.fetchone()
|
||||
await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at, attachment_ids) "
|
||||
"VALUES (%s, %s, %s, %s, %s, %s::uuid[])",
|
||||
(message_id, chat_id, user_id, _b(text), created_at, attachment_ids))
|
||||
if attachment_ids:
|
||||
await conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, position, attachment_id) "
|
||||
"SELECT %s, a.ord - 1, a.id FROM unnest(%s::uuid[]) WITH ORDINALITY AS a(id, ord)",
|
||||
(message_id, attachment_ids))
|
||||
return created_at
|
||||
|
||||
|
||||
# Attachments a message may reference (lab 5): of THIS chat and not the invisible 'pending' intent
|
||||
# (else 404); queued/processing/ready are usable, failed or quarantined -> 409.
|
||||
# FOR SHARE keeps a concurrent transition to failed from slipping in before COMMIT.
|
||||
CHECK_ATTACHMENTS = (
|
||||
"SELECT count(*), count(*) FILTER (WHERE a.lost_at IS NULL AND a.status <> 'failed') FROM ("
|
||||
"SELECT lost_at, status FROM attachments WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status <> 'pending' "
|
||||
"FOR SHARE) a")
|
||||
|
||||
|
||||
async def _check_attachments(conn: psycopg.AsyncConnection, chat_id: str, attachment_ids: list[str]) -> str:
|
||||
if not attachment_ids:
|
||||
return "ok"
|
||||
cur = await conn.execute(CHECK_ATTACHMENTS, (attachment_ids, chat_id))
|
||||
found, usable = await cur.fetchone()
|
||||
if found != len(attachment_ids):
|
||||
return "attachment_not_found"
|
||||
return "ok" if usable == found else "attachment_conflict"
|
||||
|
||||
|
||||
def _attachment_error(status: str) -> None:
|
||||
if status == "attachment_not_found":
|
||||
raise NotFound("Вложение не найдено")
|
||||
if status == "attachment_conflict":
|
||||
raise Conflict("Вложение недоступно и не может быть прикреплено")
|
||||
|
||||
|
||||
# Claim the key. A row of a concurrent transaction that is not committed yet makes this
|
||||
# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict
|
||||
# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is
|
||||
# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired).
|
||||
CLAIM_KEY = (
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) "
|
||||
"VALUES (%s, %s, %s, %s, %s) "
|
||||
"ON CONFLICT (user_id, chat_id, key) DO UPDATE SET "
|
||||
"request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() "
|
||||
"WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) "
|
||||
"RETURNING 1")
|
||||
STORED_RESULT = (
|
||||
"SELECT k.request_hash, m.id, m.text, m.created_at, m.attachment_ids FROM idempotency_keys k "
|
||||
"JOIN messages m ON m.id = k.message_id "
|
||||
"WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s")
|
||||
|
||||
|
||||
async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str,
|
||||
ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
|
||||
"""Idempotent send: the key, the message and the stored result commit in ONE transaction.
|
||||
|
||||
* first request -> key row + message + chat clock, all or nothing (no "pending" key is
|
||||
ever left behind: a crash or 5xx rolls everything back and the client may retry);
|
||||
* same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed;
|
||||
* same key, other body -> 409; the membership check comes first, so outsiders always get 404;
|
||||
* attachments (lab 4) are checked before the key is claimed (same order as post_message_v3).
|
||||
"""
|
||||
message_id = uuid.uuid4()
|
||||
attachment_ids = list(attachment_ids)
|
||||
|
||||
async def work(conn: psycopg.AsyncConnection) -> PostResult:
|
||||
await _check_member(conn, user_id, chat_id)
|
||||
check = await _check_attachments(conn, chat_id, attachment_ids)
|
||||
if check == "ok":
|
||||
cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds))
|
||||
if await cur.fetchone() is not None:
|
||||
created_at = await _insert_message(conn, message_id, user_id, chat_id, text, attachment_ids)
|
||||
return PostResult(Message(str(message_id), chat_id, user_id, text, created_at,
|
||||
tuple(attachment_ids)), False)
|
||||
cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key))
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
_attachment_error(check)
|
||||
# The key row vanished between the two statements (expired-key cleanup): retryable.
|
||||
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
|
||||
if bytes(row[0]) != request_hash:
|
||||
raise Conflict("Idempotency-Key уже использован с другим сообщением")
|
||||
return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3],
|
||||
tuple(str(a) for a in row[4])), True)
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
POST_MESSAGE_CALL = ("SELECT o_status, o_id, o_text, o_created_at, o_attachments "
|
||||
"FROM post_message_v3(%s, %s, %s, %s, %s, %s, %s, %s::uuid[])")
|
||||
|
||||
|
||||
async def post_message_single_call(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes,
|
||||
text: str, ttl_seconds: int, attachment_ids: list[str] = ()) -> PostResult:
|
||||
"""Same contract as post_message, ONE round trip: SQL function post_message_v3 (migration 0005 =
|
||||
0003 + attachment_ids) in autocommit mode, i.e. the call is the whole transaction."""
|
||||
message_id = uuid.uuid4()
|
||||
|
||||
async def work(conn: psycopg.AsyncConnection) -> tuple:
|
||||
cur = await conn.execute(POST_MESSAGE_CALL, (user_id, chat_id, key, request_hash, message_id, _b(text),
|
||||
ttl_seconds, list(attachment_ids)))
|
||||
return await cur.fetchone()
|
||||
status, stored_id, stored_text, created_at, stored_attachments = await db.run(work, transaction=False)
|
||||
if status == "not_found":
|
||||
raise NotFound("Чат не найден")
|
||||
_attachment_error(status)
|
||||
if status == "conflict":
|
||||
raise Conflict("Idempotency-Key уже использован с другим сообщением")
|
||||
if status == "retry":
|
||||
raise Unavailable("Повторите запрос с тем же Idempotency-Key")
|
||||
return PostResult(Message(str(stored_id), chat_id, user_id, _s(stored_text), created_at,
|
||||
tuple(str(a) for a in stored_attachments)), status == "replayed")
|
||||
|
||||
|
||||
async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int:
|
||||
"""Delete expired keys in bounded batches; one replica at a time (advisory xact lock)."""
|
||||
async def work(conn: psycopg.AsyncConnection) -> int:
|
||||
cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,))
|
||||
if not (await cur.fetchone())[0]:
|
||||
return 0
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY("
|
||||
"SELECT ctid FROM idempotency_keys "
|
||||
"WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))",
|
||||
(ttl_seconds, batch))
|
||||
return cur.rowcount
|
||||
return await db.run(work)
|
||||
|
||||
|
||||
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
|
||||
# would push the cursor into a Filter and late pages would scan the whole prefix.
|
||||
# attachment_ids is a column of the message row (migration 0004): still one index range scan.
|
||||
HISTORY_FIRST = (
|
||||
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
|
||||
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
|
||||
HISTORY_AFTER = (
|
||||
"SELECT id, sender_id, text, created_at, attachment_ids FROM messages "
|
||||
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
|
||||
|
||||
|
||||
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
|
||||
async def work(conn: psycopg.AsyncConnection) -> list:
|
||||
if after is None:
|
||||
cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1))
|
||||
else:
|
||||
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
|
||||
return await cur.fetchall()
|
||||
rows = await db.run(work)
|
||||
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3], tuple(str(a) for a in r[4]))
|
||||
for r in rows[:limit]]
|
||||
return items, len(rows) > limit
|
||||
@@ -0,0 +1,182 @@
|
||||
"""One-shot S3 bootstrap: `python -m messenger.s3_bootstrap` (compose service `s3-init`).
|
||||
|
||||
Idempotent, safe to run on every `docker compose up`:
|
||||
|
||||
1. waits until the S3 endpoint answers;
|
||||
2. creates the final bucket (S3_BUCKET) and the staging bucket (S3_STAGING_BUCKET) with the ADMIN
|
||||
key pair if they do not exist yet (existing buckets and their objects are left untouched);
|
||||
3. writes the sentinel object `health/sentinel` into the final bucket (the API readiness HEADs it);
|
||||
4. proves the access model and FAILS the stand (exit 1) if it is wrong:
|
||||
* anonymous (unsigned) GET of an object and anonymous LIST are refused on both buckets;
|
||||
* api: writes staging, reads final; cannot read staging, write final, list or create buckets,
|
||||
nor read another bucket (a canary bucket created here for this check);
|
||||
* worker: reads and writes both buckets; cannot list or create buckets;
|
||||
* auditor: reads and lists both buckets; cannot write.
|
||||
Only file names and verdicts are printed, never key values.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
from .config import ConfigError, Settings, _secret, s3_settings
|
||||
from .errors import Unavailable
|
||||
from .storage import SENTINEL_KEY, Storage
|
||||
|
||||
|
||||
def say(text: str) -> None:
|
||||
print(f"s3-bootstrap: {text}", flush=True)
|
||||
|
||||
|
||||
def anonymous_status(url: str) -> int:
|
||||
"""HTTP status of an UNSIGNED request (what anyone on the network could do)."""
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=3) as response:
|
||||
return response.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
async def error_code(coro) -> str:
|
||||
"""S3 error code of a call that is expected to be refused ('' if it succeeded)."""
|
||||
try:
|
||||
await coro
|
||||
return ""
|
||||
except ClientError as exc:
|
||||
return str(exc.response.get("Error", {}).get("Code", "error"))
|
||||
|
||||
|
||||
def settings_with(keys: tuple[str, str]) -> Settings:
|
||||
base = s3_settings(credentials_required=False)
|
||||
base.update(s3_access_key_id=keys[0], s3_secret_access_key=keys[1])
|
||||
return Settings(host="", port=0, env="lab", log_level="info", shutdown_timeout=0, max_body_bytes=1024, **base)
|
||||
|
||||
|
||||
def pair(prefix: str) -> tuple[str, str]:
|
||||
return _secret(f"{prefix}ACCESS_KEY_ID", True), _secret(f"{prefix}SECRET_ACCESS_KEY", True)
|
||||
|
||||
|
||||
async def ensure_bucket(admin_s3: Storage, bucket: str) -> bool:
|
||||
code = await error_code(admin_s3.client.head_bucket(Bucket=bucket))
|
||||
if code == "":
|
||||
say(f"bucket {bucket}: exists, kept as is")
|
||||
return True
|
||||
if code in ("404", "NoSuchBucket", "NotFound"):
|
||||
code = await error_code(admin_s3.client.create_bucket(Bucket=bucket))
|
||||
if code not in ("", "BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
say(f"bucket {bucket}: create failed ({code})")
|
||||
return False
|
||||
say(f"bucket {bucket}: created")
|
||||
return True
|
||||
say(f"bucket {bucket}: head failed ({code})")
|
||||
return False
|
||||
|
||||
|
||||
async def run() -> int:
|
||||
settings = settings_with(pair("S3_ADMIN_"))
|
||||
final, staging = settings.s3_bucket, settings.s3_staging_bucket
|
||||
admin_s3 = Storage(settings)
|
||||
clients = {"api": Storage(settings_with(pair("S3_"))), "worker": Storage(settings_with(pair("S3_WORKER_"))),
|
||||
"auditor": Storage(settings_with(pair("S3_AUDITOR_")))}
|
||||
for s in (admin_s3, *clients.values()):
|
||||
await s.open()
|
||||
try:
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
await admin_s3._call("list_buckets", admin_s3.client.list_buckets())
|
||||
break
|
||||
except Unavailable:
|
||||
if time.monotonic() > deadline:
|
||||
say("S3 endpoint does not answer within 60 s")
|
||||
return 1
|
||||
await asyncio.sleep(1)
|
||||
for bucket in (final, staging):
|
||||
if not await ensure_bucket(admin_s3, bucket):
|
||||
return 1
|
||||
await admin_s3.put(SENTINEL_KEY, b"ok\n", "text/plain", {})
|
||||
await admin_s3.client.put_object(Bucket=staging, Key=SENTINEL_KEY, Body=b"ok\n")
|
||||
say(f"sentinel {SENTINEL_KEY}: written")
|
||||
canary = final + "-canary"
|
||||
if await error_code(admin_s3.client.head_bucket(Bucket=canary)):
|
||||
await admin_s3.client.create_bucket(Bucket=canary)
|
||||
await admin_s3.client.put_object(Bucket=canary, Key="canary", Body=b"secret of another bucket")
|
||||
|
||||
failures = []
|
||||
base = settings.s3_endpoint.rstrip("/")
|
||||
for bucket in (final, staging):
|
||||
for what, url in ((f"anonymous GET object of {bucket}", f"{base}/{bucket}/{SENTINEL_KEY}"),
|
||||
(f"anonymous LIST of {bucket}", f"{base}/{bucket}?list-type=2")):
|
||||
status = anonymous_status(url)
|
||||
verdict = "refused" if status in (401, 403) else "ALLOWED"
|
||||
say(f"{what}: HTTP {status} -> {verdict}")
|
||||
if verdict != "refused":
|
||||
failures.append(what)
|
||||
|
||||
probe = "health/bootstrap-probe"
|
||||
|
||||
def c(who: str):
|
||||
return clients[who].client
|
||||
|
||||
# (what, coroutine, allowed?) - every refusal must be a real AccessDenied, not an outage.
|
||||
checks = [
|
||||
("api: write staging", c("api").put_object(Bucket=staging, Key=probe, Body=b"p"), True),
|
||||
("api: delete staging (janitor)", c("api").delete_object(Bucket=staging, Key=probe), True),
|
||||
("api: read staging", c("api").head_object(Bucket=staging, Key=SENTINEL_KEY), False),
|
||||
("api: read final", c("api").head_object(Bucket=final, Key=SENTINEL_KEY), True),
|
||||
("api: write final", c("api").put_object(Bucket=final, Key=probe, Body=b"p"), False),
|
||||
("api: list final", c("api").list_objects_v2(Bucket=final, MaxKeys=1), False),
|
||||
("api: list staging", c("api").list_objects_v2(Bucket=staging, MaxKeys=1), False),
|
||||
("api: create bucket", c("api").create_bucket(Bucket=final + "-x"), False),
|
||||
("api: read another bucket", c("api").head_object(Bucket=canary, Key="canary"), False),
|
||||
("worker: read staging", c("worker").head_object(Bucket=staging, Key=SENTINEL_KEY), True),
|
||||
("worker: write final", c("worker").put_object(Bucket=final, Key=probe, Body=b"p"), True),
|
||||
("worker: delete final", c("worker").delete_object(Bucket=final, Key=probe), True),
|
||||
("worker: write staging", c("worker").put_object(Bucket=staging, Key=probe, Body=b"p"), True),
|
||||
("worker: delete staging", c("worker").delete_object(Bucket=staging, Key=probe), True),
|
||||
("worker: list final", c("worker").list_objects_v2(Bucket=final, MaxKeys=1), False),
|
||||
("worker: create bucket", c("worker").create_bucket(Bucket=final + "-x"), False),
|
||||
("worker: read another bucket", c("worker").head_object(Bucket=canary, Key="canary"), False),
|
||||
("auditor: list final", c("auditor").list_objects_v2(Bucket=final, MaxKeys=1), True),
|
||||
("auditor: list staging", c("auditor").list_objects_v2(Bucket=staging, MaxKeys=1), True),
|
||||
("auditor: read final", c("auditor").head_object(Bucket=final, Key=SENTINEL_KEY), True),
|
||||
("auditor: write final", c("auditor").put_object(Bucket=final, Key=probe, Body=b"p"), False),
|
||||
("auditor: delete staging", c("auditor").delete_object(Bucket=staging, Key=SENTINEL_KEY), False),
|
||||
]
|
||||
for what, coro, allowed in checks:
|
||||
code = await error_code(coro)
|
||||
ok = (code == "") == allowed and code in ("", "AccessDenied", "403")
|
||||
say(f"{what}: {'allowed' if code == '' else 'refused (' + code + ')'} -> {'OK' if ok else 'WRONG'}")
|
||||
if not ok:
|
||||
failures.append(what)
|
||||
visible = [b["Name"] for b in (await c("api").list_buckets()).get("Buckets", [])]
|
||||
say(f"api: buckets visible in ListBuckets: {visible} -> {'OK' if canary not in visible else 'WRONG'}")
|
||||
if canary in visible:
|
||||
failures.append("api sees another bucket")
|
||||
if failures:
|
||||
say("access model is WRONG: " + ", ".join(failures))
|
||||
return 1
|
||||
say("done: private buckets, least-privilege keys verified")
|
||||
return 0
|
||||
finally:
|
||||
for s in (admin_s3, *clients.values()):
|
||||
await s.close()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
return asyncio.run(run())
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
os.environ.setdefault("PYTHONUNBUFFERED", "1")
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,76 @@
|
||||
"""Common runtime of the background processes (relay, worker): signals, heartbeat file for the
|
||||
Docker healthcheck, failure injection that fires once per job and leaves an audit row."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from . import jobs
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
|
||||
log = logging.getLogger("messenger.service")
|
||||
|
||||
HEARTBEAT_FILE = Path(os.environ.get("HEARTBEAT_FILE", "/tmp/heartbeat"))
|
||||
CRASH_EXIT_CODE = 70 # os._exit: no cleanup, no ack, no COMMIT - like SIGKILL in the middle of work
|
||||
|
||||
|
||||
def beat() -> None:
|
||||
"""Healthcheck input: `python -m messenger.healthcheck --file /tmp/heartbeat --max-age 30`."""
|
||||
try:
|
||||
HEARTBEAT_FILE.write_text(str(time.time()), encoding="ascii")
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def watch_connection(connection) -> asyncio.Event:
|
||||
"""Event set when an AMQP connection is closed by ANY side. aio-pika's `is_closed` does not turn
|
||||
True when the broker drops the connection (found by the broker-restart test: the worker kept
|
||||
"running" without a consumer), so the close callback is the signal to reconnect."""
|
||||
lost = asyncio.Event()
|
||||
connection.close_callbacks.add(lambda *_: lost.set())
|
||||
return lost
|
||||
|
||||
|
||||
def stop_event() -> asyncio.Event:
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
try:
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
except (NotImplementedError, RuntimeError):
|
||||
pass # Windows unit tests: no signal handlers
|
||||
return stop
|
||||
|
||||
|
||||
async def sleep_or_stop(stop: asyncio.Event, seconds: float) -> None:
|
||||
try:
|
||||
await asyncio.wait_for(stop.wait(), seconds)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
|
||||
async def crash_once(settings: Settings, db: Database, point: str, job_id: str, actor: str, **detail) -> None:
|
||||
"""Experiments only (FAULT_POINTS): kill THIS process at `point`, once per job (and per `detail`),
|
||||
across restarts and replicas. The decision is an audit row committed before the crash, so the
|
||||
restarted process (or another replica) continues normally and the evidence stays in job_events."""
|
||||
if point not in settings.fault_points:
|
||||
return
|
||||
|
||||
async def record(conn) -> bool:
|
||||
cur = await conn.execute(
|
||||
"SELECT 1 FROM job_events WHERE job_id = %s AND event = 'fault_injected' AND detail->>'point' = %s "
|
||||
"AND detail @> %s::jsonb", (job_id, point, jobs.json.dumps({k: str(v) for k, v in detail.items()})))
|
||||
if await cur.fetchone():
|
||||
return False
|
||||
await jobs.event(conn, job_id, "fault_injected", actor, point=point, **{k: str(v) for k, v in detail.items()})
|
||||
return True
|
||||
|
||||
if await db.run(record):
|
||||
log.warning("fault_injected_crash", extra={"fields": {"point": point, "job_id": job_id, **detail}})
|
||||
logging.shutdown()
|
||||
os._exit(CRASH_EXIT_CODE)
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Server-side sessions behind an opaque Bearer token.
|
||||
|
||||
* Token: `secrets.token_urlsafe(32)` = 256 random bits. Only SHA-256(token) is
|
||||
stored, so a leaked database/AOF does not contain usable tokens. A fast hash is
|
||||
enough for a 256-bit random value; passwords need a slow KDF (passwords.py).
|
||||
* Expiry is absolute: `expires_at = login time + SESSION_TTL_SECONDS`. Every lookup
|
||||
compares it with the current time, and a restart never extends it.
|
||||
|
||||
Two interchangeable stores (SESSION_STORE):
|
||||
|
||||
* `valkey` (compose default): the ONLY copy of a session is a Valkey key with
|
||||
`PXAT expires_at`, persisted by AOF with `appendfsync always`. There is no second
|
||||
copy (no cache over PostgreSQL), so a logout cannot be "forgotten" and a revoked
|
||||
token cannot come back after Valkey fails and recovers. If Valkey is unreachable
|
||||
a session cannot be checked: protected requests and logout get 503, never access.
|
||||
* `postgres`: table `sessions` (migrations/0001_init.sql).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from typing import Protocol
|
||||
|
||||
from redis import exceptions as redis_errors
|
||||
from redis.asyncio import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .timefmt import UTC, from_us, to_us
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def token_hash(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("ascii")).digest()
|
||||
|
||||
|
||||
def now() -> dt.datetime:
|
||||
return dt.datetime.now(UTC)
|
||||
|
||||
|
||||
class SessionStore(Protocol):
|
||||
name: str
|
||||
|
||||
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: ...
|
||||
async def user_id_for(self, token: str) -> str | None: ...
|
||||
async def revoke(self, token: str) -> bool: ...
|
||||
async def ping(self) -> bool: ...
|
||||
async def close(self) -> None: ...
|
||||
|
||||
|
||||
class PostgresSessionStore:
|
||||
name = "postgres"
|
||||
|
||||
def __init__(self, db: Database):
|
||||
self.db = db
|
||||
|
||||
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
|
||||
token = new_token()
|
||||
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
|
||||
|
||||
async def work(conn) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
|
||||
(token_hash(token), user_id, expires_at))
|
||||
# Housekeeping: drop this user's expired sessions (bounded by the user's own rows).
|
||||
await conn.execute("DELETE FROM sessions WHERE user_id = %s AND expires_at <= %s", (user_id, now()))
|
||||
await self.db.run(work)
|
||||
return token, expires_at
|
||||
|
||||
async def user_id_for(self, token: str) -> str | None:
|
||||
async def work(conn):
|
||||
cur = await conn.execute(
|
||||
"SELECT user_id FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
|
||||
return await cur.fetchone()
|
||||
row = await self.db.run(work)
|
||||
return str(row[0]) if row else None
|
||||
|
||||
async def revoke(self, token: str) -> bool:
|
||||
async def work(conn) -> bool:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
|
||||
return cur.rowcount == 1
|
||||
return await self.db.run(work)
|
||||
|
||||
async def ping(self) -> bool:
|
||||
return True # covered by the database readiness check
|
||||
|
||||
async def close(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class ValkeySessionStore:
|
||||
name = "valkey"
|
||||
PREFIX = "session:"
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
self.timeout = settings.cache_timeout_seconds
|
||||
self.client = Redis.from_url(
|
||||
settings.cache_url,
|
||||
socket_timeout=self.timeout,
|
||||
socket_connect_timeout=self.timeout,
|
||||
retry=Retry(NoBackoff(), 0), # no hidden retries: a failure is reported at once
|
||||
decode_responses=False,
|
||||
)
|
||||
|
||||
def _key(self, token: str) -> str:
|
||||
return self.PREFIX + token_hash(token).hex()
|
||||
|
||||
async def _call(self, coro):
|
||||
try:
|
||||
async with asyncio.timeout(self.timeout * 2):
|
||||
return await coro
|
||||
except (redis_errors.ConnectionError, redis_errors.TimeoutError, TimeoutError, OSError) as exc:
|
||||
log.warning("session_store_unavailable", extra={"fields": {"error": type(exc).__name__}})
|
||||
raise Unavailable("Хранилище сессий временно недоступно") from None
|
||||
|
||||
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
|
||||
token = new_token()
|
||||
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
|
||||
value = json.dumps({"user_id": user_id, "expires_at_us": to_us(expires_at)}).encode()
|
||||
# PXAT = absolute expiry time: survives restarts (AOF) without being extended.
|
||||
await self._call(self.client.set(self._key(token), value, pxat=to_us(expires_at) // 1000))
|
||||
return token, expires_at
|
||||
|
||||
async def user_id_for(self, token: str) -> str | None:
|
||||
raw = await self._call(self.client.get(self._key(token)))
|
||||
if raw is None:
|
||||
return None
|
||||
data = json.loads(raw)
|
||||
if from_us(data["expires_at_us"]) <= now():
|
||||
return None # do not rely only on Valkey's lazy expiry
|
||||
return data["user_id"]
|
||||
|
||||
async def revoke(self, token: str) -> bool:
|
||||
return await self._call(self.client.delete(self._key(token))) == 1
|
||||
|
||||
async def ping(self) -> bool:
|
||||
try:
|
||||
return bool(await self._call(self.client.ping()))
|
||||
except Unavailable:
|
||||
return False
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.client.aclose()
|
||||
|
||||
|
||||
def build_store(settings: Settings, db: Database) -> SessionStore:
|
||||
return ValkeySessionStore(settings) if settings.session_store == "valkey" else PostgresSessionStore(db)
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Private S3 bucket (SeaweedFS in compose) through aiobotocore, with a hard deadline per call.
|
||||
|
||||
* Credentials: the API's OWN key pair (S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE ->
|
||||
Docker secrets). SeaweedFS grants it only Read + Write on this one bucket: no List, no
|
||||
Admin (cannot create/delete buckets or read other buckets). The admin pair is given only
|
||||
to the S3 server and the one-shot bootstrap (infra/seaweedfs, messenger.s3_bootstrap).
|
||||
* Deadlines: botocore connect/read timeouts AND an asyncio deadline around every call, so a
|
||||
paused or unreachable S3 costs at most S3_TIMEOUT_SECONDS, then 503 `unavailable`. Unlike a
|
||||
libpq query (db.py), an aiohttp request can be cancelled safely, so the call is cancelled.
|
||||
* ONE retry inside botocore (total_max_attempts=2), within the same asyncio deadline: after S3
|
||||
restarts, pooled keep-alive connections to the old process are dead and the first request on
|
||||
such a connection fails although S3 is up again (found by test_s3_outage_..._stop on a clean
|
||||
clone). Every call here is idempotent PER KEY (PUT of the same bytes to the same server-chosen
|
||||
key, GET, HEAD, DELETE), so repeating it cannot create a second attachment.
|
||||
* Checksums: Content-MD5 is sent with every PUT, so S3 rejects bytes damaged on the way.
|
||||
botocore's newer default (aws-chunked CRC32 trailers) is switched off ("when_required"):
|
||||
plain, widely supported S3 requests.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
from aiobotocore.config import AioConfig
|
||||
from aiobotocore.session import get_session
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from .config import Settings
|
||||
from .errors import Unavailable
|
||||
|
||||
log = logging.getLogger("messenger.storage")
|
||||
|
||||
MISSING_CODES = {"NoSuchKey", "404", "NotFound"}
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""The key does not exist in the bucket (definitive answer of S3, not an outage)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ObjectHead:
|
||||
size: int
|
||||
content_type: str
|
||||
metadata: dict[str, str]
|
||||
last_modified_ts: float # POSIX seconds
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ListedObject:
|
||||
key: str
|
||||
size: int
|
||||
last_modified_ts: float # POSIX seconds
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, settings: Settings, *, access_key: str | None = None, secret_key: str | None = None,
|
||||
bucket: str | None = None):
|
||||
# Lab 5: one client per bucket - the final bucket (S3_BUCKET) or the staging bucket.
|
||||
self.bucket = bucket or settings.s3_bucket
|
||||
self.timeout = settings.s3_timeout_seconds
|
||||
self._endpoint = settings.s3_endpoint
|
||||
self._region = settings.s3_region
|
||||
self._access_key = access_key or settings.s3_access_key_id
|
||||
self._secret_key = secret_key or settings.s3_secret_access_key
|
||||
self._config = AioConfig(
|
||||
connect_timeout=settings.s3_connect_timeout_seconds,
|
||||
read_timeout=settings.s3_timeout_seconds,
|
||||
retries={"total_max_attempts": 2, "mode": "standard"},
|
||||
s3={"addressing_style": "path"},
|
||||
signature_version="s3v4",
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
max_pool_connections=settings.s3_max_connections,
|
||||
)
|
||||
self._stack: contextlib.AsyncExitStack | None = None
|
||||
self._client = None
|
||||
|
||||
async def open(self) -> None:
|
||||
self._stack = contextlib.AsyncExitStack()
|
||||
session = get_session()
|
||||
self._client = await self._stack.enter_async_context(session.create_client(
|
||||
"s3", endpoint_url=self._endpoint, region_name=self._region,
|
||||
aws_access_key_id=self._access_key, aws_secret_access_key=self._secret_key, config=self._config))
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._stack is not None:
|
||||
await self._stack.aclose()
|
||||
self._stack = None
|
||||
|
||||
@property
|
||||
def client(self):
|
||||
assert self._client is not None, "Storage.open() was not called"
|
||||
return self._client
|
||||
|
||||
async def _call(self, what: str, coro, deadline: float | None = None):
|
||||
"""Await one S3 call under the deadline; outages become Unavailable (503)."""
|
||||
try:
|
||||
return await asyncio.wait_for(coro, deadline or self.timeout)
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code in MISSING_CODES:
|
||||
raise ObjectMissing() from None
|
||||
log.warning("s3_error", extra={"fields": {"op": what, "code": code}})
|
||||
raise Unavailable("Хранилище файлов временно недоступно") from None
|
||||
except (asyncio.TimeoutError, BotoCoreError, OSError) as exc:
|
||||
log.warning("s3_unavailable", extra={"fields": {"op": what, "error": type(exc).__name__}})
|
||||
raise Unavailable("Хранилище файлов временно недоступно") from None
|
||||
|
||||
async def put(self, key: str, data: bytes | bytearray, content_type: str, metadata: dict[str, str]) -> None:
|
||||
md5 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode("ascii")
|
||||
# Time budget grows with the size: 10 MiB over a local network takes well under a second.
|
||||
deadline = self.timeout + len(data) / (8 * 1024 * 1024)
|
||||
await self._call("put", self.client.put_object(
|
||||
Bucket=self.bucket, Key=key, Body=data, ContentType=content_type, ContentLength=len(data),
|
||||
ContentMD5=md5, Metadata=metadata), deadline)
|
||||
|
||||
async def head(self, key: str) -> ObjectHead:
|
||||
response = await self._call("head", self.client.head_object(Bucket=self.bucket, Key=key))
|
||||
return ObjectHead(int(response["ContentLength"]), response.get("ContentType", ""),
|
||||
dict(response.get("Metadata", {})), response["LastModified"].timestamp())
|
||||
|
||||
async def open_stream(self, key: str) -> tuple[int, AsyncIterator[bytes]]:
|
||||
"""Start a GET: (Content-Length, async iterator of chunks). The headers arrive under the
|
||||
deadline; every chunk read afterwards has its own deadline (a stalled S3 aborts the
|
||||
download instead of hanging the connection)."""
|
||||
response = await self._call("get", self.client.get_object(Bucket=self.bucket, Key=key))
|
||||
body = response["Body"]
|
||||
size = int(response["ContentLength"])
|
||||
|
||||
async def chunks() -> AsyncIterator[bytes]:
|
||||
try:
|
||||
while True:
|
||||
chunk = await self._call("get_body", body.read(64 * 1024))
|
||||
if not chunk:
|
||||
return
|
||||
yield chunk
|
||||
finally:
|
||||
body.close()
|
||||
return size, chunks()
|
||||
|
||||
async def read_all(self, key: str, limit: int) -> bytes:
|
||||
"""Whole object (reconciler checksum verification); refuses objects above `limit`."""
|
||||
size, chunks = await self.open_stream(key)
|
||||
if size > limit:
|
||||
raise Unavailable("Объект больше допустимого размера")
|
||||
parts = [chunk async for chunk in chunks]
|
||||
return b"".join(parts)
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
"""Idempotent: deleting a missing key is a success in S3."""
|
||||
await self._call("delete", self.client.delete_object(Bucket=self.bucket, Key=key))
|
||||
|
||||
async def list_page(self, prefix: str, start_after: str | None, page_size: int) -> tuple[list[ListedObject], bool]:
|
||||
"""One ListObjectsV2 page in key order (needs the List permission: reconciler only)."""
|
||||
params = {"Bucket": self.bucket, "Prefix": prefix, "MaxKeys": page_size}
|
||||
if start_after:
|
||||
params["StartAfter"] = start_after
|
||||
response = await self._call("list", self.client.list_objects_v2(**params))
|
||||
items = [ListedObject(o["Key"], int(o["Size"]), o["LastModified"].timestamp())
|
||||
for o in response.get("Contents", [])]
|
||||
return items, bool(response.get("IsTruncated"))
|
||||
|
||||
async def ping(self, timeout: float = 1.0) -> bool:
|
||||
"""Readiness: HEAD of the sentinel object written by the bootstrap. Proves that S3
|
||||
answers, the bucket exists and the API credentials are accepted (Read permission)."""
|
||||
try:
|
||||
await self._call("ping", self.client.head_object(Bucket=self.bucket, Key=SENTINEL_KEY), timeout)
|
||||
return True
|
||||
except (Unavailable, ObjectMissing):
|
||||
return False
|
||||
|
||||
|
||||
SENTINEL_KEY = "health/sentinel"
|
||||
# Lab 4 originals: o/<random 128-bit hex> in the final bucket (still readable).
|
||||
OBJECT_PREFIX = "o/"
|
||||
# Lab 5: s/<random 128-bit hex> in the STAGING bucket (written by the API before 202) and
|
||||
# a/<attachment id>/<attempt token>/<original|thumbnail|crop|optimized> in the final bucket
|
||||
# (written by the worker; the token makes every attempt write its own keys).
|
||||
STAGING_PREFIX = "s/"
|
||||
|
||||
|
||||
def attempt_key(attachment_id: str, token: str, name: str) -> str:
|
||||
return f"a/{attachment_id}/{token}/{name}"
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Conversions between PostgreSQL timestamptz values, cursor integers and RFC 3339 strings."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
|
||||
UTC = dt.timezone.utc
|
||||
_EPOCH = dt.datetime(1970, 1, 1, tzinfo=UTC)
|
||||
|
||||
|
||||
def to_us(value: dt.datetime) -> int:
|
||||
"""Microseconds since the Unix epoch (exact: timestamptz has microsecond precision)."""
|
||||
delta = value - _EPOCH
|
||||
return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
|
||||
|
||||
|
||||
def from_us(us: int) -> dt.datetime:
|
||||
return _EPOCH + dt.timedelta(microseconds=us)
|
||||
|
||||
|
||||
def rfc3339(value: dt.datetime) -> str:
|
||||
"""RFC 3339 UTC with fixed microsecond precision, e.g. 2026-10-12T12:00:00.000001Z."""
|
||||
return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Strict request validation according to contracts/openapi.json + contracts/README.md.
|
||||
|
||||
Written by hand instead of a schema library so that every rule is visible:
|
||||
extra fields, wrong JSON types, whitespace-only strings and lengths counted in
|
||||
Unicode code points all produce 400 invalid_request.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import unicodedata
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
from .errors import InvalidRequest
|
||||
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case.
|
||||
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
|
||||
LIMIT_RE = re.compile(r"[0-9]{1,3}")
|
||||
# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}.
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
SURROGATES_RE = re.compile("[\ud800-\udfff]")
|
||||
|
||||
DEFAULT_LIMIT = 50
|
||||
MAX_LIMIT = 100
|
||||
|
||||
|
||||
def _reject_constant(name: str) -> Any:
|
||||
# Python's json accepts NaN/Infinity, which are not valid JSON.
|
||||
raise ValueError(f"invalid JSON constant {name}")
|
||||
|
||||
|
||||
def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
# {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins".
|
||||
result = dict(pairs)
|
||||
if len(result) != len(pairs):
|
||||
raise ValueError("duplicate key in JSON object")
|
||||
return result
|
||||
|
||||
|
||||
def parse_json_object(body: bytes) -> dict[str, Any]:
|
||||
"""Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400)."""
|
||||
try:
|
||||
text = body.decode("utf-8")
|
||||
value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys)
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None
|
||||
if not isinstance(value, dict):
|
||||
raise InvalidRequest("Тело запроса должно быть JSON-объектом")
|
||||
return value
|
||||
|
||||
|
||||
def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[str] = frozenset()) -> None:
|
||||
extra = set(data) - required - optional
|
||||
if extra:
|
||||
raise InvalidRequest("Неизвестные поля: " + ", ".join(sorted(extra)))
|
||||
missing = required - set(data)
|
||||
if missing:
|
||||
raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing)))
|
||||
|
||||
|
||||
def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1, allow_blank: bool = False) -> str:
|
||||
value = data[name]
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest(f"Поле {name} должно быть строкой")
|
||||
if SURROGATES_RE.search(value):
|
||||
# "\ud800" escapes decode to lone surrogates that cannot be encoded as UTF-8.
|
||||
raise InvalidRequest(f"Поле {name} содержит недопустимые символы")
|
||||
# len() of a Python str is the number of Unicode code points, as the contract requires.
|
||||
if not min_len <= len(value) <= max_len:
|
||||
raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов")
|
||||
if value.strip() == "" and not allow_blank:
|
||||
raise InvalidRequest(f"Поле {name} не может состоять только из пробелов")
|
||||
return value
|
||||
|
||||
|
||||
def username_field(data: dict[str, Any]) -> str:
|
||||
value = data["username"]
|
||||
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
|
||||
raise InvalidRequest("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
return value
|
||||
|
||||
|
||||
def password_field(data: dict[str, Any]) -> str:
|
||||
"""12-128 Unicode code points, not whitespace-only; any other character (':' too) is allowed."""
|
||||
value = data["password"]
|
||||
if not isinstance(value, str):
|
||||
raise InvalidRequest("Поле password должно быть строкой")
|
||||
if SURROGATES_RE.search(value):
|
||||
raise InvalidRequest("Поле password содержит недопустимые символы")
|
||||
if not 12 <= len(value) <= 128:
|
||||
raise InvalidRequest("Длина пароля должна быть от 12 до 128 символов")
|
||||
if value.strip() == "":
|
||||
raise InvalidRequest("Пароль не может состоять только из пробелов")
|
||||
return value
|
||||
|
||||
|
||||
def parse_uuid(value: Any, what: str = "UUID") -> str:
|
||||
if not isinstance(value, str) or UUID_RE.fullmatch(value) is None:
|
||||
raise InvalidRequest(f"Неверный {what}")
|
||||
return value.lower()
|
||||
|
||||
|
||||
def member_ids_field(data: dict[str, Any]) -> list[str]:
|
||||
value = data["member_ids"]
|
||||
if not isinstance(value, list):
|
||||
raise InvalidRequest("member_ids должен быть массивом")
|
||||
if not 1 <= len(value) <= 100:
|
||||
raise InvalidRequest("member_ids должен содержать от 1 до 100 элементов")
|
||||
ids = [parse_uuid(item, "UUID участника") for item in value]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise InvalidRequest("member_ids содержит дубликаты")
|
||||
return ids
|
||||
|
||||
|
||||
def attachment_ids_field(data: dict[str, Any]) -> list[str]:
|
||||
"""Optional `attachment_ids` (lab 4+): absent and [] are the same message (contracts/README.md);
|
||||
null or any other non-array type is 400."""
|
||||
value = data.get("attachment_ids", [])
|
||||
if not isinstance(value, list):
|
||||
raise InvalidRequest("attachment_ids должен быть массивом")
|
||||
if len(value) > 10:
|
||||
raise InvalidRequest("attachment_ids содержит больше 10 элементов")
|
||||
ids = [parse_uuid(item, "UUID вложения") for item in value]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise InvalidRequest("attachment_ids содержит дубликаты")
|
||||
return ids
|
||||
|
||||
|
||||
def idempotency_key(raw: str | None) -> str:
|
||||
if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None:
|
||||
raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}")
|
||||
return raw
|
||||
|
||||
|
||||
def single_query_param(query: list[tuple[str, str]], name: str) -> str | None:
|
||||
values = [v for k, v in query if k == name]
|
||||
if len(values) > 1:
|
||||
raise InvalidRequest(f"Параметр {name} указан несколько раз")
|
||||
return values[0] if values else None
|
||||
|
||||
|
||||
def parse_limit(raw: str | None) -> int:
|
||||
if raw is None:
|
||||
return DEFAULT_LIMIT
|
||||
if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
|
||||
raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}")
|
||||
return int(raw)
|
||||
|
||||
|
||||
# Lab 4: the display name of an attachment. Never used as a path or an S3 key, but it is shown
|
||||
# to other users, so characters that change how a name LOOKS are refused: control characters
|
||||
# (C0, DEL, C1 such as U+0085), line/paragraph separators, unpaired surrogates and bidi controls
|
||||
# ("photo<U+202E>gnp.exe" would display as "photoexe.png").
|
||||
BIDI_CONTROLS = frozenset("\u061c\u200e\u200f\u202a\u202b\u202c\u202d\u202e\u2066\u2067\u2068\u2069")
|
||||
FORBIDDEN_CATEGORIES = frozenset({"Cc", "Cs", "Zl", "Zp"})
|
||||
|
||||
|
||||
def query_items(raw: bytes) -> list[tuple[str, str]]:
|
||||
"""Strict query parsing: percent-encoded UTF-8 only (Starlette would replace bad bytes by U+FFFD)."""
|
||||
try:
|
||||
return parse_qsl(raw.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict")
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise InvalidRequest("Параметры запроса должны быть в UTF-8 с percent-encoding") from None
|
||||
|
||||
|
||||
def filename_param(query: list[tuple[str, str]]) -> str:
|
||||
value = single_query_param(query, "filename")
|
||||
if value is None:
|
||||
raise InvalidRequest("Параметр filename обязателен")
|
||||
if not 1 <= len(value) <= 255:
|
||||
raise InvalidRequest("Длина filename должна быть от 1 до 255 символов")
|
||||
if value.strip() == "" or value in {".", ".."}:
|
||||
raise InvalidRequest("filename не может состоять только из пробелов или точек")
|
||||
if "/" in value or "\\" in value:
|
||||
raise InvalidRequest("filename не может содержать / или \\")
|
||||
for char in value:
|
||||
if unicodedata.category(char) in FORBIDDEN_CATEGORIES or char in BIDI_CONTROLS:
|
||||
raise InvalidRequest("filename содержит управляющие символы")
|
||||
return value
|
||||
|
||||
|
||||
ATTACHMENT_VARIANTS = ("original", "thumbnail", "crop", "optimized")
|
||||
|
||||
|
||||
def variant_param(query: list[tuple[str, str]]) -> str:
|
||||
"""`variant` of GET /attachments/{id}/content: original (default) or a lab 5 variant name.
|
||||
Any other value is 400; a known name that a ready attachment lacks is 404 (handler)."""
|
||||
value = single_query_param(query, "variant")
|
||||
if value is None:
|
||||
return "original"
|
||||
if value in ATTACHMENT_VARIANTS:
|
||||
return value
|
||||
raise InvalidRequest("variant должен быть original, thumbnail, crop или optimized")
|
||||
@@ -0,0 +1,340 @@
|
||||
"""Image worker: `python -m messenger.worker` (compose service `worker`, separate from the API).
|
||||
|
||||
One delivery of a job message:
|
||||
|
||||
decode + validate (schema v1; invalid -> reject -> dead-letter queue)
|
||||
-> CLAIM in PostgreSQL (jobs.claim): finished/leased/not due/stale -> no-op, ack
|
||||
-> lease heartbeat every JOB_LEASE_SECONDS/3 while the attempt runs
|
||||
-> read the source (staging copy or published original), check size + SHA-256 against the DB
|
||||
-> variants in a child process (imaging.py: pixel, memory and time limits)
|
||||
-> PUT the original (first publication) and the variants under keys of THIS attempt
|
||||
-> ONE transaction: variants + attachment ready, fenced by the lease token (jobs.complete)
|
||||
-> ack -> delete the staging copy (marked in the DB first, then S3).
|
||||
|
||||
The ack comes only after the durable effect (the COMMIT). A crash anywhere before it makes
|
||||
RabbitMQ deliver the message again; the job row then says what is left to do. Failures: a
|
||||
permanent one (invalid_image, image_too_large, source_missing/corrupt) fails the job at once, a
|
||||
transient one (S3 down, timeout, killed decoder) is retried with backoff via the outbox, at most
|
||||
JOB_MAX_ATTEMPTS claims in total.
|
||||
|
||||
Concurrency: RabbitMQ prefetch = WORKER_CONCURRENCY unacked messages per worker, and at most
|
||||
WORKER_CONCURRENCY image child processes. Graceful stop (SIGTERM): stop consuming, let running
|
||||
attempts finish for WORKER_SHUTDOWN_SECONDS, give the rest back (jobs.release).
|
||||
|
||||
The worker also runs the cleanup of the pipeline objects (staging copies that are no longer
|
||||
needed, objects of attempts that never published) - always decided under a row lock.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
|
||||
import aio_pika
|
||||
from aio_pika.abc import AbstractChannel, AbstractConnection, AbstractIncomingMessage, AbstractQueue
|
||||
|
||||
from . import broker, imaging, jobs, logging_setup, service
|
||||
from .config import ConfigError, Settings
|
||||
from .db import Database
|
||||
from .errors import Unavailable
|
||||
from .imaging import ProcessingError
|
||||
from .storage import ObjectMissing, Storage
|
||||
|
||||
log = logging.getLogger("messenger.worker")
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.name = settings.instance_id
|
||||
self.actor = "worker:" + self.name
|
||||
self.db = Database(settings, name="messenger-worker")
|
||||
self.staging = Storage(settings, bucket=settings.s3_staging_bucket)
|
||||
self.final = Storage(settings)
|
||||
self.slots = asyncio.Semaphore(settings.worker_concurrency)
|
||||
self.tasks: set[asyncio.Task] = set()
|
||||
self.connection: AbstractConnection | None = None
|
||||
self.channel: AbstractChannel | None = None
|
||||
self.queue: AbstractQueue | None = None
|
||||
self.consumer_tag: str | None = None
|
||||
self.lost: asyncio.Event | None = None
|
||||
|
||||
def connected(self) -> bool:
|
||||
return (self.connection is not None and self.lost is not None and not self.lost.is_set()
|
||||
and self.channel is not None and not self.channel.is_closed)
|
||||
|
||||
# ---------------------------------------------------------------- broker
|
||||
async def connect(self) -> None:
|
||||
self.connection = await aio_pika.connect(
|
||||
self.settings.broker_url, timeout=5,
|
||||
client_properties={"connection_name": "worker@" + self.name})
|
||||
self.lost = service.watch_connection(self.connection)
|
||||
self.channel = await self.connection.channel()
|
||||
# Prefetch: the broker keeps at most this many unacked messages in flight to this worker.
|
||||
await self.channel.set_qos(prefetch_count=self.settings.worker_concurrency)
|
||||
self.queue = await self.channel.get_queue(broker.QUEUE, ensure=False)
|
||||
self.consumer_tag = await self.queue.consume(self.on_message, no_ack=False)
|
||||
log.info("consuming", extra={"fields": {"queue": broker.QUEUE, "prefetch": self.settings.worker_concurrency}})
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
if self.connection is not None:
|
||||
try:
|
||||
await self.connection.close()
|
||||
except Exception: # noqa: BLE001 - already broken
|
||||
pass
|
||||
self.connection = self.channel = self.queue = self.lost = None
|
||||
self.consumer_tag = None
|
||||
|
||||
async def on_message(self, message: AbstractIncomingMessage) -> None:
|
||||
task = asyncio.create_task(self.deliver(message))
|
||||
self.tasks.add(task)
|
||||
task.add_done_callback(self.tasks.discard)
|
||||
|
||||
@staticmethod
|
||||
async def settle(action) -> None:
|
||||
"""ack/nack/reject; the channel may be gone (broker restart): then the broker requeues the
|
||||
message by itself and the redelivery is a no-op."""
|
||||
try:
|
||||
await action
|
||||
except Exception as exc: # noqa: BLE001
|
||||
log.warning("settle_failed", extra={"fields": {"error": type(exc).__name__}})
|
||||
|
||||
async def deliver(self, message: AbstractIncomingMessage) -> None:
|
||||
try:
|
||||
body = broker.decode(message.body)
|
||||
except broker.BadMessage as exc:
|
||||
log.error("bad_message", extra={"fields": {"reason": str(exc), "message_id": message.message_id}})
|
||||
await self.settle(message.reject(requeue=False)) # -> messenger.attachment-jobs.dlq
|
||||
return
|
||||
try:
|
||||
async with self.slots:
|
||||
outcome = await self.process(body["job_id"])
|
||||
except Unavailable:
|
||||
# PostgreSQL (or S3 during a DB step) unavailable: nothing decided. Back to the queue
|
||||
# after a pause; a claim that did happen is protected by its lease.
|
||||
log.warning("job_deferred", extra={"fields": {"job_id": body["job_id"]}})
|
||||
await asyncio.sleep(2)
|
||||
await self.settle(message.nack(requeue=True))
|
||||
return
|
||||
except asyncio.CancelledError:
|
||||
# Shutdown: back to the queue. If the job was claimed it has been released (new outbox
|
||||
# row), and one of the two deliveries becomes a no-op.
|
||||
await self.settle(asyncio.shield(message.nack(requeue=True)))
|
||||
raise
|
||||
log.info("job_done", extra={"fields": {"job_id": body["job_id"], "outcome": outcome,
|
||||
"message_attempt": body["attempt"], "redelivered": message.redelivered}})
|
||||
await self.settle(message.ack())
|
||||
|
||||
# ---------------------------------------------------------------- one attempt
|
||||
async def process(self, job_id: str) -> str:
|
||||
settings = self.settings
|
||||
claim = await self.db.run(lambda c: jobs.claim(c, job_id, self.name, settings.job_lease_seconds))
|
||||
if isinstance(claim, jobs.Noop):
|
||||
return "noop_" + claim.reason
|
||||
await service.crash_once(settings, self.db, "crash_after_claim", job_id, self.actor)
|
||||
lost = asyncio.Event()
|
||||
work = asyncio.create_task(self.attempt(claim))
|
||||
beat = asyncio.create_task(self.heartbeat(claim, work, lost))
|
||||
try:
|
||||
variants = await work
|
||||
except asyncio.CancelledError:
|
||||
if lost.is_set() and not asyncio.current_task().cancelling():
|
||||
return "fenced" # lease taken over: this attempt may write nothing more
|
||||
outcome = await asyncio.shield(self.db.run(lambda c: jobs.release(c, claim, self.name)))
|
||||
log.info("job_released", extra={"fields": {"job_id": job_id, "outcome": outcome}})
|
||||
raise
|
||||
except ProcessingError as exc:
|
||||
return await self.failed(claim, exc.code, exc.permanent)
|
||||
except Unavailable:
|
||||
return await self.failed(claim, "storage_unavailable", False)
|
||||
finally:
|
||||
beat.cancel()
|
||||
await service.crash_once(settings, self.db, "crash_after_s3_write", job_id, self.actor)
|
||||
outcome = await self.db.run(lambda c: jobs.complete(c, claim, variants, self.name))
|
||||
await service.crash_once(settings, self.db, "crash_after_commit", job_id, self.actor)
|
||||
if outcome == "succeeded":
|
||||
await self.delete_staging(claim.attachment_id)
|
||||
return outcome
|
||||
|
||||
async def failed(self, claim: jobs.Claim, code: str, permanent: bool) -> str:
|
||||
s = self.settings
|
||||
outcome, delay = await self.db.run(
|
||||
lambda c: jobs.retry_or_fail(c, claim, code, permanent, self.name, s.job_retry_base_seconds,
|
||||
s.job_retry_max_seconds))
|
||||
log.info("attempt_failed", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt, "code": code,
|
||||
"permanent": permanent, "outcome": outcome,
|
||||
"delay_s": round(delay, 3)}})
|
||||
return outcome
|
||||
|
||||
async def heartbeat(self, claim: jobs.Claim, work: asyncio.Task, lost: asyncio.Event) -> None:
|
||||
"""Renew the lease while the attempt is alive and within JOB_TIMEOUT_SECONDS. A hung attempt
|
||||
stops being renewed, so the reaper can hand the job to another worker; if the lease turns out
|
||||
to be lost, the attempt is cancelled (its result would be fenced anyway)."""
|
||||
s = self.settings
|
||||
if "hang" in s.fault_points:
|
||||
return # experiment: a hung worker that no longer heartbeats
|
||||
started = time.monotonic()
|
||||
while True:
|
||||
await asyncio.sleep(s.job_lease_seconds / 3)
|
||||
if time.monotonic() - started > s.job_timeout_seconds:
|
||||
return
|
||||
try:
|
||||
alive = await self.db.run(lambda c: jobs.heartbeat(c, claim, s.job_lease_seconds))
|
||||
except Unavailable:
|
||||
continue # the lease may expire meanwhile; the fence protects the result
|
||||
if not alive:
|
||||
log.warning("lease_lost", extra={"fields": {"job_id": claim.job_id, "attempt": claim.attempt}})
|
||||
lost.set()
|
||||
work.cancel()
|
||||
return
|
||||
|
||||
async def attempt(self, claim: jobs.Claim) -> list[imaging.Rendered]:
|
||||
s = self.settings
|
||||
if "hang" in s.fault_points:
|
||||
await asyncio.sleep(s.fault_value("hang", 3600)) # stuck outside of the time limit
|
||||
try:
|
||||
return await asyncio.wait_for(self.build(claim), s.job_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
raise ProcessingError("processing_timeout", permanent=False) from None
|
||||
|
||||
async def build(self, claim: jobs.Claim) -> list[imaging.Rendered]:
|
||||
s = self.settings
|
||||
started = time.monotonic()
|
||||
if claim.size_bytes > s.max_upload_bytes:
|
||||
raise ProcessingError("image_too_large", permanent=True)
|
||||
source = self.staging if claim.source == "staging" else self.final
|
||||
try:
|
||||
data = await source.read_all(claim.source_key, limit=s.max_upload_bytes)
|
||||
except ObjectMissing:
|
||||
raise ProcessingError("source_missing", permanent=True) from None
|
||||
if len(data) != claim.size_bytes or hashlib.sha256(data).digest() != claim.sha256:
|
||||
raise ProcessingError("source_corrupt", permanent=True)
|
||||
self.injected(claim)
|
||||
if "slow" in s.fault_points and s.fault_value("slow_pipeline", claim.pipeline_version) == claim.pipeline_version:
|
||||
await asyncio.sleep(s.fault_value("slow", 10))
|
||||
remaining = max(1.0, s.job_timeout_seconds - (time.monotonic() - started))
|
||||
variants = await imaging.render(data, claim.content_type, claim.pipeline_version,
|
||||
max_pixels=s.max_image_pixels, timeout=remaining,
|
||||
memory_mb=s.image_max_memory_mb)
|
||||
meta = {"attachment-id": claim.attachment_id, "pipeline-version": str(claim.pipeline_version)}
|
||||
puts = [self.final.put(claim.key(v.name), v.data, v.content_type, meta) for v in variants]
|
||||
if claim.write_original:
|
||||
puts.append(self.final.put(claim.key("original"), data, claim.content_type,
|
||||
{**meta, "sha256": claim.sha256.hex()}))
|
||||
await asyncio.gather(*puts)
|
||||
return variants
|
||||
|
||||
def injected(self, claim: jobs.Claim) -> None:
|
||||
s = self.settings
|
||||
if "permanent_error" in s.fault_points:
|
||||
raise ProcessingError("invalid_image", permanent=True)
|
||||
if "transient_error" in s.fault_points and claim.attempt <= s.fault_value("transient_error", 1000):
|
||||
raise ProcessingError("injected_transient", permanent=False)
|
||||
|
||||
# ---------------------------------------------------------------- cleanup
|
||||
async def delete_staging(self, attachment_id: str) -> None:
|
||||
"""Mark `deleting` under the row lock (re-checked there), then S3 DELETE, then `deleted`."""
|
||||
try:
|
||||
key = await self.db.run(
|
||||
lambda c: jobs.mark_staging_deleting(c, attachment_id, self.settings.staging_retention_seconds))
|
||||
if key is None:
|
||||
return
|
||||
await self.staging.delete(key)
|
||||
await self.db.run(lambda c: jobs.mark_staging_deleted(c, attachment_id))
|
||||
log.info("staging_deleted", extra={"fields": {"attachment_id": attachment_id}})
|
||||
except Unavailable:
|
||||
pass # stays present/deleting; the next cleanup pass repeats (DELETE is idempotent)
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
s = self.settings
|
||||
ids = await self.db.run(lambda c: jobs.staging_to_delete(c, s.staging_retention_seconds))
|
||||
tokens = await self.db.run(lambda c: jobs.attempts_to_clean(c, s.orphan_grace_seconds))
|
||||
if "pause_cleanup" in s.fault_points and (ids or tokens):
|
||||
# Experiment: the race "cleanup read its candidates -> replay commits -> cleanup goes on".
|
||||
log.warning("cleanup_paused", extra={"fields": {"attachments": ids, "attempts": tokens}})
|
||||
await asyncio.sleep(s.fault_value("pause_cleanup", 10))
|
||||
for attachment_id in ids:
|
||||
await self.delete_staging(attachment_id)
|
||||
for token in tokens:
|
||||
keys = await self.db.run(lambda c: jobs.lock_attempt_for_cleanup(c, token))
|
||||
if keys is None:
|
||||
continue
|
||||
await asyncio.gather(*(self.final.delete(k) for k in keys))
|
||||
await self.db.run(lambda c: jobs.mark_attempt_cleaned(c, token))
|
||||
log.info("attempt_objects_deleted", extra={"fields": {"token": token, "objects": len(keys)}})
|
||||
|
||||
async def cleanup_loop(self, stop: asyncio.Event) -> None:
|
||||
while not stop.is_set() and self.settings.cleanup_interval_seconds > 0:
|
||||
await service.sleep_or_stop(stop, self.settings.cleanup_interval_seconds)
|
||||
try:
|
||||
await self.cleanup()
|
||||
except Unavailable:
|
||||
pass
|
||||
|
||||
# ---------------------------------------------------------------- lifecycle
|
||||
async def run(self) -> int:
|
||||
stop = service.stop_event()
|
||||
await self.db.open()
|
||||
await self.staging.open()
|
||||
await self.final.open()
|
||||
cleaner = asyncio.create_task(self.cleanup_loop(stop))
|
||||
try:
|
||||
while not stop.is_set():
|
||||
if not self.connected():
|
||||
await self.disconnect()
|
||||
try:
|
||||
await self.connect()
|
||||
except Exception as exc: # noqa: BLE001 - broker down: retry in 2 s, never crash
|
||||
log.warning("broker_unavailable", extra={"fields": {"error": type(exc).__name__}})
|
||||
await self.disconnect()
|
||||
if self.connected():
|
||||
service.beat() # healthy only while a consumer is really attached
|
||||
await service.sleep_or_stop(stop, 2.0)
|
||||
await self.shutdown()
|
||||
finally:
|
||||
cleaner.cancel()
|
||||
await self.disconnect()
|
||||
await self.final.close()
|
||||
await self.staging.close()
|
||||
await self.db.close()
|
||||
log.info("worker_stopped")
|
||||
return 0
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""Stop consuming; running attempts get WORKER_SHUTDOWN_SECONDS, the rest is released."""
|
||||
if self.queue is not None and self.consumer_tag is not None:
|
||||
try:
|
||||
await self.queue.cancel(self.consumer_tag)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
pending = set(self.tasks)
|
||||
log.info("worker_draining", extra={"fields": {"in_flight": len(pending)}})
|
||||
if pending:
|
||||
_, pending = await asyncio.wait(pending, timeout=self.settings.worker_shutdown_seconds)
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
if pending:
|
||||
await asyncio.wait(pending, timeout=10)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
settings = Settings.from_env()
|
||||
if not settings.broker_url:
|
||||
raise ConfigError("BROKER_URL_FILE (or BROKER_URL) must be set")
|
||||
except ConfigError as exc:
|
||||
print(f"configuration error: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
logging_setup.configure(settings.log_level)
|
||||
log.info("worker_starting", extra={"fields": {
|
||||
"instance_id": settings.instance_id, "concurrency": settings.worker_concurrency,
|
||||
"max_attempts": settings.job_max_attempts, "timeout_s": settings.job_timeout_seconds,
|
||||
"lease_s": settings.job_lease_seconds, "memory_mb": settings.image_max_memory_mb,
|
||||
"fault_points": sorted(settings.fault_points)}})
|
||||
return asyncio.run(Worker(settings).run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,439 @@
|
||||
"""Shared helpers for the student tests.
|
||||
|
||||
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
|
||||
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
|
||||
stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus
|
||||
`docker compose` for database checks (psql inside the postgres container), the HAProxy
|
||||
runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the
|
||||
stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP.
|
||||
|
||||
python -m pytest student-tests # everything (stand must be up)
|
||||
python -m pytest student-tests -m "not stack" # unit only
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import httpx2 as httpx
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
import lbctl # noqa: E402
|
||||
|
||||
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
|
||||
REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)")
|
||||
|
||||
|
||||
def schema_errors(value, name: str) -> list[str]:
|
||||
"""Validate against components/schemas/<name> of the course contract (JSON Schema 2020-12)."""
|
||||
schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]}
|
||||
validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER)
|
||||
return [e.message for e in validator.iter_errors(value)]
|
||||
|
||||
|
||||
def basic(username: str, password: str) -> str:
|
||||
return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ docker / psql
|
||||
def compose(*args: str, check: bool = True, timeout: float = 120, stdin: str | None = None) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
|
||||
encoding="utf-8", check=check, timeout=timeout, input=stdin)
|
||||
|
||||
|
||||
def psql(sql: str) -> str:
|
||||
"""Run SQL inside the postgres container (no DB port is published) and return unaligned output."""
|
||||
result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-At", "-v", "ON_ERROR_STOP=1", "-c", sql)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def wait_ready(timeout: float = 90) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200:
|
||||
return
|
||||
except httpx.HTTPError:
|
||||
pass
|
||||
time.sleep(0.3)
|
||||
raise AssertionError("stand did not become ready")
|
||||
|
||||
|
||||
def wait_healthy(*services: str, timeout: float = 120) -> None:
|
||||
"""Poll Docker health until EVERY container of every service is healthy again. `compose up
|
||||
--wait` is not used: it gives up at once on a container marked unhealthy while paused."""
|
||||
deadline = time.monotonic() + timeout
|
||||
pending = set(services)
|
||||
while pending and time.monotonic() < deadline:
|
||||
for service in list(pending):
|
||||
ids = compose("ps", "-q", service, check=False).stdout.split()
|
||||
if ids:
|
||||
states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids],
|
||||
capture_output=True, text=True).stdout.split()
|
||||
if states and all(state == "healthy" for state in states):
|
||||
pending.discard(service)
|
||||
time.sleep(0.5)
|
||||
assert not pending, f"not healthy: {pending}"
|
||||
|
||||
|
||||
def wait_stand(replicas: int = REPLICAS) -> None:
|
||||
"""Readiness through the LB AND every replica back in HAProxy rotation."""
|
||||
wait_ready()
|
||||
lbctl.wait_up(replicas, timeout=90)
|
||||
|
||||
|
||||
def restart_services(*services: str) -> None:
|
||||
compose("restart", *services)
|
||||
wait_healthy(*services)
|
||||
wait_stand()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def service_down(service: str, how: str = "stop"):
|
||||
"""Stop or pause a compose service; ALWAYS bring it back and wait for readiness."""
|
||||
assert how in {"stop", "pause"}
|
||||
compose(how, service)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
compose("unpause" if how == "pause" else "start", service)
|
||||
wait_healthy(service)
|
||||
wait_stand()
|
||||
|
||||
|
||||
def recreate(service: str, env: dict, scale: int | None = None) -> None:
|
||||
"""`docker compose up` of ONE service with the given environment (no dependency restarts)."""
|
||||
cmd = ["docker", "compose", "up", "-d", "--no-deps", "--wait", service]
|
||||
if scale is not None:
|
||||
cmd[4:4] = ["--scale", f"{service}={scale}"]
|
||||
subprocess.run(cmd, cwd=ROOT, env=env, capture_output=True, text=True, check=True, timeout=180)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def service_with_env(service: str, scale: int | None = None, **overrides: str):
|
||||
"""Recreate a service (api, relay, worker) with environment overrides (e.g. FAULT_POINTS of the
|
||||
worker = WORKER_FAULT_POINTS), then ALWAYS recreate it with the normal configuration."""
|
||||
clean = {k: v for k, v in os.environ.items() if k not in overrides}
|
||||
recreate(service, {**os.environ, **overrides}, scale)
|
||||
if service == "api":
|
||||
wait_stand()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
recreate(service, clean, 1 if scale is not None and service != "api" else None)
|
||||
wait_healthy(service)
|
||||
if service == "api":
|
||||
wait_stand()
|
||||
|
||||
|
||||
def api_with_env(**overrides: str):
|
||||
return service_with_env("api", **overrides)
|
||||
|
||||
|
||||
def running_replicas() -> list[dict]:
|
||||
return [r for r in lbctl.replicas() if r["state"] == "running"]
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def replica_stopped(replica: dict, mode: str = "stop"):
|
||||
"""Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again
|
||||
and wait until HAProxy has every replica UP."""
|
||||
assert mode in {"stop", "kill"}
|
||||
subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]],
|
||||
capture_output=True, text=True, check=True, timeout=60)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60)
|
||||
wait_healthy("api")
|
||||
wait_stand()
|
||||
|
||||
|
||||
def api_env(name: str) -> str:
|
||||
"""Current value of an environment variable inside the running api container."""
|
||||
return compose("exec", "-T", "api", "printenv", name).stdout.strip()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ HTTP
|
||||
class Api:
|
||||
"""HTTP helper that checks the invariants every response must satisfy."""
|
||||
|
||||
def __init__(self, base_url: str = BASE_URL):
|
||||
self.client = httpx.Client(base_url=base_url, timeout=15)
|
||||
|
||||
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
|
||||
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None,
|
||||
lb_error_ok: bool = False):
|
||||
"""lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is
|
||||
accepted without X-Instance-Id; every response of an API replica must carry it."""
|
||||
h = dict(headers or {})
|
||||
if token is not None:
|
||||
h["Authorization"] = "Bearer " + token
|
||||
if json_body is not None:
|
||||
raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8")
|
||||
h.setdefault("Content-Type", "application/json")
|
||||
response = self.client.request(method, path, content=raw, headers=h)
|
||||
allowed = (expected,) if isinstance(expected, int) else expected
|
||||
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
|
||||
request_id = response.headers.get("x-request-id")
|
||||
assert request_id, "every response must carry X-Request-Id"
|
||||
if not response.headers.get("x-instance-id"):
|
||||
assert lb_error_ok and response.status_code in (502, 503, 504), \
|
||||
("missing X-Instance-Id", method, path, response.status_code)
|
||||
if response.status_code >= 400:
|
||||
body = response.json()
|
||||
assert schema_errors(body, "Error") == [], response.text
|
||||
assert body["error"]["request_id"] == request_id
|
||||
assert "Traceback" not in response.text and "psycopg" not in response.text
|
||||
if response.status_code == 401:
|
||||
challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer"
|
||||
assert response.headers.get("www-authenticate", "").lower().startswith(challenge)
|
||||
elif schema:
|
||||
assert schema_errors(response.json(), schema) == [], response.text
|
||||
return response
|
||||
|
||||
def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict:
|
||||
username = f"{label}_{uuid.uuid4().hex[:12]}"[:32]
|
||||
body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password}
|
||||
user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json()
|
||||
return {"id": user["id"], "user": user, "username": username, "password": password}
|
||||
|
||||
def login(self, actor: dict) -> str:
|
||||
session = self.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": basic(actor["username"], actor["password"])},
|
||||
schema="Session").json()
|
||||
assert session["user"] == actor["user"] and session["token_type"] == "Bearer"
|
||||
return session["token"]
|
||||
|
||||
def user(self, label: str = "u", **kwargs) -> dict:
|
||||
actor = self.register(label, **kwargs)
|
||||
actor["token"] = self.login(actor)
|
||||
return actor
|
||||
|
||||
def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict:
|
||||
ids = [owner["id"], *(m["id"] for m in members)]
|
||||
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
|
||||
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
|
||||
|
||||
def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None,
|
||||
raw: bytes | None = None, headers: dict | None = None, **kwargs):
|
||||
"""POST a message with an Idempotency-Key (a fresh one unless `key` is given)."""
|
||||
h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})}
|
||||
if raw is not None:
|
||||
h.setdefault("Content-Type", "application/json")
|
||||
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"],
|
||||
json_body=body if raw is None else None, raw=raw, headers=h,
|
||||
schema="Message" if expected == 201 else None, **kwargs)
|
||||
|
||||
def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict:
|
||||
return self.post_message(actor, chat, {"text": text}, key=key).json()
|
||||
|
||||
def upload(self, actor: dict, chat: dict, data, expected=202, *, content_type: str | None = "image/png",
|
||||
filename: str | None = "photo.png", query: str | None = None, headers: dict | None = None):
|
||||
"""POST raw bytes to /chats/{id}/attachments. `query` replaces the whole query string
|
||||
(to send hand-made percent-encoding); `data` may be bytes or an iterator (chunked)."""
|
||||
h = dict(headers or {})
|
||||
if content_type is not None:
|
||||
h["Content-Type"] = content_type
|
||||
if query is None:
|
||||
query = "" if filename is None else "filename=" + quote(filename, safe="")
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments" + (f"?{query}" if query else "")
|
||||
if isinstance(data, (bytes, bytearray)):
|
||||
return self.call("POST", path, expected, token=actor["token"], raw=bytes(data), headers=h,
|
||||
schema="Attachment" if expected == 202 else None)
|
||||
h["Authorization"] = "Bearer " + actor["token"]
|
||||
response = self.client.post(path, content=data, headers=h)
|
||||
assert response.status_code == expected, (response.status_code, response.text[:300])
|
||||
return response
|
||||
|
||||
def attachment(self, actor: dict, attachment_id: str, expected=200) -> dict:
|
||||
return self.call("GET", f"/api/v1/attachments/{attachment_id}", expected, token=actor["token"],
|
||||
schema="Attachment" if expected == 200 else None).json()
|
||||
|
||||
def wait_status(self, actor: dict, attachment_id: str, *statuses: str, timeout: float = 60) -> dict:
|
||||
"""Poll the metadata until the status is one of `statuses` (default: a final one)."""
|
||||
wanted = statuses or ("ready", "failed")
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
att = self.attachment(actor, attachment_id)
|
||||
if att["status"] in wanted:
|
||||
return att
|
||||
assert time.monotonic() < deadline, f"attachment stayed {att['status']}, wanted {wanted}"
|
||||
time.sleep(0.3)
|
||||
|
||||
def content(self, actor: dict, attachment_id: str, variant: str | None = None, expected=200):
|
||||
query = f"?variant={variant}" if variant else ""
|
||||
return self.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
|
||||
|
||||
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
|
||||
"""Walk all pages; returns (items, number_of_pages)."""
|
||||
items, pages, cursor = [], 0, None
|
||||
while True:
|
||||
query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "")
|
||||
page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200,
|
||||
token=actor["token"], schema="MessagePage").json()
|
||||
pages += 1
|
||||
items.extend(page["items"])
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return items, pages
|
||||
assert len(page["items"]) == limit, "a non-final page must be full"
|
||||
assert len(items) < 100_000, "cursor loop"
|
||||
|
||||
|
||||
def _stand_is_up() -> bool:
|
||||
try:
|
||||
return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200
|
||||
except httpx.HTTPError:
|
||||
return False
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config, items):
|
||||
if any("stack" in item.keywords for item in items) and not _stand_is_up():
|
||||
skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait")
|
||||
for item in items:
|
||||
if "stack" in item.keywords:
|
||||
item.add_marker(skip)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def api() -> Api:
|
||||
return Api()
|
||||
|
||||
|
||||
def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess:
|
||||
"""Run scripts/<name> (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers."""
|
||||
script = str(ROOT / "scripts" / name)
|
||||
if os.name == "nt":
|
||||
command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script]
|
||||
else:
|
||||
command = [script]
|
||||
return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True,
|
||||
timeout=timeout, env={**os.environ, **env})
|
||||
|
||||
|
||||
def s3_exec(body: str, bucket: str = "final", service: str = "worker", stdin: str | None = None) -> str:
|
||||
"""Run `body` (async Python, `s` = messenger.storage.Storage of the final or the staging bucket)
|
||||
INSIDE a container that holds S3 credentials: `worker` (Read+Write both buckets, no List) or
|
||||
`auditor` (Read+List, no Write). The S3 port is not published on the host; this is how the
|
||||
tests look into the buckets and inject faults."""
|
||||
script = ("import asyncio, base64, json, sys\n"
|
||||
"from messenger.config import Settings, s3_settings\n"
|
||||
"from messenger.storage import Storage, ObjectMissing\n"
|
||||
"async def main():\n"
|
||||
" st = Settings(host='', port=0, env='lab', log_level='info', shutdown_timeout=0, max_body_bytes=1024, **s3_settings())\n"
|
||||
f" s = Storage(st, bucket=st.s3_staging_bucket if {bucket == 'staging'} else None)\n"
|
||||
" await s.open()\n"
|
||||
" try:\n" + "".join(" " + line + "\n" for line in body.splitlines()) +
|
||||
" finally:\n"
|
||||
" await s.close()\n"
|
||||
"asyncio.run(main())\n")
|
||||
if service == "auditor":
|
||||
cmd = ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service, "-c", script]
|
||||
else:
|
||||
cmd = ["exec", "-T", service, "python", "-c", script]
|
||||
return compose(*cmd, timeout=120, stdin=stdin).stdout.strip()
|
||||
|
||||
|
||||
def s3_put(key: str, data: bytes, bucket: str = "final") -> None:
|
||||
"""Write raw bytes under `key` (base64 over stdin: Windows limits command lines to 32 KiB)."""
|
||||
s3_exec(f"await s.put({key!r}, base64.b64decode(sys.stdin.read()), 'image/png', {{}})", bucket,
|
||||
stdin=base64.b64encode(data).decode())
|
||||
|
||||
|
||||
def s3_delete(key: str, bucket: str = "final") -> None:
|
||||
s3_exec(f"await s.delete({key!r})", bucket)
|
||||
|
||||
|
||||
def s3_size(key: str, bucket: str = "final") -> int | None:
|
||||
out = s3_exec(f"try:\n print((await s.head({key!r})).size)\nexcept ObjectMissing:\n print(-1)", bucket)
|
||||
size = int(out.splitlines()[-1])
|
||||
return None if size < 0 else size
|
||||
|
||||
|
||||
def s3_keys(prefix: str, bucket: str = "final") -> set[str]:
|
||||
"""Every object key under `prefix` (needs List: runs with the auditor's key pair)."""
|
||||
out = s3_exec("keys, after = [], None\n"
|
||||
"while True:\n"
|
||||
f" page, more = await s.list_page({prefix!r}, after, 1000)\n"
|
||||
" keys += [o.key for o in page]\n"
|
||||
" if not more or not page:\n"
|
||||
" break\n"
|
||||
" after = page[-1].key\n"
|
||||
"print(json.dumps(keys))", bucket, service="auditor")
|
||||
return set(json.loads(out.splitlines()[-1]))
|
||||
|
||||
|
||||
def jobctl(*args: str, expect_code: int = 0) -> dict:
|
||||
"""Operator CLI (compose service jobctl, profile tools); returns its JSON output."""
|
||||
result = compose("run", "--rm", "--no-deps", "-T", "jobctl", *args, check=False, timeout=120)
|
||||
assert result.returncode == expect_code, (result.returncode, result.stdout[-1000:], result.stderr[-1000:])
|
||||
return json.loads(result.stdout)
|
||||
|
||||
|
||||
def audit(*args: str) -> dict:
|
||||
"""Read-only invariant checker (compose service auditor); asserts that it found no violation."""
|
||||
result = compose("run", "--rm", "--no-deps", "-T", "auditor", *args, check=False, timeout=300)
|
||||
report = json.loads(result.stdout)
|
||||
assert result.returncode == 0 and report["ok"], json.dumps(report.get("violations"), ensure_ascii=False)[:3000]
|
||||
return report
|
||||
|
||||
|
||||
def job_rows(attachment_id: str) -> list[dict]:
|
||||
out = psql("SELECT coalesce(json_agg(json_build_object('id', id, 'state', state, 'attempt', attempt, 'kind', kind, "
|
||||
"'pipeline_version', pipeline_version, 'error_code', error_code) ORDER BY created_at), '[]') "
|
||||
"FROM jobs WHERE attachment_id = '%s'" % attachment_id)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def events_of(attachment_id: str) -> list[dict]:
|
||||
out = psql("SELECT coalesce(json_agg(json_build_object('event', e.event, 'attempt', e.attempt, 'actor', e.actor, "
|
||||
"'error_code', e.error_code, 'detail', e.detail, 'at', e.at) ORDER BY e.id), '[]') "
|
||||
"FROM job_events e JOIN jobs j ON j.id = e.job_id WHERE j.attachment_id = '%s'" % attachment_id)
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def queue_depth(queue: str = "messenger.attachment-jobs") -> int:
|
||||
out = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name", "messages").stdout
|
||||
for line in out.splitlines():
|
||||
name, _, count = line.partition("\t")
|
||||
if name == queue:
|
||||
return int(count)
|
||||
raise AssertionError("queue not found: " + queue)
|
||||
|
||||
|
||||
def attachment_row(attachment_id: str) -> dict | None:
|
||||
out = psql("SELECT json_build_object('status', status, 'object_key', object_key, 'size', size_bytes, "
|
||||
"'lost_reason', lost_reason, 'staging_key', staging_key, 'staging_state', staging_state, "
|
||||
"'published_version', published_version, 'error_code', error_code) "
|
||||
"FROM attachments WHERE id = '%s'" % attachment_id)
|
||||
return json.loads(out) if out else None
|
||||
|
||||
|
||||
def probe_inside(replica: dict, path: str) -> int:
|
||||
"""HTTP status of `path` asked inside the replica container itself (bypassing the LB)."""
|
||||
code = ("import urllib.request as u, urllib.error as e\n"
|
||||
"try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n"
|
||||
"except e.HTTPError as x: print(x.code)") % path
|
||||
out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True,
|
||||
timeout=30).stdout.strip()
|
||||
return int(out or 0)
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Synthetic images for the lab 4 tests: valid PNG/JPEG of any size, and the broken files the
|
||||
reviewer used against other solutions (truncated, corrupted IDAT with a valid CRC, JPEG without
|
||||
scan data, decompression-bomb headers, MIME spoofing). Nothing big is stored in Git: every file
|
||||
is generated in memory."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def chunk(ctype: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data))
|
||||
|
||||
|
||||
def ihdr(width: int, height: int, depth: int = 8, color: int = 2, interlace: int = 0) -> bytes:
|
||||
return chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace))
|
||||
|
||||
|
||||
def png(width: int = 64, height: int = 48, mode: str = "RGB", noise: bool = False, **save) -> bytes:
|
||||
if noise:
|
||||
channels = len(mode)
|
||||
image = Image.frombytes(mode, (width, height), os.urandom(width * height * channels))
|
||||
else:
|
||||
image = Image.new(mode, (width, height), (40, 120, 200) if mode == "RGB" else 128)
|
||||
out = io.BytesIO()
|
||||
image.save(out, "PNG", **save)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def png_of_size(target: int) -> bytes:
|
||||
"""A valid PNG of (about, at most) `target` bytes: random pixels do not compress."""
|
||||
side = int((target / 3) ** 0.5)
|
||||
data = png(side, side, noise=True, compress_level=1)
|
||||
while len(data) > target:
|
||||
side -= 8
|
||||
data = png(side, side, noise=True, compress_level=1)
|
||||
return data
|
||||
|
||||
|
||||
def png_exactly(size: int) -> bytes:
|
||||
"""A valid PNG of EXACTLY `size` bytes: noise image + an ancillary padding chunk."""
|
||||
base = png_of_size(size - 64)
|
||||
pad = size - len(base) - 12
|
||||
assert pad >= 0
|
||||
return base[:-12] + chunk(b"tEXt", b"pad\x00" + b"x" * (pad - 4)) + base[-12:]
|
||||
|
||||
|
||||
def jpeg(width: int = 64, height: int = 48, progressive: bool = False, noise: bool = False, quality: int = 85) -> bytes:
|
||||
if noise:
|
||||
image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3))
|
||||
else:
|
||||
image = Image.new("RGB", (width, height), (200, 30, 30))
|
||||
out = io.BytesIO()
|
||||
image.save(out, "JPEG", progressive=progressive, quality=quality)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ broken files
|
||||
def png_corrupt_idat() -> bytes:
|
||||
"""Valid signature, IHDR and CRCs; the IDAT payload is 86 bytes of garbage (reviewer's case)."""
|
||||
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", bytes(range(86))) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def png_short_pixels() -> bytes:
|
||||
"""A complete, CRC-correct PNG whose zlib stream holds fewer rows than the header promises."""
|
||||
raw = b"".join(b"\x00" + bytes(60) for _ in range(19))
|
||||
return PNG_SIGNATURE + ihdr(20, 20) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def png_bomb_header(width: int = 20000, height: int = 10000) -> bytes:
|
||||
"""Header of a huge picture: must be refused (413) from the first 33 bytes."""
|
||||
raw_row = b"\x00" + bytes(width * 3)
|
||||
body = zlib.compress(raw_row * 4, 9) # a few rows only: the decoder must never get that far
|
||||
return PNG_SIGNATURE + ihdr(width, height) + chunk(b"IDAT", body) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def jpeg_without_scan_data() -> bytes:
|
||||
data = jpeg(128, 96)
|
||||
sos = data.index(b"\xff\xda")
|
||||
length = struct.unpack(">H", data[sos + 2:sos + 4])[0]
|
||||
return data[:sos + 2 + length] + b"\xff\xd9"
|
||||
|
||||
|
||||
def jpeg_bomb_header() -> bytes:
|
||||
"""SOF0 announcing 20000 x 20000 = 400 Mpx."""
|
||||
return (b"\xff\xd8\xff\xc0\x00\x11\x08" + struct.pack(">HH", 20000, 20000)
|
||||
+ b"\x03\x01\x22\x00\x02\x11\x01\x03\x11\x01" + b"\xff\xd9")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cases the reviewer used in lab 4/5 PRs
|
||||
def png_bad_filter(filter_byte: int) -> bytes:
|
||||
"""PNG 1x1 RGB, correct CRCs and zlib stream, but the row filter byte is not 0..4."""
|
||||
raw = bytes([filter_byte]) + b"\x00\x00\x00"
|
||||
return PNG_SIGNATURE + ihdr(1, 1) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def png_idat_not_zlib() -> bytes:
|
||||
"""Correct PNG structure, IDAT = b'not-a-zlib-stream' with a recomputed CRC."""
|
||||
return PNG_SIGNATURE + ihdr(64, 64) + chunk(b"IDAT", b"not-a-zlib-stream") + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
# SOI, SOF0 1x1 grey, SOS, EOI: no tables, no scan data (27 bytes).
|
||||
JPEG_27_BYTES = bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9")
|
||||
|
||||
|
||||
def jpeg_sos_unknown_component() -> bytes:
|
||||
"""A valid JPEG whose first SOS component id (byte sos+5) is 250, absent from SOF."""
|
||||
data = bytearray(jpeg(64, 48))
|
||||
sos = data.index(b"\xff\xda")
|
||||
data[sos + 5] = 250
|
||||
return bytes(data)
|
||||
@@ -0,0 +1,344 @@
|
||||
"""Contract behaviour over real HTTP (stand must be running). Lab 1 checks moved to Bearer + new lab 2 auth."""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import PASSWORD, basic
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
|
||||
def key(item: dict) -> tuple[str, int]:
|
||||
return item["created_at"], uuid.UUID(item["id"]).int
|
||||
|
||||
|
||||
# ================================================================== registration & login
|
||||
def test_register_never_returns_password_and_login_returns_bearer(api):
|
||||
actor = api.register("alice", display_name="Алиса 👩💻")
|
||||
assert set(actor["user"]) == {"id", "username", "display_name", "created_at"}
|
||||
session = api.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": basic(actor["username"], PASSWORD)}, schema="Session").json()
|
||||
assert session["token_type"] == "Bearer" and len(session["token"]) >= 43
|
||||
assert actor["username"] not in session["token"] and actor["id"] not in session["token"]
|
||||
assert api.call("GET", "/api/v1/users/me", 200, token=session["token"], schema="User").json() == actor["user"]
|
||||
|
||||
|
||||
def test_duplicate_username_is_409(api):
|
||||
actor = api.register("dup")
|
||||
body = {"username": actor["username"], "display_name": "Другой", "password": PASSWORD}
|
||||
assert api.call("POST", "/api/v1/users", 409, json_body=body).json()["error"]["code"] == "conflict"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("password", ["short-11ch", " " * 12, " \t \n " * 3, "я" * 129])
|
||||
def test_bad_password_on_registration_is_400(api, password):
|
||||
body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": password}
|
||||
api.call("POST", "/api/v1/users", 400, json_body=body)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("password", ["я" * 128, "with:colons:inside", "Twelve chars"])
|
||||
def test_edge_passwords_register_and_login(api, password):
|
||||
actor = api.user("edge", password=password)
|
||||
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
|
||||
|
||||
|
||||
def test_wrong_password_and_unknown_user_give_the_same_401(api):
|
||||
actor = api.register("victim")
|
||||
wrong = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(actor["username"], "Wrong-password-1")})
|
||||
unknown = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("nobody_" + uuid.uuid4().hex[:8], PASSWORD)})
|
||||
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
|
||||
assert wrong.headers["www-authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header", ["Basic é", "Basic !!!", "Basic " + base64.b64encode(b"no_colon").decode(),
|
||||
"Basic " + base64.b64encode(b"\xff\xfe:x").decode(), "Bearer " + "a" * 43, ""])
|
||||
def test_malformed_basic_is_401_not_500(api, header):
|
||||
# Raw UTF-8 bytes, as curl would send them (httpx refuses non-ASCII str headers).
|
||||
api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": header.encode("utf-8")} if header else {})
|
||||
|
||||
|
||||
def test_logout_revokes_only_this_token(api):
|
||||
actor = api.user("logout")
|
||||
second = api.login(actor)
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
|
||||
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=actor["token"])
|
||||
api.call("GET", "/api/v1/users/me", 200, token=second) # other session of the same user survives
|
||||
|
||||
|
||||
def test_x_user_id_basic_and_garbage_do_not_authenticate(api):
|
||||
actor = api.user("hdr")
|
||||
for headers in ({"X-User-Id": actor["id"]},
|
||||
{"Authorization": basic(actor["username"], PASSWORD)},
|
||||
{"Authorization": "Bearer " + "A" * 43},
|
||||
{"Authorization": "Bearer"}):
|
||||
api.call("GET", "/api/v1/users/me", 401, headers=headers)
|
||||
me = api.call("GET", "/api/v1/users/me", 200, token=actor["token"], headers={"X-User-Id": str(uuid.uuid4())})
|
||||
assert me.json()["id"] == actor["id"]
|
||||
|
||||
|
||||
# ================================================================== validation / errors (from lab 1)
|
||||
@pytest.mark.parametrize("raw", [b"{", b"", b"[]", b"null", b'{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}',
|
||||
b'{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}',
|
||||
b'{"username": "abc_def", "display_name": "\\ud800", "password": "Twelve-chars"}'])
|
||||
def test_malformed_json_is_400(api, raw):
|
||||
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [
|
||||
{"username": "abc_def", "display_name": "x"}, # no password
|
||||
{"username": "abc_def", "display_name": "x", "password": "Twelve-chars", "extra": 1},
|
||||
{"username": "Abc_def", "display_name": "x", "password": "Twelve-chars"},
|
||||
{"username": "abc_def", "display_name": " ", "password": "Twelve-chars"},
|
||||
{"username": "abc_def", "display_name": "x" * 101, "password": "Twelve-chars"},
|
||||
{"username": 5, "display_name": "x", "password": "Twelve-chars"},
|
||||
])
|
||||
def test_invalid_registration_fields_are_400(api, body):
|
||||
api.call("POST", "/api/v1/users", 400, json_body=body)
|
||||
|
||||
|
||||
def test_unicode_limits_are_code_points_and_nul_is_stored(api):
|
||||
actor = api.user("uni", display_name="😀" * 100)
|
||||
assert actor["user"]["display_name"] == "😀" * 100
|
||||
nul = api.user("nul", display_name="A\u0000B")
|
||||
assert api.call("GET", "/api/v1/users/me", 200, token=nul["token"]).json()["display_name"] == "A\u0000B"
|
||||
chat = api.chat(actor, title="T\u0000itle")
|
||||
assert chat["title"] == "T\u0000itle"
|
||||
assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world"
|
||||
assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000
|
||||
api.post_message(actor, chat, {"text": "я" * 4001}, 400)
|
||||
texts = [m["text"] for m in api.history(actor, chat, 100)[0]]
|
||||
assert texts == ["hello\u0000world", "🙂" * 4000]
|
||||
|
||||
|
||||
def test_oversized_body_and_wrong_content_type(api):
|
||||
api.call("POST", "/api/v1/users", 413, json_body={"username": "big_body", "display_name": "x" * 70_000, "password": PASSWORD})
|
||||
api.call("POST", "/api/v1/users", 400, raw=b'{"username":"abc_def"}', headers={"Content-Type": "text/plain"})
|
||||
|
||||
|
||||
# ================================================================== chats & membership
|
||||
def test_chat_validation(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
for body in ({"title": "x", "member_ids": [alice["id"], bob["id"], bob["id"]]},
|
||||
{"title": "x", "member_ids": [alice["id"], str(uuid.uuid4())]},
|
||||
{"title": "x", "member_ids": [bob["id"]]},
|
||||
{"title": " ", "member_ids": [alice["id"]]},
|
||||
{"title": "x", "member_ids": [alice["id"], "bob"]}):
|
||||
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body=body)
|
||||
|
||||
|
||||
def test_foreign_chat_is_indistinguishable_from_missing(api):
|
||||
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
|
||||
chat = api.chat(alice, bob)
|
||||
for chat_id in (chat["id"], str(uuid.uuid4())):
|
||||
base = f"/api/v1/chats/{chat_id}"
|
||||
errors = [api.call("GET", base, 404, token=eve["token"]).json(),
|
||||
api.call("GET", base + "/messages", 404, token=eve["token"]).json(),
|
||||
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(),
|
||||
api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()]
|
||||
assert len({e["error"]["message"] for e in errors}) == 1
|
||||
api.call("GET", "/api/v1/chats/not-a-uuid", 401)
|
||||
api.call("GET", "/api/v1/chats/not-a-uuid", 400, token=alice["token"])
|
||||
api.call("GET", "/api/v1/chats/NOT-A-UUID/messages", 400, token=alice["token"])
|
||||
|
||||
|
||||
def test_chat_list_order_pagination_and_isolation(api):
|
||||
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
|
||||
created = [api.chat(alice, bob, title=f"Чат {i}") for i in range(5)]
|
||||
api.chat(eve, title="Чат Евы")
|
||||
walked, cursor = [], None
|
||||
while True:
|
||||
query = "?limit=2" + (f"&cursor={cursor}" if cursor else "")
|
||||
page = api.call("GET", "/api/v1/chats" + query, 200, token=bob["token"], schema="ChatPage").json()
|
||||
walked += page["items"]
|
||||
cursor = page["next_cursor"]
|
||||
if not cursor:
|
||||
break
|
||||
assert walked == created
|
||||
first = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
|
||||
api.call("GET", f"/api/v1/chats?limit=1&cursor={first}", 400, token=bob["token"]) # cursor of another user
|
||||
api.call("GET", "/api/v1/chats?limit=0", 400, token=bob["token"])
|
||||
|
||||
|
||||
# ================================================================== messages & keyset pagination
|
||||
def test_history_pages_without_loss_or_duplicates(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
sent = [api.send(alice if i % 2 else bob, chat, f"сообщение {i}") for i in range(25)]
|
||||
items, pages = api.history(bob, chat, 7)
|
||||
assert pages == 4 and items == sent and items == sorted(items, key=key)
|
||||
assert sent[0]["sender_id"] == bob["id"] and sent[1]["sender_id"] == alice["id"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("query", ["limit=0", "limit=101", "limit=abc", "limit=1&limit=2", "cursor=", "cursor=garbage",
|
||||
"cursor=djF8bXx4fDF8Mg", "cursor=" + base64.urlsafe_b64encode(b"v1|m|x|99999999999999999999|y").decode()])
|
||||
def test_bad_limit_or_cursor_is_400(api, query):
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
api.call("GET", f"/api/v1/chats/{chat['id']}/messages?{query}", 400, token=alice["token"])
|
||||
|
||||
|
||||
def test_cursor_bound_to_chat(api):
|
||||
alice = api.user("alice")
|
||||
a, b = api.chat(alice, title="A"), api.chat(alice, title="B")
|
||||
for chat in (a, b):
|
||||
api.send(alice, chat, "1")
|
||||
api.send(alice, chat, "2")
|
||||
cursor = api.call("GET", f"/api/v1/chats/{a['id']}/messages?limit=1", 200, token=alice["token"]).json()["next_cursor"]
|
||||
api.call("GET", f"/api/v1/chats/{b['id']}/messages?cursor={cursor}", 400, token=alice["token"])
|
||||
empty = api.chat(alice, title="Empty")
|
||||
api.call("GET", f"/api/v1/chats/{empty['id']}/messages?cursor=garbage", 400, token=alice["token"])
|
||||
|
||||
|
||||
def test_read_after_write(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
for i in range(5):
|
||||
message = api.send(alice, chat, f"m{i}")
|
||||
last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1]
|
||||
assert last == message
|
||||
|
||||
|
||||
# ================================================================== lab 3: inputs found by the reviewer in other PRs
|
||||
def b64(raw: bytes) -> str:
|
||||
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode()
|
||||
|
||||
|
||||
def bad_cursors(owner_id: str) -> list[str]:
|
||||
"""Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time."""
|
||||
good_uuid = str(uuid.uuid4())
|
||||
return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"),
|
||||
b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'),
|
||||
b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()),
|
||||
b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()),
|
||||
b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()),
|
||||
b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()),
|
||||
b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())]
|
||||
|
||||
|
||||
def test_malformed_cursor_values_are_400_on_both_lists(api):
|
||||
alice = api.user("cur")
|
||||
chat = api.chat(alice)
|
||||
for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]):
|
||||
quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20")
|
||||
for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
|
||||
error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json()
|
||||
assert error["error"]["code"] == "invalid_request", (path, cursor)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}',
|
||||
b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}',
|
||||
b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}',
|
||||
b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}',
|
||||
b'{"text":" \n\t "}'])
|
||||
def test_bad_message_body_is_400_and_nothing_is_stored(api, raw):
|
||||
alice = api.user("body")
|
||||
chat = api.chat(alice)
|
||||
error = api.post_message(alice, chat, None, 400, raw=raw).json()
|
||||
assert error["error"]["code"] == "invalid_request"
|
||||
assert api.history(alice, chat, 100)[0] == []
|
||||
|
||||
|
||||
def test_ff_byte_body_on_registration_is_contract_error(api):
|
||||
response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"})
|
||||
assert response.json()["error"]["request_id"] == response.headers["x-request-id"]
|
||||
|
||||
|
||||
def test_text_is_stored_verbatim_and_length_counts_raw_text(api):
|
||||
alice = api.user("verbatim")
|
||||
chat = api.chat(alice)
|
||||
padded = " строка\nс пробелами "
|
||||
assert api.send(alice, chat, padded)["text"] == padded
|
||||
api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming
|
||||
assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded]
|
||||
|
||||
|
||||
def test_unknown_attachment_ids_are_404(api):
|
||||
alice = api.user("att")
|
||||
chat = api.chat(alice)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates
|
||||
assert api.history(alice, chat, 100)[0] == []
|
||||
|
||||
|
||||
def test_member_ids_are_compared_as_uuids_not_strings(api):
|
||||
alice, bob = api.user("case"), api.user("case")
|
||||
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
|
||||
json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]})
|
||||
api.call("POST", "/api/v1/chats", 400, token=alice["token"],
|
||||
json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())})
|
||||
chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"],
|
||||
json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json()
|
||||
assert chat["member_ids"] == [alice["id"], bob["id"]]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43])
|
||||
def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header):
|
||||
api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")})
|
||||
|
||||
|
||||
def test_login_with_a_body_is_rejected(api):
|
||||
actor = api.register("login_body")
|
||||
auth = {"Authorization": basic(actor["username"], actor["password"])}
|
||||
api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"})
|
||||
api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth)
|
||||
api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session")
|
||||
|
||||
|
||||
def test_password_range_is_12_to_128_code_points(api):
|
||||
api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"})
|
||||
actor = api.user("pw128", password="я:" * 64)
|
||||
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
|
||||
|
||||
|
||||
def test_every_api_response_carries_instance_and_request_id(api):
|
||||
"""X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it)."""
|
||||
api.call("GET", "/health/live", 200)
|
||||
api.call("GET", "/health/ready", 200)
|
||||
api.call("GET", "/no/such/route", 404)
|
||||
api.call("PUT", "/api/v1/users", 405)
|
||||
api.call("GET", "/api/v1/users/me", 401)
|
||||
response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"})
|
||||
assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log
|
||||
|
||||
|
||||
@pytest.mark.parametrize("blank", [" ", " ", " \t "])
|
||||
def test_unicode_whitespace_only_strings_are_400(api, blank):
|
||||
alice = api.user("blank")
|
||||
chat = api.chat(alice)
|
||||
api.call("POST", "/api/v1/users", 400, json_body={"username": "bl_" + uuid.uuid4().hex[:10], "display_name": blank,
|
||||
"password": PASSWORD})
|
||||
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body={"title": blank, "member_ids": [alice["id"]]})
|
||||
api.post_message(alice, chat, {"text": blank}, 400)
|
||||
assert api.history(alice, chat, 100)[0] == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [b'{"username":"num_probe","display_name":"x","password":1e1000}',
|
||||
'{"username":"ff_probe","display_name":"x","password":"12345678901'.encode() + b'\xff"}',
|
||||
b'{"username":"num_probe","display_name":1,"password":"Twelve-chars-ok"}'])
|
||||
def test_odd_registration_bodies_are_400_and_the_api_survives(api, raw):
|
||||
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
|
||||
api.call("GET", "/health/ready", 200)
|
||||
|
||||
|
||||
def test_more_than_100_members_is_400(api):
|
||||
owner = api.user("big")
|
||||
others = [api.register(f"m{i}")["id"] for i in range(100)]
|
||||
api.call("POST", "/api/v1/chats", 400, token=owner["token"], json_body={"title": "x", "member_ids": [owner["id"], *others]})
|
||||
chat = api.call("POST", "/api/v1/chats", 201, token=owner["token"],
|
||||
json_body={"title": "x", "member_ids": [owner["id"], *others[:99]]}, schema="Chat").json()
|
||||
assert len(chat["member_ids"]) == 100
|
||||
|
||||
|
||||
def test_cursor_of_one_list_is_rejected_by_the_other(api):
|
||||
alice = api.user("cross")
|
||||
chats = [api.chat(alice, title=f"c{i}") for i in range(2)]
|
||||
for _ in range(2):
|
||||
api.send(alice, chats[0], "m")
|
||||
message_cursor = api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?limit=1", 200,
|
||||
token=alice["token"]).json()["next_cursor"]
|
||||
chat_cursor = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
|
||||
api.call("GET", f"/api/v1/chats?cursor={message_cursor}", 400, token=alice["token"])
|
||||
api.call("GET", f"/api/v1/chats/{chats[0]['id']}/messages?cursor={chat_cursor}", 400, token=alice["token"])
|
||||
@@ -0,0 +1,291 @@
|
||||
"""Attachments through the load balancer (lab 4, levels 3 and 4; lab 5: upload is 202 queued and
|
||||
the worker publishes the original + variants): upload/download, ACL, limits, content checks,
|
||||
filename rules and messages with attachment_ids.
|
||||
|
||||
Negative inputs include every case the reviewer used against other lab 4 solutions: corrupted
|
||||
IDAT with valid CRCs, the first 33 bytes of the fixture, JPEG without scan data, a 20000x10000
|
||||
header (DecompressionBomb), exactly 10 MiB through the LB, C1 control U+0085 and a whitespace
|
||||
filename, text=null / missing text / attachment_ids=null in a message; and the lab 4/5 review
|
||||
cases: PNG row filter 5 and 0xff, IDAT that is not a zlib stream, the 27-byte JPEG without tables
|
||||
and scan data, a JPEG whose SOS names a component missing from SOF - all 415 AT UPLOAD, never
|
||||
202 followed by failed/invalid_image."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import attachment_row, compose, s3_size
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
MiB = 1024 * 1024
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def trio(api):
|
||||
"""alice + bob share a chat; eve is an outsider with her own chat."""
|
||||
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
|
||||
return alice, bob, eve, api.chat(alice, bob), api.chat(eve)
|
||||
|
||||
|
||||
def content(api, actor, attachment_id, expected=200, query=""):
|
||||
return api.call("GET", f"/api/v1/attachments/{attachment_id}/content{query}", expected, token=actor["token"])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ positive path
|
||||
def test_png_and_jpeg_roundtrip_byte_for_byte(api, trio):
|
||||
alice, bob, _, chat, _ = trio
|
||||
jpg = imagegen.jpeg(321, 123, noise=True)
|
||||
for data, ctype in ((SAMPLE, "image/png"), (jpg, "image/jpeg")):
|
||||
att = api.upload(alice, chat, data, content_type=ctype, filename="кот 😺.bin").json()
|
||||
assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None
|
||||
assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"]
|
||||
assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
assert att["content_type"] == ctype and att["filename"] == "кот 😺.bin"
|
||||
meta = api.wait_status(bob, att["id"], "ready")
|
||||
assert {k: v for k, v in meta.items() if k not in ("status", "variants", "updated_at")} == \
|
||||
{k: v for k, v in att.items() if k not in ("status", "variants", "updated_at")}
|
||||
assert [v["name"] for v in meta["variants"]] == ["thumbnail", "crop", "optimized"]
|
||||
got = content(api, bob, att["id"])
|
||||
assert got.content == data and got.headers["content-type"] == ctype
|
||||
assert got.headers["content-length"] == str(len(data))
|
||||
assert got.headers["x-content-type-options"] == "nosniff"
|
||||
assert got.headers["content-disposition"].startswith("attachment;")
|
||||
assert "location" not in got.headers # bytes through the API, never a redirect to S3
|
||||
|
||||
|
||||
def test_metadata_reveals_no_storage_details(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(alice, att["id"], "ready")
|
||||
row = attachment_row(att["id"])
|
||||
assert row["status"] == "ready" and row["object_key"].startswith(f"a/{att['id']}/")
|
||||
assert row["staging_key"].startswith("s/")
|
||||
text = api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=alice["token"]).text
|
||||
for secret in (row["object_key"], row["staging_key"], "messenger-attachments", "messenger-staging", "a/", "s3"):
|
||||
assert secret not in text.lower(), secret
|
||||
# The staging key is random: neither the id nor the filename is part of it.
|
||||
assert att["id"].replace("-", "") not in row["staging_key"] and "photo" not in row["staging_key"]
|
||||
assert s3_size(row["object_key"]) == len(SAMPLE) # the published original IS in the final bucket
|
||||
|
||||
|
||||
def test_original_is_not_stored_on_api_containers(api, trio):
|
||||
"""The API keeps no file of the original: no recently written file of that size anywhere in
|
||||
the replica's filesystem (the bytes live only in the bucket)."""
|
||||
alice, _, _, chat, _ = trio
|
||||
data = imagegen.png_of_size(300_000)
|
||||
api.upload(alice, chat, data)
|
||||
size = len(data)
|
||||
for cid in compose("ps", "-q", "api").stdout.split():
|
||||
out = subprocess.run(
|
||||
["docker", "exec", cid, "sh", "-c", f"find / -xdev -type f -size {size}c -mmin -10 2>/dev/null | head"],
|
||||
capture_output=True, text=True, timeout=60).stdout.strip()
|
||||
assert out == "", out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ access control
|
||||
def test_outsider_and_unknown_get_404(api, trio):
|
||||
alice, _, eve, chat, _ = trio
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
for suffix in ("", "/content"):
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
|
||||
api.call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
|
||||
api.call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
|
||||
# Upload into a foreign chat: 404 before anything about the body is looked at.
|
||||
api.upload(eve, chat, SAMPLE, 404)
|
||||
api.upload(eve, chat, b"garbage", 404, content_type="text/plain")
|
||||
|
||||
|
||||
def test_attachment_ids_are_checked_per_chat(api, trio):
|
||||
alice, bob, eve, chat, eve_chat = trio
|
||||
other = api.chat(alice) # alice's second chat
|
||||
att_other = api.upload(alice, other, SAMPLE).json()
|
||||
att_eve = api.upload(eve, eve_chat, SAMPLE).json()
|
||||
# Alice is a member of both chats, but a file is bound to its chat.
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_other["id"]]}, 404)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404)
|
||||
# Eve cannot post into alice's chat even with her own file: membership first.
|
||||
api.post_message(eve, chat, {"text": "x", "attachment_ids": [att_eve["id"]]}, 404)
|
||||
# Any member may reference an attachment of the chat.
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
msg = api.post_message(bob, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
|
||||
assert msg["attachment_ids"] == [att["id"]] and msg["sender_id"] == bob["id"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ messages with attachments
|
||||
def test_message_with_attachments_and_history(api, trio):
|
||||
alice, bob, _, chat, _ = trio
|
||||
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
|
||||
m1 = api.post_message(alice, chat, {"text": "", "attachment_ids": [a2, a1]}).json()
|
||||
m2 = api.post_message(alice, chat, {"text": " ", "attachment_ids": [a1]}).json() # blank text + file: ok
|
||||
m3 = api.post_message(alice, chat, {"text": "без файлов"}).json()
|
||||
assert m1["attachment_ids"] == [a2, a1] and m1["text"] == ""
|
||||
assert m2["text"] == " " and m3["attachment_ids"] == []
|
||||
items, _ = api.history(bob, chat, 2)
|
||||
assert [m["attachment_ids"] for m in items] == [[a2, a1], [a1], []]
|
||||
|
||||
|
||||
def test_idempotent_replay_with_attachments(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
a1, a2 = (api.upload(alice, chat, SAMPLE).json()["id"] for _ in range(2))
|
||||
key = uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a1, a2]}, key=key).json()
|
||||
raw = ('{"attachment_ids":["%s","%s"],"text":"файлы"}' % (a1, a2)).encode()
|
||||
assert api.post_message(alice, chat, None, raw=raw, key=key).json() == first # key order irrelevant
|
||||
api.post_message(alice, chat, {"text": "файлы", "attachment_ids": [a2, a1]}, 409, key=key) # other body
|
||||
api.post_message(alice, chat, {"text": "файлы"}, 409, key=key)
|
||||
# A failed attachment check does not burn the key: the same key works with a valid body later.
|
||||
key2 = uuid.uuid4().hex
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404, key=key2)
|
||||
api.post_message(alice, chat, {"text": "x", "attachment_ids": [a1]}, key=key2)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [
|
||||
b'{"attachment_ids":["%(a)s"]}', # text missing
|
||||
b'{"text":null,"attachment_ids":["%(a)s"]}', # text null
|
||||
b'{"text":1,"attachment_ids":["%(a)s"]}',
|
||||
b'{"text":"x","attachment_ids":null}', # attachment_ids null
|
||||
b'{"text":"x","attachment_ids":"%(a)s"}',
|
||||
b'{"text":"x","attachment_ids":{}}',
|
||||
b'{"text":"x","attachment_ids":["%(a)s","%(a)s"]}', # duplicates
|
||||
b'{"text":"x","attachment_ids":["%(a)s",1]}',
|
||||
b'{"text":"x","attachment_ids":["not-a-uuid"]}',
|
||||
b'{"text":"","attachment_ids":[]}', # neither text nor files
|
||||
b'{"text":" "}',
|
||||
b'{"text":"x","attachment_ids":[],"sender_id":"%(a)s"}',
|
||||
], ids=lambda raw: raw.decode()[:40])
|
||||
def test_message_body_validation(api, trio, raw):
|
||||
alice, _, _, chat, _ = trio
|
||||
a = api.upload(alice, chat, SAMPLE).json()["id"]
|
||||
api.post_message(alice, chat, None, 400, raw=raw % {b"a": a.encode()})
|
||||
|
||||
|
||||
def test_at_most_ten_attachments(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
ids = [api.upload(alice, chat, imagegen.png(4, 4)).json()["id"] for _ in range(11)]
|
||||
assert len(api.post_message(alice, chat, {"text": "", "attachment_ids": ids[:10]}).json()["attachment_ids"]) == 10
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": ids}, 400)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ limits
|
||||
def test_size_limit_through_the_load_balancer(api, trio):
|
||||
alice, bob, _, chat, _ = trio
|
||||
exact = imagegen.png_exactly(10 * MiB)
|
||||
assert len(exact) == 10 * MiB
|
||||
att = api.upload(alice, chat, exact).json() # exactly the limit: accepted via LB
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
assert content(api, bob, att["id"]).content == exact
|
||||
started = time.monotonic()
|
||||
api.upload(alice, chat, exact + b"\x00", 413) # Content-Length over the limit
|
||||
assert time.monotonic() - started < 5
|
||||
over = exact + bytes(1024)
|
||||
|
||||
def chunks(): # no Content-Length: chunked body
|
||||
for i in range(0, len(over), 256 * 1024):
|
||||
yield over[i:i + 256 * 1024]
|
||||
error = api.upload(alice, chat, chunks(), 413).json()
|
||||
assert error["error"]["code"] == "payload_too_large"
|
||||
|
||||
|
||||
def test_pixel_limit_before_decoding(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
for data, ctype in ((imagegen.png_bomb_header(), "image/png"), (imagegen.jpeg_bomb_header(), "image/jpeg"),
|
||||
(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), "image/png")):
|
||||
started = time.monotonic()
|
||||
assert api.upload(alice, chat, data, 413, content_type=ctype).json()["error"]["code"] == "payload_too_large"
|
||||
assert time.monotonic() - started < 2 # refused from the header, nothing decoded
|
||||
|
||||
|
||||
def test_exactly_40_megapixels_is_accepted(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
data = imagegen.png(8000, 5000, compress_level=9) # 40 000 000 px, ~120 KB file
|
||||
att = api.upload(alice, chat, data).json()
|
||||
assert att["size_bytes"] == len(data)
|
||||
# Lab 5: the worker processes the biggest allowed picture within its memory limit.
|
||||
ready = api.wait_status(alice, att["id"], "ready", timeout=120)
|
||||
assert {v["name"]: (v["width"], v["height"]) for v in ready["variants"]} == \
|
||||
{"thumbnail": (256, 160), "crop": (128, 128), "optimized": (1280, 800)}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ content checks
|
||||
BROKEN = {
|
||||
"png: first 33 bytes of the fixture": (SAMPLE[:33], "image/png"),
|
||||
"png: truncated": (SAMPLE[:-20], "image/png"),
|
||||
"png: corrupt IDAT, valid CRC": (imagegen.png_corrupt_idat(), "image/png"),
|
||||
"png: fewer rows than IHDR": (imagegen.png_short_pixels(), "image/png"),
|
||||
"png: flipped byte": (SAMPLE[:700] + bytes([SAMPLE[700] ^ 0x5A]) + SAMPLE[701:], "image/png"),
|
||||
"png: bytes after IEND": (SAMPLE + b"<?php echo 1; ?>", "image/png"),
|
||||
"jpeg: no scan data after SOS": (imagegen.jpeg_without_scan_data(), "image/jpeg"),
|
||||
"jpeg: truncated": (imagegen.jpeg(400, 300, noise=True)[:5000], "image/jpeg"),
|
||||
"review: png row filter 5": (imagegen.png_bad_filter(5), "image/png"),
|
||||
"review: png row filter 0xff": (imagegen.png_bad_filter(0xFF), "image/png"),
|
||||
"review: png IDAT not-a-zlib-stream": (imagegen.png_idat_not_zlib(), "image/png"),
|
||||
"review: jpeg 27 bytes, no tables/scan": (imagegen.JPEG_27_BYTES, "image/jpeg"),
|
||||
"review: jpeg SOS component not in SOF": (imagegen.jpeg_sos_unknown_component(), "image/jpeg"),
|
||||
"mime: png bytes declared jpeg": (SAMPLE, "image/jpeg"),
|
||||
"mime: jpeg bytes declared png": (imagegen.jpeg(), "image/png"),
|
||||
"mime: gif declared png": (b"GIF89a" + bytes(100), "image/png"),
|
||||
"mime: html declared png": (b"<html><script>alert(1)</script></html>", "image/png"),
|
||||
"type: text/plain": (SAMPLE, "text/plain"),
|
||||
"type: octet-stream": (SAMPLE, "application/octet-stream"),
|
||||
"type: multipart": (SAMPLE, "multipart/form-data; boundary=x"),
|
||||
"type: missing": (SAMPLE, None),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", list(BROKEN))
|
||||
def test_bad_content_is_415(api, trio, name):
|
||||
alice, _, _, chat, _ = trio
|
||||
data, ctype = BROKEN[name]
|
||||
error = api.upload(alice, chat, data, 415, content_type=ctype).json()
|
||||
assert error["error"]["code"] == "unsupported_media_type"
|
||||
|
||||
|
||||
def test_empty_body_is_400(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
api.upload(alice, chat, b"", 400)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ filename
|
||||
@pytest.mark.parametrize("query", [
|
||||
"", "filename=", "filename=%20%20%20", "filename=.", "filename=..", "filename=a%2Fb.png", "filename=a%5Cb.png",
|
||||
"filename=a%00b.png", "filename=c1%C2%85.png", "filename=tab%09.png", "filename=photo%E2%80%AEgnp.exe",
|
||||
"filename=%FF%FE.png", "filename=a.png&filename=b.png", "filename=" + "a" * 256,
|
||||
], ids=lambda q: q[:40] or "missing")
|
||||
def test_bad_filename_is_400(api, trio, query):
|
||||
alice, _, _, chat, _ = trio
|
||||
api.upload(alice, chat, SAMPLE, 400, query=query)
|
||||
|
||||
|
||||
def test_unicode_filename_is_kept_and_safely_exposed(api, trio):
|
||||
alice, _, _, chat, _ = trio
|
||||
name = 'отчёт "Q3"; ok.png'
|
||||
att = api.upload(alice, chat, SAMPLE, filename=name).json()
|
||||
assert att["filename"] == name
|
||||
api.wait_status(alice, att["id"], "ready")
|
||||
disposition = content(api, alice, att["id"]).headers["content-disposition"]
|
||||
assert '"' not in disposition.split('filename="')[1].split('";')[0] # quotes neutralised in the fallback
|
||||
assert "filename*=UTF-8''" in disposition
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ variants
|
||||
def test_variant_parameter(api, trio):
|
||||
alice, _, eve, chat, _ = trio
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(alice, att["id"], "ready")
|
||||
assert content(api, alice, att["id"], query="?variant=original").content == SAMPLE
|
||||
assert content(api, alice, att["id"], query="?variant=thumbnail").headers["content-type"] == "image/jpeg"
|
||||
content(api, alice, att["id"], 400, query="?variant=huge")
|
||||
content(api, alice, att["id"], 400, query="?variant=thumbnail&variant=crop")
|
||||
content(api, eve, att["id"], 404, query="?variant=thumbnail") # outsider: 404, nothing revealed
|
||||
content(api, eve, att["id"], 404, query="?variant=huge") # outsider first, then the value
|
||||
@@ -0,0 +1,342 @@
|
||||
"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4).
|
||||
|
||||
Each test that stops, kills, rescales or isolates a replica restores the stand in `finally`
|
||||
and waits until HAProxy reports every replica UP again (conftest.wait_stand).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import httpx2 as httpx
|
||||
import pytest
|
||||
|
||||
from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \
|
||||
running_replicas, wait_healthy, wait_stand
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
|
||||
def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]:
|
||||
return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ level 3: replicas, no affinity
|
||||
def test_one_session_is_served_by_every_real_replica(api):
|
||||
actor = api.user("rr")
|
||||
seen = instances_seen(api, actor["token"])
|
||||
running = {r["hostname"] for r in running_replicas()}
|
||||
assert len(running) == REPLICAS >= 2
|
||||
assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers"
|
||||
# Round robin on ONE keep-alive client connection: consecutive requests switch replicas,
|
||||
# so there is no sticky session hiding split state.
|
||||
switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b)
|
||||
assert switches >= len(seen) // 2, seen
|
||||
assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running
|
||||
|
||||
|
||||
def test_write_on_one_replica_is_read_on_the_other(api):
|
||||
alice, bob = api.user("rw"), api.user("rw")
|
||||
chat = api.chat(alice, bob)
|
||||
for i in range(10):
|
||||
sent = api.post_message(alice, chat, {"text": f"m{i}"})
|
||||
read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])
|
||||
assert read.json()["items"][-1] == sent.json()
|
||||
logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
||||
for _ in range(6): # the revocation is visible on every replica at once (shared Valkey)
|
||||
r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
||||
assert r.headers["x-instance-id"]
|
||||
assert logout.headers["x-instance-id"]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ readiness drives the LB
|
||||
def hba_file() -> str:
|
||||
return psql("SHOW hba_file")
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def isolated_from_postgres(replica: dict):
|
||||
"""Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its
|
||||
connections), while it keeps running and answering /health/live. Always undone."""
|
||||
ip = replica["networks"]["data"] # PostgreSQL sees the replica's address on the data network
|
||||
path = hba_file()
|
||||
marker = "lab3-test-isolate"
|
||||
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}")
|
||||
try:
|
||||
psql("SELECT pg_reload_conf()")
|
||||
psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'")
|
||||
yield replica
|
||||
finally:
|
||||
compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}")
|
||||
psql("SELECT pg_reload_conf()")
|
||||
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: no restart of the replica needed
|
||||
wait_stand()
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def disconnected_from_data_network(replica: dict):
|
||||
"""`docker network disconnect <project>_data <api>`: the replica keeps its edge link to the
|
||||
LB but loses PostgreSQL AND Valkey. Always reconnected."""
|
||||
network = next(n for n in json.loads(subprocess.run(
|
||||
["docker", "inspect", "-f", "{{json .NetworkSettings.Networks}}", replica["id"]],
|
||||
capture_output=True, text=True, check=True).stdout) if n.endswith("_data"))
|
||||
subprocess.run(["docker", "network", "disconnect", network, replica["id"]], check=True, capture_output=True)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
subprocess.run(["docker", "network", "connect", network, replica["id"]], check=True, capture_output=True)
|
||||
lbctl.wait_up(REPLICAS, timeout=15) # self-heals: dead sockets are dropped (tcp_user_timeout)
|
||||
wait_stand()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("how", ["pg_hba", "network"])
|
||||
@pytest.mark.parametrize("position", range(REPLICAS))
|
||||
def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position, how):
|
||||
"""The reviewers' scenarios: one API loses PostgreSQL (pg_hba reject) or its whole data
|
||||
network (docker network disconnect) but keeps running. /health/live 200, /health/ready
|
||||
503 -> HAProxy must take it out; no business request may reach it."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
||||
actor = api.user("ready")
|
||||
isolate = isolated_from_postgres if how == "pg_hba" else disconnected_from_data_network
|
||||
with isolate(replica):
|
||||
started = time.monotonic()
|
||||
while replica["hostname"] in lbctl.up_instances():
|
||||
assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s"
|
||||
time.sleep(0.1)
|
||||
detected = time.monotonic() - started
|
||||
for _ in range(3): # the process stays responsive (event loop never blocked) ...
|
||||
started_probe = time.monotonic()
|
||||
assert probe_inside(replica, "/health/live") == 200
|
||||
assert probe_inside(replica, "/health/ready") == 503 # ... but not ready
|
||||
assert time.monotonic() - started_probe < 4.5, "replica froze while its dependencies were unreachable"
|
||||
seen = instances_seen(api, actor["token"], 40)
|
||||
assert replica["hostname"] not in seen and len(seen) == 40
|
||||
for i in range(20): # writes keep working, only through the ready replicas
|
||||
assert api.register(f"r{i}")["id"]
|
||||
print(f"{replica['name']} ({how}): unready replica out of rotation after {detected:.2f} s")
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ stopping replicas under traffic
|
||||
class Traffic:
|
||||
"""Background writers/readers through the LB; uncertain POSTs are retried with the SAME key."""
|
||||
|
||||
def __init__(self, api: Api, workers: int = 4):
|
||||
self.alice, self.bob = api.user("load"), api.user("load")
|
||||
self.chat = api.chat(self.alice, self.bob)
|
||||
self.workers = workers
|
||||
self.stop = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance)
|
||||
self.confirmed: list[dict] = []
|
||||
self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)]
|
||||
|
||||
def record(self, kind: str, status: int, instance: str | None) -> None:
|
||||
with self.lock:
|
||||
self.events.append((time.monotonic(), kind, status, instance))
|
||||
|
||||
def run(self, index: int) -> None:
|
||||
client = httpx.Client(base_url=BASE_URL, timeout=3)
|
||||
auth = {"Authorization": "Bearer " + self.alice["token"]}
|
||||
n = 0
|
||||
key, text = uuid.uuid4().hex, f"w{index}-0"
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text},
|
||||
headers={**auth, "Idempotency-Key": key})
|
||||
self.record("POST", r.status_code, r.headers.get("x-instance-id"))
|
||||
if r.status_code == 201:
|
||||
with self.lock:
|
||||
self.confirmed.append(r.json())
|
||||
n += 1
|
||||
key, text = uuid.uuid4().hex, f"w{index}-{n}"
|
||||
# any other outcome: keep key and text, retry the same request
|
||||
except httpx.HTTPError:
|
||||
self.record("POST", 0, None)
|
||||
try:
|
||||
r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth)
|
||||
self.record("GET", r.status_code, r.headers.get("x-instance-id"))
|
||||
except httpx.HTTPError:
|
||||
self.record("GET", 0, None)
|
||||
time.sleep(0.02)
|
||||
client.close()
|
||||
|
||||
def __enter__(self) -> "Traffic":
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
time.sleep(1.0)
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
self.stop.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=10)
|
||||
|
||||
def failures(self, since: float = 0.0) -> list[tuple]:
|
||||
return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)]
|
||||
|
||||
def check_history(self, api: Api) -> None:
|
||||
history = api.history(self.bob, self.chat, 100)[0]
|
||||
ids, texts = [m["id"] for m in history], [m["text"] for m in history]
|
||||
assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry"
|
||||
assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("position", range(REPLICAS))
|
||||
def test_graceful_stop_of_each_replica_loses_no_request(api, position):
|
||||
"""SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
||||
with Traffic(api) as traffic:
|
||||
started = time.monotonic()
|
||||
with replica_stopped(replica, "stop"):
|
||||
stopped_after = time.monotonic() - started
|
||||
time.sleep(3)
|
||||
after = [e for e in traffic.events if e[0] > time.monotonic() - 2]
|
||||
assert after and replica["hostname"] not in {e[3] for e in after}
|
||||
time.sleep(1)
|
||||
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
|
||||
check=True).stdout)[0]["State"]
|
||||
assert traffic.failures() == [], traffic.failures()[:10]
|
||||
assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)"
|
||||
assert state["Status"] == "running" # restored by replica_stopped
|
||||
traffic.check_history(api)
|
||||
print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("position", range(REPLICAS))
|
||||
def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position):
|
||||
"""SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within
|
||||
10 s, the killed replica gets nothing after detection, retries do not duplicate."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[position]
|
||||
with Traffic(api) as traffic:
|
||||
killed_at = time.monotonic()
|
||||
with replica_stopped(replica, "kill"):
|
||||
time.sleep(4)
|
||||
failures = traffic.failures(killed_at)
|
||||
last_failure = max((e[0] for e in failures), default=killed_at) - killed_at
|
||||
post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201]
|
||||
assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s"
|
||||
late = [e for e in traffic.events if e[0] > killed_at + 3.5]
|
||||
assert late and replica["hostname"] not in {e[3] for e in late}
|
||||
assert last_failure <= 10
|
||||
traffic.check_history(api)
|
||||
print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s")
|
||||
|
||||
|
||||
def test_graceful_shutdown_order_in_logs(api):
|
||||
"""Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0."""
|
||||
replica = sorted(running_replicas(), key=lambda r: r["name"])[-1]
|
||||
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1))
|
||||
with replica_stopped(replica, "stop"):
|
||||
logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True,
|
||||
text=True, encoding="utf-8").stdout
|
||||
lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout
|
||||
state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True,
|
||||
check=True).stdout)[0]["State"]
|
||||
events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")]
|
||||
names = [e["event"] for e in events]
|
||||
assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped")
|
||||
t = {e["event"]: e["ts"] for e in events}
|
||||
assert t["drain_started"] < t["shutdown_requested"]
|
||||
assert state["ExitCode"] == 0 and not state["OOMKilled"]
|
||||
assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ discovery: restart and scaling
|
||||
def test_full_restart_lb_rediscovers_all_replicas(api):
|
||||
actor = api.user("restart")
|
||||
before = {r["hostname"]: r["started_at"] for r in running_replicas()}
|
||||
run_hook("restart.sh")
|
||||
wait_healthy("postgres", "valkey", "api")
|
||||
wait_stand()
|
||||
after = {r["hostname"]: r["started_at"] for r in running_replicas()}
|
||||
assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted"
|
||||
assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed
|
||||
|
||||
|
||||
def test_scale_up_and_down_is_picked_up_without_errors(api):
|
||||
with Traffic(api, workers=2) as traffic:
|
||||
try:
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api",
|
||||
timeout=180)
|
||||
lbctl.wait_up(REPLICAS + 1, timeout=30)
|
||||
actor = api.user("scale")
|
||||
assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api",
|
||||
timeout=180)
|
||||
wait_stand()
|
||||
time.sleep(1)
|
||||
assert traffic.failures() == [], traffic.failures()[:10]
|
||||
traffic.check_history(api)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ level 4: deployment policy
|
||||
def test_compose_policy_limits_ports_secrets_images():
|
||||
config = json.loads(compose("config", "--format", "json").stdout)
|
||||
services = config["services"]
|
||||
# auditor and jobctl (profile "tools") are started on demand and not part of the running stand.
|
||||
assert set(services) == {"postgres", "valkey", "s3", "broker", "migrate", "s3-init", "api", "lb", "relay", "worker"}
|
||||
for name, svc in services.items():
|
||||
limits = svc.get("deploy", {}).get("resources", {}).get("limits", {})
|
||||
assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits"
|
||||
assert "container_name" not in svc, f"{name}: container_name prevents scaling"
|
||||
if name in ("api", "migrate"):
|
||||
assert "build" in svc # built from this repository's source
|
||||
elif name in ("s3-init", "relay", "worker"):
|
||||
assert svc["image"] == services["api"]["image"] # the same locally built image
|
||||
else:
|
||||
assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest"
|
||||
assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports")
|
||||
published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")}
|
||||
assert set(published) == {"lb"} and all(p["host_ip"] in ("127.0.0.1", "::1") for p in published["lb"])
|
||||
secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()}
|
||||
s3_keys = sorted(f"s3_{who}_{part}" for who in ("admin", "api", "worker", "auditor")
|
||||
for part in ("access_key_id", "secret_access_key"))
|
||||
assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], "s3": s3_keys,
|
||||
"migrate": ["database_url"], "s3-init": s3_keys, "lb": [],
|
||||
"broker": ["rabbitmq_users"],
|
||||
# the API: its own S3 key pair only (no admin/worker keys, no broker credentials)
|
||||
"api": ["cache_url", "database_url", "s3_api_access_key_id", "s3_api_secret_access_key"],
|
||||
# relay: database + its broker user only; worker: + its own S3 key pair
|
||||
"relay": ["broker_url_relay", "database_url"],
|
||||
"worker": ["broker_url_worker", "database_url", "s3_worker_access_key_id",
|
||||
"s3_worker_secret_access_key"]}
|
||||
volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"]
|
||||
for name, svc in services.items()}
|
||||
assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "s3": ["s3data"], "migrate": [],
|
||||
"s3-init": [], "api": [], "lb": [], "broker": ["rabbitmqdata"], "relay": [], "worker": []}
|
||||
networks = {name: sorted(svc.get("networks", {})) for name, svc in services.items()}
|
||||
assert networks == {"postgres": ["data"], "valkey": ["data"], "s3": ["storage"], "migrate": ["data"],
|
||||
"s3-init": ["storage"], "api": ["data", "edge", "storage"], "broker": ["queue"],
|
||||
"relay": ["data", "queue"], "worker": ["data", "queue", "storage"],
|
||||
"lb": ["edge"]}, networks # the LB cannot reach the stores; the API not the broker
|
||||
for network in ("data", "storage", "queue"):
|
||||
assert config["networks"][network].get("internal") is True, network
|
||||
|
||||
|
||||
def test_running_containers_have_limits_and_no_bypass_ports():
|
||||
ids = compose("ps", "-q").stdout.split()
|
||||
for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True,
|
||||
check=True).stdout):
|
||||
name = item["Name"]
|
||||
assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name
|
||||
bindings = item["HostConfig"]["PortBindings"] or {}
|
||||
if "-lb-" in name:
|
||||
assert all(b["HostIp"] in ("127.0.0.1", "::1") for v in bindings.values() for b in v) # loopback only
|
||||
else:
|
||||
assert not bindings, f"{name} publishes ports in bypass of the LB"
|
||||
|
||||
|
||||
def test_migrations_ran_once_by_the_migrate_service():
|
||||
applied = psql("SELECT version FROM schema_migrations ORDER BY version").split()
|
||||
assert applied == ["0001_init", "0002_idempotency_keys", "0003_post_message_function", "0004_attachments",
|
||||
"0005_pipeline"]
|
||||
migrate = compose("ps", "-a", "--format", "json", "migrate").stdout
|
||||
rows = [json.loads(line) for line in migrate.splitlines() if line.strip()]
|
||||
assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows)
|
||||
api_logs = compose("logs", "--no-log-prefix", "api").stdout
|
||||
assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Consistency of PostgreSQL and S3 around one upload (lab 4 level 4, lab 5 failure windows of the API).
|
||||
|
||||
Lab 5 upload: intent row 'pending' -> staging PUT -> ONE transaction 'queued' + job + outbox -> 202.
|
||||
Failures between the steps are injected with FAULT_POINTS of the api service (compose variable
|
||||
API_FAULT_POINTS, empty in normal runs; the API raises 503 at that exact point, as if the process
|
||||
or the network failed there):
|
||||
|
||||
* after_s3_put "after staging, before the job is written": staging object exists, row
|
||||
still pending, NO job and NO outbox row; the client got 503 -> invisible,
|
||||
the janitor removes the staging object and the row after PENDING_GRACE_SECONDS;
|
||||
* after_enqueue_commit job committed, the answer is lost (client got 503) -> nothing may be
|
||||
deleted: the worker processes the job and the attachment becomes ready.
|
||||
|
||||
Plus: an upload interrupted by the client and a stalled body leave nothing behind."""
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import api_with_env, attachment_row, psql, s3_size
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
def rows_of_chat(chat_id: str) -> list[tuple[str, str, str]]:
|
||||
out = psql(f"SELECT id || ' ' || status || ' ' || staging_key FROM attachments WHERE chat_id = '{chat_id}'")
|
||||
return [tuple(line.split()) for line in out.splitlines() if line]
|
||||
|
||||
|
||||
def jobs_of(attachment_id: str) -> tuple[int, int]:
|
||||
"""(jobs rows, outbox rows) of an attachment."""
|
||||
out = psql(f"SELECT (SELECT count(*) FROM jobs WHERE attachment_id = '{attachment_id}') || ' ' || "
|
||||
f"(SELECT count(*) FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}')")
|
||||
jobs, outbox = out.split()
|
||||
return int(jobs), int(outbox)
|
||||
|
||||
|
||||
def wait_for(condition, timeout: float, what: str) -> None:
|
||||
deadline = time.monotonic() + timeout
|
||||
while not condition():
|
||||
assert time.monotonic() < deadline, what
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def raw_upload(token: str, chat_id: str, declared: int, body: bytes, *, wait_answer: float | None) -> bytes:
|
||||
"""Hand-made HTTP request: announce `declared` bytes, send only `body`, then either close the
|
||||
socket (client gone) or keep it open and read the answer after `wait_answer` seconds."""
|
||||
sock = socket.create_connection(("127.0.0.1", 8080), timeout=30)
|
||||
head = (f"POST /api/v1/chats/{chat_id}/attachments?filename=x.png HTTP/1.1\r\nHost: localhost\r\n"
|
||||
f"Authorization: Bearer {token}\r\nContent-Type: image/png\r\nContent-Length: {declared}\r\n\r\n")
|
||||
sock.sendall(head.encode() + body)
|
||||
if wait_answer is None:
|
||||
sock.close()
|
||||
return b""
|
||||
time.sleep(wait_answer)
|
||||
answer = sock.recv(65536)
|
||||
sock.close()
|
||||
return answer
|
||||
|
||||
|
||||
def test_interrupted_and_stalled_uploads_leave_nothing(api):
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
data = imagegen.png_of_size(2_000_000)
|
||||
raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=None) # client disconnects
|
||||
with api_with_env(UPLOAD_READ_TIMEOUT_SECONDS="2"):
|
||||
answer = raw_upload(alice["token"], chat["id"], len(data), data[:300_000], wait_answer=5) # client stalls
|
||||
assert answer.startswith(b"HTTP/1.1 400") and b"invalid_request" in answer, answer[:200]
|
||||
assert b"x-instance-id" in answer.lower()
|
||||
time.sleep(1)
|
||||
assert rows_of_chat(chat["id"]) == [] # nothing was written: the body is checked before any write
|
||||
|
||||
|
||||
def test_crash_after_staging_before_job_is_cleaned_by_janitor(api):
|
||||
"""Failure window 1 of contracts/attachment-states.md: the original is in staging, the job is
|
||||
not written. No 202 was sent, so nothing was promised: the janitor removes the staging copy
|
||||
(object first) and then the intent row; no job, no message ever existed."""
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
with api_with_env(API_FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
|
||||
assert api.upload(alice, chat, SAMPLE, 503).json()["error"]["code"] == "unavailable"
|
||||
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
|
||||
assert status == "pending" and s3_size(key, "staging") == len(SAMPLE) # the orphan-to-be exists
|
||||
assert jobs_of(attachment_id) == (0, 0) # no job, no outbox row
|
||||
api.call("GET", f"/api/v1/attachments/{attachment_id}", 404, token=alice["token"]) # invisible
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [attachment_id]}, 404) # not attachable
|
||||
wait_for(lambda: rows_of_chat(chat["id"]) == [], 30, "janitor did not remove the stale intent")
|
||||
assert s3_size(key, "staging") is None # object removed first
|
||||
# Normal configuration again: an upload in the same chat works and is not touched.
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
assert api.wait_status(alice, att["id"], "ready")["status"] == "ready"
|
||||
|
||||
|
||||
def test_lost_answer_after_job_commit_still_becomes_ready(api):
|
||||
"""The job COMMIT succeeded, the answer was lost (503). Nothing compensates in the request path
|
||||
(the janitor only touches 'pending' rows); the worker processes the job as usual."""
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
with api_with_env(API_FAULT_POINTS="after_enqueue_commit", PENDING_GRACE_SECONDS="3", JANITOR_INTERVAL_SECONDS="1"):
|
||||
api.upload(alice, chat, SAMPLE, 503)
|
||||
[(attachment_id, status, key)] = rows_of_chat(chat["id"])
|
||||
assert status in ("queued", "processing", "ready") and jobs_of(attachment_id)[0] == 1
|
||||
att = api.wait_status(bob, attachment_id, "ready")
|
||||
time.sleep(4) # several janitor rounds
|
||||
assert attachment_row(attachment_id)["status"] == "ready"
|
||||
got = api.call("GET", f"/api/v1/attachments/{attachment_id}/content", 200, token=bob["token"])
|
||||
assert got.content == SAMPLE and att["sha256"] == __import__("hashlib").sha256(SAMPLE).hexdigest()
|
||||
|
||||
|
||||
def test_fresh_pending_upload_is_not_touched_by_janitor(api):
|
||||
"""An intent younger than PENDING_GRACE_SECONDS is an upload in progress: never removed."""
|
||||
alice = api.user("alice")
|
||||
chat = api.chat(alice)
|
||||
with api_with_env(API_FAULT_POINTS="after_s3_put", PENDING_GRACE_SECONDS="600", JANITOR_INTERVAL_SECONDS="1"):
|
||||
api.upload(alice, chat, SAMPLE, 503)
|
||||
time.sleep(4)
|
||||
[(_, status, key)] = rows_of_chat(chat["id"])
|
||||
assert status == "pending" and s3_size(key, "staging") == len(SAMPLE)
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Database-level guarantees, checked over HTTP and verified directly in PostgreSQL / Valkey."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import PASSWORD, Api, compose, psql
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
|
||||
def run_concurrently(workers: int, target) -> list:
|
||||
barrier = threading.Barrier(workers)
|
||||
results: list = [None] * workers
|
||||
|
||||
def body(index: int) -> None:
|
||||
local = Api() # one HTTP client (connection) per worker
|
||||
barrier.wait() # all workers fire at the same moment
|
||||
try:
|
||||
results[index] = target(local, index)
|
||||
except BaseException as exc: # noqa: BLE001 - reported by the test
|
||||
results[index] = exc
|
||||
|
||||
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
return results
|
||||
|
||||
|
||||
def test_registration_race_exactly_one_201_and_seven_409():
|
||||
username = "race_" + uuid.uuid4().hex[:10]
|
||||
|
||||
def register(client: Api, _index: int) -> int:
|
||||
body = {"username": username, "display_name": "Гонка", "password": PASSWORD}
|
||||
response = client.client.post("/api/v1/users", json=body)
|
||||
return response.status_code
|
||||
|
||||
statuses = run_concurrently(8, register)
|
||||
# Real HTTP codes are compared: a timeout, 500 or exception fails the test instead of counting as 409.
|
||||
assert sorted(statuses, key=str) == [201] + [409] * 7, statuses
|
||||
assert psql(f"SELECT count(*) FROM users WHERE username = '{username}'") == "1"
|
||||
|
||||
|
||||
def _ins_counter() -> int:
|
||||
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
|
||||
|
||||
|
||||
def test_create_chat_rolls_back_after_the_chat_row_was_inserted(api):
|
||||
"""The chat INSERT runs first; the unknown member then breaks the membership INSERT (FK).
|
||||
|
||||
Proof that the first INSERT really happened and was rolled back: PostgreSQL counts
|
||||
attempted inserts in pg_stat_user_tables.n_tup_ins even for aborted transactions,
|
||||
while the table contains no such chat and no new membership rows.
|
||||
"""
|
||||
alice = api.user("rollback")
|
||||
title = "rollback-" + uuid.uuid4().hex
|
||||
members_before = int(psql("SELECT count(*) FROM chat_members"))
|
||||
attempts_before = _ins_counter()
|
||||
error = api.call("POST", "/api/v1/chats", 400, token=alice["token"],
|
||||
json_body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}).json()
|
||||
assert "несуществующих" in error["error"]["message"]
|
||||
deadline = time.monotonic() + 20 # statistics are flushed by the API backend asynchronously
|
||||
while _ins_counter() == attempts_before and time.monotonic() < deadline:
|
||||
time.sleep(0.5)
|
||||
assert _ins_counter() > attempts_before, "the chat INSERT was never attempted"
|
||||
assert psql(f"SELECT count(*) FROM chats WHERE title = convert_to('{title}', 'UTF8')") == "0"
|
||||
assert int(psql("SELECT count(*) FROM chat_members")) == members_before
|
||||
|
||||
|
||||
def test_migrations_rerun_is_a_noop():
|
||||
users_before = psql("SELECT count(*) FROM users")
|
||||
result = compose("run", "--rm", "--no-deps", "migrate")
|
||||
assert "0001_init: already applied, skipped" in result.stdout
|
||||
assert "0002_idempotency_keys: already applied, skipped" in result.stdout
|
||||
assert "0003_post_message_function: already applied, skipped" in result.stdout
|
||||
assert "0004_attachments: already applied, skipped" in result.stdout
|
||||
assert "0005_pipeline: already applied, skipped" in result.stdout
|
||||
assert psql("SELECT count(*) FROM schema_migrations") == "5"
|
||||
assert psql("SELECT count(*) FROM users") == users_before
|
||||
|
||||
|
||||
def test_100_messages_from_10_clients_keep_order_and_cursor_walk(api):
|
||||
clients = [api.user(f"w{i}") for i in range(10)]
|
||||
chat = api.chat(clients[0], *clients[1:])
|
||||
|
||||
def writer(client: Api, index: int) -> list[dict]:
|
||||
return [client.send(clients[index], chat, f"c{index}-m{j}") for j in range(10)]
|
||||
|
||||
results = run_concurrently(10, writer)
|
||||
assert all(isinstance(r, list) for r in results), results
|
||||
confirmed = [m for batch in results for m in batch]
|
||||
history, pages = api.history(clients[5], chat, 7)
|
||||
ids = [m["id"] for m in history]
|
||||
assert len(confirmed) == 100 and len(history) == 100 and len(set(ids)) == 100 and pages == 15
|
||||
assert set(ids) == {m["id"] for m in confirmed}
|
||||
keys = [(m["created_at"], uuid.UUID(m["id"]).int) for m in history]
|
||||
assert all(a < b for a, b in zip(keys, keys[1:])), "history is not strictly ordered"
|
||||
stamps = [m["created_at"] for m in history]
|
||||
assert len(set(stamps)) == 100, "per-chat clock must give unique timestamps"
|
||||
for i, client in enumerate(clients):
|
||||
own = [m["text"] for m in history if m["sender_id"] == client["id"]]
|
||||
assert own == [f"c{i}-m{j}" for j in range(10)]
|
||||
|
||||
|
||||
def test_secrets_are_not_stored_in_plain_text(api):
|
||||
actor = api.user("store")
|
||||
stored = psql(f"SELECT password_hash FROM users WHERE id = '{actor['id']}'")
|
||||
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and PASSWORD not in stored
|
||||
token_hex = actor["token"].encode().hex()
|
||||
keys = compose("exec", "-T", "valkey", "sh", "-c",
|
||||
'REDISCLI_AUTH="$(cat /run/secrets/valkey_password)" valkey-cli --scan --pattern "session:*"').stdout.split()
|
||||
assert keys and all(len(k) == len("session:") + 64 for k in keys)
|
||||
assert not any(actor["token"] in k or token_hex in k for k in keys)
|
||||
@@ -0,0 +1,35 @@
|
||||
"""The history index really bounds the work of a late page (>= 10 000 messages, generic plan)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
import explain_history # noqa: E402
|
||||
|
||||
|
||||
def section(text: str, number: int) -> str:
|
||||
parts = re.split(r"^=== ", text, flags=re.M)
|
||||
return next(p for p in parts if p.startswith(f"{number}."))
|
||||
|
||||
|
||||
def buffers(plan: str) -> int:
|
||||
return int(re.search(r"Buffers: shared hit=(\d+)", plan).group(1))
|
||||
|
||||
|
||||
def test_late_page_uses_index_range_not_filter():
|
||||
text = explain_history.run()
|
||||
assert "20000" in text # the target chat holds 20 000 messages
|
||||
app_late = section(text, 2)
|
||||
assert "Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))" in app_late
|
||||
assert "Rows Removed by Filter" not in app_late and "Sort" not in app_late
|
||||
rejected = section(text, 3)
|
||||
assert "Rows Removed by Filter: 19950" in rejected
|
||||
# The app query touches a handful of pages; the rejected variants touch the whole prefix.
|
||||
assert buffers(app_late) <= 10 < buffers(rejected) and buffers(app_late) <= 10 < buffers(section(text, 4))
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards.
|
||||
|
||||
Every test stops/pauses a real container and restores it in `finally` (service_down).
|
||||
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
|
||||
|
||||
Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer
|
||||
503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same
|
||||
contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401
|
||||
for a valid token. Liveness/readiness of the processes are checked inside each container.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead
|
||||
CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
|
||||
|
||||
|
||||
def timed(api, *args, **kwargs):
|
||||
started = time.monotonic()
|
||||
response = api.call(*args, lb_error_ok=True, **kwargs)
|
||||
return response, time.monotonic() - started
|
||||
|
||||
|
||||
def processes_alive_but_not_ready() -> None:
|
||||
for replica in running_replicas():
|
||||
assert probe_inside(replica, "/health/live") == 200
|
||||
assert probe_inside(replica, "/health/ready") == 503
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def prepared(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
chat = api.chat(alice, bob)
|
||||
message = api.send(alice, chat, "до отказа")
|
||||
return alice, bob, chat, message
|
||||
|
||||
|
||||
def check_intact(api, alice, bob, chat, message):
|
||||
"""After recovery the SAME tokens work and confirmed data is unchanged."""
|
||||
assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"]
|
||||
items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"]
|
||||
assert items == [message]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("how", ["stop", "pause"])
|
||||
def test_valkey_failure(api, prepared, how):
|
||||
alice, bob, chat, message = prepared
|
||||
with service_down("valkey", how):
|
||||
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
|
||||
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
|
||||
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
|
||||
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
|
||||
api.call("GET", "/health/ready", 503, lb_error_ok=True)
|
||||
processes_alive_but_not_ready()
|
||||
check_intact(api, alice, bob, chat, message)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("how", ["stop", "pause"])
|
||||
def test_postgres_failure(api, prepared, how):
|
||||
alice, bob, chat, message = prepared
|
||||
with service_down("postgres", how):
|
||||
for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
|
||||
response, took = timed(api, "GET", path, 503, token=alice["token"])
|
||||
assert response.json()["error"]["code"] == "unavailable", path
|
||||
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
|
||||
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access
|
||||
api.call("GET", "/health/ready", 503, lb_error_ok=True)
|
||||
processes_alive_but_not_ready()
|
||||
check_intact(api, alice, bob, chat, message)
|
||||
|
||||
|
||||
def test_locked_table_gives_bounded_503(api, prepared):
|
||||
"""A statement stuck behind a lock is stopped by lock_timeout, not after minutes."""
|
||||
alice, bob, chat, message = prepared
|
||||
locker = subprocess.Popen(
|
||||
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
|
||||
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"],
|
||||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
try:
|
||||
time.sleep(1.5) # let the lock be taken
|
||||
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
|
||||
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
|
||||
assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready"
|
||||
finally:
|
||||
locker.wait(timeout=30)
|
||||
check_intact(api, alice, bob, chat, message)
|
||||
|
||||
|
||||
def test_logout_during_valkey_outage_is_never_lost(api, prepared):
|
||||
"""Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten':
|
||||
during the outage logout answers 503 (nothing revoked, the client knows it failed);
|
||||
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
|
||||
alice, bob, chat, message = prepared
|
||||
with service_down("valkey", "stop"):
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True)
|
||||
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
||||
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
||||
restart_services("valkey", "api")
|
||||
for _ in range(10): # repeated checks: the revoked token must never come back
|
||||
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
|
||||
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"})
|
||||
api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart
|
||||
@@ -0,0 +1,281 @@
|
||||
"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer.
|
||||
|
||||
Strict assertions: exact HTTP codes for every request, identical bodies for replays,
|
||||
the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica
|
||||
answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always
|
||||
undone in `finally`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import httpx2 as httpx
|
||||
import pytest
|
||||
|
||||
from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
|
||||
def messages_in_db(chat: dict) -> int:
|
||||
return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
|
||||
|
||||
|
||||
def key_rows(actor: dict, key: str) -> int:
|
||||
return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pair(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
return alice, bob, api.chat(alice, bob)
|
||||
|
||||
|
||||
def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair):
|
||||
alice, bob, chat = pair
|
||||
key = "replay-" + uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, {"text": "один раз"}, key=key)
|
||||
served = {first.headers["x-instance-id"]}
|
||||
for _ in range(20):
|
||||
again = api.post_message(alice, chat, {"text": "один раз"}, key=key)
|
||||
assert again.status_code == 201 and again.json() == first.json()
|
||||
served.add(again.headers["x-instance-id"])
|
||||
assert len(served) >= 2, "replays must be answered by different replicas, not a local cache"
|
||||
assert api.history(bob, chat, 100)[0] == [first.json()]
|
||||
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair):
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
original = api.send(alice, chat, "оригинал", key=key)
|
||||
for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}):
|
||||
assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict"
|
||||
assert api.send(alice, chat, "оригинал", key=key) == original
|
||||
assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair):
|
||||
alice, _, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json()
|
||||
for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}',
|
||||
'{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'):
|
||||
assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first
|
||||
assert messages_in_db(chat) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"])
|
||||
def test_missing_or_invalid_key_is_400(api, pair, value):
|
||||
alice, _, chat = pair
|
||||
raw_value = value.encode("utf-8")
|
||||
error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
|
||||
json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json()
|
||||
assert error["error"]["code"] == "invalid_request"
|
||||
assert messages_in_db(chat) == 0
|
||||
|
||||
|
||||
def test_repeated_key_header_is_400(pair):
|
||||
alice, _, chat = pair
|
||||
response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10,
|
||||
headers=[("Authorization", "Bearer " + alice["token"]),
|
||||
("Idempotency-Key", "one"), ("Idempotency-Key", "two")])
|
||||
assert response.status_code == 400 and response.headers["x-instance-id"]
|
||||
assert messages_in_db(chat) == 0
|
||||
|
||||
|
||||
def test_longest_key_and_every_allowed_character(api, pair):
|
||||
alice, _, chat = pair
|
||||
for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex):
|
||||
first = api.send(alice, chat, key[:20], key=key)
|
||||
assert api.send(alice, chat, key[:20], key=key) == first
|
||||
assert messages_in_db(chat) == 2
|
||||
|
||||
|
||||
def test_scope_is_user_plus_chat_plus_key(api, pair):
|
||||
alice, bob, chat = pair
|
||||
other = api.chat(alice, bob, title="второй")
|
||||
key = "shared-key"
|
||||
a1 = api.send(alice, chat, "текст", key=key)
|
||||
b1 = api.send(bob, chat, "текст", key=key) # another user: independent
|
||||
a2 = api.send(alice, other, "текст", key=key) # another chat: independent
|
||||
assert len({a1["id"], b1["id"], a2["id"]}) == 3
|
||||
assert messages_in_db(chat) == 2 and messages_in_db(other) == 1
|
||||
|
||||
|
||||
def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair):
|
||||
alice, bob, chat = pair
|
||||
eve = api.user("eve")
|
||||
key = uuid.uuid4().hex
|
||||
api.send(alice, chat, "секрет", key=key)
|
||||
for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message
|
||||
assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found"
|
||||
api.post_message(eve, chat, {"text": "другое"}, 404, key=key)
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
|
||||
api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again
|
||||
assert messages_in_db(chat) == 1
|
||||
|
||||
|
||||
def fire_together(workers: int, request) -> list:
|
||||
barrier = threading.Barrier(workers)
|
||||
results: list = [None] * workers
|
||||
|
||||
def body(index: int) -> None:
|
||||
client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them
|
||||
barrier.wait()
|
||||
try:
|
||||
results[index] = request(client, index)
|
||||
except BaseException as exc: # noqa: BLE001 - asserted by the test
|
||||
results[index] = exc
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
return results
|
||||
|
||||
|
||||
def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair):
|
||||
alice, bob, chat = pair
|
||||
key = "race-" + uuid.uuid4().hex
|
||||
|
||||
def post(client: httpx.Client, _index: int):
|
||||
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"},
|
||||
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
||||
return r.status_code, r.headers.get("x-instance-id"), r.json()
|
||||
|
||||
for _ in range(3): # three rounds: the first creates, later rounds replay concurrently
|
||||
results = fire_together(24, post)
|
||||
assert all(isinstance(r, tuple) for r in results), results
|
||||
assert [r[0] for r in results] == [201] * 24, [r[0] for r in results]
|
||||
assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message"
|
||||
assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas"
|
||||
assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_concurrent_same_key_different_bodies_one_wins(pair):
|
||||
alice, _, chat = pair
|
||||
key = "conflict-" + uuid.uuid4().hex
|
||||
|
||||
def post(client: httpx.Client, index: int):
|
||||
r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"},
|
||||
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
||||
return r.status_code, r.json()
|
||||
|
||||
results = fire_together(20, post)
|
||||
winners = [body for status, body in results if status == 201]
|
||||
losers = [body for status, body in results if status == 409]
|
||||
assert len(winners) + len(losers) == 20, results
|
||||
assert winners and len({str(b) for b in winners}) == 1
|
||||
assert all(b["error"]["code"] == "conflict" for b in losers)
|
||||
assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"}
|
||||
assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"])
|
||||
assert messages_in_db(chat) == 1
|
||||
|
||||
|
||||
def inserts_into_keys() -> int:
|
||||
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'"))
|
||||
|
||||
|
||||
def test_failure_after_the_key_insert_rolls_everything_back(api, pair):
|
||||
"""A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same
|
||||
transaction. Expected: 500 for the client, no key row and no message left behind
|
||||
(n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds."""
|
||||
alice, bob, chat = pair
|
||||
marker = "boom-" + uuid.uuid4().hex
|
||||
key = uuid.uuid4().hex
|
||||
psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ "
|
||||
f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; "
|
||||
"RETURN NEW; END $$; "
|
||||
"CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()")
|
||||
try:
|
||||
before = inserts_into_keys()
|
||||
error = api.post_message(alice, chat, {"text": marker}, 500, key=key)
|
||||
assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text
|
||||
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
|
||||
deadline = time.monotonic() + 20 # table statistics are flushed asynchronously
|
||||
while inserts_into_keys() == before and time.monotonic() < deadline:
|
||||
time.sleep(0.5)
|
||||
assert inserts_into_keys() > before, "the key INSERT never ran before the failure"
|
||||
finally:
|
||||
psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()")
|
||||
retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded
|
||||
assert api.send(alice, chat, marker, key=key) == retried
|
||||
assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen:
|
||||
sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; "
|
||||
f"SELECT pg_sleep({seconds}); COMMIT;")
|
||||
return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d",
|
||||
"messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def wait_lock_taken(chat: dict) -> None:
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' "
|
||||
"AND pid <> pg_backend_pid()") != "0":
|
||||
return
|
||||
time.sleep(0.2)
|
||||
raise AssertionError("the row lock was not taken")
|
||||
|
||||
|
||||
def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair):
|
||||
"""Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row
|
||||
lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one."""
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
locker = lock_chat_row(chat, 1.0)
|
||||
try:
|
||||
wait_lock_taken(chat)
|
||||
with pytest.raises(httpx.TimeoutException):
|
||||
httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4,
|
||||
headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key})
|
||||
finally:
|
||||
locker.wait(timeout=30)
|
||||
deadline = time.monotonic() + 10
|
||||
while messages_in_db(chat) == 0 and time.monotonic() < deadline:
|
||||
time.sleep(0.2)
|
||||
stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'")
|
||||
assert stored, "the server-side transaction was expected to commit after the lock was released"
|
||||
retried = api.send(alice, chat, "неопределённо", key=key)
|
||||
assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair):
|
||||
"""The row lock outlives lock_timeout: the API answers 503 within its deadline and stores
|
||||
nothing; after the lock is gone the same key creates the message exactly once."""
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
locker = lock_chat_row(chat, 4.0)
|
||||
try:
|
||||
wait_lock_taken(chat)
|
||||
started = time.monotonic()
|
||||
api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key)
|
||||
assert time.monotonic() - started < 3.6
|
||||
assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0
|
||||
finally:
|
||||
locker.wait(timeout=30)
|
||||
created = api.send(alice, chat, "после блокировки", key=key)
|
||||
assert api.send(alice, chat, "после блокировки", key=key) == created
|
||||
assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1
|
||||
|
||||
|
||||
def test_replay_after_the_serving_replica_is_stopped(api, pair):
|
||||
alice, bob, chat = pair
|
||||
key = uuid.uuid4().hex
|
||||
first = api.post_message(alice, chat, {"text": "до остановки"}, key=key)
|
||||
served_by = first.headers["x-instance-id"]
|
||||
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
|
||||
with replica_stopped(replica):
|
||||
for _ in range(5):
|
||||
again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key)
|
||||
assert again.json() == first.json() and again.headers["x-instance-id"] != served_by
|
||||
assert messages_in_db(chat) == 1
|
||||
@@ -0,0 +1,501 @@
|
||||
"""Lab 5 pipeline against the running stand: 202 -> queued -> ready, variants by the contract,
|
||||
the failure windows of contracts/attachment-states.md, retries, the failed registry and replay,
|
||||
lease expiry of a hung worker, cleanup that never deletes accepted work.
|
||||
|
||||
Failures are injected with the real mechanisms - docker stop/pause/restart of a service, and
|
||||
FAULT_POINTS of ONE service recreated with an override (conftest.service_with_env) - and every
|
||||
test restores the normal stand in `finally`. Each test checks exact HTTP codes AND the rows in
|
||||
PostgreSQL AND the objects in S3, then the read-only invariant audit (I1-I7) of the whole stand."""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
import imagegen
|
||||
from conftest import (attachment_row, audit, compose, events_of, job_rows, jobctl, psql, queue_depth, s3_keys,
|
||||
s3_size, service_down, service_with_env, wait_healthy, wait_stand)
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
FAST_LEASE = {"JOB_LEASE_SECONDS": "6"} # the reaper notices a dead attempt within ~11 s
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def pair(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
return alice, bob, api.chat(alice, bob)
|
||||
|
||||
|
||||
def wait_for(condition, timeout: float, what: str):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
value = condition()
|
||||
if value:
|
||||
return value
|
||||
assert time.monotonic() < deadline, what
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def check_variants(api, actor, att: dict, source: bytes) -> dict[str, bytes]:
|
||||
"""Download every variant: real format and size decoded with Pillow, size/sha256 = metadata."""
|
||||
expected = {"thumbnail": ("image/jpeg", "JPEG"), "crop": ("image/jpeg", "JPEG"), "optimized": ("image/webp", "WEBP")}
|
||||
assert [v["name"] for v in att["variants"]] == list(expected)
|
||||
src = Image.open(io.BytesIO(source))
|
||||
w, h = src.size
|
||||
out = {}
|
||||
for v in att["variants"]:
|
||||
got = api.content(actor, att["id"], v["name"])
|
||||
assert got.headers["content-type"] == expected[v["name"]][0] == v["content_type"]
|
||||
assert len(got.content) == v["size_bytes"] and hashlib.sha256(got.content).hexdigest() == v["sha256"]
|
||||
img = Image.open(io.BytesIO(got.content))
|
||||
assert img.format == expected[v["name"]][1] and img.size == (v["width"], v["height"])
|
||||
out[v["name"]] = got.content
|
||||
def fit(box):
|
||||
if w <= box and h <= box:
|
||||
return w, h
|
||||
return (box, max(1, h * box // w)) if w >= h else (max(1, w * box // h), box)
|
||||
sizes = {v["name"]: (v["width"], v["height"]) for v in att["variants"]}
|
||||
assert sizes == {"thumbnail": fit(256), "crop": (128, 128), "optimized": fit(1280)}
|
||||
original = api.content(actor, att["id"])
|
||||
assert original.content == source # the original is published byte for byte
|
||||
return out
|
||||
|
||||
|
||||
def published_keys(attachment_id: str) -> list[str]:
|
||||
return psql(f"SELECT object_key FROM attachment_variants WHERE attachment_id = '{attachment_id}' "
|
||||
f"UNION ALL SELECT object_key FROM attachments WHERE id = '{attachment_id}' ORDER BY 1").split()
|
||||
|
||||
|
||||
def attempts(attachment_id: str) -> list[dict]:
|
||||
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', a.attempt, 'outcome', a.outcome, "
|
||||
"'error_code', a.error_code, 'token', a.token, 'cleaned', a.objects_cleaned_at IS NOT NULL, "
|
||||
"'keys', a.planned_keys) ORDER BY a.started_at), '[]') FROM job_attempts a JOIN jobs j ON j.id = a.job_id "
|
||||
f"WHERE j.attachment_id = '{attachment_id}'")
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def outbox_of(attachment_id: str) -> list[dict]:
|
||||
out = psql("SELECT coalesce(json_agg(json_build_object('attempt', o.attempt, 'published', o.published_at IS NOT NULL) "
|
||||
f"ORDER BY o.id), '[]') FROM outbox o JOIN jobs j ON j.id = o.job_id WHERE j.attachment_id = '{attachment_id}'")
|
||||
return json.loads(out)
|
||||
|
||||
|
||||
def names(events: list[dict]) -> list[str]:
|
||||
return [e["event"] for e in events]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ level 3: the happy path
|
||||
def test_upload_is_202_queued_then_ready_with_contract_variants(api, pair):
|
||||
alice, bob, chat = pair
|
||||
data = imagegen.png(900, 600, noise=True)
|
||||
response = api.upload(alice, chat, data)
|
||||
assert response.status_code == 202
|
||||
att = response.json()
|
||||
assert att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None
|
||||
ready = api.wait_status(bob, att["id"], "ready")
|
||||
assert ready["error_code"] is None and ready["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
check_variants(api, bob, ready, data)
|
||||
row = attachment_row(att["id"])
|
||||
assert row["published_version"] == 1 and row["staging_state"] == "deleted" # staging cleaned after ready
|
||||
assert s3_size(row["staging_key"], "staging") is None
|
||||
[job] = job_rows(att["id"])
|
||||
assert job["state"] == "succeeded" and job["attempt"] == 1 and job["kind"] == "upload"
|
||||
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
|
||||
|
||||
|
||||
def test_queued_attachment_is_attachable_failed_is_not(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_down("worker"): # stays queued
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
msg = api.post_message(alice, chat, {"text": "", "attachment_ids": [att["id"]]}).json()
|
||||
assert msg["attachment_ids"] == [att["id"]]
|
||||
api.wait_status(alice, att["id"], "ready")
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
||||
bad = api.upload(alice, chat, SAMPLE).json()
|
||||
failed = api.wait_status(bob, bad["id"], "failed")
|
||||
assert failed["error_code"] == "invalid_image" and failed["variants"] == []
|
||||
api.post_message(alice, chat, {"text": "", "attachment_ids": [bad["id"]]}, 409)
|
||||
api.content(bob, bad["id"], expected=409)
|
||||
api.content(bob, bad["id"], "thumbnail", expected=409)
|
||||
|
||||
|
||||
def test_worker_stopped_upload_202_content_409_restart_api_and_broker_then_ready(api, pair):
|
||||
"""The defence scenario: worker stopped -> upload 202 -> content 409, other messages keep working
|
||||
-> the message with the attachment is created -> API and broker restarted WITH volumes ->
|
||||
the job is still in the queue -> worker started -> ready, thumbnail downloadable."""
|
||||
alice, bob, chat = pair
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
response = api.upload(alice, chat, SAMPLE)
|
||||
att = response.json()
|
||||
assert response.status_code == 202 and att["status"] == "queued"
|
||||
for variant in (None, "thumbnail"):
|
||||
assert api.content(bob, att["id"], variant, expected=409).json()["error"]["code"] == "conflict"
|
||||
api.send(bob, chat, "текст работает без worker")
|
||||
msg = api.post_message(alice, chat, {"text": "фото", "attachment_ids": [att["id"]]}).json()
|
||||
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 15, "relay did not publish the job")
|
||||
# rabbitmqctl counters are refreshed every ~5 s (statistics interval): wait, do not sample once.
|
||||
wait_for(lambda: queue_depth() >= 1, 20, "no message in the queue") # durable message waits in the queue
|
||||
compose("restart", "api", "broker") # volumes preserved
|
||||
wait_healthy("api", "broker", "relay")
|
||||
wait_stand()
|
||||
wait_for(lambda: queue_depth() >= 1, 20, "the accepted job was lost by the broker restart")
|
||||
assert api.attachment(bob, att["id"])["status"] == "queued"
|
||||
finally:
|
||||
compose("start", "worker")
|
||||
wait_healthy("worker")
|
||||
ready = api.wait_status(bob, att["id"], "ready")
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
items, _ = api.history(bob, chat, 50)
|
||||
assert any(m["id"] == msg["id"] and m["attachment_ids"] == [att["id"]] for m in items)
|
||||
[job] = job_rows(att["id"])
|
||||
assert job["attempt"] == 1 # no duplicate work
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ failure windows
|
||||
def test_broker_down_commit_to_publish_gap_is_closed_by_the_outbox(api, pair):
|
||||
"""Window "after COMMIT, before publish": the broker is down, the API still answers 202 (it never
|
||||
talks to the broker), the outbox row stays unpublished. No manual action: when the broker is
|
||||
back the relay publishes it and the worker finishes the job."""
|
||||
alice, bob, chat = pair
|
||||
with service_down("broker"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
assert att["status"] == "queued"
|
||||
time.sleep(3)
|
||||
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
|
||||
api.send(alice, chat, "сообщения работают без брокера")
|
||||
api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
assert outbox_of(att["id"]) == [{"attempt": 1, "published": True}]
|
||||
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
|
||||
|
||||
|
||||
def test_relay_down_job_waits_in_outbox_not_lost(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_down("relay"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
time.sleep(3)
|
||||
assert outbox_of(att["id"]) == [{"attempt": 1, "published": False}]
|
||||
assert api.attachment(bob, att["id"])["status"] == "queued"
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
|
||||
|
||||
def test_crash_after_publish_before_confirm_is_recorded_gives_a_harmless_duplicate(api, pair):
|
||||
"""Window "after publish, before confirm": the relay dies after the broker confirmed the message
|
||||
but before outbox.published_at is committed. The restarted relay publishes it AGAIN; the
|
||||
duplicate is a no-op (one claim, one set of variants)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("relay", RELAY_FAULT_POINTS="crash_after_publish"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
wait_for(lambda: all(o["published"] for o in outbox_of(att["id"])), 30, "outbox not marked after restart")
|
||||
events = events_of(att["id"])
|
||||
assert names(events).count("fault_injected") == 1 and names(events).count("claimed") == 1
|
||||
assert [j["attempt"] for j in job_rows(att["id"])] == [1]
|
||||
assert len(published_keys(att["id"])) == 4 # original + 3 variants, once
|
||||
|
||||
|
||||
def test_duplicate_and_reordered_deliveries_are_noops(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("relay", RELAY_FAULT_POINTS="duplicate_publish,reverse_order"):
|
||||
atts = [api.upload(alice, chat, imagegen.png(50 + i, 40)).json() for i in range(4)]
|
||||
for att in atts:
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
time.sleep(2) # let the duplicates be consumed
|
||||
for att in atts:
|
||||
assert names(events_of(att["id"])) == ["created", "claimed", "succeeded"]
|
||||
assert len(published_keys(att["id"])) == 4
|
||||
logs = compose("logs", "--no-log-prefix", "worker").stdout
|
||||
assert '"noop_finished"' in logs or '"noop_leased"' in logs # duplicates were seen and acked
|
||||
|
||||
|
||||
def test_worker_crash_before_side_effect_lease_expires_and_job_completes(api, pair):
|
||||
"""The worker dies right after the claim. The redelivered message finds a valid lease (no-op);
|
||||
the reaper returns the job to queued when the lease expires; attempt 2 publishes."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_claim", **FAST_LEASE):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
ev = names(events_of(att["id"]))
|
||||
assert ev.count("fault_injected") == 1 and "lease_expired" in ev and ev[-1] == "succeeded"
|
||||
tries = attempts(att["id"])
|
||||
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
|
||||
assert tries[0]["keys"] and all(s3_size(k) is None for k in tries[0]["keys"]) # nothing written
|
||||
|
||||
|
||||
def test_worker_crash_after_s3_write_before_commit_republishes_without_duplicates(api, pair):
|
||||
"""Window "after the S3 write, before the result COMMIT": objects of attempt 1 exist under ITS keys
|
||||
but are never published; attempt 2 publishes its own; the cleanup deletes attempt 1's objects
|
||||
after ORPHAN_GRACE_SECONDS (they can never become published: the token is fenced)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_s3_write", ORPHAN_GRACE_SECONDS="2",
|
||||
CLEANUP_INTERVAL_SECONDS="1", **FAST_LEASE):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
first, second = attempts(att["id"])
|
||||
assert (first["outcome"], second["outcome"]) == ("abandoned", "succeeded")
|
||||
assert set(published_keys(att["id"])) == set(second["keys"])
|
||||
wait_for(lambda: attempts(att["id"])[0]["cleaned"], 30, "orphans of attempt 1 not cleaned")
|
||||
assert all(s3_size(k) is None for k in first["keys"])
|
||||
assert all(s3_size(k) is not None for k in second["keys"])
|
||||
assert s3_keys(f"a/{att['id']}/") == set(second["keys"]) # exactly one set of objects
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
|
||||
|
||||
def test_worker_crash_after_commit_before_ack_redelivery_is_noop(api, pair):
|
||||
"""Window "after the result COMMIT, before the ack": the message comes back; the job is
|
||||
succeeded -> ack without any work; the same result stays published."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="crash_after_commit"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
before = published_keys(att["id"])
|
||||
wait_healthy("worker")
|
||||
wait_for(lambda: '"noop_finished"' in compose("logs", "--no-log-prefix", "worker").stdout, 30,
|
||||
"the redelivered message was not acknowledged as a no-op")
|
||||
ev = names(events_of(att["id"]))
|
||||
assert ev == ["created", "claimed", "fault_injected", "succeeded"] or ev == ["created", "claimed", "succeeded", "fault_injected"]
|
||||
assert published_keys(att["id"]) == before and [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
|
||||
assert api.attachment(bob, att["id"]) == ready
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ retries and failures
|
||||
def test_transient_errors_are_retried_with_jittered_backoff(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error=2", JOB_RETRY_BASE_SECONDS="1",
|
||||
JOB_RETRY_MAX_SECONDS="8"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
events = events_of(att["id"])
|
||||
retries = [e for e in events if e["event"] == "retry_scheduled"]
|
||||
assert [(e["attempt"], e["error_code"]) for e in retries] == [(1, "injected_transient"), (2, "injected_transient")]
|
||||
for e in retries:
|
||||
assert 0 <= e["detail"]["delay_s"] <= 1 * 2 ** (e["attempt"] - 1) # full jitter window
|
||||
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "retry"), (2, "retry"), (3, "succeeded")]
|
||||
assert [o["attempt"] for o in outbox_of(att["id"])] == [1, 2, 3] # every retry went through the outbox
|
||||
|
||||
|
||||
def test_retry_exhausted_failed_registry_then_replay(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="transient_error", JOB_RETRY_BASE_SECONDS="0.2",
|
||||
JOB_RETRY_MAX_SECONDS="0.5"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
failed = api.wait_status(bob, att["id"], "failed", timeout=60)
|
||||
assert failed["error_code"] == "retry_exhausted" and failed["variants"] == []
|
||||
[job] = job_rows(att["id"])
|
||||
assert (job["state"], job["attempt"], job["error_code"]) == ("failed", 5, "retry_exhausted") # JOB_MAX_ATTEMPTS
|
||||
registry = jobctl("list-failed", "--limit", "200")["failed"]
|
||||
assert any(r["attachment_id"] == att["id"] and r["error_code"] == "retry_exhausted" for r in registry)
|
||||
assert attachment_row(att["id"])["staging_state"] == "present" # kept for the replay
|
||||
# Operator replay: a NEW job (replay_of = the failed one), audit with operator and reason.
|
||||
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "fault removed")
|
||||
assert result["result"] == "queued" and result["replay_of"] == job["id"]
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
jobs = job_rows(att["id"])
|
||||
assert [(j["kind"], j["state"]) for j in jobs] == [("upload", "failed"), ("replay", "succeeded")]
|
||||
ev = events_of(att["id"])
|
||||
req = next(e for e in ev if e["event"] == "replay_requested")
|
||||
assert req["actor"] == "jobctl:alice" and req["detail"]["reason"] == "fault removed"
|
||||
# "Repeat" never re-runs a finished job: a replay of a ready attachment is refused, nothing created.
|
||||
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "again", expect_code=3)
|
||||
assert refused == {"result": "refused", "reason": "already_ready"}
|
||||
assert len(job_rows(att["id"])) == 2
|
||||
|
||||
|
||||
def test_permanent_error_fails_at_once_without_retries(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
failed = api.wait_status(bob, att["id"], "failed")
|
||||
assert failed["error_code"] == "invalid_image"
|
||||
assert [(a["attempt"], a["outcome"]) for a in attempts(att["id"])] == [(1, "failed")]
|
||||
|
||||
|
||||
def test_memory_and_time_limits_of_the_worker(api, pair):
|
||||
"""40 Mpx PNG (valid, accepted by the API) with IMAGE_MAX_MEMORY_MB=200: the child process cannot
|
||||
allocate the raster -> image_too_large (permanent); the worker itself keeps running.
|
||||
JOB_TIMEOUT_SECONDS=3 with a 10 s stall -> processing_timeout (transient, retried)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", IMAGE_MAX_MEMORY_MB="200"):
|
||||
att = api.upload(alice, chat, imagegen.png(8000, 5000, compress_level=9)).json()
|
||||
failed = api.wait_status(bob, att["id"], "failed", timeout=90)
|
||||
assert failed["error_code"] == "image_too_large"
|
||||
ok = api.upload(alice, chat, SAMPLE).json() # the same worker still works
|
||||
api.wait_status(bob, ok["id"], "ready")
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="slow=10", JOB_TIMEOUT_SECONDS="3"):
|
||||
slow = api.upload(alice, chat, SAMPLE).json()
|
||||
wait_for(lambda: any(a["error_code"] == "processing_timeout" for a in attempts(slow["id"])), 60,
|
||||
"no processing_timeout")
|
||||
api.wait_status(bob, slow["id"], "ready", timeout=120) # normal worker finishes it
|
||||
|
||||
|
||||
def test_bad_message_is_dead_lettered_not_processed(api, pair):
|
||||
"""A message that does not match jobs.schema.json (published here with the relay's own broker
|
||||
user) is rejected by the worker into the DLQ; nothing is processed."""
|
||||
before = queue_depth("messenger.attachment-jobs.dlq")
|
||||
script = ("import asyncio, aio_pika\n"
|
||||
"async def m():\n"
|
||||
" url = open('/run/secrets/broker_url').read().strip()\n"
|
||||
" c = await aio_pika.connect(url)\n"
|
||||
" ch = await c.channel(publisher_confirms=True)\n"
|
||||
" ex = await ch.get_exchange('messenger.jobs', ensure=False)\n"
|
||||
" await ex.publish(aio_pika.Message(b'{\"url\":\"http://evil/x.png\"}'), routing_key='attachment.process.v1')\n"
|
||||
" await c.close()\n"
|
||||
"asyncio.run(m())\n")
|
||||
compose("exec", "-T", "relay", "python", "-c", script)
|
||||
wait_for(lambda: queue_depth("messenger.attachment-jobs.dlq") == before + 1, 20, "not dead-lettered")
|
||||
assert '"bad_message"' in compose("logs", "--no-log-prefix", "worker").stdout
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ lease, hung worker, shutdown, concurrency
|
||||
def worker_containers() -> dict[str, str]:
|
||||
"""hostname (= lease_owner) -> container id of every running worker."""
|
||||
ids = compose("ps", "-q", "worker").stdout.split()
|
||||
out = subprocess.run(["docker", "inspect", "-f", "{{.Config.Hostname}} {{.Id}}", *ids], capture_output=True,
|
||||
text=True, check=True).stdout
|
||||
return dict(line.split() for line in out.splitlines())
|
||||
|
||||
|
||||
def test_hung_worker_loses_its_lease_and_cannot_publish(api, pair):
|
||||
"""Two workers. The one processing the job is frozen (docker pause: no heartbeat). Its lease
|
||||
expires, the reaper re-queues the job, the other worker publishes it. When the frozen worker
|
||||
resumes, its result is fenced by the token: the published result does not change."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", scale=2, WORKER_FAULT_POINTS="slow=8", **FAST_LEASE):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
owner = wait_for(lambda: psql(f"SELECT lease_owner FROM jobs WHERE attachment_id = '{att['id']}' "
|
||||
"AND state = 'processing'"), 30, "job never claimed")
|
||||
frozen = worker_containers()[owner]
|
||||
subprocess.run(["docker", "pause", frozen], check=True, capture_output=True)
|
||||
try:
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=120)
|
||||
published = published_keys(att["id"])
|
||||
finally:
|
||||
subprocess.run(["docker", "unpause", frozen], check=True, capture_output=True)
|
||||
time.sleep(12) # the resumed attempt finishes its sleep and tries to publish
|
||||
assert published_keys(att["id"]) == published and api.attachment(bob, att["id"]) == ready
|
||||
tries = attempts(att["id"])
|
||||
assert [(a["attempt"], a["outcome"]) for a in tries] == [(1, "abandoned"), (2, "succeeded")]
|
||||
assert "lease_expired" in names(events_of(att["id"]))
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
|
||||
|
||||
def test_graceful_worker_stop_finishes_the_running_attempt(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="slow=5"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
wait_for(lambda: api.attachment(bob, att["id"])["status"] == "processing", 30, "never processing")
|
||||
started = time.monotonic()
|
||||
compose("stop", "worker") # SIGTERM: drain, not kill
|
||||
assert time.monotonic() - started < 30
|
||||
assert attachment_row(att["id"])["status"] == "ready" # committed before the exit
|
||||
compose("start", "worker")
|
||||
wait_healthy("worker")
|
||||
assert [a["outcome"] for a in attempts(att["id"])] == ["succeeded"]
|
||||
|
||||
|
||||
def test_concurrency_is_bounded_by_prefetch(api, pair):
|
||||
"""WORKER_CONCURRENCY=2: never more than 2 jobs processing and 2 unacked messages at once."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="slow=3"):
|
||||
atts = [api.upload(alice, chat, imagegen.png(30 + i, 30)).json() for i in range(6)]
|
||||
peak = 0
|
||||
deadline = time.monotonic() + 90
|
||||
while time.monotonic() < deadline:
|
||||
states = [api.attachment(bob, a["id"])["status"] for a in atts]
|
||||
peak = max(peak, states.count("processing"))
|
||||
unacked = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "list_queues", "name",
|
||||
"messages_unacknowledged").stdout
|
||||
for line in unacked.splitlines():
|
||||
if line.startswith("messenger.attachment-jobs\t"):
|
||||
assert int(line.split("\t")[1]) <= 2, line
|
||||
if all(s == "ready" for s in states):
|
||||
break
|
||||
time.sleep(0.3)
|
||||
assert all(api.attachment(bob, a["id"])["status"] == "ready" for a in atts)
|
||||
assert peak == 2
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cleanup never deletes accepted work
|
||||
def test_queued_staging_is_never_deleted_even_after_retention(api, pair):
|
||||
"""Reviewer's case in another PR: the worker was down longer than the retention and the cleanup
|
||||
deleted the original of a job that was still queued. Here: the job waits queued (relay down)
|
||||
while the cleanup runs every second with retention 0 - the staging copy stays, and the
|
||||
database itself refuses marking it (CHECK attachments_active_has_source)."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", STAGING_RETENTION_SECONDS="0", CLEANUP_INTERVAL_SECONDS="1"):
|
||||
with service_down("relay"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
time.sleep(5) # several cleanup passes
|
||||
row = attachment_row(att["id"])
|
||||
assert row["status"] == "queued" and row["staging_state"] == "present"
|
||||
assert s3_size(row["staging_key"], "staging") == len(SAMPLE)
|
||||
refused = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
|
||||
f"UPDATE attachments SET staging_state = 'deleting' WHERE id = '{att['id']}'",
|
||||
check=False)
|
||||
assert refused.returncode != 0 and "attachments_active_has_source" in refused.stderr
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=60)
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
|
||||
|
||||
def test_retention_deletes_failed_staging_then_replay_is_refused(api, pair):
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error", STAGING_RETENTION_SECONDS="1",
|
||||
CLEANUP_INTERVAL_SECONDS="1"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "failed")
|
||||
wait_for(lambda: attachment_row(att["id"])["staging_state"] == "deleted", 30, "staging not cleaned")
|
||||
assert s3_size(attachment_row(att["id"])["staging_key"], "staging") is None
|
||||
refused = jobctl("replay", att["id"], "--operator", "alice", "--reason", "too late", expect_code=3)
|
||||
assert refused == {"result": "refused", "reason": "source_missing"}
|
||||
assert api.attachment(bob, att["id"])["status"] == "failed" and len(job_rows(att["id"])) == 1
|
||||
|
||||
|
||||
def test_cleanup_racing_a_replay_never_deletes_the_replayed_source(api, pair):
|
||||
"""Reviewer's interleaving in another PR: the cleanup READ its candidates (an expired failed
|
||||
attachment), a replay committed queued, the cleanup went on and deleted the staging copy.
|
||||
Here the cleanup pauses exactly there (pause_cleanup); the replay commits during the pause;
|
||||
the cleanup then re-checks under the row lock and skips the row. The replay finishes ready."""
|
||||
alice, bob, chat = pair
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="permanent_error"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(bob, att["id"], "failed")
|
||||
with service_with_env("worker", WORKER_FAULT_POINTS="pause_cleanup=8", STAGING_RETENTION_SECONDS="1",
|
||||
CLEANUP_INTERVAL_SECONDS="1"):
|
||||
wait_for(lambda: '"cleanup_paused"' in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout
|
||||
and att["id"] in compose("logs", "--no-log-prefix", "--since", "60s", "worker").stdout,
|
||||
30, "cleanup did not pick the failed attachment")
|
||||
result = jobctl("replay", att["id"], "--operator", "alice", "--reason", "race test")
|
||||
assert result["result"] == "queued"
|
||||
ready = api.wait_status(bob, att["id"], "ready", timeout=90)
|
||||
check_variants(api, bob, ready, SAMPLE)
|
||||
assert attachment_row(att["id"])["staging_state"] == "deleted" # deleted only AFTER ready
|
||||
|
||||
|
||||
def test_reprocess_of_a_not_ready_upload_is_refused(api, pair):
|
||||
"""Reviewer's case in another PR: worker stopped -> upload 202 -> reprocess with pipeline 2 ->
|
||||
the attachment stayed queued forever. Here the command is refused and v1 completes."""
|
||||
alice, bob, chat = pair
|
||||
with service_down("worker"):
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
refused = jobctl("reprocess", att["id"], "--pipeline", "2", "--operator", "alice", "--reason", "x",
|
||||
expect_code=3)
|
||||
assert refused == {"result": "refused", "reason": "not_ready:queued"}
|
||||
assert len(job_rows(att["id"])) == 1
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
assert attachment_row(att["id"])["published_version"] == 1
|
||||
|
||||
|
||||
def test_zz_invariants_hold_after_all_scenarios():
|
||||
report = audit("--verify-checksums")
|
||||
assert report["objects"]["published"] > 0
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Session lifetime: expiry with a short SESSION_TTL_SECONDS, no extension by restarts,
|
||||
and the alternative SESSION_STORE=postgres. The api container is recreated with the
|
||||
override and ALWAYS recreated back with the normal configuration (api_with_env)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as dt
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import api_env, api_with_env, restart_services, service_down
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
|
||||
def seconds_left(expires_at: str) -> float:
|
||||
moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00"))
|
||||
return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds()
|
||||
|
||||
|
||||
def login_session(api, actor) -> dict:
|
||||
from conftest import basic
|
||||
return api.call("POST", "/api/v1/auth/sessions", 201,
|
||||
headers={"Authorization": basic(actor["username"], actor["password"])}, schema="Session").json()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("store", ["valkey", "postgres"])
|
||||
def test_short_ttl_expires(api, store):
|
||||
with api_with_env(SESSION_TTL_SECONDS="2", SESSION_STORE=store):
|
||||
assert api_env("SESSION_TTL_SECONDS") == "2" # the contract variable reaches the container
|
||||
actor = api.register("ttl")
|
||||
session = login_session(api, actor)
|
||||
assert 1.0 < seconds_left(session["expires_at"]) <= 2.1
|
||||
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
|
||||
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
|
||||
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=session["token"])
|
||||
|
||||
|
||||
def test_restart_does_not_extend_ttl(api):
|
||||
# 30 s: a restart of Valkey + both API replicas (readiness now also waits for S3) takes ~12-15 s.
|
||||
with api_with_env(SESSION_TTL_SECONDS="30"):
|
||||
actor = api.register("ttlrestart")
|
||||
session = login_session(api, actor)
|
||||
restart_services("valkey", "api")
|
||||
assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it"
|
||||
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
|
||||
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
|
||||
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
|
||||
|
||||
|
||||
def test_postgres_session_store(api):
|
||||
"""SESSION_STORE=postgres: sessions survive an API restart, logout revokes, and a DB outage
|
||||
is 503 for a valid token (not 401 and not anonymous access)."""
|
||||
with api_with_env(SESSION_STORE="postgres"):
|
||||
assert api_env("SESSION_STORE") == "postgres"
|
||||
actor = api.user("pgstore")
|
||||
restart_services("api")
|
||||
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
|
||||
with service_down("postgres", "stop"):
|
||||
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"],
|
||||
lb_error_ok=True).json()["error"]["code"] == "unavailable"
|
||||
api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True)
|
||||
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
|
||||
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
|
||||
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
|
||||
@@ -0,0 +1,112 @@
|
||||
"""S3 outages, private buckets and failover of attachments (lab 4, levels 3 and 4; lab 5: two
|
||||
buckets, upload 202 + worker).
|
||||
|
||||
Every injected failure is undone in `finally` (conftest.service_down / replica_stopped), and the
|
||||
stand is waited for until HAProxy sees all replicas UP again."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
import imagegen
|
||||
from conftest import attachment_row, compose, probe_inside, replica_stopped, running_replicas, service_down
|
||||
|
||||
pytestmark = pytest.mark.stack
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def pair(api):
|
||||
alice, bob = api.user("alice"), api.user("bob")
|
||||
return alice, bob, api.chat(alice, bob)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("how", ["stop", "pause"])
|
||||
def test_s3_outage_is_503_and_text_keeps_working(api, pair, how):
|
||||
"""S3 down (stopped: connection refused; paused: no answer at all):
|
||||
* readiness is 503 on every replica (and through the LB), quickly;
|
||||
* uploads/downloads are 503 `unavailable` within the S3 deadline, never 500;
|
||||
* the LB's degraded mode keeps chats/text messages working (HAProxy backend api_core);
|
||||
* after recovery the earlier file is intact."""
|
||||
alice, bob, chat = pair
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(alice, att["id"], "ready")
|
||||
with service_down("s3", how):
|
||||
deadline = time.monotonic() + 15
|
||||
while api.client.get("/health/ready").status_code != 503:
|
||||
assert time.monotonic() < deadline, "readiness did not turn 503"
|
||||
time.sleep(0.5)
|
||||
for replica in running_replicas():
|
||||
assert probe_inside(replica, "/health/ready") == 503
|
||||
assert probe_inside(replica, "/health/ready/core") == 200
|
||||
started = time.monotonic()
|
||||
error = api.upload(alice, chat, SAMPLE, 503).json()
|
||||
assert error["error"]["code"] == "unavailable"
|
||||
assert time.monotonic() - started < 8
|
||||
started = time.monotonic()
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"])
|
||||
assert time.monotonic() - started < 8
|
||||
api.call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]) # metadata: PostgreSQL only
|
||||
api.send(alice, chat, "текст работает без S3") # degraded mode
|
||||
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
assert got.content == SAMPLE
|
||||
api.upload(alice, chat, SAMPLE)
|
||||
|
||||
|
||||
def test_upload_survives_failover_of_the_serving_replica(api, pair):
|
||||
"""Upload -> stop the replica that served it -> another replica downloads the same bytes
|
||||
(nothing of the file lives in the replica)."""
|
||||
alice, bob, chat = pair
|
||||
data = imagegen.png_of_size(500_000)
|
||||
response = api.upload(alice, chat, data)
|
||||
att, served_by = response.json(), response.headers["x-instance-id"]
|
||||
replica = next(r for r in running_replicas() if r["hostname"] == served_by)
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
with replica_stopped(replica, "kill"):
|
||||
for _ in range(5):
|
||||
got = api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
assert got.headers["x-instance-id"] != served_by and got.content == data
|
||||
|
||||
|
||||
def test_bucket_is_private_outside_the_api(api, pair):
|
||||
"""Inside the storage network (where the API is): an UNSIGNED GET of a real object and an
|
||||
unsigned LIST are refused; the same GET signed with the API key pair works (control).
|
||||
SeaweedFS admin endpoints (master 9333, filer 8888, volume 8080) are not reachable."""
|
||||
alice, _, chat = pair
|
||||
att = api.upload(alice, chat, SAMPLE).json()
|
||||
api.wait_status(alice, att["id"], "ready")
|
||||
key = attachment_row(att["id"])["object_key"]
|
||||
probe = (
|
||||
"import urllib.request as u, urllib.error as e, socket\n"
|
||||
"def st(url):\n"
|
||||
" try: return u.urlopen(url, timeout=3).status\n"
|
||||
" except e.HTTPError as x: return x.code\n"
|
||||
f"print(st('http://s3:8333/messenger-attachments/{key}'), st('http://s3:8333/messenger-attachments?list-type=2'))\n"
|
||||
"for port in (9333, 8888, 8080):\n"
|
||||
" s = socket.socket(); s.settimeout(2)\n"
|
||||
" print(port, s.connect_ex(('s3', port)))\n")
|
||||
out = compose("exec", "-T", "api", "python", "-c", probe).stdout.split("\n")
|
||||
get_status, list_status = out[0].split()
|
||||
assert get_status in ("401", "403") and list_status in ("401", "403"), out[0]
|
||||
for line in out[1:4]:
|
||||
port, code = line.split()
|
||||
assert code != "0", f"admin port {port} reachable"
|
||||
signed = compose("exec", "-T", "api", "python", "-c",
|
||||
"import asyncio\nfrom messenger.config import Settings\nfrom messenger.storage import Storage\n"
|
||||
"async def m():\n s=Storage(Settings.from_env()); await s.open()\n"
|
||||
f" print((await s.head('{key}')).size); await s.close()\nasyncio.run(m())").stdout.strip()
|
||||
assert signed == str(len(SAMPLE))
|
||||
|
||||
|
||||
def test_s3_restart_keeps_objects(api, pair):
|
||||
alice, bob, chat = pair
|
||||
data = imagegen.png_of_size(200_000)
|
||||
att = api.upload(alice, chat, data).json()
|
||||
api.wait_status(bob, att["id"], "ready")
|
||||
with service_down("s3", "stop"):
|
||||
pass
|
||||
assert api.call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]).content == data
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Unit tests of the image checks (src/messenger/images.py) and the filename rules. No services.
|
||||
|
||||
The JPEG decoder (djpeg) exists in the API image; locally these tests are skipped without it
|
||||
(the same cases run against the stand in test_attachments.py)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
|
||||
import imagegen # noqa: E402
|
||||
from messenger import images, validation # noqa: E402
|
||||
from messenger.errors import InvalidRequest # noqa: E402
|
||||
|
||||
LIMIT = 40_000_000
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
has_djpeg = pytest.mark.skipif(shutil.which("djpeg") is None, reason="djpeg (libjpeg-turbo-progs) not installed")
|
||||
|
||||
|
||||
def status(data: bytes) -> int:
|
||||
"""201 if accepted, else the HTTP status of the ImageError (413/415)."""
|
||||
try:
|
||||
info = images.sniff(data, LIMIT)
|
||||
if info is None:
|
||||
return 415 # header never completed
|
||||
asyncio.run(images.verify(data, info))
|
||||
return 201
|
||||
except images.ImageError as exc:
|
||||
return exc.status
|
||||
|
||||
|
||||
def test_public_fixture_is_accepted():
|
||||
assert images.sniff(SAMPLE, LIMIT) == images.ImageInfo("image/png", 320, 200)
|
||||
assert status(SAMPLE) == 201
|
||||
|
||||
|
||||
@pytest.mark.parametrize("data", [
|
||||
imagegen.png(37, 23, "RGBA"), imagegen.png(33, 17, "P"), imagegen.png(33, 17, "1"),
|
||||
imagegen.png(33, 17, "L"), imagegen.png(81, 57, "RGB", noise=True), imagegen.png(1, 1),
|
||||
], ids=["rgba", "palette", "1bit", "gray", "noise", "1x1"])
|
||||
def test_valid_png_kinds(data):
|
||||
assert status(data) == 201
|
||||
|
||||
|
||||
def test_interlaced_png_is_accepted():
|
||||
# Pillow cannot write Adam7; build one by hand: 9 x 9 grey, every pass with filter 0.
|
||||
from messenger.images import _png_segments
|
||||
raw = b"".join(b"\x00" + bytes(stride - 1) for size, stride in _png_segments(9, 9, 8, 0, 1)
|
||||
for _ in range(size // stride))
|
||||
data = imagegen.PNG_SIGNATURE + imagegen.ihdr(9, 9, 8, 0, 1) + imagegen.chunk(b"IDAT", __import__("zlib").compress(raw)) \
|
||||
+ imagegen.chunk(b"IEND", b"")
|
||||
assert status(data) == 201
|
||||
|
||||
|
||||
BROKEN_PNG = {
|
||||
"first 33 bytes of the fixture": SAMPLE[:33],
|
||||
"truncated (no IEND)": SAMPLE[:-12],
|
||||
"truncated in IDAT": SAMPLE[:len(SAMPLE) // 2],
|
||||
"flipped byte -> CRC error": SAMPLE[:500] + bytes([SAMPLE[500] ^ 0xFF]) + SAMPLE[501:],
|
||||
"garbage IDAT with valid CRC": imagegen.png_corrupt_idat(),
|
||||
"fewer rows than IHDR": imagegen.png_short_pixels(),
|
||||
"bytes after IEND (polyglot)": SAMPLE + b"PK\x03\x04zip",
|
||||
"text": b"just text, definitely not an image",
|
||||
"gif": b"GIF89a" + bytes(64),
|
||||
"two bytes of a signature": b"\x89P",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", list(BROKEN_PNG))
|
||||
def test_broken_png_is_415(name):
|
||||
assert status(BROKEN_PNG[name]) == 415, name
|
||||
|
||||
|
||||
def test_pixel_limit_is_checked_from_the_header():
|
||||
with pytest.raises(images.ImageError) as exc:
|
||||
images.sniff(imagegen.png_bomb_header()[:33], LIMIT) # 33 bytes are enough
|
||||
assert exc.value.status == 413
|
||||
exact = imagegen.PNG_SIGNATURE + imagegen.ihdr(8000, 5000) # exactly 40 000 000: allowed
|
||||
assert images.sniff(exact, LIMIT).width == 8000
|
||||
with pytest.raises(images.ImageError) as exc:
|
||||
images.sniff(imagegen.PNG_SIGNATURE + imagegen.ihdr(8001, 5000), LIMIT)
|
||||
assert exc.value.status == 413
|
||||
with pytest.raises(images.ImageError) as exc:
|
||||
images.sniff(imagegen.jpeg_bomb_header(), LIMIT)
|
||||
assert exc.value.status == 413
|
||||
|
||||
|
||||
def test_png_verification_is_streaming_and_fast():
|
||||
"""40 Mpx RGB = 120 MB of raw pixels: inflated in 1 MiB steps and discarded."""
|
||||
big = imagegen.png(8000, 5000, compress_level=9)
|
||||
started = time.perf_counter()
|
||||
assert status(big) == 201
|
||||
assert time.perf_counter() - started < 5
|
||||
|
||||
|
||||
def test_jpeg_header_is_parsed_incrementally():
|
||||
data = imagegen.jpeg(320, 200)
|
||||
assert images.sniff(data[:4], LIMIT) is None # SOF not arrived yet
|
||||
assert images.sniff(data, LIMIT) == images.ImageInfo("image/jpeg", 320, 200)
|
||||
|
||||
|
||||
@has_djpeg
|
||||
@pytest.mark.parametrize("data", [imagegen.jpeg(), imagegen.jpeg(640, 480, progressive=True)], ids=["base", "prog"])
|
||||
def test_valid_jpeg(data):
|
||||
assert status(data) == 201
|
||||
|
||||
|
||||
@has_djpeg
|
||||
@pytest.mark.parametrize("name,data", [
|
||||
("no scan data after SOS", imagegen.jpeg_without_scan_data()),
|
||||
("truncated", imagegen.jpeg(320, 240, noise=True)[:3000]),
|
||||
], ids=lambda v: v if isinstance(v, str) else "")
|
||||
def test_broken_jpeg_is_415(name, data):
|
||||
assert status(data) == 415, name
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ filename
|
||||
@pytest.mark.parametrize("name", ["photo.png", "фото отпуска.jpg", "a" * 255, "emoji 👨👩👧.png", " x ", "..."[:2] + "x"])
|
||||
def test_good_filenames(name):
|
||||
assert validation.filename_param([("filename", name)]) == name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["", " ", " ", ".", "..", "a/b.png", "a\\b.png", "a\x00b", "c1\x85.png",
|
||||
"tab\t.png", "del\x7f.png", "x
y", "photognp.exe", "a" * 256])
|
||||
def test_bad_filenames(name):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.filename_param([("filename", name)])
|
||||
|
||||
|
||||
def test_filename_must_be_single_and_present():
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.filename_param([])
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.filename_param([("filename", "a.png"), ("filename", "b.png")])
|
||||
|
||||
|
||||
def test_query_must_be_valid_utf8():
|
||||
assert validation.query_items(b"filename=%D1%84.png") == [("filename", "ф.png")]
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.query_items(b"filename=%FF%FE.png")
|
||||
@@ -0,0 +1,224 @@
|
||||
"""Unit tests of lab 5 without services: variant geometry and encoding (imaging.build, the same
|
||||
code the worker runs in its child process), the job message schema, backoff with jitter and the
|
||||
pipeline configuration."""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from jsonschema import Draft202012Validator
|
||||
from PIL import Image
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "src"))
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
|
||||
import imagegen # noqa: E402
|
||||
from messenger import broker, imaging, jobs # noqa: E402
|
||||
from messenger.config import ConfigError, Settings # noqa: E402
|
||||
from messenger.pipeline import PIPELINES, center_square, fit # noqa: E402
|
||||
|
||||
SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes()
|
||||
SCHEMA = json.loads((ROOT / "contracts" / "jobs.schema.json").read_text(encoding="utf-8"))
|
||||
LIMIT = 40_000_000
|
||||
|
||||
|
||||
def variants(data: bytes, content_type: str = "image/png", version: int = 1) -> dict[str, imaging.Rendered]:
|
||||
return {v.name: v for v in imaging.build(data, content_type, version, LIMIT)}
|
||||
|
||||
|
||||
def decoded(v: imaging.Rendered) -> Image.Image:
|
||||
img = Image.open(io.BytesIO(v.data))
|
||||
img.load()
|
||||
return img
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ geometry (contract rules)
|
||||
@pytest.mark.parametrize("size, box, expected", [
|
||||
((320, 200), 256, (256, 160)), # public fixture
|
||||
((200, 320), 256, (160, 256)),
|
||||
((100, 50), 256, (100, 50)), # never upscale
|
||||
((256, 256), 256, (256, 256)),
|
||||
((1000, 1), 256, (256, 1)), # 0.256 px -> minimum 1 px
|
||||
((1, 1000), 256, (1, 256)),
|
||||
((257, 255), 256, (256, 254)), # 255 * 256 / 257 = 254.0 (floor)
|
||||
((3000, 2000), 1280, (1280, 853)), # 853.33 -> 853
|
||||
])
|
||||
def test_fit_rounds_down_without_upscaling(size, box, expected):
|
||||
assert fit(*size, box) == expected
|
||||
|
||||
|
||||
def test_center_square():
|
||||
assert center_square(320, 200) == (60, 0, 260, 200)
|
||||
assert center_square(7, 10) == (0, 1, 7, 8)
|
||||
|
||||
|
||||
def test_public_fixture_variants_match_the_contract():
|
||||
out = variants(SAMPLE)
|
||||
assert list(out) == ["thumbnail", "crop", "optimized"]
|
||||
assert (out["thumbnail"].content_type, decoded(out["thumbnail"]).format, decoded(out["thumbnail"]).size) == \
|
||||
("image/jpeg", "JPEG", (256, 160))
|
||||
assert (out["crop"].content_type, decoded(out["crop"]).size) == ("image/jpeg", (128, 128))
|
||||
assert (out["optimized"].content_type, decoded(out["optimized"]).format, decoded(out["optimized"]).size) == \
|
||||
("image/webp", "WEBP", (320, 200)) # fits 1280 already: no upscale
|
||||
for v in out.values():
|
||||
assert (v.width, v.height) == decoded(v).size
|
||||
|
||||
|
||||
def test_small_image_is_not_upscaled_but_crop_is():
|
||||
out = variants(imagegen.png(40, 30))
|
||||
assert decoded(out["thumbnail"]).size == (40, 30)
|
||||
assert decoded(out["optimized"]).size == (40, 30)
|
||||
assert decoded(out["crop"]).size == (128, 128)
|
||||
|
||||
|
||||
def test_big_image_is_bounded_by_1280():
|
||||
out = variants(imagegen.png(3000, 2000))
|
||||
assert decoded(out["optimized"]).size == (1280, 853)
|
||||
assert decoded(out["thumbnail"]).size == (256, 170)
|
||||
|
||||
|
||||
def test_exif_orientation_is_applied_and_dropped():
|
||||
# 60x20 stored landscape with Orientation=6 (rotate 90 CW): displayed portrait 20x60.
|
||||
img = Image.new("RGB", (60, 20), (200, 30, 30))
|
||||
exif = Image.Exif()
|
||||
exif[0x0112] = 6
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "JPEG", exif=exif.tobytes(), quality=95)
|
||||
out = variants(buf.getvalue(), "image/jpeg")
|
||||
assert decoded(out["thumbnail"]).size == (20, 60)
|
||||
assert decoded(out["optimized"]).size == (20, 60)
|
||||
for v in out.values():
|
||||
assert not decoded(v).getexif(), v.name
|
||||
|
||||
|
||||
def test_alpha_is_composited_on_white_for_jpeg_and_kept_for_webp():
|
||||
img = Image.new("RGBA", (64, 64), (255, 0, 0, 0)) # fully transparent red
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "PNG")
|
||||
out = variants(buf.getvalue())
|
||||
thumb = decoded(out["thumbnail"]).convert("RGB")
|
||||
assert min(thumb.getpixel((32, 32))) >= 250, "transparent pixels must become white, not red/black"
|
||||
webp = decoded(out["optimized"])
|
||||
assert webp.mode == "RGBA" and webp.getpixel((10, 10))[3] == 0
|
||||
|
||||
|
||||
def test_metadata_is_not_copied():
|
||||
img = Image.new("RGB", (300, 300), (10, 200, 10))
|
||||
exif = Image.Exif()
|
||||
exif[0x010F] = "SecretCamera" # Make
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "JPEG", exif=exif.tobytes(), icc_profile=b"\0" * 128, comment=b"private")
|
||||
out = variants(buf.getvalue(), "image/jpeg")
|
||||
for name in ("thumbnail", "crop"):
|
||||
data = out[name].data
|
||||
assert b"Exif" not in data and b"ICC_PROFILE" not in data and b"SecretCamera" not in data, name
|
||||
assert b"EXIF" not in out["optimized"].data and b"ICCP" not in out["optimized"].data
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["RGBA", "P", "1", "L", "LA", "RGB"])
|
||||
def test_png_modes_are_processed(mode):
|
||||
out = variants(imagegen.png(33, 17, mode))
|
||||
assert decoded(out["thumbnail"]).size == (33, 17)
|
||||
|
||||
|
||||
def test_16_bit_grey_is_scaled_not_clipped():
|
||||
img = Image.new("I;16", (8, 8), 32768) # 50 % grey in 16 bits
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, "PNG")
|
||||
out = variants(buf.getvalue())
|
||||
value = decoded(out["thumbnail"]).convert("L").getpixel((4, 4))
|
||||
assert 120 <= value <= 135
|
||||
|
||||
|
||||
def test_pipeline_versions_differ_in_bytes_not_in_geometry():
|
||||
v1, v2 = variants(SAMPLE, version=1), variants(SAMPLE, version=2)
|
||||
for name in v1:
|
||||
assert (v1[name].width, v1[name].height) == (v2[name].width, v2[name].height)
|
||||
assert v1[name].sha256 != v2[name].sha256
|
||||
assert variants(SAMPLE, version=1)["thumbnail"].sha256 == v1["thumbnail"].sha256 # deterministic
|
||||
|
||||
|
||||
def test_pixel_limit_is_checked_before_decoding():
|
||||
# 8001 x 5000 = 40.005 Mpx with only a few rows of data: refused from the header (TooLarge);
|
||||
# 20000 x 10000 is refused even earlier by Pillow's own bomb check (> 2 x MAX_IMAGE_PIXELS).
|
||||
with pytest.raises(imaging.TooLarge):
|
||||
imaging.build(imagegen.png_bomb_header(8001, 5000), "image/png", 1, LIMIT)
|
||||
with pytest.raises((imaging.TooLarge, Image.DecompressionBombError)):
|
||||
imaging.build(imagegen.png_bomb_header(), "image/png", 1, LIMIT)
|
||||
|
||||
|
||||
def test_child_process_protocol_round_trip():
|
||||
out = list(variants(SAMPLE).values())
|
||||
raw = json.dumps({"variants": [{"name": v.name, "content_type": v.content_type, "width": v.width,
|
||||
"height": v.height, "size": len(v.data)} for v in out]}).encode() + b"\n"
|
||||
raw += b"".join(v.data for v in out)
|
||||
parsed = imaging.parse_output(raw)
|
||||
assert [(p.name, p.sha256) for p in parsed] == [(v.name, v.sha256) for v in out]
|
||||
with pytest.raises(imaging.ProcessingError):
|
||||
imaging.parse_output(raw[:-1])
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ message format
|
||||
def test_encoded_message_matches_jobs_schema_and_carries_ids_only():
|
||||
import datetime as dt
|
||||
raw = broker.encode(str(uuid.uuid4()), str(uuid.uuid4()), 1, 2, dt.datetime.now(dt.timezone.utc))
|
||||
body = json.loads(raw)
|
||||
assert not list(Draft202012Validator(SCHEMA, format_checker=Draft202012Validator.FORMAT_CHECKER).iter_errors(body))
|
||||
assert set(body) == {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"}
|
||||
assert broker.decode(raw) == body
|
||||
|
||||
|
||||
@pytest.mark.parametrize("change", [
|
||||
{"schema_version": 2}, {"type": "other"}, {"job_id": "not-a-uuid"}, {"attempt": 0}, {"attempt": True},
|
||||
{"pipeline_version": "1"}, {"created_at": "yesterday"}, {"url": "http://evil"}, {"object_key": "a/b"},
|
||||
{"traceparent": "bad"},
|
||||
])
|
||||
def test_invalid_messages_are_rejected(change):
|
||||
body = json.loads(broker.encode(str(uuid.uuid4()), str(uuid.uuid4()), 1, 1,
|
||||
__import__("datetime").datetime(2026, 1, 1, tzinfo=__import__("datetime").timezone.utc)))
|
||||
body.update(change)
|
||||
with pytest.raises(broker.BadMessage):
|
||||
broker.decode(json.dumps(body).encode())
|
||||
# The contract schema refuses the same messages (date-time needs an optional jsonschema extra).
|
||||
checker = Draft202012Validator.FORMAT_CHECKER
|
||||
assert list(Draft202012Validator(SCHEMA, format_checker=checker).iter_errors(body)) or "created_at" in change
|
||||
|
||||
|
||||
def test_garbage_is_rejected():
|
||||
for raw in (b"", b"[]", b"{", b"\xff", b"x" * 5000):
|
||||
with pytest.raises(broker.BadMessage):
|
||||
broker.decode(raw)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ backoff and configuration
|
||||
def test_backoff_is_full_jitter_and_capped():
|
||||
assert jobs.backoff_delay(1, 2, 60, rnd=lambda: 1.0) == 2
|
||||
assert jobs.backoff_delay(3, 2, 60, rnd=lambda: 1.0) == 8
|
||||
assert jobs.backoff_delay(10, 2, 60, rnd=lambda: 1.0) == 60
|
||||
assert jobs.backoff_delay(4, 2, 60, rnd=lambda: 0.0) == 0
|
||||
samples = [jobs.backoff_delay(3, 2, 60) for _ in range(500)]
|
||||
assert 0 <= min(samples) < 2 and 6 < max(samples) <= 8 # spread over the whole window
|
||||
|
||||
|
||||
def test_pipeline_settings(monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST")
|
||||
monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests")
|
||||
s = Settings.from_env()
|
||||
assert (s.job_max_attempts, s.job_timeout_seconds, s.worker_concurrency, s.staging_retention_seconds) == (5, 60, 2, 86400)
|
||||
assert s.s3_staging_bucket == "messenger-staging" and s.pipeline_version == 1
|
||||
monkeypatch.setenv("FAULT_POINTS", "transient_error=3,slow")
|
||||
s = Settings.from_env()
|
||||
assert s.fault_points == {"transient_error", "slow"} and s.fault_value("transient_error") == 3
|
||||
for name, value in (("PIPELINE_VERSION", "9"), ("JOB_MAX_ATTEMPTS", "0"), ("FAULT_POINTS", "slow=x"),
|
||||
("S3_STAGING_BUCKET", "messenger-attachments"), ("JOB_RETRY_BASE_SECONDS", "100")):
|
||||
monkeypatch.setenv(name, value)
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
monkeypatch.delenv(name)
|
||||
assert set(PIPELINES) == {1, 2, 3}
|
||||
@@ -0,0 +1,225 @@
|
||||
"""Pure functions, no services: validation, cursor, Authorization parsing, Argon2id, config."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import datetime as dt
|
||||
|
||||
import pytest
|
||||
|
||||
from messenger import authhdr, cursor, passwords, validation
|
||||
from messenger.config import ConfigError, Settings
|
||||
from messenger.errors import InvalidRequest, Unauthorized
|
||||
from messenger.timefmt import UTC, from_us, rfc3339, to_us
|
||||
|
||||
|
||||
def b64(raw: bytes) -> str:
|
||||
return base64.b64encode(raw).decode()
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ password rules
|
||||
@pytest.mark.parametrize("password", ["Twelve-chars", "я" * 128, "пароль:с:двоеточиями", "😀" * 12, "a b c d e f g"])
|
||||
def test_valid_passwords(password):
|
||||
assert validation.password_field({"password": password}) == password
|
||||
|
||||
|
||||
@pytest.mark.parametrize("password", ["short-11ch", "я" * 129, " " * 12, "\t\n " * 5, "\ud800" * 12, None, 123456789012])
|
||||
def test_invalid_passwords(password):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.password_field({"password": password})
|
||||
|
||||
|
||||
def test_text_may_contain_nul_but_not_lone_surrogates():
|
||||
assert validation.text_field({"t": "a\u0000b"}, "t", 10) == "a\u0000b"
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.text_field({"t": "a\ud800"}, "t", 10)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ cursor
|
||||
CHAT = "3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11"
|
||||
|
||||
|
||||
def test_cursor_roundtrip_and_binding():
|
||||
key = (to_us(dt.datetime(2026, 10, 12, 12, 0, 0, 123456, tzinfo=UTC)), "11111111-1111-4111-8111-111111111111")
|
||||
token = cursor.encode(cursor.MESSAGES, CHAT, key)
|
||||
assert cursor.decode(token, cursor.MESSAGES, CHAT) == key
|
||||
with pytest.raises(InvalidRequest):
|
||||
cursor.decode(token, cursor.MESSAGES, "00000000-0000-4000-8000-000000000000") # other chat
|
||||
with pytest.raises(InvalidRequest):
|
||||
cursor.decode(token, cursor.CHATS, CHAT) # other resource
|
||||
|
||||
|
||||
@pytest.mark.parametrize("payload", [
|
||||
b"v1|m|" + CHAT.encode() + b"|253402300800000000|11111111-1111-4111-8111-111111111111", # 1 us after 9999-12-31
|
||||
b"v1|m|" + CHAT.encode() + b"|9999999999999999999|11111111-1111-4111-8111-111111111111", # int64 edge
|
||||
b"v1|m|" + CHAT.encode() + b"|-1|11111111-1111-4111-8111-111111111111",
|
||||
b"v1|m|" + CHAT.encode() + b"|1|not-a-uuid",
|
||||
b"v1|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-11111111111G",
|
||||
b"v2|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-111111111111",
|
||||
])
|
||||
def test_broken_cursors_are_400_not_500(payload):
|
||||
token = base64.urlsafe_b64encode(payload).rstrip(b"=").decode()
|
||||
with pytest.raises(InvalidRequest):
|
||||
cursor.decode(token, cursor.MESSAGES, CHAT)
|
||||
|
||||
|
||||
def test_time_helpers_are_exact():
|
||||
moment = dt.datetime(2026, 2, 28, 23, 59, 59, 999999, tzinfo=UTC)
|
||||
assert from_us(to_us(moment)) == moment
|
||||
assert rfc3339(moment) == "2026-02-28T23:59:59.999999Z"
|
||||
assert from_us(cursor.MAX_TS_US).year == 9999
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Authorization header
|
||||
def test_basic_is_parsed_and_password_may_contain_colons():
|
||||
assert authhdr.parse_basic("Basic " + b64("alice:pa:ss:wörd".encode())) == ("alice", "pa:ss:wörd")
|
||||
assert authhdr.parse_basic("basic " + b64(b"bob_1:x" * 1)) == ("bob_1", "x")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header", [
|
||||
None, "", "Basic", "Basic é", "Basic !!!!", "Basic " + b64(b"no-colon"), "Basic " + b64(b"\xff\xfe:pw"),
|
||||
"Basic " + b64(b":password"), "Basic " + b64(b"alice:"), "Basic " + b64("Алиса:pw".encode()),
|
||||
"Bearer " + "a" * 43, "Basic " + "A" * 2000,
|
||||
])
|
||||
def test_bad_basic_is_401_with_basic_challenge(header):
|
||||
with pytest.raises(Unauthorized) as info:
|
||||
authhdr.parse_basic(header)
|
||||
assert info.value.headers["WWW-Authenticate"].startswith("Basic realm=\"messenger\"")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("header", [None, "", "Bearer", "Bearer short", "Bearer " + "a" * 43 + "!",
|
||||
"Basic " + b64(b"alice:password-123"), "Token " + "a" * 43, "Bearer " + "é" * 43])
|
||||
def test_bad_bearer_is_401_with_bearer_challenge(header):
|
||||
with pytest.raises(Unauthorized) as info:
|
||||
authhdr.parse_bearer(header)
|
||||
assert info.value.headers["WWW-Authenticate"] == 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def test_bearer_ok():
|
||||
token = "A_b-" + "x" * 39
|
||||
assert authhdr.parse_bearer("Bearer " + token) == token
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ Argon2id
|
||||
def test_argon2id_hash_and_verify():
|
||||
async def scenario():
|
||||
stored = await passwords.hash_password("я" * 128)
|
||||
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$")
|
||||
assert await passwords.verify_password(stored, "я" * 128)
|
||||
assert not await passwords.verify_password(stored, "я" * 127)
|
||||
assert not await passwords.verify_password(None, "anything") # unknown user, dummy hash
|
||||
assert not await passwords.verify_password("not-a-hash", "anything")
|
||||
assert await passwords.hash_password("same password!") != await passwords.hash_password("same password!") # salt
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ config
|
||||
@pytest.fixture(autouse=True)
|
||||
def s3_credentials(monkeypatch):
|
||||
"""Lab 4: Settings require the API's S3 key pair; synthetic values for the unit tests."""
|
||||
monkeypatch.setenv("S3_ACCESS_KEY_ID", "AKTEST")
|
||||
monkeypatch.setenv("S3_SECRET_ACCESS_KEY", "secret-for-unit-tests")
|
||||
|
||||
|
||||
def test_s3_settings_are_validated_and_contract_limits_cannot_be_raised(monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
settings = Settings.from_env()
|
||||
assert (settings.max_upload_bytes, settings.max_image_pixels) == (10 * 1024 * 1024, 40_000_000)
|
||||
assert settings.fault_points == frozenset() and settings.upload_concurrency == 4
|
||||
for name, value in (("MAX_UPLOAD_BYTES", "10485761"), ("MAX_IMAGE_PIXELS", "40000001"),
|
||||
("FAULT_POINTS", "drop_database"), ("S3_ENDPOINT", "ftp://s3"), ("S3_BUCKET", "Bad_Bucket")):
|
||||
monkeypatch.setenv(name, value)
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
monkeypatch.delenv(name)
|
||||
monkeypatch.delenv("S3_SECRET_ACCESS_KEY")
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
|
||||
|
||||
def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
|
||||
url_file = tmp_path / "database_url"
|
||||
url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8")
|
||||
monkeypatch.setenv("DATABASE_URL_FILE", str(url_file))
|
||||
monkeypatch.setenv("SESSION_STORE", "redis")
|
||||
monkeypatch.setenv("CACHE_URL", "redis://:p@valkey:6379/0")
|
||||
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
|
||||
settings = Settings.from_env()
|
||||
assert settings.database_url == "postgresql://u:p@db/x"
|
||||
assert settings.session_store == "valkey" and settings.session_ttl_seconds == 2
|
||||
monkeypatch.setenv("SESSION_TTL_SECONDS", "0")
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
|
||||
monkeypatch.setenv("DATABASE_URL_FILE", str(tmp_path / "missing"))
|
||||
with pytest.raises(ConfigError) as info:
|
||||
Settings.from_env()
|
||||
assert "u:p" not in str(info.value)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id
|
||||
@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"])
|
||||
def test_valid_idempotency_keys(key):
|
||||
assert validation.idempotency_key(key) == key
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"])
|
||||
def test_invalid_idempotency_keys(key):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.idempotency_key(key)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}'])
|
||||
def test_duplicate_keys_and_garbage_json_are_rejected(raw):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.parse_json_object(raw)
|
||||
|
||||
|
||||
def test_attachment_ids_absent_equals_empty_and_is_validated():
|
||||
assert validation.attachment_ids_field({"text": "x"}) == []
|
||||
assert validation.attachment_ids_field({"attachment_ids": []}) == []
|
||||
one = "11111111-1111-4111-8111-111111111111"
|
||||
assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one]
|
||||
for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11):
|
||||
with pytest.raises(InvalidRequest):
|
||||
validation.attachment_ids_field({"attachment_ids": bad})
|
||||
|
||||
|
||||
def test_request_fingerprint_ignores_key_order_and_default_attachments():
|
||||
from messenger.app import request_fingerprint
|
||||
assert request_fingerprint("привет", []) == request_fingerprint("привет", [])
|
||||
assert request_fingerprint("привет", []) != request_fingerprint("привет ", [])
|
||||
assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", [])
|
||||
one = "11111111-1111-4111-8111-111111111111"
|
||||
assert request_fingerprint("x", [one]) != request_fingerprint("x", [])
|
||||
|
||||
|
||||
def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch):
|
||||
import socket
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
monkeypatch.delenv("INSTANCE_ID", raising=False)
|
||||
assert Settings.from_env().instance_id == socket.gethostname()
|
||||
monkeypatch.setenv("INSTANCE_ID", "replica-a")
|
||||
assert Settings.from_env().instance_id == "replica-a"
|
||||
for bad in ("with space", "x" * 65, "ip/1"):
|
||||
monkeypatch.setenv("INSTANCE_ID", bad)
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
|
||||
|
||||
def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600")
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
|
||||
|
||||
def test_message_write_path_setting(monkeypatch):
|
||||
monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z")
|
||||
monkeypatch.delenv("MESSAGE_WRITE_PATH", raising=False)
|
||||
assert Settings.from_env().message_write_path == "function"
|
||||
monkeypatch.setenv("MESSAGE_WRITE_PATH", "multi")
|
||||
assert Settings.from_env().message_write_path == "multi"
|
||||
monkeypatch.setenv("MESSAGE_WRITE_PATH", "fast")
|
||||
with pytest.raises(ConfigError):
|
||||
Settings.from_env()
|
||||
Reference in New Issue
Block a user