forked from mtusi2026/mtusi2026lab5
ЛР 5 — БПИ2303 — Лазарев — на 4
Асинхронная обработка вложений: RabbitMQ 4.1 (quorum-очереди, publisher confirms, manual ack, DLQ), transactional outbox, отдельный worker. - upload: оригинал в staging S3, затем одной транзакцией queued + jobs + outbox -> 202; API к брокеру не подключается - relay: публикация outbox с confirm, reaper истёкших аренд, redrive - worker: claim по строке jobs, аренда с heartbeat и fencing по lease_token, transform в подпроцессе с RLIMIT_AS/RLIMIT_CPU/таймаутом, thumbnail 256, crop 128, WebP 1280 по детерминированным ключам - retry с backoff и jitter, постоянные/временные ошибки, реестр failed, ops replay (ready не повторяется), DLQ для невалидных сообщений - janitor: очистка staging и сирот; миграция 004 с backfill вложений lab4 - тесты отказов: окна COMMIT/publish, крахи worker до/после S3 и ack, заморозка worker, всплеск загрузок, права брокера; evidence/ и REPORT.md
This commit is contained in:
+70
-9
@@ -1,25 +1,86 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# scripts/init-secrets.sh делает это сам и создаёт файлы в secrets/ (Docker secrets).
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
# Пароль БД НЕ задаётся здесь: он в файле secrets/postgres_password (Docker secret),
|
||||
# в контейнерах виден как /run/secrets/postgres_password (POSTGRES_PASSWORD_FILE).
|
||||
# Вместо DATABASE_URL_FILE из skeleton используется POSTGRES_* + POSTGRES_PASSWORD_FILE;
|
||||
# приложение понимает и DATABASE_URL_FILE (см. REPORT.md).
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_USER=messenger
|
||||
# Сессии хранятся в PostgreSQL (lab2), Redis/Valkey не используется:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
DB_POOL_SIZE=10
|
||||
# Graceful shutdown реплики: пауза с ready=503 и лимит на дообслуживание запросов.
|
||||
DRAIN_SECONDS=3
|
||||
SHUTDOWN_TIMEOUT_SECONDS=10
|
||||
LB_PORT=8080
|
||||
APP_VERSION=1
|
||||
# Защита от OOM при шторме входов: одновременных scrypt на реплику и ожидание слота (с).
|
||||
PASSWORD_HASH_CONCURRENCY=2
|
||||
PASSWORD_HASH_WAIT_SECONDS=0.5
|
||||
# Readiness: фоновый пинг БД (с) и сколько секунд без успешного пинга реплика ещё «готова».
|
||||
DB_HEALTH_INTERVAL_SECONDS=1
|
||||
READY_STALE_SECONDS=3
|
||||
# Попыток POST /auth/sessions за 10 с с одного IP на LB (сверх — 429).
|
||||
LOGIN_RATE_LIMIT=20
|
||||
|
||||
# --- Lab4: S3 (SeaweedFS) и вложения ---
|
||||
# S3 API SeaweedFS слушает 8333 (не 9000, как у MinIO); наружу не публикуется.
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
# Ключи API (права только Read+Write на bucket). Значения — в secrets/, здесь только пути.
|
||||
# У worker свой ключ (s3_worker_*), у janitor — свой (s3_janitor_*).
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_api_access_key
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_api_secret_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
# Весь приём тела upload, с; одновременных загрузок и JPEG-декодирований на реплику.
|
||||
UPLOAD_TIMEOUT_SECONDS=60
|
||||
UPLOAD_CONCURRENCY=8
|
||||
DECODE_CONCURRENCY=2
|
||||
# Одновременных S3 PUT на реплику (~20 MiB RAM каждый), остальные ждут в очереди.
|
||||
S3_PUT_CONCURRENCY=3
|
||||
S3_CONNECT_TIMEOUT_SECONDS=2
|
||||
# < timeout server 10s у LB: на зависший S3 API сам отвечает 503, а не LB 504.
|
||||
S3_READ_TIMEOUT_SECONDS=5
|
||||
# Janitor: возраст, после которого pending-строка/объект без строки считаются брошенными, и период обхода.
|
||||
ATTACHMENT_GRACE_SECONDS=3600
|
||||
JANITOR_INTERVAL_SECONDS=300
|
||||
# Только для эксперимента отказов (student-tests/consistency_faults.py). В обычной работе 0.
|
||||
FAULT_INJECTION=0
|
||||
|
||||
# --- Lab5: брокер, worker, задания ---
|
||||
# Брокер RabbitMQ 4.1 (service broker). URL с паролем — в secrets/broker_<relay|worker|ops>_url,
|
||||
# у каждого сервиса свой пользователь; здесь только путь (переопределяется в compose по сервису).
|
||||
BROKER_URL_FILE=/run/secrets/broker_worker_url
|
||||
# Реплик worker и потоков-потребителей в каждой (prefetch 1 на поток).
|
||||
WORKER_REPLICAS=2
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=1
|
||||
# Попыток на задание; таймаут одной обработки (с); аренда и её продление (heartbeat = lease/3).
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=30
|
||||
# Backoff между попытками: min(MAX, BASE*2^(n-1)), равномерно в [d/2, d] (equal jitter).
|
||||
JOB_RETRY_BASE_SECONDS=2
|
||||
JOB_RETRY_MAX_SECONDS=60
|
||||
PIPELINE_VERSION=1
|
||||
# Лимиты подпроцесса преобразования: адресное пространство (MiB) и CPU-время (с).
|
||||
TRANSFORM_MEMORY_LIMIT_MB=768
|
||||
TRANSFORM_CPU_SECONDS=30
|
||||
# Сколько хранить staging failed-заданий (для replay), с.
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
# Relay: опрос outbox без NOTIFY (с), период reaper (с), повторная публикация «застрявших» queued (с).
|
||||
RELAY_POLL_SECONDS=1
|
||||
REAPER_INTERVAL_SECONDS=5
|
||||
JOB_REDRIVE_AFTER_SECONDS=900
|
||||
OUTBOX_RETENTION_SECONDS=86400
|
||||
# Только для экспериментов отказов. В обычной работе пусто/0.
|
||||
RELAY_FAULT=
|
||||
WORKER_FAULT_INJECTION=0
|
||||
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
# Lab5: мессенджер на Python + PostgreSQL + S3 + RabbitMQ. Один образ для API-реплик, relay,
|
||||
# worker, janitor, ops-CLI и job-ов migrate (схема), s3-init (bucket).
|
||||
FROM python:3.12.7-slim@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d
|
||||
|
||||
# MALLOC_ARENA_MAX=2: glibc не заводит арену на каждый поток ThreadingHTTPServer —
|
||||
# освобождённые буферы upload возвращаются в общий пул, RSS не растёт с числом потоков.
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
MALLOC_ARENA_MAX=2 \
|
||||
APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Сначала зависимости (закреплённые версии) — слой кэшируется, пока не изменится requirements.txt.
|
||||
COPY requirements.txt /app/
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY src/ /app/src/
|
||||
COPY migrations/ /app/migrations/
|
||||
# Временные файлы upload (безымянные O_TMPFILE): каталог в writable-слое контейнера.
|
||||
# Рабочие файлы worker (staging-копия и результаты одного задания, удаляются после него).
|
||||
RUN mkdir -p /tmp/uploads /tmp/work && chmod 1777 /tmp/uploads /tmp/work
|
||||
ENV UPLOAD_TMP_DIR=/tmp/uploads
|
||||
|
||||
# Метка версии сборки (rolling update: scripts/rolling-update.sh собирает с новым значением).
|
||||
ARG APP_VERSION=1
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
RUN useradd --no-create-home --uid 10001 app
|
||||
USER app
|
||||
|
||||
EXPOSE 8080
|
||||
# Python-процесс — PID 1 (exec-форма), сам обрабатывает SIGTERM (graceful shutdown).
|
||||
STOPSIGNAL SIGTERM
|
||||
HEALTHCHECK --interval=5s --timeout=2s --start-period=10s --retries=2 \
|
||||
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"]
|
||||
CMD ["python", "/app/src/server.py"]
|
||||
@@ -1,37 +1,322 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Лазарев Юрий Сергеевич, БПИ2303
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **4**, трек 5 не заявлен
|
||||
- Commit/PR: ветка `solution/bpi2303-lazarev`; ссылка на PR — после публикации в fork.
|
||||
- Использованные источники, библиотеки и помощь ИИ: Python 3.12 (стандартная библиотека), `psycopg[binary]==3.2.3`, `psycopg-pool==3.2.4`, `minio==7.2.15`, `urllib3==2.3.0`, `Pillow==11.1.0` (проверка JPEG и варианты: JPEG, WebP), **`pika==1.3.2`** (AMQP 0-9-1). Образы: `python:3.12.7-slim@sha256:60d9996b…b50d`, `postgres:16.4-alpine@sha256:5660c2cb…e64c`, `haproxy:3.0.5-alpine@sha256:43a2591a…e949`, `chrislusf/seaweedfs:4.50@sha256:c2658820…4860`, **`rabbitmq:4.1.4-alpine@sha256:b736d649…63f2`** (полные digest — в `compose.yaml` и `Dockerfile`). Документация: RabbitMQ (Reliability, Publisher Confirms and Consumer Acknowledgements, Quorum Queues, Dead Letter Exchanges, Definitions, Access Control), Pillow (Image.draft, ImageOps.exif_transpose/fit, WebP/JPEG plugins), PostgreSQL (`FOR UPDATE SKIP LOCKED`, `LISTEN/NOTIFY`, advisory locks), AWS Architecture Blog «Exponential Backoff and Jitter», шаблон transactional outbox. Код lab4 перенесён и доработан с помощью ИИ-ассистента (Claude); решения, запуск и проверки — автор.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Проверено: MacBook Air M3, 16 GB RAM, macOS 26.4.1; Docker Desktop 28.0.4 (VM 8 vCPU, 7.7 GiB), Compose v2.34.0; Python 3.12 на хосте только для тестов (стандартная библиотека). Порты `127.0.0.1:8080` (LB) и `127.0.0.1:8404` (статистика HAProxy) должны быть свободны — стенд lab4 остановить (`docker compose stop` в его каталоге).
|
||||
|
||||
```sh
|
||||
# 0. Локальные секреты и .env (не коммитятся; значения не печатаются; повтор ничего не перезаписывает).
|
||||
# Пароль БД; ключи S3 admin/api/worker/janitor и s3_identities.json; пароли и URL пользователей
|
||||
# RabbitMQ relay/worker/ops и rabbitmq_definitions.json (пользователи с хешами, vhost, права, топология).
|
||||
./scripts/init-secrets.sh
|
||||
|
||||
# 1. Проверка конфигурации, сборка, запуск, ожидание готовности (~30–40 с)
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env build
|
||||
docker compose --env-file .env up -d --wait
|
||||
# порядок: db, s3, broker (healthy) -> migrate, s3-init (exit 0) -> api x2, relay, worker x2, janitor -> lb
|
||||
docker compose ps # migrate и s3-init Exited (0); порты только у lb
|
||||
|
||||
# 2. Обязательные проверки
|
||||
make check && make test
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh # рестарт db, s3, broker, api, relay, worker, janitor, lb
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh && ./scripts/start-all.sh
|
||||
|
||||
# 3. Сценарий защиты уровня 3 (worker остановлен -> 202 -> рестарт API и брокера -> worker -> ready -> дубль)
|
||||
python3 student-tests/worker_outage.py --output evidence/worker-outage.json
|
||||
# вручную то же: ./scripts/stop-worker.sh ; upload ; docker compose restart api broker ; ./scripts/start-worker.sh
|
||||
|
||||
# 4. Проверки уровня 4 (каждая сама включает/выключает учебные инъекции отказов и возвращает .env-конфигурацию)
|
||||
docker compose --profile tools run --rm checks /app/student-tests/variants_check.py --base-url http://lb:8080 --output /app/evidence/variants-check.json
|
||||
docker compose --profile tools run --rm checks /app/student-tests/transform_limits.py --output /app/evidence/transform-limits.json
|
||||
python3 student-tests/worker_faults.py --output evidence/worker-faults.json
|
||||
python3 student-tests/outbox_gap.py --output evidence/outbox-gap.json
|
||||
python3 student-tests/lease_takeover.py --output evidence/lease-takeover.json
|
||||
python3 student-tests/burst.py --count 24 --output evidence/burst.json
|
||||
python3 student-tests/broker_permissions.py --output evidence/broker-permissions.json
|
||||
python3 student-tests/attachments_negative.py --output evidence/attachments-negative.json # регрессия lab4
|
||||
python3 student-tests/attachment_failover.py --output evidence/attachment-failover.json && ./scripts/start-all.sh
|
||||
docker compose --profile tools run --rm checks /app/student-tests/s3_private_check.py
|
||||
|
||||
# 5. Оператор
|
||||
docker compose --profile tools run --rm ops failed # реестр failed
|
||||
docker compose --profile tools run --rm ops inspect <attachment_id> # job, события, outbox, варианты, объекты S3
|
||||
docker compose --profile tools run --rm ops replay <attachment_id> --dry-run
|
||||
docker compose --profile tools run --rm ops replay <attachment_id> --reason "S3 восстановлен"
|
||||
docker compose --profile tools run --rm ops dlq [--replay] [--purge-invalid]
|
||||
docker compose exec broker rabbitmqctl -p messenger list_queues name type messages messages_ready messages_unacknowledged consumers
|
||||
docker compose run --rm --no-deps janitor python /app/src/janitor.py --once --dry-run
|
||||
|
||||
# 6. Безопасная остановка (данные остаются в volumes pgdata, s3data, brokerdata)
|
||||
docker compose stop # или docker compose down (контейнеры удаляются, volumes остаются)
|
||||
|
||||
# 7. Намеренное удаление учебных данных (НЕОБРАТИМО: БД, объекты S3, очереди)
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
Один образ `mtusi-messenger:lab5` используется восемью сервисами; собирает его только `migrate` (у остальных `pull_policy: never`). Раньше `build` стоял у всех, и на пустом Docker параллельная сборка одного тега упала с `image ... already exists`. Можно и одной командой `up -d --build --wait`. Во время работы Docker Desktop дважды аварийно останавливался и один раз завис (сборка висела на «load metadata»); после `docker desktop restart` всё собралось за секунды, а стенд поднялся сам (`restart: unless-stopped`) с сохранёнными данными. Миграции, bucket и топология брокера создаются автоматически (job `migrate`, job `s3-init`, definitions при старте RabbitMQ) и безопасно повторяются: миграции учитываются в `schema_migrations`, bucket создаётся только если его нет, definitions импортируются идемпотентно.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ─HTTP─> lb (HAProxy) ─> api × 2 ──(1) PUT staging/<uuid>──────────────> s3 (SeaweedFS, s3data)
|
||||
│ (2) ОДНА транзакция: attachments queued + jobs + outbox
|
||||
└──────────────────────────────────────────> db (PostgreSQL, pgdata)
|
||||
LISTEN/NOTIFY, SKIP LOCKED │
|
||||
relay ─────── publish + confirm ─────> broker (RabbitMQ, quorum, brokerdata)
|
||||
(reaper аренд, redrive) │ prefetch 1, manual ack
|
||||
worker × 2 (по 2 потребителя)
|
||||
staging ──> transform.py (подпроцесс, RLIMIT) ──> attachments/<id>/v1/* ──> s3
|
||||
COMMIT ready + attachment_variants (fencing по lease_token) ──> db, затем ack
|
||||
janitor: сверка БД <-> S3, очистка staging/сирот; ops: inspect/failed/replay/dlq
|
||||
```
|
||||
|
||||
| Роль | Service | Образ | Health/readiness | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| LB | `lb` | `haproxy:3.0.5-alpine` | проверка конфигурации; сам проверяет `/health/ready` API | — | frontend+backend; `127.0.0.1:8080`, `:8404` | нет |
|
||||
| API | `api` ×2 | `mtusi-messenger:lab5` | `/health/live`, `/health/ready` (только БД) | — | backend, `8080` внутри | `postgres_password`, `s3_api_*`; **брокера нет** |
|
||||
| Брокер | `broker` | `rabbitmq:4.1.4-alpine` | `check_running`, `check_local_alarms`, очередь объявлена | `brokerdata` | backend, `5672` внутри | `rabbitmq_definitions.json` (хеши паролей) |
|
||||
| Relay | `relay` | `mtusi-messenger:lab5` | heartbeat-файл ≤ 30 с | — | backend | `postgres_password`, `broker_relay_url` |
|
||||
| Worker | `worker` ×2 | тот же | heartbeat-файл ≤ 30 с, только если все потребители живы | — (временные файлы в слое контейнера) | backend | `postgres_password`, `broker_worker_url`, `s3_worker_*` |
|
||||
| БД | `db` | `postgres:16.4-alpine` | `pg_isready` | `pgdata` | backend | `postgres_password` |
|
||||
| S3 | `s3` | `chrislusf/seaweedfs:4.50` | `/healthz` и запись в filer | `s3data` | backend, только `:8333` | `s3_identities.json` |
|
||||
| Миграции / bucket | `migrate`, `s3-init` | тот же | exit 0 | — | backend | `postgres_password` / `s3_admin_*` |
|
||||
| Сверка | `janitor` | тот же | heartbeat ≤ 90 с | — | backend | `postgres_password`, `s3_janitor_*` |
|
||||
| Оператор, проверки | `ops`, `checks` (profile `tools`) | тот же | — | `checks`: bind `student-tests`, `evidence` | backend | `ops`: `broker_ops_url`, `s3_janitor_*`, БД |
|
||||
|
||||
### Выбор брокера: RabbitMQ 4.1, quorum-очередь
|
||||
|
||||
Нужны persistence и подтверждения в обе стороны. У RabbitMQ это штатно: **publisher confirm** (брокер записал сообщение в quorum-очередь — Raft-журнал на диске — и только после этого присылает ack издателю) и **manual consumer ack** (сообщение остаётся у брокера, пока потребитель не подтвердит обработку; при обрыве соединения возвращается в очередь). Quorum-очередь durable по определению, переживает рестарт с volume. Дополнительно используются её `x-delivery-limit` (защита от crash-loop) и dead-lettering `at-least-once`.
|
||||
|
||||
Топология (`secrets/rabbitmq_definitions.json`, генерирует `init-secrets.sh`, импортируется при старте):
|
||||
- exchange `attachments` (direct) → очередь `attachments.process.v1` (quorum; `x-delivery-limit=10`, `x-dead-letter-exchange=attachments.dlx`, `x-dead-letter-strategy=at-least-once`, `x-overflow=reject-publish`, `x-max-length=100000`);
|
||||
- exchange `attachments.dlx` (fanout) → очередь `attachments.dlq` (quorum);
|
||||
- vhost `messenger`; `hostname: broker` (имя узла `rabbit@broker` неизменно, данные volume находятся после пересоздания контейнера).
|
||||
|
||||
| Пользователь | Где | configure | write | read |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `relay` | relay | — | `attachments` | — |
|
||||
| `worker` | worker | — | — | `attachments.process.v1` |
|
||||
| `ops` | ops-CLI | — | `attachments` | `attachments.dlq` |
|
||||
| `guest` | не создаётся | | | |
|
||||
|
||||
Права проверены `broker_permissions.py` (см. «Доказательства»). API к брокеру не подключается вовсе: недоступность RabbitMQ не влияет на приём файлов — задания копятся в outbox.
|
||||
|
||||
### Что сохранено в момент 202
|
||||
|
||||
`POST /chats/{id}/attachments` (`src/server.py`):
|
||||
1. проверки lab4 без изменений: сессия, членство, filename, Content-Type, Content-Length ≤ 10 MiB, потоковый приём во временный безымянный файл с SHA-256, сигнатура, пиксели до распаковки, полный валидатор PNG/JPEG;
|
||||
2. `INSERT attachments … status='pending'` (скрыто от API, как pending в lab4) — БД знает о будущем объекте **до** записи в S3;
|
||||
3. полный проверенный оригинал → **staging** `staging/<случайный uuid>` в приватном bucket (S3 — устойчивый staging: SeaweedFS подтверждает PUT после записи на volume);
|
||||
4. **одна транзакция**: `attachments.status='queued'` + строка `jobs` + строка `outbox` (сообщение по `jobs.schema.json`) + событие `job_events`. COMMIT;
|
||||
5. ответ **202** со снимком `status=queued`, `variants=[]`, `error_code=null`.
|
||||
|
||||
Значит, к 202 устойчиво сохранены: полный оригинал (staging, S3), метаданные (размер, SHA-256, тип, размеры), задание и намерение его опубликовать (outbox) — всё в PostgreSQL. Сам брокер к этому моменту может ещё ничего не знать: это нормально, outbox будет опубликован relay. Остановка API после ответа ничего не теряет (проверено `outbox_gap.py`, случай D: API убит после COMMIT, но до отправки 202 — работа всё равно выполнена).
|
||||
|
||||
Если шаги 3–4 не удались: PUT неудачен → удаляем объект, затем pending-строку (как в lab4); ошибка на шаге 4 → **staging не удаляем** (COMMIT мог пройти при обрыве соединения на COMMIT; удалить оригинал принятой работы нельзя), отвечаем 503, pending-остаток уберёт janitor после grace. Ключ конечного оригинала назначается сразу и детерминирован: `attachments/<attachment_id>/v<pipeline_version>/original`.
|
||||
|
||||
### Состояния и модель данных (`migrations/004_async_processing.sql`)
|
||||
|
||||
- `attachments.status`: `pending` (внутреннее) → `queued` → `processing` → `ready` | `failed`; новые колонки `staging_key`, `error_code`, `pipeline_version`.
|
||||
- `jobs` — **источник истины по заданию**: `(attachment_id, pipeline_version)` уникально (одна логическая работа = один `job_id`, не меняется при redelivery/retry/replay), `attempt` (растёт при каждом захвате), `max_attempts`, `next_attempt_at`, аренда `lease_owner` (`hostname:boot_id`), `lease_token` (fencing), `lease_until`, `error_code`, `last_error`, `replay_count`; `CHECK ((status='processing') = (lease_token IS NOT NULL))`.
|
||||
- `attachment_variants` — **ключ эффекта** `(attachment_id, pipeline_version, name)` = PRIMARY KEY: второго набора вариантов быть не может.
|
||||
- `outbox` — `payload jsonb`, `available_at` (отложенная публикация для retry), `published_at`, `publish_count`; триггер `NOTIFY outbox` после вставки.
|
||||
- `job_events` — audit всех переходов (кто, когда, попытка, причина; без секретов и bytes).
|
||||
- Переход lab4 → lab5: готовые вложения lab4 не имеют вариантов. Миграция делает их оригинал staging-объектом, вложение — `queued`, создаёт задание и outbox (готовность не заявляется без вариантов). Проверено на данных формы lab4: `evidence/migration-backfill.txt`.
|
||||
|
||||
Публичный API: GET метаданных показывает `queued|processing|ready|failed`; `variants` — только у `ready`, `error_code` — только у `failed`. `content` для queued/processing → 409 (+`Retry-After: 1`), для failed → 409; `variant=thumbnail|crop|optimized` у ready — байты варианта, неизвестный variant → 400, отсутствующий у ready → 404. Сообщение можно отправить с queued/processing/ready вложением, с failed → 409, с pending/чужим → 404.
|
||||
|
||||
### Outbox relay (`src/relay.py`)
|
||||
|
||||
Цикл: `SELECT … FROM outbox WHERE published_at IS NULL AND available_at <= now() ORDER BY available_at, id LIMIT 100 FOR UPDATE SKIP LOCKED` → для каждой строки `basic_publish(persistent, mandatory)` на канале с `confirm_delivery()` (вызов возвращается после ack брокера, при nack/unroutable — исключение) → `UPDATE outbox SET published_at=now()` → COMMIT. Ошибка брокера — откат транзакции, строки остаются, переподключение с паузой до 5 с. Будит relay `NOTIFY` (задержка публикации — миллисекунды), без уведомления — опрос раз в секунду. Несколько relay безопасны (`SKIP LOCKED`).
|
||||
|
||||
Окна отказа: крах между COMMIT API и publish, между publish и confirm, между confirm и `UPDATE outbox` → строка не помечена → будет опубликована ещё раз. Последнее даёт **дубль сообщения** — это нормальный at-least-once, его гасит worker (см. ниже).
|
||||
|
||||
**Reaper** (в relay, раз в 5 с, `pg_try_advisory_xact_lock` — один на всех):
|
||||
- `processing` с истёкшей арендой → `queued` с backoff и новой строкой outbox (или `failed/retry_exhausted`, если попытки кончились);
|
||||
- `queued` без движения дольше `JOB_REDRIVE_AFTER_SECONDS=900` и без неопубликованного outbox → повторная публикация (если сообщение потеряно вместе с volume брокера);
|
||||
- опубликованные строки outbox старше суток удаляются.
|
||||
|
||||
### Worker (`src/worker.py`, `src/jobs.py`, `src/transform.py`)
|
||||
|
||||
- `WORKER_CONCURRENCY=2` потока-потребителя на контейнер, у каждого своё AMQP-соединение и `basic.qos(prefetch_count=1)`; реплик 2 → одновременно не больше 4 заданий. Задание выполняется в отдельном потоке, а поток потребителя продолжает `process_data_events` — AMQP heartbeat (20 с) не рвётся на долгой обработке.
|
||||
- **Claim** (транзакция, `SELECT … FOR UPDATE` строки jobs). Решение принимается по БД, а не по факту доставки:
|
||||
- `ready` / `failed` → no-op, ack (`duplicate_ignored`);
|
||||
- `processing` с живой арендой другого процесса → ack (`busy_ignored`): если владелец умер, задание вернёт reaper;
|
||||
- `processing` с живой арендой **предыдущего процесса этого же контейнера** (тот же hostname, другой boot id) → перехват сразу (`lease_taken_over`): два процесса одного контейнера одновременно не живут;
|
||||
- `processing` с истёкшей арендой → перехват;
|
||||
- `queued`, но `next_attempt_at` ещё не наступил → старый дубль, ack (запланированное сообщение relay опубликует в срок);
|
||||
- иначе `processing`, `attempt+1`, новый `lease_token`, `lease_until=now()+30 с`. COMMIT.
|
||||
- **Аренда и зависание.** Отдельный поток продлевает аренду раз в `JOB_LEASE_SECONDS/3` (10 с), но не дольше `JOB_TIMEOUT_SECONDS+30` от начала. Долгий, но живой worker аренду держит — второй её не трогает; замороженный/зависший процесс продлевать перестаёт, аренда истекает, reaper возвращает задание в очередь. Опоздавший процесс узнаёт о потере аренды (heartbeat вернул 0 строк) и прекращает работу, а `complete` записывает результат **только при совпадении `lease_token`** (fencing). Так два «неконтролируемых» дубля исключены: побочные эффекты в S3 идемпотентны (те же ключи), а публикация результата в БД — только у владельца аренды.
|
||||
- **Обработка**: staging → временный файл (не больше `size_bytes` из БД; размер и SHA-256 должны совпасть с принятым upload, иначе `staging_corrupted`) → `transform.py` **в подпроцессе** с `RLIMIT_AS=768 MiB`, `RLIMIT_CPU=30 с` и wall-clock таймаутом `JOB_TIMEOUT_SECONDS=60` → PUT оригинала и вариантов по **детерминированным ключам** `attachments/<id>/v1/{original,thumbnail.jpg,crop.jpg,optimized.webp}` → `complete` (одна транзакция: jobs ready + 3 строки `attachment_variants` + attachments ready) → удаление staging → ack.
|
||||
- Сначала все объекты, потом COMMIT ready: ready никогда не ссылается на отсутствующий объект. Повтор после краха перезаписывает те же 4 объекта: второго набора нет ни в S3, ни в БД.
|
||||
|
||||
### Варианты (`src/transform.py`)
|
||||
|
||||
| Вариант | Формат | Размер | Параметры |
|
||||
| --- | --- | --- | --- |
|
||||
| thumbnail | JPEG | вписан в 256×256, без увеличения; целочисленно, вниз, минимум 1 px (320×200 → 256×160) | quality 85, optimize |
|
||||
| crop | JPEG | центрированный квадрат 128×128 (`ImageOps.fit`; маленькие исходники увеличиваются) | quality 85 |
|
||||
| optimized | WebP | вписан в 1280×1280 без увеличения | quality 80, method 4; alpha сохраняется |
|
||||
|
||||
Общее: EXIF orientation применяется (`exif_transpose`, размеры считаются после поворота); alpha (RGBA, LA, палитра с прозрачностью) для JPEG компонуется на белый; **EXIF, XMP, ICC и комментарии не переносятся** (Pillow пишет их, только если передать явно). Цвет считается sRGB: профиль ICC широкого охвата теряется — учебное упрощение. Уменьшение каждого маленького файла не обещается. JPEG декодируется в режиме `draft` (сразу в 2/4/8 раз меньше, но не меньше 1280 по обеим сторонам) — для 36 Мпикс нужно в разы меньше памяти; все варианты делаются из промежуточной копии ≤ 1280.
|
||||
|
||||
Лимиты «плохих» входов: API (lab4) отклоняет > 10 MiB (413), > 40 Мпикс по заголовку (413), битые и чужие форматы (415) ещё до очереди. Worker — вторая линия: лимит байт из метаданных, `Image.MAX_IMAGE_PIXELS=40M` (превышение — ошибка, а не предупреждение), `RLIMIT_AS`, `RLIMIT_CPU`, таймаут, контейнер 1 CPU / 1 GiB. Бомба убивает только подпроцесс, а не worker с его соединениями и арендами.
|
||||
|
||||
### Retry, ошибки, DLQ, replay
|
||||
|
||||
| Класс | error_code | Поведение |
|
||||
| --- | --- | --- |
|
||||
| постоянные | `invalid_image`, `image_too_large`, `staging_corrupted`, `staging_missing`, `resource_limit` | сразу `failed`, без повторов |
|
||||
| временные | `storage_unavailable`, `processing_timeout`, `processing_failed`, `lease_expired` | `queued` + backoff, пока `attempt < JOB_MAX_ATTEMPTS=5`; затем `failed/retry_exhausted` |
|
||||
| нет БД | — | результат записать нельзя: `nack(requeue)` после паузы 2 с; аренда истечёт, reaper вернёт задание |
|
||||
| невалидное сообщение / неизвестный job | — | `basic.reject(requeue=False)` → DLX → `attachments.dlq` |
|
||||
| crash-loop (10 доставок без ack) | — | quorum `x-delivery-limit` → DLQ; задание в БД остаётся и будет redrive |
|
||||
|
||||
Backoff: `d = min(60, 2·2^(attempt−1))` с, задержка равномерно в `[d/2, d]` (equal jitter: пауза гарантирована, повторы многих заданий разнесены во времени). Отложенная попытка — строка outbox с `available_at = now()+delay`; relay публикует её в срок. Отдельной retry-очереди с TTL не нужно.
|
||||
|
||||
**Реестр failed** — сами строки `jobs` со `status='failed'` (`ops failed`: код, попытки, причина, можно ли повторить). **DLQ брокера** — для сообщений, которые нельзя связать с заданием. **Replay** (`ops replay`) — транзакция над строкой jobs: только `failed` → `queued`, `max_attempts = attempt + 5`, `replay_count+1`, audit `replayed`, новая строка outbox, **тот же job_id и ключ эффекта**. `ready`, `queued`, `processing` не трогаются (`refused_status_ready`…): «повторить» не означает исполнить дубль успешной работы. Replay невозможен после удаления staging по retention (`refused_staging_expired`). `ops dlq --replay` разбирает сообщения DLQ по состоянию jobs: failed → replay, остальные → отбросить с audit; `--purge-invalid` удаляет мусор.
|
||||
|
||||
### Staging: retention и cleanup
|
||||
|
||||
- ready → worker удаляет staging сразу после COMMIT; не вышло (крах, S3) — janitor (`staging_ready_cleaned`);
|
||||
- failed → staging хранится `STAGING_RETENTION_SECONDS=86400` (для replay), затем janitor удаляет;
|
||||
- pending старше grace (1 ч; API упал до COMMIT, 202 не было) → janitor удаляет staging и строку;
|
||||
- объекты `staging/`, `originals/`, `attachments/` без ссылки из БД старше grace → удаляются; объекты вложений в `queued/processing` не трогаются никогда;
|
||||
- ready без объекта оригинала или варианта → только отчёт (`missing_objects`).
|
||||
|
||||
### Health, лимиты, таймауты (новое относительно lab4)
|
||||
|
||||
| Параметр | Значение | Почему |
|
||||
| --- | --- | --- |
|
||||
| `/health/ready` API | только PostgreSQL | брокер и worker не нужны синхронному пути: без них upload → 202, задания ждут |
|
||||
| healthcheck broker | `check_running` + `check_local_alarms` + очередь объявлена | relay/worker стартуют после импорта definitions |
|
||||
| Worker | 2 × 2 потребителя, prefetch 1; 1 CPU / 1 GiB; `stop_grace_period 75s` (SIGTERM: текущее задание дорабатывается) | ограниченная конкуренция; память подпроцессов (замер ниже) |
|
||||
| Подпроцесс transform | `RLIMIT_AS 768 MiB`, `RLIMIT_CPU 30 с`, таймаут 60 с | 36 Мпикс PNG — 0,7 с; бомбы ограничены |
|
||||
| Аренда | 30 с, продление каждые 10 с, предел `JOB_TIMEOUT+30` | зависший теряет аренду ≤ 30 с + 5 с reaper |
|
||||
| Relay / reaper | NOTIFY + опрос 1 с; reaper 5 с; redrive 900 с | |
|
||||
| RabbitMQ | `vm_memory_high_watermark 0.6`, `disk_free_limit 256MB`, 1 CPU / 512 MiB; `x-max-length 100000` + `reject-publish` | при переполнении relay получает nack, задания остаются в outbox |
|
||||
| S3 у worker | read timeout 10 с; ключ `worker`: Read+Write без List | |
|
||||
|
||||
### Что перенесено из lab4 и что изменено
|
||||
|
||||
Перенесены без изменения поведения: весь REST API, сессии, идемпотентность сообщений, `X-Instance-Id`, graceful shutdown, HAProxy, потоковый upload и валидаторы PNG/JPEG, S3Health, janitor-подход, скрипты `stop-one.sh`, `failover-each.sh`, `rolling-update.sh`. Изменено (объявленный переход 4 → 5): upload **201 ready → 202 queued**; оригинал идёт в staging; GET показывает асинхронные состояния и варианты; `content` у неготовых → 409; queued-вложение можно прикрепить, failed → 409. `restart.sh` перезапускает и брокер, relay, worker; `start-all.sh` поднимает и worker; новые `stop-worker.sh`/`start-worker.sh`. Тесты lab4 `attachments_negative.py`, `attachment_failover.py`, `upload_memory.py` переведены на 202 + ожидание ready; `s3-check` стал сервисом `checks` и проверяет ещё ключ worker.
|
||||
|
||||
### Соответствие `.env.example`
|
||||
|
||||
| Переменная skeleton | Реализация |
|
||||
| --- | --- |
|
||||
| `APP_*`, `SESSION_*`, `IDEMPOTENCY_TTL_SECONDS`, `API_REPLICAS`, `S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | как в lab4 (`S3_ENDPOINT=http://s3:8333`) |
|
||||
| `DATABASE_URL_FILE` | поддерживается; в compose — `POSTGRES_*` + `POSTGRES_PASSWORD_FILE` |
|
||||
| `CACHE_URL_FILE` | не используется (Redis/Valkey нет) |
|
||||
| `BROKER_URL_FILE` | у каждого сервиса свой файл: `/run/secrets/broker_relay_url`, `broker_worker_url`, `broker_ops_url` |
|
||||
| `WORKER_CONCURRENCY=2` | потоков-потребителей на контейнер (prefetch 1 каждый), реплик — `WORKER_REPLICAS=2` |
|
||||
| `JOB_MAX_ATTEMPTS=5`, `JOB_TIMEOUT_SECONDS=60` | как есть |
|
||||
| `STAGING_RETENTION_SECONDS=86400` | retention staging failed-заданий (janitor) |
|
||||
| добавлены | `WORKER_REPLICAS`, `WORKER_PREFETCH`, `JOB_LEASE_SECONDS`, `JOB_RETRY_BASE_SECONDS`, `JOB_RETRY_MAX_SECONDS`, `PIPELINE_VERSION`, `TRANSFORM_MEMORY_LIMIT_MB`, `TRANSFORM_CPU_SECONDS`, `RELAY_POLL_SECONDS`, `REAPER_INTERVAL_SECONDS`, `JOB_REDRIVE_AFTER_SECONDS`, `OUTBOX_RETENTION_SECONDS`; учебные `RELAY_FAULT`, `WORKER_FAULT_INJECTION`, `FAULT_SLOW_SECONDS` (по умолчанию выключены) |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Все прогоны — 13.11.2026 на итоговом коде одной командой `./student-tests/run_all.sh` (результат `overall=0`, 18/18). Машина и лимиты — как выше. Лог каждой команды — `evidence/<имя>.txt`, подробности — `evidence/<имя>.json`.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `make check`, `make test` | PASS: 202 → ready, оригинал побайтно, настоящий JPEG thumbnail 256×160 | `evidence/check.txt`, `evidence/smoke.txt` |
|
||||
| 3: минимум lab4 сохранён (201 → 202), restart с volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS, тот же токен и тот же файл; `StartedAt` изменился у **всех 10** контейнеров (db, s3, broker, api×2, relay, worker×2, janitor, lb) | `evidence/persistence.txt`, `evidence/containers-{before,after}-restart.txt` |
|
||||
| 3: failover API (lab3) | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: 2 реплики, 131 запись, 0 неуспешных, восстановление 3,7 с | `evidence/failover.txt` |
|
||||
| 3: регрессия вложений lab4 | `attachments_negative.py`, `attachment_failover.py` | 60/60, **0 ответов 5xx**; download через другую реплику 2/2 раунда | `evidence/attachments-negative.json`, `evidence/attachment-failover.json` |
|
||||
| 3: 202 только после устойчивого приёма; остановка API после ответа не теряет работу | `outbox_gap.py` D | API убит (`os._exit`) после COMMIT до отправки 202: клиент получил 502 от LB, задание дошло до ready | `evidence/outbox-gap.json` |
|
||||
| 3: отдельные broker и worker, durable, at-least-once, ack после эффекта | `compose.yaml`; `worker_outage.py`; `rabbitmqctl list_queues` | quorum-очереди durable; при остановленных worker сообщение лежит `messages_ready=1`, `consumers=0` | `evidence/worker-outage.json`, `evidence/queues.txt` |
|
||||
| 3: недоступность worker не делает resize в API | `worker_outage.py` | при остановленных worker upload 202 queued; content и thumbnail 409; сообщение с queued-вложением и обычное — 201 | `evidence/worker-outage.json` |
|
||||
| 3: queued/processing/ready/failed в API; оригинал + thumbnail по контракту | `make test`, `variants_check.py`, `worker_faults.py` | все 4 состояния наблюдаются; 7 входов: размеры, формат, SHA-256 совпадают; оригинал неизменен | `evidence/variants-check.json`, `evidence/worker-faults.json` |
|
||||
| 3: ограниченный retry, финальное failed с error_code; повтор не создаёт второй набор | `worker_faults.py` transient-always, повторная доставка в `worker_outage.py` | 5 попыток → `failed/retry_exhausted`; дубль job → `duplicate_ignored`, 3 варианта, 4 объекта. Во всём bucket **436 объектов `attachments/` = 109 ready × 4** | `evidence/worker-faults.json`, `evidence/janitor-dry-run.txt` |
|
||||
| 3: демонстрация worker stop → 202 → API restart → worker start → ready; рестарт broker с volume | `worker_outage.py` | 202 на 2,7 с; рестарт API (16 с), рестарт брокера (21 с) — сообщение на месте; worker запущен — ready через 0,03 с | `evidence/worker-outage.json` |
|
||||
| 4: crop 128×128 и WebP по контракту, параметры качества и метаданные | `variants_check.py` | crop 128×128 JPEG, optimized WebP ≤ 1280 без увеличения; EXIF orientation применена; прозрачное → белое; EXIF/ICC в вариантах нет | `evidence/variants-check.json` |
|
||||
| 4: лимиты bytes / пикселей / времени / памяти worker | `transform_limits.py` (+ лимиты API lab4) | 49 Мпикс → `image_too_large`; память 160 MiB → `resource_limit`; CPU 1 с → убит SIGXCPU; таймаут 0,5 с → `processing_timeout`; битые → `invalid_image`. 9/9 | `evidence/transform-limits.json` |
|
||||
| 4: outbox; crash между COMMIT и publish → ready без нового задания | `outbox_gap.py` A, B, C | A: relay остановлен — outbox не опубликован, очередь пуста; после запуска — ready, одно задание. B: relay умер посреди батча (RestartCount 0 → 1) — ready. C: дубль публикации → `duplicate_ignored` | `evidence/outbox-gap.json` |
|
||||
| 4: backoff + jitter, постоянные ≠ временные, DLQ/реестр, безопасный replay | `worker_faults.py` | задержки 0,95 / 1,51 с в границах [0,5; 1] и [1; 2]; подмена staging → `staging_corrupted` с 1-й попытки; `ops failed`; replay → ready; replay ready → `refused_status_ready`; мусор → DLQ → `--purge-invalid` | `evidence/worker-faults.json`, `evidence/failed-registry.txt` |
|
||||
| 4: остановка worker после записи объекта до ack | `worker_faults.py` crash-after-s3-put, crash-after-commit | после S3 до COMMIT: аренда истекла, попытка 2 записала **те же** 4 объекта; после COMMIT до ack: redelivery → no-op | `evidence/worker-faults.json` |
|
||||
| 4: timeout/lease для зависшего processing | `lease_takeover.py` | долгий worker держит аренду (heartbeat); замороженный (`docker pause`) теряет: задание завершено другим worker через 25,1 с; после разморозки — `aborted_lease_lost`, результат не изменён | `evidence/lease-takeover.json` |
|
||||
| 4: ограниченная конкуренция/prefetch | `burst.py --count 24`, `rabbitmqctl list_consumers` | 4 потребителя, prefetch 1; одновременно в обработке **максимум 4**; 24 файла по 6,75 MB — все ready за 7,2 с | `evidence/burst.json` |
|
||||
| 4: очистка staging после безопасной фиксации | `ops inspect`, `burst.py`, janitor | `staging_deleted` после ready во всех сценариях; staging остался только у 3 failed (retention для replay); crash-after-commit — staging убрал janitor | `evidence/burst.json`, `evidence/janitor-dry-run.txt` |
|
||||
| Минимальные права брокера и S3 | `broker_permissions.py`, `s3_private_check.py` | матрица relay/worker/ops как в таблице, configure — ни у кого, guest нет, 5672 на хост не опубликован; S3: 26/26 (новый ключ worker: Get/Put/Delete да, List/CreateBucket нет) | `evidence/broker-permissions.json`, `evidence/s3-private-check.txt` |
|
||||
| Переход lab4 → lab5 | миграция 004 на данных lab4 | ready-вложение lab4 → queued + задание + outbox по схеме | `evidence/migration-backfill.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Окна отказа (сводка; методика — учебные инъекции, включаемые только тестами)
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Инъекции: у API — заголовок `X-Fault-Inject` при `FAULT_INJECTION=1` (из lab4); у relay — `RELAY_FAULT=<точка>` (один раз за жизнь контейнера); у worker — имя файла `fault-<точка>.png` при `WORKER_FAULT_INJECTION=1`, крах — `os._exit(86)` один раз на задание (как SIGKILL, без finally). Каждый тест возвращает сервисы к конфигурации `.env`.
|
||||
|
||||
| Воздействие (окно из attachment-states.md) | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| API умер после staging до записи задания | 202 нет, работа не принята; остаток убирается | pending-строка без задания, GET 404; janitor (grace 2 с в тесте): dry-run нашёл, боевой удалил строку и объект | grace (1 ч по умолчанию) / 0 принятых |
|
||||
| API умер после COMMIT до 202 | работа принята, хотя клиент не узнал | клиент: 502 от LB; задание ready | секунды / 0 |
|
||||
| Relay остановлен (COMMIT есть, publish нет) | outbox ждёт | `published_at=null`, очередь 0; после старта — ready, одно задание | до запуска relay / 0 |
|
||||
| Relay умер между COMMIT и publish | повтор публикации | RestartCount 0 → 1, ready | ~1 с (рестарт контейнера) / 0 |
|
||||
| Relay умер после confirm до `published_at` | дубль сообщения | вторая доставка → `duplicate_ignored`, 3 варианта | 0 / 0, дубль погашен |
|
||||
| Worker умер после claim | аренда истекает, задание повторяется | `busy_ignored` → `lease_expired` → попытка 2 → ready | ~33 с (аренда 30 с + reaper) / 0 |
|
||||
| Worker умер после записи S3 до COMMIT | повтор перезаписывает те же ключи | попытка 2, ровно 4 объекта и 3 строки вариантов | ~35 с / 0 |
|
||||
| Worker умер после COMMIT до ack | redelivery — no-op | `duplicate_ignored`; staging убрал janitor | секунды / 0 |
|
||||
| Временная ошибка ×2 | backoff с jitter, ready | задержки 0,95 и 1,51 с, ready на 3-й попытке | ~3 с / 0 |
|
||||
| Временная ошибка всегда | 5 попыток, failed, replay | `retry_exhausted`; content 409, прикрепление 409; replay → ready (attempt 6, replay_count 1) | по решению оператора / 0 |
|
||||
| Подмена staging | постоянная ошибка, без повторов | `staging_corrupted` с 1-й попытки | — (нужен человек) |
|
||||
| Мусорное сообщение | DLQ | reject → DLQ → `ops dlq --purge-invalid` | — |
|
||||
| Worker заморожен (`docker pause`) | аренда истекает, другой worker завершает, опоздавший не пишет | ready через 25,1 с после паузы; разморозка → `aborted_lease_lost`, метаданные те же | 25 с / 0 |
|
||||
| Рестарт брокера с volume при остановленных worker | сообщение сохраняется | `messages_ready=1` до и после | — / 0 |
|
||||
| Рестарт всего стенда (persistence hook) и двукратный аварийный рестарт Docker Desktop | данные, сессии, очередь целы | PASS; после аварий стенд поднялся сам, данные на месте | ~40 с / 0 |
|
||||
|
||||
Почему восстановление после краха worker ~33 с, а не мгновенно. После `os._exit` контейнер перезапускается, но redelivered-сообщение брокер чаще отдаёт потребителю **другого** контейнера. Тот видит чужую живую аренду и не трогает задание (`busy_ignored`): отличить мёртвого владельца от долгого он не может. Ждём истечения аренды (30 с) + проход reaper (≤ 5 с). Мгновенный перехват срабатывает, только когда сообщение получил перезапущенный контейнер-владелец (тот же hostname, новый boot id). Компромисс: меньший `JOB_LEASE_SECONDS` ускоряет восстановление, но чаще ложно отбирает задания у живых worker при паузах GC или БД.
|
||||
|
||||
### Эксперимент: замороженный worker и fencing (`lease_takeover.py`)
|
||||
|
||||
Гипотеза: аренда с heartbeat отличает долгий процесс от зависшего, а `lease_token` не даёт опоздавшему перезаписать результат. Методика: аренда 10 с, `fault-slow` (25 с работы после claim), `docker pause` владельца через 9 с.
|
||||
|
||||
Первый прогон нашёл два недостатка:
|
||||
1. **Сообщение, выданное по prefetch второму (свободному) потребителю замороженного контейнера, висело ~35 с** — до таймаута AMQP heartbeat (20 с), только потом брокер вернул его в очередь. Исправлено: heartbeat 10 с.
|
||||
2. **Опоздавший worker после разморозки успевал сделать шаг**: он читал staging (уже удалённый победителем). Флаг потери аренды выставлял фоновый поток раз в 10 с. Запись результата всё равно отклонилась по `lease_token` (`error_ignored_lease_lost`), но правильнее не делать побочных эффектов вовсе. Исправлено: перед чтением staging, перед записью в конечный prefix и перед COMMIT аренда синхронно продлевается/проверяется в БД.
|
||||
|
||||
После исправлений: ready через 25,1 с после паузы (было ~55 с), опоздавший — `aborted_lease_lost` **до** записи в S3, вариантов 3, объектов 4.
|
||||
|
||||
### Всплеск загрузок (`burst.py`)
|
||||
|
||||
24 несжимаемых PNG 1500×1500 (6,75 MB), 8 параллельных загрузок через LB; стенд с лимитами выше, клиент на той же машине, один прогон — иллюстрация.
|
||||
|
||||
| Метрика | Значение |
|
||||
| --- | --- |
|
||||
| приём 24 × 202 | 2,3 с |
|
||||
| все ready | 7,2 с от начала (3,3 задания/с) |
|
||||
| потребителей × prefetch | 4 × 1 |
|
||||
| одновременно в обработке (по `job_events`) | **max 4** |
|
||||
| задержка очереди (enqueued → claimed) | p50 2,4 с, p95 4,1 с |
|
||||
| enqueued → ready | p50 2,9 с, p95 4,9 с |
|
||||
| обработка одного задания | p50 0,77 с, max 1,65 с |
|
||||
| пик памяти контейнера worker (docker stats, шаг 1 с) | 42 / 100 MiB при лимите 1 GiB |
|
||||
| staging после всех ready | 0 |
|
||||
|
||||
Вывод: конкуренция ограничена именно числом потребителей × prefetch, лишняя работа ждёт в очереди (растёт задержка, а не память). Пиковая память — грубая оценка: выборка раз в секунду пропускает короткие пики подпроцесса; верхнюю границу задают `RLIMIT_AS 768 MiB` × 2 потока и лимит контейнера 1 GiB. Худший допустимый вход (36 Мпикс PNG) проходит transform за 0,7 с (`transform_limits.py`).
|
||||
|
||||
## Вопросы защиты
|
||||
|
||||
- **Publisher confirm vs consumer ack.** Confirm — граница «брокер принял»: RabbitMQ записал сообщение в quorum-очередь и отвечает ack **издателю** (relay); только после него relay ставит `published_at`. Ack потребителя — граница «работа устойчиво выполнена»: worker подтверждает **брокеру** только после COMMIT ready (или записи failed/retry) в PostgreSQL; до этого сообщение у брокера и вернётся при обрыве. Это разные стороны и разные события; ни одно не заменяет другое.
|
||||
- **Что сохранено в момент 202:** полный проверенный оригинал в `staging/` S3, метаданные вложения, строка `jobs` и строка `outbox` — одним COMMIT. Брокер может ещё не знать о задании (публикует relay).
|
||||
- **Почему повторная доставка нормальна и где ключ идемпотентности.** At-least-once: дубль возникает при крахе между publish и отметкой outbox, между COMMIT и ack worker, при redrive, при replay DLQ. Ключ — `job_id` (строка `jobs`, решение по её состоянию под `FOR UPDATE`) и ключ эффекта `(attachment_id, pipeline_version, variant)` (PK `attachment_variants` и детерминированные ключи S3). Это не exactly-once доставка: эффект однократен за счёт идемпотентности, а не брокера.
|
||||
- **Зависший vs долго работающий.** Долго работающий продлевает аренду heartbeat-ом (каждые 10 с) — второй worker видит живую аренду и не трогает задание. Зависший (замороженный, мёртвый) продлевать перестаёт — аренда истекает, reaper возвращает задание. Продление ограничено `JOB_TIMEOUT+30` с, а transform убивается таймаутом, поэтому «живой, но бесконечный» процесс тоже теряет аренду. Опоздавший процесс не может опубликовать результат: `complete` требует его `lease_token` (fencing).
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Выдерживается:** остановка или падение API-реплики (в том числе после COMMIT до ответа), relay (в любом окне), worker (в любом окне), зависание или заморозка worker, рестарт брокера, БД, S3 и всего стенда с volumes, дубли и переупорядочивание сообщений (решение — по строке jobs), временные ошибки S3 (retry), невалидные изображения и сообщения.
|
||||
- **Не выдерживается:** потеря volume `pgdata` (источник истины: задания и outbox) или `s3data`; отказ единственного PostgreSQL, S3, брокера или LB (работа ждёт, но не теряется, пока целы volumes); потеря Docker-host. RabbitMQ — один узел: quorum-очередь здесь даёт durability на диске, но не репликацию.
|
||||
- **Потеря volume брокера** не теряет принятые задания: они в PostgreSQL. Reaper повторно публикует queued-задания без движения через `JOB_REDRIVE_AFTER_SECONDS=900`. Это не проверялось экспериментом (`down -v` только для брокера не выполнялся) — гипотеза по коду.
|
||||
- **Восстановление после краха worker — до ~35 с** (аренда 30 с + reaper 5 с), если redelivery достался другому контейнеру; см. «Эксперименты».
|
||||
- **Это не exactly-once доставка.** Сообщения могут приходить повторно и не по порядку. Однократен эффект: решение принимается по строке jobs, ключ эффекта — PK вариантов и детерминированные ключи S3. Опоздавший worker теоретически может перезаписать объект S3 теми же байтами в окне между последней проверкой аренды и PUT. Байты детерминированы (тот же вход, те же параметры, тот же Pillow), а метаданные он не публикует. При смене версии Pillow байты могут отличаться — тогда это задача versioned pipeline (трек 2), здесь не реализовано.
|
||||
- **Один pipeline_version=1.** Колонки и проверки для нескольких версий есть (`superseded`, `pipeline_version <=`), но сценарии нескольких версий не проверялись.
|
||||
- **NOTIFY не гарантирует доставку** — relay дополнительно опрашивает outbox раз в секунду.
|
||||
- **Учебные упрощения:** ICC-профили в вариантах теряются (sRGB предполагается); `docker stats` раз в секунду — грубая оценка памяти; хуки отказов (`FAULT_INJECTION`, `RELAY_FAULT`, `WORKER_FAULT_INJECTION`) в обычной конфигурации выключены, но в lab7 их код стоит удалить.
|
||||
- **Права SeaweedFS грубее IAM:** у worker `Write` = Put + Delete на весь bucket (нужен для удаления staging), разделения по prefix нет.
|
||||
- **Следующие шаги:** трек 1 (нагрузка с разными concurrency/prefetch и backpressure) или трек 2 (versioned pipeline и автоматический fault-injection); метрики очереди, outbox lag и аренд для lab6; ускорение восстановления (короче аренда, `basic.nack` на чужой живой аренде с задержкой).
|
||||
|
||||
+397
@@ -0,0 +1,397 @@
|
||||
# Lab5: HAProxy -> API x N (по умолчанию 2) -> PostgreSQL (данные, сессии, вложения, jobs, outbox)
|
||||
# -> SeaweedFS S3 (staging/ и attachments/, приватный bucket)
|
||||
# relay: outbox (PostgreSQL) -> RabbitMQ (quorum queue, publisher confirms) + reaper аренд
|
||||
# worker x N: RabbitMQ -> staging -> transform (thumbnail/crop/optimized) -> S3 + PostgreSQL
|
||||
# Запуск: ./scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait
|
||||
name: mtusi-lab5
|
||||
|
||||
# Один образ на восемь сервисов. Собирает его ТОЛЬКО migrate (первый job стенда), остальные
|
||||
# используют готовый тег: параллельная сборка одного тега несколькими сервисами давала
|
||||
# «image ... already exists» на пустом Docker.
|
||||
x-app-image: &app-image
|
||||
image: mtusi-messenger:lab5
|
||||
pull_policy: never
|
||||
|
||||
x-db-env: &db-env
|
||||
POSTGRES_HOST: db
|
||||
POSTGRES_PORT: "5432"
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
|
||||
x-s3-env: &s3-env
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-2}
|
||||
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-5}
|
||||
|
||||
x-job-env: &job-env
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-30}
|
||||
JOB_RETRY_BASE_SECONDS: ${JOB_RETRY_BASE_SECONDS:-2}
|
||||
JOB_RETRY_MAX_SECONDS: ${JOB_RETRY_MAX_SECONDS:-60}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
|
||||
services:
|
||||
db:
|
||||
image: postgres:16.4-alpine@sha256:5660c2cbfea50c7a9127d17dc4e48543eedd3d7a41a595a2dfa572471e37e64c
|
||||
environment:
|
||||
POSTGRES_DB: ${POSTGRES_DB:-messenger}
|
||||
POSTGRES_USER: ${POSTGRES_USER:-messenger}
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
|
||||
# S3 (как в lab4): master/volume/filer только на 127.0.0.1 контейнера, наружу — S3 API :8333.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:4.50@sha256:c2658820e763294a9e9de7b36c81e59b879e3459148351aee4e31a126ca64860
|
||||
command:
|
||||
- server
|
||||
- -dir=/data
|
||||
- -ip=127.0.0.1
|
||||
- -ip.bind=127.0.0.1
|
||||
- -master.volumeSizeLimitMB=1024
|
||||
- -master.electionTimeout=2s
|
||||
- -volume.preStopSeconds=2
|
||||
- -volume.max=0
|
||||
- -s3
|
||||
- -s3.ip.bind=0.0.0.0
|
||||
- -s3.port=8333
|
||||
- -s3.config=/run/secrets/s3_identities.json
|
||||
- -s3.iam=false
|
||||
- -s3.port.iceberg=0
|
||||
- -s3.port.lance=0
|
||||
- -s3.concurrentUploadLimitMB=64
|
||||
- -filer.concurrentUploadLimitMB=64
|
||||
- -volume.concurrentUploadLimitMB=64
|
||||
- -s3.cacheCapacityMB=16
|
||||
- -s3.readerCacheSizeMB=32
|
||||
secrets: [s3_identities.json]
|
||||
environment:
|
||||
GOMEMLIMIT: 512MiB
|
||||
volumes:
|
||||
- s3data:/data
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz && curl -sf -m 2 -X PUT --data-binary ok http://127.0.0.1:8888/.healthcheck/probe -o /dev/null"]
|
||||
interval: 5s
|
||||
start_interval: 1s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 60s
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 768M}
|
||||
|
||||
# Брокер: RabbitMQ 4.1, quorum-очередь (durable, Raft-журнал на диске), publisher confirms.
|
||||
# Топология и пользователи — из definitions (secret). Порты на хост не публикуются.
|
||||
broker:
|
||||
image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2
|
||||
hostname: broker # имя узла rabbit@broker неизменно -> данные volume находятся после пересоздания
|
||||
volumes:
|
||||
- brokerdata:/var/lib/rabbitmq
|
||||
- ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/conf.d/90-lab5.conf:ro
|
||||
secrets:
|
||||
- source: rabbitmq_definitions.json
|
||||
target: /run/secrets/rabbitmq_definitions.json
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
# Узел запущен, нет алармов памяти/диска, и рабочая очередь объявлена.
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms && rabbitmqctl -q -p messenger list_queues name | grep -q attachments.process.v1"]
|
||||
interval: 10s
|
||||
start_interval: 2s
|
||||
timeout: 15s
|
||||
retries: 5
|
||||
start_period: 60s
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
|
||||
migrate:
|
||||
<<: *app-image
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-1}
|
||||
pull_policy: build
|
||||
command: ["python", "/app/src/migrate.py"]
|
||||
environment:
|
||||
<<: *db-env
|
||||
secrets: [postgres_password]
|
||||
networks: [backend]
|
||||
healthcheck: {disable: true}
|
||||
restart: "no"
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
|
||||
s3-init:
|
||||
<<: *app-image
|
||||
command: ["python", "/app/src/s3_bootstrap.py"]
|
||||
environment:
|
||||
<<: *s3-env
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_key
|
||||
secrets: [s3_admin_access_key, s3_admin_secret_key]
|
||||
networks: [backend]
|
||||
healthcheck: {disable: true}
|
||||
restart: "no"
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
|
||||
api:
|
||||
<<: *app-image
|
||||
expose: ["8080"]
|
||||
environment:
|
||||
<<: [*db-env, *s3-env, *job-env]
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DB_POOL_SIZE: ${DB_POOL_SIZE:-10}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-3}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-10}
|
||||
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2}
|
||||
PASSWORD_HASH_WAIT_SECONDS: ${PASSWORD_HASH_WAIT_SECONDS:-0.5}
|
||||
DB_HEALTH_INTERVAL_SECONDS: ${DB_HEALTH_INTERVAL_SECONDS:-1}
|
||||
READY_STALE_SECONDS: ${READY_STALE_SECONDS:-3}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_api_access_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_api_secret_key
|
||||
UPLOAD_TIMEOUT_SECONDS: ${UPLOAD_TIMEOUT_SECONDS:-60}
|
||||
UPLOAD_CONCURRENCY: ${UPLOAD_CONCURRENCY:-8}
|
||||
DECODE_CONCURRENCY: ${DECODE_CONCURRENCY:-2}
|
||||
S3_PUT_CONCURRENCY: ${S3_PUT_CONCURRENCY:-3}
|
||||
FAULT_INJECTION: ${FAULT_INJECTION:-0}
|
||||
# У API НЕТ доступа к брокеру: задание пишется в outbox PostgreSQL.
|
||||
secrets: [postgres_password, s3_api_access_key, s3_api_secret_key]
|
||||
networks: [backend]
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 20s
|
||||
depends_on:
|
||||
db: {condition: service_healthy}
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 256M}
|
||||
|
||||
# Outbox relay + reaper аренд (src/relay.py). Можно масштабировать: SKIP LOCKED + advisory lock.
|
||||
relay:
|
||||
<<: *app-image
|
||||
command: ["python", "/app/src/relay.py"]
|
||||
environment:
|
||||
<<: [*db-env, *job-env]
|
||||
SERVICE_NAME: relay
|
||||
BROKER_URL_FILE: /run/secrets/broker_relay_url
|
||||
RELAY_POLL_SECONDS: ${RELAY_POLL_SECONDS:-1}
|
||||
REAPER_INTERVAL_SECONDS: ${REAPER_INTERVAL_SECONDS:-5}
|
||||
JOB_REDRIVE_AFTER_SECONDS: ${JOB_REDRIVE_AFTER_SECONDS:-900}
|
||||
OUTBOX_RETENTION_SECONDS: ${OUTBOX_RETENTION_SECONDS:-86400}
|
||||
RELAY_FAULT: ${RELAY_FAULT:-}
|
||||
secrets: [postgres_password, broker_relay_url]
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/relay.heartbeat') < 30 else 1)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M}
|
||||
|
||||
# Worker (src/worker.py): WORKER_CONCURRENCY потоков x prefetch 1 на контейнер.
|
||||
worker:
|
||||
<<: *app-image
|
||||
command: ["python", "/app/src/worker.py"]
|
||||
environment:
|
||||
<<: [*db-env, *s3-env, *job-env]
|
||||
SERVICE_NAME: worker
|
||||
BROKER_URL_FILE: /run/secrets/broker_worker_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_worker_access_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_worker_secret_key
|
||||
# Чтение 10 MiB из staging и запись результатов: дольше, чем upload-таймауты API.
|
||||
S3_READ_TIMEOUT_SECONDS: ${WORKER_S3_READ_TIMEOUT_SECONDS:-10}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
TRANSFORM_MEMORY_LIMIT_MB: ${TRANSFORM_MEMORY_LIMIT_MB:-768}
|
||||
TRANSFORM_CPU_SECONDS: ${TRANSFORM_CPU_SECONDS:-30}
|
||||
WORKER_TMP_DIR: /tmp/work
|
||||
WORKER_FAULT_INJECTION: ${WORKER_FAULT_INJECTION:-0}
|
||||
FAULT_SLOW_SECONDS: ${FAULT_SLOW_SECONDS:-20}
|
||||
secrets: [postgres_password, broker_worker_url, s3_worker_access_key, s3_worker_secret_key]
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/worker.heartbeat') < 30 else 1)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
restart: unless-stopped
|
||||
# SIGTERM: потребители дорабатывают текущее задание (<= JOB_TIMEOUT 60 с) и выходят.
|
||||
stop_grace_period: 75s
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
broker: {condition: service_healthy}
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 1G}
|
||||
|
||||
janitor:
|
||||
<<: *app-image
|
||||
command: ["python", "/app/src/janitor.py"]
|
||||
environment:
|
||||
<<: [*db-env, *s3-env]
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_janitor_access_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_janitor_secret_key
|
||||
ATTACHMENT_GRACE_SECONDS: ${ATTACHMENT_GRACE_SECONDS:-3600}
|
||||
JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-300}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
secrets: [postgres_password, s3_janitor_access_key, s3_janitor_secret_key]
|
||||
networks: [backend]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time() - os.path.getmtime('/tmp/janitor.heartbeat') < 90 else 1)"]
|
||||
interval: 15s
|
||||
timeout: 3s
|
||||
retries: 2
|
||||
start_period: 20s
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.5-alpine@sha256:43a2591a6a62a41dd22edb4452880e9c70de489fbd7c54ffb52d035cccbfe949
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
environment:
|
||||
LOGIN_RATE_LIMIT: ${LOGIN_RATE_LIMIT:-20}
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
- "127.0.0.1:8404:8404"
|
||||
networks: [frontend, backend]
|
||||
restart: unless-stopped
|
||||
healthcheck:
|
||||
test: ["CMD", "haproxy", "-c", "-q", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
depends_on:
|
||||
api: {condition: service_started}
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
|
||||
# Операторский CLI (не часть стенда): docker compose --profile tools run --rm ops <команда>
|
||||
# inspect / failed / replay / dlq. Ключ S3 janitor (List, без чтения), брокер — пользователь ops.
|
||||
ops:
|
||||
<<: *app-image
|
||||
profiles: [tools]
|
||||
entrypoint: ["python", "/app/src/ops.py"]
|
||||
environment:
|
||||
<<: [*db-env, *s3-env, *job-env]
|
||||
SERVICE_NAME: ops
|
||||
BROKER_URL_FILE: /run/secrets/broker_ops_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_janitor_access_key
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_janitor_secret_key
|
||||
secrets: [postgres_password, broker_ops_url, s3_janitor_access_key, s3_janitor_secret_key]
|
||||
networks: [backend]
|
||||
healthcheck: {disable: true}
|
||||
restart: "no"
|
||||
|
||||
# Проверки внутри backend-сети (Pillow для попиксельной проверки вариантов, права S3).
|
||||
checks:
|
||||
<<: *app-image
|
||||
profiles: [tools]
|
||||
entrypoint: ["python"]
|
||||
environment:
|
||||
<<: *s3-env
|
||||
volumes:
|
||||
- ./student-tests:/app/student-tests:ro
|
||||
- ./tests:/app/tests:ro
|
||||
- ./contracts:/app/contracts:ro
|
||||
- ./lab.json:/app/lab.json:ro
|
||||
- ./evidence:/app/evidence
|
||||
secrets: [s3_api_access_key, s3_api_secret_key, s3_worker_access_key, s3_worker_secret_key,
|
||||
s3_janitor_access_key, s3_janitor_secret_key]
|
||||
networks: [backend]
|
||||
healthcheck: {disable: true}
|
||||
restart: "no"
|
||||
user: "0"
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
s3_identities.json:
|
||||
file: ./secrets/s3_identities.json
|
||||
s3_admin_access_key:
|
||||
file: ./secrets/s3_admin_access_key
|
||||
s3_admin_secret_key:
|
||||
file: ./secrets/s3_admin_secret_key
|
||||
s3_api_access_key:
|
||||
file: ./secrets/s3_api_access_key
|
||||
s3_api_secret_key:
|
||||
file: ./secrets/s3_api_secret_key
|
||||
s3_worker_access_key:
|
||||
file: ./secrets/s3_worker_access_key
|
||||
s3_worker_secret_key:
|
||||
file: ./secrets/s3_worker_secret_key
|
||||
s3_janitor_access_key:
|
||||
file: ./secrets/s3_janitor_access_key
|
||||
s3_janitor_secret_key:
|
||||
file: ./secrets/s3_janitor_secret_key
|
||||
rabbitmq_definitions.json:
|
||||
file: ./secrets/rabbitmq_definitions.json
|
||||
broker_relay_url:
|
||||
file: ./secrets/broker_relay_url
|
||||
broker_worker_url:
|
||||
file: ./secrets/broker_worker_url
|
||||
broker_ops_url:
|
||||
file: ./secrets/broker_ops_url
|
||||
|
||||
networks:
|
||||
frontend: {}
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
s3data: {}
|
||||
brokerdata: {}
|
||||
@@ -0,0 +1,73 @@
|
||||
{
|
||||
"replicas": [
|
||||
"7aa798b287f4",
|
||||
"9e76aae00a09"
|
||||
],
|
||||
"rounds": [
|
||||
{
|
||||
"uploaded_by": "7aa798b287f4",
|
||||
"stopped": "7aa798b287f4",
|
||||
"downloads": [
|
||||
{
|
||||
"instance": "9e76aae00a09",
|
||||
"sha256_ok": true,
|
||||
"ms": 8.5
|
||||
},
|
||||
{
|
||||
"instance": "9e76aae00a09",
|
||||
"sha256_ok": true,
|
||||
"ms": 18.4
|
||||
},
|
||||
{
|
||||
"instance": "9e76aae00a09",
|
||||
"sha256_ok": true,
|
||||
"ms": 6.5
|
||||
},
|
||||
{
|
||||
"instance": "9e76aae00a09",
|
||||
"sha256_ok": true,
|
||||
"ms": 6.8
|
||||
},
|
||||
{
|
||||
"instance": "9e76aae00a09",
|
||||
"sha256_ok": true,
|
||||
"ms": 5.7
|
||||
}
|
||||
],
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"uploaded_by": "9e76aae00a09",
|
||||
"stopped": "9e76aae00a09",
|
||||
"downloads": [
|
||||
{
|
||||
"instance": "7aa798b287f4",
|
||||
"sha256_ok": true,
|
||||
"ms": 7.6
|
||||
},
|
||||
{
|
||||
"instance": "7aa798b287f4",
|
||||
"sha256_ok": true,
|
||||
"ms": 7.4
|
||||
},
|
||||
{
|
||||
"instance": "7aa798b287f4",
|
||||
"sha256_ok": true,
|
||||
"ms": 7.4
|
||||
},
|
||||
{
|
||||
"instance": "7aa798b287f4",
|
||||
"sha256_ok": true,
|
||||
"ms": 17.8
|
||||
},
|
||||
{
|
||||
"instance": "7aa798b287f4",
|
||||
"sha256_ok": true,
|
||||
"ms": 7.5
|
||||
}
|
||||
],
|
||||
"pass": true
|
||||
}
|
||||
],
|
||||
"pass": true
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
# 2026-11-13T13:25:43Z python3 student-tests/attachment_failover.py --output evidence/attachment-failover.json
|
||||
{"uploaded_by": "7aa798b287f4", "stopped": "7aa798b287f4", "downloads": [{"instance": "9e76aae00a09", "sha256_ok": true, "ms": 8.5}, {"instance": "9e76aae00a09", "sha256_ok": true, "ms": 18.4}, {"instance": "9e76aae00a09", "sha256_ok": true, "ms": 6.5}, {"instance": "9e76aae00a09", "sha256_ok": true, "ms": 6.8}, {"instance": "9e76aae00a09", "sha256_ok": true, "ms": 5.7}], "pass": true}
|
||||
{"uploaded_by": "9e76aae00a09", "stopped": "9e76aae00a09", "downloads": [{"instance": "7aa798b287f4", "sha256_ok": true, "ms": 7.6}, {"instance": "7aa798b287f4", "sha256_ok": true, "ms": 7.4}, {"instance": "7aa798b287f4", "sha256_ok": true, "ms": 7.4}, {"instance": "7aa798b287f4", "sha256_ok": true, "ms": 17.8}, {"instance": "7aa798b287f4", "sha256_ok": true, "ms": 7.5}], "pass": true}
|
||||
{"pass": true, "rounds": 2}
|
||||
exit=0
|
||||
@@ -0,0 +1,430 @@
|
||||
{
|
||||
"summary": {
|
||||
"cases": 60,
|
||||
"failed": 0,
|
||||
"server_errors_5xx": 0,
|
||||
"pass": true
|
||||
},
|
||||
"results": [
|
||||
{
|
||||
"case": "valid PNG fixture",
|
||||
"expected": 202,
|
||||
"status": 202,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "valid JPEG",
|
||||
"expected": 202,
|
||||
"status": 202,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "unicode filename",
|
||||
"expected": 202,
|
||||
"status": 202,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "PNG near 10 MiB limit (10482871 B)",
|
||||
"expected": 202,
|
||||
"status": 202,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "too large: 10 MiB + 1 B",
|
||||
"expected": 413,
|
||||
"status": 413,
|
||||
"error_code": "payload_too_large",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "pixel bomb header 100000x100000",
|
||||
"expected": 413,
|
||||
"status": 413,
|
||||
"error_code": "payload_too_large",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "PNG 7000x6000 = 42 Mpx (> 40 Mpx)",
|
||||
"expected": 413,
|
||||
"status": 413,
|
||||
"error_code": "payload_too_large",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "zlib bomb: 100x100 header, 50 MB inflated",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "corrupted PNG (byte flipped, bad CRC)",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "truncated PNG",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "PNG without IEND",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "PNG with appended ZIP (polyglot)",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "truncated JPEG",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "JPEG header only",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "empty body",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "JPEG bytes declared image/png",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "PNG bytes declared image/jpeg",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "text declared image/png",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "GIF declared image/png",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "PNG declared text/html",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "PNG declared image/svg+xml",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "multipart/form-data instead of raw",
|
||||
"expected": 415,
|
||||
"status": 415,
|
||||
"error_code": "unsupported_media_type",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: ../etc/passwd",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: slash",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: backslash",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: NUL",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: newline",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: bidi override U+202E",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: ..",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: empty",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: whitespace only",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: 256 code points",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: missing filename",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: filename twice",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "filename: invalid UTF-8",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "eve uploads to her own chat",
|
||||
"expected": 202,
|
||||
"status": 202,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "non-member uploads to alice chat",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "upload without session",
|
||||
"expected": 401,
|
||||
"status": 401,
|
||||
"error_code": "unauthorized",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "upload to unknown chat",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "foreign attachment metadata",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "foreign attachment content",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "eve reads alice attachment",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "message with foreign attachment_id",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "message mixing own and foreign ids",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "attachment of OTHER chat of the same user",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "unknown attachment_id",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "malformed attachment_id",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "duplicate attachment_ids",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "11 attachment_ids",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "attachment_ids not a list",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "empty text without attachments",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "empty text with attachment (bob links alice's file)",
|
||||
"expected": 201,
|
||||
"status": 201,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "idempotent send with attachments",
|
||||
"expected": 201,
|
||||
"status": 201,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "same key, same body (key order changed)",
|
||||
"expected": 201,
|
||||
"status": 201,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "same key, different attachment_ids -> 409",
|
||||
"expected": 409,
|
||||
"status": 409,
|
||||
"error_code": "conflict",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "variant=thumbnail (ready, lab5)",
|
||||
"expected": 200,
|
||||
"status": 200,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "variant=bogus",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "variant=original",
|
||||
"expected": 200,
|
||||
"status": 200,
|
||||
"error_code": null,
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "malformed attachment id in path",
|
||||
"expected": 400,
|
||||
"status": 400,
|
||||
"error_code": "invalid_request",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"case": "unknown attachment id",
|
||||
"expected": 404,
|
||||
"status": 404,
|
||||
"error_code": "not_found",
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
# 2026-11-13T13:25:41Z python3 student-tests/attachments_negative.py --output evidence/attachments-negative.json
|
||||
PASS valid PNG fixture -> 202
|
||||
PASS valid JPEG -> 202
|
||||
PASS unicode filename -> 202
|
||||
PASS PNG near 10 MiB limit (10482871 B) -> 202
|
||||
PASS too large: 10 MiB + 1 B -> 413 payload_too_large
|
||||
PASS pixel bomb header 100000x100000 -> 413 payload_too_large
|
||||
PASS PNG 7000x6000 = 42 Mpx (> 40 Mpx) -> 413 payload_too_large
|
||||
PASS zlib bomb: 100x100 header, 50 MB inflated -> 415 unsupported_media_type
|
||||
PASS corrupted PNG (byte flipped, bad CRC) -> 415 unsupported_media_type
|
||||
PASS truncated PNG -> 415 unsupported_media_type
|
||||
PASS PNG without IEND -> 415 unsupported_media_type
|
||||
PASS PNG with appended ZIP (polyglot) -> 415 unsupported_media_type
|
||||
PASS truncated JPEG -> 415 unsupported_media_type
|
||||
PASS JPEG header only -> 415 unsupported_media_type
|
||||
PASS empty body -> 415 unsupported_media_type
|
||||
PASS JPEG bytes declared image/png -> 415 unsupported_media_type
|
||||
PASS PNG bytes declared image/jpeg -> 415 unsupported_media_type
|
||||
PASS text declared image/png -> 415 unsupported_media_type
|
||||
PASS GIF declared image/png -> 415 unsupported_media_type
|
||||
PASS PNG declared text/html -> 415 unsupported_media_type
|
||||
PASS PNG declared image/svg+xml -> 415 unsupported_media_type
|
||||
PASS multipart/form-data instead of raw -> 415 unsupported_media_type
|
||||
PASS filename: ../etc/passwd -> 400 invalid_request
|
||||
PASS filename: slash -> 400 invalid_request
|
||||
PASS filename: backslash -> 400 invalid_request
|
||||
PASS filename: NUL -> 400 invalid_request
|
||||
PASS filename: newline -> 400 invalid_request
|
||||
PASS filename: bidi override U+202E -> 400 invalid_request
|
||||
PASS filename: .. -> 400 invalid_request
|
||||
PASS filename: empty -> 400 invalid_request
|
||||
PASS filename: whitespace only -> 400 invalid_request
|
||||
PASS filename: 256 code points -> 400 invalid_request
|
||||
PASS filename: missing filename -> 400 invalid_request
|
||||
PASS filename: filename twice -> 400 invalid_request
|
||||
PASS filename: invalid UTF-8 -> 400 invalid_request
|
||||
PASS eve uploads to her own chat -> 202
|
||||
PASS non-member uploads to alice chat -> 404 not_found
|
||||
PASS upload without session -> 401 unauthorized
|
||||
PASS upload to unknown chat -> 404 not_found
|
||||
PASS foreign attachment metadata -> 404 not_found
|
||||
PASS foreign attachment content -> 404 not_found
|
||||
PASS eve reads alice attachment -> 404 not_found
|
||||
PASS message with foreign attachment_id -> 404 not_found
|
||||
PASS message mixing own and foreign ids -> 404 not_found
|
||||
PASS attachment of OTHER chat of the same user -> 404 not_found
|
||||
PASS unknown attachment_id -> 404 not_found
|
||||
PASS malformed attachment_id -> 400 invalid_request
|
||||
PASS duplicate attachment_ids -> 400 invalid_request
|
||||
PASS 11 attachment_ids -> 400 invalid_request
|
||||
PASS attachment_ids not a list -> 400 invalid_request
|
||||
PASS empty text without attachments -> 400 invalid_request
|
||||
PASS empty text with attachment (bob links alice's file) -> 201
|
||||
PASS idempotent send with attachments -> 201
|
||||
PASS same key, same body (key order changed) -> 201
|
||||
PASS same key, different attachment_ids -> 409 -> 409 conflict
|
||||
PASS variant=thumbnail (ready, lab5) -> 200
|
||||
PASS variant=bogus -> 400 invalid_request
|
||||
PASS variant=original -> 200
|
||||
PASS malformed attachment id in path -> 400 invalid_request
|
||||
PASS unknown attachment id -> 404 not_found
|
||||
{"cases": 60, "failed": 0, "server_errors_5xx": 0, "pass": true}
|
||||
exit=0
|
||||
@@ -0,0 +1,49 @@
|
||||
{
|
||||
"relay": {
|
||||
"got": {
|
||||
"publish": true,
|
||||
"get_work_queue": false,
|
||||
"get_dlq": false,
|
||||
"declare_queue": false
|
||||
},
|
||||
"expected": {
|
||||
"publish": true,
|
||||
"get_work_queue": false,
|
||||
"get_dlq": false,
|
||||
"declare_queue": false
|
||||
},
|
||||
"pass": true
|
||||
},
|
||||
"worker": {
|
||||
"got": {
|
||||
"publish": false,
|
||||
"get_work_queue": true,
|
||||
"get_dlq": false,
|
||||
"declare_queue": false
|
||||
},
|
||||
"expected": {
|
||||
"publish": false,
|
||||
"get_work_queue": true,
|
||||
"get_dlq": false,
|
||||
"declare_queue": false
|
||||
},
|
||||
"pass": true
|
||||
},
|
||||
"ops": {
|
||||
"got": {
|
||||
"publish": true,
|
||||
"get_work_queue": false,
|
||||
"get_dlq": true,
|
||||
"declare_queue": false
|
||||
},
|
||||
"expected": {
|
||||
"publish": true,
|
||||
"get_work_queue": false,
|
||||
"get_dlq": true,
|
||||
"declare_queue": false
|
||||
},
|
||||
"pass": true
|
||||
},
|
||||
"guest_absent": true,
|
||||
"broker_port_published_to_host": false
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-11-13T13:25:36Z python3 student-tests/broker_permissions.py --output evidence/broker-permissions.json
|
||||
PASS relay {'publish': True, 'get_work_queue': False, 'get_dlq': False, 'declare_queue': False}
|
||||
PASS worker {'publish': False, 'get_work_queue': True, 'get_dlq': False, 'declare_queue': False}
|
||||
PASS ops {'publish': True, 'get_work_queue': False, 'get_dlq': True, 'declare_queue': False}
|
||||
guest absent: True | port 5672 on host: False
|
||||
PASS
|
||||
exit=0
|
||||
@@ -0,0 +1,108 @@
|
||||
{
|
||||
"scenario": "burst of uploads: bounded worker concurrency",
|
||||
"date_utc": "2026-11-13T13:25:36+00:00",
|
||||
"steps": [
|
||||
{
|
||||
"t": 1.1,
|
||||
"utc": "13:25:25.872",
|
||||
"step": "consumers",
|
||||
"count": 4,
|
||||
"prefetch": [
|
||||
1
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 1.64,
|
||||
"utc": "13:25:26.409",
|
||||
"step": "generated images",
|
||||
"bytes": [
|
||||
6753618,
|
||||
6753618,
|
||||
6753618,
|
||||
6753618
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 4.75,
|
||||
"utc": "13:25:29.527",
|
||||
"step": "all uploads accepted (202)",
|
||||
"count": 24,
|
||||
"seconds": 2.31,
|
||||
"queue_right_after": {
|
||||
"messages": 0,
|
||||
"messages_ready": 0,
|
||||
"messages_unacknowledged": 0,
|
||||
"consumers": 4
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11.68,
|
||||
"utc": "13:25:36.456",
|
||||
"step": "result",
|
||||
"count": 24,
|
||||
"image_bytes": 6753618,
|
||||
"upload_seconds": 2.31,
|
||||
"total_seconds_to_all_ready": 7.23,
|
||||
"throughput_jobs_per_s": 3.32,
|
||||
"consumers": 4,
|
||||
"prefetch_per_consumer": [
|
||||
1
|
||||
],
|
||||
"max_concurrent_processing": 4,
|
||||
"queue_lag_s": {
|
||||
"p50": 2.401,
|
||||
"p95": 4.069,
|
||||
"max": 4.353
|
||||
},
|
||||
"enqueued_to_ready_s": {
|
||||
"p50": 2.895,
|
||||
"p95": 4.889,
|
||||
"max": 4.895
|
||||
},
|
||||
"processing_s": {
|
||||
"p50": 0.765,
|
||||
"max": 1.649,
|
||||
"mean": 0.834
|
||||
},
|
||||
"worker_peak_mem_mib": {
|
||||
"mtusi-lab5-worker-1": 42.2,
|
||||
"mtusi-lab5-worker-2": 100.4
|
||||
},
|
||||
"worker_mem_limit": "1GiB",
|
||||
"staging_left": 0
|
||||
}
|
||||
],
|
||||
"result": {
|
||||
"count": 24,
|
||||
"image_bytes": 6753618,
|
||||
"upload_seconds": 2.31,
|
||||
"total_seconds_to_all_ready": 7.23,
|
||||
"throughput_jobs_per_s": 3.32,
|
||||
"consumers": 4,
|
||||
"prefetch_per_consumer": [
|
||||
1
|
||||
],
|
||||
"max_concurrent_processing": 4,
|
||||
"queue_lag_s": {
|
||||
"p50": 2.401,
|
||||
"p95": 4.069,
|
||||
"max": 4.353
|
||||
},
|
||||
"enqueued_to_ready_s": {
|
||||
"p50": 2.895,
|
||||
"p95": 4.889,
|
||||
"max": 4.895
|
||||
},
|
||||
"processing_s": {
|
||||
"p50": 0.765,
|
||||
"max": 1.649,
|
||||
"mean": 0.834
|
||||
},
|
||||
"worker_peak_mem_mib": {
|
||||
"mtusi-lab5-worker-1": 42.2,
|
||||
"mtusi-lab5-worker-2": 100.4
|
||||
},
|
||||
"worker_mem_limit": "1GiB",
|
||||
"staging_left": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-11-13T13:25:24Z python3 student-tests/burst.py --count 24 --output evidence/burst.json
|
||||
[ 1.10s] consumers {"count": 4, "prefetch": [1]}
|
||||
[ 1.64s] generated images {"bytes": [6753618, 6753618, 6753618, 6753618]}
|
||||
[ 4.75s] all uploads accepted (202) {"count": 24, "seconds": 2.31, "queue_right_after": {"messages": 0, "messages_ready": 0, "messages_unacknowledged": 0, "consumers": 4}}
|
||||
[ 11.68s] result {"count": 24, "image_bytes": 6753618, "upload_seconds": 2.31, "total_seconds_to_all_ready": 7.23, "throughput_jobs_per_s": 3.32, "consumers": 4, "prefetch_per_consumer": [1], "max_concurrent_processing": 4, "queue_lag_s": {"p50": 2.401, "p95": 4.069, "max": 4.353}, "enqueued_to_ready_s": {"p50": 2.895, "p95": 4.889, "max": 4.895}, "processing_s": {"p50": 0.765, "max": 1.649, "mean": 0.834}, "worker_peak_mem_mib": {"mtusi-lab5-worker-1": 42.2, "mtusi-lab5-worker-2": 100.4}, "worker_mem_limit": "1GiB", "staging_left": 0}
|
||||
PASS: 24 jobs, max concurrent 4 <= 4 consumers
|
||||
exit=0
|
||||
@@ -0,0 +1,4 @@
|
||||
# 2026-11-13T13:19:14Z make check
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
@@ -0,0 +1,21 @@
|
||||
# containers after persistence hook
|
||||
mtusi-lab5-api-1 running
|
||||
mtusi-lab5-api-2 running
|
||||
mtusi-lab5-broker-1 running
|
||||
mtusi-lab5-db-1 running
|
||||
mtusi-lab5-janitor-1 running
|
||||
mtusi-lab5-lb-1 running
|
||||
mtusi-lab5-relay-1 running
|
||||
mtusi-lab5-s3-1 running
|
||||
mtusi-lab5-worker-1 running
|
||||
mtusi-lab5-worker-2 running
|
||||
/mtusi-lab5-api-1 StartedAt=2026-11-13T13:19:57.047261594Z
|
||||
/mtusi-lab5-api-2 StartedAt=2026-11-13T13:19:57.066645345Z
|
||||
/mtusi-lab5-broker-1 StartedAt=2026-11-13T13:19:18.587341132Z
|
||||
/mtusi-lab5-db-1 StartedAt=2026-11-13T13:19:17.540371965Z
|
||||
/mtusi-lab5-janitor-1 StartedAt=2026-11-13T13:20:03.852909333Z
|
||||
/mtusi-lab5-lb-1 StartedAt=2026-11-13T13:19:53.963829093Z
|
||||
/mtusi-lab5-relay-1 StartedAt=2026-11-13T13:19:54.774267885Z
|
||||
/mtusi-lab5-s3-1 StartedAt=2026-11-13T13:19:34.799779792Z
|
||||
/mtusi-lab5-worker-1 StartedAt=2026-11-13T13:19:54.68599976Z
|
||||
/mtusi-lab5-worker-2 StartedAt=2026-11-13T13:19:54.204608051Z
|
||||
@@ -0,0 +1,21 @@
|
||||
# containers before persistence hook
|
||||
mtusi-lab5-api-1 running
|
||||
mtusi-lab5-api-2 running
|
||||
mtusi-lab5-broker-1 running
|
||||
mtusi-lab5-db-1 running
|
||||
mtusi-lab5-janitor-1 running
|
||||
mtusi-lab5-lb-1 running
|
||||
mtusi-lab5-relay-1 running
|
||||
mtusi-lab5-s3-1 running
|
||||
mtusi-lab5-worker-1 running
|
||||
mtusi-lab5-worker-2 running
|
||||
/mtusi-lab5-api-1 StartedAt=2026-11-13T13:18:58.823119012Z
|
||||
/mtusi-lab5-api-2 StartedAt=2026-11-13T13:18:58.721958845Z
|
||||
/mtusi-lab5-broker-1 StartedAt=2026-11-13T13:16:34.919270459Z
|
||||
/mtusi-lab5-db-1 StartedAt=2026-11-13T13:16:34.884065126Z
|
||||
/mtusi-lab5-janitor-1 StartedAt=2026-11-13T13:18:58.727139929Z
|
||||
/mtusi-lab5-lb-1 StartedAt=2026-11-13T13:18:58.963141679Z
|
||||
/mtusi-lab5-relay-1 StartedAt=2026-11-13T13:18:58.71470797Z
|
||||
/mtusi-lab5-s3-1 StartedAt=2026-11-13T13:16:34.886978376Z
|
||||
/mtusi-lab5-worker-1 StartedAt=2026-11-13T13:18:58.841973137Z
|
||||
/mtusi-lab5-worker-2 StartedAt=2026-11-13T13:18:58.721509054Z
|
||||
@@ -0,0 +1,40 @@
|
||||
# 2026-11-13T13:26:15Z docker compose --profile tools run --rm -T ops failed --limit 10
|
||||
{
|
||||
"failed": [
|
||||
{
|
||||
"attachment_id": "dddc059c-a98e-4150-b866-dbdbfeb875f2",
|
||||
"job_id": "7c4b6bfa-565d-4399-8d9e-57f58210b657",
|
||||
"error_code": "staging_corrupted",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"replay_count": 0,
|
||||
"last_error": "size/sha256 differ from accepted upload",
|
||||
"updated_at": "2026-11-13T13:22:53.830976Z",
|
||||
"replayable": true
|
||||
},
|
||||
{
|
||||
"attachment_id": "ed395364-f044-42c0-aa9c-d61649968951",
|
||||
"job_id": "b82c635b-27b0-4d6f-9413-689c659bbb89",
|
||||
"error_code": "staging_corrupted",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"replay_count": 0,
|
||||
"last_error": "size/sha256 differ from accepted upload",
|
||||
"updated_at": "2026-11-13T12:41:52.426094Z",
|
||||
"replayable": true
|
||||
},
|
||||
{
|
||||
"attachment_id": "4f03beaa-a713-46c4-88e8-3551f98be852",
|
||||
"job_id": "937971a7-ac66-4c49-94e7-3c0d9188afdf",
|
||||
"error_code": "staging_corrupted",
|
||||
"attempt": 1,
|
||||
"max_attempts": 5,
|
||||
"replay_count": 0,
|
||||
"last_error": "size/sha256 differ from accepted upload",
|
||||
"updated_at": "2026-11-13T12:39:23.836327Z",
|
||||
"replayable": true
|
||||
}
|
||||
],
|
||||
"count": 3
|
||||
}
|
||||
exit=0
|
||||
@@ -0,0 +1,6 @@
|
||||
# 2026-11-13T13:20:04Z make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.725}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
@@ -0,0 +1,3 @@
|
||||
# 2026-11-13T13:26:12Z docker compose exec -T janitor python /app/src/janitor.py --once --dry-run
|
||||
{"dry_run": true, "grace_seconds": 3600, "staging_retention_seconds": 86400, "abandoned_pending": 0, "staging_ready_cleaned": 0, "staging_failed_expired": 0, "orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_rows": 0, "errors": 0, "rows_by_status": {"ready": 109, "failed": 3}, "jobs_by_status": {"ready": 109, "failed": 3}, "outbox_unpublished": 0, "abandoned_ids": [], "objects_total": 439, "objects_by_prefix": {"staging/": 3, "originals/": 0, "attachments/": 436}, "orphan_keys_sample": [], "missing_ids_sample": [], "duration_ms": 94.9}
|
||||
exit=0
|
||||
@@ -0,0 +1,208 @@
|
||||
{
|
||||
"scenario": "lease expiry of a frozen worker and fencing of its late result",
|
||||
"date_utc": "2026-11-13T13:25:17+00:00",
|
||||
"steps": [
|
||||
{
|
||||
"t": 7.61,
|
||||
"utc": "13:24:41.036",
|
||||
"step": "workers recreated",
|
||||
"env": {
|
||||
"WORKER_FAULT_INJECTION": "1",
|
||||
"JOB_LEASE_SECONDS": "10",
|
||||
"FAULT_SLOW_SECONDS": "25"
|
||||
},
|
||||
"workers": [
|
||||
"5b3a6ceb1830",
|
||||
"526f1f18e42d"
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 8.41,
|
||||
"utc": "13:24:41.840",
|
||||
"step": "claimed",
|
||||
"attachment_id": "a7ea720b-3a62-4aba-89bb-7ee47cc4ff77",
|
||||
"lease_owner": "526f1f18e42d:5a6bf2a7",
|
||||
"lease_until": "2026-11-13T13:24:51.070995Z"
|
||||
},
|
||||
{
|
||||
"t": 17.22,
|
||||
"utc": "13:24:50.653",
|
||||
"step": "long-running job keeps its lease (heartbeat)",
|
||||
"lease_until": "2026-11-13T13:24:57.769248Z",
|
||||
"status": "processing"
|
||||
},
|
||||
{
|
||||
"t": 17.37,
|
||||
"utc": "13:24:50.797",
|
||||
"step": "owner container paused (frozen, no heartbeat)",
|
||||
"container": "526f1f18e42d"
|
||||
},
|
||||
{
|
||||
"t": 43.29,
|
||||
"utc": "13:25:16.719",
|
||||
"step": "lease expired -> requeued -> other worker finished",
|
||||
"seconds_after_pause": 25.1,
|
||||
"new_owner": "5b3a6ceb1830:80c565a7",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"lease_expired",
|
||||
"requeued_after_lease",
|
||||
"claimed",
|
||||
"busy_ignored",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 43.32,
|
||||
"utc": "13:25:16.748",
|
||||
"step": "owner container unpaused"
|
||||
},
|
||||
{
|
||||
"t": 44.17,
|
||||
"utc": "13:25:17.600",
|
||||
"step": "late worker fenced, result unchanged",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"lease_expired",
|
||||
"requeued_after_lease",
|
||||
"claimed",
|
||||
"busy_ignored",
|
||||
"ready",
|
||||
"staging_deleted",
|
||||
"aborted_lease_lost"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 2
|
||||
}
|
||||
}
|
||||
],
|
||||
"attachment_id": "a7ea720b-3a62-4aba-89bb-7ee47cc4ff77",
|
||||
"paused_container": "526f1f18e42d",
|
||||
"events": [
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:24:41.056799Z",
|
||||
"event": "enqueued",
|
||||
"attempt": 1,
|
||||
"actor": "api",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:24:41.074354Z",
|
||||
"event": "claimed",
|
||||
"attempt": 1,
|
||||
"actor": "526f1f18e42d:5a6bf2a7",
|
||||
"detail": {
|
||||
"redelivered_message_attempt": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:25:00.154971Z",
|
||||
"event": "lease_expired",
|
||||
"attempt": 1,
|
||||
"actor": "reaper:6e17e167ffa4:c0758d5b",
|
||||
"detail": {
|
||||
"lease_owner": "526f1f18e42d:5a6bf2a7"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:25:00.160740Z",
|
||||
"event": "requeued_after_lease",
|
||||
"attempt": 1,
|
||||
"actor": "reaper:6e17e167ffa4:c0758d5b",
|
||||
"detail": {
|
||||
"error_code": "lease_expired",
|
||||
"next_attempt": 2,
|
||||
"delay_seconds": 1.243
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:25:15.712439Z",
|
||||
"event": "claimed",
|
||||
"attempt": 2,
|
||||
"actor": "5b3a6ceb1830:80c565a7",
|
||||
"detail": {
|
||||
"redelivered_message_attempt": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:25:15.713469Z",
|
||||
"event": "busy_ignored",
|
||||
"attempt": 1,
|
||||
"actor": "5b3a6ceb1830:80c565a7",
|
||||
"detail": {
|
||||
"lease_owner": "5b3a6ceb1830:80c565a7"
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:25:15.860857Z",
|
||||
"event": "ready",
|
||||
"attempt": 2,
|
||||
"actor": "5b3a6ceb1830:80c565a7",
|
||||
"detail": {
|
||||
"variants": {
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
320,
|
||||
200
|
||||
],
|
||||
"thumbnail": [
|
||||
256,
|
||||
160
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:25:15.862651Z",
|
||||
"event": "staging_deleted",
|
||||
"attempt": 2,
|
||||
"actor": "5b3a6ceb1830:80c565a7",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"job_id": "2d2c94ac-bf98-435b-b5c0-5c0d95613cbb",
|
||||
"at": "2026-11-13T13:25:16.745294Z",
|
||||
"event": "aborted_lease_lost",
|
||||
"attempt": 1,
|
||||
"actor": "526f1f18e42d:5a6bf2a7",
|
||||
"detail": null
|
||||
}
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 2
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
# 2026-11-13T13:24:33Z python3 student-tests/lease_takeover.py --output evidence/lease-takeover.json
|
||||
[ 7.61s] workers recreated {"env": {"WORKER_FAULT_INJECTION": "1", "JOB_LEASE_SECONDS": "10", "FAULT_SLOW_SECONDS": "25"}, "workers": ["5b3a6ceb1830", "526f1f18e42d"]}
|
||||
[ 8.41s] claimed {"attachment_id": "a7ea720b-3a62-4aba-89bb-7ee47cc4ff77", "lease_owner": "526f1f18e42d:5a6bf2a7", "lease_until": "2026-11-13T13:24:51.070995Z"}
|
||||
[ 17.22s] long-running job keeps its lease (heartbeat) {"lease_until": "2026-11-13T13:24:57.769248Z", "status": "processing"}
|
||||
[ 17.37s] owner container paused (frozen, no heartbeat) {"container": "526f1f18e42d"}
|
||||
[ 43.29s] lease expired -> requeued -> other worker finished {"seconds_after_pause": 25.1, "new_owner": "5b3a6ceb1830:80c565a7", "events": ["enqueued", "claimed", "lease_expired", "requeued_after_lease", "claimed", "busy_ignored", "ready", "staging_deleted"]}
|
||||
[ 43.32s] owner container unpaused
|
||||
[ 44.17s] late worker fenced, result unchanged {"events": ["enqueued", "claimed", "lease_expired", "requeued_after_lease", "claimed", "busy_ignored", "ready", "staging_deleted", "aborted_lease_lost"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 2}}
|
||||
PASS: frozen worker loses lease, job recovered, late result fenced
|
||||
exit=0
|
||||
@@ -0,0 +1,8 @@
|
||||
# 13.11.2026. Переход lab4 -> lab5: миграция 004 на данных формы lab4 (отдельный postgres:16.4-alpine).
|
||||
# 001..003 применены, вставлено ready-вложение lab4 (object_key originals/abc), затем 004 в одной транзакции (-1).
|
||||
$ psql -tA -c "select status, staging_key, object_key from attachments; select status, attempt, max_attempts from jobs; select payload from outbox;"
|
||||
queued|originals/abc|attachments/33333333-3333-4333-8333-333333333333/v1/original
|
||||
queued|0|5
|
||||
{"type": "attachment.process.v1", "job_id": "0b3091d5-c9d7-4008-8df3-43675d45f07e", "attempt": 1, "created_at": "2026-11-13T12:41:36Z", "attachment_id": "33333333-3333-4333-8333-333333333333", "schema_version": 1, "pipeline_version": 1}
|
||||
# Итог: оригинал lab4 стал staging-объектом, вложение queued (не ready без вариантов), задание и outbox созданы;
|
||||
# relay опубликует его, worker создаст варианты и перенесёт оригинал в attachments/<id>/v1/original.
|
||||
@@ -0,0 +1,358 @@
|
||||
{
|
||||
"scenario": "outbox: COMMIT/publish gap and API crash windows",
|
||||
"date_utc": "2026-11-13T13:24:33+00:00",
|
||||
"steps": [
|
||||
{
|
||||
"t": 6.31,
|
||||
"utc": "13:23:24.684",
|
||||
"step": "A: relay stopped, upload 202",
|
||||
"attachment_id": "41a53cd8-7e57-44b8-9c9f-12320b323750",
|
||||
"outbox": [
|
||||
{
|
||||
"id": 105,
|
||||
"attempt": "1",
|
||||
"available_at": "2026-11-13T13:23:19.956776Z",
|
||||
"created_at": "2026-11-13T13:23:19.956776Z",
|
||||
"published_at": null,
|
||||
"publish_count": 0
|
||||
}
|
||||
],
|
||||
"queue": {
|
||||
"messages": 0,
|
||||
"messages_ready": 0,
|
||||
"messages_unacknowledged": 0,
|
||||
"consumers": 4
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12.77,
|
||||
"utc": "13:23:31.150",
|
||||
"step": "A: relay started -> ready, same single job",
|
||||
"attachment_id": "41a53cd8-7e57-44b8-9c9f-12320b323750",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"outbox": [
|
||||
{
|
||||
"id": 105,
|
||||
"attempt": "1",
|
||||
"available_at": "2026-11-13T13:23:19.956776Z",
|
||||
"created_at": "2026-11-13T13:23:19.956776Z",
|
||||
"published_at": "2026-11-13T13:23:26.328881Z",
|
||||
"publish_count": 1
|
||||
}
|
||||
],
|
||||
"jobs": 1,
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 23.48,
|
||||
"utc": "13:23:41.861",
|
||||
"step": "B: relay crashed between COMMIT and publish -> restarted -> ready",
|
||||
"attachment_id": "a2401b04-a5db-4026-adc4-01d5040fdd2c",
|
||||
"relay_restart_count": [
|
||||
[
|
||||
0
|
||||
],
|
||||
[
|
||||
1
|
||||
]
|
||||
],
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"outbox": [
|
||||
{
|
||||
"id": 106,
|
||||
"attempt": "1",
|
||||
"available_at": "2026-11-13T13:23:38.382769Z",
|
||||
"created_at": "2026-11-13T13:23:38.382769Z",
|
||||
"published_at": "2026-11-13T13:23:40.445416Z",
|
||||
"publish_count": 1
|
||||
}
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 32.98,
|
||||
"utc": "13:23:51.354",
|
||||
"step": "C: published twice (crash before marking) -> duplicate is a no-op",
|
||||
"attachment_id": "85b3ce6d-b06a-49ab-bc80-07be18cfd5d9",
|
||||
"relay_restart_count": [
|
||||
1
|
||||
],
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted",
|
||||
"duplicate_ignored"
|
||||
],
|
||||
"outbox": [
|
||||
{
|
||||
"id": 107,
|
||||
"attempt": "1",
|
||||
"available_at": "2026-11-13T13:23:48.361567Z",
|
||||
"created_at": "2026-11-13T13:23:48.361567Z",
|
||||
"published_at": "2026-11-13T13:23:49.734047Z",
|
||||
"publish_count": 1
|
||||
}
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 49.85,
|
||||
"utc": "13:24:08.230",
|
||||
"step": "D: API died after COMMIT before 202 -> work still completed",
|
||||
"attachment_id": "5778567a-0d6c-4322-b0e0-31a17a40994f",
|
||||
"client_saw": "missing X-Instance-Id",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 50.15,
|
||||
"utc": "13:24:08.530",
|
||||
"step": "E: API died after staging before COMMIT",
|
||||
"attachment_id": "5b7070f4-ece0-49d9-bef4-1d704222955f",
|
||||
"client_saw": "missing X-Instance-Id",
|
||||
"status": "pending",
|
||||
"jobs": 0,
|
||||
"api_get": 404
|
||||
},
|
||||
{
|
||||
"t": 58.86,
|
||||
"utc": "13:24:17.232",
|
||||
"step": "E: janitor removed pending row and staging object",
|
||||
"attachment_id": "5b7070f4-ece0-49d9-bef4-1d704222955f",
|
||||
"janitor_dry_run": {
|
||||
"abandoned_pending": 1,
|
||||
"orphan_objects": 0,
|
||||
"deleted_objects": 0,
|
||||
"deleted_rows": 0
|
||||
},
|
||||
"janitor": {
|
||||
"abandoned_pending": 1,
|
||||
"orphan_objects": 0,
|
||||
"deleted_objects": 1,
|
||||
"deleted_rows": 1,
|
||||
"missing_objects": 0
|
||||
},
|
||||
"row_gone": true
|
||||
},
|
||||
{
|
||||
"t": 74.96,
|
||||
"utc": "13:24:33.332",
|
||||
"step": "relay and api restored to .env configuration"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"A": {
|
||||
"attachment_id": "41a53cd8-7e57-44b8-9c9f-12320b323750",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"outbox": [
|
||||
{
|
||||
"id": 105,
|
||||
"attempt": "1",
|
||||
"available_at": "2026-11-13T13:23:19.956776Z",
|
||||
"created_at": "2026-11-13T13:23:19.956776Z",
|
||||
"published_at": "2026-11-13T13:23:26.328881Z",
|
||||
"publish_count": 1
|
||||
}
|
||||
],
|
||||
"jobs": 1,
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
"B": {
|
||||
"attachment_id": "a2401b04-a5db-4026-adc4-01d5040fdd2c",
|
||||
"relay_restart_count": [
|
||||
[
|
||||
0
|
||||
],
|
||||
[
|
||||
1
|
||||
]
|
||||
],
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"outbox": [
|
||||
{
|
||||
"id": 106,
|
||||
"attempt": "1",
|
||||
"available_at": "2026-11-13T13:23:38.382769Z",
|
||||
"created_at": "2026-11-13T13:23:38.382769Z",
|
||||
"published_at": "2026-11-13T13:23:40.445416Z",
|
||||
"publish_count": 1
|
||||
}
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
"C": {
|
||||
"attachment_id": "85b3ce6d-b06a-49ab-bc80-07be18cfd5d9",
|
||||
"relay_restart_count": [
|
||||
1
|
||||
],
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted",
|
||||
"duplicate_ignored"
|
||||
],
|
||||
"outbox": [
|
||||
{
|
||||
"id": 107,
|
||||
"attempt": "1",
|
||||
"available_at": "2026-11-13T13:23:48.361567Z",
|
||||
"created_at": "2026-11-13T13:23:48.361567Z",
|
||||
"published_at": "2026-11-13T13:23:49.734047Z",
|
||||
"publish_count": 1
|
||||
}
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
"D": {
|
||||
"attachment_id": "5778567a-0d6c-4322-b0e0-31a17a40994f",
|
||||
"client_saw": "missing X-Instance-Id",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
"E": {
|
||||
"attachment_id": "5b7070f4-ece0-49d9-bef4-1d704222955f",
|
||||
"janitor_dry_run": {
|
||||
"abandoned_pending": 1,
|
||||
"orphan_objects": 0,
|
||||
"deleted_objects": 0,
|
||||
"deleted_rows": 0
|
||||
},
|
||||
"janitor": {
|
||||
"abandoned_pending": 1,
|
||||
"orphan_objects": 0,
|
||||
"deleted_objects": 1,
|
||||
"deleted_rows": 1,
|
||||
"missing_objects": 0
|
||||
},
|
||||
"row_gone": true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
# 2026-11-13T13:23:18Z python3 student-tests/outbox_gap.py --output evidence/outbox-gap.json
|
||||
[ 6.31s] A: relay stopped, upload 202 {"attachment_id": "41a53cd8-7e57-44b8-9c9f-12320b323750", "outbox": [{"id": 105, "attempt": "1", "available_at": "2026-11-13T13:23:19.956776Z", "created_at": "2026-11-13T13:23:19.956776Z", "published_at": null, "publish_count": 0}], "queue": {"messages": 0, "messages_ready": 0, "messages_unacknowledged": 0, "consumers": 4}}
|
||||
[ 12.77s] A: relay started -> ready, same single job {"attachment_id": "41a53cd8-7e57-44b8-9c9f-12320b323750", "events": ["enqueued", "claimed", "ready", "staging_deleted"], "outbox": [{"id": 105, "attempt": "1", "available_at": "2026-11-13T13:23:19.956776Z", "created_at": "2026-11-13T13:23:19.956776Z", "published_at": "2026-11-13T13:23:26.328881Z", "publish_count": 1}], "jobs": 1, "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}}
|
||||
[ 23.48s] B: relay crashed between COMMIT and publish -> restarted -> ready {"attachment_id": "a2401b04-a5db-4026-adc4-01d5040fdd2c", "relay_restart_count": [[0], [1]], "events": ["enqueued", "claimed", "ready", "staging_deleted"], "outbox": [{"id": 106, "attempt": "1", "available_at": "2026-11-13T13:23:38.382769Z", "created_at": "2026-11-13T13:23:38.382769Z", "published_at": "2026-11-13T13:23:40.445416Z", "publish_count": 1}], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}}
|
||||
[ 32.98s] C: published twice (crash before marking) -> duplicate is a no-op {"attachment_id": "85b3ce6d-b06a-49ab-bc80-07be18cfd5d9", "relay_restart_count": [1], "events": ["enqueued", "claimed", "ready", "staging_deleted", "duplicate_ignored"], "outbox": [{"id": 107, "attempt": "1", "available_at": "2026-11-13T13:23:48.361567Z", "created_at": "2026-11-13T13:23:48.361567Z", "published_at": "2026-11-13T13:23:49.734047Z", "publish_count": 1}], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}}
|
||||
[ 49.85s] D: API died after COMMIT before 202 -> work still completed {"attachment_id": "5778567a-0d6c-4322-b0e0-31a17a40994f", "client_saw": "missing X-Instance-Id", "events": ["enqueued", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}}
|
||||
[ 50.15s] E: API died after staging before COMMIT {"attachment_id": "5b7070f4-ece0-49d9-bef4-1d704222955f", "client_saw": "missing X-Instance-Id", "status": "pending", "jobs": 0, "api_get": 404}
|
||||
[ 58.86s] E: janitor removed pending row and staging object {"attachment_id": "5b7070f4-ece0-49d9-bef4-1d704222955f", "janitor_dry_run": {"abandoned_pending": 1, "orphan_objects": 0, "deleted_objects": 0, "deleted_rows": 0}, "janitor": {"abandoned_pending": 1, "orphan_objects": 0, "deleted_objects": 1, "deleted_rows": 1, "missing_objects": 0}, "row_gone": true}
|
||||
[ 74.96s] relay and api restored to .env configuration
|
||||
PASS: outbox closes COMMIT/publish gap; API crash windows handled
|
||||
exit=0
|
||||
@@ -0,0 +1,5 @@
|
||||
# 2026-11-13T13:19:16Z make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
@@ -0,0 +1,5 @@
|
||||
# 2026-11-13T13:26:14Z docker compose exec -T broker rabbitmqctl -q -p messenger list_queues name type durable messages consumers
|
||||
name type durable messages consumers
|
||||
attachments.dlq quorum true 0 0
|
||||
attachments.process.v1 quorum true 0 4
|
||||
exit=0
|
||||
@@ -0,0 +1,29 @@
|
||||
# 2026-11-13T13:25:40Z docker compose --profile tools run --rm -T checks /app/student-tests/s3_private_check.py
|
||||
PASS api: PutObject в свой bucket expected=allowed got=allowed
|
||||
PASS anonymous GET object expected=403 got=403
|
||||
PASS anonymous ListBucket expected=403 got=403
|
||||
PASS anonymous ListAllMyBuckets expected=403 got=403
|
||||
PASS anonymous PUT object expected=403 got=403
|
||||
PASS api: GetObject expected=allowed got=allowed
|
||||
PASS api: ListBucket expected=denied got=AccessDenied
|
||||
PASS api: CreateBucket expected=denied got=AccessDenied
|
||||
PASS api: DeleteBucket expected=denied got=AccessDenied
|
||||
PASS api: PutBucketPolicy (сделать публичным) expected=denied got=AccessDenied
|
||||
PASS janitor: GetObject (чтение содержимого) expected=denied got=AccessDenied
|
||||
PASS janitor: ListBucket expected=allowed got=allowed
|
||||
PASS janitor: CreateBucket expected=denied got=AccessDenied
|
||||
PASS worker: GetObject (чтение staging) expected=allowed got=allowed
|
||||
PASS worker: PutObject (результат) expected=allowed got=allowed
|
||||
PASS worker: DeleteObject (очистка staging) expected=allowed got=allowed
|
||||
PASS worker: ListBucket expected=denied got=AccessDenied
|
||||
PASS worker: CreateBucket expected=denied got=AccessDenied
|
||||
PASS api: DeleteObject (компенсация) expected=allowed got=allowed
|
||||
PASS port 9333 (master UI/API) из backend-сети expected=closed got=closed
|
||||
PASS port 8080 (volume) из backend-сети expected=closed got=closed
|
||||
PASS port 8888 (filer UI/API) из backend-сети expected=closed got=closed
|
||||
PASS port 8111 (IAM API) из backend-сети expected=closed got=closed
|
||||
PASS port 8181 (Iceberg) из backend-сети expected=closed got=closed
|
||||
PASS port 9101 (Lance) из backend-сети expected=closed got=closed
|
||||
PASS port 8333 (S3 API) из backend-сети expected=open got=open
|
||||
{"checks": 26, "failed": 0}
|
||||
exit=0
|
||||
@@ -0,0 +1,4 @@
|
||||
# 2026-11-13T13:19:14Z make test
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,98 @@
|
||||
{
|
||||
"cases": [
|
||||
{
|
||||
"exit": 0,
|
||||
"class": "ok",
|
||||
"seconds": 0.13,
|
||||
"case": "valid 640x480",
|
||||
"expected": "ok",
|
||||
"input_bytes": 1949,
|
||||
"env": {},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": 4,
|
||||
"class": "image_too_large",
|
||||
"seconds": 0.1,
|
||||
"case": "49 Mpx PNG (> MAX_IMAGE_PIXELS)",
|
||||
"expected": "image_too_large",
|
||||
"input_bytes": 159675,
|
||||
"env": {},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": 0,
|
||||
"class": "ok",
|
||||
"seconds": 0.67,
|
||||
"case": "36 Mpx PNG, normal limits",
|
||||
"expected": "ok",
|
||||
"input_bytes": 119592,
|
||||
"env": {},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": 5,
|
||||
"class": "resource_limit",
|
||||
"seconds": 0.11,
|
||||
"case": "36 Mpx PNG, memory limit 160 MiB",
|
||||
"expected": "resource_limit",
|
||||
"input_bytes": 119592,
|
||||
"env": {
|
||||
"TRANSFORM_MEMORY_LIMIT_MB": "160"
|
||||
},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": -24,
|
||||
"class": "killed_by_signal_24",
|
||||
"seconds": 1.0,
|
||||
"case": "40 Mpx RGBA noise PNG, CPU limit 1 s",
|
||||
"expected": "killed_by_signal_24",
|
||||
"input_bytes": 151852446,
|
||||
"env": {
|
||||
"TRANSFORM_CPU_SECONDS": "1"
|
||||
},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": "timeout",
|
||||
"class": "processing_timeout",
|
||||
"seconds": 0.51,
|
||||
"case": "40 Mpx RGBA noise PNG, wall timeout 0.5 s",
|
||||
"expected": "processing_timeout",
|
||||
"input_bytes": 151852446,
|
||||
"env": {},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": 3,
|
||||
"class": "invalid_image",
|
||||
"seconds": 0.1,
|
||||
"case": "truncated PNG",
|
||||
"expected": "invalid_image",
|
||||
"input_bytes": 974,
|
||||
"env": {},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": 3,
|
||||
"class": "invalid_image",
|
||||
"seconds": 0.1,
|
||||
"case": "garbage declared as JPEG",
|
||||
"expected": "invalid_image",
|
||||
"input_bytes": 5003,
|
||||
"env": {},
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"exit": 3,
|
||||
"class": "invalid_image",
|
||||
"seconds": 0.11,
|
||||
"case": "PNG declared as JPEG",
|
||||
"expected": "invalid_image",
|
||||
"input_bytes": 1949,
|
||||
"env": {},
|
||||
"pass": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
# 2026-11-13T13:20:56Z docker compose --profile tools run --rm -T checks /app/student-tests/transform_limits.py --output /app/evidence/transform-limits.json
|
||||
PASS valid 640x480 -> ok (0.13s)
|
||||
PASS 49 Mpx PNG (> MAX_IMAGE_PIXELS) -> image_too_large (0.10s)
|
||||
PASS 36 Mpx PNG, normal limits -> ok (0.67s)
|
||||
PASS 36 Mpx PNG, memory limit 160 MiB -> resource_limit (0.11s)
|
||||
PASS 40 Mpx RGBA noise PNG, CPU limit 1 s -> killed_by_signal_24 (1.00s)
|
||||
PASS 40 Mpx RGBA noise PNG, wall timeout 0.5 s -> processing_timeout (0.51s)
|
||||
PASS truncated PNG -> invalid_image (0.10s)
|
||||
PASS garbage declared as JPEG -> invalid_image (0.10s)
|
||||
PASS PNG declared as JPEG -> invalid_image (0.11s)
|
||||
PASS 9/9
|
||||
exit=0
|
||||
@@ -0,0 +1,287 @@
|
||||
{
|
||||
"cases": [
|
||||
{
|
||||
"case": "fixture 320x200 PNG",
|
||||
"source_oriented": [
|
||||
320,
|
||||
200
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"size": [
|
||||
256,
|
||||
160
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 4679,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"crop": {
|
||||
"size": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 1820,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"optimized": {
|
||||
"size": [
|
||||
320,
|
||||
200
|
||||
],
|
||||
"format": "WEBP",
|
||||
"bytes": 2462,
|
||||
"no_exif_icc": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "RGBA 300x300 half transparent",
|
||||
"source_oriented": [
|
||||
300,
|
||||
300
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"size": [
|
||||
256,
|
||||
256
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 957,
|
||||
"no_exif_icc": true,
|
||||
"transparent_pixel_rgb": [
|
||||
255,
|
||||
255,
|
||||
255
|
||||
]
|
||||
},
|
||||
"crop": {
|
||||
"size": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 619,
|
||||
"no_exif_icc": true,
|
||||
"transparent_pixel_rgb": [
|
||||
255,
|
||||
255,
|
||||
255
|
||||
]
|
||||
},
|
||||
"optimized": {
|
||||
"size": [
|
||||
300,
|
||||
300
|
||||
],
|
||||
"format": "WEBP",
|
||||
"bytes": 350,
|
||||
"no_exif_icc": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "JPEG 400x200 EXIF orientation=6",
|
||||
"source_oriented": [
|
||||
200,
|
||||
400
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"size": [
|
||||
128,
|
||||
256
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 861,
|
||||
"no_exif_icc": true,
|
||||
"left_rgb": [
|
||||
0,
|
||||
0,
|
||||
254
|
||||
],
|
||||
"right_rgb": [
|
||||
254,
|
||||
0,
|
||||
0
|
||||
]
|
||||
},
|
||||
"crop": {
|
||||
"size": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 578,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"optimized": {
|
||||
"size": [
|
||||
200,
|
||||
400
|
||||
],
|
||||
"format": "WEBP",
|
||||
"bytes": 374,
|
||||
"no_exif_icc": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "tiny PNG 10x5 (no upscale)",
|
||||
"source_oriented": [
|
||||
10,
|
||||
5
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"size": [
|
||||
10,
|
||||
5
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 288,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"crop": {
|
||||
"size": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 383,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"optimized": {
|
||||
"size": [
|
||||
10,
|
||||
5
|
||||
],
|
||||
"format": "WEBP",
|
||||
"bytes": 68,
|
||||
"no_exif_icc": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "JPEG 3000x2000",
|
||||
"source_oriented": [
|
||||
3000,
|
||||
2000
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"size": [
|
||||
256,
|
||||
170
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 2556,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"crop": {
|
||||
"size": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 1230,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"optimized": {
|
||||
"size": [
|
||||
1280,
|
||||
853
|
||||
],
|
||||
"format": "WEBP",
|
||||
"bytes": 5144,
|
||||
"no_exif_icc": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "tall PNG 100x1000",
|
||||
"source_oriented": [
|
||||
100,
|
||||
1000
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"size": [
|
||||
25,
|
||||
256
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 334,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"crop": {
|
||||
"size": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 382,
|
||||
"no_exif_icc": true
|
||||
},
|
||||
"optimized": {
|
||||
"size": [
|
||||
100,
|
||||
1000
|
||||
],
|
||||
"format": "WEBP",
|
||||
"bytes": 272,
|
||||
"no_exif_icc": true
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"case": "palette PNG with transparency",
|
||||
"source_oriented": [
|
||||
64,
|
||||
64
|
||||
],
|
||||
"variants": {
|
||||
"thumbnail": {
|
||||
"size": [
|
||||
64,
|
||||
64
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 308,
|
||||
"no_exif_icc": true,
|
||||
"transparent_pixel_rgb": [
|
||||
255,
|
||||
255,
|
||||
255
|
||||
]
|
||||
},
|
||||
"crop": {
|
||||
"size": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"format": "JPEG",
|
||||
"bytes": 380,
|
||||
"no_exif_icc": true,
|
||||
"transparent_pixel_rgb": [
|
||||
255,
|
||||
255,
|
||||
255
|
||||
]
|
||||
},
|
||||
"optimized": {
|
||||
"size": [
|
||||
64,
|
||||
64
|
||||
],
|
||||
"format": "WEBP",
|
||||
"bytes": 130,
|
||||
"no_exif_icc": true
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"unknown_variant_status": 400
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
# 2026-11-13T13:20:54Z docker compose --profile tools run --rm -T checks /app/student-tests/variants_check.py --base-url http://lb:8080 --output /app/evidence/variants-check.json
|
||||
PASS fixture 320x200 PNG {'thumbnail': (256, 160), 'crop': (128, 128), 'optimized': (320, 200)}
|
||||
PASS RGBA 300x300 half transparent {'thumbnail': (256, 256), 'crop': (128, 128), 'optimized': (300, 300)}
|
||||
PASS JPEG 400x200 EXIF orientation=6 {'thumbnail': (128, 256), 'crop': (128, 128), 'optimized': (200, 400)}
|
||||
PASS tiny PNG 10x5 (no upscale) {'thumbnail': (10, 5), 'crop': (128, 128), 'optimized': (10, 5)}
|
||||
PASS JPEG 3000x2000 {'thumbnail': (256, 170), 'crop': (128, 128), 'optimized': (1280, 853)}
|
||||
PASS tall PNG 100x1000 {'thumbnail': (25, 256), 'crop': (128, 128), 'optimized': (100, 1000)}
|
||||
PASS palette PNG with transparency {'thumbnail': (64, 64), 'crop': (128, 128), 'optimized': (64, 64)}
|
||||
PASS: 7 inputs, all variants match contract
|
||||
exit=0
|
||||
@@ -0,0 +1,416 @@
|
||||
{
|
||||
"scenario": "worker faults, retries, failed registry, replay, DLQ",
|
||||
"date_utc": "2026-11-13T13:23:18+00:00",
|
||||
"steps": [
|
||||
{
|
||||
"t": 7.26,
|
||||
"utc": "13:21:13.790",
|
||||
"step": "workers recreated with fault injection",
|
||||
"env": {
|
||||
"WORKER_FAULT_INJECTION": "1",
|
||||
"JOB_RETRY_BASE_SECONDS": "1",
|
||||
"JOB_RETRY_MAX_SECONDS": "8"
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 40.75,
|
||||
"utc": "13:21:47.283",
|
||||
"step": "crash-after-claim",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"fault_injected",
|
||||
"busy_ignored",
|
||||
"lease_expired",
|
||||
"requeued_after_lease",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 76.1,
|
||||
"utc": "13:22:22.637",
|
||||
"step": "crash-after-s3-put",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"fault_injected",
|
||||
"busy_ignored",
|
||||
"lease_expired",
|
||||
"requeued_after_lease",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 2
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 80.78,
|
||||
"utc": "13:22:27.315",
|
||||
"step": "crash-after-commit",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"fault_injected",
|
||||
"duplicate_ignored",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 80.92,
|
||||
"utc": "13:22:27.454",
|
||||
"step": "worker containers after crashes (restart: unless-stopped)",
|
||||
"containers": [
|
||||
"/mtusi-lab5-worker-1 1 running",
|
||||
"/mtusi-lab5-worker-2 2 running"
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 85.17,
|
||||
"utc": "13:22:31.709",
|
||||
"step": "transient-2: retried with backoff",
|
||||
"delays": [
|
||||
0.952,
|
||||
1.513
|
||||
],
|
||||
"bounds": [
|
||||
[
|
||||
0.5,
|
||||
1
|
||||
],
|
||||
[
|
||||
1.0,
|
||||
2
|
||||
]
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 3
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 101.04,
|
||||
"utc": "13:22:47.577",
|
||||
"step": "transient-always: failed after max attempts",
|
||||
"error_code": "retry_exhausted",
|
||||
"attempts": 5,
|
||||
"content": 409,
|
||||
"attach": 409,
|
||||
"registry_entry": {
|
||||
"attachment_id": "8a6dc0d1-e4eb-4187-9618-09148e913ebc",
|
||||
"job_id": "1a212af1-07fd-4afe-a5bd-4342839a3e8e",
|
||||
"error_code": "retry_exhausted",
|
||||
"attempt": 5,
|
||||
"max_attempts": 5,
|
||||
"replay_count": 0,
|
||||
"last_error": "last error: storage_unavailable injected transient error",
|
||||
"updated_at": "2026-11-13T13:22:45.859850Z",
|
||||
"replayable": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 104.53,
|
||||
"utc": "13:22:51.067",
|
||||
"step": "replay -> ready; replay of ready refused",
|
||||
"attachment_id": "8a6dc0d1-e4eb-4187-9618-09148e913ebc",
|
||||
"dry_run": "would_replay",
|
||||
"replay": "replayed",
|
||||
"second_replay": "refused_status_ready",
|
||||
"replay_count": 1,
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"failed",
|
||||
"replayed",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 6
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 113.33,
|
||||
"utc": "13:22:59.866",
|
||||
"step": "tampered staging -> failed without retries",
|
||||
"attachment_id": "dddc059c-a98e-4150-b866-dbdbfeb875f2",
|
||||
"error_code": "staging_corrupted",
|
||||
"attempts": 1,
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"failed"
|
||||
]
|
||||
},
|
||||
{
|
||||
"t": 125.06,
|
||||
"utc": "13:23:11.590",
|
||||
"step": "poison message dead-lettered and purged",
|
||||
"dlq_before": 0,
|
||||
"dlq_after_publish": 1,
|
||||
"listed": [
|
||||
{
|
||||
"reason": "rejected",
|
||||
"delivery_count": 1,
|
||||
"action": "kept",
|
||||
"invalid": "extra fields"
|
||||
}
|
||||
],
|
||||
"purged": [
|
||||
"purged_invalid"
|
||||
],
|
||||
"dlq_after_purge": 0
|
||||
},
|
||||
{
|
||||
"t": 131.72,
|
||||
"utc": "13:23:18.259",
|
||||
"step": "workers restored to .env configuration"
|
||||
}
|
||||
],
|
||||
"results": {
|
||||
"crash-after-claim": {
|
||||
"attachment_id": "93dccce9-b04b-4317-a72e-68f43e719634",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"fault_injected",
|
||||
"busy_ignored",
|
||||
"lease_expired",
|
||||
"requeued_after_lease",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 2
|
||||
}
|
||||
},
|
||||
"crash-after-s3-put": {
|
||||
"attachment_id": "da8e1461-70e0-431d-81da-d993cca01288",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"fault_injected",
|
||||
"busy_ignored",
|
||||
"lease_expired",
|
||||
"requeued_after_lease",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 2
|
||||
}
|
||||
},
|
||||
"crash-after-commit": {
|
||||
"attachment_id": "1379cce6-0c65-40cf-a575-61a7f643ea07",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"fault_injected",
|
||||
"duplicate_ignored",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
},
|
||||
"transient-2": {
|
||||
"attachment_id": "58756a3d-8bb3-4aad-bb47-f1b4c4b0696e",
|
||||
"delays": [
|
||||
0.952,
|
||||
1.513
|
||||
],
|
||||
"bounds": [
|
||||
[
|
||||
0.5,
|
||||
1
|
||||
],
|
||||
[
|
||||
1.0,
|
||||
2
|
||||
]
|
||||
],
|
||||
"claims_at": [
|
||||
"2026-11-13T13:22:27.471179Z",
|
||||
"2026-11-13T13:22:28.490332Z",
|
||||
"2026-11-13T13:22:30.531633Z"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 3
|
||||
}
|
||||
},
|
||||
"transient-always+replay": {
|
||||
"attachment_id": "8a6dc0d1-e4eb-4187-9618-09148e913ebc",
|
||||
"dry_run": "would_replay",
|
||||
"replay": "replayed",
|
||||
"second_replay": "refused_status_ready",
|
||||
"replay_count": 1,
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"retry_scheduled",
|
||||
"claimed",
|
||||
"failed",
|
||||
"replayed",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 6
|
||||
}
|
||||
},
|
||||
"staging-tamper": {
|
||||
"attachment_id": "dddc059c-a98e-4150-b866-dbdbfeb875f2",
|
||||
"error_code": "staging_corrupted",
|
||||
"attempts": 1,
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"failed"
|
||||
]
|
||||
},
|
||||
"poison": {
|
||||
"dlq_before": 0,
|
||||
"dlq_after_publish": 1,
|
||||
"listed": [
|
||||
{
|
||||
"reason": "rejected",
|
||||
"delivery_count": 1,
|
||||
"action": "kept",
|
||||
"invalid": "extra fields"
|
||||
}
|
||||
],
|
||||
"purged": [
|
||||
"purged_invalid"
|
||||
],
|
||||
"dlq_after_purge": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
# 2026-11-13T13:21:06Z python3 student-tests/worker_faults.py --output evidence/worker-faults.json
|
||||
[ 7.26s] workers recreated with fault injection {"env": {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"}}
|
||||
[ 40.75s] crash-after-claim {"events": ["enqueued", "claimed", "fault_injected", "busy_ignored", "lease_expired", "requeued_after_lease", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 2}}
|
||||
[ 76.10s] crash-after-s3-put {"events": ["enqueued", "claimed", "fault_injected", "busy_ignored", "lease_expired", "requeued_after_lease", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 2}}
|
||||
[ 80.78s] crash-after-commit {"events": ["enqueued", "claimed", "ready", "fault_injected", "duplicate_ignored", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}}
|
||||
[ 80.92s] worker containers after crashes (restart: unless-stopped) {"containers": ["/mtusi-lab5-worker-1 1 running", "/mtusi-lab5-worker-2 2 running"]}
|
||||
[ 85.17s] transient-2: retried with backoff {"delays": [0.952, 1.513], "bounds": [[0.5, 1], [1.0, 2]], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 3}}
|
||||
[ 101.04s] transient-always: failed after max attempts {"error_code": "retry_exhausted", "attempts": 5, "content": 409, "attach": 409, "registry_entry": {"attachment_id": "8a6dc0d1-e4eb-4187-9618-09148e913ebc", "job_id": "1a212af1-07fd-4afe-a5bd-4342839a3e8e", "error_code": "retry_exhausted", "attempt": 5, "max_attempts": 5, "replay_count": 0, "last_error": "last error: storage_unavailable injected transient error", "updated_at": "2026-11-13T13:22:45.859850Z", "replayable": true}}
|
||||
[ 104.53s] replay -> ready; replay of ready refused {"attachment_id": "8a6dc0d1-e4eb-4187-9618-09148e913ebc", "dry_run": "would_replay", "replay": "replayed", "second_replay": "refused_status_ready", "replay_count": 1, "events": ["enqueued", "claimed", "retry_scheduled", "claimed", "retry_scheduled", "claimed", "retry_scheduled", "claimed", "retry_scheduled", "claimed", "failed", "replayed", "claimed", "ready", "staging_deleted"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 6}}
|
||||
[ 113.33s] tampered staging -> failed without retries {"attachment_id": "dddc059c-a98e-4150-b866-dbdbfeb875f2", "error_code": "staging_corrupted", "attempts": 1, "events": ["enqueued", "claimed", "failed"]}
|
||||
[ 125.06s] poison message dead-lettered and purged {"dlq_before": 0, "dlq_after_publish": 1, "listed": [{"reason": "rejected", "delivery_count": 1, "action": "kept", "invalid": "extra fields"}], "purged": ["purged_invalid"], "dlq_after_purge": 0}
|
||||
[ 131.72s] workers restored to .env configuration
|
||||
PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ
|
||||
exit=0
|
||||
@@ -0,0 +1,185 @@
|
||||
{
|
||||
"scenario": "worker outage -> restart api+broker -> worker start -> ready -> redelivery",
|
||||
"date_utc": "2026-11-13T13:20:54+00:00",
|
||||
"steps": [
|
||||
{
|
||||
"t": 1.69,
|
||||
"utc": "13:20:27.997",
|
||||
"step": "worker stopped",
|
||||
"running": []
|
||||
},
|
||||
{
|
||||
"t": 2.72,
|
||||
"utc": "13:20:29.021",
|
||||
"step": "upload while worker is down",
|
||||
"http": 202,
|
||||
"attachment_id": "d5d38cb6-d3e7-424b-8dc6-33270c9df865",
|
||||
"status": "queued"
|
||||
},
|
||||
{
|
||||
"t": 2.72,
|
||||
"utc": "13:20:29.031",
|
||||
"step": "metadata and content while queued",
|
||||
"status": "queued",
|
||||
"content": 409,
|
||||
"thumbnail": 409,
|
||||
"error_code": "conflict"
|
||||
},
|
||||
{
|
||||
"t": 2.73,
|
||||
"utc": "13:20:29.041",
|
||||
"step": "messages while worker is down",
|
||||
"with_queued_attachment": true,
|
||||
"plain_message_id": "10afa020-6afb-4516-853f-2254f37ed7c7"
|
||||
},
|
||||
{
|
||||
"t": 6.56,
|
||||
"utc": "13:20:32.862",
|
||||
"step": "job is durable in broker",
|
||||
"queue": {
|
||||
"messages": 1,
|
||||
"messages_ready": 1,
|
||||
"messages_unacknowledged": 0,
|
||||
"consumers": 0
|
||||
},
|
||||
"outbox_published": true,
|
||||
"job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c",
|
||||
"job_status": "queued"
|
||||
},
|
||||
{
|
||||
"t": 16.02,
|
||||
"utc": "13:20:42.330",
|
||||
"step": "api restarted",
|
||||
"status": "queued"
|
||||
},
|
||||
{
|
||||
"t": 21.07,
|
||||
"utc": "13:20:47.381",
|
||||
"step": "broker restarted with volume",
|
||||
"queue": {
|
||||
"messages": 1,
|
||||
"messages_ready": 1,
|
||||
"messages_unacknowledged": 0,
|
||||
"consumers": 0
|
||||
},
|
||||
"status": "queued"
|
||||
},
|
||||
{
|
||||
"t": 26.86,
|
||||
"utc": "13:20:53.167",
|
||||
"step": "worker started"
|
||||
},
|
||||
{
|
||||
"t": 26.89,
|
||||
"utc": "13:20:53.196",
|
||||
"step": "ready after worker start",
|
||||
"variants": [
|
||||
[
|
||||
"thumbnail",
|
||||
256,
|
||||
160
|
||||
],
|
||||
[
|
||||
"crop",
|
||||
128,
|
||||
128
|
||||
],
|
||||
[
|
||||
"optimized",
|
||||
320,
|
||||
200
|
||||
]
|
||||
],
|
||||
"original_sha256_ok": true,
|
||||
"thumbnail_bytes": 4679
|
||||
},
|
||||
{
|
||||
"t": 28.47,
|
||||
"utc": "13:20:54.779",
|
||||
"step": "redelivery of the same job is a no-op",
|
||||
"events": [
|
||||
"enqueued",
|
||||
"claimed",
|
||||
"ready",
|
||||
"staging_deleted",
|
||||
"duplicate_ignored"
|
||||
],
|
||||
"invariants": {
|
||||
"variants": [
|
||||
"crop",
|
||||
"optimized",
|
||||
"thumbnail"
|
||||
],
|
||||
"variants_ok": true,
|
||||
"final_objects": 4,
|
||||
"final_objects_ok": true,
|
||||
"staging_left": false,
|
||||
"job_status": "ready",
|
||||
"attempt": 1
|
||||
}
|
||||
}
|
||||
],
|
||||
"attachment_id": "d5d38cb6-d3e7-424b-8dc6-33270c9df865",
|
||||
"job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c",
|
||||
"events": [
|
||||
{
|
||||
"job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c",
|
||||
"at": "2026-11-13T13:20:29.018662Z",
|
||||
"event": "enqueued",
|
||||
"attempt": 1,
|
||||
"actor": "api",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c",
|
||||
"at": "2026-11-13T13:20:47.938036Z",
|
||||
"event": "claimed",
|
||||
"attempt": 1,
|
||||
"actor": "c2f424ff165b:fa6a01e0",
|
||||
"detail": {
|
||||
"redelivered_message_attempt": 1
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c",
|
||||
"at": "2026-11-13T13:20:48.074885Z",
|
||||
"event": "ready",
|
||||
"attempt": 1,
|
||||
"actor": "c2f424ff165b:fa6a01e0",
|
||||
"detail": {
|
||||
"variants": {
|
||||
"crop": [
|
||||
128,
|
||||
128
|
||||
],
|
||||
"optimized": [
|
||||
320,
|
||||
200
|
||||
],
|
||||
"thumbnail": [
|
||||
256,
|
||||
160
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c",
|
||||
"at": "2026-11-13T13:20:48.076077Z",
|
||||
"event": "staging_deleted",
|
||||
"attempt": 1,
|
||||
"actor": "c2f424ff165b:fa6a01e0",
|
||||
"detail": null
|
||||
},
|
||||
{
|
||||
"job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c",
|
||||
"at": "2026-11-13T13:20:53.879984Z",
|
||||
"event": "duplicate_ignored",
|
||||
"attempt": 1,
|
||||
"actor": "c2f424ff165b:fa6a01e0",
|
||||
"detail": {
|
||||
"status": "ready"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# 2026-11-13T13:20:26Z python3 student-tests/worker_outage.py --output evidence/worker-outage.json
|
||||
[ 1.69s] worker stopped {"running": []}
|
||||
[ 2.72s] upload while worker is down {"http": 202, "attachment_id": "d5d38cb6-d3e7-424b-8dc6-33270c9df865", "status": "queued"}
|
||||
[ 2.72s] metadata and content while queued {"status": "queued", "content": 409, "thumbnail": 409, "error_code": "conflict"}
|
||||
[ 2.73s] messages while worker is down {"with_queued_attachment": true, "plain_message_id": "10afa020-6afb-4516-853f-2254f37ed7c7"}
|
||||
[ 6.56s] job is durable in broker {"queue": {"messages": 1, "messages_ready": 1, "messages_unacknowledged": 0, "consumers": 0}, "outbox_published": true, "job_id": "41ad5bea-d05d-42b4-a450-d47690ef998c", "job_status": "queued"}
|
||||
[ 16.02s] api restarted {"status": "queued"}
|
||||
[ 21.07s] broker restarted with volume {"queue": {"messages": 1, "messages_ready": 1, "messages_unacknowledged": 0, "consumers": 0}, "status": "queued"}
|
||||
[ 26.86s] worker started
|
||||
[ 26.89s] ready after worker start {"variants": [["thumbnail", 256, 160], ["crop", 128, 128], ["optimized", 320, 200]], "original_sha256_ok": true, "thumbnail_bytes": 4679}
|
||||
[ 28.47s] redelivery of the same job is a no-op {"events": ["enqueued", "claimed", "ready", "staging_deleted", "duplicate_ignored"], "invariants": {"variants": ["crop", "optimized", "thumbnail"], "variants_ok": true, "final_objects": 4, "final_objects_ok": true, "staging_left": false, "job_status": "ready", "attempt": 1}}
|
||||
PASS: worker outage, api+broker restart, ready, duplicate delivery no-op
|
||||
exit=0
|
||||
@@ -0,0 +1,106 @@
|
||||
# Lab3/4: единая точка входа перед API-репликами.
|
||||
#
|
||||
# Lab4: upload вложений — тело идёт потоком (без http-buffer-request); для upload timeout server
|
||||
# увеличен до 75 с (> UPLOAD_TIMEOUT_SECONDS=60 у API + запись в S3). Content-Length
|
||||
# больше лимита API отклоняет сам, не читая тело.
|
||||
#
|
||||
# Обнаружение реплик: DNS Docker (127.0.0.11) отдаёт по имени сервиса `api` адреса
|
||||
# ВСЕХ его контейнеров; server-template создаёт до 8 слотов и заполняет их по DNS.
|
||||
# Поэтому `docker compose up --scale api=N` не требует правки конфигурации.
|
||||
#
|
||||
# Readiness: активная проверка GET /health/ready каждую 1 с; 2 неудачи подряд -> DOWN,
|
||||
# 2 успеха -> UP. Максимальное время обнаружения отказа:
|
||||
# inter(1s) + timeout check(1s) + fastinter(0.5s) + timeout check(1s) ~= 3.5 с;
|
||||
# ready=503 или refused без ожидания таймаута -> ~1.5 с.
|
||||
# Sticky sessions НЕ используются: balance roundrobin без cookie/stick-table для backend
|
||||
# (stick-table во frontend считает только частоту входов по IP, на выбор реплики не влияет).
|
||||
#
|
||||
# Перегрузка: maxconn 10 на реплику (= DB_POOL_SIZE) — лишние запросы ждут в очереди LB
|
||||
# (timeout queue 5s), а не плодят потоки внутри Python-процесса. Переполнение -> 503 JSON.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold other 2s
|
||||
hold refused 2s
|
||||
hold nx 2s
|
||||
hold timeout 2s
|
||||
hold obsolete 2s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
# Внутри одного Docker-host соединение устанавливается за доли мс: 500 мс хватает с запасом,
|
||||
# а при внезапно исчезнувшей реплике запрос быстро уходит в retry на другую.
|
||||
timeout connect 500ms
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 5s
|
||||
timeout check 1s
|
||||
# Повтор только если соединение с backend НЕ установлено (запрос точно не дошёл
|
||||
# до приложения) — это безопасно даже для неидемпотентных POST. Повтор уходит
|
||||
# на другую реплику (redispatch). Ответы/таймауты после отправки не повторяются.
|
||||
retries 2
|
||||
retry-on conn-failure
|
||||
option redispatch 1
|
||||
|
||||
# Ошибки, которые формирует сам HAProxy (нет живых реплик, обрыв backend, timeout,
|
||||
# переполнение очереди), отдаются в общем JSON-формате ошибки контракта, а не HTML.
|
||||
# Ответы приложения (в т.ч. его 503) эти правила не затрагивают.
|
||||
unique-id-format %[uuid()]
|
||||
http-error status 400 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"invalid_request","message":"Неверный запрос","request_id":"%ID"}}'
|
||||
http-error status 408 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"invalid_request","message":"Запрос не получен вовремя","request_id":"%ID"}}'
|
||||
http-error status 413 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"payload_too_large","message":"Файл больше допустимого размера","request_id":"%ID"}}'
|
||||
http-error status 429 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "10" lf-string '{"error":{"code":"rate_limited","message":"Слишком много запросов, повторите позже","request_id":"%ID"}}'
|
||||
http-error status 500 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" lf-string '{"error":{"code":"internal_error","message":"Внутренняя ошибка сервера","request_id":"%ID"}}'
|
||||
http-error status 502 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "1" lf-string '{"error":{"code":"unavailable","message":"Сервис временно недоступен","request_id":"%ID"}}'
|
||||
http-error status 503 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "1" lf-string '{"error":{"code":"unavailable","message":"Сервис временно недоступен","request_id":"%ID"}}'
|
||||
http-error status 504 content-type "application/json; charset=utf-8" hdr X-Request-Id "%ID" hdr Retry-After "1" lf-string '{"error":{"code":"unavailable","message":"Сервис временно недоступен","request_id":"%ID"}}'
|
||||
|
||||
frontend fe_api
|
||||
bind :8080
|
||||
# X-Request-Id формирует само приложение; LB только пробрасывает адрес клиента.
|
||||
option forwardfor
|
||||
# Защита входа от перебора паролей и шторма scrypt: не более LOGIN_RATE_LIMIT
|
||||
# попыток POST /auth/sessions за 10 с с одного IP, сверх — 429 rate_limited.
|
||||
stick-table type ip size 100k expire 60s store http_req_rate(10s)
|
||||
acl is_login method POST
|
||||
acl is_login_path path /api/v1/auth/sessions
|
||||
# Явно вычисляем unique-id до deny: иначе в ответе 429 request_id остаётся пустым.
|
||||
http-request set-var(txn.request_id) unique-id
|
||||
http-request track-sc0 src if is_login is_login_path
|
||||
http-request deny deny_status 429 if is_login is_login_path { sc_http_req_rate(0) gt "${LOGIN_RATE_LIMIT}" }
|
||||
# Lab4: лимит размера upload проверяет API по Content-Length ДО чтения тела (ответ 413
|
||||
# с X-Instance-Id по контракту). Дублирующий deny на LB убран: его ответ без X-Instance-Id.
|
||||
default_backend be_api
|
||||
|
||||
backend be_api
|
||||
balance roundrobin
|
||||
http-request set-timeout server 75s if METH_POST { path_reg ^/api/v1/chats/[^/]+/attachments$ }
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
|
||||
http-check expect status 200
|
||||
# on-marked-down shutdown-sessions НЕ используется: при штатной остановке реплика ещё
|
||||
# дообслуживает начатые запросы (drain), и обрыв их сессий дал бы лишние 502.
|
||||
default-server maxconn 10 check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr libc,none
|
||||
server-template api 8 api:8080
|
||||
|
||||
# Статистика LB (только на 127.0.0.1 хоста): видно, какие реплики UP/DOWN.
|
||||
frontend stats
|
||||
bind :8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
stats show-legends
|
||||
@@ -0,0 +1,16 @@
|
||||
# RabbitMQ 4.1 для lab5. Топология и пользователи — из definitions (Docker secret,
|
||||
# генерирует scripts/init-secrets.sh): exchange attachments -> quorum-очередь
|
||||
# attachments.process.v1 (DLX attachments.dlx -> attachments.dlq).
|
||||
load_definitions = /run/secrets/rabbitmq_definitions.json
|
||||
# guest (если создан) пускается только с localhost внутри контейнера.
|
||||
loopback_users.guest = true
|
||||
listeners.tcp.default = 5672
|
||||
# Защита брокера от переполнения памяти/диска: при достижении порога publishers блокируются
|
||||
# (relay ждёт, задания копятся в outbox PostgreSQL — работа не теряется).
|
||||
vm_memory_high_watermark.relative = 0.6
|
||||
disk_free_limit.absolute = 256MB
|
||||
# Неподтверждённое сообщение дольше 30 мин -> канал закрывается, сообщение возвращается.
|
||||
consumer_timeout = 1800000
|
||||
log.console = true
|
||||
log.console.level = info
|
||||
log.file = false
|
||||
@@ -0,0 +1,40 @@
|
||||
-- Lab2: начальная схема. Источник истины — PostgreSQL.
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
position integer NOT NULL, -- порядок участников как в запросе
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
-- Индекс под keyset-пагинацию истории: (created_at, id) внутри чата.
|
||||
CREATE INDEX messages_chat_order_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY, -- SHA-256 от токена; сам токен не хранится
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz
|
||||
);
|
||||
@@ -0,0 +1,17 @@
|
||||
-- Lab3: общая (для всех реплик) идемпотентность POST сообщения.
|
||||
-- Миграция только добавляет таблицу: код lab2 продолжает работать на новой схеме
|
||||
-- (expand-only), поэтому её можно применить до замены реплик при rolling update.
|
||||
--
|
||||
-- Область уникальности по контракту: пользователь + метод + путь чата + ключ.
|
||||
-- Метод всегда POST, путь однозначно задаётся chat_id, поэтому ключ PK — (user_id, chat_id, idem_key).
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
idem_key text NOT NULL,
|
||||
request_hash bytea NOT NULL, -- SHA-256 канонического JSON тела запроса
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, idem_key)
|
||||
);
|
||||
CREATE INDEX idempotency_keys_expires_idx ON idempotency_keys (expires_at);
|
||||
@@ -0,0 +1,40 @@
|
||||
-- Lab4: метаданные вложений (PostgreSQL) отдельно от байтов (приватный S3-bucket).
|
||||
-- Expand-only: только новые таблицы, код lab3 продолжает работать на этой схеме.
|
||||
--
|
||||
-- Жизненный цикл строки attachments:
|
||||
-- pending — намерение записано ДО S3 PUT (известны размер, SHA-256, object_key);
|
||||
-- ready — объект в S3 подтверждён, строка видна API и может быть прикреплена.
|
||||
-- Если процесс упал/БД отказала между PUT и переводом в ready, строка остаётся pending.
|
||||
-- Её находит janitor (src/janitor.py) по created_at старше grace period и удаляет
|
||||
-- сначала объект в S3, затем строку. Поэтому в S3 нет объектов, о которых не знает БД.
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width integer NOT NULL CHECK (width > 0),
|
||||
height integer NOT NULL CHECK (height > 0),
|
||||
-- Ключ назначает сервер (случайный), filename в нём не участвует.
|
||||
object_key text NOT NULL UNIQUE,
|
||||
status text NOT NULL CHECK (status IN ('pending', 'ready')),
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
|
||||
-- Janitor ищет только «зависшие» pending: частичный индекс маленький.
|
||||
CREATE INDEX attachments_pending_idx ON attachments (created_at) WHERE status = 'pending';
|
||||
|
||||
-- Ссылки сообщения на вложения (порядок как в запросе).
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages(id),
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
position integer NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
|
||||
-- Отпечаток тела в ключе идемпотентности уже включает attachment_ids (= [] для lab3),
|
||||
-- поэтому существующие ключи остаются валидными без пересчёта.
|
||||
@@ -0,0 +1,126 @@
|
||||
-- Lab5: асинхронная обработка вложений (очередь RabbitMQ + worker).
|
||||
--
|
||||
-- Жизненный цикл attachments.status:
|
||||
-- pending — внутреннее: строка записана ДО PUT в staging, API её не показывает (как в lab4);
|
||||
-- queued — оригинал в staging + задание + запись outbox зафиксированы ОДНОЙ транзакцией, ответ 202;
|
||||
-- processing — worker взял аренду (lease) задания;
|
||||
-- ready — оригинал и все варианты в конечном prefix, метаданные вариантов записаны;
|
||||
-- failed — постоянная ошибка или исчерпаны попытки (error_code).
|
||||
-- Источник истины по состоянию задания — таблица jobs; сообщение брокера — только «пинок»
|
||||
-- с job_id: worker все решения принимает по строке jobs (FOR UPDATE).
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('pending', 'queued', 'processing', 'ready', 'failed'));
|
||||
-- Ключ staging-объекта (staging/<uuid>); NULL после безопасной очистки.
|
||||
ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE;
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
-- Версия pipeline, результат которой сейчас опубликован/ожидается (основной курс: 1).
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1;
|
||||
|
||||
-- Одна логическая работа = (attachment_id, pipeline_version). job_id не меняется при
|
||||
-- redelivery и retry; attempt — номер управляемой попытки (растёт при каждом захвате).
|
||||
CREATE TABLE jobs (
|
||||
id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0),
|
||||
max_attempts integer NOT NULL CHECK (max_attempts >= 1),
|
||||
next_attempt_at timestamptz NOT NULL,
|
||||
-- Аренда: кто обрабатывает (host:boot), токен попытки (fencing) и срок.
|
||||
lease_owner text,
|
||||
lease_token uuid,
|
||||
lease_until timestamptz,
|
||||
error_code text,
|
||||
last_error text,
|
||||
replay_count integer NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL,
|
||||
finished_at timestamptz,
|
||||
UNIQUE (attachment_id, pipeline_version),
|
||||
CHECK ((status = 'processing') = (lease_token IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX jobs_lease_idx ON jobs (lease_until) WHERE status = 'processing';
|
||||
CREATE INDEX jobs_queued_idx ON jobs (updated_at) WHERE status = 'queued';
|
||||
CREATE INDEX jobs_failed_idx ON jobs (updated_at) WHERE status = 'failed';
|
||||
|
||||
-- Результаты: ключ эффекта (attachment_id, pipeline_version, variant) — PRIMARY KEY.
|
||||
-- Повтор job не может создать второй набор вариантов.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments(id),
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
|
||||
width integer NOT NULL CHECK (width > 0),
|
||||
height integer NOT NULL CHECK (height > 0),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name)
|
||||
);
|
||||
|
||||
-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и
|
||||
-- задание. Relay публикует с publisher confirm и только потом ставит published_at.
|
||||
-- available_at позволяет отложить публикацию (retry с backoff).
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs(id),
|
||||
payload jsonb NOT NULL,
|
||||
available_at timestamptz NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
publish_count integer NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX outbox_unpublished_idx ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
CREATE INDEX outbox_job_idx ON outbox (job_id);
|
||||
|
||||
-- Журнал переходов (audit): кто, когда, какая попытка, причина. Без секретов и bytes.
|
||||
CREATE TABLE job_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES jobs(id),
|
||||
at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
event text NOT NULL,
|
||||
attempt integer,
|
||||
actor text,
|
||||
detail jsonb
|
||||
);
|
||||
CREATE INDEX job_events_job_idx ON job_events (job_id, id);
|
||||
|
||||
-- Relay ждёт NOTIFY, а не опрашивает таблицу каждые N мс. Уведомление доставляется
|
||||
-- только после COMMIT; при потере уведомления relay всё равно делает опрос раз в секунду.
|
||||
CREATE FUNCTION outbox_notify() RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
PERFORM pg_notify('outbox', '');
|
||||
RETURN NULL;
|
||||
END $$;
|
||||
CREATE TRIGGER outbox_notify_trg AFTER INSERT ON outbox
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION outbox_notify();
|
||||
|
||||
-- Переход lab4 -> lab5: готовые вложения lab4 не имеют вариантов. Их оригинал
|
||||
-- (originals/<uuid>) становится staging-объектом, вложение — queued, создаётся задание.
|
||||
-- Готовность заявляется только когда все варианты уровня существуют.
|
||||
WITH legacy AS (
|
||||
UPDATE attachments
|
||||
SET staging_key = object_key,
|
||||
object_key = 'attachments/' || id || '/v1/original',
|
||||
status = 'queued',
|
||||
updated_at = now()
|
||||
WHERE status = 'ready'
|
||||
RETURNING id
|
||||
), new_jobs AS (
|
||||
INSERT INTO jobs (id, attachment_id, pipeline_version, status, attempt, max_attempts,
|
||||
next_attempt_at, created_at, updated_at)
|
||||
SELECT gen_random_uuid(), id, 1, 'queued', 0, 5, now(), now(), now() FROM legacy
|
||||
RETURNING id, attachment_id, created_at
|
||||
), events AS (
|
||||
INSERT INTO job_events (job_id, event, attempt, actor, detail)
|
||||
SELECT id, 'enqueued', 1, 'migration', '{"reason":"lab4 backfill"}'::jsonb FROM new_jobs
|
||||
)
|
||||
INSERT INTO outbox (job_id, payload, available_at)
|
||||
SELECT id,
|
||||
jsonb_build_object('schema_version', 1, 'job_id', id, 'type', 'attachment.process.v1',
|
||||
'attachment_id', attachment_id, 'pipeline_version', 1, 'attempt', 1,
|
||||
'created_at', to_char(created_at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"')),
|
||||
now()
|
||||
FROM new_jobs;
|
||||
@@ -0,0 +1,11 @@
|
||||
# Закреплённые версии.
|
||||
# Драйвер PostgreSQL для Python (из lab3).
|
||||
psycopg[binary]==3.2.3
|
||||
psycopg-pool==3.2.4
|
||||
# Lab4: S3-клиент (SigV4, multipart) и проверка JPEG (draft-декодирование).
|
||||
minio==7.2.15
|
||||
urllib3==2.3.0
|
||||
# Lab5: Pillow также делает варианты (JPEG, WebP — libwebp входит в wheel).
|
||||
Pillow==11.1.0
|
||||
# Lab5: клиент AMQP 0-9-1 для RabbitMQ (publisher confirms, manual ack, basic.qos).
|
||||
pika==1.3.2
|
||||
Executable
+50
@@ -0,0 +1,50 @@
|
||||
#!/bin/sh
|
||||
# Уровень 4: ПОСЛЕДОВАТЕЛЬНО останавливает КАЖДУЮ API-реплику под нагрузкой,
|
||||
# восстанавливая предыдущую перед следующей. Режимы:
|
||||
# stop — docker stop (SIGTERM -> graceful drain), штатная остановка/деплой;
|
||||
# kill — docker kill (SIGKILL), внезапное падение процесса без drain.
|
||||
# Результаты: evidence/failover-<mode>-<container>.json (+ .action.log).
|
||||
# Использование: scripts/failover-each.sh [stop|kill] ... (по умолчанию: stop kill)
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
MODES=${*:-stop kill}
|
||||
CONCURRENCY=${CONCURRENCY:-8}
|
||||
DURATION=${DURATION:-40}
|
||||
mkdir -p evidence
|
||||
|
||||
wait_all_up() {
|
||||
# API-контейнеры healthy и HAProxy видит их все UP (rise 2 * inter 1s).
|
||||
./scripts/start-all.sh >/dev/null
|
||||
n=$(docker compose ps -q api | wc -l | tr -d ' ')
|
||||
for _ in $(seq 1 60); do
|
||||
up=$(curl -s 'http://127.0.0.1:8404/stats;csv' | awk -F, '$1=="be_api" && $18=="UP"' | wc -l | tr -d ' ')
|
||||
[ "$up" -ge "$n" ] && return 0
|
||||
sleep 1
|
||||
done
|
||||
echo "HAProxy did not mark all $n api servers UP" >&2; return 1
|
||||
}
|
||||
|
||||
wait_all_up
|
||||
# Список фиксируется один раз: остановленная реплика пропала бы из `docker compose ps`.
|
||||
NAMES=$(docker compose ps --format '{{.Name}}' api | sort)
|
||||
for mode in $MODES; do
|
||||
for name in $NAMES; do
|
||||
wait_all_up
|
||||
cid=$(docker inspect -f '{{.Id}}' "$name"); short=$(echo "$cid" | cut -c1-12)
|
||||
case $mode in
|
||||
stop) action="docker stop -t 20 $cid" ;;
|
||||
kill) action="docker kill $cid" ;;
|
||||
*) echo "unknown mode $mode" >&2; exit 2 ;;
|
||||
esac
|
||||
echo "== $mode $name ($short) $(date -u +%FT%TZ)"
|
||||
out=evidence/failover-$mode-$name.json
|
||||
python3 student-tests/failover_bench.py --label "$mode $name" \
|
||||
--concurrency "$CONCURRENCY" --duration "$DURATION" --action-at 10 \
|
||||
--action "$action" --target-instance "$short" \
|
||||
--action-log "${out%.json}.action.log" --output "$out" \
|
||||
| python3 -c 'import json,sys; t=sys.stdin.read(); s=json.loads(t[:t.rindex("}")+1]); print(json.dumps({k:s[k] for k in ("baseline_ok_rps","post_action_ok_rps","capacity_ratio","errors_total","errors_by_kind","error_window_s","recovery_s","target_last_seen_s","lb_events","lost_confirmed","duplicate_texts","pass")},ensure_ascii=False)); sys.exit(0 if s["pass"] else 1)'
|
||||
echo "-- state after action:"; docker compose ps -a --format '{{.Name}} {{.Status}}' api
|
||||
done
|
||||
done
|
||||
wait_all_up
|
||||
echo "== restored: $(docker compose ps --format '{{.Name}} {{.Status}}' api | tr '\n' ';')"
|
||||
Executable
+112
@@ -0,0 +1,112 @@
|
||||
#!/bin/sh
|
||||
# Создаёт локальные секреты и .env для учебного стенда. Повторный запуск ничего не перезаписывает.
|
||||
# Значения секретов не печатаются.
|
||||
#
|
||||
# S3 (SeaweedFS) — РАЗНЫЕ ключи с разными правами (secrets/s3_identities.json):
|
||||
# admin — Admin/Read/Write/List: только job s3-init (создание bucket);
|
||||
# api — Read+Write на bucket: staging PUT и выдача байтов, без List и без управления bucket;
|
||||
# worker — Read+Write на bucket: чтение staging, запись результатов, удаление staging;
|
||||
# janitor — List+Write на bucket: обход и удаление, без чтения содержимого.
|
||||
# RabbitMQ — пользователи с минимальными правами в vhost messenger (secrets/rabbitmq_definitions.json):
|
||||
# relay — только publish в exchange attachments;
|
||||
# worker — только consume из attachments.process.v1;
|
||||
# ops — consume из DLQ и publish в attachments (replay/учебные проверки).
|
||||
# Ни у кого из них нет права configure: топология создаётся из definitions при старте брокера.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
chmod 700 secrets
|
||||
gen() { # gen <file> <bytes>
|
||||
if [ ! -s "secrets/$1" ]; then
|
||||
python3 -c "import secrets,sys; print(secrets.token_urlsafe(int(sys.argv[1])))" "$2" > "secrets/$1"
|
||||
echo "created secrets/$1"
|
||||
fi
|
||||
}
|
||||
gen postgres_password 24
|
||||
for who in admin api worker janitor; do
|
||||
if [ ! -s "secrets/s3_${who}_access_key" ]; then
|
||||
python3 -c "import secrets; print('MTUSI' + secrets.token_hex(8).upper())" > "secrets/s3_${who}_access_key"
|
||||
echo "created secrets/s3_${who}_access_key"
|
||||
fi
|
||||
gen "s3_${who}_secret_key" 30
|
||||
done
|
||||
for who in relay worker ops; do
|
||||
gen "broker_${who}_password" 24
|
||||
done
|
||||
BUCKET=$(grep -E '^S3_BUCKET=' .env 2>/dev/null | cut -d= -f2 || true)
|
||||
BUCKET=${BUCKET:-messenger-attachments}
|
||||
# Производные файлы пересобираются из базовых секретов (детерминированно, кроме соли хеша).
|
||||
python3 - "$BUCKET" <<'PY'
|
||||
import base64, hashlib, json, os, sys
|
||||
b = sys.argv[1]
|
||||
def read(name):
|
||||
return open("secrets/" + name).read().strip()
|
||||
def ident(name, actions):
|
||||
return {"name": name, "credentials": [{"accessKey": read("s3_%s_access_key" % name),
|
||||
"secretKey": read("s3_%s_secret_key" % name)}], "actions": actions}
|
||||
cfg = {"identities": [
|
||||
ident("admin", ["Admin", "Read", "List", "Tagging", "Write"]),
|
||||
ident("api", ["Read:" + b, "Write:" + b]),
|
||||
ident("worker", ["Read:" + b, "Write:" + b]),
|
||||
ident("janitor", ["List:" + b, "Write:" + b]),
|
||||
]}
|
||||
with open("secrets/s3_identities.json", "w") as f:
|
||||
json.dump(cfg, f, indent=1)
|
||||
|
||||
# RabbitMQ: rabbit_password_hashing_sha256 = base64(salt4 + sha256(salt4 + password)).
|
||||
def rmq_hash(password):
|
||||
salt = os.urandom(4)
|
||||
return base64.b64encode(salt + hashlib.sha256(salt + password.encode()).digest()).decode()
|
||||
vhost = "messenger"
|
||||
users, perms = [], []
|
||||
rights = {
|
||||
"relay": {"configure": "^$", "write": "^attachments$", "read": "^$"},
|
||||
"worker": {"configure": "^$", "write": "^$", "read": r"^attachments\.process\.v1$"},
|
||||
"ops": {"configure": "^$", "write": "^attachments$", "read": r"^attachments\.dlq$"},
|
||||
}
|
||||
for user, r in rights.items():
|
||||
password = read("broker_%s_password" % user)
|
||||
users.append({"name": user, "password_hash": rmq_hash(password),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []})
|
||||
perms.append(dict(user=user, vhost=vhost, **r))
|
||||
with open("secrets/broker_%s_url" % user, "w") as f:
|
||||
f.write("amqp://%s:%s@broker:5672/%s\n" % (user, password, vhost))
|
||||
x = lambda name, kind: {"name": name, "vhost": vhost, "type": kind, "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}}
|
||||
definitions = {
|
||||
"users": users,
|
||||
"vhosts": [{"name": vhost}],
|
||||
"permissions": perms,
|
||||
"exchanges": [x("attachments", "direct"), x("attachments.dlx", "fanout")],
|
||||
"queues": [
|
||||
# Quorum-очередь: durable, сообщения пишутся на диск (Raft-журнал) до publisher confirm.
|
||||
# delivery-limit: сообщение, которое 10 раз вернулось без ack (crash-loop worker),
|
||||
# уходит в DLQ. at-least-once dead-lettering требует overflow=reject-publish.
|
||||
{"name": "attachments.process.v1", "vhost": vhost, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "quorum", "x-delivery-limit": 10,
|
||||
"x-dead-letter-exchange": "attachments.dlx",
|
||||
"x-dead-letter-strategy": "at-least-once", "x-overflow": "reject-publish",
|
||||
"x-max-length": 100000}},
|
||||
{"name": "attachments.dlq", "vhost": vhost, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "quorum"}},
|
||||
],
|
||||
"bindings": [
|
||||
{"source": "attachments", "vhost": vhost, "destination": "attachments.process.v1",
|
||||
"destination_type": "queue", "routing_key": "attachment.process.v1", "arguments": {}},
|
||||
{"source": "attachments.dlx", "vhost": vhost, "destination": "attachments.dlq",
|
||||
"destination_type": "queue", "routing_key": "", "arguments": {}},
|
||||
],
|
||||
}
|
||||
if not os.path.exists("secrets/rabbitmq_definitions.json"):
|
||||
with open("secrets/rabbitmq_definitions.json", "w") as f:
|
||||
json.dump(definitions, f, indent=1)
|
||||
print("created secrets/rabbitmq_definitions.json")
|
||||
PY
|
||||
# Файлы монтируются в контейнеры (uid 70 postgres, uid 10001 app, uid 100 rabbitmq, root в seaweedfs) —
|
||||
# нужны на чтение; от других пользователей хоста их закрывает каталог secrets/ с правами 700.
|
||||
chmod 644 secrets/*
|
||||
if [ ! -f .env ]; then
|
||||
cp .env.example .env
|
||||
echo "created .env from .env.example"
|
||||
fi
|
||||
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Перезапуск ВСЕХ хранилищ (PostgreSQL, SeaweedFS S3, RabbitMQ) и всех потребителей
|
||||
# (API-реплики, relay, worker, janitor, LB) с сохранением volumes pgdata, s3data, brokerdata.
|
||||
# Не выполняет down -v, не пересоздаёт БД/bucket/очереди и не повторяет seed.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart db s3 broker
|
||||
# Потребители стартуют, когда хранилища действительно готовы (healthcheck s3 проверяет запись,
|
||||
# healthcheck broker — запущенный узел и объявленную очередь).
|
||||
docker compose up -d --no-deps --no-recreate --wait db s3 broker
|
||||
docker compose restart api relay worker janitor lb
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/sh
|
||||
# Поочерёдная замена API-реплик на новую версию образа без остановки сервиса.
|
||||
# scripts/rolling-update.sh <APP_VERSION>
|
||||
# Порядок (expand -> replace):
|
||||
# 1. собрать новый образ (тот же тег mtusi-messenger:lab5, build-arg APP_VERSION);
|
||||
# 2. ОДИН раз применить миграции (job migrate). Допускаются только обратно совместимые
|
||||
# (expand-only) миграции: старые реплики продолжают работать на новой схеме;
|
||||
# 3. для каждой старой реплики: поднять +1 новую -> дождаться healthy и UP в HAProxy ->
|
||||
# штатно остановить (SIGTERM, drain) и удалить старую. Число обслуживающих реплик
|
||||
# не опускается ниже исходного.
|
||||
# 4. записать APP_VERSION в .env, чтобы следующий `docker compose up` не пересоздал всё разом.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
VERSION=${1:?usage: rolling-update.sh <APP_VERSION>}
|
||||
export APP_VERSION="$VERSION"
|
||||
ts() { date -u +%FT%TZ; }
|
||||
|
||||
lb_up_count() {
|
||||
curl -s 'http://127.0.0.1:8404/stats;csv' | awk -F, '$1=="be_api" && $18=="UP"' | wc -l | tr -d ' '
|
||||
}
|
||||
# UP ли в HAProxy слот с адресом конкретного контейнера. Проверяем именно адрес, а не число
|
||||
# UP-серверов: новый контейнер может получить IP только что удалённого, и счётчик обманет.
|
||||
lb_has_up() {
|
||||
curl -s 'http://127.0.0.1:8404/stats;csv' | awk -F, -v a="$1:8080" '$1=="be_api" && $18=="UP" && $74==a' | grep -q .
|
||||
}
|
||||
|
||||
echo "$(ts) build version $VERSION"
|
||||
docker compose --env-file .env build api >/dev/null
|
||||
echo "$(ts) migrate (once)"
|
||||
docker compose --env-file .env run --rm --no-deps migrate
|
||||
|
||||
OLD=$(docker compose ps -q api)
|
||||
N=$(echo "$OLD" | wc -l | tr -d ' ')
|
||||
echo "$(ts) replacing $N replicas"
|
||||
for old in $OLD; do
|
||||
before=$(docker compose ps -q api | sort)
|
||||
docker compose --env-file .env up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1
|
||||
new=$(docker compose ps -q api | sort | grep -v -F "$before" | head -1)
|
||||
[ -n "$new" ] || { echo "new replica was not created" >&2; exit 1; }
|
||||
for _ in $(seq 1 60); do
|
||||
[ "$(docker inspect -f '{{.State.Health.Status}}' "$new")" = healthy ] && break; sleep 1
|
||||
done
|
||||
[ "$(docker inspect -f '{{.State.Health.Status}}' "$new")" = healthy ] || { echo "new replica not healthy" >&2; exit 1; }
|
||||
ip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$new")
|
||||
for _ in $(seq 1 30); do lb_has_up "$ip" && break; sleep 0.5; done
|
||||
lb_has_up "$ip" || { echo "HAProxy did not mark new replica $ip UP" >&2; exit 1; }
|
||||
echo "$(ts) up $(docker inspect -f '{{.Name}}' "$new" | tr -d /) version=$(docker exec "$new" printenv APP_VERSION) ip=$ip UP in LB (UP slots: $(lb_up_count))"
|
||||
name=$(docker inspect -f '{{.Name}}' "$old" | tr -d /)
|
||||
docker stop -t 20 "$old" >/dev/null && docker rm "$old" >/dev/null
|
||||
echo "$(ts) gone $name (graceful SIGTERM)"
|
||||
done
|
||||
if grep -q '^APP_VERSION=' .env; then
|
||||
sed -i.bak "s/^APP_VERSION=.*/APP_VERSION=$VERSION/" .env && rm -f .env.bak
|
||||
else
|
||||
echo "APP_VERSION=$VERSION" >> .env
|
||||
fi
|
||||
echo "$(ts) done: $(docker compose ps --format '{{.Name}}' api | tr '\n' ' ')"
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Восстанавливает остановленные API-реплики и worker (после stop-one.sh / stop-worker.sh) и ждёт готовности.
|
||||
# --no-deps: хранилища и job-ы не трогаются; --no-recreate: те же контейнеры (тот же образ).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-deps --no-recreate --wait api worker
|
||||
docker compose ps --format '{{.Name}} {{.Status}}' api worker
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Запустить worker после stop-worker.sh (те же контейнеры) и дождаться готовности.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose up -d --no-deps --no-recreate --wait worker
|
||||
docker compose ps --format '{{.Name}} {{.Status}}' worker
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
# Останавливает ОДНУ API-реплику, обслужившую тест, и оставляет её остановленной.
|
||||
# TARGET_INSTANCE_ID = X-Instance-Id = hostname контейнера = короткий ID контейнера.
|
||||
# DB, LB и volumes не трогаются. Восстановление: scripts/start-all.sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]*|'') echo "unexpected instance id format" >&2; exit 2 ;;
|
||||
esac
|
||||
# Проверяем, что это действительно запущенный контейнер сервиса api ЭТОГО compose-проекта.
|
||||
# Список берём у самого compose: работает и при другом имени проекта (-p / COMPOSE_PROJECT_NAME).
|
||||
cid=$(docker compose ps -q --status running api | grep "^${TARGET_INSTANCE_ID}" || true)
|
||||
[ -n "$cid" ] || { echo "no running api container for $TARGET_INSTANCE_ID" >&2; exit 3; }
|
||||
name=$(docker inspect -f '{{.Name}}' "$cid")
|
||||
echo "$(date -u +%FT%TZ) stopping api container ${name#/} (${TARGET_INSTANCE_ID})" >&2
|
||||
# SIGTERM -> graceful shutdown приложения (drain 3 с + дообслуживание), затем выход.
|
||||
docker stop -t 20 "$cid" >/dev/null
|
||||
echo "$(date -u +%FT%TZ) stopped ${name#/}" >&2
|
||||
Executable
+7
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Сценарий защиты: остановить ВСЕ worker (SIGTERM, текущие задания дорабатываются).
|
||||
# API, relay и брокер продолжают работать: upload -> 202, задания копятся в очереди.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop worker
|
||||
docker compose ps --format '{{.Name}} {{.State}}' worker
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Общее для фоновых процессов lab5 (relay, worker, ops): конфигурация, БД, брокер.
|
||||
|
||||
API (server.py) брокер не использует вовсе: он только пишет outbox в PostgreSQL.
|
||||
Поэтому недоступность RabbitMQ не влияет на приём файлов — задания копятся в outbox.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
from psycopg.conninfo import make_conninfo
|
||||
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
logging.getLogger("pika").setLevel(logging.WARNING)
|
||||
|
||||
# Топология брокера (объявляется декларативно в secrets/rabbitmq_definitions.json,
|
||||
# приложения её не создают — у их пользователей нет права configure).
|
||||
EXCHANGE = "attachments"
|
||||
ROUTING_KEY = "attachment.process.v1"
|
||||
QUEUE = "attachments.process.v1"
|
||||
DLQ = "attachments.dlq"
|
||||
|
||||
HOSTNAME = socket.gethostname()
|
||||
# Идентификатор процесса для lease_owner: hostname контейнера + случайный boot id.
|
||||
BOOT_ID = uuid.uuid4().hex[:8]
|
||||
OWNER = "%s:%s" % (HOSTNAME, BOOT_ID)
|
||||
|
||||
|
||||
def env_int(name, default):
|
||||
return int(os.environ.get(name, str(default)))
|
||||
|
||||
|
||||
def env_float(name, default):
|
||||
return float(os.environ.get(name, str(default)))
|
||||
|
||||
|
||||
def secret(name):
|
||||
"""Значение из NAME_FILE (Docker secret) либо из NAME."""
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read().strip()
|
||||
return os.environ.get(name, "")
|
||||
|
||||
|
||||
def database_conninfo():
|
||||
options = "-c statement_timeout=%d" % env_int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
url = secret("DATABASE_URL")
|
||||
if url:
|
||||
return make_conninfo(url, connect_timeout=3, options=options)
|
||||
return make_conninfo(
|
||||
host=os.environ.get("POSTGRES_HOST", "localhost"),
|
||||
port=os.environ.get("POSTGRES_PORT", "5432"),
|
||||
dbname=os.environ.get("POSTGRES_DB", "messenger"),
|
||||
user=os.environ.get("POSTGRES_USER", "messenger"),
|
||||
password=secret("POSTGRES_PASSWORD"),
|
||||
connect_timeout=3,
|
||||
options=options,
|
||||
)
|
||||
|
||||
|
||||
def broker_parameters():
|
||||
"""amqp URL из BROKER_URL_FILE: у relay, worker и ops — разные пользователи RabbitMQ."""
|
||||
params = pika.URLParameters(secret("BROKER_URL"))
|
||||
# heartbeat: брокер закроет соединение мёртвого/замороженного клиента и вернёт его unacked
|
||||
# сообщения в очередь (в т.ч. выданные заранее по prefetch). 10 с: эксперимент
|
||||
# lease_takeover показал, что при 20 с такое сообщение висело ~35 с. Worker обрабатывает
|
||||
# задание в отдельном потоке и продолжает обслуживать heartbeat (process_data_events),
|
||||
# поэтому долгая обработка соединение не рвёт.
|
||||
params.heartbeat = env_int("BROKER_HEARTBEAT_SECONDS", 10)
|
||||
params.blocked_connection_timeout = 30
|
||||
params.socket_timeout = 5
|
||||
params.connection_attempts = 1
|
||||
params.client_properties = {"connection_name": "%s-%s" % (os.environ.get("SERVICE_NAME", "app"), HOSTNAME)}
|
||||
return params
|
||||
|
||||
|
||||
def connect_broker(log, stop=None):
|
||||
"""Подключение с повторами и паузой до 5 с; stop — threading.Event для выхода."""
|
||||
delay = 0.5
|
||||
while True:
|
||||
try:
|
||||
return pika.BlockingConnection(broker_parameters())
|
||||
except Exception as exc:
|
||||
log.warning("broker unavailable (%s), retry in %.1fs", type(exc).__name__, delay)
|
||||
if stop is not None and stop.wait(delay):
|
||||
return None
|
||||
if stop is None:
|
||||
time.sleep(delay)
|
||||
delay = min(5.0, delay * 2)
|
||||
|
||||
|
||||
def touch(path):
|
||||
with open(path, "w") as f:
|
||||
f.write(str(time.time()))
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
"""Проверка загружаемых изображений по СОДЕРЖИМОМУ, с ограниченной памятью.
|
||||
|
||||
Заголовок Content-Type клиента — только заявление: настоящий формат определяется
|
||||
по сигнатуре и структуре файла. Лимит пикселей проверяется по заголовку (PNG IHDR,
|
||||
JPEG SOF) ДО распаковки, поэтому «бомба» 100 000×100 000 не декодируется вовсе.
|
||||
|
||||
PNG проверяется собственным потоковым валидатором (стандартная библиотека):
|
||||
структура и CRC каждого chunk, допустимые IHDR, IDAT распаковывается zlib кусками
|
||||
по 1 MiB без сохранения результата; объём распакованных данных должен ТОЧНО совпасть
|
||||
с размером, который следует из IHDR (больше — бомба/мусор, меньше — обрезанный файл).
|
||||
|
||||
JPEG: разбор маркеров до SOF (размеры), затем пробное декодирование Pillow в режиме
|
||||
draft (масштаб 1/8): проверяются энтропийные данные, а памяти нужно ~1/64 от полного
|
||||
кадра (40 Мпикс -> ~2 MiB вместо ~120 MiB).
|
||||
"""
|
||||
import struct
|
||||
import zlib
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_SOI = b"\xff\xd8\xff"
|
||||
READ_CHUNK = 64 * 1024
|
||||
INFLATE_STEP = 1024 * 1024
|
||||
|
||||
# Допустимые глубины цвета по типу цвета PNG и число каналов.
|
||||
_PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}}
|
||||
_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
# Adam7: (x0, y0, dx, dy)
|
||||
_ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
_JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF}
|
||||
|
||||
|
||||
class InvalidImage(Exception):
|
||||
"""Файл не является корректным PNG/JPEG заявленного типа -> 415."""
|
||||
|
||||
|
||||
class TooManyPixels(Exception):
|
||||
"""Декодированный размер превышает лимит -> 413."""
|
||||
|
||||
|
||||
def sniff(head):
|
||||
"""Тип по сигнатуре первых байтов: 'image/png', 'image/jpeg' или None."""
|
||||
if head.startswith(PNG_SIGNATURE):
|
||||
return "image/png"
|
||||
if head.startswith(JPEG_SOI):
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def png_header_pixels(head):
|
||||
"""Ранняя проверка по первым 33 байтам (сигнатура + IHDR): (w, h) или None."""
|
||||
if len(head) < 33 or not head.startswith(PNG_SIGNATURE) or head[12:16] != b"IHDR":
|
||||
return None
|
||||
return struct.unpack(">II", head[16:24])
|
||||
|
||||
|
||||
def _read_exact(f, n):
|
||||
data = f.read(n)
|
||||
if len(data) != n:
|
||||
raise InvalidImage("truncated")
|
||||
return data
|
||||
|
||||
|
||||
def _png_expected_bytes(w, h, bits_per_pixel, interlace):
|
||||
def raw(pw, ph):
|
||||
return ph * (1 + (pw * bits_per_pixel + 7) // 8) if pw and ph else 0
|
||||
if interlace == 0:
|
||||
return raw(w, h)
|
||||
total = 0
|
||||
for x0, y0, dx, dy in _ADAM7:
|
||||
pw = (w - x0 + dx - 1) // dx if w > x0 else 0
|
||||
ph = (h - y0 + dy - 1) // dy if h > y0 else 0
|
||||
total += raw(pw, ph)
|
||||
return total
|
||||
|
||||
|
||||
def validate_png(f, size, max_pixels):
|
||||
if _read_exact(f, 8) != PNG_SIGNATURE:
|
||||
raise InvalidImage("signature")
|
||||
pos = 8
|
||||
width = height = None
|
||||
expected = produced = 0
|
||||
inflater = None
|
||||
seen_plte = idat_done = False
|
||||
color_type = None
|
||||
last_type = None
|
||||
while True:
|
||||
if pos + 12 > size:
|
||||
raise InvalidImage("truncated")
|
||||
length, ctype = struct.unpack(">I4s", _read_exact(f, 8))
|
||||
pos += 8
|
||||
if length > 0x7FFFFFFF or pos + length + 4 > size:
|
||||
raise InvalidImage("chunk length")
|
||||
if not ctype.isalpha():
|
||||
raise InvalidImage("chunk type")
|
||||
if width is None and ctype != b"IHDR":
|
||||
raise InvalidImage("IHDR must be first")
|
||||
crc = zlib.crc32(ctype)
|
||||
if ctype == b"IDAT":
|
||||
if idat_done or (color_type == 3 and not seen_plte):
|
||||
raise InvalidImage("IDAT order")
|
||||
remaining = length
|
||||
while remaining:
|
||||
piece = _read_exact(f, min(READ_CHUNK, remaining))
|
||||
remaining -= len(piece)
|
||||
crc = zlib.crc32(piece, crc)
|
||||
buf = piece
|
||||
while buf:
|
||||
if inflater.eof:
|
||||
raise InvalidImage("data after zlib stream")
|
||||
try:
|
||||
out = inflater.decompress(buf, INFLATE_STEP)
|
||||
except zlib.error:
|
||||
raise InvalidImage("zlib")
|
||||
produced += len(out)
|
||||
if produced > expected:
|
||||
raise InvalidImage("too much pixel data") # бомба/мусор
|
||||
buf = inflater.unconsumed_tail
|
||||
else:
|
||||
if last_type == b"IDAT":
|
||||
idat_done = True
|
||||
data = _read_exact(f, length)
|
||||
crc = zlib.crc32(data, crc)
|
||||
if ctype == b"IHDR":
|
||||
if width is not None or length != 13:
|
||||
raise InvalidImage("IHDR")
|
||||
width, height, depth, color_type, comp, filt, interlace = struct.unpack(">IIBBBBB", data)
|
||||
if not (1 <= width <= 0x7FFFFFFF and 1 <= height <= 0x7FFFFFFF):
|
||||
raise InvalidImage("dimensions")
|
||||
if width * height > max_pixels:
|
||||
raise TooManyPixels()
|
||||
if depth not in _PNG_DEPTHS.get(color_type, ()) or comp != 0 or filt != 0 or interlace not in (0, 1):
|
||||
raise InvalidImage("IHDR values")
|
||||
expected = _png_expected_bytes(width, height, _PNG_CHANNELS[color_type] * depth, interlace)
|
||||
inflater = zlib.decompressobj()
|
||||
elif ctype == b"PLTE":
|
||||
seen_plte = True
|
||||
elif ctype == b"IEND":
|
||||
pass
|
||||
elif ctype[0:1].isupper():
|
||||
raise InvalidImage("unknown critical chunk") # декодер обязан его понимать
|
||||
if struct.unpack(">I", _read_exact(f, 4))[0] != crc & 0xFFFFFFFF:
|
||||
raise InvalidImage("crc")
|
||||
pos += length + 4
|
||||
last_type = ctype
|
||||
if ctype == b"IEND":
|
||||
break
|
||||
if inflater is None or not inflater.eof or produced != expected:
|
||||
raise InvalidImage("incomplete pixel data")
|
||||
if pos != size:
|
||||
raise InvalidImage("trailing data") # полиглоты PNG+что-то ещё не принимаем
|
||||
return width, height
|
||||
|
||||
|
||||
def jpeg_dimensions(f, size):
|
||||
"""Размеры из первого SOF без декодирования. Ограничено размером файла."""
|
||||
if _read_exact(f, 2) != b"\xff\xd8":
|
||||
raise InvalidImage("signature")
|
||||
pos = 2
|
||||
while pos < size:
|
||||
b = _read_exact(f, 1)
|
||||
pos += 1
|
||||
if b != b"\xff":
|
||||
raise InvalidImage("marker")
|
||||
marker = 0xFF
|
||||
while marker == 0xFF:
|
||||
marker = _read_exact(f, 1)[0]
|
||||
pos += 1
|
||||
if marker == 0x01 or 0xD0 <= marker <= 0xD7:
|
||||
continue
|
||||
if marker in (0xD8, 0xD9, 0xDA):
|
||||
raise InvalidImage("no SOF before scan")
|
||||
seg_len = struct.unpack(">H", _read_exact(f, 2))[0]
|
||||
if seg_len < 2 or pos + seg_len > size:
|
||||
raise InvalidImage("segment length")
|
||||
if marker in _JPEG_SOF:
|
||||
if seg_len < 8:
|
||||
raise InvalidImage("SOF")
|
||||
_precision, height, width = struct.unpack(">BHH", _read_exact(f, 5))
|
||||
if width == 0 or height == 0:
|
||||
raise InvalidImage("dimensions") # DNL-высота не поддерживается
|
||||
return width, height
|
||||
f.seek(seg_len - 2, 1)
|
||||
pos += seg_len
|
||||
raise InvalidImage("no SOF")
|
||||
|
||||
|
||||
def validate_jpeg(f, size, max_pixels):
|
||||
width, height = jpeg_dimensions(f, size)
|
||||
if width * height > max_pixels:
|
||||
raise TooManyPixels()
|
||||
from PIL import Image # импорт по требованию: Pillow нужен только для JPEG
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
f.seek(0)
|
||||
try:
|
||||
with Image.open(f, formats=("JPEG",)) as im:
|
||||
if im.size != (width, height):
|
||||
raise InvalidImage("size mismatch")
|
||||
im.draft("RGB", (max(1, width // 8), max(1, height // 8)))
|
||||
im.load() # обрезанные/битые энтропийные данные -> OSError
|
||||
except InvalidImage:
|
||||
raise
|
||||
except Exception:
|
||||
# Pillow бросает разные исключения (OSError, SyntaxError, ValueError,
|
||||
# DecompressionBombError...) — для клиента это одинаково «не JPEG».
|
||||
raise InvalidImage("decode")
|
||||
return width, height
|
||||
|
||||
|
||||
def validate(f, size, content_type, max_pixels):
|
||||
"""(width, height) или InvalidImage/TooManyPixels. f — seekable файл, позиция 0."""
|
||||
f.seek(0)
|
||||
if content_type == "image/png":
|
||||
return validate_png(f, size, max_pixels)
|
||||
if content_type == "image/jpeg":
|
||||
return validate_jpeg(f, size, max_pixels)
|
||||
raise InvalidImage("type")
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сверка PostgreSQL <-> S3 и очистка (compose-сервис `janitor`). Lab5.
|
||||
|
||||
Prefix-ы bucket:
|
||||
staging/<uuid> принятые оригиналы до публикации результата (attachments.staging_key);
|
||||
attachments/<id>/v<ver>/<name> конечные объекты: original и варианты;
|
||||
originals/<uuid> оригиналы lab4 (после миграции 004 — staging-объекты backfill-заданий);
|
||||
health/, bootstrap/ служебные, не трогаются.
|
||||
|
||||
За один проход (под pg_try_advisory_lock):
|
||||
1. abandoned: pending-строки старше grace (API упал до постановки в очередь, 202 не было)
|
||||
-> удалить staging-объект, затем строку;
|
||||
2. staging ready: у ready-вложения остался staging (worker не смог удалить после COMMIT)
|
||||
-> удалить объект, staging_key = NULL;
|
||||
3. staging failed: failed дольше STAGING_RETENTION_SECONDS -> удалить staging (replay после
|
||||
этого невозможен: ops replay ответит refused_staging_expired);
|
||||
4. orphan: объекты staging/, originals/, attachments/ старше grace, на которые не ссылается
|
||||
ни одна строка (staging_key, object_key ready-вложения, attachment_variants), и вложение
|
||||
не queued/processing -> удалить;
|
||||
5. missing: ready без объекта оригинала или варианта -> ТОЛЬКО отчёт.
|
||||
Текущую работу (queued/processing) janitor не удаляет никогда: её staging нужен worker.
|
||||
Grace (ATTACHMENT_GRACE_SECONDS, 3600 с) многократно больше UPLOAD_TIMEOUT_SECONDS и
|
||||
JOB_TIMEOUT_SECONDS, поэтому идущая загрузка/обработка не считается брошенной.
|
||||
|
||||
python src/janitor.py --once --dry-run | --once | --interval 300
|
||||
"""
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
import jobs
|
||||
from server import OBJ, STORE
|
||||
|
||||
log = logging.getLogger("messenger.janitor")
|
||||
JANITOR_LOCK_ID = 727002
|
||||
PREFIXES = ("staging/", "originals/", "attachments/")
|
||||
HEARTBEAT = "/tmp/janitor.heartbeat"
|
||||
STAGING_RETENTION = int(os.environ.get("STAGING_RETENTION_SECONDS", "86400"))
|
||||
_FINAL_RE = re.compile(r"^attachments/([0-9a-f-]{36})/")
|
||||
|
||||
|
||||
def heartbeat():
|
||||
with open(HEARTBEAT, "w") as f:
|
||||
f.write(str(time.time()))
|
||||
|
||||
|
||||
def _delete(report, key, dry_run):
|
||||
if dry_run:
|
||||
return True
|
||||
try:
|
||||
OBJ.delete(key)
|
||||
report["deleted_objects"] += 1
|
||||
return True
|
||||
except Exception as exc:
|
||||
report["errors"] += 1
|
||||
log.warning("object not deleted (%s)", type(exc).__name__)
|
||||
return False
|
||||
|
||||
|
||||
def run_once(grace_seconds, dry_run, staging_retention=STAGING_RETENTION):
|
||||
started = time.monotonic()
|
||||
report = {"dry_run": dry_run, "grace_seconds": grace_seconds, "staging_retention_seconds": staging_retention,
|
||||
"abandoned_pending": 0, "staging_ready_cleaned": 0, "staging_failed_expired": 0,
|
||||
"orphan_objects": 0, "missing_objects": 0, "deleted_objects": 0, "deleted_rows": 0, "errors": 0}
|
||||
with STORE.pool.connection() as conn:
|
||||
conn.autocommit = True
|
||||
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK_ID,)).fetchone()["ok"]:
|
||||
report["skipped"] = "another janitor is running"
|
||||
return report
|
||||
try:
|
||||
now_db = conn.execute("SELECT now() AS t").fetchone()["t"]
|
||||
cutoff = now_db - dt.timedelta(seconds=grace_seconds)
|
||||
report["rows_by_status"] = {r["status"]: r["n"] for r in conn.execute(
|
||||
"SELECT status, count(*) AS n FROM attachments GROUP BY status")}
|
||||
report["jobs_by_status"] = {r["status"]: r["n"] for r in conn.execute(
|
||||
"SELECT status, count(*) AS n FROM jobs GROUP BY status")}
|
||||
report["outbox_unpublished"] = conn.execute(
|
||||
"SELECT count(*) AS n FROM outbox WHERE published_at IS NULL").fetchone()["n"]
|
||||
|
||||
# 1. брошенные pending
|
||||
abandoned = conn.execute(
|
||||
"""SELECT id, staging_key FROM attachments
|
||||
WHERE status = 'pending' AND created_at < %s ORDER BY created_at""", (cutoff,)).fetchall()
|
||||
report["abandoned_pending"] = len(abandoned)
|
||||
report["abandoned_ids"] = [str(r["id"]) for r in abandoned[:20]]
|
||||
for row in abandoned:
|
||||
if dry_run:
|
||||
continue
|
||||
if row["staging_key"] and not _delete(report, row["staging_key"], dry_run):
|
||||
continue
|
||||
report["deleted_rows"] += conn.execute(
|
||||
"DELETE FROM attachments WHERE id = %s AND status = 'pending'", (row["id"],)).rowcount
|
||||
|
||||
# 2/3. staging, который больше не нужен
|
||||
stale = conn.execute(
|
||||
"""SELECT a.id, a.staging_key, a.status, j.id AS job_id FROM attachments a
|
||||
LEFT JOIN jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
|
||||
WHERE a.staging_key IS NOT NULL AND (a.status = 'ready'
|
||||
OR (a.status = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))""",
|
||||
(staging_retention,)).fetchall()
|
||||
for row in stale:
|
||||
key = "staging_ready_cleaned" if row["status"] == "ready" else "staging_failed_expired"
|
||||
report[key] += 1
|
||||
if dry_run or not _delete(report, row["staging_key"], dry_run):
|
||||
continue
|
||||
with conn.transaction():
|
||||
n = conn.execute(
|
||||
"""UPDATE attachments SET staging_key = NULL
|
||||
WHERE id = %s AND staging_key = %s AND status = %s""",
|
||||
(row["id"], row["staging_key"], row["status"])).rowcount
|
||||
if n and row["job_id"]:
|
||||
jobs.add_event(conn, row["job_id"], "staging_deleted", None, "janitor", {"status": row["status"]})
|
||||
|
||||
# 4/5. обход bucket. Момент начала — по часам БД: ready раньше этого момента обязан
|
||||
# иметь все объекты в листинге (объекты пишутся ДО COMMIT ready).
|
||||
list_started = conn.execute("SELECT now() AS t").fetchone()["t"]
|
||||
listed = {}
|
||||
for prefix in PREFIXES:
|
||||
for obj in OBJ.client.list_objects(OBJ.bucket, prefix=prefix, recursive=True):
|
||||
listed[obj.object_name] = obj.last_modified
|
||||
report["objects_total"] = len(listed)
|
||||
report["objects_by_prefix"] = {p: sum(1 for k in listed if k.startswith(p)) for p in PREFIXES}
|
||||
known = set()
|
||||
busy = set() # вложения с текущей работой: их объекты не трогаем
|
||||
keys = list(listed)
|
||||
for i in range(0, len(keys), 1000):
|
||||
batch = keys[i:i + 1000]
|
||||
for r in conn.execute(
|
||||
"""SELECT staging_key AS k FROM attachments WHERE staging_key = ANY(%(b)s)
|
||||
UNION SELECT object_key FROM attachments WHERE status = 'ready' AND object_key = ANY(%(b)s)
|
||||
UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%(b)s)""",
|
||||
{"b": batch}):
|
||||
known.add(r["k"])
|
||||
ids = {m.group(1) for k in keys for m in [_FINAL_RE.match(k)] if m}
|
||||
if ids:
|
||||
for r in conn.execute(
|
||||
"SELECT id FROM attachments WHERE id = ANY(%s::uuid[]) AND status IN ('pending','queued','processing')",
|
||||
(list(ids),)):
|
||||
busy.add(str(r["id"]))
|
||||
orphans = []
|
||||
for k, mtime in listed.items():
|
||||
if k in known or mtime is None or mtime >= cutoff:
|
||||
continue
|
||||
m = _FINAL_RE.match(k)
|
||||
if m and m.group(1) in busy:
|
||||
continue
|
||||
orphans.append(k)
|
||||
report["orphan_objects"] = len(orphans)
|
||||
report["orphan_keys_sample"] = sorted(orphans)[:5]
|
||||
for key in orphans:
|
||||
_delete(report, key, dry_run)
|
||||
|
||||
missing = []
|
||||
for r in conn.execute(
|
||||
"""SELECT a.id, a.object_key AS k FROM attachments a WHERE a.status = 'ready' AND a.updated_at < %s
|
||||
UNION ALL
|
||||
SELECT v.attachment_id, v.object_key FROM attachment_variants v
|
||||
JOIN attachments a ON a.id = v.attachment_id
|
||||
WHERE a.status = 'ready' AND a.updated_at < %s""", (list_started, list_started)):
|
||||
if r["k"] not in listed:
|
||||
missing.append(str(r["id"]))
|
||||
report["missing_objects"] = len(missing)
|
||||
report["missing_ids_sample"] = sorted(set(missing))[:5]
|
||||
if missing:
|
||||
log.error("ready attachments without object: %d (manual investigation needed)", len(missing))
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK_ID,))
|
||||
report["duration_ms"] = round((time.monotonic() - started) * 1000, 1)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--once", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
parser.add_argument("--grace-seconds", type=int,
|
||||
default=int(os.environ.get("ATTACHMENT_GRACE_SECONDS", "3600")))
|
||||
parser.add_argument("--staging-retention-seconds", type=int, default=STAGING_RETENTION)
|
||||
parser.add_argument("--interval", type=int, default=int(os.environ.get("JANITOR_INTERVAL_SECONDS", "300")))
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 1:
|
||||
parser.error("grace period must be positive")
|
||||
STORE.open()
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
print(json.dumps(run_once(args.grace_seconds, args.dry_run, args.staging_retention_seconds),
|
||||
ensure_ascii=False), flush=True)
|
||||
except Exception as exc:
|
||||
log.warning("janitor pass failed (%s)", type(exc).__name__)
|
||||
if args.once:
|
||||
return 1
|
||||
if args.once:
|
||||
return 0
|
||||
wake = time.monotonic() + args.interval
|
||||
while time.monotonic() < wake:
|
||||
heartbeat()
|
||||
time.sleep(min(30, max(0.1, wake - time.monotonic())))
|
||||
finally:
|
||||
STORE.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+364
@@ -0,0 +1,364 @@
|
||||
"""Задания обработки вложений: схема сообщения, переходы состояний, retry, lease, replay.
|
||||
|
||||
Используется API (постановка в очередь), worker, relay (reaper) и ops-CLI. Все переходы —
|
||||
SQL-транзакции над строкой jobs под FOR UPDATE, поэтому решение «что делать с сообщением»
|
||||
принимается по устойчивому состоянию в PostgreSQL, а не по факту доставки.
|
||||
|
||||
Сообщение брокера (contracts/jobs.schema.json) — только ссылка: job_id, attachment_id,
|
||||
pipeline_version, attempt. Bytes, credentials, URL и ключи S3 в сообщение не попадают:
|
||||
worker берёт доверенные storage metadata из БД.
|
||||
|
||||
Переходы (contracts/attachment-states.md):
|
||||
enqueue нет -> queued API, одна транзакция с outbox
|
||||
claim queued -> processing worker: attempt+1, lease_owner/token/until
|
||||
heartbeat processing (продление) worker, только со своим lease_token
|
||||
complete processing -> ready worker, только со своим lease_token (fencing)
|
||||
retry processing -> queued временная ошибка, backoff + jitter, outbox с available_at
|
||||
fail processing -> failed постоянная ошибка или исчерпаны попытки
|
||||
lease expired processing -> queued reaper (relay), если аренду никто не продлил
|
||||
replay failed -> queued оператор (ops.py), audit, ready не трогается
|
||||
"""
|
||||
import datetime as dt
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
VARIANT_NAMES = ("thumbnail", "crop", "optimized")
|
||||
# Постоянные ошибки: повтор не поможет (тот же вход даст тот же результат).
|
||||
PERMANENT_ERRORS = {"invalid_image", "image_too_large", "staging_corrupted", "staging_missing",
|
||||
"resource_limit"}
|
||||
_UUID_FIELDS = ("job_id", "attachment_id")
|
||||
_ALLOWED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt",
|
||||
"created_at", "traceparent", "tracestate"}
|
||||
|
||||
|
||||
class InvalidMessage(Exception):
|
||||
"""Сообщение не соответствует jobs.schema.json -> reject в DLQ."""
|
||||
|
||||
|
||||
def utc_now_text():
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def make_payload(job_id, attachment_id, pipeline_version, attempt, created_at=None):
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"job_id": str(job_id),
|
||||
"type": JOB_TYPE,
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": int(pipeline_version),
|
||||
"attempt": int(attempt),
|
||||
"created_at": created_at or utc_now_text(),
|
||||
}
|
||||
|
||||
|
||||
def parse_message(body):
|
||||
"""bytes -> dict по jobs.schema.json (без внешних библиотек). Иначе InvalidMessage."""
|
||||
try:
|
||||
msg = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise InvalidMessage("not json")
|
||||
if not isinstance(msg, dict):
|
||||
raise InvalidMessage("not object")
|
||||
if set(msg) - _ALLOWED:
|
||||
raise InvalidMessage("extra fields")
|
||||
for field in ("schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"):
|
||||
if field not in msg:
|
||||
raise InvalidMessage("missing " + field)
|
||||
if msg["schema_version"] != SCHEMA_VERSION or msg["type"] != JOB_TYPE:
|
||||
raise InvalidMessage("unsupported type/version")
|
||||
for field in _UUID_FIELDS:
|
||||
try:
|
||||
if not isinstance(msg[field], str):
|
||||
raise ValueError
|
||||
uuid.UUID(msg[field])
|
||||
except ValueError:
|
||||
raise InvalidMessage("bad " + field)
|
||||
for field in ("pipeline_version", "attempt"):
|
||||
v = msg[field]
|
||||
if not isinstance(v, int) or isinstance(v, bool) or v < 1:
|
||||
raise InvalidMessage("bad " + field)
|
||||
if not isinstance(msg["created_at"], str):
|
||||
raise InvalidMessage("bad created_at")
|
||||
try:
|
||||
dt.datetime.fromisoformat(msg["created_at"].replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
raise InvalidMessage("bad created_at")
|
||||
return msg
|
||||
|
||||
|
||||
def backoff_seconds(attempt, base, cap, rng=random):
|
||||
"""Экспоненциальный backoff с «equal jitter»: d = min(cap, base*2^(attempt-1)),
|
||||
задержка равномерно в [d/2, d]. Половина гарантирует паузу (не долбим упавший S3),
|
||||
случайная половина разносит повторы многих заданий во времени (нет «стада»)."""
|
||||
d = min(float(cap), float(base) * (2 ** max(0, attempt - 1)))
|
||||
return d / 2 + rng.uniform(0, d / 2)
|
||||
|
||||
|
||||
def add_event(conn, job_id, event, attempt=None, actor=None, detail=None):
|
||||
conn.execute(
|
||||
"INSERT INTO job_events (job_id, event, attempt, actor, detail) VALUES (%s, %s, %s, %s, %s)",
|
||||
(job_id, event, attempt, actor, json.dumps(detail, ensure_ascii=False) if detail is not None else None))
|
||||
|
||||
|
||||
def add_outbox(conn, job_id, attachment_id, pipeline_version, attempt, delay_seconds=0.0):
|
||||
payload = make_payload(job_id, attachment_id, pipeline_version, attempt)
|
||||
conn.execute(
|
||||
"""INSERT INTO outbox (job_id, payload, available_at)
|
||||
VALUES (%s, %s, now() + make_interval(secs => %s))""",
|
||||
(job_id, json.dumps(payload), delay_seconds))
|
||||
return payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# API: постановка в очередь
|
||||
# ---------------------------------------------------------------------------
|
||||
def enqueue(conn, attachment_id, pipeline_version, max_attempts):
|
||||
"""В ТЕКУЩЕЙ транзакции: pending -> queued, jobs, outbox, событие. job_id или None."""
|
||||
row = conn.execute(
|
||||
"""UPDATE attachments SET status = 'queued', updated_at = now()
|
||||
WHERE id = %s AND status = 'pending' RETURNING id""", (attachment_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
job_id = uuid.uuid4()
|
||||
conn.execute(
|
||||
"""INSERT INTO jobs (id, attachment_id, pipeline_version, status, attempt, max_attempts,
|
||||
next_attempt_at, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, 'queued', 0, %s, now(), now(), now())""",
|
||||
(job_id, attachment_id, pipeline_version, max_attempts))
|
||||
add_event(conn, job_id, "enqueued", 1, "api")
|
||||
add_outbox(conn, job_id, attachment_id, pipeline_version, 1)
|
||||
return job_id
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Worker
|
||||
# ---------------------------------------------------------------------------
|
||||
JOB_COLUMNS = """j.id AS job_id, j.attachment_id, j.pipeline_version, j.status AS job_status, j.attempt,
|
||||
j.max_attempts, j.next_attempt_at, j.error_code, j.lease_owner, j.lease_token, j.lease_until, j.replay_count,
|
||||
j.lease_until > now() AS lease_alive, j.next_attempt_at > now() + interval '1 second' AS too_early,
|
||||
a.status AS attachment_status, a.content_type, a.size_bytes, a.sha256, a.width, a.height,
|
||||
a.object_key, a.staging_key, a.filename, a.pipeline_version AS current_pipeline_version"""
|
||||
|
||||
|
||||
def load_job(conn, job_id, lock=False):
|
||||
return conn.execute(
|
||||
"SELECT " + JOB_COLUMNS + " FROM jobs j JOIN attachments a ON a.id = j.attachment_id WHERE j.id = %s"
|
||||
+ (" FOR UPDATE OF j" if lock else ""), (job_id,)).fetchone()
|
||||
|
||||
|
||||
def _owner_host(owner):
|
||||
return (owner or "").split(":", 1)[0]
|
||||
|
||||
|
||||
def claim(conn, msg, owner, lease_seconds):
|
||||
"""Решение по сообщению. Возвращает (action, job):
|
||||
'process' — аренда взята (job со свежим lease_token), можно работать;
|
||||
'ack' — ничего делать не нужно (уже ready/failed, занято живой арендой, ранний дубль);
|
||||
'reject' — сообщение не соответствует заданию (в DLQ, разбирает оператор).
|
||||
Вызывается в транзакции; при 'process' коммит фиксирует processing ДО начала работы."""
|
||||
job = load_job(conn, msg["job_id"], lock=True)
|
||||
if job is None:
|
||||
return "reject", "unknown_job"
|
||||
if str(job["attachment_id"]) != msg["attachment_id"] or job["pipeline_version"] != msg["pipeline_version"]:
|
||||
return "reject", "job_mismatch"
|
||||
status = job["job_status"]
|
||||
if status == "ready":
|
||||
add_event(conn, job["job_id"], "duplicate_ignored", msg["attempt"], owner, {"status": "ready"})
|
||||
return "ack", job
|
||||
if status == "failed":
|
||||
add_event(conn, job["job_id"], "duplicate_ignored", msg["attempt"], owner, {"status": "failed"})
|
||||
return "ack", job
|
||||
if job["current_pipeline_version"] > job["pipeline_version"]:
|
||||
# Устаревшая job после новой версии pipeline: результат не публикуем.
|
||||
conn.execute("""UPDATE jobs SET status = 'failed', error_code = 'superseded', lease_owner = NULL,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = now(), finished_at = now()
|
||||
WHERE id = %s""", (job["job_id"],))
|
||||
add_event(conn, job["job_id"], "superseded", msg["attempt"], owner)
|
||||
return "ack", job
|
||||
if status == "processing" and job["lease_alive"]:
|
||||
prev = job["lease_owner"] or ""
|
||||
# Тот же контейнер (hostname), но другой процесс (boot id): прежняя инкарнация этого
|
||||
# worker умерла (crash/restart) — два процесса одного контейнера одновременно не живут.
|
||||
if not (_owner_host(prev) == _owner_host(owner) and prev != owner):
|
||||
add_event(conn, job["job_id"], "busy_ignored", msg["attempt"], owner, {"lease_owner": prev})
|
||||
return "ack", job
|
||||
add_event(conn, job["job_id"], "lease_taken_over", job["attempt"], owner,
|
||||
{"previous_owner": prev, "reason": "previous process of this container died"})
|
||||
elif status == "processing":
|
||||
add_event(conn, job["job_id"], "lease_taken_over", job["attempt"], owner,
|
||||
{"previous_owner": job["lease_owner"], "reason": "lease expired"})
|
||||
elif job["too_early"]:
|
||||
# queued, но время следующей попытки ещё не пришло: это старый дубль, а запланированное
|
||||
# сообщение relay опубликует само (outbox.available_at).
|
||||
add_event(conn, job["job_id"], "early_duplicate_ignored", msg["attempt"], owner)
|
||||
return "ack", job
|
||||
if job["attempt"] >= job["max_attempts"]:
|
||||
_finish_failed(conn, job, "retry_exhausted", owner, "attempts exhausted before claim")
|
||||
return "ack", job
|
||||
token = uuid.uuid4()
|
||||
conn.execute(
|
||||
"""UPDATE jobs SET status = 'processing', attempt = attempt + 1, lease_owner = %s, lease_token = %s,
|
||||
lease_until = now() + make_interval(secs => %s), updated_at = now()
|
||||
WHERE id = %s""", (owner, token, lease_seconds, job["job_id"]))
|
||||
conn.execute("UPDATE attachments SET status = 'processing', updated_at = now() WHERE id = %s",
|
||||
(job["attachment_id"],))
|
||||
job = dict(job, attempt=job["attempt"] + 1, lease_token=token, job_status="processing")
|
||||
add_event(conn, job["job_id"], "claimed", job["attempt"], owner,
|
||||
{"redelivered_message_attempt": msg["attempt"]})
|
||||
return "process", job
|
||||
|
||||
|
||||
def heartbeat(conn, job_id, token, lease_seconds):
|
||||
"""Продлить аренду. False — аренда потеряна (reaper/другой worker), работу надо бросить."""
|
||||
cur = conn.execute(
|
||||
"""UPDATE jobs SET lease_until = now() + make_interval(secs => %s)
|
||||
WHERE id = %s AND lease_token = %s AND status = 'processing'""", (lease_seconds, job_id, token))
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
def complete(conn, job, variants, owner):
|
||||
"""processing -> ready атомарно с метаданными вариантов. Только владелец аренды (fencing по
|
||||
lease_token): опоздавший процесс, у которого аренду забрали, ничего не перезапишет.
|
||||
Возвращает 'ready' | 'already_ready' | 'lease_lost'."""
|
||||
cur = conn.execute(
|
||||
"""UPDATE jobs SET status = 'ready', lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
error_code = NULL, last_error = NULL, updated_at = now(), finished_at = now()
|
||||
WHERE id = %s AND status = 'processing' AND lease_token = %s""", (job["job_id"], job["lease_token"]))
|
||||
if cur.rowcount != 1:
|
||||
now = load_job(conn, job["job_id"])
|
||||
add_event(conn, job["job_id"], "commit_rejected_lease_lost", job["attempt"], owner,
|
||||
{"current_status": now["job_status"]})
|
||||
return "already_ready" if now["job_status"] == "ready" else "lease_lost"
|
||||
with conn.cursor() as cur:
|
||||
# Ключ эффекта — PK (attachment_id, pipeline_version, name): второго набора не будет.
|
||||
cur.executemany(
|
||||
"""INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type,
|
||||
size_bytes, width, height, sha256, created_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now())
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO UPDATE SET
|
||||
object_key = EXCLUDED.object_key, content_type = EXCLUDED.content_type,
|
||||
size_bytes = EXCLUDED.size_bytes, width = EXCLUDED.width, height = EXCLUDED.height,
|
||||
sha256 = EXCLUDED.sha256""",
|
||||
[(job["attachment_id"], job["pipeline_version"], v["name"], v["key"], v["content_type"],
|
||||
v["size_bytes"], v["width"], v["height"], v["sha256"]) for v in variants])
|
||||
# Не откатываем более новую версию pipeline (на случай нескольких версий, трек 5).
|
||||
conn.execute(
|
||||
"""UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = now()
|
||||
WHERE id = %s AND pipeline_version <= %s""", (job["attachment_id"], job["pipeline_version"]))
|
||||
add_event(conn, job["job_id"], "ready", job["attempt"], owner,
|
||||
{"variants": {v["name"]: [v["width"], v["height"]] for v in variants}})
|
||||
return "ready"
|
||||
|
||||
|
||||
def _finish_failed(conn, job, code, actor, detail):
|
||||
conn.execute(
|
||||
"""UPDATE jobs SET status = 'failed', error_code = %s, last_error = %s, lease_owner = NULL,
|
||||
lease_token = NULL, lease_until = NULL, updated_at = now(), finished_at = now()
|
||||
WHERE id = %s""", (code, (detail or "")[:500], job["job_id"]))
|
||||
conn.execute("UPDATE attachments SET status = 'failed', error_code = %s, updated_at = now() WHERE id = %s",
|
||||
(code, job["attachment_id"]))
|
||||
add_event(conn, job["job_id"], "failed", job["attempt"], actor, {"error_code": code, "detail": detail})
|
||||
|
||||
|
||||
def fail_or_retry(conn, job, code, detail, owner, base, cap, rng=random):
|
||||
"""Ошибка попытки. Постоянная -> failed(code); временная -> queued с backoff, пока есть
|
||||
попытки, иначе failed(retry_exhausted). Только владелец аренды.
|
||||
Возвращает 'failed' | 'retry' | 'lease_lost' и задержку."""
|
||||
cur = load_job(conn, job["job_id"], lock=True)
|
||||
if cur["job_status"] != "processing" or cur["lease_token"] != job["lease_token"]:
|
||||
add_event(conn, job["job_id"], "error_ignored_lease_lost", job["attempt"], owner, {"error_code": code})
|
||||
return "lease_lost", None
|
||||
if code in PERMANENT_ERRORS:
|
||||
_finish_failed(conn, job, code, owner, detail)
|
||||
return "failed", None
|
||||
if cur["attempt"] >= cur["max_attempts"]:
|
||||
_finish_failed(conn, job, "retry_exhausted", owner, "last error: %s %s" % (code, detail or ""))
|
||||
return "failed", None
|
||||
delay = backoff_seconds(cur["attempt"], base, cap, rng)
|
||||
_schedule_retry(conn, cur, delay, code, detail, owner, "retry_scheduled")
|
||||
return "retry", delay
|
||||
|
||||
|
||||
def _schedule_retry(conn, job, delay, code, detail, actor, event):
|
||||
conn.execute(
|
||||
"""UPDATE jobs SET status = 'queued', lease_owner = NULL, lease_token = NULL, lease_until = NULL,
|
||||
error_code = NULL, last_error = %s, next_attempt_at = now() + make_interval(secs => %s),
|
||||
updated_at = now()
|
||||
WHERE id = %s""", (("%s: %s" % (code, detail or ""))[:500], delay, job["job_id"]))
|
||||
conn.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = %s",
|
||||
(job["attachment_id"],))
|
||||
add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1, delay)
|
||||
add_event(conn, job["job_id"], event, job["attempt"], actor,
|
||||
{"error_code": code, "delay_seconds": round(delay, 3), "next_attempt": job["attempt"] + 1})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reaper (в relay): зависшие processing и потерянные сообщения
|
||||
# ---------------------------------------------------------------------------
|
||||
def reap_expired_leases(conn, base, cap, actor="reaper", limit=100):
|
||||
"""processing с истёкшей арендой (worker умер/завис и не продлевал lease) -> queued
|
||||
с backoff или failed(retry_exhausted). Возвращает число обработанных."""
|
||||
rows = conn.execute(
|
||||
"SELECT " + JOB_COLUMNS + """ FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.status = 'processing' AND j.lease_until < now()
|
||||
ORDER BY j.lease_until LIMIT %s FOR UPDATE OF j SKIP LOCKED""", (limit,)).fetchall()
|
||||
for job in rows:
|
||||
add_event(conn, job["job_id"], "lease_expired", job["attempt"], actor, {"lease_owner": job["lease_owner"]})
|
||||
if job["attempt"] >= job["max_attempts"]:
|
||||
_finish_failed(conn, job, "retry_exhausted", actor, "lease expired on last attempt")
|
||||
else:
|
||||
_schedule_retry(conn, job, backoff_seconds(job["attempt"], base, cap), "lease_expired",
|
||||
"worker did not renew lease", actor, "requeued_after_lease")
|
||||
return len(rows)
|
||||
|
||||
|
||||
def redrive_stale_queued(conn, after_seconds, actor="reaper", limit=100):
|
||||
"""queued-задание, по которому давно нет ни неопубликованного outbox, ни движения:
|
||||
сообщение могло потеряться в брокере (например, после потери его volume). Публикуем
|
||||
ещё раз — дубль безопасен, решение примет worker по строке jobs."""
|
||||
rows = conn.execute(
|
||||
"""SELECT j.id, j.attachment_id, j.pipeline_version, j.attempt FROM jobs j
|
||||
WHERE j.status = 'queued' AND j.updated_at < now() - make_interval(secs => %s)
|
||||
AND j.next_attempt_at < now() - make_interval(secs => %s)
|
||||
AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.id AND o.published_at IS NULL)
|
||||
ORDER BY j.updated_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""",
|
||||
(after_seconds, after_seconds, limit)).fetchall()
|
||||
for r in rows:
|
||||
conn.execute("UPDATE jobs SET updated_at = now() WHERE id = %s", (r["id"],))
|
||||
add_outbox(conn, r["id"], r["attachment_id"], r["pipeline_version"], r["attempt"] + 1)
|
||||
add_event(conn, r["id"], "redriven", r["attempt"] + 1, actor, {"stale_after_seconds": after_seconds})
|
||||
return len(rows)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Оператор: replay
|
||||
# ---------------------------------------------------------------------------
|
||||
def replay(conn, attachment_id, actor, extra_attempts, reason, dry_run=False):
|
||||
"""failed -> queued. Новая попытка того же job_id (прежний ключ эффекта), max_attempts
|
||||
увеличивается на extra_attempts. ready/queued/processing НЕ трогаются: «повторить» не
|
||||
означает выполнить дубль успешной работы. Возвращает (result, job)."""
|
||||
job = conn.execute(
|
||||
"SELECT " + JOB_COLUMNS + """ FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.attachment_id = %s AND j.pipeline_version = a.pipeline_version FOR UPDATE OF j""",
|
||||
(attachment_id,)).fetchone()
|
||||
if job is None:
|
||||
return "not_found", None
|
||||
if job["job_status"] != "failed":
|
||||
return "refused_status_" + job["job_status"], job
|
||||
if job["staging_key"] is None:
|
||||
return "refused_staging_expired", job
|
||||
if dry_run:
|
||||
return "would_replay", job
|
||||
conn.execute(
|
||||
"""UPDATE jobs SET status = 'queued', max_attempts = attempt + %s, error_code = NULL,
|
||||
next_attempt_at = now(), replay_count = replay_count + 1, updated_at = now(), finished_at = NULL
|
||||
WHERE id = %s""", (extra_attempts, job["job_id"]))
|
||||
conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() WHERE id = %s",
|
||||
(job["attachment_id"],))
|
||||
add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1)
|
||||
add_event(conn, job["job_id"], "replayed", job["attempt"] + 1, actor,
|
||||
{"reason": reason, "previous_error": job.get("error_code"), "replay_count": job["replay_count"] + 1})
|
||||
return "replayed", job
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Однократное применение миграций (compose-сервис `migrate`).
|
||||
|
||||
Запускается ОДИН раз до старта/замены API-реплик, а не каждой репликой.
|
||||
Дополнительно миграции выполняются под pg_advisory_xact_lock: даже если
|
||||
оператор случайно запустит job дважды параллельно, гонки не будет.
|
||||
Каждый файл применяется в своей транзакции и фиксируется в schema_migrations.
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
|
||||
from server import MIGRATIONS_DIR, STORE
|
||||
|
||||
log = logging.getLogger("messenger.migrate")
|
||||
|
||||
|
||||
def main():
|
||||
STORE.open()
|
||||
try:
|
||||
before = STORE.pending_migrations(MIGRATIONS_DIR)
|
||||
STORE.migrate(MIGRATIONS_DIR)
|
||||
log.info("migrations done; applied now: %s", ", ".join(before) or "nothing")
|
||||
finally:
|
||||
STORE.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+182
@@ -0,0 +1,182 @@
|
||||
"""Доступ к S3-совместимому хранилищу (SeaweedFS) через клиент minio-py.
|
||||
|
||||
API работает с bucket под ОТДЕЛЬНЫМИ credentials с правами только Read+Write на этот
|
||||
bucket (без List, без создания/удаления bucket, без политик и IAM) — см.
|
||||
scripts/init-secrets.sh и REPORT.md. Публичных URL/redirect нет: байты всегда идут
|
||||
через авторизованный API.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import urllib3
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
|
||||
log = logging.getLogger("messenger.s3")
|
||||
|
||||
# minio-py для объектов больше part_size делает multipart и держит в памяти
|
||||
# один part. 5 MiB — минимум S3: файл 10 MiB -> 2 части по 5 MiB.
|
||||
PART_SIZE = 5 * 1024 * 1024
|
||||
STREAM_CHUNK = 64 * 1024
|
||||
# Служебные объекты S3Health: health/<реплика> (вне prefix originals/, janitor их не трогает).
|
||||
HEALTH_PREFIX = "health/"
|
||||
|
||||
|
||||
class StorageUnavailable(Exception):
|
||||
"""S3 недоступен/таймаут/ошибка сервера -> 503."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""Метаданные есть, объекта нет (рассогласование) -> 503 + лог."""
|
||||
|
||||
|
||||
def _secret(name):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
return f.read().strip()
|
||||
return os.environ.get(name, "")
|
||||
|
||||
|
||||
def make_client(access_env="S3_ACCESS_KEY_ID", secret_env="S3_SECRET_ACCESS_KEY",
|
||||
connect_timeout=None, read_timeout=None):
|
||||
endpoint = urlsplit(os.environ.get("S3_ENDPOINT", "http://s3:8333"))
|
||||
connect_timeout = connect_timeout or float(os.environ.get("S3_CONNECT_TIMEOUT_SECONDS", "2"))
|
||||
read_timeout = read_timeout or float(os.environ.get("S3_READ_TIMEOUT_SECONDS", "5"))
|
||||
# Короткие таймауты и без собственных повторов: зависший S3 не держит поток
|
||||
# дольше таймаута LB, повтор решает клиент (для upload — новым запросом).
|
||||
http = urllib3.PoolManager(
|
||||
timeout=urllib3.Timeout(connect=connect_timeout, read=read_timeout),
|
||||
retries=urllib3.Retry(total=0, connect=0, read=0, redirect=0, status=0),
|
||||
maxsize=int(os.environ.get("S3_POOL_SIZE", "16")),
|
||||
block=False)
|
||||
return Minio(endpoint.netloc, access_key=_secret(access_env), secret_key=_secret(secret_env),
|
||||
secure=endpoint.scheme == "https", region=os.environ.get("S3_REGION", "us-east-1"),
|
||||
http_client=http)
|
||||
|
||||
|
||||
def _is_transport_error(exc):
|
||||
return isinstance(exc, (urllib3.exceptions.HTTPError, OSError))
|
||||
|
||||
|
||||
class ObjectStore:
|
||||
def __init__(self, client, bucket):
|
||||
self.client = client
|
||||
self.bucket = bucket
|
||||
|
||||
def put(self, key, fileobj, length, content_type):
|
||||
try:
|
||||
self.client.put_object(self.bucket, key, fileobj, length, content_type=content_type,
|
||||
part_size=PART_SIZE)
|
||||
except S3Error as exc:
|
||||
log.warning("s3 put failed: %s", exc.code)
|
||||
raise StorageUnavailable()
|
||||
except Exception as exc:
|
||||
if _is_transport_error(exc):
|
||||
log.warning("s3 put failed: %s", type(exc).__name__)
|
||||
raise StorageUnavailable()
|
||||
raise
|
||||
|
||||
def open(self, key):
|
||||
"""Поток объекта (urllib3 response). Закрывать: resp.close(); resp.release_conn()."""
|
||||
try:
|
||||
return self.client.get_object(self.bucket, key)
|
||||
except S3Error as exc:
|
||||
if exc.code in ("NoSuchKey", "NoSuchObject"):
|
||||
raise ObjectMissing()
|
||||
log.warning("s3 get failed: %s", exc.code)
|
||||
raise StorageUnavailable()
|
||||
except Exception as exc:
|
||||
if _is_transport_error(exc):
|
||||
log.warning("s3 get failed: %s", type(exc).__name__)
|
||||
raise StorageUnavailable()
|
||||
raise
|
||||
|
||||
def delete(self, key):
|
||||
"""Идемпотентно: отсутствие объекта — не ошибка (S3 DELETE так и работает)."""
|
||||
try:
|
||||
self.client.remove_object(self.bucket, key)
|
||||
except S3Error as exc:
|
||||
if exc.code in ("NoSuchKey", "NoSuchObject"):
|
||||
return
|
||||
raise StorageUnavailable()
|
||||
except Exception as exc:
|
||||
if _is_transport_error(exc):
|
||||
raise StorageUnavailable()
|
||||
raise
|
||||
|
||||
def ping(self, probe_key):
|
||||
"""PUT + GET маленького служебного объекта health/<реплика>: проверяет S3,
|
||||
credentials и РЕАЛЬНЫЕ пути записи и чтения. HEAD bucket/healthz для этого мало:
|
||||
у SeaweedFS после рестарта они успешны, а первая запись ещё ~5 с висит
|
||||
(выбор raft-лидера master, выдача volume под запись)."""
|
||||
import io
|
||||
self.client.put_object(self.bucket, probe_key, io.BytesIO(b"ok"), 2, content_type="text/plain")
|
||||
resp = self.client.get_object(self.bucket, probe_key)
|
||||
try:
|
||||
if resp.read() != b"ok":
|
||||
raise RuntimeError("probe mismatch")
|
||||
finally:
|
||||
resp.close()
|
||||
resp.release_conn()
|
||||
return True
|
||||
|
||||
|
||||
class S3Health:
|
||||
"""Фоновая проверка S3, как DbHealth для PostgreSQL.
|
||||
|
||||
S3 НЕ входит в /health/ready: он общий для всех реплик, и его отказ не лечится
|
||||
исключением реплики из LB, а текстовые сообщения без него работают. Флаг нужен,
|
||||
чтобы upload при известном отказе S3 сразу получал 503, не принимая 10 MiB."""
|
||||
|
||||
def __init__(self, store, interval, stale_after, instance_id):
|
||||
self.store = store
|
||||
self.probe_key = HEALTH_PREFIX + instance_id
|
||||
self.interval = interval
|
||||
self.stale_after = stale_after
|
||||
self.last_ok = None
|
||||
self._stop = threading.Event()
|
||||
self._ok_event = threading.Event() # выставлен, пока последняя проверка успешна
|
||||
|
||||
def start(self):
|
||||
threading.Thread(target=self._loop, name="s3-health", daemon=True).start()
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
|
||||
def ok(self):
|
||||
last = self.last_ok
|
||||
return last is not None and time.monotonic() - last <= self.stale_after
|
||||
|
||||
def wait_startup(self, timeout):
|
||||
"""Только на старте реплики (успешной проверки ещё не было): подождать до timeout,
|
||||
пока S3 станет доступен для записи. При подтверждённом отказе — не ждать."""
|
||||
if self.last_ok is None:
|
||||
self._ok_event.wait(timeout)
|
||||
return self.ok()
|
||||
|
||||
def known_down(self):
|
||||
"""S3 был доступен и перестал отвечать. На старте реплики (успешной проверки ещё
|
||||
не было) — False: чтение может работать раньше, чем прогреется запись."""
|
||||
last = self.last_ok
|
||||
return last is not None and time.monotonic() - last > self.stale_after
|
||||
|
||||
def _loop(self):
|
||||
healthy = None
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
self.store.ping(self.probe_key)
|
||||
self.last_ok = time.monotonic()
|
||||
self._ok_event.set()
|
||||
if healthy is not True:
|
||||
log.info("s3 health: ok")
|
||||
healthy = True
|
||||
except Exception as exc:
|
||||
self._ok_event.clear()
|
||||
if healthy is not False:
|
||||
log.warning("s3 health: failing (%s)", type(exc).__name__)
|
||||
healthy = False
|
||||
self._stop.wait(self.interval)
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Операторский CLI (compose-сервис `ops`, profile tools). Вывод — JSON без секретов.
|
||||
|
||||
ops inspect <attachment_id> состояние: attachment, job, события, варианты, outbox, объекты S3
|
||||
ops failed [--limit N] реестр failed-заданий (DLQ на стороне БД)
|
||||
ops replay <attachment_id>... [--all-failed] [--error-code C] [--dry-run] [--reason TEXT]
|
||||
явный replay failed -> queued (ready/queued/processing не трогаются)
|
||||
ops dlq [--replay] [--purge-invalid]
|
||||
сообщения в RabbitMQ DLQ: показать; --replay — разобрать каждое
|
||||
по состоянию jobs (failed -> replay, остальное -> отбросить с audit)
|
||||
ops publish-raw <json> учебное: опубликовать произвольное тело (проверка DLQ)
|
||||
|
||||
Replay всегда проходит через строку jobs под FOR UPDATE: повтор ready-задания отклоняется,
|
||||
поэтому «повторить» не означает исполнить дубль уже успешной работы.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import common
|
||||
import jobs
|
||||
import objstore
|
||||
|
||||
EXTRA_ATTEMPTS = common.env_int("JOB_MAX_ATTEMPTS", 5)
|
||||
|
||||
|
||||
def db():
|
||||
return psycopg.connect(common.database_conninfo(), row_factory=dict_row)
|
||||
|
||||
|
||||
def jsonable(value):
|
||||
if isinstance(value, dict):
|
||||
return {k: jsonable(v) for k, v in value.items()}
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [jsonable(v) for v in value]
|
||||
if isinstance(value, (str, int, float, bool)) or value is None:
|
||||
return value
|
||||
if hasattr(value, "isoformat"):
|
||||
return value.isoformat().replace("+00:00", "Z")
|
||||
return str(value)
|
||||
|
||||
|
||||
def out(data):
|
||||
print(json.dumps(jsonable(data), ensure_ascii=False, indent=1))
|
||||
|
||||
|
||||
def s3_objects(prefix):
|
||||
"""Только имена/размеры (ключ janitor: List, без чтения содержимого)."""
|
||||
store = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments"))
|
||||
try:
|
||||
return sorted((o.object_name, o.size) for o in store.client.list_objects(store.bucket, prefix=prefix, recursive=True))
|
||||
except Exception as exc:
|
||||
return [("error", type(exc).__name__)]
|
||||
|
||||
|
||||
def cmd_inspect(args):
|
||||
with db() as conn:
|
||||
att = conn.execute(
|
||||
"""SELECT id, chat_id, status, error_code, content_type, size_bytes, sha256, width, height,
|
||||
object_key, staging_key, pipeline_version, created_at, updated_at
|
||||
FROM attachments WHERE id = %s""", (args.attachment_id,)).fetchone()
|
||||
if att is None:
|
||||
out({"error": "not_found"})
|
||||
return 1
|
||||
job = conn.execute(
|
||||
"""SELECT id, pipeline_version, status, attempt, max_attempts, next_attempt_at, lease_owner,
|
||||
lease_until, error_code, last_error, replay_count, created_at, updated_at, finished_at
|
||||
FROM jobs WHERE attachment_id = %s ORDER BY pipeline_version""", (args.attachment_id,)).fetchall()
|
||||
job_ids = [j["id"] for j in job]
|
||||
events = conn.execute(
|
||||
"SELECT job_id, at, event, attempt, actor, detail FROM job_events WHERE job_id = ANY(%s) ORDER BY id",
|
||||
(job_ids,)).fetchall()
|
||||
outbox = conn.execute(
|
||||
"""SELECT id, payload->>'attempt' AS attempt, available_at, created_at, published_at, publish_count
|
||||
FROM outbox WHERE job_id = ANY(%s) ORDER BY id""", (job_ids,)).fetchall()
|
||||
variants = conn.execute(
|
||||
"""SELECT pipeline_version, name, object_key, content_type, size_bytes, width, height, sha256
|
||||
FROM attachment_variants WHERE attachment_id = %s ORDER BY pipeline_version, name""",
|
||||
(args.attachment_id,)).fetchall()
|
||||
objects = s3_objects("attachments/%s/" % args.attachment_id)
|
||||
staging = s3_objects(att["staging_key"]) if att["staging_key"] else []
|
||||
out({"attachment": att, "jobs": job, "events": events, "outbox": outbox, "variants": variants,
|
||||
"s3_final_objects": objects, "s3_staging_objects": staging})
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_failed(args):
|
||||
with db() as conn:
|
||||
rows = conn.execute(
|
||||
"""SELECT j.attachment_id, j.id AS job_id, j.error_code, j.attempt, j.max_attempts, j.replay_count,
|
||||
j.last_error, j.updated_at, a.staging_key IS NOT NULL AS replayable
|
||||
FROM jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.status = 'failed' ORDER BY j.updated_at DESC LIMIT %s""", (args.limit,)).fetchall()
|
||||
out({"failed": rows, "count": len(rows)})
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_replay(args):
|
||||
results = []
|
||||
with db() as conn:
|
||||
ids = list(args.attachment_ids)
|
||||
if args.all_failed:
|
||||
q = "SELECT attachment_id FROM jobs WHERE status = 'failed'"
|
||||
params = []
|
||||
if args.error_code:
|
||||
q += " AND error_code = %s"
|
||||
params.append(args.error_code)
|
||||
ids += [str(r["attachment_id"]) for r in conn.execute(q, params)]
|
||||
for att_id in ids:
|
||||
with conn.transaction():
|
||||
result, job = jobs.replay(conn, att_id, "ops:" + common.HOSTNAME, EXTRA_ATTEMPTS,
|
||||
args.reason, dry_run=args.dry_run)
|
||||
results.append({"attachment_id": att_id, "result": result,
|
||||
"job_id": str(job["job_id"]) if job else None})
|
||||
out({"replay": results})
|
||||
return 0 if all(r["result"] in ("replayed", "would_replay") for r in results) else 3
|
||||
|
||||
|
||||
def broker_channel():
|
||||
conn = pika.BlockingConnection(common.broker_parameters())
|
||||
return conn, conn.channel()
|
||||
|
||||
|
||||
def cmd_dlq(args):
|
||||
conn, ch = broker_channel()
|
||||
seen = []
|
||||
tags = []
|
||||
try:
|
||||
with db() as dbc:
|
||||
while True:
|
||||
method, props, body = ch.basic_get(common.DLQ, auto_ack=False)
|
||||
if method is None:
|
||||
break
|
||||
deaths = (props.headers or {}).get("x-death") or []
|
||||
reason = deaths[0].get("reason") if deaths else None
|
||||
item = {"reason": reason.decode() if isinstance(reason, bytes) else reason,
|
||||
"delivery_count": (props.headers or {}).get("x-delivery-count"), "action": "kept"}
|
||||
try:
|
||||
msg = jobs.parse_message(body)
|
||||
item["job_id"] = msg["job_id"]
|
||||
if args.replay:
|
||||
with dbc.transaction():
|
||||
job = jobs.load_job(dbc, msg["job_id"], lock=True)
|
||||
if job is None:
|
||||
item["action"] = "kept_unknown_job"
|
||||
elif job["job_status"] == "failed":
|
||||
result, _ = jobs.replay(dbc, str(job["attachment_id"]), "ops-dlq", EXTRA_ATTEMPTS,
|
||||
"dlq replay")
|
||||
item["action"] = result
|
||||
else:
|
||||
# queued/processing/ready: состояние уже ведёт БД, дубль не нужен.
|
||||
jobs.add_event(dbc, job["job_id"], "dlq_message_dropped", msg["attempt"], "ops-dlq",
|
||||
{"job_status": job["job_status"], "reason": item["reason"]})
|
||||
item["action"] = "dropped_job_" + job["job_status"]
|
||||
except jobs.InvalidMessage as exc:
|
||||
item["invalid"] = str(exc)
|
||||
if args.purge_invalid:
|
||||
item["action"] = "purged_invalid"
|
||||
seen.append(item)
|
||||
if item["action"].startswith("kept"):
|
||||
tags.append(method.delivery_tag) # вернём в DLQ после обхода
|
||||
else:
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
for tag in tags:
|
||||
ch.basic_nack(tag, requeue=True)
|
||||
finally:
|
||||
conn.close()
|
||||
out({"dlq_messages": seen, "count": len(seen)})
|
||||
return 0
|
||||
|
||||
|
||||
def cmd_publish_raw(args):
|
||||
conn, ch = broker_channel()
|
||||
try:
|
||||
ch.confirm_delivery()
|
||||
ch.basic_publish(common.EXCHANGE, common.ROUTING_KEY, args.body.encode(),
|
||||
pika.BasicProperties(delivery_mode=2, content_type="application/json"), mandatory=True)
|
||||
finally:
|
||||
conn.close()
|
||||
out({"published": True})
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(prog="ops")
|
||||
sub = p.add_subparsers(dest="cmd", required=True)
|
||||
s = sub.add_parser("inspect")
|
||||
s.add_argument("attachment_id")
|
||||
s.set_defaults(func=cmd_inspect)
|
||||
s = sub.add_parser("failed")
|
||||
s.add_argument("--limit", type=int, default=50)
|
||||
s.set_defaults(func=cmd_failed)
|
||||
s = sub.add_parser("replay")
|
||||
s.add_argument("attachment_ids", nargs="*")
|
||||
s.add_argument("--all-failed", action="store_true")
|
||||
s.add_argument("--error-code")
|
||||
s.add_argument("--dry-run", action="store_true")
|
||||
s.add_argument("--reason", default="operator replay")
|
||||
s.set_defaults(func=cmd_replay)
|
||||
s = sub.add_parser("dlq")
|
||||
s.add_argument("--replay", action="store_true")
|
||||
s.add_argument("--purge-invalid", action="store_true")
|
||||
s.set_defaults(func=cmd_dlq)
|
||||
s = sub.add_parser("publish-raw")
|
||||
s.add_argument("body")
|
||||
s.set_defaults(func=cmd_publish_raw)
|
||||
args = p.parse_args()
|
||||
return args.func(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Outbox relay + reaper (compose-сервис `relay`).
|
||||
|
||||
Закрывает разрыв PostgreSQL <-> broker (transactional outbox):
|
||||
API в ОДНОЙ транзакции пишет queued-вложение, jobs и outbox. Relay берёт неопубликованные
|
||||
строки (FOR UPDATE SKIP LOCKED — несколько relay не мешают друг другу), публикует каждую
|
||||
в RabbitMQ с publisher confirm (persistent, mandatory) и только ПОСЛЕ подтверждения
|
||||
брокера ставит published_at в той же транзакции. Крах между COMMIT API и publish, между
|
||||
publish и confirm или между confirm и UPDATE outbox -> строка остаётся неопубликованной и
|
||||
будет опубликована снова: возможен дубль сообщения (at-least-once), его гасит worker по
|
||||
состоянию jobs.
|
||||
|
||||
Будит relay NOTIFY из триггера outbox; без уведомления — опрос раз в RELAY_POLL_SECONDS.
|
||||
Строки с available_at в будущем (retry с backoff) публикуются, когда наступит срок.
|
||||
|
||||
Reaper (раз в REAPER_INTERVAL_SECONDS, под advisory-блокировкой — один на всех):
|
||||
* processing с истёкшей арендой -> queued (backoff) или failed(retry_exhausted);
|
||||
* queued без движения дольше JOB_REDRIVE_AFTER_SECONDS и без неопубликованного outbox
|
||||
-> ещё одна публикация (сообщение могло потеряться вместе с volume брокера);
|
||||
* опубликованные строки outbox старше OUTBOX_RETENTION_SECONDS удаляются.
|
||||
|
||||
Учебная инъекция отказа: RELAY_FAULT=crash-before-publish | crash-after-publish —
|
||||
один раз за жизнь контейнера (маркер в /tmp) процесс делает os._exit посреди батча.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
import pika
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import common
|
||||
import jobs
|
||||
|
||||
log = logging.getLogger("relay")
|
||||
POLL_SECONDS = common.env_float("RELAY_POLL_SECONDS", 1)
|
||||
BATCH = common.env_int("RELAY_BATCH", 100)
|
||||
REAPER_INTERVAL = common.env_float("REAPER_INTERVAL_SECONDS", 5)
|
||||
REDRIVE_AFTER = common.env_int("JOB_REDRIVE_AFTER_SECONDS", 900)
|
||||
OUTBOX_RETENTION = common.env_int("OUTBOX_RETENTION_SECONDS", 86400)
|
||||
RETRY_BASE = common.env_float("JOB_RETRY_BASE_SECONDS", 2)
|
||||
RETRY_MAX = common.env_float("JOB_RETRY_MAX_SECONDS", 60)
|
||||
FAULT = os.environ.get("RELAY_FAULT", "")
|
||||
FAULT_MARKER = "/tmp/relay-fault.done"
|
||||
HEARTBEAT = "/tmp/relay.heartbeat"
|
||||
REAPER_LOCK_ID = 727003
|
||||
|
||||
|
||||
def fault(point):
|
||||
if FAULT != point or os.path.exists(FAULT_MARKER):
|
||||
return
|
||||
with open(FAULT_MARKER, "w") as f:
|
||||
f.write(point)
|
||||
log.warning("FAULT INJECTION %s: exiting without finishing the batch", point)
|
||||
os._exit(86)
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self):
|
||||
self.conninfo = common.database_conninfo()
|
||||
self.db = None
|
||||
self.listen = None
|
||||
self.broker = None
|
||||
self.channel = None
|
||||
self.next_reap = 0.0
|
||||
self.stop = threading.Event()
|
||||
|
||||
# --- соединения ---
|
||||
def ensure_db(self):
|
||||
if self.db is None or self.db.closed:
|
||||
self.db = psycopg.connect(self.conninfo, row_factory=dict_row)
|
||||
if self.listen is None or self.listen.closed:
|
||||
self.listen = psycopg.connect(self.conninfo, autocommit=True)
|
||||
self.listen.execute("LISTEN outbox")
|
||||
|
||||
def ensure_broker(self):
|
||||
if self.broker is None or self.broker.is_closed or self.channel is None or self.channel.is_closed:
|
||||
self.close_broker()
|
||||
self.broker = common.connect_broker(log, self.stop)
|
||||
if self.broker is None:
|
||||
return
|
||||
self.channel = self.broker.channel()
|
||||
# Publisher confirms: basic_publish ждёт ack брокера (сообщение записано в
|
||||
# quorum-очередь) и бросает NackError/UnroutableError при отказе.
|
||||
self.channel.confirm_delivery()
|
||||
log.info("connected to broker")
|
||||
|
||||
def close_broker(self):
|
||||
try:
|
||||
if self.broker is not None and not self.broker.is_closed:
|
||||
self.broker.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.broker = self.channel = None
|
||||
|
||||
def close_db(self):
|
||||
for conn in (self.db, self.listen):
|
||||
try:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.db = self.listen = None
|
||||
|
||||
# --- публикация ---
|
||||
def publish_batch(self):
|
||||
"""Один батч в одной транзакции. Возвращает число опубликованных строк."""
|
||||
with self.db.transaction():
|
||||
rows = self.db.execute(
|
||||
"""SELECT id, job_id, payload FROM outbox
|
||||
WHERE published_at IS NULL AND available_at <= now()
|
||||
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", (BATCH,)).fetchall()
|
||||
if not rows:
|
||||
return 0
|
||||
fault("crash-before-publish") # COMMIT задания есть, publish ещё нет
|
||||
for row in rows:
|
||||
payload = row["payload"]
|
||||
self.channel.basic_publish(
|
||||
exchange=common.EXCHANGE, routing_key=common.ROUTING_KEY,
|
||||
body=json.dumps(payload, separators=(",", ":")).encode(),
|
||||
properties=pika.BasicProperties(
|
||||
content_type="application/json", delivery_mode=pika.DeliveryMode.Persistent,
|
||||
message_id="%s:%s" % (payload["job_id"], payload["attempt"]), type=jobs.JOB_TYPE,
|
||||
headers={"x-outbox-id": row["id"]}),
|
||||
mandatory=True)
|
||||
fault("crash-after-publish") # confirm получен, published_at не записан -> дубль
|
||||
self.db.execute(
|
||||
"UPDATE outbox SET published_at = now(), publish_count = publish_count + 1 WHERE id = ANY(%s)",
|
||||
([r["id"] for r in rows],))
|
||||
log.info("published %d job message(s)", len(rows))
|
||||
return len(rows)
|
||||
|
||||
def reap(self):
|
||||
with self.db.transaction():
|
||||
if not self.db.execute("SELECT pg_try_advisory_xact_lock(%s) AS ok", (REAPER_LOCK_ID,)).fetchone()["ok"]:
|
||||
return
|
||||
expired = jobs.reap_expired_leases(self.db, RETRY_BASE, RETRY_MAX, actor="reaper:" + common.OWNER)
|
||||
redriven = jobs.redrive_stale_queued(self.db, REDRIVE_AFTER, actor="reaper:" + common.OWNER)
|
||||
purged = self.db.execute(
|
||||
"DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)",
|
||||
(OUTBOX_RETENTION,)).rowcount
|
||||
if expired or redriven or purged:
|
||||
log.info("reaper: expired_leases=%d redriven=%d outbox_purged=%d", expired, redriven, purged)
|
||||
|
||||
def wait_notify(self, timeout):
|
||||
"""Ждём NOTIFY outbox не дольше timeout; параллельно обслуживаем heartbeat брокера."""
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if self.broker is not None:
|
||||
self.broker.process_data_events(time_limit=0)
|
||||
for _ in self.listen.notifies(timeout=min(0.25, max(0.01, deadline - time.monotonic())), stop_after=1):
|
||||
return
|
||||
|
||||
def run(self):
|
||||
log.info("relay started owner=%s fault=%s", common.OWNER, FAULT or "-")
|
||||
delay = 0.5
|
||||
while not self.stop.is_set():
|
||||
common.touch(HEARTBEAT)
|
||||
try:
|
||||
self.ensure_db()
|
||||
self.ensure_broker()
|
||||
if self.broker is None:
|
||||
continue
|
||||
while self.publish_batch() == BATCH:
|
||||
pass
|
||||
if time.monotonic() >= self.next_reap:
|
||||
self.reap()
|
||||
self.next_reap = time.monotonic() + REAPER_INTERVAL
|
||||
self.wait_notify(POLL_SECONDS)
|
||||
delay = 0.5
|
||||
except (pika.exceptions.AMQPError, pika.exceptions.UnroutableError, pika.exceptions.NackError) as exc:
|
||||
# Транзакция батча откатилась: строки остались неопубликованными.
|
||||
log.warning("broker publish failed (%s), retry in %.1fs", type(exc).__name__, delay)
|
||||
self.close_broker()
|
||||
self.stop.wait(delay)
|
||||
delay = min(5.0, delay * 2)
|
||||
except psycopg.Error as exc:
|
||||
log.warning("database error (%s), retry in %.1fs", type(exc).__name__, delay)
|
||||
self.close_db()
|
||||
self.stop.wait(delay)
|
||||
delay = min(5.0, delay * 2)
|
||||
self.close_broker()
|
||||
self.close_db()
|
||||
|
||||
|
||||
def main():
|
||||
import signal
|
||||
relay = Relay()
|
||||
signal.signal(signal.SIGTERM, lambda *_: relay.stop.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: relay.stop.set())
|
||||
relay.run()
|
||||
log.info("relay stopped")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,84 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Идемпотентный bootstrap S3 (однократный compose-job `s3-init`).
|
||||
|
||||
Работает под admin-credentials, которые есть ТОЛЬКО у этого job (API и janitor их не
|
||||
получают). Повторный запуск ничего не ломает:
|
||||
1. ждёт доступности S3 (до 60 с);
|
||||
2. создаёт bucket, если его нет (уже существующий — не ошибка);
|
||||
3. проверяет приватность: анонимные GET объекта и ListBucket должны получить 403.
|
||||
Если bucket вдруг публичен — exit 1, и API не стартует (depends_on).
|
||||
Права пользователей (api: Read+Write, janitor: List+Write) задаются декларативно в
|
||||
secrets/s3_identities.json (генерирует scripts/init-secrets.sh).
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
from minio.error import S3Error
|
||||
|
||||
import objstore
|
||||
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
log = logging.getLogger("messenger.s3-init")
|
||||
|
||||
|
||||
def anonymous_status(url):
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=3) as resp:
|
||||
return resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main():
|
||||
bucket = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
endpoint = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/")
|
||||
# Первый PUT в новый bucket ждёт, пока SeaweedFS выделит volume-ы коллекции: таймаут больше.
|
||||
client = objstore.make_client(read_timeout=30)
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
exists = client.bucket_exists(bucket)
|
||||
break
|
||||
except Exception as exc:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log.info("s3 not ready (%s), retrying", type(exc).__name__)
|
||||
time.sleep(1)
|
||||
created = False
|
||||
if not exists:
|
||||
try:
|
||||
client.make_bucket(bucket)
|
||||
created = True
|
||||
except S3Error as exc:
|
||||
if exc.code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
raise
|
||||
# Контрольный объект: на нём проверяем, что анонимный GET не проходит.
|
||||
probe = "bootstrap/probe"
|
||||
for attempt in range(1, 6):
|
||||
try:
|
||||
client.put_object(bucket, probe, io.BytesIO(b"private"), 7, content_type="text/plain")
|
||||
break
|
||||
except Exception as exc:
|
||||
if attempt == 5:
|
||||
raise
|
||||
log.info("probe put failed (%s), retry %d", type(exc).__name__, attempt)
|
||||
time.sleep(2)
|
||||
checks = {
|
||||
"anonymous_get_object": anonymous_status("%s/%s/%s" % (endpoint, bucket, probe)),
|
||||
"anonymous_list_bucket": anonymous_status("%s/%s?list-type=2" % (endpoint, bucket)),
|
||||
}
|
||||
client.remove_object(bucket, probe)
|
||||
private = all(code in (401, 403) for code in checks.values())
|
||||
print(json.dumps({"bucket": bucket, "created": created, "already_existed": exists,
|
||||
"anonymous_http_status": checks, "private": private}), flush=True)
|
||||
return 0 if private else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,53 @@
|
||||
"""Пароли (scrypt из стандартной библиотеки) и сессионные токены."""
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import os
|
||||
import secrets
|
||||
|
||||
# Параметры scrypt: N=2^15, r=8, p=1 (~32 МБ памяти и ~0.1 с на проверку).
|
||||
SCRYPT_LOG_N = 15
|
||||
SCRYPT_R = 8
|
||||
SCRYPT_P = 1
|
||||
_MAXMEM = 128 * 1024 * 1024
|
||||
|
||||
|
||||
def _scrypt(password, salt, log_n, r, p):
|
||||
return hashlib.scrypt(password.encode("utf-8"), salt=salt, n=2 ** log_n, r=r, p=p,
|
||||
maxmem=_MAXMEM, dklen=32)
|
||||
|
||||
|
||||
def hash_password(password):
|
||||
"""Возвращает строку scrypt$logN$r$p$salt$hash; plaintext нигде не хранится."""
|
||||
salt = os.urandom(16)
|
||||
digest = _scrypt(password, salt, SCRYPT_LOG_N, SCRYPT_R, SCRYPT_P)
|
||||
return "scrypt$%d$%d$%d$%s$%s" % (
|
||||
SCRYPT_LOG_N, SCRYPT_R, SCRYPT_P,
|
||||
base64.b64encode(salt).decode(), base64.b64encode(digest).decode())
|
||||
|
||||
|
||||
def verify_password(password, stored):
|
||||
try:
|
||||
scheme, log_n, r, p, salt, digest = stored.split("$")
|
||||
if scheme != "scrypt":
|
||||
return False
|
||||
expected = base64.b64decode(digest)
|
||||
actual = _scrypt(password, base64.b64decode(salt), int(log_n), int(r), int(p))
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return hmac.compare_digest(actual, expected) # сравнение за постоянное время
|
||||
|
||||
|
||||
# Хеш несуществующего пользователя: неверный логин тратит столько же времени,
|
||||
# сколько неверный пароль, и не выдаёт, существует ли username.
|
||||
DUMMY_HASH = hash_password(secrets.token_urlsafe(16))
|
||||
|
||||
|
||||
def new_token():
|
||||
"""Непрозрачный случайный токен: 32 байта = 256 бит энтропии."""
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def token_hash(token):
|
||||
"""В БД лежит только SHA-256 токена: утечка таблицы не даёт готовых сессий."""
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
+1079
File diff suppressed because it is too large
Load Diff
+413
@@ -0,0 +1,413 @@
|
||||
"""Работа с PostgreSQL: пользователи, чаты, сообщения, сессии, метаданные вложений.
|
||||
|
||||
Lab5: вложение создаётся как pending (скрыто), после записи оригинала в staging одной
|
||||
транзакцией становится queued вместе с заданием и записью outbox (src/jobs.py).
|
||||
|
||||
Источник истины — база данных. В памяти процесса состояния нет, поэтому
|
||||
перезапуск API (и самой БД с сохранённым volume) не теряет данные.
|
||||
"""
|
||||
import datetime as dt
|
||||
import logging
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import jobs
|
||||
from psycopg_pool import ConnectionPool
|
||||
|
||||
log = logging.getLogger("messenger.store")
|
||||
|
||||
EPOCH = dt.datetime(1970, 1, 1, tzinfo=dt.timezone.utc)
|
||||
MIGRATIONS_LOCK_ID = 727001 # произвольный номер advisory-блокировки для миграций
|
||||
|
||||
|
||||
class Conflict(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class UnknownMember(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class AttachmentNotFound(Exception):
|
||||
"""Вложение неизвестно, не принято (pending) или принадлежит другому чату -> 404."""
|
||||
|
||||
|
||||
class AttachmentFailed(Exception):
|
||||
"""Вложение в состоянии failed: прикреплять нельзя -> 409."""
|
||||
|
||||
|
||||
def format_ts(value):
|
||||
"""datetime -> RFC 3339 UTC с микросекундами и суффиксом Z."""
|
||||
return value.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
||||
|
||||
|
||||
def to_micros(value):
|
||||
delta = value - EPOCH
|
||||
return (delta.days * 86400 + delta.seconds) * 1_000_000 + delta.microseconds
|
||||
|
||||
|
||||
def from_micros(micros):
|
||||
return EPOCH + dt.timedelta(microseconds=micros)
|
||||
|
||||
|
||||
def _user(row):
|
||||
# Пароль/hash и сессионные данные никогда не входят в User.
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": row["display_name"],
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _chat(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": row["title"],
|
||||
"member_ids": [str(m) for m in row["member_ids"]],
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message(row):
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": row["text"],
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
"attachment_ids": [str(a) for a in (row["attachment_ids"] or [])],
|
||||
}
|
||||
|
||||
|
||||
def _attachment(row):
|
||||
# Наружу не отдаются object_key, bucket и внутренние адреса.
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
# Варианты показываются только у ready (их наличие = готовность уровня).
|
||||
"variants": list(row["variants"] or []) if row["status"] == "ready" else [],
|
||||
"error_code": row["error_code"] if row["status"] == "failed" else None,
|
||||
"created_at": format_ts(row["created_at"]),
|
||||
"updated_at": format_ts(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
ATTACHMENT_COLUMNS = """a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256,
|
||||
a.status, a.error_code, a.object_key, a.created_at, a.updated_at,
|
||||
(SELECT json_agg(json_build_object('name', v.name, 'content_type', v.content_type,
|
||||
'size_bytes', v.size_bytes, 'width', v.width, 'height', v.height, 'sha256', v.sha256)
|
||||
ORDER BY array_position(ARRAY['thumbnail','crop','optimized'], v.name))
|
||||
FROM attachment_variants v
|
||||
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variants"""
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
(SELECT array_agg(ma.attachment_id ORDER BY ma.position) FROM message_attachments ma
|
||||
WHERE ma.message_id = m.id) AS attachment_ids"""
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
(SELECT array_agg(m.user_id ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) AS member_ids"""
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, conninfo, pool_size=10, pool_timeout=3):
|
||||
# check: перед выдачей соединение проверяется. После перезапуска PostgreSQL
|
||||
# «мёртвые» соединения отбрасываются, и запрос не получает ложный 503.
|
||||
self.pool = ConnectionPool(
|
||||
conninfo, min_size=1, max_size=pool_size, timeout=pool_timeout,
|
||||
kwargs={"row_factory": dict_row}, open=False,
|
||||
check=ConnectionPool.check_connection)
|
||||
|
||||
# --- запуск ---
|
||||
def open(self, wait_seconds=60):
|
||||
"""Ждём готовности БД (она может стартовать дольше API) и открываем пул."""
|
||||
self.pool.open()
|
||||
deadline = time.monotonic() + wait_seconds
|
||||
while True:
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
return
|
||||
except Exception as exc:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log.info("database is not ready yet (%s), retrying", type(exc).__name__)
|
||||
time.sleep(1)
|
||||
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
|
||||
def migrate(self, directory):
|
||||
"""Применяет *.sql по порядку имён; каждая миграция — одна транзакция, один раз."""
|
||||
files = sorted(Path(directory).glob("*.sql"))
|
||||
with self.pool.connection() as conn:
|
||||
# Advisory-блокировка (сессионная): параллельный повторный запуск job не устроит гонку.
|
||||
conn.autocommit = True
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK_ID,))
|
||||
try:
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())""")
|
||||
done = {r["version"] for r in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for path in files:
|
||||
if path.name in done:
|
||||
continue
|
||||
# Миграция и её отметка — одна транзакция: либо целиком, либо никак.
|
||||
with conn.transaction():
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
|
||||
log.info("migration applied: %s", path.name)
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK_ID,))
|
||||
conn.autocommit = False
|
||||
|
||||
def pending_migrations(self, directory):
|
||||
"""Миграции из каталога образа, ещё не применённые к БД (API их НЕ применяет)."""
|
||||
files = sorted(p.name for p in Path(directory).glob("*.sql"))
|
||||
with self.pool.connection() as conn:
|
||||
exists = conn.execute("SELECT to_regclass('schema_migrations') IS NOT NULL AS ok").fetchone()["ok"]
|
||||
done = {r["version"] for r in conn.execute("SELECT version FROM schema_migrations")} if exists else set()
|
||||
return [f for f in files if f not in done]
|
||||
|
||||
def ping(self):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
# --- пользователи ---
|
||||
def create_user(self, username, display_name, password_hash):
|
||||
try:
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (%s, %s, %s, %s, clock_timestamp())
|
||||
RETURNING id, username, display_name, created_at""",
|
||||
(uuid.uuid4(), username, display_name, password_hash)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
# Уникальность гарантирует сама БД — гонка двух регистраций безопасна.
|
||||
raise Conflict()
|
||||
return _user(row)
|
||||
|
||||
def get_credentials(self, username):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,)).fetchone()
|
||||
if row is None:
|
||||
return None, None
|
||||
return _user(row), row["password_hash"]
|
||||
|
||||
# --- сессии ---
|
||||
def create_session(self, user_id, token_hash, ttl_seconds):
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""INSERT INTO sessions (token_hash, user_id, created_at, expires_at)
|
||||
VALUES (%s, %s, now(), now() + make_interval(secs => %s))
|
||||
RETURNING expires_at""",
|
||||
(token_hash, user_id, ttl_seconds)).fetchone()
|
||||
return format_ts(row["expires_at"])
|
||||
|
||||
def get_session_user(self, token_hash):
|
||||
"""Пользователь действующей сессии или None (неизвестна/истекла/отозвана)."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""SELECT u.id, u.username, u.display_name, u.created_at
|
||||
FROM sessions s JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > now()""",
|
||||
(token_hash,)).fetchone()
|
||||
return _user(row) if row else None
|
||||
|
||||
def revoke_session(self, token_hash):
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"UPDATE sessions SET revoked_at = now() WHERE token_hash = %s AND revoked_at IS NULL",
|
||||
(token_hash,))
|
||||
|
||||
# --- чаты ---
|
||||
def create_chat(self, title, member_ids):
|
||||
"""Чат и его участники создаются в одной транзакции (атомарно)."""
|
||||
chat_id = uuid.uuid4()
|
||||
with self.pool.connection() as conn:
|
||||
found = conn.execute(
|
||||
"SELECT count(*) AS n FROM users WHERE id = ANY(%s::uuid[])", (member_ids,)).fetchone()["n"]
|
||||
if found != len(member_ids):
|
||||
raise UnknownMember()
|
||||
conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES (%s, %s, clock_timestamp())",
|
||||
(chat_id, title))
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)",
|
||||
[(chat_id, m, i) for i, m in enumerate(member_ids)])
|
||||
row = conn.execute("SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id = %s", (chat_id,)).fetchone()
|
||||
return _chat(row)
|
||||
|
||||
def get_chat_for(self, user_id, chat_id):
|
||||
"""Чат, если пользователь — участник; иначе None (чужой и несуществующий неразличимы)."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + """ FROM chats c
|
||||
WHERE c.id = %s AND EXISTS (
|
||||
SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = %s)""",
|
||||
(chat_id, user_id)).fetchone()
|
||||
return _chat(row) if row else None
|
||||
|
||||
def list_chats(self, user_id, after, limit):
|
||||
params = [user_id]
|
||||
where = ""
|
||||
if after:
|
||||
where = "AND (c.created_at, c.id) > (%s, %s::uuid)"
|
||||
params += [from_micros(after[0]), after[1]]
|
||||
params.append(limit + 1) # +1 запись: узнаём, есть ли следующая страница
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT " + CHAT_COLUMNS + """ FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||
WHERE true """ + where + " ORDER BY c.created_at, c.id LIMIT %s", params).fetchall()
|
||||
return self._page(rows, limit, _chat)
|
||||
|
||||
# --- сообщения ---
|
||||
def add_message(self, user_id, chat_id, text, attachment_ids, idem_key, request_hash, idem_ttl_seconds):
|
||||
"""Идемпотентная отправка. Возвращает (message, replayed) или None, если чата нет/он чужой.
|
||||
|
||||
Всё в ОДНОЙ транзакции: проверка доступа, поиск ключа, вставка сообщения и запись
|
||||
ключа. Либо фиксируется и сообщение, и ключ, либо ничего (транзиентная ошибка
|
||||
не запоминается как результат — клиент повторит с тем же ключом).
|
||||
Конкурентные запросы с одним ключом через разные реплики сериализуются
|
||||
блокировкой строки чата (FOR UPDATE), второй после COMMIT первого видит ключ.
|
||||
PK таблицы ключей — страховка: даже без блокировки дубль не зафиксируется.
|
||||
Lab4: в той же транзакции проверяется, что все attachment_ids — ready-вложения
|
||||
ЭТОГО чата (FOR SHARE: строка не исчезнет до COMMIT), и пишутся ссылки.
|
||||
"""
|
||||
with self.pool.connection() as conn:
|
||||
# FOR UPDATE по строке чата: вставки в один чат идут по очереди, поэтому
|
||||
# created_at растёт в порядке фиксации и keyset-курсор не пропускает записи.
|
||||
allowed = conn.execute(
|
||||
"""SELECT 1 FROM chats c WHERE c.id = %s AND EXISTS (
|
||||
SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = %s)
|
||||
FOR UPDATE OF c""", (chat_id, user_id)).fetchone()
|
||||
if allowed is None:
|
||||
return None
|
||||
existing = conn.execute(
|
||||
"""SELECT k.request_hash, k.expires_at > now() AS alive, """ + MESSAGE_COLUMNS + """
|
||||
FROM idempotency_keys k JOIN messages m ON m.id = k.message_id
|
||||
WHERE k.user_id = %s AND k.chat_id = %s AND k.idem_key = %s""",
|
||||
(user_id, chat_id, idem_key)).fetchone()
|
||||
if existing is not None:
|
||||
if existing["alive"]:
|
||||
if bytes(existing["request_hash"]) != request_hash:
|
||||
raise Conflict()
|
||||
return _message(existing), True
|
||||
# Ключ старше TTL (24 ч) — освобождаем его и обрабатываем как новый.
|
||||
conn.execute(
|
||||
"DELETE FROM idempotency_keys WHERE user_id = %s AND chat_id = %s AND idem_key = %s",
|
||||
(user_id, chat_id, idem_key))
|
||||
if attachment_ids:
|
||||
# Lab5: queued/processing/ready можно прикреплять; failed — 409;
|
||||
# pending (upload ещё не принят) — как неизвестное.
|
||||
found = conn.execute(
|
||||
"""SELECT id, status FROM attachments
|
||||
WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status <> 'pending'
|
||||
FOR SHARE""", (attachment_ids, chat_id)).fetchall()
|
||||
if len(found) != len(attachment_ids):
|
||||
# Чужой чат, неизвестный или непринятый id — одинаково 404.
|
||||
raise AttachmentNotFound()
|
||||
if any(r["status"] == "failed" for r in found):
|
||||
raise AttachmentFailed()
|
||||
row = conn.execute(
|
||||
"""INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES (%s, %s, %s, %s, clock_timestamp())
|
||||
RETURNING id, chat_id, sender_id, text, created_at""",
|
||||
(uuid.uuid4(), chat_id, user_id, text)).fetchone()
|
||||
if attachment_ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""INSERT INTO message_attachments (message_id, attachment_id, position)
|
||||
VALUES (%s, %s, %s)""",
|
||||
[(row["id"], a, i) for i, a in enumerate(attachment_ids)])
|
||||
row = dict(row, attachment_ids=list(attachment_ids))
|
||||
conn.execute(
|
||||
"""INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id, expires_at)
|
||||
VALUES (%s, %s, %s, %s, %s, now() + make_interval(secs => %s))""",
|
||||
(user_id, chat_id, idem_key, request_hash, row["id"], idem_ttl_seconds))
|
||||
return _message(row), False
|
||||
|
||||
def list_messages(self, chat_id, after, limit):
|
||||
params = [chat_id]
|
||||
where = ""
|
||||
if after:
|
||||
where = "AND (m.created_at, m.id) > (%s, %s::uuid)"
|
||||
params += [from_micros(after[0]), after[1]]
|
||||
params.append(limit + 1)
|
||||
with self.pool.connection() as conn:
|
||||
rows = conn.execute(
|
||||
"SELECT " + MESSAGE_COLUMNS + " FROM messages m WHERE m.chat_id = %s "
|
||||
+ where + " ORDER BY m.created_at, m.id LIMIT %s", params).fetchall()
|
||||
return self._page(rows, limit, _message)
|
||||
|
||||
# --- вложения ---
|
||||
def create_pending_attachment(self, att_id, chat_id, owner_id, filename, content_type,
|
||||
size_bytes, sha256, width, height, object_key, staging_key):
|
||||
"""Шаг 1: намерение фиксируется ДО PUT в staging (как в lab4). Строка pending
|
||||
API не показывает; брошенную (крах до постановки в очередь) убирает janitor."""
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute(
|
||||
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes,
|
||||
sha256, width, height, object_key, staging_key, status, created_at, updated_at)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'pending', now(), now())""",
|
||||
(att_id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
width, height, object_key, staging_key))
|
||||
|
||||
def enqueue_attachment(self, att_id, pipeline_version, max_attempts, before_commit=None):
|
||||
"""Шаг 3 (после PUT в staging): ОДНА транзакция pending -> queued + jobs + outbox.
|
||||
После COMMIT работа принята: даже если API сразу упадёт, relay опубликует задание.
|
||||
None — строку уже убрал janitor (ответ 503)."""
|
||||
with self.pool.connection() as conn:
|
||||
job_id = jobs.enqueue(conn, att_id, pipeline_version, max_attempts)
|
||||
if job_id is None:
|
||||
return None
|
||||
if before_commit:
|
||||
before_commit()
|
||||
row = conn.execute("SELECT " + ATTACHMENT_COLUMNS + " FROM attachments a WHERE a.id = %s",
|
||||
(att_id,)).fetchone()
|
||||
return _attachment(row)
|
||||
|
||||
def delete_pending_attachment(self, att_id):
|
||||
"""Компенсация: убрать непринятое намерение (staging-объект удалён/не создан)."""
|
||||
with self.pool.connection() as conn:
|
||||
conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'pending'", (att_id,))
|
||||
|
||||
def get_attachment_for(self, user_id, att_id):
|
||||
"""(attachment, object_key) для участника чата; иначе None. Pending не виден."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"SELECT " + ATTACHMENT_COLUMNS + """ FROM attachments a
|
||||
WHERE a.id = %s AND a.status <> 'pending' AND EXISTS (
|
||||
SELECT 1 FROM chat_members m WHERE m.chat_id = a.chat_id AND m.user_id = %s)""",
|
||||
(att_id, user_id)).fetchone()
|
||||
return (_attachment(row), row["object_key"]) if row else None
|
||||
|
||||
def get_variant_key(self, att_id, name):
|
||||
"""object_key варианта текущей версии pipeline (или None)."""
|
||||
with self.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
"""SELECT v.object_key FROM attachment_variants v JOIN attachments a ON a.id = v.attachment_id
|
||||
WHERE v.attachment_id = %s AND v.name = %s AND v.pipeline_version = a.pipeline_version""",
|
||||
(att_id, name)).fetchone()
|
||||
return row["object_key"] if row else None
|
||||
|
||||
@staticmethod
|
||||
def _page(rows, limit, convert):
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
last = (to_micros(rows[-1]["created_at"]), str(rows[-1]["id"])) if rows and has_more else None
|
||||
return [convert(r) for r in rows], last
|
||||
@@ -0,0 +1,120 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Преобразование одного изображения в отдельном ПРОЦЕССЕ с жёсткими лимитами.
|
||||
|
||||
python transform.py <input> <output_dir> <content_type> <max_pixels>
|
||||
-> stdout: JSON {"variants": [...]} ; код выхода — класс ошибки.
|
||||
|
||||
Worker запускает его через subprocess с таймаутом JOB_TIMEOUT_SECONDS (wall clock), а сам
|
||||
процесс до импорта Pillow ограничивает себе адресное пространство (RLIMIT_AS) и CPU-время
|
||||
(RLIMIT_CPU). «Бомба», которую не поймали проверки заголовка, убивает только этот процесс,
|
||||
а не worker с его соединениями и арендами.
|
||||
|
||||
Коды выхода: 0 — успех; 3 — invalid_image; 4 — image_too_large; 5 — resource_limit
|
||||
(MemoryError). Убит сигналом (SIGXCPU/SIGKILL) -> resource_limit; таймаут — решает worker.
|
||||
|
||||
Варианты (contracts/README.md, «Фоновая обработка»):
|
||||
thumbnail — JPEG q=85, вписан в 256x256 с сохранением пропорций, без увеличения;
|
||||
размеры считаются целочисленно с округлением вниз, минимум 1 px
|
||||
(320x200 -> 256x160); EXIF orientation применяется; alpha -> на белый фон;
|
||||
crop — JPEG q=85, центрированный квадрат 128x128 (маленькие исходники увеличиваются);
|
||||
optimized — WebP q=80 method=4, вписан в 1280x1280 без увеличения; alpha сохраняется.
|
||||
Во все варианты НЕ переносятся EXIF/XMP/ICC/комментарии: Pillow пишет метаданные только
|
||||
если их явно передать (exif=, icc_profile=), мы их не передаём. Цвет считается sRGB.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
|
||||
MEMORY_LIMIT_MB = int(os.environ.get("TRANSFORM_MEMORY_LIMIT_MB", "768"))
|
||||
CPU_LIMIT_SECONDS = int(os.environ.get("TRANSFORM_CPU_SECONDS", "30"))
|
||||
THUMB = 256
|
||||
CROP = 128
|
||||
OPTIMIZED = 1280
|
||||
JPEG_QUALITY = 85
|
||||
WEBP_QUALITY = 80
|
||||
EXIT_INVALID, EXIT_TOO_LARGE, EXIT_RESOURCE = 3, 4, 5
|
||||
|
||||
|
||||
def fit_within(w, h, box):
|
||||
"""Вписать (w, h) в box x box без увеличения; целочисленно, вниз, минимум 1."""
|
||||
if w <= box and h <= box:
|
||||
return w, h
|
||||
if w >= h:
|
||||
return box, max(1, h * box // w)
|
||||
return max(1, w * box // h), box
|
||||
|
||||
|
||||
def fail(code, error, detail=""):
|
||||
print(json.dumps({"error": error, "detail": detail[:200]}))
|
||||
sys.stdout.flush()
|
||||
os._exit(code)
|
||||
|
||||
|
||||
def main():
|
||||
src, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
|
||||
limit = MEMORY_LIMIT_MB * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (CPU_LIMIT_SECONDS, CPU_LIMIT_SECONDS + 1))
|
||||
|
||||
import warnings
|
||||
from PIL import Image, ImageOps
|
||||
# Больше max_pixels — ошибка, а не предупреждение (по умолчанию Pillow только warn до 2x).
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
warnings.simplefilter("error", Image.DecompressionBombWarning)
|
||||
fmt = {"image/png": "PNG", "image/jpeg": "JPEG"}.get(content_type)
|
||||
if fmt is None:
|
||||
fail(EXIT_INVALID, "invalid_image", "unsupported content type")
|
||||
try:
|
||||
im = Image.open(src, formats=(fmt,))
|
||||
w0, h0 = im.size
|
||||
if w0 * h0 > max_pixels:
|
||||
fail(EXIT_TOO_LARGE, "image_too_large", "%dx%d" % (w0, h0))
|
||||
orientation = im.getexif().get(0x0112, 1)
|
||||
swap = orientation in (5, 6, 7, 8)
|
||||
ow, oh = (h0, w0) if swap else (w0, h0) # размеры ПОСЛЕ применения EXIF orientation
|
||||
if fmt == "JPEG":
|
||||
# draft: декодер JPEG сразу уменьшает в 2/4/8 раз, но не меньше 1280 по обеим
|
||||
# сторонам — памяти нужно в 4–64 раза меньше, качество вариантов не страдает.
|
||||
im.draft("RGB", (OPTIMIZED, OPTIMIZED))
|
||||
im.load()
|
||||
if orientation in range(2, 9):
|
||||
im = ImageOps.exif_transpose(im)
|
||||
has_alpha = im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info)
|
||||
work = im.convert("RGBA") if has_alpha else im.convert("RGB")
|
||||
del im
|
||||
# Промежуточная копия <= 1280: из неё делаются все варианты (экономия памяти и CPU).
|
||||
mw, mh = fit_within(ow, oh, OPTIMIZED)
|
||||
if work.size != (mw, mh):
|
||||
work = work.resize((mw, mh), Image.LANCZOS, reducing_gap=3.0)
|
||||
if has_alpha:
|
||||
flat = Image.new("RGB", work.size, (255, 255, 255))
|
||||
flat.paste(work, mask=work.getchannel("A"))
|
||||
else:
|
||||
flat = work
|
||||
variants = []
|
||||
tw, th = fit_within(ow, oh, THUMB)
|
||||
thumb = flat if flat.size == (tw, th) else flat.resize((tw, th), Image.LANCZOS)
|
||||
path = os.path.join(out_dir, "thumbnail.jpg")
|
||||
thumb.save(path, "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
variants.append({"name": "thumbnail", "path": path, "content_type": "image/jpeg", "width": tw, "height": th})
|
||||
crop = ImageOps.fit(flat, (CROP, CROP), method=Image.LANCZOS, centering=(0.5, 0.5))
|
||||
path = os.path.join(out_dir, "crop.jpg")
|
||||
crop.save(path, "JPEG", quality=JPEG_QUALITY, optimize=True)
|
||||
variants.append({"name": "crop", "path": path, "content_type": "image/jpeg", "width": CROP, "height": CROP})
|
||||
path = os.path.join(out_dir, "optimized.webp")
|
||||
work.save(path, "WEBP", quality=WEBP_QUALITY, method=4)
|
||||
variants.append({"name": "optimized", "path": path, "content_type": "image/webp", "width": mw, "height": mh})
|
||||
except MemoryError:
|
||||
fail(EXIT_RESOURCE, "resource_limit", "memory limit %d MiB" % MEMORY_LIMIT_MB)
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
|
||||
fail(EXIT_TOO_LARGE, "image_too_large", type(exc).__name__)
|
||||
except Exception as exc:
|
||||
# Битые/усечённые данные: OSError, SyntaxError, ValueError, struct.error...
|
||||
fail(EXIT_INVALID, "invalid_image", type(exc).__name__)
|
||||
print(json.dumps({"variants": variants, "source": [ow, oh]}))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+431
@@ -0,0 +1,431 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Worker обработки вложений (compose-сервис `worker`, отдельный процесс от API).
|
||||
|
||||
WORKER_CONCURRENCY потоков-потребителей; у каждого своё соединение AMQP, prefetch=1
|
||||
(не больше одного неподтверждённого сообщения на поток) и своё соединение PostgreSQL.
|
||||
Итого одновременно в работе не больше WORKER_CONCURRENCY заданий на контейнер.
|
||||
|
||||
Обработка одного сообщения (at-least-once, ack только после устойчивого эффекта):
|
||||
1. разбор по jobs.schema.json; мусор -> basic.reject(requeue=False) -> DLQ;
|
||||
2. claim в транзакции (src/jobs.py): по строке jobs решаем, нужна ли работа. ready/failed,
|
||||
чужая живая аренда, ранний дубль -> ack без работы. Иначе processing, attempt+1,
|
||||
lease_token (fencing) и lease_until, COMMIT;
|
||||
3. пока идёт работа, отдельный поток продлевает аренду (heartbeat) не дольше
|
||||
JOB_TIMEOUT_SECONDS: зависший процесс аренду теряет, долгий, но живой — нет;
|
||||
4. staging -> временный файл (лимит байт = size_bytes из БД, проверка SHA-256);
|
||||
5. transform.py в подпроцессе (RLIMIT_AS, RLIMIT_CPU, таймаут) -> thumbnail/crop/optimized;
|
||||
6. PUT оригинала и вариантов по ДЕТЕРМИНИРОВАННЫМ ключам attachments/<id>/v<ver>/<name>:
|
||||
повтор перезаписывает те же объекты, второго набора не появляется;
|
||||
7. complete: processing -> ready + attachment_variants одной транзакцией и только со своим
|
||||
lease_token; иначе (аренду забрали) — ничего не пишем;
|
||||
8. удаление staging (после COMMIT), затем ack. Крах до ack -> redelivery -> claim видит
|
||||
ready -> no-op ack.
|
||||
Ошибки: постоянные -> failed(error_code); временные -> queued с backoff+jitter через outbox
|
||||
(relay опубликует в срок) или failed(retry_exhausted). Нет доступа к БД -> сообщение
|
||||
возвращается в очередь (nack) после паузы; если так и не записалось — reaper/redrive.
|
||||
|
||||
Учебные отказы (WORKER_FAULT_INJECTION=1): filename вида fault-<point>.png задаёт точку:
|
||||
crash-after-claim, crash-after-s3-put, crash-after-commit (os._exit один раз на job),
|
||||
transient-<N> (временная ошибка в первых N попытках), transient-always (до replay),
|
||||
slow (пауза FAULT_SLOW_SECONDS после claim, аренда продлевается).
|
||||
"""
|
||||
import concurrent.futures
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import common
|
||||
import jobs
|
||||
import objstore
|
||||
from objstore import ObjectMissing, StorageUnavailable
|
||||
|
||||
log = logging.getLogger("worker")
|
||||
CONCURRENCY = common.env_int("WORKER_CONCURRENCY", 2)
|
||||
PREFETCH = common.env_int("WORKER_PREFETCH", 1)
|
||||
LEASE_SECONDS = common.env_int("JOB_LEASE_SECONDS", 30)
|
||||
JOB_TIMEOUT = common.env_float("JOB_TIMEOUT_SECONDS", 60)
|
||||
RETRY_BASE = common.env_float("JOB_RETRY_BASE_SECONDS", 2)
|
||||
RETRY_MAX = common.env_float("JOB_RETRY_MAX_SECONDS", 60)
|
||||
MAX_UPLOAD_BYTES = common.env_int("MAX_UPLOAD_BYTES", 10485760)
|
||||
MAX_IMAGE_PIXELS = common.env_int("MAX_IMAGE_PIXELS", 40000000)
|
||||
REQUEUE_PAUSE = common.env_float("WORKER_REQUEUE_PAUSE_SECONDS", 2)
|
||||
WORK_DIR = os.environ.get("WORKER_TMP_DIR") or None
|
||||
FAULT_INJECTION = os.environ.get("WORKER_FAULT_INJECTION", "0") == "1"
|
||||
FAULT_SLOW_SECONDS = common.env_float("FAULT_SLOW_SECONDS", 20)
|
||||
TRANSFORM = os.path.join(os.path.dirname(os.path.abspath(__file__)), "transform.py")
|
||||
HEARTBEAT = "/tmp/worker.heartbeat"
|
||||
EXT = {"image/jpeg": "jpg", "image/webp": "webp", "image/png": "png"}
|
||||
TRANSFORM_ERRORS = {3: "invalid_image", 4: "image_too_large", 5: "resource_limit"}
|
||||
|
||||
OBJ = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments"))
|
||||
STOP = threading.Event()
|
||||
|
||||
|
||||
class JobError(Exception):
|
||||
def __init__(self, code, detail=""):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
class LeaseLost(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Lease:
|
||||
"""Продление аренды отдельным соединением раз в LEASE_SECONDS/3, но не дольше
|
||||
JOB_TIMEOUT_SECONDS (+запас на загрузку в S3) от начала: после этого аренда истекает
|
||||
и reaper вернёт задание в очередь, даже если процесс жив, но завис."""
|
||||
|
||||
def __init__(self, conninfo, job):
|
||||
self.conninfo = conninfo
|
||||
self.job = job
|
||||
self.lost = False
|
||||
self.deadline = time.monotonic() + JOB_TIMEOUT + 30
|
||||
self._stop = threading.Event()
|
||||
self._thread = threading.Thread(target=self._loop, daemon=True)
|
||||
|
||||
def __enter__(self):
|
||||
self._thread.start()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
self._stop.set()
|
||||
self._thread.join(timeout=5)
|
||||
|
||||
def check(self, conn=None):
|
||||
"""Перед каждым побочным эффектом: флаг фонового heartbeat И синхронная проверка
|
||||
(продление) аренды в БД. Иначе замороженный процесс после разморозки успел бы
|
||||
сделать шаг до того, как фоновый поток заметит потерю аренды."""
|
||||
if self.lost:
|
||||
raise LeaseLost()
|
||||
if conn is not None:
|
||||
with conn.transaction():
|
||||
ok = jobs.heartbeat(conn, self.job["job_id"], self.job["lease_token"], LEASE_SECONDS)
|
||||
if not ok:
|
||||
self.lost = True
|
||||
raise LeaseLost()
|
||||
|
||||
def _loop(self):
|
||||
conn = None
|
||||
while not self._stop.wait(max(1.0, LEASE_SECONDS / 3)):
|
||||
if time.monotonic() > self.deadline:
|
||||
log.warning("job %s exceeded hard deadline, lease is no longer renewed", self.job["job_id"])
|
||||
return
|
||||
try:
|
||||
if conn is None or conn.closed:
|
||||
conn = psycopg.connect(self.conninfo, autocommit=True)
|
||||
if not jobs.heartbeat(conn, self.job["job_id"], self.job["lease_token"], LEASE_SECONDS):
|
||||
self.lost = True
|
||||
log.warning("lease lost for job %s", self.job["job_id"])
|
||||
return
|
||||
except psycopg.Error as exc:
|
||||
log.warning("lease heartbeat failed (%s)", type(exc).__name__)
|
||||
conn = None
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
class Consumer(threading.Thread):
|
||||
def __init__(self, index):
|
||||
super().__init__(name="consumer-%d" % index, daemon=True)
|
||||
self.conninfo = common.database_conninfo()
|
||||
self.db = None
|
||||
self.last_beat = time.monotonic()
|
||||
self.pool = concurrent.futures.ThreadPoolExecutor(max_workers=1)
|
||||
|
||||
# ------------------------------------------------------------------ AMQP
|
||||
def run(self):
|
||||
while not STOP.is_set():
|
||||
conn = common.connect_broker(log, STOP)
|
||||
if conn is None:
|
||||
return
|
||||
try:
|
||||
ch = conn.channel()
|
||||
ch.basic_qos(prefetch_count=PREFETCH)
|
||||
log.info("%s consuming %s owner=%s", self.name, common.QUEUE, common.OWNER)
|
||||
for method, _props, body in ch.consume(common.QUEUE, inactivity_timeout=1):
|
||||
self.last_beat = time.monotonic()
|
||||
if STOP.is_set():
|
||||
break
|
||||
if method is None:
|
||||
continue
|
||||
outcome = self.run_job(conn, body)
|
||||
if outcome == "ack":
|
||||
ch.basic_ack(method.delivery_tag)
|
||||
elif outcome == "reject":
|
||||
ch.basic_reject(method.delivery_tag, requeue=False) # -> DLX -> DLQ
|
||||
else:
|
||||
self.pause(conn, REQUEUE_PAUSE)
|
||||
ch.basic_nack(method.delivery_tag, requeue=True)
|
||||
ch.cancel()
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
log.warning("%s broker loop error (%s), reconnecting", self.name, type(exc).__name__)
|
||||
try:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
STOP.wait(1)
|
||||
|
||||
def pause(self, conn, seconds):
|
||||
end = time.monotonic() + seconds
|
||||
while time.monotonic() < end:
|
||||
conn.process_data_events(time_limit=min(0.5, end - time.monotonic()))
|
||||
self.last_beat = time.monotonic()
|
||||
|
||||
def run_job(self, conn, body):
|
||||
"""Работа в отдельном потоке; этот поток обслуживает heartbeat AMQP-соединения."""
|
||||
future = self.pool.submit(self.process_message, body)
|
||||
while not future.done():
|
||||
conn.process_data_events(time_limit=0.5)
|
||||
self.last_beat = time.monotonic()
|
||||
try:
|
||||
return future.result()
|
||||
except Exception:
|
||||
log.exception("unexpected worker error")
|
||||
return "requeue"
|
||||
|
||||
# ------------------------------------------------------------------ DB
|
||||
def database(self):
|
||||
if self.db is None or self.db.closed:
|
||||
self.db = psycopg.connect(self.conninfo, row_factory=dict_row)
|
||||
return self.db
|
||||
|
||||
def drop_db(self):
|
||||
try:
|
||||
if self.db is not None:
|
||||
self.db.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.db = None
|
||||
|
||||
# ------------------------------------------------------------------ job
|
||||
def process_message(self, body):
|
||||
try:
|
||||
msg = jobs.parse_message(body)
|
||||
except jobs.InvalidMessage as exc:
|
||||
log.warning("invalid message rejected to DLQ (%s)", exc)
|
||||
return "reject"
|
||||
try:
|
||||
db = self.database()
|
||||
with db.transaction():
|
||||
action, job = jobs.claim(db, msg, common.OWNER, LEASE_SECONDS)
|
||||
except psycopg.Error as exc:
|
||||
log.warning("claim failed, message requeued (%s)", type(exc).__name__)
|
||||
self.drop_db()
|
||||
return "requeue"
|
||||
if action == "reject":
|
||||
log.warning("message for job %s rejected to DLQ (%s)", msg["job_id"], job)
|
||||
return "reject"
|
||||
if action == "ack":
|
||||
return "ack"
|
||||
log.info("job %s attempt %d claimed (attachment %s)", job["job_id"], job["attempt"], job["attachment_id"])
|
||||
try:
|
||||
return self.execute(db, job)
|
||||
except psycopg.Error as exc:
|
||||
# Результат не записан; аренда истечёт, reaper вернёт задание. Сообщение — назад.
|
||||
log.warning("database error during job %s (%s)", job["job_id"], type(exc).__name__)
|
||||
self.drop_db()
|
||||
return "requeue"
|
||||
|
||||
def fault_point(self, job):
|
||||
if not FAULT_INJECTION:
|
||||
return None
|
||||
name = job["filename"] or ""
|
||||
if not name.startswith("fault-"):
|
||||
return None
|
||||
return name[len("fault-"):].rsplit(".", 1)[0]
|
||||
|
||||
def crash_once(self, db, job, point, wanted):
|
||||
if point != wanted:
|
||||
return
|
||||
with db.transaction():
|
||||
seen = db.execute(
|
||||
"""SELECT 1 FROM job_events WHERE job_id = %s AND event = 'fault_injected'
|
||||
AND detail->>'point' = %s""", (job["job_id"], point)).fetchone()
|
||||
if seen:
|
||||
return
|
||||
jobs.add_event(db, job["job_id"], "fault_injected", job["attempt"], common.OWNER, {"point": point})
|
||||
log.warning("FAULT INJECTION %s job=%s: process exits now", point, job["job_id"])
|
||||
os._exit(86)
|
||||
|
||||
def execute(self, db, job):
|
||||
point = self.fault_point(job)
|
||||
started = time.monotonic()
|
||||
with Lease(self.conninfo, job) as lease:
|
||||
try:
|
||||
self.crash_once(db, job, point, "crash-after-claim")
|
||||
if point == "slow" and job["attempt"] == 1:
|
||||
log.warning("FAULT INJECTION slow job=%s sleeping %.0fs", job["job_id"], FAULT_SLOW_SECONDS)
|
||||
time.sleep(FAULT_SLOW_SECONDS)
|
||||
if point and point.startswith("transient-"):
|
||||
n = point.split("-", 1)[1]
|
||||
if (n == "always" and job["replay_count"] == 0) or (n.isdigit() and job["attempt"] <= int(n)):
|
||||
raise JobError("storage_unavailable", "injected transient error")
|
||||
lease.check(db)
|
||||
variants = self.do_work(db, job, lease, started)
|
||||
self.crash_once(db, job, point, "crash-after-s3-put")
|
||||
lease.check(db)
|
||||
except LeaseLost:
|
||||
with db.transaction():
|
||||
jobs.add_event(db, job["job_id"], "aborted_lease_lost", job["attempt"], common.OWNER)
|
||||
return "ack"
|
||||
except JobError as err:
|
||||
with db.transaction():
|
||||
result, delay = jobs.fail_or_retry(db, job, err.code, err.detail, common.OWNER,
|
||||
RETRY_BASE, RETRY_MAX)
|
||||
log.warning("job %s attempt %d: %s (%s) -> %s%s", job["job_id"], job["attempt"], err.code,
|
||||
err.detail, result, " in %.1fs" % delay if delay else "")
|
||||
return "ack"
|
||||
with db.transaction():
|
||||
result = jobs.complete(db, job, variants, common.OWNER)
|
||||
self.crash_once(db, job, point, "crash-after-commit")
|
||||
log.info("job %s attempt %d -> %s in %.0fms", job["job_id"], job["attempt"], result,
|
||||
(time.monotonic() - started) * 1000)
|
||||
if result == "ready":
|
||||
self.cleanup_staging(db, job)
|
||||
return "ack"
|
||||
|
||||
def cleanup_staging(self, db, job):
|
||||
"""Staging удаляется только после COMMIT ready. Не вышло — доберёт janitor."""
|
||||
key = job["staging_key"]
|
||||
if not key:
|
||||
return
|
||||
try:
|
||||
OBJ.delete(key)
|
||||
except StorageUnavailable:
|
||||
log.warning("staging cleanup postponed for %s", job["attachment_id"])
|
||||
return
|
||||
with db.transaction():
|
||||
db.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'",
|
||||
(job["attachment_id"],))
|
||||
jobs.add_event(db, job["job_id"], "staging_deleted", job["attempt"], common.OWNER)
|
||||
|
||||
def do_work(self, db, job, lease, started):
|
||||
tmpdir = tempfile.mkdtemp(prefix="job-", dir=WORK_DIR)
|
||||
try:
|
||||
src = os.path.join(tmpdir, "source")
|
||||
self.fetch_staging(job, src)
|
||||
lease.check(db)
|
||||
produced = self.transform(job, src, tmpdir, started)
|
||||
lease.check(db) # последняя проверка перед записью в конечный prefix
|
||||
ver = job["pipeline_version"]
|
||||
base = "attachments/%s/v%d/" % (job["attachment_id"], ver)
|
||||
if job["object_key"] != base + "original":
|
||||
raise JobError("staging_corrupted", "unexpected object key")
|
||||
variants = []
|
||||
for v in produced:
|
||||
path = v["path"]
|
||||
variants.append({"name": v["name"], "key": base + "%s.%s" % (v["name"], EXT[v["content_type"]]),
|
||||
"content_type": v["content_type"], "width": v["width"], "height": v["height"],
|
||||
"size_bytes": os.path.getsize(path), "sha256": sha256_file(path), "path": path})
|
||||
try:
|
||||
# Сначала все объекты, потом COMMIT в БД: ready никогда не ссылается на
|
||||
# отсутствующий объект. Ключи детерминированы — повтор перезаписывает их же.
|
||||
with open(src, "rb") as f:
|
||||
OBJ.put(job["object_key"], f, job["size_bytes"], job["content_type"])
|
||||
for v in variants:
|
||||
with open(v["path"], "rb") as f:
|
||||
OBJ.put(v["key"], f, v["size_bytes"], v["content_type"])
|
||||
except StorageUnavailable:
|
||||
raise JobError("storage_unavailable", "put result")
|
||||
return variants
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
def fetch_staging(self, job, path):
|
||||
key = job["staging_key"]
|
||||
if not key:
|
||||
raise JobError("staging_missing", "staging already removed")
|
||||
limit = min(job["size_bytes"], MAX_UPLOAD_BYTES)
|
||||
try:
|
||||
resp = OBJ.open(key)
|
||||
except ObjectMissing:
|
||||
raise JobError("staging_missing", "no staging object")
|
||||
except StorageUnavailable:
|
||||
raise JobError("storage_unavailable", "get staging")
|
||||
h = hashlib.sha256()
|
||||
size = 0
|
||||
try:
|
||||
with open(path, "wb") as f:
|
||||
for chunk in resp.stream(objstore.STREAM_CHUNK):
|
||||
size += len(chunk)
|
||||
if size > limit:
|
||||
raise JobError("staging_corrupted", "staging larger than metadata")
|
||||
h.update(chunk)
|
||||
f.write(chunk)
|
||||
except JobError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise JobError("storage_unavailable", "read staging: %s" % type(exc).__name__)
|
||||
finally:
|
||||
resp.close()
|
||||
resp.release_conn()
|
||||
if size != job["size_bytes"] or h.hexdigest() != job["sha256"]:
|
||||
raise JobError("staging_corrupted", "size/sha256 differ from accepted upload")
|
||||
|
||||
def transform(self, job, src, tmpdir, started):
|
||||
out = os.path.join(tmpdir, "out")
|
||||
os.mkdir(out)
|
||||
left = max(1.0, JOB_TIMEOUT - (time.monotonic() - started))
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[sys.executable, TRANSFORM, src, out, job["content_type"], str(MAX_IMAGE_PIXELS)],
|
||||
capture_output=True, timeout=left, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise JobError("processing_timeout", "transform exceeded %.0fs" % left)
|
||||
if proc.returncode < 0:
|
||||
raise JobError("resource_limit", "transform killed by signal %d" % -proc.returncode)
|
||||
if proc.returncode in TRANSFORM_ERRORS:
|
||||
raise JobError(TRANSFORM_ERRORS[proc.returncode], proc.stdout.decode(errors="replace")[:200])
|
||||
if proc.returncode != 0:
|
||||
raise JobError("processing_failed", "transform exit %d" % proc.returncode)
|
||||
result = json.loads(proc.stdout)
|
||||
names = sorted(v["name"] for v in result["variants"])
|
||||
if names != sorted(jobs.VARIANT_NAMES):
|
||||
raise JobError("processing_failed", "unexpected variants")
|
||||
return result["variants"]
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGTERM, lambda *_: STOP.set())
|
||||
signal.signal(signal.SIGINT, lambda *_: STOP.set())
|
||||
log.info("worker starting owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%.0fs fault_injection=%s",
|
||||
common.OWNER, CONCURRENCY, PREFETCH, LEASE_SECONDS, JOB_TIMEOUT, FAULT_INJECTION)
|
||||
consumers = [Consumer(i) for i in range(CONCURRENCY)]
|
||||
for c in consumers:
|
||||
c.start()
|
||||
while not STOP.is_set():
|
||||
# healthcheck: файл обновляется, только пока все потребители живы и не зависли.
|
||||
if all(c.is_alive() and time.monotonic() - c.last_beat < 30 for c in consumers):
|
||||
common.touch(HEARTBEAT)
|
||||
STOP.wait(2)
|
||||
log.info("SIGTERM: finishing in-flight jobs")
|
||||
for c in consumers:
|
||||
c.join(timeout=JOB_TIMEOUT + 30)
|
||||
log.info("worker stopped")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload -> остановка ИМЕННО обслужившей upload реплики -> download через другую.
|
||||
|
||||
python3 student-tests/attachment_failover.py [--output evidence/attachment-failover.json]
|
||||
|
||||
Для каждой API-реплики по очереди: загрузить файл так, чтобы его обслужила эта реплика
|
||||
(round-robin LB, повторяем до попадания), остановить её scripts/stop-one.sh
|
||||
(SIGTERM, контейнер остаётся остановленным), затем 5 раз скачать оригинал через LB и
|
||||
проверить SHA-256 и что отвечает уже другая реплика. Восстановление — scripts/start-all.sh.
|
||||
Доказывает, что на API нет обязательного локального файла оригинала.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
CONTENT = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
instances = set()
|
||||
for _ in range(10):
|
||||
instances.add(c.call("GET", "/health/ready")["headers"]["X-Instance-Id"])
|
||||
check(len(instances) >= 2, "need >=2 replicas")
|
||||
rounds = []
|
||||
for target in sorted(instances):
|
||||
for _ in range(20): # round-robin: попадаем в нужную реплику за пару попыток
|
||||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=failover.png" % chat["id"], 202, actor=alice,
|
||||
raw=CONTENT, headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
if r["headers"]["X-Instance-Id"] == target:
|
||||
break
|
||||
else:
|
||||
raise SystemExit("could not hit replica " + target)
|
||||
att = c.wait_ready(alice, r["json"]) # lab5: 202 queued -> ждём ready, затем отказ реплики
|
||||
stop = subprocess.run([str(ROOT / "scripts/stop-one.sh")], env=dict(os.environ, TARGET_INSTANCE_ID=target),
|
||||
capture_output=True, text=True, timeout=60)
|
||||
check(stop.returncode == 0, "stop-one failed")
|
||||
downloads = []
|
||||
for _ in range(5):
|
||||
d = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)
|
||||
downloads.append({"instance": d["headers"]["X-Instance-Id"],
|
||||
"sha256_ok": hashlib.sha256(d["bytes"]).hexdigest() == att["sha256"] == hashlib.sha256(CONTENT).hexdigest(),
|
||||
"ms": round(d["elapsed"] * 1000, 1)})
|
||||
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
|
||||
ok = all(x["sha256_ok"] and x["instance"] != target for x in downloads) and meta["status"] == "ready"
|
||||
rounds.append({"uploaded_by": target, "stopped": target, "downloads": downloads, "pass": ok})
|
||||
print(json.dumps(rounds[-1]), flush=True)
|
||||
subprocess.run([str(ROOT / "scripts/start-all.sh")], capture_output=True, timeout=120, check=True)
|
||||
time.sleep(4) # HAProxy: rise 2 * inter 1s
|
||||
summary = {"replicas": sorted(instances), "rounds": rounds, "pass": all(r["pass"] for r in rounds)}
|
||||
print(json.dumps({"pass": summary["pass"], "rounds": len(rounds)}))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps(summary, indent=1))
|
||||
return 0 if summary["pass"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Негативные и граничные сценарии вложений (из lab4; в lab5 upload -> 202, затем ready). Запуск на учебном стенде:
|
||||
|
||||
python3 student-tests/attachments_negative.py [--base-url http://localhost:8080] [--output evidence/attachments-negative.json]
|
||||
|
||||
Использует публичный tests/client.py: каждый ответ проверяется на код, X-Request-Id,
|
||||
X-Instance-Id и (для ошибок) формат {"error":{code,message,request_id}}. Отдельно
|
||||
проверяется, что ни один пользовательский ввод не дал 5xx. Все файлы синтетические и
|
||||
генерируются на лету (большие в Git не хранятся).
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
JPEG = (ROOT / "student-tests/fixtures/sample.jpg").read_bytes()
|
||||
MAX_BYTES = 10 * 1024 * 1024
|
||||
results = []
|
||||
|
||||
|
||||
def chunk(ctype, data):
|
||||
return struct.pack(">I", len(data)) + ctype + data + struct.pack(">I", zlib.crc32(ctype + data) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def png(width, height, raw, color_type=2, depth=8):
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, depth, color_type, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", raw) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def noise_png(target_bytes):
|
||||
"""Несжимаемый RGB PNG размером чуть меньше target_bytes (zlib level 0)."""
|
||||
width = 1024
|
||||
row = 1 + width * 3
|
||||
height = (target_bytes - 200) // (row + 6 * 0 + 1)
|
||||
while True:
|
||||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||||
data = png(width, height, zlib.compress(raw, 0))
|
||||
if len(data) <= target_bytes:
|
||||
return data
|
||||
height -= 16
|
||||
|
||||
|
||||
class Suite:
|
||||
def __init__(self, c):
|
||||
self.c = c
|
||||
|
||||
def case(self, name, method, path, expected, actor=None, **kw):
|
||||
try:
|
||||
r = self.c.call(method, path, expected, actor=actor, **kw)
|
||||
code = r["json"]["error"]["code"] if r["json"] and "error" in r["json"] else None
|
||||
results.append({"case": name, "expected": expected, "status": r["status"], "error_code": code, "pass": True})
|
||||
return r
|
||||
except Failure as exc:
|
||||
results.append({"case": name, "expected": expected, "error": str(exc), "pass": False})
|
||||
return None
|
||||
|
||||
def upload(self, name, actor, chat, data, expected, ctype="image/png", filename="f.png", raw_query=None):
|
||||
q = raw_query if raw_query is not None else "filename=" + quote(filename, safe="")
|
||||
return self.case(name, "POST", "/api/v1/chats/%s/attachments?%s" % (chat["id"], q), expected,
|
||||
actor=actor, raw=data, headers={"Content-Type": ctype},
|
||||
schema="Attachment" if expected == 202 else None)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
c = Client(args.base_url, timeout=60)
|
||||
s = Suite(c)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
chat2 = c.call("POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "второй чат", "member_ids": [alice["id"], bob["id"]]}, schema="Chat")["json"]
|
||||
eve_chat = c.call("POST", "/api/v1/chats", 201, actor=eve,
|
||||
body={"title": "чат Евы", "member_ids": [eve["id"]]}, schema="Chat")["json"]
|
||||
|
||||
# --- позитивные контрольные ---
|
||||
ok_png = s.upload("valid PNG fixture", alice, chat, PNG, 202)
|
||||
ok_jpg = s.upload("valid JPEG", bob, chat, JPEG, 202, ctype="image/jpeg", filename="photo.jpg")
|
||||
uni = s.upload("unicode filename", alice, chat, PNG, 202, filename="фото отпуска 🙂.png")
|
||||
if uni:
|
||||
check(uni["json"]["filename"] == "фото отпуска 🙂.png", "unicode filename changed")
|
||||
def ready(r): # lab5: upload -> 202 queued, байты доступны после ready
|
||||
if r:
|
||||
c.wait_ready(alice if r["json"]["owner_id"] == alice["id"] else bob, r["json"])
|
||||
for r in (ok_png, ok_jpg):
|
||||
ready(r)
|
||||
if ok_jpg:
|
||||
d = c.call("GET", "/api/v1/attachments/%s/content" % ok_jpg["json"]["id"], actor=alice)
|
||||
check(d["bytes"] == JPEG and d["headers"].get_content_type() == "image/jpeg", "jpeg download mismatch")
|
||||
check(d["headers"].get("X-Content-Type-Options") == "nosniff", "nosniff missing")
|
||||
near = noise_png(MAX_BYTES)
|
||||
big = s.upload("PNG near 10 MiB limit (%d B)" % len(near), alice, chat, near, 202, filename="big.png")
|
||||
if big:
|
||||
check(big["json"]["sha256"] == hashlib.sha256(near).hexdigest(), "big sha mismatch")
|
||||
ready(big)
|
||||
d = c.call("GET", "/api/v1/attachments/%s/content" % big["json"]["id"], actor=bob)
|
||||
check(hashlib.sha256(d["bytes"]).hexdigest() == big["json"]["sha256"], "big download mismatch")
|
||||
|
||||
# --- размер и пиксели ---
|
||||
s.upload("too large: 10 MiB + 1 B", alice, chat, near + b"\0" * (MAX_BYTES + 1 - len(near)), 413)
|
||||
s.upload("pixel bomb header 100000x100000", alice, chat, png(100000, 100000, zlib.compress(b"\0" * 1000)), 413)
|
||||
s.upload("PNG 7000x6000 = 42 Mpx (> 40 Mpx)", alice, chat, png(7000, 6000, zlib.compress(b"\0" * 1000)), 413)
|
||||
s.upload("zlib bomb: 100x100 header, 50 MB inflated", alice, chat,
|
||||
png(100, 100, zlib.compress(b"\0" * 50_000_000, 9), color_type=0), 415)
|
||||
|
||||
# --- повреждённые изображения ---
|
||||
flipped = bytearray(PNG); flipped[5000] ^= 0xFF
|
||||
s.upload("corrupted PNG (byte flipped, bad CRC)", alice, chat, bytes(flipped), 415)
|
||||
s.upload("truncated PNG", alice, chat, PNG[:-4000], 415)
|
||||
s.upload("PNG without IEND", alice, chat, PNG[:-12], 415)
|
||||
s.upload("PNG with appended ZIP (polyglot)", alice, chat, PNG + b"PK\x03\x04" + os.urandom(64), 415)
|
||||
s.upload("truncated JPEG", alice, chat, JPEG[:len(JPEG) // 2], 415, ctype="image/jpeg")
|
||||
s.upload("JPEG header only", alice, chat, JPEG[:200], 415, ctype="image/jpeg")
|
||||
s.upload("empty body", alice, chat, b"", 415)
|
||||
|
||||
# --- подмена MIME ---
|
||||
s.upload("JPEG bytes declared image/png", alice, chat, JPEG, 415)
|
||||
s.upload("PNG bytes declared image/jpeg", alice, chat, PNG, 415, ctype="image/jpeg")
|
||||
s.upload("text declared image/png", alice, chat, b"<html><script>alert(1)</script></html>" * 4, 415)
|
||||
s.upload("GIF declared image/png", alice, chat, b"GIF89a" + os.urandom(200), 415)
|
||||
s.upload("PNG declared text/html", alice, chat, PNG, 415, ctype="text/html")
|
||||
s.upload("PNG declared image/svg+xml", alice, chat, PNG, 415, ctype="image/svg+xml")
|
||||
s.upload("multipart/form-data instead of raw", alice, chat, b"--x\r\n\r\n" + PNG + b"\r\n--x--", 415,
|
||||
ctype="multipart/form-data; boundary=x")
|
||||
|
||||
# --- опасные filename ---
|
||||
for label, query in (("../etc/passwd", "filename=" + quote("../passwd.png", safe="")),
|
||||
("slash", "filename=" + quote("a/b.png", safe="")),
|
||||
("backslash", "filename=" + quote("a\\b.png", safe="")),
|
||||
("NUL", "filename=a%00.png"),
|
||||
("newline", "filename=a%0A.png"),
|
||||
("bidi override U+202E", "filename=" + quote("gnp.exe", safe="")),
|
||||
("..", "filename=.."),
|
||||
("empty", "filename="),
|
||||
("whitespace only", "filename=%20%20"),
|
||||
("256 code points", "filename=" + quote("я" * 256, safe="")),
|
||||
("missing filename", ""),
|
||||
("filename twice", "filename=a.png&filename=b.png"),
|
||||
("invalid UTF-8", "filename=%FF%FE.png")):
|
||||
s.upload("filename: " + label, alice, chat, PNG, 400, raw_query=query)
|
||||
|
||||
# --- доступ и чужие объекты ---
|
||||
eve_att = s.upload("eve uploads to her own chat", eve, eve_chat, PNG, 202)
|
||||
s.upload("non-member uploads to alice chat", eve, chat, PNG, 404)
|
||||
s.case("upload without session", "POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 401,
|
||||
raw=PNG, headers={"Content-Type": "image/png"})
|
||||
s.upload("upload to unknown chat", alice, {"id": str(uuid.uuid4())}, PNG, 404)
|
||||
if eve_att and ok_png:
|
||||
eid, aid = eve_att["json"]["id"], ok_png["json"]["id"]
|
||||
s.case("foreign attachment metadata", "GET", "/api/v1/attachments/" + eid, 404, actor=alice)
|
||||
s.case("foreign attachment content", "GET", "/api/v1/attachments/%s/content" % eid, 404, actor=alice)
|
||||
s.case("eve reads alice attachment", "GET", "/api/v1/attachments/%s/content" % aid, 404, actor=eve)
|
||||
key = lambda: {"Idempotency-Key": uuid.uuid4().hex} # noqa: E731
|
||||
s.case("message with foreign attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404,
|
||||
actor=alice, body={"text": "x", "attachment_ids": [eid]}, headers=key())
|
||||
s.case("message mixing own and foreign ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404,
|
||||
actor=alice, body={"text": "x", "attachment_ids": [aid, eid]}, headers=key())
|
||||
s.case("attachment of OTHER chat of the same user", "POST", "/api/v1/chats/%s/messages" % chat2["id"], 404,
|
||||
actor=alice, body={"text": "x", "attachment_ids": [aid]}, headers=key())
|
||||
s.case("unknown attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 404, actor=alice,
|
||||
body={"text": "x", "attachment_ids": [str(uuid.uuid4())]}, headers=key())
|
||||
s.case("malformed attachment_id", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||||
body={"text": "x", "attachment_ids": ["not-a-uuid"]}, headers=key())
|
||||
s.case("duplicate attachment_ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||||
body={"text": "x", "attachment_ids": [aid, aid]}, headers=key())
|
||||
s.case("11 attachment_ids", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||||
body={"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, headers=key())
|
||||
s.case("attachment_ids not a list", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice,
|
||||
body={"text": "x", "attachment_ids": aid}, headers=key())
|
||||
s.case("empty text without attachments", "POST", "/api/v1/chats/%s/messages" % chat["id"], 400,
|
||||
actor=alice, body={"text": " "}, headers=key())
|
||||
m = s.case("empty text with attachment (bob links alice's file)", "POST",
|
||||
"/api/v1/chats/%s/messages" % chat["id"], 201, actor=bob,
|
||||
body={"text": "", "attachment_ids": [aid]}, headers=key(), schema="Message")
|
||||
if m:
|
||||
check(m["json"]["attachment_ids"] == [aid], "attachment_ids lost")
|
||||
k = key()
|
||||
s.case("idempotent send with attachments", "POST", "/api/v1/chats/%s/messages" % chat["id"], 201,
|
||||
actor=alice, body={"attachment_ids": [aid], "text": "повтор"}, headers=k, schema="Message")
|
||||
s.case("same key, same body (key order changed)", "POST", "/api/v1/chats/%s/messages" % chat["id"], 201,
|
||||
actor=alice, body={"text": "повтор", "attachment_ids": [aid]}, headers=k, schema="Message")
|
||||
s.case("same key, different attachment_ids -> 409", "POST", "/api/v1/chats/%s/messages" % chat["id"], 409,
|
||||
actor=alice, body={"text": "повтор", "attachment_ids": []}, headers=k)
|
||||
s.case("variant=thumbnail (ready, lab5)", "GET",
|
||||
"/api/v1/attachments/%s/content?variant=thumbnail" % aid, 200, actor=bob)
|
||||
s.case("variant=bogus", "GET", "/api/v1/attachments/%s/content?variant=bogus" % aid, 400, actor=bob)
|
||||
s.case("variant=original", "GET", "/api/v1/attachments/%s/content?variant=original" % aid, 200, actor=bob)
|
||||
s.case("malformed attachment id in path", "GET", "/api/v1/attachments/zzz", 400, actor=bob)
|
||||
s.case("unknown attachment id", "GET", "/api/v1/attachments/" + str(uuid.uuid4()), 404, actor=bob)
|
||||
|
||||
failed = [r for r in results if not r["pass"]]
|
||||
server_errors = [r for r in results if r.get("status", 0) >= 500]
|
||||
for r in results:
|
||||
print(("PASS " if r["pass"] else "FAIL ") + "%-55s -> %s %s" % (
|
||||
r["case"], r.get("status", "-"), r.get("error_code") or r.get("error", "")))
|
||||
summary = {"cases": len(results), "failed": len(failed), "server_errors_5xx": len(server_errors),
|
||||
"pass": not failed and not server_errors}
|
||||
print(json.dumps(summary, ensure_ascii=False))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps({"summary": summary, "results": results}, ensure_ascii=False, indent=1))
|
||||
return 0 if summary["pass"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Права пользователей RabbitMQ (минимальные привилегии). Хост-оркестратор:
|
||||
|
||||
python3 student-tests/broker_permissions.py --output evidence/broker-permissions.json
|
||||
|
||||
Внутри контейнеров relay, worker и ops (каждый — со СВОИМ BROKER_URL_FILE) выполняется
|
||||
проба (--probe): publish в exchange attachments (с mandatory и неизвестным routing key —
|
||||
сообщение никуда не попадает, но право write проверяется), basic.get из рабочей очереди и
|
||||
из DLQ (право read), queue.declare (право configure). Ожидаемая матрица:
|
||||
relay — publish; worker — read рабочей очереди; ops — publish + read DLQ; configure — никто.
|
||||
Также: пользователя guest нет; порты брокера на хост не опубликованы.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
EXPECT = {
|
||||
"relay": {"publish": True, "get_work_queue": False, "get_dlq": False, "declare_queue": False},
|
||||
"worker": {"publish": False, "get_work_queue": True, "get_dlq": False, "declare_queue": False},
|
||||
"ops": {"publish": True, "get_work_queue": False, "get_dlq": True, "declare_queue": False},
|
||||
}
|
||||
|
||||
|
||||
def probe():
|
||||
sys.path.insert(0, "/app/src")
|
||||
import pika
|
||||
import common
|
||||
out = {}
|
||||
conn = pika.BlockingConnection(common.broker_parameters())
|
||||
|
||||
def attempt(name, fn):
|
||||
ch = conn.channel()
|
||||
try:
|
||||
fn(ch)
|
||||
out[name] = True
|
||||
except pika.exceptions.UnroutableError:
|
||||
out[name] = True # write разрешён, просто нет binding
|
||||
except pika.exceptions.ChannelClosedByBroker as exc:
|
||||
out[name] = False if exc.reply_code == 403 else "error %d" % exc.reply_code
|
||||
finally:
|
||||
if ch.is_open:
|
||||
ch.close()
|
||||
|
||||
def publish(ch):
|
||||
ch.confirm_delivery()
|
||||
ch.basic_publish(common.EXCHANGE, "permission.check.unbound", b"{}", mandatory=True)
|
||||
|
||||
def get(queue):
|
||||
def fn(ch):
|
||||
m, _p, _b = ch.basic_get(queue, auto_ack=False)
|
||||
if m is not None:
|
||||
ch.basic_nack(m.delivery_tag, requeue=True)
|
||||
return fn
|
||||
|
||||
attempt("publish", publish)
|
||||
attempt("get_work_queue", get(common.QUEUE))
|
||||
attempt("get_dlq", get(common.DLQ))
|
||||
attempt("declare_queue", lambda ch: ch.queue_declare("evil-queue", durable=True))
|
||||
conn.close()
|
||||
print(json.dumps(out))
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--probe", action="store_true")
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
if args.probe:
|
||||
return probe()
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
script = (root / "student-tests/broker_permissions.py").read_text()
|
||||
results, ok = {}, True
|
||||
runners = {"relay": ["docker", "compose", "exec", "-T", "relay", "python", "-", "--probe"],
|
||||
"worker": ["docker", "compose", "exec", "-T", "worker", "python", "-", "--probe"],
|
||||
"ops": ["docker", "compose", "--profile", "tools", "run", "--rm", "-T", "--entrypoint", "python",
|
||||
"ops", "-", "--probe"]}
|
||||
for user, cmd in runners.items():
|
||||
r = subprocess.run(cmd, input=script, capture_output=True, text=True, cwd=root, timeout=120)
|
||||
got = json.loads(r.stdout.strip().splitlines()[-1])
|
||||
results[user] = {"got": got, "expected": EXPECT[user], "pass": got == EXPECT[user]}
|
||||
ok &= got == EXPECT[user]
|
||||
print("%s %-6s %s" % ("PASS" if got == EXPECT[user] else "FAIL", user, got))
|
||||
users = subprocess.run(["docker", "compose", "exec", "-T", "broker", "rabbitmqctl", "-q", "list_users"],
|
||||
capture_output=True, text=True, cwd=root).stdout.split()
|
||||
ports = subprocess.run(["docker", "compose", "port", "broker", "5672"], capture_output=True, text=True, cwd=root)
|
||||
results["guest_absent"] = "guest" not in users
|
||||
results["broker_port_published_to_host"] = bool(ports.stdout.strip())
|
||||
ok &= results["guest_absent"] and not results["broker_port_published_to_host"]
|
||||
print("guest absent:", results["guest_absent"], "| port 5672 on host:", results["broker_port_published_to_host"])
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps(results, ensure_ascii=False, indent=1) + "\n")
|
||||
print("PASS" if ok else "FAIL")
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main() or 0)
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: ограниченная конкуренция/prefetch, очистка staging, память worker под всплеском.
|
||||
|
||||
python3 student-tests/burst.py --count 24 --output evidence/burst.json
|
||||
|
||||
N несжимаемых PNG 1500x1500 (~6.8 MB) загружаются 8 потоками. Во время обработки раз в
|
||||
секунду снимается память контейнеров worker (docker stats). По job_events считаются:
|
||||
максимальное число одновременно обрабатываемых заданий (claimed..ready), задержка очереди
|
||||
(enqueued -> claimed) и время до ready. Ожидание: одновременно <= WORKER_REPLICAS x
|
||||
WORKER_CONCURRENCY (prefetch=1 на потребителя), staging всех заданий удалён.
|
||||
Иллюстрация на одной машине, не статистика.
|
||||
"""
|
||||
import argparse
|
||||
import concurrent.futures
|
||||
import datetime as dt
|
||||
import json
|
||||
import statistics
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
|
||||
from lab5lib import Client, Log, check, compose, noise_png_fast, queues, sql, upload, wait_status
|
||||
|
||||
|
||||
def ts(text):
|
||||
return dt.datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def mem_sampler(stop, samples):
|
||||
while not stop.is_set():
|
||||
ids = compose("ps", "-q", "worker").stdout.split()
|
||||
out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}} {{.CPUPerc}}"]
|
||||
+ ids, capture_output=True, text=True).stdout
|
||||
for line in out.splitlines():
|
||||
name, used, _slash, limit, cpu = line.split()[:5]
|
||||
samples.append({"name": name, "used": used, "limit": limit, "cpu": cpu})
|
||||
stop.wait(1)
|
||||
|
||||
|
||||
def mib(text):
|
||||
num = float("".join(ch for ch in text if ch.isdigit() or ch == "."))
|
||||
return num / 1024 if text.endswith("KiB") else num * 1024 if text.endswith("GiB") else num
|
||||
|
||||
|
||||
def pct(values, p):
|
||||
values = sorted(values)
|
||||
return round(values[min(len(values) - 1, int(round(p / 100 * (len(values) - 1))))], 3)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--count", type=int, default=24)
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
log = Log("burst of uploads: bounded worker concurrency")
|
||||
c = Client(args.base_url, timeout=60)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
consumers = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "-p", "messenger", "list_consumers",
|
||||
"queue_name", "prefetch_count", "ack_required", "--formatter", "json").stdout
|
||||
consumers = json.loads(consumers)
|
||||
log.step("consumers", count=len(consumers), prefetch=sorted({x["prefetch_count"] for x in consumers}))
|
||||
images = [noise_png_fast(1500, 1500) for _ in range(4)]
|
||||
log.step("generated images", bytes=[len(x) for x in images])
|
||||
stop = threading.Event()
|
||||
samples = []
|
||||
sampler = threading.Thread(target=mem_sampler, args=(stop, samples), daemon=True)
|
||||
sampler.start()
|
||||
t0 = time.monotonic()
|
||||
with concurrent.futures.ThreadPoolExecutor(8) as pool:
|
||||
atts = list(pool.map(lambda i: upload(c, alice, chat, images[i % 4], "burst-%d.png" % i),
|
||||
range(args.count)))
|
||||
t_up = time.monotonic() - t0
|
||||
q = queues()["attachments.process.v1"]
|
||||
log.step("all uploads accepted (202)", count=len(atts), seconds=round(t_up, 2), queue_right_after=q)
|
||||
for a in atts:
|
||||
st = wait_status(c, bob, a["id"], {"ready", "failed"}, timeout=180, interval=1)
|
||||
check(st["status"] == "ready", "burst job failed")
|
||||
total = time.monotonic() - t0
|
||||
stop.set()
|
||||
sampler.join()
|
||||
ids = ",".join("'%s'" % a["id"] for a in atts)
|
||||
rows = sql("""SELECT j.attachment_id, e.event, to_char(e.at AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS.US"Z"')
|
||||
FROM job_events e JOIN jobs j ON j.id = e.job_id
|
||||
WHERE j.attachment_id IN (%s) AND e.event IN ('enqueued','claimed','ready') ORDER BY e.id""" % ids)
|
||||
per = {}
|
||||
for att_id, ev, at in rows:
|
||||
per.setdefault(att_id, {})[ev] = ts(at)
|
||||
intervals = [(v["claimed"], v["ready"]) for v in per.values()]
|
||||
points = sorted([(a, 1) for a, _ in intervals] + [(b, -1) for _, b in intervals], key=lambda x: (x[0], x[1]))
|
||||
cur = peak = 0
|
||||
for _, d in points:
|
||||
cur += d
|
||||
peak = max(peak, cur)
|
||||
lag = [(v["claimed"] - v["enqueued"]).total_seconds() for v in per.values()]
|
||||
to_ready = [(v["ready"] - v["enqueued"]).total_seconds() for v in per.values()]
|
||||
work = [(v["ready"] - v["claimed"]).total_seconds() for v in per.values()]
|
||||
staging_left = sql("SELECT count(*) FROM attachments WHERE id IN (%s) AND staging_key IS NOT NULL" % ids)[0][0]
|
||||
limit = len(consumers)
|
||||
by_name = {}
|
||||
for s in samples:
|
||||
by_name[s["name"]] = max(by_name.get(s["name"], 0), mib(s["used"]))
|
||||
result = {
|
||||
"count": args.count, "image_bytes": len(images[0]), "upload_seconds": round(t_up, 2),
|
||||
"total_seconds_to_all_ready": round(total, 2),
|
||||
"throughput_jobs_per_s": round(args.count / total, 2),
|
||||
"consumers": limit, "prefetch_per_consumer": sorted({x["prefetch_count"] for x in consumers}),
|
||||
"max_concurrent_processing": peak,
|
||||
"queue_lag_s": {"p50": pct(lag, 50), "p95": pct(lag, 95), "max": round(max(lag), 3)},
|
||||
"enqueued_to_ready_s": {"p50": pct(to_ready, 50), "p95": pct(to_ready, 95), "max": round(max(to_ready), 3)},
|
||||
"processing_s": {"p50": pct(work, 50), "max": round(max(work), 3), "mean": round(statistics.mean(work), 3)},
|
||||
"worker_peak_mem_mib": {k: round(v, 1) for k, v in by_name.items()},
|
||||
"worker_mem_limit": samples[0]["limit"] if samples else None,
|
||||
"staging_left": int(staging_left),
|
||||
}
|
||||
check(peak <= limit, "concurrency exceeded consumers x prefetch")
|
||||
check(result["staging_left"] == 0, "staging not cleaned")
|
||||
log.step("result", **result)
|
||||
log.save(args.output, result=result)
|
||||
print("PASS: %d jobs, max concurrent %d <= %d consumers" % (args.count, peak, limit))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,179 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Отказы между записью в S3 и PostgreSQL и очистка janitor (уровень 4).
|
||||
|
||||
Требует API с FAULT_INJECTION=1 (учебный хук, по умолчанию выключен):
|
||||
FAULT_INJECTION=1 docker compose --env-file .env up -d --no-deps --wait api
|
||||
python3 student-tests/consistency_faults.py --output evidence/consistency-faults.json
|
||||
docker compose --env-file .env up -d --no-deps --wait api # вернуть FAULT_INJECTION=0
|
||||
|
||||
Сценарии (каждый — реальный HTTP upload через LB):
|
||||
db-fail-after-s3-put PUT прошёл, «COMMIT» ready упал -> 503; компенсация удаляет объект
|
||||
и pending-строку -> ничего не остаётся;
|
||||
crash-after-s3-put процесс API умирает (os._exit) после PUT -> pending + объект;
|
||||
crash-after-pending процесс умирает до PUT -> pending без объекта;
|
||||
обрыв клиента Content-Length 5 MiB, отправлен 1 MiB, сокет закрыт -> ничего;
|
||||
orphan pending-строку удаляем вручную -> объект без строки;
|
||||
missing объект ready-вложения удаляем в S3 -> только отчёт, download = 503.
|
||||
Затем janitor: dry-run и боевой проход с grace 3600 (свежее не трогает), ожидание
|
||||
grace-seconds, dry-run -> боевой -> повторный (идемпотентность). Контрольное ready-вложение
|
||||
должно остаться доступным побайтно.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check # noqa: E402
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
log = []
|
||||
|
||||
|
||||
def sh(*cmd, timeout=120):
|
||||
r = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, timeout=timeout)
|
||||
if r.returncode != 0:
|
||||
raise SystemExit("command failed: %s\n%s" % (" ".join(cmd[:4]), r.stderr[-500:]))
|
||||
return r.stdout
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return sh("docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tAc "$0"', sql).strip()
|
||||
|
||||
|
||||
def janitor(grace, dry_run):
|
||||
cmd = ["docker", "compose", "run", "--rm", "--no-deps", "janitor", "python", "/app/src/janitor.py",
|
||||
"--once", "--grace-seconds", str(grace)] + (["--dry-run"] if dry_run else [])
|
||||
out = sh(*cmd)
|
||||
report = json.loads([line for line in out.splitlines() if line.startswith("{")][-1])
|
||||
keep = ("dry_run", "grace_seconds", "rows_by_status", "objects_total", "abandoned_pending",
|
||||
"orphan_objects", "missing_objects", "deleted_objects", "deleted_rows", "errors", "duration_ms")
|
||||
return {k: report.get(k) for k in keep}
|
||||
|
||||
|
||||
def step(name, **data):
|
||||
entry = dict(step=name, t=round(time.time(), 1), **data)
|
||||
log.append(entry)
|
||||
print(json.dumps(entry, ensure_ascii=False), flush=True)
|
||||
return entry
|
||||
|
||||
|
||||
def wait_api_healthy():
|
||||
deadline = time.time() + 90
|
||||
while time.time() < deadline:
|
||||
states = sh("docker", "compose", "ps", "-a", "--format", "{{.Health}}", "api").split()
|
||||
if states and all(s == "healthy" for s in states):
|
||||
time.sleep(3) # HAProxy rise
|
||||
return
|
||||
time.sleep(1)
|
||||
raise SystemExit("api did not become healthy")
|
||||
|
||||
|
||||
def upload(c, actor, chat, fault=None, expected=201):
|
||||
h = {"Content-Type": "image/png"}
|
||||
if fault:
|
||||
h["X-Fault-Inject"] = fault
|
||||
try:
|
||||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=f.png" % chat["id"], expected,
|
||||
actor=actor, raw=PNG, headers=h, schema="Attachment" if expected == 201 else None)
|
||||
return r["status"], r["json"]
|
||||
except Exception as exc: # обрыв соединения при crash-* тоже допустимый исход
|
||||
return type(exc).__name__, str(exc)[:120]
|
||||
|
||||
|
||||
def interrupted_upload(base, token, chat_id):
|
||||
host, port = base.split("//")[1].split(":")
|
||||
s = socket.create_connection((host, int(port)), timeout=10)
|
||||
s.sendall(("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: %s\r\n"
|
||||
"Authorization: Bearer %s\r\nContent-Type: image/png\r\nContent-Length: %d\r\n\r\n"
|
||||
% (chat_id, host, token, 5 * 1024 * 1024)).encode())
|
||||
s.sendall(PNG[:8] + b"\0" * (1024 * 1024 - 8))
|
||||
s.close() # клиент «пропал» на середине
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--grace-seconds", type=int, default=20)
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
env = sh("docker", "compose", "exec", "-T", "api", "printenv", "FAULT_INJECTION").strip()
|
||||
check(env == "1", "start api with FAULT_INJECTION=1 first (see docstring)")
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
started = psql("SELECT now()")
|
||||
base = janitor(3600, True)
|
||||
step("baseline janitor dry-run", report=base)
|
||||
|
||||
_, control = upload(c, alice, chat)
|
||||
_, victim = upload(c, alice, chat)
|
||||
step("control + victim uploaded (ready)", control=control["id"], victim=victim["id"])
|
||||
|
||||
st, body = upload(c, alice, chat, "db-fail-after-s3-put", expected=503)
|
||||
after_comp = janitor(3600, True)
|
||||
step("db-fail-after-s3-put -> compensation", status=st, report=after_comp,
|
||||
leftover_pending=after_comp["rows_by_status"].get("pending", 0) - base["rows_by_status"].get("pending", 0),
|
||||
leftover_objects=after_comp["objects_total"] - base["objects_total"] - 2)
|
||||
|
||||
for fault in ("crash-after-s3-put", "crash-after-s3-put", "crash-after-pending"):
|
||||
st, body = upload(c, alice, chat, fault, expected=[502, 503])
|
||||
step(fault, client_saw=st)
|
||||
wait_api_healthy()
|
||||
|
||||
interrupted_upload(args.base_url, alice["token"], chat["id"])
|
||||
time.sleep(2)
|
||||
step("client aborted upload at 1 of 5 MiB", report=janitor(3600, True))
|
||||
|
||||
pending = psql("SELECT id || ' ' || (object_key IS NOT NULL) FROM attachments WHERE status='pending' "
|
||||
"AND created_at >= '%s' ORDER BY created_at" % started).splitlines()
|
||||
step("pending rows left by crashes", pending=len(pending))
|
||||
check(len(pending) == 3, "expected 3 pending rows from crashes")
|
||||
orphan_id = pending[0].split()[0]
|
||||
psql("DELETE FROM attachments WHERE id = '%s'" % orphan_id)
|
||||
step("orphan injected: pending row deleted, object kept", row=orphan_id)
|
||||
victim_key = psql("SELECT object_key FROM attachments WHERE id = '%s'" % victim["id"])
|
||||
sh("docker", "compose", "--profile", "tools", "run", "--rm", "--entrypoint", "python", "s3-check", "-c",
|
||||
"import sys; sys.argv=['x']; exec(open('/app/student-tests/s3_private_check.py').read().split('results = []')[0]);"
|
||||
"client('api').remove_object(BUCKET, %r)" % victim_key)
|
||||
dl = c.call("GET", "/api/v1/attachments/%s/content" % victim["id"], 503, actor=bob)
|
||||
step("missing injected: victim object deleted in S3", download_status=dl["status"])
|
||||
|
||||
fresh_dry = janitor(3600, True)
|
||||
fresh_run = janitor(3600, False)
|
||||
step("janitor grace=3600 (all fresh): dry-run", report=fresh_dry)
|
||||
step("janitor grace=3600 (all fresh): real run deletes nothing", report=fresh_run)
|
||||
check(fresh_run["deleted_objects"] == 0 and fresh_run["deleted_rows"] == 0, "fresh data deleted")
|
||||
|
||||
time.sleep(args.grace_seconds + 2)
|
||||
dry = janitor(args.grace_seconds, True)
|
||||
step("after grace: dry-run", report=dry)
|
||||
real = janitor(args.grace_seconds, False)
|
||||
step("after grace: real run", report=real)
|
||||
again = janitor(args.grace_seconds, False)
|
||||
step("after grace: repeated run (idempotent)", report=again)
|
||||
d = c.call("GET", "/api/v1/attachments/%s/content" % control["id"], actor=bob)
|
||||
control_ok = hashlib.sha256(d["bytes"]).hexdigest() == control["sha256"]
|
||||
still = psql("SELECT count(*) FROM attachments WHERE status='pending' AND created_at >= '%s'" % started)
|
||||
step("control attachment still intact", sha256_ok=control_ok, pending_left=int(still))
|
||||
|
||||
ok = (log[2]["leftover_pending"] == 0 and log[2]["leftover_objects"] == 0
|
||||
and dry["abandoned_pending"] >= 2 and dry["orphan_objects"] >= 1 and dry["missing_objects"] >= 1
|
||||
and dry["deleted_objects"] == 0
|
||||
and real["deleted_rows"] >= 2 and again["abandoned_pending"] == 0 and again["orphan_objects"] == 0
|
||||
and again["deleted_objects"] == 0 and again["missing_objects"] >= 1
|
||||
and control_ok and int(still) == 0)
|
||||
print(json.dumps({"pass": ok}))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps({"pass": ok, "steps": log}, ensure_ascii=False, indent=1))
|
||||
return 0 if ok else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка диапазона дат cursor: некорректная дата должна давать 400, а не 500."""
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client, arguments, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments(__doc__)
|
||||
client = Client(args.base_url, ca_file=args.ca_file)
|
||||
alice, _bob, _eve, chat = client.bootstrap()
|
||||
checks = 0
|
||||
for kind, scope, path in (
|
||||
("messages", chat["id"], "/api/v1/chats/" + chat["id"] + "/messages"),
|
||||
("chats", alice["id"], "/api/v1/chats"),
|
||||
):
|
||||
for timestamp, expected in ((-10**17, 400), (-62135596800000001, 400), (0, 200)):
|
||||
payload = {"k": kind, "s": scope, "t": timestamp, "i": str(uuid.uuid4())}
|
||||
cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=")
|
||||
client.call("GET", path + "?cursor=" + quote(cursor, safe=""), expected, actor=alice)
|
||||
checks += 1
|
||||
print("PASS: %d cursor date boundary checks; invalid dates return 400." % checks)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Отказ/замена API-реплики ПОД НАГРУЗКОЙ: RPS, ошибки, время обнаружения и восстановления (lab3, уровень 4).
|
||||
|
||||
Что делает:
|
||||
* CONCURRENCY потоков в closed-loop шлют смесь запросов через LB: POST сообщения
|
||||
(с Idempotency-Key) и GET истории (limit=20) в пропорции 1:1;
|
||||
* после неуспешного/неопределённого POST поток повторяет ТОТ ЖЕ ключ и тело (как
|
||||
требует контракт), поэтому подтверждённых дублей быть не должно;
|
||||
* на секунде --action-at запускается --action (например, docker stop/kill контейнера
|
||||
или scripts/rolling-update.sh), его stdout/stderr пишутся в отдельный лог;
|
||||
* параллельно каждые 200 мс опрашивается CSV статистики HAProxy: фиксируются моменты
|
||||
UP -> DOWN / DOWN -> UP серверов backend (время обнаружения отказа LB);
|
||||
* в конце читается вся история чата (тем же токеном) и проверяется: все подтверждённые
|
||||
сообщения на месте, дублей нет.
|
||||
|
||||
Метрики (все времена — от старта action):
|
||||
errors_total / error_rate — неуспешные попытки (HTTP != ожидаемого, timeout, разрыв);
|
||||
error_window_s — от первой до последней ошибки после старта action;
|
||||
recovery_s — конец окна ошибок (дальше все запросы успешны), 0 если ошибок нет;
|
||||
post_action_ok_rps/capacity_ratio — успешный RPS после действия и его доля от базового;
|
||||
lb_down_detected_s — когда HAProxy пометил сервер DOWN;
|
||||
target_last_seen_s — последний ответ от остановленной реплики.
|
||||
|
||||
Ограничения: closed-loop генератор (coordinated omission), клиент на том же хосте, что и
|
||||
стенд; цифры RPS сравнимы только между прогонами этого же скрипта на этом же ноутбуке.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
TIMEOUT = 2.0
|
||||
|
||||
|
||||
def percentile(values, p):
|
||||
if not values:
|
||||
return None
|
||||
s = sorted(values)
|
||||
return round(s[min(len(s) - 1, int(round(p / 100 * (len(s) - 1))))] * 1000, 1)
|
||||
|
||||
|
||||
class Bench:
|
||||
def __init__(self, a):
|
||||
self.a = a
|
||||
self.base = a.base_url.rstrip("/")
|
||||
self.lock = threading.Lock()
|
||||
self.t0 = None
|
||||
self.stop = threading.Event()
|
||||
self.per_sec = defaultdict(lambda: {"ok": 0, "err": 0, "inst": Counter(), "ver": Counter()})
|
||||
self.errors = [] # (t, kind)
|
||||
self.lat = [] # успешные задержки, с
|
||||
self.confirmed = {} # message id -> text
|
||||
self.retried_posts = 0
|
||||
self.lb_events = []
|
||||
self.inst_last_seen = {}
|
||||
|
||||
def now(self):
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def record(self, ok, dt, inst=None, ver=None, kind=None):
|
||||
t = self.now()
|
||||
with self.lock:
|
||||
b = self.per_sec[int(t)]
|
||||
if ok:
|
||||
b["ok"] += 1
|
||||
self.lat.append(dt)
|
||||
if inst:
|
||||
b["inst"][inst] += 1
|
||||
self.inst_last_seen[inst] = t
|
||||
if ver:
|
||||
b["ver"][ver] += 1
|
||||
else:
|
||||
b["err"] += 1
|
||||
self.errors.append((round(t, 3), kind))
|
||||
|
||||
def request(self, method, path, body=None, headers=None, expected=200):
|
||||
h = {"Authorization": "Bearer " + self.token}
|
||||
h.update(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(self.base + path, data=data, method=method, headers=h)
|
||||
start = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||
payload = json.loads(r.read())
|
||||
ok = r.status == expected
|
||||
self.record(ok, time.monotonic() - start, r.headers.get("X-Instance-Id"),
|
||||
r.headers.get("X-App-Version"), None if ok else "http_%d" % r.status)
|
||||
return payload if ok else None
|
||||
except urllib.error.HTTPError as e:
|
||||
self.record(False, 0, kind="http_%d" % e.code)
|
||||
except Exception as e: # noqa: BLE001
|
||||
self.record(False, 0, kind=type(e).__name__)
|
||||
return None
|
||||
|
||||
def worker(self, n):
|
||||
chat_path = "/api/v1/chats/" + self.chat["id"] + "/messages"
|
||||
key = text = None
|
||||
i = 0
|
||||
while not self.stop.is_set():
|
||||
i += 1
|
||||
if i % 2:
|
||||
if key is None: # новый логический запрос; после ошибки ключ НЕ меняется
|
||||
key = "bench-%d-%s" % (n, uuid.uuid4().hex)
|
||||
text = "Нагрузка " + key
|
||||
else:
|
||||
with self.lock:
|
||||
self.retried_posts += 1
|
||||
msg = self.request("POST", chat_path, {"text": text}, {"Idempotency-Key": key}, 201)
|
||||
if msg is not None:
|
||||
with self.lock:
|
||||
self.confirmed[msg["id"]] = msg["text"]
|
||||
key = None
|
||||
else:
|
||||
self.request("GET", chat_path + "?limit=20")
|
||||
|
||||
def poll_lb(self):
|
||||
prev = {}
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
with urllib.request.urlopen(self.a.stats_url, timeout=1) as r:
|
||||
rows = list(csv.reader(io.StringIO(r.read().decode().lstrip("# "))))
|
||||
head = rows[0]
|
||||
ix = {k: head.index(k) for k in ("pxname", "svname", "status", "addr")}
|
||||
for row in rows[1:]:
|
||||
if len(row) <= ix["addr"] or row[ix["pxname"]] != "be_api" or row[ix["svname"]] == "BACKEND":
|
||||
continue
|
||||
sv, st, addr = row[ix["svname"]], row[ix["status"]].split(" ")[0], row[ix["addr"]]
|
||||
if sv in prev and prev[sv] != (st, addr):
|
||||
self.lb_events.append({"t": round(self.now(), 2), "server": sv, "addr": addr,
|
||||
"from": prev[sv][0], "to": st})
|
||||
prev[sv] = (st, addr)
|
||||
except Exception: # noqa: BLE001 — статистика вспомогательная
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
|
||||
def history(self):
|
||||
c, path, items = self.client, "/api/v1/chats/" + self.chat["id"] + "/messages?limit=100", []
|
||||
while True:
|
||||
page = c.call("GET", path, actor=self.alice, schema="MessagePage")["json"]
|
||||
items += page["items"]
|
||||
if page["next_cursor"] is None:
|
||||
return items
|
||||
path = "/api/v1/chats/" + self.chat["id"] + "/messages?limit=100&cursor=" + quote(page["next_cursor"], safe="")
|
||||
|
||||
def run(self):
|
||||
a = self.a
|
||||
self.client = Client(self.base, timeout=10)
|
||||
self.alice, _bob, _eve, self.chat = self.client.bootstrap()
|
||||
self.token = self.alice["token"]
|
||||
self.t0 = time.monotonic()
|
||||
threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(a.concurrency)]
|
||||
threads.append(threading.Thread(target=self.poll_lb, daemon=True))
|
||||
for t in threads:
|
||||
t.start()
|
||||
time.sleep(a.action_at)
|
||||
action = {"command": a.action, "started_s": round(self.now(), 3)}
|
||||
log = open(a.action_log, "w", encoding="utf-8") if a.action_log else subprocess.DEVNULL
|
||||
proc = subprocess.Popen(shlex.split(a.action), stdout=log, stderr=subprocess.STDOUT)
|
||||
rc = proc.wait(timeout=a.duration)
|
||||
action.update(finished_s=round(self.now(), 3), exit_code=rc)
|
||||
remaining = a.duration - self.now()
|
||||
if remaining > 0:
|
||||
time.sleep(remaining)
|
||||
self.stop.set()
|
||||
for t in threads:
|
||||
t.join(timeout=TIMEOUT + 1)
|
||||
return self.summarize(action)
|
||||
|
||||
def summarize(self, action):
|
||||
a, t_act = self.a, action["started_s"]
|
||||
secs = sorted(self.per_sec)
|
||||
base = [self.per_sec[s]["ok"] for s in secs if 1 <= s < int(t_act)] # 1-я секунда — прогрев
|
||||
baseline = sum(base) / len(base) if base else 0
|
||||
after = [(round(t - t_act, 3), k) for t, k in self.errors if t >= t_act]
|
||||
# Восстановление = конец окна ошибок: после этого момента все запросы до конца
|
||||
# прогона успешны. Если ошибок не было — 0. Мощность после отказа меряем отдельно:
|
||||
# при меньшем числе реплик RPS законно ниже (лимиты CPU на реплику).
|
||||
recovery = after[-1][0] if after else 0.0
|
||||
tail_secs = [s for s in secs if int(action.get("finished_s", t_act)) + 2 <= s < a.duration - 1]
|
||||
post = [self.per_sec[s]["ok"] for s in tail_secs]
|
||||
post_rps = sum(post) / len(post) if post else 0
|
||||
writes_after = any(self.per_sec[s]["ok"] > 0 for s in tail_secs)
|
||||
target = a.target_instance
|
||||
hist = self.history()
|
||||
ids = [m["id"] for m in hist]
|
||||
texts = [m["text"] for m in hist]
|
||||
lost = [i for i in self.confirmed if i not in set(ids)]
|
||||
timeline = [{"t": s, "ok": self.per_sec[s]["ok"], "err": self.per_sec[s]["err"],
|
||||
"instances": dict(self.per_sec[s]["inst"]), "versions": dict(self.per_sec[s]["ver"])}
|
||||
for s in secs]
|
||||
total_ok = sum(b["ok"] for b in self.per_sec.values())
|
||||
total_err = len(self.errors)
|
||||
summary = {
|
||||
"label": a.label,
|
||||
"concurrency": a.concurrency, "duration_s": a.duration, "request_timeout_s": TIMEOUT,
|
||||
"action": action,
|
||||
"baseline_ok_rps": round(baseline, 1),
|
||||
"overall_ok_rps": round(total_ok / a.duration, 1),
|
||||
"requests_ok": total_ok, "errors_total": total_err,
|
||||
"error_rate": round(total_err / max(1, total_ok + total_err), 5),
|
||||
"errors_by_kind": dict(Counter(k for _, k in self.errors)),
|
||||
"error_window_s": [after[0][0], after[-1][0]] if after else None,
|
||||
"recovery_s": recovery,
|
||||
"post_action_ok_rps": round(post_rps, 1),
|
||||
"capacity_ratio": round(post_rps / baseline, 2) if baseline else None,
|
||||
"latency_ok_ms": {"p50": percentile(self.lat, 50), "p95": percentile(self.lat, 95),
|
||||
"p99": percentile(self.lat, 99)},
|
||||
"target_instance": target,
|
||||
"target_last_seen_s": (round(self.inst_last_seen[target] - t_act, 3)
|
||||
if target in self.inst_last_seen else None),
|
||||
"lb_events": [dict(e, t=round(e["t"] - t_act, 2)) for e in self.lb_events],
|
||||
"posts_retried_same_key": self.retried_posts,
|
||||
"confirmed_messages": len(self.confirmed),
|
||||
"history_messages": len(hist), "lost_confirmed": len(lost),
|
||||
"duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)),
|
||||
"timeline": timeline,
|
||||
}
|
||||
summary["pass"] = (summary["lost_confirmed"] == 0 and summary["duplicate_ids"] == 0
|
||||
and summary["duplicate_texts"] == 0 and action["exit_code"] == 0
|
||||
and writes_after and recovery <= 10)
|
||||
return summary
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="Отказ/замена реплики под нагрузкой")
|
||||
p.add_argument("--base-url", default="http://localhost:8080")
|
||||
p.add_argument("--stats-url", default="http://127.0.0.1:8404/stats;csv")
|
||||
p.add_argument("--concurrency", type=int, default=8)
|
||||
p.add_argument("--duration", type=float, default=40)
|
||||
p.add_argument("--action-at", type=float, default=10)
|
||||
p.add_argument("--action", required=True, help="команда без shell, например 'docker stop -t 20 <id>'")
|
||||
p.add_argument("--action-log")
|
||||
p.add_argument("--target-instance", help="X-Instance-Id реплики, которую останавливает action")
|
||||
p.add_argument("--label", default="")
|
||||
p.add_argument("--output")
|
||||
a = p.parse_args()
|
||||
s = Bench(a).run()
|
||||
brief = {k: v for k, v in s.items() if k != "timeline"}
|
||||
print(json.dumps(brief, ensure_ascii=False, indent=2))
|
||||
if a.output:
|
||||
Path(a.output).write_text(json.dumps(s, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print("PASS" if s["pass"] else "FAIL")
|
||||
return 0 if s["pass"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 8.1 KiB |
@@ -0,0 +1,130 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Конкурентная идемпотентность через разные реплики (lab3, уровни 3–4).
|
||||
|
||||
Сценарий на каждый раунд:
|
||||
A) PARALLEL запросов одновременно отправляют ОДИН И ТОТ ЖЕ ключ и одинаковое тело.
|
||||
Ожидаем: все 201, у всех один и тот же Message.id, ответы пришли от >=2 реплик,
|
||||
в истории ровно одно сообщение с этим текстом.
|
||||
B) Тот же ключ, но половина запросов с другим телом. Ожидаем: побеждает ровно одно
|
||||
тело; запросы с ним — 201 и один id, запросы с другим телом — 409; дублей нет.
|
||||
|
||||
Каждый запрос — отдельное TCP-соединение, поэтому HAProxy (roundrobin) раскладывает их
|
||||
по репликам. Токены только в памяти; синтетические пользователи создаются заново.
|
||||
|
||||
python3 student-tests/idempotency_concurrency.py --rounds 20 --parallel 16
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
|
||||
from client import Client # noqa: E402 (публичный клиент тестов: bootstrap пользователей/чата)
|
||||
|
||||
|
||||
def post(base, token, chat_id, key, text, barrier, out, idx):
|
||||
data = json.dumps({"text": text}).encode()
|
||||
req = urllib.request.Request(
|
||||
base + "/api/v1/chats/" + chat_id + "/messages", data=data, method="POST",
|
||||
headers={"Authorization": "Bearer " + token, "Content-Type": "application/json",
|
||||
"Idempotency-Key": key, "Connection": "close"})
|
||||
barrier.wait()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
out[idx] = (r.status, json.loads(r.read()), r.headers.get("X-Instance-Id"), text)
|
||||
except urllib.error.HTTPError as e:
|
||||
out[idx] = (e.code, None, e.headers.get("X-Instance-Id"), text)
|
||||
except Exception as e: # noqa: BLE001 — считаем любую сетевую ошибку
|
||||
out[idx] = (0, repr(e), None, text)
|
||||
|
||||
|
||||
def burst(base, token, chat_id, key, texts):
|
||||
barrier = threading.Barrier(len(texts))
|
||||
out = [None] * len(texts)
|
||||
threads = [threading.Thread(target=post, args=(base, token, chat_id, key, t, barrier, out, i))
|
||||
for i, t in enumerate(texts)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
return out
|
||||
|
||||
|
||||
def history_texts(c, actor, chat):
|
||||
from urllib.parse import quote
|
||||
texts, path = [], "/api/v1/chats/" + chat["id"] + "/messages?limit=100"
|
||||
while True:
|
||||
page = c.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
texts += [m["text"] for m in page["items"]]
|
||||
if page["next_cursor"] is None:
|
||||
return texts
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100&cursor=" + quote(page["next_cursor"], safe="")
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
p.add_argument("--base-url", default="http://localhost:8080")
|
||||
p.add_argument("--rounds", type=int, default=20)
|
||||
p.add_argument("--parallel", type=int, default=16)
|
||||
p.add_argument("--output")
|
||||
a = p.parse_args()
|
||||
base = a.base_url.rstrip("/")
|
||||
c = Client(base)
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
token = alice["token"]
|
||||
problems, instances = [], Counter()
|
||||
stats = {"same_body": Counter(), "conflicting_body": Counter()}
|
||||
|
||||
for rnd in range(a.rounds):
|
||||
# A) одинаковое тело
|
||||
key, text = "same-" + uuid.uuid4().hex, "Конкурентный повтор " + uuid.uuid4().hex
|
||||
res = burst(base, token, chat["id"], key, [text] * a.parallel)
|
||||
stats["same_body"].update(str(r[0]) for r in res)
|
||||
ids = {r[1]["id"] for r in res if r[0] == 201}
|
||||
seen = {r[2] for r in res if r[2]}
|
||||
instances.update(r[2] for r in res if r[2])
|
||||
if any(r[0] != 201 for r in res) or len(ids) != 1:
|
||||
problems.append("round %d same-body: statuses=%s ids=%d" % (rnd, Counter(r[0] for r in res), len(ids)))
|
||||
if len(seen) < 2:
|
||||
problems.append("round %d: only %d replica(s) answered" % (rnd, len(seen)))
|
||||
|
||||
# B) тот же ключ, два разных тела вперемешку
|
||||
key = "mixed-" + uuid.uuid4().hex
|
||||
t1, t2 = "Тело A " + key, "Тело B " + key
|
||||
res = burst(base, token, chat["id"], key, [t1, t2] * (a.parallel // 2))
|
||||
stats["conflicting_body"].update(str(r[0]) for r in res)
|
||||
winners = {r[3] for r in res if r[0] == 201}
|
||||
ids = {r[1]["id"] for r in res if r[0] == 201}
|
||||
bad = [r[0] for r in res if r[0] not in (201, 409)]
|
||||
losers_ok = all(r[0] == 409 for r in res if r[3] not in winners)
|
||||
if len(winners) != 1 or len(ids) != 1 or bad or not losers_ok:
|
||||
problems.append("round %d mixed-body: winners=%d ids=%d bad=%s" % (rnd, len(winners), len(ids), bad))
|
||||
|
||||
texts = history_texts(c, alice, chat)
|
||||
dupes = [t for t, n in Counter(texts).items() if n > 1]
|
||||
if dupes:
|
||||
problems.append("duplicated messages in history: %d" % len(dupes))
|
||||
summary = {
|
||||
"rounds": a.rounds, "parallel_per_burst": a.parallel,
|
||||
"requests_total": a.rounds * a.parallel * 2,
|
||||
"status_counts": {k: dict(v) for k, v in stats.items()},
|
||||
"replicas_answered": len(instances), "requests_per_replica": sorted(instances.values()),
|
||||
"messages_in_history": len(texts), "expected_messages": a.rounds * 2,
|
||||
"duplicates": len(dupes), "problems": problems,
|
||||
}
|
||||
if len(texts) != a.rounds * 2:
|
||||
problems.append("history has %d messages, expected %d" % (len(texts), a.rounds * 2))
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||||
if a.output:
|
||||
Path(a.output).write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
print("PASS" if not problems else "FAIL")
|
||||
return 1 if problems else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Общие помощники проверок lab5 (запускаются на хосте, управляют стендом через docker compose).
|
||||
|
||||
Токены сессий живут только в памяти процесса и не печатаются. В evidence пишутся ID,
|
||||
статусы, времена и события заданий — без секретов и содержимого файлов.
|
||||
"""
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure, check # noqa: E402,F401
|
||||
|
||||
SAMPLE_PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def now():
|
||||
return dt.datetime.now(dt.timezone.utc).strftime("%H:%M:%S.%f")[:-3]
|
||||
|
||||
|
||||
class Log:
|
||||
def __init__(self, name):
|
||||
self.name = name
|
||||
self.steps = []
|
||||
self.started = time.monotonic()
|
||||
|
||||
def step(self, text, **data):
|
||||
item = {"t": round(time.monotonic() - self.started, 2), "utc": now(), "step": text}
|
||||
item.update(data)
|
||||
self.steps.append(item)
|
||||
print("[%7.2fs] %s %s" % (item["t"], text, json.dumps(data, ensure_ascii=False) if data else ""), flush=True)
|
||||
|
||||
def save(self, path, **extra):
|
||||
if not path:
|
||||
return
|
||||
out = {"scenario": self.name, "date_utc": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"),
|
||||
"steps": self.steps}
|
||||
out.update(extra)
|
||||
Path(path).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(path).write_text(json.dumps(out, ensure_ascii=False, indent=1) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def compose(*args, env=None, timeout=300, ok=True):
|
||||
full_env = dict(os.environ)
|
||||
full_env.update(env or {})
|
||||
proc = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
|
||||
env=full_env, timeout=timeout)
|
||||
if ok and proc.returncode != 0:
|
||||
raise Failure("docker compose %s failed: %s" % (" ".join(args[:3]), proc.stderr.strip()[-400:]))
|
||||
return proc
|
||||
|
||||
|
||||
def recreate(service, env=None):
|
||||
"""Пересоздать сервис с переопределёнными переменными (shell env важнее .env)."""
|
||||
compose("up", "-d", "--no-deps", "--force-recreate", "--wait", service, env=env, timeout=400)
|
||||
|
||||
|
||||
def sql(query):
|
||||
proc = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "db", "sh", "-c",
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -tA -F "|" -v ON_ERROR_STOP=1'],
|
||||
cwd=ROOT, input=query, capture_output=True, text=True, timeout=60)
|
||||
if proc.returncode != 0:
|
||||
raise Failure("psql failed: " + proc.stderr.strip()[-300:])
|
||||
return [line.split("|") for line in proc.stdout.strip().splitlines() if line]
|
||||
|
||||
|
||||
def ops(*args, ok=True):
|
||||
proc = compose("--profile", "tools", "run", "--rm", "-T", "ops", *args, ok=False, timeout=120)
|
||||
if ok and proc.returncode not in (0, 3):
|
||||
raise Failure("ops %s failed: %s" % (args[0], proc.stderr.strip()[-400:]))
|
||||
try:
|
||||
return json.loads(proc.stdout)
|
||||
except ValueError:
|
||||
raise Failure("ops output is not JSON: " + proc.stdout[-300:] + proc.stderr[-300:])
|
||||
|
||||
|
||||
def queues():
|
||||
proc = compose("exec", "-T", "broker", "rabbitmqctl", "-q", "-p", "messenger", "list_queues", "name",
|
||||
"messages", "messages_ready", "messages_unacknowledged", "consumers", "--formatter", "json")
|
||||
return {q["name"]: {k: q[k] for k in ("messages", "messages_ready", "messages_unacknowledged", "consumers")}
|
||||
for q in json.loads(proc.stdout)}
|
||||
|
||||
|
||||
def container_ids(service, running_only=True):
|
||||
args = ["ps", "-q"] + (["--status", "running"] if running_only else []) + [service]
|
||||
return [x for x in compose(*args).stdout.split() if x]
|
||||
|
||||
|
||||
def upload(c, actor, chat, data, filename="photo.png", ctype="image/png", expected=202):
|
||||
return c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], filename), expected,
|
||||
actor=actor, raw=data, headers={"Content-Type": ctype},
|
||||
schema="Attachment" if expected == 202 else None)["json"]
|
||||
|
||||
|
||||
def get_att(c, actor, att_id):
|
||||
return c.call("GET", "/api/v1/attachments/" + att_id, actor=actor, schema="Attachment")["json"]
|
||||
|
||||
|
||||
def wait_status(c, actor, att_id, statuses, timeout=90, interval=0.3):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
att = get_att(c, actor, att_id)
|
||||
if att["status"] in statuses:
|
||||
return att
|
||||
time.sleep(interval)
|
||||
raise Failure("attachment %s did not reach %s in %ss (last %s)" % (att_id, statuses, timeout, att["status"]))
|
||||
|
||||
|
||||
def wait_until(fn, timeout, interval=0.5, what="condition"):
|
||||
deadline = time.monotonic() + timeout
|
||||
last = None
|
||||
while time.monotonic() < deadline:
|
||||
last = fn()
|
||||
if last:
|
||||
return last
|
||||
time.sleep(interval)
|
||||
raise Failure("timeout waiting for " + what)
|
||||
|
||||
|
||||
def events(info):
|
||||
return [e["event"] for e in info["events"]]
|
||||
|
||||
|
||||
def invariants(info):
|
||||
"""Ключ эффекта: ровно 3 варианта и 4 конечных объекта на (attachment, v1), staging убран."""
|
||||
names = sorted(v["name"] for v in info["variants"])
|
||||
finals = [o[0] for o in info["s3_final_objects"]]
|
||||
return {
|
||||
"variants": names,
|
||||
"variants_ok": names == ["crop", "optimized", "thumbnail"],
|
||||
"final_objects": len(finals),
|
||||
"final_objects_ok": len(finals) == 4,
|
||||
"staging_left": info["attachment"]["staging_key"] is not None,
|
||||
"job_status": info["jobs"][0]["status"],
|
||||
"attempt": info["jobs"][0]["attempt"],
|
||||
}
|
||||
|
||||
|
||||
def png(width, height, rgba=False, noise=False, color=(40, 120, 200), seed=1):
|
||||
"""PNG стандартной библиотекой: шум (несжимаемый) или заливка."""
|
||||
channels = 4 if rgba else 3
|
||||
rng = random.Random(seed)
|
||||
rows = []
|
||||
if noise:
|
||||
for _ in range(height):
|
||||
rows.append(b"\x00" + bytes(rng.getrandbits(8) for _ in range(width * channels)))
|
||||
else:
|
||||
px = bytes(color[:3]) + (b"\x80" if rgba else b"")
|
||||
row = b"\x00" + px * width
|
||||
rows = [row] * height
|
||||
raw = b"".join(rows)
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 6 if rgba else 2, 0, 0, 0)
|
||||
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 6))
|
||||
+ chunk(b"IEND", b""))
|
||||
|
||||
|
||||
def noise_png_fast(width, height, seed=1):
|
||||
"""Несжимаемый PNG через os.urandom (быстро для больших размеров)."""
|
||||
raw = b"".join(b"\x00" + os.urandom(width * 3) for _ in range(height))
|
||||
|
||||
def chunk(kind, data):
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")
|
||||
@@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: зависший processing (lease) и fencing опоздавшего worker.
|
||||
|
||||
python3 student-tests/lease_takeover.py --output evidence/lease-takeover.json
|
||||
|
||||
Worker пересоздаются с WORKER_FAULT_INJECTION=1, JOB_LEASE_SECONDS=10, FAULT_SLOW_SECONDS=25.
|
||||
1. upload fault-slow.png: worker берёт задание и «долго работает» (продлевает аренду);
|
||||
2. пока аренда продлевается, второй worker задание не трогает (долгий != зависший);
|
||||
3. docker pause контейнера-владельца: процесс жив, но заморожен — аренду никто не продлевает;
|
||||
4. через ~lease reaper (relay) переводит задание в queued, второй worker берёт его -> ready;
|
||||
5. docker unpause: опоздавший worker обнаруживает потерю аренды и ничего не записывает
|
||||
(aborted_lease_lost / commit_rejected_lease_lost); вариантов по-прежнему 3, объектов 4.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, container_ids, events, get_att, invariants, ops,
|
||||
recreate, upload, wait_status, wait_until)
|
||||
|
||||
ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_LEASE_SECONDS": "10", "FAULT_SLOW_SECONDS": "25"}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
log = Log("lease expiry of a frozen worker and fencing of its late result")
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
recreate("worker", ENV)
|
||||
log.step("workers recreated", env=ENV, workers=[x[:12] for x in container_ids("worker")])
|
||||
paused = None
|
||||
try:
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-slow.png")
|
||||
info = wait_until(lambda: (lambda i: i if i["jobs"][0]["status"] == "processing" else None)(
|
||||
ops("inspect", att["id"])), 30, 0.5, "processing")
|
||||
owner = info["jobs"][0]["lease_owner"]
|
||||
lease1 = info["jobs"][0]["lease_until"]
|
||||
log.step("claimed", attachment_id=att["id"], lease_owner=owner, lease_until=lease1)
|
||||
time.sleep(8)
|
||||
info = ops("inspect", att["id"])
|
||||
check(info["jobs"][0]["status"] == "processing" and info["jobs"][0]["lease_owner"] == owner, "long job stolen")
|
||||
log.step("long-running job keeps its lease (heartbeat)", lease_until=info["jobs"][0]["lease_until"],
|
||||
status=info["jobs"][0]["status"])
|
||||
host = owner.split(":")[0]
|
||||
paused = [cid for cid in container_ids("worker") if cid.startswith(host)][0]
|
||||
subprocess.run(["docker", "pause", paused], check=True, capture_output=True)
|
||||
t_pause = time.monotonic()
|
||||
log.step("owner container paused (frozen, no heartbeat)", container=paused[:12])
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
|
||||
took = round(time.monotonic() - t_pause, 1)
|
||||
info = ops("inspect", att["id"])
|
||||
check(ready["status"] == "ready", "not ready after lease expiry")
|
||||
new_owner = [e["actor"] for e in info["events"] if e["event"] == "claimed"][-1]
|
||||
check(new_owner.split(":")[0] != host, "must be taken by the other worker")
|
||||
log.step("lease expired -> requeued -> other worker finished", seconds_after_pause=took,
|
||||
new_owner=new_owner, events=events(info))
|
||||
subprocess.run(["docker", "unpause", paused], check=True, capture_output=True)
|
||||
paused_id, paused = paused, None
|
||||
log.step("owner container unpaused")
|
||||
info = wait_until(lambda: (lambda i: i if {"aborted_lease_lost", "commit_rejected_lease_lost", "error_ignored_lease_lost"} & set(events(i))
|
||||
else None)(ops("inspect", att["id"])), 60, 1, "late worker fenced")
|
||||
inv = invariants(info)
|
||||
after = get_att(c, bob, att["id"])
|
||||
check(inv["variants_ok"] and inv["final_objects_ok"] and after["status"] == "ready", "invariants")
|
||||
check(after["variants"] == ready["variants"], "late worker changed published result")
|
||||
log.step("late worker fenced, result unchanged", events=events(info), invariants=inv)
|
||||
log.save(args.output, attachment_id=att["id"], paused_container=paused_id[:12], events=info["events"],
|
||||
invariants=inv)
|
||||
finally:
|
||||
if paused:
|
||||
subprocess.run(["docker", "unpause", paused], capture_output=True)
|
||||
recreate("worker")
|
||||
print("PASS: frozen worker loses lease, job recovered, late result fenced")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Шторм входов: много параллельных POST /auth/sessions с неверным паролем через LB.
|
||||
|
||||
Зачем: scrypt (N=2^15, r=8) берёт ~32 MiB на одну проверку пароля, лимит API-реплики
|
||||
256 MiB. Без ограничения параллельных проверок несколько одновременных входов
|
||||
приводят к OOM-kill реплики. Скрипт показывает, переживают ли реплики такой шторм.
|
||||
|
||||
Что измеряется:
|
||||
* статусы ответов шторма (ожидаем 401 и, при перегрузке, 503 с Retry-After) и
|
||||
ответы не в JSON-формате ошибки контракта (например HTML-страницы HAProxy);
|
||||
* «жертва»: параллельный поток раз в 50 мс читает /users/me действующим токеном —
|
||||
обычные пользователи не должны получать ошибки из-за шторма входов;
|
||||
* состояние API-контейнеров до/после: OOMKilled, RestartCount, StartedAt.
|
||||
|
||||
Пример: python3 student-tests/login_flood.py --parallel 24 --duration 20 --output evidence/login-flood.json
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from collections import Counter
|
||||
|
||||
TIMEOUT = 10.0
|
||||
|
||||
|
||||
def http(base, method, path, headers=None, body=None):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
h = dict(headers or {})
|
||||
if data is not None:
|
||||
h["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(base + path, data=data, method=method, headers=h)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||
return r.status, r.headers, r.read()
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, e.headers, e.read()
|
||||
|
||||
|
||||
def is_contract_error(headers, raw):
|
||||
if not (headers.get("Content-Type") or "").startswith("application/json"):
|
||||
return False
|
||||
try:
|
||||
err = json.loads(raw)["error"]
|
||||
return bool(err["code"] and err["message"] and err["request_id"])
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return False
|
||||
|
||||
|
||||
def api_containers():
|
||||
ids = subprocess.run(["docker", "compose", "ps", "-a", "-q", "api"], capture_output=True,
|
||||
text=True, check=True).stdout.split()
|
||||
out = {}
|
||||
for cid in ids:
|
||||
info = json.loads(subprocess.run(["docker", "inspect", cid], capture_output=True, text=True,
|
||||
check=True).stdout)[0]
|
||||
out[info["Name"].lstrip("/")] = {
|
||||
"status": info["State"]["Status"], "oom_killed": info["State"]["OOMKilled"],
|
||||
"restart_count": info["RestartCount"], "started_at": info["State"]["StartedAt"]}
|
||||
return out
|
||||
|
||||
|
||||
def oom_events(since):
|
||||
"""Имена контейнеров, получивших OOM с момента since (unix time): флаг OOMKilled
|
||||
сбрасывается после рестарта по restart-policy, поэтому смотрим события Docker."""
|
||||
out = subprocess.run(["docker", "events", "--since", str(int(since)), "--until", str(int(time.time()) + 1),
|
||||
"--filter", "type=container", "--filter", "event=oom",
|
||||
"--format", "{{.Actor.Attributes.name}}"], capture_output=True, text=True).stdout
|
||||
return sorted(Counter(out.split()).items())
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://localhost:8080")
|
||||
p.add_argument("--parallel", type=int, default=24)
|
||||
p.add_argument("--duration", type=float, default=20)
|
||||
p.add_argument("--output")
|
||||
a = p.parse_args()
|
||||
base = a.base_url.rstrip("/")
|
||||
|
||||
username = "flood_" + uuid.uuid4().hex[:10]
|
||||
password = "correct-horse-" + uuid.uuid4().hex[:8]
|
||||
st, _, _ = http(base, "POST", "/api/v1/users",
|
||||
body={"username": username, "display_name": "Flood", "password": password})
|
||||
assert st == 201, st
|
||||
basic = base64.b64encode(("%s:%s" % (username, password)).encode()).decode()
|
||||
st, _, raw = http(base, "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
assert st == 201, st
|
||||
token = json.loads(raw)["token"]
|
||||
bad = base64.b64encode(("%s:wrong-password-123" % username).encode()).decode()
|
||||
|
||||
before = api_containers()
|
||||
wall0 = time.time()
|
||||
stop = threading.Event()
|
||||
lock = threading.Lock()
|
||||
flood, victim = Counter(), Counter()
|
||||
non_json_errors = Counter()
|
||||
retry_after_503 = [0, 0] # [503 всего, из них с Retry-After]
|
||||
|
||||
def record(counter, status, headers, raw):
|
||||
with lock:
|
||||
counter[str(status)] += 1
|
||||
if status >= 400 and not is_contract_error(headers, raw):
|
||||
non_json_errors[str(status)] += 1
|
||||
if status == 503 and counter is flood:
|
||||
retry_after_503[0] += 1
|
||||
retry_after_503[1] += bool(headers.get("Retry-After"))
|
||||
|
||||
def flooder():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
record(flood, *http(base, "POST", "/api/v1/auth/sessions",
|
||||
headers={"Authorization": "Basic " + bad}))
|
||||
except Exception as exc: # обрыв соединения, timeout
|
||||
with lock:
|
||||
flood["exc:" + type(exc).__name__] += 1
|
||||
|
||||
def prober():
|
||||
while not stop.is_set():
|
||||
try:
|
||||
record(victim, *http(base, "GET", "/api/v1/users/me",
|
||||
headers={"Authorization": "Bearer " + token}))
|
||||
except Exception as exc:
|
||||
with lock:
|
||||
victim["exc:" + type(exc).__name__] += 1
|
||||
time.sleep(0.05)
|
||||
|
||||
threads = [threading.Thread(target=flooder, daemon=True) for _ in range(a.parallel)]
|
||||
threads.append(threading.Thread(target=prober, daemon=True))
|
||||
t0 = time.monotonic()
|
||||
for t in threads:
|
||||
t.start()
|
||||
time.sleep(a.duration)
|
||||
stop.set()
|
||||
for t in threads:
|
||||
t.join(TIMEOUT + 1)
|
||||
elapsed = time.monotonic() - t0
|
||||
time.sleep(3) # дать restart-policy отработать, если реплика упала
|
||||
after = api_containers()
|
||||
|
||||
restarted = [n for n in after if n in before and (
|
||||
after[n]["restart_count"] != before[n]["restart_count"]
|
||||
or after[n]["started_at"] != before[n]["started_at"])]
|
||||
victim_ok = victim.get("200", 0)
|
||||
victim_total = sum(victim.values())
|
||||
summary = {
|
||||
"parallel": a.parallel, "duration_s": round(elapsed, 1),
|
||||
"flood_statuses": dict(flood), "flood_rps": round(sum(flood.values()) / elapsed, 1),
|
||||
"flood_503_with_retry_after": "%d/%d" % (retry_after_503[1], retry_after_503[0]),
|
||||
"victim_statuses": dict(victim),
|
||||
"victim_error_rate": round(1 - victim_ok / victim_total, 4) if victim_total else None,
|
||||
"non_contract_error_bodies": dict(non_json_errors),
|
||||
"containers_before": before, "containers_after": after,
|
||||
"oom_events": dict(oom_events(wall0)),
|
||||
"restarted": restarted,
|
||||
}
|
||||
summary["pass"] = (not summary["oom_events"] and not restarted and victim_total > 0
|
||||
and victim_ok == victim_total and not non_json_errors)
|
||||
text = json.dumps(summary, ensure_ascii=False, indent=2)
|
||||
print(text)
|
||||
if a.output:
|
||||
with open(a.output, "w", encoding="utf-8") as f:
|
||||
f.write(text + "\n")
|
||||
return 0 if summary["pass"] else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: разрыв PostgreSQL <-> broker закрыт transactional outbox. Окна отказа:
|
||||
|
||||
python3 student-tests/outbox_gap.py --output evidence/outbox-gap.json
|
||||
|
||||
A. relay остановлен: COMMIT есть, publish нет -> outbox не опубликован, очередь пуста;
|
||||
relay запущен -> задание доходит до ready без ручного создания нового задания;
|
||||
B. RELAY_FAULT=crash-before-publish: relay выбрал строки outbox и умер до publish
|
||||
(внедрённый crash между COMMIT и publish) -> рестарт контейнера -> ready;
|
||||
C. RELAY_FAULT=crash-after-publish: publish + confirm прошли, published_at не записан ->
|
||||
после рестарта сообщение опубликовано ещё раз -> worker гасит дубль, вариантов 3;
|
||||
D. API crash-after-commit (FAULT_INJECTION=1): клиент не получил 202, но задание принято
|
||||
и доходит до ready;
|
||||
E. API crash-after-staging: оригинал в staging, задания нет (202 не было) -> pending-строку
|
||||
и staging-объект убирает janitor после grace.
|
||||
"""
|
||||
import argparse
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
from lab5lib import (Client, Failure, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops,
|
||||
queues, recreate, sql, upload, wait_status, wait_until)
|
||||
|
||||
|
||||
def restart_count(service):
|
||||
ids = compose("ps", "-q", "-a", service).stdout.split()
|
||||
out = subprocess.run(["docker", "inspect", "-f", "{{.RestartCount}}"] + ids, capture_output=True, text=True)
|
||||
return [int(x) for x in out.stdout.split()]
|
||||
|
||||
|
||||
def find_by_filename(name):
|
||||
rows = sql("SELECT id, status, staging_key IS NOT NULL FROM attachments WHERE filename = '%s'" % name)
|
||||
return rows[0] if rows else None
|
||||
|
||||
|
||||
def raw_upload(c, actor, chat, name, fault):
|
||||
try:
|
||||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], name), [502, 503, 504],
|
||||
actor=actor, raw=SAMPLE_PNG, headers={"Content-Type": "image/png", "X-Fault-Inject": fault})
|
||||
return "http %d" % r["status"]
|
||||
except Failure as exc:
|
||||
return str(exc)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
log = Log("outbox: COMMIT/publish gap and API crash windows")
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
res = {}
|
||||
try:
|
||||
# A ---------------------------------------------------------------------------
|
||||
compose("stop", "relay")
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "gap-relay-stopped.png")
|
||||
time.sleep(3)
|
||||
info = ops("inspect", att["id"])
|
||||
q = queues()["attachments.process.v1"]
|
||||
check(info["outbox"][0]["published_at"] is None and get_att(c, bob, att["id"])["status"] == "queued",
|
||||
"A: must stay unpublished")
|
||||
log.step("A: relay stopped, upload 202", attachment_id=att["id"], outbox=info["outbox"], queue=q)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "relay")
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 60)
|
||||
info = ops("inspect", att["id"])
|
||||
check(ready["status"] == "ready" and len(info["jobs"]) == 1, "A: not ready / extra job")
|
||||
res["A"] = {"attachment_id": att["id"], "events": events(info), "outbox": info["outbox"],
|
||||
"jobs": len(info["jobs"]), "invariants": invariants(info)}
|
||||
log.step("A: relay started -> ready, same single job", **res["A"])
|
||||
|
||||
# B ---------------------------------------------------------------------------
|
||||
recreate("relay", {"RELAY_FAULT": "crash-before-publish"})
|
||||
rc0 = restart_count("relay")
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-before-publish.png")
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90)
|
||||
info = ops("inspect", att["id"])
|
||||
logs = compose("logs", "--no-log-prefix", "relay").stdout
|
||||
check(ready["status"] == "ready" and "FAULT INJECTION crash-before-publish" in logs, "B")
|
||||
res["B"] = {"attachment_id": att["id"], "relay_restart_count": [rc0, restart_count("relay")],
|
||||
"events": events(info), "outbox": info["outbox"], "invariants": invariants(info)}
|
||||
log.step("B: relay crashed between COMMIT and publish -> restarted -> ready", **res["B"])
|
||||
|
||||
# C ---------------------------------------------------------------------------
|
||||
recreate("relay", {"RELAY_FAULT": "crash-after-publish"})
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "gap-crash-after-publish.png")
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, 90)
|
||||
info = wait_until(lambda: (lambda i: i if "duplicate_ignored" in events(i) or
|
||||
"busy_ignored" in events(i) else None)(ops("inspect", att["id"])),
|
||||
60, 1, "duplicate handled")
|
||||
inv = invariants(info)
|
||||
check(ready["status"] == "ready" and inv["variants_ok"] and inv["final_objects_ok"], "C")
|
||||
res["C"] = {"attachment_id": att["id"], "relay_restart_count": restart_count("relay"),
|
||||
"events": events(info), "outbox": info["outbox"], "invariants": inv}
|
||||
log.step("C: published twice (crash before marking) -> duplicate is a no-op", **res["C"])
|
||||
recreate("relay")
|
||||
|
||||
# D ---------------------------------------------------------------------------
|
||||
recreate("api", {"FAULT_INJECTION": "1"})
|
||||
name = "gap-api-crash-after-commit-%d.png" % time.time()
|
||||
outcome = raw_upload(c, alice, chat, name, "crash-after-commit")
|
||||
row = wait_until(lambda: find_by_filename(name), 20, what="attachment row")
|
||||
ready = wait_status(c, bob, row[0], {"ready", "failed"}, 90)
|
||||
info = ops("inspect", row[0])
|
||||
check(ready["status"] == "ready", "D")
|
||||
res["D"] = {"attachment_id": row[0], "client_saw": outcome, "events": events(info),
|
||||
"invariants": invariants(info)}
|
||||
log.step("D: API died after COMMIT before 202 -> work still completed", **res["D"])
|
||||
|
||||
# E ---------------------------------------------------------------------------
|
||||
name = "gap-api-crash-after-staging-%d.png" % time.time()
|
||||
outcome = raw_upload(c, alice, chat, name, "crash-after-staging")
|
||||
row = wait_until(lambda: find_by_filename(name), 20, what="pending row")
|
||||
jobs_n = sql("SELECT count(*) FROM jobs WHERE attachment_id = '%s'" % row[0])[0][0]
|
||||
check(row[1] == "pending" and jobs_n == "0", "E: expected pending row without job")
|
||||
c.call("GET", "/api/v1/attachments/" + row[0], 404, actor=bob)
|
||||
log.step("E: API died after staging before COMMIT", attachment_id=row[0], client_saw=outcome,
|
||||
status=row[1], jobs=int(jobs_n), api_get=404)
|
||||
time.sleep(3)
|
||||
dry = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once", "--dry-run",
|
||||
"--grace-seconds", "2").stdout
|
||||
real = compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once",
|
||||
"--grace-seconds", "2").stdout
|
||||
import json as _j
|
||||
dry, real = _j.loads(dry.strip().splitlines()[-1]), _j.loads(real.strip().splitlines()[-1])
|
||||
gone = find_by_filename(name) is None
|
||||
check(row[0] in dry["abandoned_ids"] and gone, "E: janitor did not clean")
|
||||
res["E"] = {"attachment_id": row[0], "janitor_dry_run": {k: dry[k] for k in ("abandoned_pending", "orphan_objects",
|
||||
"deleted_objects", "deleted_rows")},
|
||||
"janitor": {k: real[k] for k in ("abandoned_pending", "orphan_objects", "deleted_objects",
|
||||
"deleted_rows", "missing_objects")}, "row_gone": gone}
|
||||
log.step("E: janitor removed pending row and staging object", **res["E"])
|
||||
finally:
|
||||
recreate("relay")
|
||||
recreate("api")
|
||||
log.step("relay and api restored to .env configuration")
|
||||
log.save(args.output, results=res)
|
||||
print("PASS: outbox closes COMMIT/publish gap; API crash windows handled")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/bin/sh
|
||||
# Полный прогон проверок lab5 на запущенном стенде с записью результатов в evidence/.
|
||||
# Каждый сценарий сам включает и выключает свои учебные инъекции отказов.
|
||||
# ./student-tests/run_all.sh
|
||||
set -u
|
||||
cd "$(dirname "$0")/.."
|
||||
E=evidence
|
||||
mkdir -p "$E"
|
||||
status=0
|
||||
run() { # run <name> <command...>
|
||||
name=$1; shift
|
||||
printf '=== %s\n' "$name"
|
||||
{ echo "# $(date -u +%FT%TZ) $*"; "$@" 2>&1; rc=$?; echo "exit=$rc"; } > "$E/$name.txt"
|
||||
tail -n 3 "$E/$name.txt"
|
||||
grep -q '^exit=0$' "$E/$name.txt" || status=1
|
||||
}
|
||||
states() { docker compose ps --format '{{.Name}} {{.State}}' | sort; \
|
||||
for c in $(docker compose ps -q); do docker inspect -f '{{.Name}} StartedAt={{.State.StartedAt}}' "$c"; done | sort; }
|
||||
|
||||
run check make check
|
||||
run smoke make test
|
||||
{ echo "# containers before persistence hook"; states; } > "$E/containers-before-restart.txt"
|
||||
run persistence make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
{ echo "# containers after persistence hook"; states; } > "$E/containers-after-restart.txt"
|
||||
run failover make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
./scripts/start-all.sh >/dev/null 2>&1
|
||||
run worker-outage python3 student-tests/worker_outage.py --output "$E/worker-outage.json"
|
||||
run variants-check docker compose --profile tools run --rm -T checks /app/student-tests/variants_check.py \
|
||||
--base-url http://lb:8080 --output /app/evidence/variants-check.json
|
||||
run transform-limits docker compose --profile tools run --rm -T checks /app/student-tests/transform_limits.py \
|
||||
--output /app/evidence/transform-limits.json
|
||||
run worker-faults python3 student-tests/worker_faults.py --output "$E/worker-faults.json"
|
||||
run outbox-gap python3 student-tests/outbox_gap.py --output "$E/outbox-gap.json"
|
||||
run lease-takeover python3 student-tests/lease_takeover.py --output "$E/lease-takeover.json"
|
||||
run burst python3 student-tests/burst.py --count 24 --output "$E/burst.json"
|
||||
run broker-permissions python3 student-tests/broker_permissions.py --output "$E/broker-permissions.json"
|
||||
run s3-private-check docker compose --profile tools run --rm -T checks /app/student-tests/s3_private_check.py
|
||||
run attachments-negative python3 student-tests/attachments_negative.py --output "$E/attachments-negative.json"
|
||||
run attachment-failover python3 student-tests/attachment_failover.py --output "$E/attachment-failover.json"
|
||||
./scripts/start-all.sh >/dev/null 2>&1
|
||||
run janitor-dry-run docker compose exec -T janitor python /app/src/janitor.py --once --dry-run
|
||||
run queues docker compose exec -T broker rabbitmqctl -q -p messenger list_queues name type durable messages consumers
|
||||
run failed-registry docker compose --profile tools run --rm -T ops failed --limit 10
|
||||
echo "overall=$status"
|
||||
exit $status
|
||||
@@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Недоступность S3: stop (порт закрыт) и pause (S3 «завис», TCP не отвечает).
|
||||
|
||||
python3 student-tests/s3_outage.py [--output evidence/s3-outage.json]
|
||||
|
||||
Ожидания: upload и download — быстрый 503 unavailable c Retry-After (не 500 и не
|
||||
таймаут LB), текстовые сообщения и /health/ready продолжают работать (S3 не входит в
|
||||
readiness), после восстановления S3 загрузка снова 201 без перезапуска API.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
PNG = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
log = []
|
||||
|
||||
|
||||
def probe(c, alice, chat, att):
|
||||
out = {}
|
||||
for name, args in (
|
||||
("upload", ("POST", "/api/v1/chats/%s/attachments?filename=o.png" % chat["id"])),
|
||||
("download", ("GET", "/api/v1/attachments/%s/content" % att["id"])),
|
||||
("text_message", ("POST", "/api/v1/chats/%s/messages" % chat["id"])),
|
||||
("ready", ("GET", "/health/ready"))):
|
||||
kw = {}
|
||||
if name == "upload":
|
||||
kw = {"raw": PNG, "headers": {"Content-Type": "image/png"}}
|
||||
if name == "text_message":
|
||||
kw = {"body": {"text": "без S3"}, "headers": {"Idempotency-Key": uuid.uuid4().hex}}
|
||||
t = time.monotonic()
|
||||
r = c.call(*args, [200, 201, 503, 502, 504], actor=alice if name != "ready" else None, **kw)
|
||||
out[name] = {"status": r["status"], "ms": round((time.monotonic() - t) * 1000),
|
||||
"code": (r["json"] or {}).get("error", {}).get("code"),
|
||||
"retry_after": r["headers"].get("Retry-After")}
|
||||
return out
|
||||
|
||||
|
||||
def phase(name, c, alice, chat, att):
|
||||
entry = {"phase": name, "probe": probe(c, alice, chat, att)}
|
||||
log.append(entry)
|
||||
print(json.dumps(entry, ensure_ascii=False), flush=True)
|
||||
return entry
|
||||
|
||||
|
||||
def recover(c, alice, chat, started):
|
||||
while time.monotonic() - started < 120:
|
||||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=r.png" % chat["id"], [201, 503], actor=alice,
|
||||
raw=PNG, headers={"Content-Type": "image/png"})
|
||||
if r["status"] == 201:
|
||||
return round(time.monotonic() - started, 1)
|
||||
time.sleep(0.5)
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=a.png" % chat["id"], 201, actor=alice, raw=PNG,
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")["json"]
|
||||
phase("baseline", c, alice, chat, att)
|
||||
for action, undo in ((["stop", "s3"], ["start", "s3"]), (["pause", "s3"], ["unpause", "s3"])):
|
||||
subprocess.run(["docker", "compose"] + action, cwd=ROOT, check=True, capture_output=True)
|
||||
time.sleep(1)
|
||||
phase("immediately after `%s` (S3Health not yet stale)" % " ".join(action), c, alice, chat, att)
|
||||
time.sleep(4)
|
||||
phase("S3 down for 5 s (`%s`)" % " ".join(action), c, alice, chat, att)
|
||||
subprocess.run(["docker", "compose"] + undo, cwd=ROOT, check=True, capture_output=True)
|
||||
t0 = time.monotonic()
|
||||
rec = recover(c, alice, chat, t0)
|
||||
log.append({"phase": "recovery after `%s`" % " ".join(undo), "upload_201_after_s": rec})
|
||||
print(json.dumps(log[-1]), flush=True)
|
||||
# Оставляем стенд рабочим: ждём, пока healthcheck S3 снова станет healthy (после pause
|
||||
# он несколько проверок подряд был unhealthy, а `docker compose up api` при нездоровой
|
||||
# зависимости не запускает реплики).
|
||||
cid = subprocess.run(["docker", "compose", "ps", "-q", "s3"], cwd=ROOT, capture_output=True, text=True).stdout.strip()
|
||||
t0 = time.monotonic()
|
||||
while time.monotonic() - t0 < 120:
|
||||
st = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid], capture_output=True, text=True).stdout.strip()
|
||||
if st == "healthy":
|
||||
break
|
||||
time.sleep(1)
|
||||
log.append({"phase": "s3 healthcheck healthy again", "after_s": round(time.monotonic() - t0, 1), "status": st})
|
||||
print(json.dumps(log[-1]), flush=True)
|
||||
down = [e for e in log if e.get("phase", "").startswith("S3 down")]
|
||||
ok = all(e["probe"]["upload"]["status"] == 503 and e["probe"]["download"]["status"] == 503
|
||||
and e["probe"]["text_message"]["status"] == 201 and e["probe"]["ready"]["status"] == 200
|
||||
for e in down) and all(e.get("upload_201_after_s") is not None for e in log if "upload_201_after_s" in e)
|
||||
no5xx = all(p["status"] in (200, 201, 503) for e in log if "probe" in e for p in e["probe"].values())
|
||||
ok = ok and st == "healthy"
|
||||
print(json.dumps({"pass": ok and no5xx, "only_200_201_503": no5xx}))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps({"pass": ok and no5xx, "phases": log}, ensure_ascii=False, indent=1))
|
||||
return 0 if ok and no5xx else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка приватности bucket и ограничений ключей ВНЕ API (внутри backend-сети).
|
||||
|
||||
docker compose --profile tools run --rm checks /app/student-tests/s3_private_check.py
|
||||
|
||||
Проверяет напрямую на S3 (минуя API и его ACL):
|
||||
* анонимные GET объекта / ListBucket / PUT -> 403;
|
||||
* ключ api: Put/Get/Delete своего тестового объекта можно; ListBucket, CreateBucket,
|
||||
DeleteBucket, PutBucketPolicy — AccessDenied; другой bucket — AccessDenied;
|
||||
* ключ worker (lab5): Get/Put/Delete можно; ListBucket, CreateBucket — AccessDenied;
|
||||
* ключ janitor: List и Delete можно; GetObject (чтение содержимого) — AccessDenied;
|
||||
* служебные порты SeaweedFS (master 9333, volume 8080, filer 8888, IAM 8111,
|
||||
Iceberg 8181, Lance 9101) из backend-сети недоступны — открыт только S3 :8333.
|
||||
Выход 0 только если все ожидания выполнены. Ключи не печатаются.
|
||||
"""
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
sys.path.insert(0, "/app/src")
|
||||
from minio import Minio # noqa: E402
|
||||
from minio.error import S3Error # noqa: E402
|
||||
|
||||
BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:8333").rstrip("/")
|
||||
HOST = urlsplit(ENDPOINT).netloc
|
||||
|
||||
|
||||
def key(name):
|
||||
with open("/run/secrets/" + name) as f:
|
||||
return f.read().strip()
|
||||
|
||||
|
||||
def client(who):
|
||||
return Minio(HOST, access_key=key("s3_%s_access_key" % who), secret_key=key("s3_%s_secret_key" % who),
|
||||
secure=False, region=os.environ.get("S3_REGION", "us-east-1"))
|
||||
|
||||
|
||||
results = []
|
||||
|
||||
|
||||
def expect(name, should_succeed, fn):
|
||||
try:
|
||||
fn()
|
||||
ok, got = should_succeed, "allowed"
|
||||
except S3Error as exc:
|
||||
ok, got = not should_succeed, exc.code
|
||||
except Exception as exc: # noqa: BLE001
|
||||
ok, got = False, type(exc).__name__
|
||||
results.append({"check": name, "expected": "allowed" if should_succeed else "denied", "got": got, "pass": ok})
|
||||
|
||||
|
||||
def http_status(method, url, data=None):
|
||||
req = urllib.request.Request(url, data=data, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=3) as resp:
|
||||
return resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main():
|
||||
api, janitor, worker = client("api"), client("janitor"), client("worker")
|
||||
probe = "originals/s3check-" + uuid.uuid4().hex
|
||||
expect("api: PutObject в свой bucket", True, lambda: api.put_object(BUCKET, probe, io.BytesIO(b"x" * 16), 16))
|
||||
for name, method, url, data in (
|
||||
("anonymous GET object", "GET", "%s/%s/%s" % (ENDPOINT, BUCKET, probe), None),
|
||||
("anonymous ListBucket", "GET", "%s/%s?list-type=2" % (ENDPOINT, BUCKET), None),
|
||||
("anonymous ListAllMyBuckets", "GET", ENDPOINT + "/", None),
|
||||
("anonymous PUT object", "PUT", "%s/%s/anon-%s" % (ENDPOINT, BUCKET, uuid.uuid4().hex), b"z")):
|
||||
code = http_status(method, url, data)
|
||||
results.append({"check": name, "expected": "403", "got": code, "pass": code in (401, 403)})
|
||||
expect("api: GetObject", True, lambda: api.get_object(BUCKET, probe).read())
|
||||
expect("api: ListBucket", False, lambda: list(api.list_objects(BUCKET, recursive=True)))
|
||||
expect("api: CreateBucket", False, lambda: api.make_bucket("evil-" + uuid.uuid4().hex[:8]))
|
||||
expect("api: DeleteBucket", False, lambda: api.remove_bucket(BUCKET))
|
||||
expect("api: PutBucketPolicy (сделать публичным)", False, lambda: api.set_bucket_policy(BUCKET, json.dumps({
|
||||
"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": "*", "Action": ["s3:GetObject"],
|
||||
"Resource": ["arn:aws:s3:::%s/*" % BUCKET]}]})))
|
||||
expect("janitor: GetObject (чтение содержимого)", False, lambda: janitor.get_object(BUCKET, probe).read())
|
||||
expect("janitor: ListBucket", True, lambda: list(janitor.list_objects(BUCKET, prefix="originals/", recursive=True)))
|
||||
expect("janitor: CreateBucket", False, lambda: janitor.make_bucket("evil-" + uuid.uuid4().hex[:8]))
|
||||
expect("worker: GetObject (чтение staging)", True, lambda: worker.get_object(BUCKET, probe).read())
|
||||
expect("worker: PutObject (результат)", True, lambda: worker.put_object(BUCKET, probe + ".w", io.BytesIO(b"y"), 1))
|
||||
expect("worker: DeleteObject (очистка staging)", True, lambda: worker.remove_object(BUCKET, probe + ".w"))
|
||||
expect("worker: ListBucket", False, lambda: list(worker.list_objects(BUCKET, recursive=True)))
|
||||
expect("worker: CreateBucket", False, lambda: worker.make_bucket("evil-" + uuid.uuid4().hex[:8]))
|
||||
expect("api: DeleteObject (компенсация)", True, lambda: api.remove_object(BUCKET, probe))
|
||||
s3_host = HOST.split(":")[0]
|
||||
for port, role in ((9333, "master UI/API"), (8080, "volume"), (8888, "filer UI/API"), (8111, "IAM API"),
|
||||
(8181, "Iceberg"), (9101, "Lance"), (8333, "S3 API")):
|
||||
try:
|
||||
socket.create_connection((s3_host, port), timeout=2).close()
|
||||
got = "open"
|
||||
except OSError:
|
||||
got = "closed"
|
||||
want = "open" if port == 8333 else "closed"
|
||||
results.append({"check": "port %d (%s) из backend-сети" % (port, role), "expected": want, "got": got,
|
||||
"pass": got == want})
|
||||
for r in results:
|
||||
print(("PASS " if r["pass"] else "FAIL ") + "%-45s expected=%-8s got=%s" % (r["check"], r["expected"], r["got"]))
|
||||
failed = [r for r in results if not r["pass"]]
|
||||
print(json.dumps({"checks": len(results), "failed": len(failed)}))
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Учебная порча staging-объекта (контейнер checks, ключ api): подменяет байты на другие
|
||||
байты той же длины. Нужна worker_faults.py для проверки постоянной ошибки staging_corrupted.
|
||||
|
||||
docker compose --profile tools run --rm checks /app/student-tests/s3_tamper.py staging/<hex>
|
||||
"""
|
||||
import io
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app/src")
|
||||
os.environ.setdefault("S3_ACCESS_KEY_ID_FILE", "/run/secrets/s3_api_access_key")
|
||||
os.environ.setdefault("S3_SECRET_ACCESS_KEY_FILE", "/run/secrets/s3_api_secret_key")
|
||||
import objstore # noqa: E402
|
||||
|
||||
key = sys.argv[1]
|
||||
if not re.fullmatch(r"staging/[0-9a-f]{32}", key):
|
||||
sys.exit("refusing: not a staging key")
|
||||
store = objstore.ObjectStore(objstore.make_client(), os.environ.get("S3_BUCKET", "messenger-attachments"))
|
||||
resp = store.open(key)
|
||||
data = resp.read()
|
||||
resp.close()
|
||||
resp.release_conn()
|
||||
bad = bytes(b ^ 0x5A for b in data)
|
||||
store.put(key, io.BytesIO(bad), len(bad), "application/octet-stream")
|
||||
print("tampered", key, len(bad))
|
||||
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Лимиты worker на «плохие» входы: пиксели, память, CPU-время, повреждённые данные.
|
||||
Запускает src/transform.py так же, как worker (подпроцесс), В КОНТЕЙНЕРЕ checks:
|
||||
|
||||
docker compose --profile tools run --rm checks /app/student-tests/transform_limits.py \
|
||||
--output /app/evidence/transform-limits.json
|
||||
|
||||
API отсекает > 10 MiB и > 40 Мпикс ещё до очереди (lab4); здесь проверяется вторая линия
|
||||
обороны — сам worker, если в staging всё же оказался такой файл.
|
||||
"""
|
||||
import argparse
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
TRANSFORM = "/app/src/transform.py"
|
||||
EXPECT = {0: "ok", 3: "invalid_image", 4: "image_too_large", 5: "resource_limit"}
|
||||
|
||||
|
||||
def run(data, ctype, env=None, max_pixels=40000000, timeout=60):
|
||||
"""Как worker: подпроцесс; timeout — wall-clock (у worker JOB_TIMEOUT_SECONDS)."""
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
src = os.path.join(d, "in")
|
||||
Path(src).write_bytes(data)
|
||||
out = os.path.join(d, "out")
|
||||
os.mkdir(out)
|
||||
e = dict(os.environ, **(env or {}))
|
||||
t = time.monotonic()
|
||||
try:
|
||||
p = subprocess.run([sys.executable, TRANSFORM, src, out, ctype, str(max_pixels)], capture_output=True,
|
||||
env=e, timeout=timeout)
|
||||
rc = p.returncode
|
||||
except subprocess.TimeoutExpired:
|
||||
rc = "timeout"
|
||||
dur = round(time.monotonic() - t, 2)
|
||||
if rc == "timeout":
|
||||
return {"exit": rc, "class": "processing_timeout", "seconds": round(time.monotonic() - t, 2)}
|
||||
cls = EXPECT.get(rc, "killed_by_signal_%d" % -rc if isinstance(rc, int) and rc < 0 else str(rc))
|
||||
return {"exit": rc, "class": cls, "seconds": dur}
|
||||
|
||||
|
||||
def png(size, mode="RGB", color=(10, 20, 30)):
|
||||
buf = io.BytesIO()
|
||||
Image.new(mode, size, color).save(buf, "PNG")
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
ok = png((640, 480))
|
||||
big = png((6000, 6000)) # 36 Мпикс: допустимо, ~108 MiB в RGB
|
||||
# Несжимаемый RGBA-шум ~40 Мпикс: декодирование, alpha-композиция и resize — секунды CPU.
|
||||
buf = io.BytesIO()
|
||||
n = Image.effect_noise((6300, 6300), 80)
|
||||
Image.merge("RGBA", [n, n.transpose(Image.Transpose.ROTATE_90), n, n]).save(buf, "PNG", compress_level=1)
|
||||
heavy = buf.getvalue()
|
||||
cases = [
|
||||
("valid 640x480", ok, "image/png", {}, 40000000, "ok"),
|
||||
("49 Mpx PNG (> MAX_IMAGE_PIXELS)", png((7000, 7000)), "image/png", {}, 40000000, "image_too_large"),
|
||||
("36 Mpx PNG, normal limits", big, "image/png", {}, 40000000, "ok"),
|
||||
("36 Mpx PNG, memory limit 160 MiB", big, "image/png", {"TRANSFORM_MEMORY_LIMIT_MB": "160"}, 40000000,
|
||||
"resource_limit"),
|
||||
("40 Mpx RGBA noise PNG, CPU limit 1 s", heavy, "image/png", {"TRANSFORM_CPU_SECONDS": "1"}, 40000000,
|
||||
"killed_by_signal_24"),
|
||||
("40 Mpx RGBA noise PNG, wall timeout 0.5 s", heavy, "image/png", {}, 40000000, "processing_timeout"),
|
||||
("truncated PNG", ok[: len(ok) // 2], "image/png", {}, 40000000, "invalid_image"),
|
||||
("garbage declared as JPEG", b"\xff\xd8\xff" + os.urandom(5000), "image/jpeg", {}, 40000000, "invalid_image"),
|
||||
("PNG declared as JPEG", ok, "image/jpeg", {}, 40000000, "invalid_image"),
|
||||
]
|
||||
results = []
|
||||
failed = 0
|
||||
for name, data, ctype, env, mp, want in cases:
|
||||
r = run(data, ctype, env, mp, timeout=0.5 if want == "processing_timeout" else 60)
|
||||
r.update(case=name, expected=want, input_bytes=len(data), env=env)
|
||||
r["pass"] = r["class"] == want or (want.startswith("killed") and r["class"] in ("resource_limit", want))
|
||||
failed += not r["pass"]
|
||||
results.append(r)
|
||||
print("%s %-36s -> %s (%.2fs)" % ("PASS" if r["pass"] else "FAIL", name, r["class"], r["seconds"]), flush=True)
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps({"cases": results}, ensure_ascii=False, indent=1) + "\n")
|
||||
print("PASS" if not failed else "FAIL", "%d/%d" % (len(cases) - failed, len(cases)))
|
||||
sys.exit(1 if failed else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,177 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Пик памяти API при одновременных загрузках крупных файлов (уровень 4).
|
||||
|
||||
python3 student-tests/upload_memory.py --concurrency 12 --rounds 3 --output evidence/upload-memory-c12.json
|
||||
|
||||
Гипотеза: потоковый приём (буфер 64 KiB -> безымянный временный файл), multipart PUT
|
||||
частями по 5 MiB и draft-декодирование JPEG держат RAM реплики ограниченной и не
|
||||
зависящей линейно от числа одновременных загрузок; 10+ загрузок по ~10 MiB не
|
||||
приближают процесс к лимиту контейнера 256 MiB.
|
||||
|
||||
Методика: перед прогоном API-реплики перезапускаются (VmHWM считается с нуля). В каждой
|
||||
реплике фоново (docker exec ... while sleep 0.05) читаются VmRSS процесса (PID 1) и
|
||||
anon/file из cgroup memory.stat. Нагрузка — concurrency потоков, каждый грузит свой
|
||||
файл через LB; файлы: несжимаемый PNG ~9.99 MiB и (если на хосте есть Pillow) JPEG
|
||||
6000x6000 = 36 Мпикс. В конце — VmHWM (пиковый RSS процесса) и memory.peak cgroup.
|
||||
file-память cgroup — page cache временных файлов, она вытесняема и OOM не вызывает.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
from client import Client # noqa: E402
|
||||
from attachments_negative import noise_png # noqa: E402
|
||||
|
||||
SAMPLER = ("while true; do r=$(grep VmRSS /proc/1/status | tr -s ' ' | cut -d' ' -f2); "
|
||||
"a=$(grep '^anon ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); "
|
||||
"f=$(grep '^file ' /sys/fs/cgroup/memory.stat | cut -d' ' -f2); echo \"$r $a $f\"; sleep 0.05; done")
|
||||
|
||||
|
||||
def sh(*cmd):
|
||||
return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, check=True).stdout
|
||||
|
||||
|
||||
def containers():
|
||||
return sh("docker", "compose", "ps", "-q", "api").split()
|
||||
|
||||
|
||||
def proc_status(cid, field):
|
||||
out = sh("docker", "exec", cid, "grep", field, "/proc/1/status")
|
||||
return int(out.split()[1]) // 1024 # kB -> MiB
|
||||
|
||||
|
||||
class Sampler:
|
||||
def __init__(self, cid):
|
||||
self.cid, self.samples = cid, []
|
||||
self.p = subprocess.Popen(["docker", "exec", cid, "sh", "-c", SAMPLER], stdout=subprocess.PIPE, text=True)
|
||||
threading.Thread(target=self._read, daemon=True).start()
|
||||
|
||||
def _read(self):
|
||||
for line in self.p.stdout:
|
||||
parts = line.split()
|
||||
if len(parts) == 3:
|
||||
self.samples.append((time.monotonic(), int(parts[0]) / 1024, int(parts[1]) / 2**20, int(parts[2]) / 2**20))
|
||||
|
||||
def stop(self):
|
||||
self.p.terminate()
|
||||
|
||||
|
||||
def make_jpeg():
|
||||
try:
|
||||
from PIL import Image
|
||||
except ImportError:
|
||||
return None
|
||||
im = Image.effect_noise((6000, 6000), 40).convert("RGB")
|
||||
for q in (60, 45, 30, 20):
|
||||
b = io.BytesIO()
|
||||
im.save(b, "JPEG", quality=q)
|
||||
if b.tell() <= 10 * 1024 * 1024:
|
||||
return b.getvalue()
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--base-url", default="http://localhost:8080")
|
||||
ap.add_argument("--concurrency", type=int, default=12)
|
||||
ap.add_argument("--rounds", type=int, default=3)
|
||||
ap.add_argument("--no-restart", action="store_true")
|
||||
ap.add_argument("--output")
|
||||
args = ap.parse_args()
|
||||
files = [("image/png", noise_png(10 * 1024 * 1024))]
|
||||
jpeg = make_jpeg()
|
||||
if jpeg:
|
||||
files.append(("image/jpeg", jpeg))
|
||||
if not args.no_restart:
|
||||
sh("docker", "compose", "restart", "api")
|
||||
sh("docker", "compose", "up", "-d", "--no-deps", "--no-recreate", "--wait", "api")
|
||||
time.sleep(4)
|
||||
c = Client(args.base_url, timeout=120)
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
# прогрев: один upload каждого типа (импорт Pillow, пулы соединений)
|
||||
for ctype, data in files:
|
||||
c.call("POST", "/api/v1/chats/%s/attachments?filename=w" % chat["id"], 202, actor=alice, raw=data,
|
||||
headers={"Content-Type": ctype})
|
||||
cids = containers()
|
||||
idle = {cid[:12]: {"rss_mib": proc_status(cid, "VmRSS"), "hwm_mib_after_warmup": proc_status(cid, "VmHWM")}
|
||||
for cid in cids}
|
||||
s3_cid = sh("docker", "compose", "ps", "-q", "s3").split()[0]
|
||||
s3_before = {"rss_mib": proc_status(s3_cid, "VmRSS"), "oom_killed": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip()}
|
||||
samplers = [Sampler(cid) for cid in cids]
|
||||
s3_sampler = Sampler(s3_cid)
|
||||
time.sleep(1)
|
||||
results, lock = [], threading.Lock()
|
||||
|
||||
def worker(i):
|
||||
ctype, data = files[i % len(files)]
|
||||
t = time.monotonic()
|
||||
try:
|
||||
r = c.call("POST", "/api/v1/chats/%s/attachments?filename=m%d" % (chat["id"], i), [202, 503], actor=alice,
|
||||
raw=data, headers={"Content-Type": ctype}, schema=None)
|
||||
st, inst = r["status"], r["headers"].get("X-Instance-Id")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
st, inst = type(exc).__name__, None
|
||||
with lock:
|
||||
results.append({"type": ctype, "status": st, "instance": inst, "s": round(time.monotonic() - t, 2)})
|
||||
|
||||
started = time.monotonic()
|
||||
for _ in range(args.rounds):
|
||||
threads = [threading.Thread(target=worker, args=(i,)) for i in range(args.concurrency)]
|
||||
for th in threads:
|
||||
th.start()
|
||||
for th in threads:
|
||||
th.join()
|
||||
wall = time.monotonic() - started
|
||||
time.sleep(1)
|
||||
for s in samplers + [s3_sampler]:
|
||||
s.stop()
|
||||
per = {}
|
||||
for cid, s in zip(cids, samplers):
|
||||
k = cid[:12]
|
||||
per[k] = dict(idle[k],
|
||||
samples=len(s.samples),
|
||||
peak_rss_mib_sampled=round(max(x[1] for x in s.samples), 1),
|
||||
peak_anon_mib=round(max(x[2] for x in s.samples), 1),
|
||||
peak_file_cache_mib=round(max(x[3] for x in s.samples), 1),
|
||||
vmhwm_mib=proc_status(cid, "VmHWM"),
|
||||
cgroup_memory_peak_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak")) / 2**20, 1),
|
||||
memory_limit_mib=round(int(sh("docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20),
|
||||
uploads_served=sum(1 for r in results if r["instance"] == k))
|
||||
durations = sorted(r["s"] for r in results if r["status"] == 202)
|
||||
summary = {
|
||||
"concurrency": args.concurrency, "rounds": args.rounds,
|
||||
"files": [{"type": t, "bytes": len(d), "sha256": hashlib.sha256(d).hexdigest()[:16]} for t, d in files],
|
||||
"uploads": len(results), "status_counts": {str(k): sum(1 for r in results if r["status"] == k)
|
||||
for k in {r["status"] for r in results}},
|
||||
"megabytes_uploaded_ok": round(sum(len(files[i % len(files)][1]) for i, r in enumerate(results)
|
||||
if r["status"] == 202) / 2**20, 1),
|
||||
"wall_s": round(wall, 1),
|
||||
"latency_s_p50_max": [durations[len(durations) // 2], durations[-1]] if durations else None,
|
||||
"replicas": per,
|
||||
"s3": dict(s3_before,
|
||||
peak_rss_mib_sampled=round(max(x[1] for x in s3_sampler.samples), 1) if s3_sampler.samples else None,
|
||||
peak_anon_mib=round(max(x[2] for x in s3_sampler.samples), 1) if s3_sampler.samples else None,
|
||||
restarts=int(sh("docker", "inspect", "-f", "{{.RestartCount}}", s3_cid)),
|
||||
oom_killed_after=sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", s3_cid).strip(),
|
||||
memory_limit_mib=round(int(sh("docker", "exec", s3_cid, "cat", "/sys/fs/cgroup/memory.max")) / 2**20)),
|
||||
"oom_kills": sh("docker", "inspect", "-f", "{{.State.OOMKilled}}", *cids).split(),
|
||||
}
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка вариантов по контракту (запускается В КОНТЕЙНЕРЕ checks, где есть Pillow):
|
||||
|
||||
docker compose --profile tools run --rm checks /app/student-tests/variants_check.py \
|
||||
--base-url http://lb:8080 --output /app/evidence/variants-check.json
|
||||
|
||||
Для каждого входа: ready, ровно 3 варианта, метаданные = фактическим байтам (размер, SHA-256,
|
||||
декодированные размеры, формат), оригинал неизменен. Thumbnail: JPEG, вписан в 256x256 без
|
||||
увеличения, округление вниз; EXIF orientation применена; alpha -> белый. Crop: JPEG 128x128.
|
||||
Optimized: WebP, вписан в 1280x1280 без увеличения. В вариантах нет EXIF/ICC.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, "/app/tests")
|
||||
from client import Client, check # noqa: E402
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
FIXTURE = Path("/app/tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def fit(w, h, box):
|
||||
if w <= box and h <= box:
|
||||
return w, h
|
||||
return (box, max(1, h * box // w)) if w >= h else (max(1, w * box // h), box)
|
||||
|
||||
|
||||
def encode(img, fmt, **kw):
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, fmt, **kw)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def cases():
|
||||
out = [("fixture 320x200 PNG", FIXTURE, "image/png", (320, 200), None)]
|
||||
# RGBA: левая половина прозрачная -> в JPEG должна стать белой.
|
||||
rgba = Image.new("RGBA", (300, 300), (200, 20, 20, 255))
|
||||
rgba.paste((0, 0, 0, 0), (0, 0, 150, 300))
|
||||
out.append(("RGBA 300x300 half transparent", encode(rgba, "PNG"), "image/png", (300, 300), "alpha"))
|
||||
# EXIF orientation 6 (поворот 90° по часовой): хранится 400x200, показывается 200x400.
|
||||
# Верхняя половина ХРАНИМОГО кадра красная -> после поворота красной становится правая половина.
|
||||
raw = Image.new("RGB", (400, 200), (0, 0, 255))
|
||||
raw.paste((255, 0, 0), (0, 0, 400, 100))
|
||||
exif = Image.Exif()
|
||||
exif[0x0112] = 6
|
||||
exif[0x010F] = "MTUSI test camera" # Make: проверяем, что EXIF не утёк в варианты
|
||||
out.append(("JPEG 400x200 EXIF orientation=6", encode(raw, "JPEG", quality=95, exif=exif.tobytes()),
|
||||
"image/jpeg", (200, 400), "orientation"))
|
||||
out.append(("tiny PNG 10x5 (no upscale)", encode(Image.new("RGB", (10, 5), (9, 200, 9)), "PNG"),
|
||||
"image/png", (10, 5), None))
|
||||
big = Image.radial_gradient("L").resize((3000, 2000)).convert("RGB")
|
||||
out.append(("JPEG 3000x2000", encode(big, "JPEG", quality=90), "image/jpeg", (3000, 2000), None))
|
||||
tall = Image.new("RGB", (100, 1000), (50, 60, 70))
|
||||
out.append(("tall PNG 100x1000", encode(tall, "PNG"), "image/png", (100, 1000), None))
|
||||
pal = Image.new("P", (64, 64), 1)
|
||||
pal.putpalette([255, 255, 255, 10, 200, 10] + [0] * 762)
|
||||
out.append(("palette PNG with transparency", encode(pal, "PNG", transparency=1), "image/png", (64, 64), "alpha"))
|
||||
return out
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://lb:8080")
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
results = []
|
||||
pending = []
|
||||
for name, data, ctype, (ow, oh), special in cases():
|
||||
att = c.call("POST", "/api/v1/chats/%s/attachments?filename=v.%s" % (chat["id"], ctype.split("/")[1]), 202,
|
||||
actor=alice, raw=data, headers={"Content-Type": ctype}, schema="Attachment")["json"]
|
||||
pending.append((name, data, ctype, ow, oh, special, att["id"]))
|
||||
for name, data, ctype, ow, oh, special, att_id in pending:
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
att = c.call("GET", "/api/v1/attachments/" + att_id, actor=bob, schema="Attachment")["json"]
|
||||
if att["status"] in ("ready", "failed") or time.monotonic() > deadline:
|
||||
break
|
||||
time.sleep(0.3)
|
||||
check(att["status"] == "ready", "%s: status %s" % (name, att["status"]))
|
||||
r = {"case": name, "source_oriented": [ow, oh], "variants": {}}
|
||||
orig = c.call("GET", "/api/v1/attachments/%s/content" % att_id, actor=bob)
|
||||
check(orig["bytes"] == data and orig["headers"].get_content_type() == ctype, name + ": original changed")
|
||||
check(sorted(v["name"] for v in att["variants"]) == ["crop", "optimized", "thumbnail"], name + ": variants")
|
||||
expect = {"thumbnail": ("image/jpeg", "JPEG", fit(ow, oh, 256)),
|
||||
"crop": ("image/jpeg", "JPEG", (128, 128)),
|
||||
"optimized": ("image/webp", "WEBP", fit(ow, oh, 1280))}
|
||||
for v in att["variants"]:
|
||||
ct, fmt, size = expect[v["name"]]
|
||||
body = c.call("GET", "/api/v1/attachments/%s/content?variant=%s" % (att_id, v["name"]), actor=bob)
|
||||
b = body["bytes"]
|
||||
img = Image.open(io.BytesIO(b))
|
||||
img.load()
|
||||
check(v["content_type"] == ct and body["headers"].get_content_type() == ct, name + ": content type")
|
||||
check(img.format == fmt, name + ": real format " + str(img.format))
|
||||
check((v["width"], v["height"]) == size == img.size, "%s %s: size %s meta %s expected %s" % (
|
||||
name, v["name"], img.size, (v["width"], v["height"]), size))
|
||||
check(v["size_bytes"] == len(b) and v["sha256"] == hashlib.sha256(b).hexdigest(), name + ": checksum")
|
||||
no_meta = not img.getexif() and "icc_profile" not in img.info and "exif" not in img.info
|
||||
check(no_meta, name + ": metadata leaked into " + v["name"])
|
||||
item = {"size": list(img.size), "format": fmt, "bytes": len(b), "no_exif_icc": no_meta}
|
||||
if special == "alpha" and v["name"] in ("thumbnail", "crop"):
|
||||
px = img.convert("RGB").getpixel((1, img.size[1] // 2))
|
||||
check(min(px) >= 245, name + ": transparent area is not white: %s" % (px,))
|
||||
item["transparent_pixel_rgb"] = list(px)
|
||||
if special == "orientation" and v["name"] == "thumbnail":
|
||||
w, h = img.size
|
||||
left, right = img.getpixel((w // 4, h // 2)), img.getpixel((3 * w // 4, h // 2))
|
||||
check(right[0] > 200 and right[2] < 60 and left[2] > 200, name + ": orientation not applied")
|
||||
item["left_rgb"], item["right_rgb"] = list(left), list(right)
|
||||
r["variants"][v["name"]] = item
|
||||
results.append(r)
|
||||
print("PASS %-36s %s" % (name, {k: tuple(x["size"]) for k, x in r["variants"].items()}), flush=True)
|
||||
c.call("GET", "/api/v1/attachments/%s/content?variant=bogus" % pending[0][6], 400, actor=bob)
|
||||
if args.output:
|
||||
Path(args.output).write_text(json.dumps({"cases": results, "unknown_variant_status": 400},
|
||||
ensure_ascii=False, indent=1) + "\n")
|
||||
print("PASS: %d inputs, all variants match contract" % len(results))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,147 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 4: отказы worker, retry/backoff, failed + replay, DLQ.
|
||||
|
||||
python3 student-tests/worker_faults.py --output evidence/worker-faults.json
|
||||
|
||||
На время теста worker пересоздаются с WORKER_FAULT_INJECTION=1 и JOB_RETRY_BASE_SECONDS=1
|
||||
(короче ожидание backoff); в конце возвращаются к значениям из .env.
|
||||
Точка отказа задаётся именем файла fault-<point>.png (src/worker.py).
|
||||
|
||||
crash-after-claim процесс умер в processing до любых эффектов
|
||||
crash-after-s3-put все объекты записаны в S3, COMMIT ready не сделан, ack не отправлен
|
||||
crash-after-commit ready зафиксирован, ack не отправлен (и staging не удалён)
|
||||
transient-2 временная ошибка в попытках 1–2: backoff с jitter, ready с 3-й
|
||||
transient-always временная ошибка до replay: 5 попыток -> failed(retry_exhausted)
|
||||
-> content 409, прикрепление 409 -> ops replay -> ready; повторный replay отклонён
|
||||
staging tamper staging подменён до обработки -> failed(staging_corrupted) без повторов
|
||||
poison message невалидное сообщение -> reject -> DLQ -> ops dlq --purge-invalid
|
||||
Инварианты после каждого: 3 варианта, 4 конечных объекта, staging убран (с janitor).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from lab5lib import (Client, Log, SAMPLE_PNG, check, compose, events, get_att, invariants, ops, png, queues,
|
||||
recreate, upload, wait_status, wait_until)
|
||||
|
||||
FAULT_ENV = {"WORKER_FAULT_INJECTION": "1", "JOB_RETRY_BASE_SECONDS": "1", "JOB_RETRY_MAX_SECONDS": "8"}
|
||||
|
||||
|
||||
def inspect_until(att_id, pred, timeout=60, what="state"):
|
||||
return wait_until(lambda: (lambda i: i if pred(i) else None)(ops("inspect", att_id)), timeout, 1, what)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
log = Log("worker faults, retries, failed registry, replay, DLQ")
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
recreate("worker", FAULT_ENV)
|
||||
log.step("workers recreated with fault injection", env=FAULT_ENV)
|
||||
report = {}
|
||||
try:
|
||||
# --- крахи процесса в разных окнах -------------------------------------------------
|
||||
for point in ("crash-after-claim", "crash-after-s3-put", "crash-after-commit"):
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-%s.png" % point)
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
|
||||
check(ready["status"] == "ready", point + ": not ready")
|
||||
if point == "crash-after-commit":
|
||||
# ready уже был зафиксирован; redelivery после рестарта -> no-op ack.
|
||||
inspect_until(att["id"], lambda i: "duplicate_ignored" in events(i), 60, "redelivery no-op")
|
||||
compose("exec", "-T", "janitor", "python", "/app/src/janitor.py", "--once")
|
||||
info = ops("inspect", att["id"])
|
||||
inv = invariants(info)
|
||||
check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], point + ": invariants")
|
||||
report[point] = {"attachment_id": att["id"], "events": events(info), "invariants": inv}
|
||||
log.step(point, events=events(info), invariants=inv)
|
||||
ids = compose("ps", "-q", "worker").stdout.split()
|
||||
rc = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.RestartCount}} {{.State.Status}}"] + ids,
|
||||
capture_output=True, text=True).stdout.split("\n")
|
||||
log.step("worker containers after crashes (restart: unless-stopped)", containers=[x for x in rc if x])
|
||||
|
||||
# --- временные ошибки: backoff + jitter ---------------------------------------------
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-2.png")
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=90)
|
||||
check(ready["status"] == "ready", "transient-2 not ready")
|
||||
info = ops("inspect", att["id"])
|
||||
delays = [e["detail"]["delay_seconds"] for e in info["events"] if e["event"] == "retry_scheduled"]
|
||||
claims = [e["at"] for e in info["events"] if e["event"] == "claimed"]
|
||||
bounds = [[0.5 * min(8, 2 ** (n - 1)), min(8, 2 ** (n - 1))] for n in (1, 2)]
|
||||
check(len(delays) == 2 and all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "backoff bounds")
|
||||
inv = invariants(info)
|
||||
check(inv["attempt"] == 3 and inv["variants_ok"], "transient-2 attempts")
|
||||
report["transient-2"] = {"attachment_id": att["id"], "delays": delays, "bounds": bounds, "claims_at": claims,
|
||||
"invariants": inv}
|
||||
log.step("transient-2: retried with backoff", delays=delays, bounds=bounds, invariants=inv)
|
||||
|
||||
# --- исчерпание попыток -> failed -> replay ----------------------------------------
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "fault-transient-always.png")
|
||||
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=120)
|
||||
check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "expected retry_exhausted")
|
||||
r409 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
|
||||
m409 = c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice,
|
||||
body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "f-" + att["id"]})
|
||||
registry = ops("failed")
|
||||
in_registry = [r for r in registry["failed"] if r["attachment_id"] == att["id"]]
|
||||
check(in_registry and in_registry[0]["replayable"], "failed registry")
|
||||
info = ops("inspect", att["id"])
|
||||
log.step("transient-always: failed after max attempts", error_code=failed["error_code"],
|
||||
attempts=info["jobs"][0]["attempt"], content=r409["status"], attach=m409["status"],
|
||||
registry_entry=in_registry[0])
|
||||
dry = ops("replay", att["id"], "--dry-run")
|
||||
rep = ops("replay", att["id"], "--reason", "test: S3 restored")
|
||||
check(rep["replay"][0]["result"] == "replayed", "replay")
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
|
||||
check(ready["status"] == "ready", "not ready after replay")
|
||||
again = ops("replay", att["id"])
|
||||
check(again["replay"][0]["result"] == "refused_status_ready", "replay of ready must be refused")
|
||||
info = ops("inspect", att["id"])
|
||||
inv = invariants(info)
|
||||
check(inv["variants_ok"] and inv["final_objects_ok"], "replay invariants")
|
||||
report["transient-always+replay"] = {"attachment_id": att["id"], "dry_run": dry["replay"][0]["result"],
|
||||
"replay": rep["replay"][0]["result"],
|
||||
"second_replay": again["replay"][0]["result"],
|
||||
"replay_count": info["jobs"][0]["replay_count"],
|
||||
"events": events(info), "invariants": inv}
|
||||
log.step("replay -> ready; replay of ready refused", **report["transient-always+replay"])
|
||||
|
||||
# --- подмена staging: постоянная ошибка без повторов --------------------------------
|
||||
compose("stop", "worker")
|
||||
att = upload(c, alice, chat, png(50, 40), "tamper.png")
|
||||
info = ops("inspect", att["id"])
|
||||
key = info["attachment"]["staging_key"]
|
||||
compose("--profile", "tools", "run", "--rm", "-T", "checks", "/app/student-tests/s3_tamper.py", key)
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker", env=FAULT_ENV)
|
||||
failed = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
|
||||
info = ops("inspect", att["id"])
|
||||
check(failed["error_code"] == "staging_corrupted" and info["jobs"][0]["attempt"] == 1, "tamper -> permanent")
|
||||
report["staging-tamper"] = {"attachment_id": att["id"], "error_code": failed["error_code"],
|
||||
"attempts": info["jobs"][0]["attempt"], "events": events(info)}
|
||||
log.step("tampered staging -> failed without retries", **report["staging-tamper"])
|
||||
|
||||
# --- poison message -> DLQ ----------------------------------------------------------
|
||||
before = queues()["attachments.dlq"]["messages"]
|
||||
ops("publish-raw", json.dumps({"schema_version": 1, "hello": "not a job"}))
|
||||
q = wait_until(lambda: (lambda x: x if x["messages"] > before else None)(queues()["attachments.dlq"]),
|
||||
30, what="message in DLQ")
|
||||
listed = ops("dlq")
|
||||
purged = ops("dlq", "--purge-invalid")
|
||||
# Счётчики quorum-очереди в rabbitmqctl обновляются с задержкой в секунды.
|
||||
after = wait_until(lambda: (lambda n: "0" if n == 0 else None)(queues()["attachments.dlq"]["messages"]),
|
||||
20, what="empty DLQ") and 0
|
||||
report["poison"] = {"dlq_before": before, "dlq_after_publish": q["messages"], "listed": listed["dlq_messages"],
|
||||
"purged": [m["action"] for m in purged["dlq_messages"]], "dlq_after_purge": after}
|
||||
log.step("poison message dead-lettered and purged", **report["poison"])
|
||||
check(after == 0, "DLQ not empty after purge")
|
||||
finally:
|
||||
recreate("worker")
|
||||
log.step("workers restored to .env configuration")
|
||||
log.save(args.output, results=report)
|
||||
print("PASS: worker crashes, backoff, retry_exhausted -> replay, staging tamper, DLQ")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Уровень 3, сценарий защиты: worker остановлен -> upload 202 -> сообщение с вложением ->
|
||||
рестарт API и брокера (volumes сохраняются) -> worker запущен -> ready -> повторная доставка.
|
||||
|
||||
python3 student-tests/worker_outage.py --output evidence/worker-outage.json
|
||||
|
||||
Проверяется:
|
||||
* 202 queued при остановленных worker; GET queued; content и thumbnail -> 409;
|
||||
* сообщение с queued-вложением и обычное сообщение -> 201 (остальной API работает);
|
||||
* задание лежит в durable quorum-очереди (messages_ready=1) и переживает рестарт брокера;
|
||||
* после рестарта API и брокера и запуска worker вложение ready, thumbnail 256x160,
|
||||
оригинал побайтно совпадает;
|
||||
* повторная доставка (тот же job_id ещё раз в очередь) -> no-op, вариантов по-прежнему 3.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from lab5lib import (SAMPLE_PNG, Client, Log, check, compose, events, get_att, invariants, ops, queues,
|
||||
upload, wait_status, wait_until)
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default=None)
|
||||
p.add_argument("--output")
|
||||
args = p.parse_args()
|
||||
log = Log("worker outage -> restart api+broker -> worker start -> ready -> redelivery")
|
||||
c = Client(args.base_url, timeout=30)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
|
||||
compose("stop", "worker")
|
||||
log.step("worker stopped", running=compose("ps", "-q", "--status", "running", "worker").stdout.split())
|
||||
before = queues()["attachments.process.v1"]
|
||||
|
||||
att = upload(c, alice, chat, SAMPLE_PNG, "outage.png")
|
||||
check(att["status"] == "queued", "expected queued")
|
||||
log.step("upload while worker is down", http=202, attachment_id=att["id"], status=att["status"])
|
||||
meta = get_att(c, bob, att["id"])
|
||||
r1 = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob)
|
||||
r2 = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], 409, actor=bob)
|
||||
log.step("metadata and content while queued", status=meta["status"], content=r1["status"],
|
||||
thumbnail=r2["status"], error_code=r1["json"]["error"]["code"])
|
||||
m1 = c.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||||
m2 = c.message(bob, chat, "другие сообщения работают")["json"]
|
||||
log.step("messages while worker is down", with_queued_attachment=m1["attachment_ids"] == [att["id"]],
|
||||
plain_message_id=m2["id"])
|
||||
|
||||
q = wait_until(lambda: (lambda x: x if x["messages_ready"] >= before["messages_ready"] + 1 else None)(
|
||||
queues()["attachments.process.v1"]), 15, what="job message in queue")
|
||||
info = ops("inspect", att["id"])
|
||||
log.step("job is durable in broker", queue=q, outbox_published=info["outbox"][0]["published_at"] is not None,
|
||||
job_id=info["jobs"][0]["id"], job_status=info["jobs"][0]["status"])
|
||||
job_id = info["jobs"][0]["id"]
|
||||
|
||||
compose("restart", "api")
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "api")
|
||||
log.step("api restarted", status=get_att(c, alice, att["id"])["status"])
|
||||
compose("restart", "broker")
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "broker")
|
||||
q2 = queues()["attachments.process.v1"]
|
||||
check(q2["messages_ready"] >= 1, "job message lost after broker restart")
|
||||
log.step("broker restarted with volume", queue=q2, status=get_att(c, alice, att["id"])["status"])
|
||||
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "worker")
|
||||
log.step("worker started")
|
||||
ready = wait_status(c, bob, att["id"], {"ready", "failed"}, timeout=60)
|
||||
check(ready["status"] == "ready" and ready["error_code"] is None, "attachment not ready")
|
||||
thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0]
|
||||
content = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)["bytes"]
|
||||
tb = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)["bytes"]
|
||||
check(content == SAMPLE_PNG, "original changed")
|
||||
check(hashlib.sha256(tb).hexdigest() == thumb["sha256"], "thumbnail checksum")
|
||||
log.step("ready after worker start", variants=[(v["name"], v["width"], v["height"]) for v in ready["variants"]],
|
||||
original_sha256_ok=True, thumbnail_bytes=len(tb))
|
||||
|
||||
# Повторная доставка: то же сообщение (тот же job_id) ещё раз в очередь.
|
||||
payload = {"schema_version": 1, "job_id": job_id, "type": "attachment.process.v1", "attachment_id": att["id"],
|
||||
"pipeline_version": 1, "attempt": 1, "created_at": "2026-11-13T00:00:00Z"}
|
||||
ops("publish-raw", json.dumps(payload))
|
||||
info = wait_until(lambda: (lambda i: i if "duplicate_ignored" in events(i) else None)(ops("inspect", att["id"])),
|
||||
30, interval=1, what="duplicate_ignored event")
|
||||
inv = invariants(info)
|
||||
check(inv["variants_ok"] and inv["final_objects_ok"] and not inv["staging_left"], "invariants violated")
|
||||
log.step("redelivery of the same job is a no-op", events=events(info), invariants=inv)
|
||||
after = get_att(c, bob, att["id"])
|
||||
check(after == ready, "metadata changed after redelivery")
|
||||
log.save(args.output, attachment_id=att["id"], job_id=job_id, events=info["events"])
|
||||
print("PASS: worker outage, api+broker restart, ready, duplicate delivery no-op")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user