Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2f0a5e221f | ||
|
|
9863bb0505 |
+40
-4
@@ -6,20 +6,56 @@ APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey (this solution: Redis since lab2):
|
||||
SESSION_STORE=redis
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
# SeaweedFS S3 (since lab4)
|
||||
S3_ENDPOINT=http://s3:8333
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
|
||||
# lab5: broker and worker. The API has NO broker credentials (it writes the transactional outbox);
|
||||
# compose.yaml passes per-service files: dispatcher -> /run/secrets/broker_url_dispatcher,
|
||||
# worker -> /run/secrets/broker_url_worker. This variable is therefore empty for api/janitor.
|
||||
BROKER_URL_FILE=
|
||||
WORKER_CONCURRENCY=2
|
||||
WORKER_PREFETCH=1
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
JOB_LEASE_SECONDS=15
|
||||
JOB_BACKOFF_BASE_SECONDS=2
|
||||
JOB_BACKOFF_MAX_SECONDS=60
|
||||
WORKER_MEMORY_LIMIT_MB=512
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
PIPELINE_VERSION=1
|
||||
# Fault injection (worker/dispatcher only, triggered by files in the faults volume): compose.yaml enables it on the
|
||||
# training stand by default; set FAULT_INJECTION=0 here to disable.
|
||||
|
||||
# Pools and timeouts (same as lab3)
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_WAIT_SECONDS=3
|
||||
DB_CONNECT_TIMEOUT_SECONDS=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
REDIS_POOL_MAX=20
|
||||
REDIS_TIMEOUT_MS=500
|
||||
SHUTDOWN_DRAIN_SECONDS=3
|
||||
SHUTDOWN_TIMEOUT_SECONDS=10
|
||||
|
||||
# lab4: S3 client and uploads
|
||||
S3_CONNECT_TIMEOUT_SECONDS=1
|
||||
S3_READ_TIMEOUT_SECONDS=5
|
||||
S3_MAX_ATTEMPTS=2
|
||||
S3_POOL_MAX=20
|
||||
UPLOAD_TIMEOUT_SECONDS=60
|
||||
UPLOAD_IDLE_TIMEOUT_SECONDS=10
|
||||
DECODE_CONCURRENCY=2
|
||||
# Cleanup of abandoned uploads and staging (janitor service): grace period and pass interval
|
||||
ATTACHMENT_GRACE_SECONDS=3600
|
||||
JANITOR_INTERVAL_SECONDS=300
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
*.sh text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.example text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
@@ -1,14 +1,14 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Бабиченко Максим, БВТ2303
|
||||
Целевая оценка (3/4/5): 4
|
||||
Трек 5, если заявлен: не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено: из lab4 — весь API, сессии в Redis, балансировка, приём/проверка файлов, приватный S3, janitor. Новое: RabbitMQ (quorum-очередь + DLQ, definitions с минимальными правами), transactional outbox + dispatcher (publisher confirms), отдельный worker (2 потока, prefetch 1, ручной ack, аренда с fencing, backoff с jitter, постоянные/временные ошибки), варианты thumbnail/crop/optimized в дочернем процессе с RLIMIT, replay и разбор DLQ (`ops.py`), очистка staging. Upload: 201 → 202 queued.
|
||||
Команды запуска из чистого clone: `Copy-Item .env.example .env; python scripts/init_secrets.py; docker compose up -d --build --wait; python student-tests/collect_evidence.py` (подробно — REPORT.md, раздел «Запуск»).
|
||||
Результаты проверок и ссылка на REPORT.md: REPORT.md, раздел «Доказательства критериев»; `evidence/summary.md`.
|
||||
Ограничения / известные ошибки: один узел RabbitMQ; worker использует ограниченный S3-ключ API; точки внедрения сбоев включены на учебном стенде (`FAULT_INJECTION=1`). См. REPORT.md, «Ограничения».
|
||||
Источники и использование ИИ: REPORT.md, шапка.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
FROM python:3.11.13-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt
|
||||
|
||||
COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py \
|
||||
pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py ./
|
||||
COPY migrations/ ./migrations/
|
||||
|
||||
RUN useradd --create-home --shell /usr/sbin/nologin appuser \
|
||||
&& mkdir -p /faults && chown appuser:appuser /faults
|
||||
# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher); владелец appuser
|
||||
# копируется в пустой named volume при первом монтировании.
|
||||
|
||||
USER appuser
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)"
|
||||
|
||||
CMD ["python", "main.py"]
|
||||
@@ -1,37 +1,293 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Бабиченко Максим, БВТ2303
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран
|
||||
- Commit/PR: <<ссылка на MR в Gitea после публикации>>
|
||||
- Использованные источники, библиотеки и помощь ИИ: всё из lab4 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py, boto3/botocore 1.43.116, Pillow 12.3.0; образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine, haproxy:3.2.25-alpine, chrislusf/seaweedfs:3.80). Новое: брокер RabbitMQ (образ `rabbitmq:4.1.4-alpine`), клиент pika 1.3.2. Документация: RabbitMQ — Reliability Guide, Consumer Acknowledgements and Publisher Confirms, Quorum Queues (`x-delivery-limit`, at-least-once dead-lettering, `x-overflow=reject-publish`), Definitions import, Access Control (configure/write/read); PostgreSQL `SELECT ... FOR UPDATE SKIP LOCKED`, `LISTEN/NOTIFY`; паттерн transactional outbox; W3C Trace Context (`traceparent`); Pillow (`Image.draft`, `ImageOps.exif_transpose`, параметры JPEG/WebP); `setrlimit(2)`. При написании кода и тестов использовался ИИ-ассистент (Claude); весь код проверен и прогнан мной, объяснить и изменить его могу на защите.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Проверял на Windows, все команды в PowerShell. Python нужен только для тестов (стандартная библиотека; Pillow на хосте не нужен — изображения для тестов генерирует и проверяет пробник внутри контейнера worker). Перед запуском остановить стенд lab4 (`docker compose down` в его папке), иначе занят порт 8080. Имя проекта Compose — `messenger-lab5`, поэтому volumes отдельные от lab4.
|
||||
|
||||
```powershell
|
||||
git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab5.git
|
||||
cd mtusi2026lab5
|
||||
git checkout solution/bvt2303-babichenko
|
||||
$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают
|
||||
Copy-Item .env.example .env
|
||||
python scripts/init_secrets.py # пароли PostgreSQL/Redis, ключи S3, пользователи RabbitMQ и
|
||||
# secrets/rabbitmq_definitions.json (папка secrets/ в .gitignore)
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose up -d --build --wait # postgres, s3, rabbitmq -> migrate, s3-init -> api x2, janitor,
|
||||
# dispatcher, worker -> lb
|
||||
docker compose ps -a # migrate и s3-init: Exited (0), остальные healthy
|
||||
|
||||
python student-tests/collect_evidence.py # все проверки lab1-5 по очереди (~15 мин), вывод в evidence/,
|
||||
# таблицы замеров — в этот отчёт
|
||||
```
|
||||
|
||||
Отдельные проверки (make на Windows нет, ниже то же самое без make):
|
||||
|
||||
| Команда из README | То же в PowerShell |
|
||||
| --- | --- |
|
||||
| `make check` | `python scripts/check_contract.py` |
|
||||
| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` |
|
||||
| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` |
|
||||
| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` |
|
||||
| все сценарии lab5 | `python student-tests/test_jobs.py` |
|
||||
| один сценарий lab5 | `python student-tests/test_jobs.py worker-down` (список — в начале файла) |
|
||||
| состояние pipeline | `docker compose exec dispatcher python ops.py status` |
|
||||
| реестр failed / журнал job | `docker compose exec dispatcher python ops.py failed`, `... ops.py events <job_id>` |
|
||||
| replay failed / разбор DLQ | `docker compose exec dispatcher python ops.py replay <job_id>`, `... ops.py dlq`, `... ops.py dlq-replay` |
|
||||
| остановить / запустить worker | `scripts\worker.cmd stop`, `scripts\worker.cmd start` (Linux/macOS: `scripts/worker.sh`) |
|
||||
|
||||
После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть: `docker compose up -d --wait`.
|
||||
|
||||
Bootstrap: миграции — одноразовый сервис `migrate` (теперь и `005_jobs.sql`), bucket — `s3-init`, топология RabbitMQ (exchange, очереди, DLQ, пользователи и права) — декларативный файл `secrets/rabbitmq_definitions.json`, который брокер импортирует при каждом старте (`infra/rabbitmq.conf`). Всё идемпотентно: повторный `docker compose run --rm migrate` / `s3-init` и рестарт `rabbitmq` ничего не ломают. Сервисы не имеют права `configure` в брокере и только проверяют топологию пассивным declare — создать или удалить очередь они не могут.
|
||||
|
||||
Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults` остаются). Безопасная остановка worker: `docker compose stop worker` — SIGTERM, потоки доделывают текущее задание (не дольше `JOB_TIMEOUT_SECONDS`, `stop_grace_period: 70s`), неподтверждённое сообщение брокер доставит снова. Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py` (новые пароли не подходят к старым volumes, поэтому удаляются вместе).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
Клиент --HTTP--> lb (HAProxy) --> api x2 --+--> postgres: метаданные, attachment_jobs, outbox, job_events,
|
||||
| варианты (ИСТОЧНИК ИСТИНЫ, volume pgdata)
|
||||
+--> s3 staging/<hex>: полный проверенный оригинал до 202
|
||||
+--> redis: сессии
|
||||
COMMIT(queued + job + outbox) --NOTIFY--> dispatcher --publish+confirm--> rabbitmq
|
||||
(outbox, (quorum queue attachments.process.v1,
|
||||
аренды, DLQ attachments.process.v1.dlq,
|
||||
сверка) volume rabbitmqdata)
|
||||
|
|
||||
worker x2 потока <-+ prefetch 1, ручной ack
|
||||
| claim (аренда + fencing-токен) в postgres
|
||||
| staging -> imaging.py (дочерний процесс, лимиты)
|
||||
| PUT attachments/<id>/v1/{original,thumbnail.jpg,crop.jpg,optimized.webp}
|
||||
| COMMIT ready (+варианты) -> ack -> удалить staging
|
||||
janitor: брошенные uploading, staging ready/failed по retention, сироты в bucket, старый outbox
|
||||
```
|
||||
|
||||
| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` | нет | edge; `127.0.0.1:8080`, `127.0.0.1:8404` | не нужны |
|
||||
| API ×2 | `api` | python:3.11.13-slim | `/health/ready` (PostgreSQL + Redis; брокер и S3 не входят) | нет | edge + data | `database_url`, `cache_url`, ключ S3 API. **Доступа к брокеру нет** |
|
||||
| Брокер | `rabbitmq` | rabbitmq:4.1.4-alpine, `hostname: rabbitmq` | узел запущен и очередь `attachments.process.v1` загружена | `rabbitmqdata` | только data, 5672 не публикуется | `rabbitmq_definitions` (хеши паролей, права) |
|
||||
| Dispatcher | `dispatcher` (образ приложения) | python:3.11.13-slim | файл-пульс `/tmp/alive` не старше 30 с | `faults` | data | `database_url`, `broker_url_dispatcher` (write только в exchange `attachments`, read только DLQ) |
|
||||
| Worker | `worker` (образ приложения), 2 потока | python:3.11.13-slim | файл-пульс | `faults`; `/tmp` — tmpfs 256 MiB | data | `database_url`, `broker_url_worker` (read только основной очереди, write нет), ключ S3 API |
|
||||
| Janitor | `janitor` (образ приложения) | python:3.11.13-slim | файл-пульс `/tmp/alive` (обновляется перед проходом и каждые 10 с между проходами) | нет | data | как в lab4: `database_url`, ключ S3 API |
|
||||
| S3, s3-init, migrate, postgres, redis | как в lab4 | | | | | |
|
||||
|
||||
Почему RabbitMQ: durable quorum-очередь (Raft-журнал на диске, fsync до подтверждения издателю), publisher confirms, ручной ack, `x-delivery-limit` и dead-lettering «at-least-once» — ровно те границы, о которых спрашивают на защите. Почему dispatcher, а не publish из API: API фиксирует вложение, задание и строку outbox **одной транзакцией** и сразу отвечает 202; брокер в этом пути не участвует, поэтому его недоступность не мешает приёму, а API не нужны его пароли.
|
||||
|
||||
Что перенесено из lab4 без изменений по смыслу: пользователи, сессии в Redis, чаты, сообщения, идемпотентность, балансировка, приём и потоковая проверка файла (лимиты bytes/пикселей до распаковки, MIME по содержимому), приватный bucket и ограниченный ключ, janitor.
|
||||
|
||||
Что изменено в lab5:
|
||||
- Новые файлы: `pipeline.py` (формат сообщения, outbox, переходы состояний, replay), `broker.py` (RabbitMQ, publisher confirms), `dispatcher.py`, `worker.py`, `imaging.py` (варианты, отдельный процесс), `faults.py` (учебное внедрение сбоев), `ops.py` (операторские команды), `migrations/005_jobs.sql`, `infra/rabbitmq.conf`.
|
||||
- `attachments.py` / `db.py`: исходник пишется в `staging/<hex>`, затем одной транзакцией `state=queued` + `attachment_jobs` + `outbox` + `NOTIFY`; ответ `202` (переход 201 → 202 из COURSE.md). GET отдаёт `queued/processing/ready/failed`, варианты текущей версии pipeline. Сообщение можно создать с queued/processing-вложением, с failed — 409.
|
||||
- `main.py`: `variant=original|thumbnail|crop|optimized`; для не-ready — 409, отсутствующий вариант ready — 404.
|
||||
- `janitor.py`: очистка staging после безопасной фиксации и по retention для failed; сироты под `attachments/<id>/...`; очистка опубликованного outbox.
|
||||
- `compose.yaml`: сервисы `rabbitmq`, `dispatcher`, `worker`, volumes `rabbitmqdata`, `faults`, три новых секрета; `scripts/init_secrets.py` генерирует пользователей RabbitMQ и definitions; restart-hook перезапускает ещё `rabbitmq`, `dispatcher`, `worker`.
|
||||
- `.env.example`: в skeleton стояли `SESSION_STORE=postgres`, `S3_ENDPOINT=http://s3:9000` и `BROKER_URL_FILE=/run/secrets/broker_url`, с которыми API этого решения не стартует (сессии в Redis с lab2, S3 — SeaweedFS на 8333, у API нет брокера). Значения приведены к реальной конфигурации, добавлены переменные lab5.
|
||||
- Тесты lab4: хелпер `upload()` ждёт `202`, затем опрашивает GET до `ready` — проверки «файл скачивается побайтно» сохранены; `test_consistency` имитирует падение в `enqueue_upload` вместо `finish_upload`.
|
||||
|
||||
### Задание, сообщение и состояния
|
||||
|
||||
Сообщение брокера — ровно `contracts/jobs.schema.json`: `schema_version`, `job_id`, `type=attachment.process.v1`, `attachment_id`, `pipeline_version=1`, `attempt`, `created_at`, `traceparent`. Ни байтов, ни URL, ни ключей S3, ни имени файла: worker берёт `source_key`, размер и SHA-256 из PostgreSQL. `traceparent` дублируется в native header AMQP; `message_id = <job_id>:<attempt>`; `delivery_mode=2` (persistent). Worker строго проверяет сообщение по схеме (лишние поля, типы, форматы) — не прошедшее уходит в DLQ.
|
||||
|
||||
Таблицы (`005_jobs.sql`): `attachment_jobs` (одна строка на `(attachment_id, pipeline_version)`, `job_id` не меняется при повторах; `attempt`, `max_attempts`, `available_at`, `lease_token`/`lease_owner`/`lease_until`, `error_code`, `last_error`, `replays`), `attachment_variants` с **первичным ключом `(attachment_id, pipeline_version, name)`** — это и есть ключ эффекта из контракта, второй набор вариантов физически не вставится, `outbox`, `job_events` (журнал: created, claimed, retry_scheduled, lease_expired, duplicate_ignored, stale_result_rejected, ready, failed, replayed, staging_expired). CHECK-ограничения: `ready ⇔ object_key`, `failed ⇔ error_code`, `processing ⇔ lease`, до ready исходник обязан быть в staging.
|
||||
|
||||
| Переход | Где | Что устойчиво сохранено |
|
||||
| --- | --- | --- |
|
||||
| нет → queued | API: PUT в staging, затем одна транзакция | исходник в `staging/<hex>` (SHA-256 сверяет S3 при PUT), строка вложения, job, outbox |
|
||||
| queued → processing | worker `claim` под `FOR UPDATE` | attempt, новый `lease_token`, `lease_until = now()+15 с`, событие claimed |
|
||||
| processing → queued | временная ошибка / истёкшая аренда | причина (код), `available_at = now()+backoff`, новая строка outbox |
|
||||
| processing → ready | worker после PUT всех объектов | варианты + оригинал + ready одним COMMIT, **только если `lease_token` ещё наш** |
|
||||
| processing → failed | постоянная ошибка или попытки исчерпаны | `error_code`, запись в журнале, исходник хранится 24 ч для replay |
|
||||
| failed → queued | `ops.py replay` | событие replayed (audit), новая попытка того же job_id через outbox |
|
||||
|
||||
Повторная доставка ready/failed — no-op с ack (событие `duplicate_ignored`). Устаревшее сообщение прежней попытки (job ещё ждёт backoff) — ack без работы: своё сообщение придёт из outbox. Ready никогда не откатывается: `finish_ready` обновляет только `state='processing'` и не перезаписывает более новую `pipeline_version`.
|
||||
|
||||
### Где граница «202»
|
||||
|
||||
К моменту ответа 202 сохранено: полный проверенный оригинал в S3 (staging — приватный prefix того же bucket, persistent volume `s3data`; это честная граница: оригинал уже в S3, worker «публикует» его копией в конечный prefix `attachments/<id>/v1/original`), строка вложения `queued`, задание и сообщение в outbox — всё в одной COMMIT PostgreSQL (`synchronous_commit=on`). Брокер на этот момент может ещё ничего не знать — это нормально: строку outbox опубликует dispatcher. Перезапуск/падение API после ответа не теряет работу.
|
||||
|
||||
### Transactional outbox и окна отказа
|
||||
|
||||
Dispatcher просыпается по `NOTIFY outbox` (сразу после COMMIT в API) или раз в 1 с, берёт строки `published_at IS NULL AND available_at <= now()` под `FOR UPDATE SKIP LOCKED`, публикует в канал с publisher confirms (`mandatory=True`: непривязанное сообщение — ошибка, nack при переполнении очереди — ошибка) и только после confirm ставит `published_at`.
|
||||
|
||||
| Окно | Что происходит | Проверка |
|
||||
| --- | --- | --- |
|
||||
| после staging, до записи задания | клиент получил 503/обрыв, 202 не было; строка `uploading` невидима, janitor удалит объект и строку после grace | `test_consistency.py crash` (lab4, теперь с `enqueue_upload`) |
|
||||
| после COMMIT, до publish | строка outbox не отмечена, после рестарта dispatcher публикует её | `test_jobs.py outbox-crash` (`dispatcher_crash_before_publish`) |
|
||||
| после publish, до confirm / после confirm, до отметки | сообщение опубликовано, а отметки нет → публикуется ещё раз → дубль, worker видит ready и делает ack | `test_jobs.py outbox-crash` (`dispatcher_crash_after_confirm`) |
|
||||
| после записи S3, до COMMIT результата | ack не отправлен, брокер доставляет снова; живая аренда → no-op; аренда истекает → новая попытка пишет **те же ключи** | `test_jobs.py crash-after-put` |
|
||||
| после COMMIT результата, до ack | повторная доставка видит ready → ack, второго набора нет; staging доочищает janitor | `test_jobs.py crash-after-commit`, `staging` |
|
||||
|
||||
Дополнительная страховка: раз в 30 с dispatcher ищет `queued`-задания старше `JOB_RECONCILE_SECONDS=120` без сообщения в полёте (например, сообщение ушло в DLQ после `x-delivery-limit`, пока PostgreSQL был недоступен) и публикует заново. Дубль безопасен, потеря — нет.
|
||||
|
||||
**Publisher confirm и consumer ack — разные границы:** confirm значит «брокер принял сообщение и записал в quorum-журнал» (граница dispatcher → broker, после неё можно отметить outbox); ack значит «worker устойчиво выполнил эффект — объекты в S3 и COMMIT в PostgreSQL» (граница broker → worker, после неё брокер удаляет сообщение). At-least-once + идемпотентный эффект — это не exactly-once доставка: дубли сообщений происходят (и видны в журнале как `duplicate_ignored`), но не дают второго эффекта.
|
||||
|
||||
### Повторы, аренда, DLQ, replay
|
||||
|
||||
- Попытки: `JOB_MAX_ATTEMPTS=5`. Backoff «equal jitter»: `потолок = min(60, 2·2^(n-1))`, задержка = половина потолка + случайная половина: 1–2 с, 2–4 с, 4–8 с, 8–16 с. Задержка реализована через `available_at` строки outbox, а не через sleep в worker: поток не занят ожиданием.
|
||||
- Постоянные ошибки (сразу failed, без повторов): `invalid_image`, `image_too_large`, `resource_limit` (не хватило памяти под RLIMIT_AS), `source_missing`, `source_corrupted` (staging не совпал по SHA-256 с принятым). Временные (повтор): `storage_unavailable`, `processing_timeout`, `processing_failed`/`processing_killed`, ошибки PostgreSQL. После последней попытки — `failed/retry_exhausted`. Наружу в `error_code` уходит только финальный код, без деталей.
|
||||
- Аренда: `JOB_LEASE_SECONDS=15`, отдельный поток продлевает её каждые 5 с, пока идёт работа. **Долго работающий worker** продлевает аренду (проверено: зависание imaging 60 с — ни одного `lease_expired`), его ограничивает жёсткий `JOB_TIMEOUT_SECONDS=60`. **Зависший/умерший** перестаёт продлевать → через 15 с dispatcher возвращает job в queued (попытка засчитана) или worker сам перехватывает её при повторной доставке. Если «воскресший» исполнитель всё же дойдёт до COMMIT, fencing-проверка `lease_token` отклонит его результат (`stale_result_rejected`) — два неконтролируемых дубля не фиксируются. Его PUT в S3 перезаписал бы те же ключи теми же байтами (Pillow детерминирован), поэтому опасности для опубликованных объектов нет.
|
||||
- RabbitMQ `consumer_timeout = 300 с`: канал, который не прислал ack дольше, закрывается брокером — последняя страховка от вечного зависания.
|
||||
- DLQ `attachments.process.v1.dlq` (classic durable) получает: сообщения не по схеме, неизвестный `job_id`, превышение `x-delivery-limit=20` (бесконечный nack при недоступной БД). Реестр failed — `attachment_jobs` со `status='failed'` + `job_events` (`ops.py failed`, `ops.py events`).
|
||||
- Replay (`ops.py replay <job_id>`): только для `failed` с исходником в staging; та же job получает новую попытку и ещё 5 попыток, событие `replayed`; ready/queued/processing не трогаются («повторить» ≠ исполнить дубль успешной job). `ops.py dlq-replay`: для валидных сообщений DLQ решает по строке job в PostgreSQL (failed → replay, ready → удалить как no-op, queued/processing → удалить, работу ведёт outbox); невалидные и неизвестные остаются в DLQ.
|
||||
|
||||
### Варианты изображения (pipeline_version = 1)
|
||||
|
||||
| Вариант | Формат | Правило | Параметры |
|
||||
| --- | --- | --- | --- |
|
||||
| thumbnail | JPEG | вписан в 256×256, без увеличения, размеры вниз (минимум 1 px); 320×200 → 256×160 | quality 85, `optimize=True`, LANCZOS |
|
||||
| crop | JPEG | центрированный квадрат 128×128 (маленькие увеличиваются) | quality 85, `optimize=True` |
|
||||
| optimized | WebP | вписан в 1280×1280 без увеличения; прозрачность сохраняется | quality 80, method 4 |
|
||||
|
||||
Во всех вариантах применяется EXIF orientation (1–8, как `ImageOps.exif_transpose`), альфа для JPEG компонуется на белый, EXIF/XMP/комментарии **не записываются** — в `save()` передаётся только ICC-профиль (цвет). Оригинал хранится побайтно как загружен (его EXIF не трогается — это требование контракта). Размер варианта считается от размеров исходника после поворота, а не от промежуточной копии. Большой JPEG декодируется сразу в уменьшенном масштабе (`draft`), сначала уменьшение до ≤1280, потом поворот. Не обещаю, что optimized меньше маленького исходника.
|
||||
|
||||
### Лимиты worker
|
||||
|
||||
- Bytes: API не принимает больше 10 MiB (413); worker скачивает staging не дальше `MAX_UPLOAD_BYTES` и сверяет размер и SHA-256 с метаданными.
|
||||
- Пиксели: API проверяет по заголовку до распаковки (413); worker повторно — `Image.MAX_IMAGE_PIXELS` и явная проверка `w·h ≤ 40 000 000` (защита в глубину, `image_too_large`).
|
||||
- Память: обработка идёт в дочернем процессе `imaging.py`, который **до импорта Pillow** ставит себе `RLIMIT_AS = WORKER_MEMORY_LIMIT_MB` (512 MiB), `RLIMIT_CPU`, `RLIMIT_FSIZE = 64 MiB`. Не хватило памяти → `MemoryError` → `resource_limit`, worker и его соединение с брокером живы. Контейнер worker: 2 CPU / 1536 MiB ≥ 2 потока × 512 MiB + сам worker + tmpfs.
|
||||
- Время: worker убивает дочерний процесс по `JOB_TIMEOUT_SECONDS=60` (`processing_timeout`, временная ошибка).
|
||||
- Конкуренция: `WORKER_CONCURRENCY=2` потока, у каждого своё соединение и `prefetch_count=1` → одновременно не больше 2 заданий на контейнер, остальное ждёт в очереди; очередь ограничена `x-max-length=100000` с `reject-publish` (переполнение — nack издателю, строка outbox ждёт, не потеря).
|
||||
|
||||
### Очистка staging
|
||||
|
||||
Worker удаляет `staging/<hex>` сразу после COMMIT ready и ack. Если не успел (упал, S3 недоступен) — janitor удаляет исходник ready-вложения, обновлённого больше 60 с назад. Исходник failed хранится `STAGING_RETENTION_SECONDS=86400` (для replay), потом удаляется с событием `staging_expired`. queued/processing janitor не трогает никогда. Объекты под `attachments/<id>/` удаляются, только если вложения нет, оно ready и ключ не входит в опубликованный набор, или оно failed и исходник уже удалён по retention (replay невозможен — конечное состояние). Повторная проверка и DELETE выполняются в одной транзакции под `FOR SHARE` на строке вложения: replay (`FOR UPDATE`) и worker ждут её завершения.
|
||||
|
||||
### Соответствие `.env.example`
|
||||
|
||||
| Переменная | Что делает в этом решении |
|
||||
| --- | --- |
|
||||
| APP_*, SESSION_*, DATABASE_URL_FILE, CACHE_URL_FILE, API_REPLICAS, S3_*, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS, пулы и таймауты | как в lab4 |
|
||||
| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) |
|
||||
| BROKER_URL_FILE | пусто для api/janitor (брокера у них нет); compose задаёт свой файл dispatcher (`broker_url_dispatcher`) и worker (`broker_url_worker`) |
|
||||
| WORKER_CONCURRENCY, WORKER_PREFETCH | 2 потока-потребителя, prefetch 1 на поток |
|
||||
| JOB_MAX_ATTEMPTS, JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS | 5 попыток, 60 с на обработку изображения, аренда 15 с |
|
||||
| JOB_BACKOFF_BASE_SECONDS, JOB_BACKOFF_MAX_SECONDS | 2 с и 60 с (equal jitter) |
|
||||
| WORKER_MEMORY_LIMIT_MB | RLIMIT_AS процесса обработки, 512 MiB |
|
||||
| STAGING_RETENTION_SECONDS | сколько хранится исходник failed-вложения для replay, 86400 с |
|
||||
| PIPELINE_VERSION | 1 (версионирование нескольких pipeline — трек 5, не заявлен) |
|
||||
| FAULT_INJECTION | учебные точки сбоев, на стенде включены по умолчанию (срабатывают только по файлу в volume `faults`, доступном лишь worker/dispatcher); `FAULT_INJECTION=0` — выключить |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
`python student-tests/collect_evidence.py` создаёт `evidence/*.txt` (вывод тестов как есть, с датой, commit и кодом выхода) и `evidence/summary.md`. Сценарии lab5 — `student-tests/test_jobs.py`, сбои внедряются через `faults.py` (файл в volume `faults` → процесс в нужной точке делает `os._exit(137)`, как SIGKILL/OOM, или временную ошибку). Каждый сценарий проверяет инвариант «ровно 3 строки вариантов и ровно 4 объекта под `attachments/<id>/v1/`, SHA-256 совпадают».
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke | `python tests/smoke.py` | PASS: публичный сценарий lab5 через балансировщик (202 → ready); размеры вариантов (thumbnail 256×160 для 320×200) — в jobs-variants.txt | smoke.txt |
|
||||
| Контракт и публичные тесты не изменены | `python scripts/check_contract.py`; `git diff` по `contracts/`, `tests/` | PASS; не менялись | check.txt |
|
||||
| 3: минимум lab4 сохранён (201 → 202) | `test_attachments.py`, `test_api.py`, `test_race.py`, `test_idempotency.py`, `test_balancing.py`, публичные persistence/failover | все PASS; upload ждёт 202 и затем ready | attachments.txt, student-tests.txt, race.txt, idempotency.txt, balancing.txt, persistence.txt, failover.txt |
|
||||
| 3: 202 только после устойчивого staging и задания; остановка API после ответа не теряет работу | `test_jobs.py worker-down` | staging-объект есть до 202; job и outbox в БД; после рестарта api и rabbitmq тот же job_id → ready | jobs-recovery.txt |
|
||||
| 3: отдельные broker и worker, durable, at-least-once, ack после эффекта, API не делает resize | `check_stack.py`, `test_jobs.py worker-down`, `broker-down` | rabbitmq/worker/dispatcher — отдельные сервисы; при остановленном worker вложение остаётся queued (API не обрабатывает); без брокера 202 и /health/ready 200 | stack.txt, jobs-recovery.txt |
|
||||
| 3: queued/processing/ready/failed в API; оригинал неизменен; thumbnail по контракту | `test_jobs.py variants`, smoke | оригинал побайтно; 320×200 → 256×160; content 409 для queued и failed | jobs-variants.txt |
|
||||
| 3: ограниченный retry, failed с безопасным error_code, повтор не создаёт второго набора | `test_jobs.py retry`, `exhausted-replay`, все сценарии сбоев | 5 попыток → `retry_exhausted`; в каждом сценарии ровно один набор | jobs-retry.txt, jobs-crash.txt |
|
||||
| 3: демонстрация worker off → 202 → сообщение → рестарт API → worker → ready; брокер с volume не теряет задание | `test_jobs.py worker-down` | сообщение с вложением 201, другие сообщения работают, после `restart rabbitmq` в очереди 1 сообщение, затем ready и thumbnail | jobs-recovery.txt |
|
||||
| 4: crop 128×128 и optimized WebP, параметры описаны | `test_jobs.py variants` | 6 исходников (PNG/JPEG, EXIF orientation 6, alpha, 40×30, 3000×2000, 5000×2): размеры по правилам, EXIF/XMP в вариантах 0, alpha на белом | jobs-variants.txt |
|
||||
| 4: лимиты по bytes, пикселям, времени и памяти worker | `test_jobs.py limits` | 10 MiB+ → 413; 40 Мпикс PNG — пик RSS дочернего процесса 227 MiB при RLIMIT 512 MiB; лимит пикселей в worker — exit 4; «бомба памяти» → `resource_limit`; зависание → убито через 60 с, затем ready | jobs-limits.txt |
|
||||
| 4: outbox, crash между COMMIT и publish, восстановление без ручного задания | `test_jobs.py outbox-crash` | dispatcher убит до publish → после рестарта тот же job опубликован и ready (attempt 1); убит после confirm → дубль, `duplicate_ignored`, один набор | jobs-outbox.txt |
|
||||
| 4: backoff с jitter; постоянные/временные ошибки; DLQ/реестр failed; безопасный replay | `test_jobs.py retry exhausted-replay permanent dlq` | задержки в диапазонах 1–2 и 2–4 с; `invalid_image` — failed с attempt 1; replay → ready; replay ready — отказ без изменений; невалидное сообщение и неизвестный job → DLQ, dlq-replay их не исполняет | jobs-retry.txt |
|
||||
| 4: остановка worker после записи объекта до ack; повтор завершает тот же результат | `test_jobs.py crash-after-put`, `crash-after-commit` | после PUT: аренда истекла, attempt 2 перезаписал те же ключи, 4 объекта; после COMMIT: redelivery → no-op ack, очередь пуста | jobs-crash.txt |
|
||||
| 4: timeout/lease для зависшего processing | `test_jobs.py lease`, `limits` | зависший не продлевает аренду → через ~15 с job возвращена и выполнена другим потоком, поздний результат отклонён fencing; живой долгий — аренда продлевается | jobs-crash.txt, jobs-limits.txt |
|
||||
| 4: ограниченная конкуренция/prefetch | `test_jobs.py concurrency` | 2 consumer, `prefetch_count=1`, ручной ack; из 8 накопленных заданий в processing одновременно ровно 2 | jobs-concurrency.txt |
|
||||
| 4: очистка staging после безопасной фиксации | `test_jobs.py worker-down`, `staging` | staging удалён worker'ом после COMMIT; после crash до очистки — janitor | jobs-recovery.txt, jobs-crash.txt |
|
||||
| Исправлено по ревью: гонка janitor и replay | `test_janitor_replay.py` | на старом коде janitor удалял crop.jpg после replay (503), теперь все 4 файла 200; после истечения retention остатки удаляются, replay отказывает | evidence/jobs-janitor-replay.txt |
|
||||
| Выбранный трек 5 | — | не выбран | — |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
**Где проверено сейчас.** Все строки таблицы выше подтверждены на Docker-стенде из этого репозитория (Windows 10, Docker Desktop, Compose 2.35, образы из `compose.yaml`, 2 реплики API за балансировщиком): 24.10.2026 один полный прогон `python student-tests/collect_evidence.py` — все 26 шагов PASS, сводка в `evidence/summary.md`, вывод каждого шага в `evidence/*.txt`, таблицы замеров ниже взяты из того же прогона. Перед ним на том же стенде отдельно прошли `docker compose up -d --build --wait` (migrate и s3-init — Exited (0), остальные healthy), `python tests/smoke.py` и `python student-tests/test_jobs.py worker-down`. Ранний прогон процессами без Docker (PostgreSQL 16, RabbitMQ 3.12, SeaweedFS 3.80, Redis) сохранён в `evidence/native-sandbox/` для сравнения.
|
||||
|
||||
Что исправлено по итогам первого запуска в Docker (без Docker эти места не проявлялись): путь к definitions в `infra/rabbitmq.conf` (Compose монтирует секрет как `/run/secrets/rabbitmq_definitions`, без `.json`, — брокер не стартовал); у janitor появился healthcheck по файлу-пульсу (с `healthcheck: disable: true` команда `up --wait` завершалась ошибкой); в тестах — пробник `test_jobs.py` запускается одноразовым контейнером worker, когда сам worker остановлен, `upload_memory.py` ждёт 202 вместо 201 из lab4, а сценарий `staging` проверяет результат очистки независимо от того, какой проход janitor её сделал.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Гипотезы:
|
||||
1. Принятая (202) работа не теряется ни при каком одиночном отказе процесса (API, dispatcher, worker, broker) в любом из окон контракта, а повторная доставка не создаёт второго эффекта.
|
||||
2. Память worker ограничена независимо от входа: худший допустимый вход (PNG 8000×5000) помещается в RLIMIT_AS 512 MiB, а вход сверх лимита ломает только свою попытку, не процесс worker.
|
||||
|
||||
Методика: каждый сценарий `test_jobs.py` взводит одну точку сбоя (`faults.py`), загружает синтетический файл через балансировщик, ждёт конечного состояния через публичный GET и проверяет журнал `job_events`, строки outbox, глубину очередей (`rabbitmqctl list_queues`) и объекты в bucket (пробник в контейнере worker). Пик памяти обработки — `ru_maxrss` дочернего процесса `imaging.py` (`getrusage(RUSAGE_CHILDREN)`).
|
||||
|
||||
<!-- measurements:begin -->
|
||||
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-24, commit 276dbda + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB.
|
||||
|
||||
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
|
||||
|
||||
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 1 | 5aa14958c864 | 2 | 123 | 0 | 4.406 |
|
||||
| 2 | b9ad6d33d190 | 2 | 125 | 0 | 4.188 |
|
||||
|
||||
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
|
||||
|
||||
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.45 / 1.56 | 5aa14958c864: 5 | 96.0 | 109.7 | 13.7 | 149.6 |
|
||||
| | | | | b9ad6d33d190: 5 | 94.5 | 108.9 | 14.4 | 142.1 |
|
||||
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.04 / 1.31 | 5aa14958c864: 5 | 96.6 | 97.6 | 1.0 | 89.6 |
|
||||
| | | | | b9ad6d33d190: 5 | 96.2 | 98.6 | 2.3 | 88.9 |
|
||||
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 4.96 / 5.2 | 5aa14958c864: 10 | 94.7 | 114.9 | 20.3 | 155.4 |
|
||||
| | | | | b9ad6d33d190: 10 | 97.0 | 121.8 | 24.8 | 176.9 |
|
||||
|
||||
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
|
||||
|
||||
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 243.885 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.406, 4.188 с (hook ждёт остановки ~4 с); потерь нет |
|
||||
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 58.5 с до готовности; потерь нет |
|
||||
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19922 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
|
||||
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14860 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
|
||||
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
|
||||
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
|
||||
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
|
||||
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
|
||||
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action | без ручного действия |
|
||||
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
|
||||
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
|
||||
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
|
||||
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
|
||||
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready |
|
||||
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
|
||||
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
|
||||
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
|
||||
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
|
||||
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены |
|
||||
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - |
|
||||
<!-- measurements:end -->
|
||||
|
||||
Результаты прогона на нативном стенде (`evidence/native-sandbox/`, 1 API-реплика без балансировщика):
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
|
||||
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
|
||||
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action | без ручного действия |
|
||||
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
|
||||
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
|
||||
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
|
||||
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
|
||||
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready |
|
||||
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
|
||||
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
|
||||
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
|
||||
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
|
||||
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены |
|
||||
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) | - |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Выдерживается: падение/рестарт любого одного процесса (api, dispatcher, worker, rabbitmq, postgres с volume) в любом окне отказа из контракта; недоступность брокера при приёме; зависание и нехватка памяти в обработке изображения; временные ошибки S3.
|
||||
- Не выдерживается: потеря volume `pgdata` (источник истины) или `s3data` — нужен бэкап; один узел RabbitMQ (quorum-очередь из одного члена — durability есть, отказоустойчивости узла нет); при недоступном PostgreSQL новые загрузки и обработка останавливаются (это осознанно: без источника истины нельзя ни принять, ни зафиксировать).
|
||||
- Учебные упрощения: worker использует тот же ограниченный S3-ключ, что и API (права только на свой bucket; отдельный ключ worker — следующий шаг); точки внедрения сбоев включены на стенде (`FAULT_INJECTION=1`), в эксплуатации выключаются; replay — CLI в контейнере dispatcher, а не API с авторизацией оператора.
|
||||
- Подтверждено измерением: память обработки ограничена, конкуренция равна `WORKER_CONCURRENCY`, отказы не дают потерь и дублей эффекта. Гипотеза без измерения: пропускная способность pipeline под перегрузкой и выбор concurrency/prefetch — это трек 5.1, не заявлен.
|
||||
|
||||
## Ответы на вопросы защиты (кратко)
|
||||
|
||||
- **Publisher confirm vs consumer ack.** Confirm — брокер принял и сохранил сообщение (после него dispatcher отмечает outbox). Ack — worker устойчиво выполнил эффект (объекты в S3 + COMMIT ready); до ack сообщение остаётся у брокера и будет доставлено снова.
|
||||
- **Что сохранено в момент 202.** Полный проверенный оригинал в S3-prefix `staging/`, строка вложения `queued`, задание `attachment_jobs` и сообщение в `outbox` — одной транзакцией PostgreSQL. Брокер может ещё ничего не знать.
|
||||
- **Почему повторная доставка нормальна и где ключ идемпотентности.** At-least-once: ack может потеряться после эффекта, dispatcher может опубликовать дважды. Ключ — `job_id` (одна логическая работа) и ключ эффекта `(attachment_id, pipeline_version, variant)`: первичный ключ `attachment_variants` и детерминированные ключи S3 `attachments/<id>/v<N>/<variant>`; состояние job в PostgreSQL решает, работать или подтвердить как no-op.
|
||||
- **Зависший vs долго работающий.** Живой продлевает аренду каждые 5 с из отдельного потока, его ограничивает только `JOB_TIMEOUT_SECONDS`. Зависший/умерший перестаёт продлевать — через 15 с job перехватывают; если он «оживёт», fencing по `lease_token` не даст зафиксировать результат.
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202.
|
||||
|
||||
Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого
|
||||
объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение:
|
||||
|
||||
1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503.
|
||||
2. PUT исходника в staging/<hex> — упало: удаляем объект (PUT мог пройти)
|
||||
и строку, клиенту 503. Если и это не вышло —
|
||||
uploading-строку уберёт janitor после grace.
|
||||
3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT
|
||||
— упало/неизвестно: клиенту 503, строка
|
||||
остаётся uploading -> janitor удалит объект
|
||||
и строку. Если COMMIT всё же прошёл, задание
|
||||
есть и будет обработано (повтор клиента даст
|
||||
второе вложение — в контракте нет ключа).
|
||||
4. 202 Attachment(status=queued) — только после шага 3.
|
||||
|
||||
Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный
|
||||
брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно.
|
||||
"""
|
||||
import logging
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import db
|
||||
import settings
|
||||
import storage
|
||||
from errors import unavailable
|
||||
from uploads import ImageInfo, Upload
|
||||
|
||||
log = logging.getLogger("messenger.attachments")
|
||||
|
||||
|
||||
def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict:
|
||||
attachment_id = uuid4()
|
||||
key = storage.new_key(uuid4().hex)
|
||||
db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type,
|
||||
upload.size, upload.sha256, image.width, image.height, key)
|
||||
try:
|
||||
storage.put(key, upload.file, upload.size, image.content_type, upload.sha256)
|
||||
except storage.StorageUnavailable:
|
||||
compensate(attachment_id, key)
|
||||
raise unavailable("File storage is temporarily unavailable") from None
|
||||
row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID)
|
||||
if row is None:
|
||||
# janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace).
|
||||
log.warning("Upload %s was reclaimed before it was queued", attachment_id)
|
||||
raise unavailable()
|
||||
log.info("Attachment %s accepted into staging and queued", attachment_id)
|
||||
return {**row, "variants": []}
|
||||
|
||||
|
||||
def compensate(attachment_id: UUID, key: str) -> None:
|
||||
"""Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor."""
|
||||
try:
|
||||
storage.delete(key, quick=True)
|
||||
db.delete_upload_intent(attachment_id)
|
||||
log.info("Upload %s rolled back after S3 failure", attachment_id)
|
||||
except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту
|
||||
log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__)
|
||||
|
||||
|
||||
def public(row: dict) -> dict:
|
||||
"""Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей)."""
|
||||
return {
|
||||
"id": row["id"],
|
||||
"chat_id": row["chat_id"],
|
||||
"owner_id": row["owner_id"],
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["state"],
|
||||
"variants": [
|
||||
{key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")}
|
||||
for v in row.get("variants", [])
|
||||
],
|
||||
"error_code": row["error_code"] if row["state"] == "failed" else None,
|
||||
"created_at": row["created_at"],
|
||||
"updated_at": row["updated_at"],
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import logging
|
||||
import secrets
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from uuid import UUID
|
||||
|
||||
import redis
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
import settings
|
||||
from errors import BASIC_CHALLENGE, unauthorized, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.auth")
|
||||
|
||||
hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1)
|
||||
|
||||
DUMMY_HASH = hasher.hash("there-is-no-such-user")
|
||||
|
||||
PASSWORD_MIN_LENGTH = 12
|
||||
PASSWORD_MAX_LENGTH = 128
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return hasher.hash(password)
|
||||
|
||||
|
||||
def password_matches(password_hash: str | None, password: str) -> bool:
|
||||
try:
|
||||
hasher.verify(password_hash or DUMMY_HASH, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
return password_hash is not None
|
||||
|
||||
def parse_basic(header: str | None) -> tuple[str, str] | None:
|
||||
"""'Basic base64(username:password)' -> (username, password) или None.
|
||||
|
||||
Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617).
|
||||
"""
|
||||
scheme, _, value = (header or "").strip().partition(" ")
|
||||
if scheme.lower() != "basic" or not value or len(value) > 1024:
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
return None
|
||||
username, colon, password = decoded.partition(":")
|
||||
if not colon or not username or not username.isascii():
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(header: str | None) -> str | None:
|
||||
scheme, _, value = (header or "").strip().partition(" ")
|
||||
value = value.strip()
|
||||
if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
|
||||
KEY_PREFIX = "session:"
|
||||
|
||||
_redis = redis.Redis(
|
||||
connection_pool=redis.BlockingConnectionPool.from_url(
|
||||
settings.REDIS_URL or "redis://localhost:6379/0",
|
||||
max_connections=settings.REDIS_POOL_MAX,
|
||||
timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
socket_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
decode_responses=True,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
user_id: UUID
|
||||
key: str
|
||||
|
||||
|
||||
def session_key(token: str) -> str:
|
||||
return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def create_session(user_id: UUID) -> tuple[str, datetime]:
|
||||
"""Новая сессия: случайный токен и абсолютное время истечения."""
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS)
|
||||
try:
|
||||
_redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000))
|
||||
except redis.RedisError as exc:
|
||||
log.warning("Redis is unavailable on login: %s", type(exc).__name__)
|
||||
raise unavailable() from None
|
||||
return token, expires_at
|
||||
|
||||
|
||||
def check_bearer(header: str | None) -> Session:
|
||||
token = parse_bearer(header)
|
||||
if token is None:
|
||||
raise unauthorized("Bearer token is required")
|
||||
key = session_key(token)
|
||||
try:
|
||||
user_id = _redis.get(key)
|
||||
except redis.RedisError as exc:
|
||||
log.warning("Redis is unavailable: %s", type(exc).__name__)
|
||||
raise unavailable() from None
|
||||
if user_id is None:
|
||||
raise unauthorized("Session is invalid or expired")
|
||||
return Session(UUID(user_id), key)
|
||||
|
||||
|
||||
def delete_session(session: Session) -> None:
|
||||
try:
|
||||
_redis.delete(session.key)
|
||||
except redis.RedisError:
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def ping_redis() -> None:
|
||||
try:
|
||||
_redis.ping()
|
||||
except redis.RedisError:
|
||||
raise unavailable() from None
|
||||
|
||||
|
||||
def login_failed():
|
||||
return unauthorized("Invalid username or password", BASIC_CHALLENGE)
|
||||
|
||||
|
||||
def close() -> None:
|
||||
_redis.close()
|
||||
_redis.connection_pool.disconnect()
|
||||
@@ -0,0 +1,105 @@
|
||||
"""RabbitMQ: подключение, проверка топологии, публикация с publisher confirms.
|
||||
|
||||
Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует
|
||||
scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure:
|
||||
они только проверяют её пассивным declare и не могут создать/удалить очередь.
|
||||
|
||||
exchange attachments (direct, durable)
|
||||
--attachment.process.v1--> queue attachments.process.v1 (quorum, durable)
|
||||
x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once),
|
||||
x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря)
|
||||
exchange attachments.dlx (direct, durable)
|
||||
--attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py
|
||||
через basic_get + nack не расходует delivery-limit)
|
||||
|
||||
Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до
|
||||
подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
|
||||
import pika
|
||||
from pika.exceptions import AMQPError
|
||||
|
||||
import pipeline
|
||||
import settings
|
||||
|
||||
log = logging.getLogger("messenger.broker")
|
||||
logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал
|
||||
|
||||
BrokerError = (AMQPError, OSError)
|
||||
|
||||
|
||||
def parameters(url: str | None = None) -> pika.URLParameters:
|
||||
params = pika.URLParameters(url or settings.BROKER_URL)
|
||||
params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS
|
||||
params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно
|
||||
params.socket_timeout = 5
|
||||
params.connection_attempts = 1
|
||||
params.client_properties = {"connection_name": settings.INSTANCE_ID}
|
||||
return params
|
||||
|
||||
|
||||
def connect(url: str | None = None) -> pika.BlockingConnection:
|
||||
return pika.BlockingConnection(parameters(url))
|
||||
|
||||
|
||||
def check_topology(channel) -> dict:
|
||||
"""Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях."""
|
||||
channel.exchange_declare(pipeline.EXCHANGE, passive=True)
|
||||
main = channel.queue_declare(pipeline.QUEUE, passive=True).method
|
||||
dlq = channel.queue_declare(pipeline.DLQ, passive=True).method
|
||||
return {"queue": main.message_count, "consumers": main.consumer_count, "dlq": dlq.message_count}
|
||||
|
||||
|
||||
def properties(message: dict) -> pika.BasicProperties:
|
||||
headers = {}
|
||||
if "traceparent" in message:
|
||||
headers["traceparent"] = message["traceparent"] # тот же carrier и в native headers
|
||||
return pika.BasicProperties(
|
||||
delivery_mode=pika.DeliveryMode.Persistent,
|
||||
content_type="application/json",
|
||||
message_id=f"{message['job_id']}:{message['attempt']}",
|
||||
type=message["type"],
|
||||
headers=headers or None,
|
||||
)
|
||||
|
||||
|
||||
class Publisher:
|
||||
"""Канал в режиме confirm: basic_publish возвращает управление только после ack брокера.
|
||||
|
||||
mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError.
|
||||
Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях
|
||||
строка outbox остаётся неопубликованной и будет отправлена позже.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
|
||||
def open(self):
|
||||
if self.channel is None or self.channel.is_closed or self.connection.is_closed:
|
||||
self.close()
|
||||
self.connection = connect()
|
||||
self.channel = self.connection.channel()
|
||||
self.channel.confirm_delivery()
|
||||
check_topology(self.channel)
|
||||
log.info("connected to broker, publisher confirms enabled")
|
||||
return self.channel
|
||||
|
||||
def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY):
|
||||
body = json.dumps(message, separators=(",", ":")).encode("utf-8")
|
||||
self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True)
|
||||
|
||||
def pump(self):
|
||||
if self.connection is not None and self.connection.is_open:
|
||||
self.connection.process_data_events(time_limit=0) # heartbeat в паузах
|
||||
|
||||
def close(self):
|
||||
try:
|
||||
if self.connection is not None and self.connection.is_open:
|
||||
self.connection.close()
|
||||
except BrokerError:
|
||||
pass
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
+320
@@ -0,0 +1,320 @@
|
||||
name: messenger-lab5
|
||||
|
||||
x-app-build: &app-build
|
||||
context: .
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-dev}
|
||||
|
||||
# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py.
|
||||
x-pipeline-env: &pipeline-env
|
||||
PIPELINE_VERSION: ${PIPELINE_VERSION:-1}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-1}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
|
||||
JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2}
|
||||
JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
# Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults,
|
||||
# доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0.
|
||||
FAULT_INJECTION: ${FAULT_INJECTION:-1}
|
||||
FAULTS_DIR: /faults
|
||||
|
||||
x-alive-check: &alive-check
|
||||
# Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад.
|
||||
test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"]
|
||||
interval: 10s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
start_interval: 1s
|
||||
|
||||
x-s3-api-secrets: &s3-api-secrets
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
|
||||
services:
|
||||
migrate:
|
||||
build: *app-build
|
||||
command: ["python", "migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
|
||||
# S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений.
|
||||
# Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config
|
||||
# (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket.
|
||||
# Анонимного доступа нет: identity "anonymous" в конфиге не объявлена.
|
||||
s3:
|
||||
image: chrislusf/seaweedfs:3.80
|
||||
entrypoint: ["/usr/bin/weed"]
|
||||
command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0",
|
||||
"-master.volumeSizeLimitMB=1024", "-volume.max=0",
|
||||
"-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"]
|
||||
secrets: [s3_config]
|
||||
volumes: ["s3data:/data"]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
# S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт»)
|
||||
test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 24
|
||||
start_period: 15s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
|
||||
# Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки.
|
||||
s3-init:
|
||||
build: *app-build
|
||||
command: ["python", "s3_init.py"]
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user
|
||||
S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
s3: {condition: service_healthy}
|
||||
healthcheck:
|
||||
disable: true
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M}
|
||||
|
||||
api:
|
||||
build: *app-build
|
||||
env_file: .env
|
||||
environment:
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL
|
||||
<<: *s3-api-secrets
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [edge, data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
|
||||
# Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ.
|
||||
janitor:
|
||||
build: *app-build
|
||||
command: ["python", "janitor.py"]
|
||||
env_file: .env
|
||||
environment:
|
||||
CACHE_URL_FILE: "" # janitor не ходит в Redis
|
||||
BROKER_URL_FILE: "" # и в брокер
|
||||
<<: *s3-api-secrets
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 192M}
|
||||
|
||||
# Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права —
|
||||
# в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте.
|
||||
# hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume.
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.1.4-alpine
|
||||
hostname: rabbitmq
|
||||
volumes:
|
||||
- ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro
|
||||
- rabbitmqdata:/var/lib/rabbitmq
|
||||
secrets: [rabbitmq_definitions]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
# Узел запущен, и vhost с очередями загружен (а не только «порт открыт»).
|
||||
test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 24
|
||||
start_period: 20s
|
||||
start_interval: 2s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
|
||||
# Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает
|
||||
# просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет.
|
||||
dispatcher:
|
||||
build: *app-build
|
||||
command: ["python", "dispatcher.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url_dispatcher
|
||||
<<: *pipeline-env
|
||||
secrets: [database_url, broker_url_dispatcher]
|
||||
volumes: ["faults:/faults"]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.25", memory: 128M}
|
||||
|
||||
# Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем
|
||||
# процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS.
|
||||
# Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker.
|
||||
worker:
|
||||
build: *app-build
|
||||
command: ["python", "worker.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
BROKER_URL_FILE: /run/secrets/broker_url_worker
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512}
|
||||
<<: [*pipeline-env, *s3-api-secrets]
|
||||
secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key]
|
||||
volumes: ["faults:/faults"]
|
||||
tmpfs:
|
||||
- /tmp:size=256m,mode=1777
|
||||
networks: [data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
s3-init: {condition: service_completed_successfully}
|
||||
rabbitmq: {condition: service_healthy}
|
||||
healthcheck: *alive-check
|
||||
restart: unless-stopped
|
||||
# SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят;
|
||||
# если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова.
|
||||
stop_grace_period: 70s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "2.00", memory: 1536M}
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.25-alpine
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
- "127.0.0.1:8404:8404"
|
||||
volumes:
|
||||
- ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
networks: [edge]
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
start_interval: 1s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 15s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
|
||||
postgres:
|
||||
image: postgres:17.11-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"]
|
||||
secrets: [postgres_password]
|
||||
volumes: ["pgdata:/var/lib/postgresql/data"]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.00", memory: 512M}
|
||||
|
||||
redis:
|
||||
image: redis:7.4.11-alpine
|
||||
command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
|
||||
volumes:
|
||||
- ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro
|
||||
- redisdata:/data
|
||||
secrets: [redis_auth]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.50", memory: 128M}
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
data:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
redisdata: {}
|
||||
s3data: {}
|
||||
rabbitmqdata: {}
|
||||
faults: {}
|
||||
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
redis_auth: {file: ./secrets/redis_auth}
|
||||
s3_root_user: {file: ./secrets/s3_root_user}
|
||||
s3_root_password: {file: ./secrets/s3_root_password}
|
||||
s3_access_key_id: {file: ./secrets/s3_access_key_id}
|
||||
s3_secret_access_key: {file: ./secrets/s3_secret_access_key}
|
||||
s3_config: {file: ./secrets/s3_config.json}
|
||||
broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher}
|
||||
broker_url_worker: {file: ./secrets/broker_url_worker}
|
||||
rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json}
|
||||
@@ -0,0 +1,391 @@
|
||||
import logging
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import psycopg
|
||||
from psycopg import errors as pg_errors
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool, PoolTimeout
|
||||
|
||||
import pipeline
|
||||
import settings
|
||||
from errors import bad_request, conflict, not_found, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.db")
|
||||
|
||||
|
||||
pool = ConnectionPool(
|
||||
conninfo=settings.DATABASE_URL,
|
||||
min_size=settings.DB_POOL_MIN,
|
||||
max_size=settings.DB_POOL_MAX,
|
||||
timeout=settings.DB_POOL_WAIT_SECONDS,
|
||||
kwargs={
|
||||
"connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||
"row_factory": dict_row,
|
||||
"options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}"
|
||||
" -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC",
|
||||
},
|
||||
check=ConnectionPool.check_connection,
|
||||
open=False,
|
||||
)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def connection(wait_seconds: float | None = None):
|
||||
try:
|
||||
with pool.connection(timeout=wait_seconds) as conn:
|
||||
yield conn
|
||||
except (psycopg.OperationalError, PoolTimeout) as exc:
|
||||
log.warning("PostgreSQL is unavailable: %s", type(exc).__name__)
|
||||
raise unavailable() from None
|
||||
except psycopg.DataError as exc:
|
||||
log.warning("PostgreSQL rejected a value: %s", type(exc).__name__)
|
||||
raise bad_request("Invalid request data") from None
|
||||
|
||||
|
||||
TEXT_FIELDS = ("display_name", "title", "text", "filename")
|
||||
|
||||
|
||||
def to_db(value: str) -> bytes:
|
||||
return value.encode("utf-8")
|
||||
|
||||
|
||||
def from_db(row: dict | None) -> dict | None:
|
||||
"""Строка из БД -> dict для API: байты текстовых полей обратно в str."""
|
||||
if row is None:
|
||||
return None
|
||||
return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()}
|
||||
|
||||
|
||||
def ping() -> None:
|
||||
with connection(wait_seconds=1) as conn:
|
||||
conn.execute("SELECT 1")
|
||||
|
||||
|
||||
MIGRATIONS_LOCK = 20260202
|
||||
|
||||
|
||||
def migration_files() -> list:
|
||||
return sorted(settings.MIGRATIONS_DIR.glob("*.sql"))
|
||||
|
||||
|
||||
def run_migrations() -> list[str]:
|
||||
files = migration_files()
|
||||
applied = []
|
||||
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,))
|
||||
try:
|
||||
conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version text PRIMARY KEY,
|
||||
applied_at timestamptz NOT NULL DEFAULT now())""")
|
||||
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
for path in files:
|
||||
if path.name in done:
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(path.read_text(encoding="utf-8"))
|
||||
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,))
|
||||
applied.append(path.name)
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,))
|
||||
return applied
|
||||
|
||||
|
||||
def wait_and_migrate(max_wait_seconds: int = 60) -> None:
|
||||
deadline = time.monotonic() + max_wait_seconds
|
||||
while True:
|
||||
try:
|
||||
applied = run_migrations()
|
||||
log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)")
|
||||
return
|
||||
except psycopg.OperationalError:
|
||||
if time.monotonic() > deadline:
|
||||
raise
|
||||
log.warning("PostgreSQL is not ready yet, retry in 1 s")
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def missing_migrations() -> list[str]:
|
||||
expected = [path.name for path in migration_files()]
|
||||
with psycopg.connect(settings.DATABASE_URL, autocommit=True,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn:
|
||||
try:
|
||||
done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")}
|
||||
except pg_errors.UndefinedTable:
|
||||
return expected
|
||||
return [name for name in expected if name not in done]
|
||||
|
||||
|
||||
def wait_for_schema(max_wait_seconds: int) -> None:
|
||||
deadline = time.monotonic() + max_wait_seconds
|
||||
while True:
|
||||
try:
|
||||
missing = missing_migrations()
|
||||
except psycopg.OperationalError:
|
||||
missing = None
|
||||
if missing == []:
|
||||
log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files()))
|
||||
return
|
||||
if time.monotonic() > deadline:
|
||||
raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}")
|
||||
log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None
|
||||
else "missing " + ", ".join(missing))
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
|
||||
USER_COLUMNS = "id, username, display_name, created_at"
|
||||
|
||||
|
||||
def create_user(username: str, display_name: str, password_hash: str) -> dict:
|
||||
try:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(
|
||||
f"""INSERT INTO users (id, username, display_name, password_hash)
|
||||
VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""",
|
||||
(uuid4(), username, to_db(display_name), password_hash),
|
||||
).fetchone())
|
||||
except pg_errors.UniqueViolation:
|
||||
raise conflict("Username already exists") from None
|
||||
|
||||
|
||||
def get_user(user_id: UUID) -> dict | None:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone())
|
||||
|
||||
|
||||
def get_user_with_password(username: str) -> dict | None:
|
||||
with connection() as conn:
|
||||
return from_db(conn.execute(
|
||||
f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,)
|
||||
).fetchone())
|
||||
|
||||
|
||||
CHAT_COLUMNS = """c.id, c.title, c.created_at,
|
||||
ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids"""
|
||||
|
||||
|
||||
def create_chat(title: str, member_ids: list[UUID]) -> dict:
|
||||
chat_id = uuid4()
|
||||
members = sorted(member_ids)
|
||||
try:
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
"INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at",
|
||||
(chat_id, to_db(title)),
|
||||
).fetchone()
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
|
||||
[(chat_id, member_id) for member_id in members],
|
||||
)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise bad_request("Unknown member") from None
|
||||
return {**from_db(row), "member_ids": members}
|
||||
|
||||
|
||||
def get_chat(chat_id: UUID, user_id: UUID) -> dict:
|
||||
with connection() as conn:
|
||||
chat = conn.execute(
|
||||
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||
WHERE c.id = %s""",
|
||||
(user_id, chat_id),
|
||||
).fetchone()
|
||||
if chat is None:
|
||||
raise not_found("Chat not found")
|
||||
return from_db(chat)
|
||||
|
||||
|
||||
def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||
"""Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё»."""
|
||||
where, params = "", [user_id]
|
||||
if after is not None:
|
||||
where = "WHERE (c.created_at, c.id) > (%s, %s)"
|
||||
params += list(after)
|
||||
with connection() as conn:
|
||||
rows = conn.execute(
|
||||
f"""SELECT {CHAT_COLUMNS} FROM chats c
|
||||
JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s
|
||||
{where}
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT %s""",
|
||||
(*params, limit + 1),
|
||||
).fetchall()
|
||||
return [from_db(row) for row in rows]
|
||||
|
||||
|
||||
|
||||
MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids"""
|
||||
|
||||
|
||||
def is_member(conn, chat_id: UUID, user_id: UUID) -> bool:
|
||||
return conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)
|
||||
).fetchone() is not None
|
||||
|
||||
|
||||
def check_member(chat_id: UUID, user_id: UUID) -> None:
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise not_found("Chat not found")
|
||||
|
||||
|
||||
def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID],
|
||||
idempotency_key: str, request_hash: bytes) -> dict:
|
||||
try:
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404
|
||||
raise not_found("Chat not found")
|
||||
if attachment_ids:
|
||||
# Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409).
|
||||
# FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние.
|
||||
found = conn.execute(
|
||||
"""SELECT id, state FROM attachments
|
||||
WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""",
|
||||
(attachment_ids, chat_id),
|
||||
).fetchall()
|
||||
if len(found) != len(attachment_ids):
|
||||
raise not_found("Attachment not found")
|
||||
failed = any(row["state"] == "failed" for row in found)
|
||||
else:
|
||||
failed = False
|
||||
saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m
|
||||
WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s"""
|
||||
if failed:
|
||||
# Повтор уже созданного сообщения возвращает прежний 201, даже если вложение
|
||||
# с тех пор стало failed. Новое сообщение с failed-вложением — 409.
|
||||
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
|
||||
if saved is None:
|
||||
raise conflict("Failed attachment cannot be attached")
|
||||
created = None
|
||||
else:
|
||||
created = conn.execute(
|
||||
"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash)
|
||||
VALUES (%s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING
|
||||
RETURNING id, chat_id, sender_id, text, created_at""",
|
||||
(uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash),
|
||||
).fetchone()
|
||||
if created is None:
|
||||
saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone()
|
||||
if created is not None:
|
||||
if attachment_ids:
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany(
|
||||
"""INSERT INTO message_attachments (message_id, chat_id, attachment_id, position)
|
||||
VALUES (%s, %s, %s, %s)""",
|
||||
[(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)],
|
||||
)
|
||||
return {**from_db(created), "attachment_ids": list(attachment_ids)}
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise not_found("Chat not found") from None
|
||||
if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен
|
||||
raise unavailable()
|
||||
if bytes(saved.pop("request_hash")) != request_hash:
|
||||
raise conflict("Idempotency-Key was already used with a different request")
|
||||
return from_db(saved)
|
||||
|
||||
|
||||
def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]:
|
||||
"""История чата после курсора (keyset-пагинация), до limit+1 строк."""
|
||||
where, params = "m.chat_id = %s", [chat_id]
|
||||
if after is not None:
|
||||
where += " AND (m.created_at, m.id) > (%s, %s)"
|
||||
params += list(after)
|
||||
with connection() as conn:
|
||||
if not is_member(conn, chat_id, user_id):
|
||||
raise not_found("Chat not found")
|
||||
rows = conn.execute(
|
||||
f"""SELECT {MESSAGE_COLUMNS} FROM messages m
|
||||
WHERE {where}
|
||||
ORDER BY m.created_at, m.id
|
||||
LIMIT %s""",
|
||||
(*params, limit + 1),
|
||||
).fetchall()
|
||||
return [from_db(row) for row in rows]
|
||||
|
||||
|
||||
# ---------- вложения ----------
|
||||
|
||||
ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, "
|
||||
"pipeline_version, created_at, updated_at, object_key")
|
||||
VISIBLE_STATES = ("queued", "processing", "ready", "failed")
|
||||
|
||||
|
||||
def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str,
|
||||
size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None:
|
||||
"""Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging.
|
||||
|
||||
Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт
|
||||
и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД).
|
||||
"""
|
||||
try:
|
||||
with connection() as conn:
|
||||
conn.execute(
|
||||
"""INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256,
|
||||
width, height, source_key, state)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""",
|
||||
(attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256,
|
||||
width, height, source_key),
|
||||
)
|
||||
except pg_errors.ForeignKeyViolation:
|
||||
raise not_found("Chat not found") from None
|
||||
|
||||
|
||||
def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None:
|
||||
"""Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией.
|
||||
|
||||
После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL,
|
||||
сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace).
|
||||
"""
|
||||
with connection() as conn:
|
||||
with conn.transaction():
|
||||
row = conn.execute(
|
||||
f"""UPDATE attachments SET state = 'queued', updated_at = now()
|
||||
WHERE id = %s AND state = 'uploading'
|
||||
RETURNING {ATTACHMENT_COLUMNS}""",
|
||||
(attachment_id,),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
pipeline.enqueue_new(conn, attachment_id, actor)
|
||||
return from_db(row)
|
||||
|
||||
|
||||
def delete_upload_intent(attachment_id: UUID) -> None:
|
||||
with connection(wait_seconds=1) as conn:
|
||||
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,))
|
||||
|
||||
|
||||
def get_attachment(attachment_id: UUID, user_id: UUID) -> dict:
|
||||
"""Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404.
|
||||
|
||||
Возвращает строку и варианты текущей версии pipeline (они есть только у ready).
|
||||
"""
|
||||
with connection() as conn:
|
||||
row = conn.execute(
|
||||
f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))}
|
||||
FROM attachments a
|
||||
JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s
|
||||
WHERE a.id = %s AND a.state = ANY(%s)""",
|
||||
(user_id, attachment_id, list(VISIBLE_STATES)),
|
||||
).fetchone()
|
||||
if row is None:
|
||||
raise not_found("Attachment not found")
|
||||
variants = []
|
||||
if row["state"] == "ready":
|
||||
variants = conn.execute(
|
||||
"""SELECT name, content_type, size_bytes, width, height, sha256, object_key
|
||||
FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s
|
||||
ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""",
|
||||
(attachment_id, row["pipeline_version"]),
|
||||
).fetchall()
|
||||
return {**from_db(row), "variants": variants}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания.
|
||||
|
||||
Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной
|
||||
транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором
|
||||
намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox
|
||||
остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса,
|
||||
ни ключей брокера, и недоступность брокера не мешает приёму файлов.
|
||||
|
||||
Цикл:
|
||||
1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP
|
||||
LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT.
|
||||
Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения,
|
||||
который worker распознаёт по состоянию job (at-least-once, а не exactly-once).
|
||||
2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed.
|
||||
3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages).
|
||||
Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру.
|
||||
"""
|
||||
import logging
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import broker
|
||||
import faults
|
||||
import pipeline
|
||||
import settings
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
log = logging.getLogger("messenger.dispatcher")
|
||||
|
||||
STOP = threading.Event()
|
||||
ACTOR = "dispatcher@" + settings.INSTANCE_ID
|
||||
|
||||
|
||||
def connect_db():
|
||||
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||||
|
||||
|
||||
def publish_due(conn, publisher: broker.Publisher) -> int:
|
||||
published = 0
|
||||
while not STOP.is_set():
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""SELECT id, job_id, attempt, payload FROM outbox
|
||||
WHERE published_at IS NULL AND available_at <= now()
|
||||
ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""",
|
||||
(settings.DISPATCHER_BATCH,),
|
||||
).fetchall()
|
||||
if not rows:
|
||||
return published
|
||||
for row in rows:
|
||||
faults.crash("dispatcher_crash_before_publish")
|
||||
try:
|
||||
publisher.publish(row["payload"])
|
||||
except broker.BrokerError as exc:
|
||||
log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__)
|
||||
publisher.close()
|
||||
raise
|
||||
faults.crash("dispatcher_crash_after_confirm")
|
||||
conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],))
|
||||
published += 1
|
||||
log.info("published %s message(s)", len(rows))
|
||||
if len(rows) < settings.DISPATCHER_BATCH:
|
||||
return published
|
||||
return published
|
||||
|
||||
|
||||
def note_failure(conn) -> None:
|
||||
"""Счётчик неудачных публикаций для наблюдения (ops.py status)."""
|
||||
try:
|
||||
conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1
|
||||
WHERE published_at IS NULL AND available_at <= now()""")
|
||||
except psycopg.Error:
|
||||
pass
|
||||
|
||||
|
||||
def touch_alive() -> None:
|
||||
try:
|
||||
Path(settings.ALIVE_FILE).touch()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def run() -> None:
|
||||
publisher = broker.Publisher()
|
||||
conn = None
|
||||
last_leases = last_reconcile = 0.0
|
||||
pause = 1.0
|
||||
while not STOP.is_set():
|
||||
try:
|
||||
if conn is None or conn.closed:
|
||||
conn = connect_db()
|
||||
conn.execute("LISTEN outbox")
|
||||
log.info("connected to PostgreSQL, listening for outbox notifications")
|
||||
touch_alive()
|
||||
publish_due(conn, publisher)
|
||||
now = time.monotonic()
|
||||
if now - last_leases >= settings.DISPATCHER_POLL_SECONDS:
|
||||
last_leases = now
|
||||
if recovered := pipeline.recover_expired_leases(conn, ACTOR):
|
||||
log.warning("recovered %s job(s) with expired lease", recovered)
|
||||
if now - last_reconcile >= 30:
|
||||
last_reconcile = now
|
||||
if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS):
|
||||
log.warning("re-published %s queued job(s) without a message in flight", lost)
|
||||
pause = 1.0
|
||||
publisher.pump()
|
||||
# Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса.
|
||||
for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1):
|
||||
pass
|
||||
except broker.BrokerError:
|
||||
if conn is not None and not conn.closed:
|
||||
note_failure(conn)
|
||||
log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause)
|
||||
STOP.wait(pause)
|
||||
pause = min(pause * 2, 15)
|
||||
except psycopg.Error as exc:
|
||||
log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause)
|
||||
try:
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
except psycopg.Error:
|
||||
pass
|
||||
conn = None
|
||||
STOP.wait(pause)
|
||||
pause = min(pause * 2, 15)
|
||||
publisher.close()
|
||||
if conn is not None:
|
||||
conn.close()
|
||||
log.info("stopped")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not settings.DATABASE_URL or not settings.BROKER_URL:
|
||||
raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required")
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
signal.signal(sig, lambda *_: STOP.set())
|
||||
log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID,
|
||||
"ON" if settings.FAULT_INJECTION else "off")
|
||||
run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,40 @@
|
||||
# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт.
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status_code = status_code
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def bad_request(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError:
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def not_found(message: str) -> ApiError:
|
||||
return ApiError(404, "not_found", message)
|
||||
|
||||
|
||||
def conflict(message: str) -> ApiError:
|
||||
return ApiError(409, "conflict", message)
|
||||
|
||||
|
||||
def unavailable(message: str = "Service temporarily unavailable") -> ApiError:
|
||||
return ApiError(503, "unavailable", message)
|
||||
|
||||
|
||||
def payload_too_large(message: str) -> ApiError:
|
||||
return ApiError(413, "payload_too_large", message)
|
||||
|
||||
|
||||
def unsupported_media_type(message: str) -> ApiError:
|
||||
return ApiError(415, "unsupported_media_type", message)
|
||||
@@ -0,0 +1,96 @@
|
||||
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
|
||||
# date: 2026-10-24 15:37:16, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_attachments.py
|
||||
# exit code: 0, 10.2 s
|
||||
|
||||
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
|
||||
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
|
||||
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
|
||||
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
|
||||
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
|
||||
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
|
||||
exactly 10485760 bytes (10 MiB) -> 202, then ready
|
||||
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
|
||||
PASS: the 10 MiB limit is exact
|
||||
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
|
||||
PASS: oversized streamed body is cut off, the API keeps working
|
||||
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready f338bc0d
|
||||
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
|
||||
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
|
||||
PASS: pixel limit is checked before decompression
|
||||
PNG cut in half (55908 bytes, image/png) -> 415
|
||||
PNG without IEND (111805 bytes, image/png) -> 415
|
||||
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
|
||||
PNG with garbage after IEND (111824 bytes, image/png) -> 415
|
||||
PNG signature only (8 bytes, image/png) -> 415
|
||||
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
|
||||
JPEG cut in half (746 bytes, image/jpeg) -> 415
|
||||
JPEG header only (20 bytes, image/jpeg) -> 415
|
||||
random bytes (1600 bytes, image/png) -> 415
|
||||
plain text (28 bytes, image/jpeg) -> 415
|
||||
empty body -> 400
|
||||
PASS: damaged and foreign files are rejected with 415, never 500
|
||||
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
|
||||
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
|
||||
empty PLTE (79 bytes, valid CRCs) -> 415
|
||||
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
|
||||
palette image without PLTE (67 bytes, valid CRCs) -> 415
|
||||
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
|
||||
PLTE after IDAT (85 bytes, valid CRCs) -> 415
|
||||
two PLTE chunks (103 bytes, valid CRCs) -> 415
|
||||
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
|
||||
tRNS before PLTE (98 bytes, valid CRCs) -> 415
|
||||
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
|
||||
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
|
||||
IEND with data (70 bytes, valid CRCs) -> 415
|
||||
unknown critical chunk (81 bytes, valid CRCs) -> 415
|
||||
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
|
||||
gAMA after IDAT (85 bytes, valid CRCs) -> 415
|
||||
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
|
||||
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
|
||||
private chunk after IDAT (181 bytes, valid CRCs) -> 415
|
||||
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
|
||||
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
|
||||
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
|
||||
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
|
||||
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
|
||||
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
|
||||
palette depth 2 with 4 entries -> 201, downloaded byte for byte
|
||||
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
|
||||
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
|
||||
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
|
||||
PNG bytes sent as image/jpeg -> 415
|
||||
JPEG bytes sent as image/png -> 415
|
||||
GIF bytes sent as image/png -> 415
|
||||
PNG sent as image/gif -> 415
|
||||
PNG sent as text/html -> 415
|
||||
PNG sent as application/octet-stream -> 415
|
||||
PNG sent as multipart/form-data -> 415
|
||||
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
|
||||
PASS: the type is taken from the content; a declared type that does not match is refused
|
||||
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
|
||||
no filename -> 400
|
||||
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
|
||||
PASS: filename never becomes a path; dangerous names are refused
|
||||
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
|
||||
upload into someone else's / unknown chat -> 404, without token -> 401
|
||||
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
|
||||
PASS: only chat members can see an attachment
|
||||
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
|
||||
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
|
||||
attachment of another chat (author is a member of both) -> 404
|
||||
own + foreign attachment -> 404
|
||||
unknown attachment_id -> 404
|
||||
outsider referencing an attachment of the chat -> 404
|
||||
duplicate ids -> 400
|
||||
11 ids -> 400
|
||||
bad UUID -> 400
|
||||
empty text without attachments -> 400
|
||||
whitespace text without attachments -> 400
|
||||
PASS: only attachments of the same chat can be linked; rejected requests change nothing
|
||||
same key + same attachments -> same 201; other attachments -> 409; missing field == []
|
||||
PASS: Idempotency-Key covers attachment_ids
|
||||
sent 1.1 MB of a declared 5 MB body and closed the connection
|
||||
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
|
||||
PASS: an interrupted upload is dropped
|
||||
12 passed, 0 failed
|
||||
@@ -0,0 +1,18 @@
|
||||
# balancing: balancing, shared sessions/data/files, X-Instance-Id
|
||||
# date: 2026-10-24 15:40:58, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_balancing.py
|
||||
# exit code: 0, 2.1 s
|
||||
|
||||
20 x GET /users/me with one token -> replicas: {'b9ad6d33d190': 10, '5aa14958c864': 10}
|
||||
PASS: one session works on every replica, no second login (sessions are shared in Redis)
|
||||
message written by 5aa14958c864; read back by ['5aa14958c864', 'b9ad6d33d190']
|
||||
PASS: a message written via one replica is read right away via another (shared PostgreSQL)
|
||||
logout on b9ad6d33d190; old token then rejected by ['5aa14958c864', 'b9ad6d33d190']
|
||||
PASS: logout on one replica revokes the session on all of them
|
||||
first POST on b9ad6d33d190 -> 201 af45ed44-ef90-4258-b144-77628e56bfed; same key on 5aa14958c864 -> 201 af45ed44-ef90-4258-b144-77628e56bfed
|
||||
PASS: Idempotency-Key is shared: a retry through another replica returns the same message
|
||||
file uploaded via b9ad6d33d190; downloaded byte for byte via ['5aa14958c864', 'b9ad6d33d190']
|
||||
PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)
|
||||
statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['5aa14958c864', 'b9ad6d33d190']; api containers: ['5aa14958c864', 'b9ad6d33d190']
|
||||
PASS: X-Instance-Id is in every response and points to a real container
|
||||
6 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# check: offline contract check (make check)
|
||||
# date: 2026-10-24 15:30:17, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe scripts/check_contract.py
|
||||
# exit code: 0, 0.1 s
|
||||
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,47 @@
|
||||
# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage
|
||||
# date: 2026-10-24 15:40:02, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_consistency.py
|
||||
# exit code: 0, 156.4 s
|
||||
|
||||
declared 9 000 000 bytes, sent 211817, closed the connection
|
||||
declared 9 000 000 bytes, sent 3111817, closed the connection
|
||||
rows in attachments: 219 -> 219; objects in bucket: 860 -> 860
|
||||
temporary upload files in api containers: {'b9ad6d33d190': 0, '5aa14958c864': 0} (removed when the request ends)
|
||||
api containers have no volumes: the original is never stored locally
|
||||
PASS: an interrupted upload leaves no row, no object and no local file
|
||||
crash after S3 PUT: row 3a083fe9 state=uploading, object staging/5bda5e392029444d915a233aef8e7af9 exists in S3
|
||||
the half-written attachment is invisible: GET -> 404, attach to a message -> 404
|
||||
janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 865, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.31} -> fresh row and object are kept (the upload might still be running)
|
||||
row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 865, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.301} -> nothing deleted
|
||||
janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 864, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.29} -> row and object deleted
|
||||
janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 864, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.278} -> idempotent
|
||||
the ready attachment of the same chat is untouched
|
||||
PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period
|
||||
object staging/bea3e5320fc84973a5b2b762fe96ddd8 put into the bucket without a row in PostgreSQL
|
||||
janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.292} -> fresh object kept
|
||||
after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.298} -> reported, kept
|
||||
janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.295} -> orphan deleted
|
||||
ready attachments older than the grace period are untouched (they have rows)
|
||||
object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500)
|
||||
PASS: orphans are found and deleted only after the grace period; missing objects are reported
|
||||
--- S3 stopped (connection refused / name not resolved): docker compose stop s3
|
||||
upload -> 503 unavailable in 19922 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose start s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 201
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 879, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.29}
|
||||
PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
--- S3 frozen (no answer, timeouts): docker compose pause s3
|
||||
upload -> 503 unavailable in 14860 ms, no storage details in the body
|
||||
download of an existing file -> 503
|
||||
metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes
|
||||
uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API
|
||||
docker compose unpause s3
|
||||
after S3 recovery the old file downloads byte for byte
|
||||
new upload -> 201
|
||||
janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 887, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.284}
|
||||
PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up
|
||||
5 passed, 0 failed
|
||||
@@ -0,0 +1,21 @@
|
||||
# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations
|
||||
# date: 2026-10-24 15:41:16, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_db.py
|
||||
# exit code: 0, 10.7 s
|
||||
|
||||
chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 185 -> 186
|
||||
PASS: chat row was inserted and then rolled back together with members
|
||||
direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey"
|
||||
PASS: the database itself rejects a message from a non-member
|
||||
stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97)
|
||||
PASS: password is stored as Argon2id hash
|
||||
unique constraint on users: UNIQUE (username)
|
||||
PASS: username is unique in the database
|
||||
applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; running the migrate service once more
|
||||
after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; log: Migrations applied: none (schema is up to date)
|
||||
PASS: migrations are applied once, a repeated run does nothing
|
||||
api replicas that checked the schema at start: 28; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago
|
||||
PASS: only the migrate service applies migrations, replicas wait for the schema
|
||||
unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key)
|
||||
PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database
|
||||
7 passed, 0 failed
|
||||
@@ -0,0 +1,99 @@
|
||||
=== Data volume
|
||||
messages_total | messages_in_big_chat
|
||||
----------------+----------------------
|
||||
29169 | 15000
|
||||
|
||||
|
||||
=== History index
|
||||
indexdef
|
||||
----------------------------------------------------------------------------------------
|
||||
CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id)
|
||||
|
||||
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.005..0.012 rows=51 loops=1)
|
||||
Buffers: shared hit=14
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1)
|
||||
Index Cond: (chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid)
|
||||
Buffers: shared hit=14
|
||||
Planning:
|
||||
Buffers: shared hit=6
|
||||
Planning Time: 0.058 ms
|
||||
Execution Time: 0.020 ms
|
||||
|
||||
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.006..0.012 rows=51 loops=1)
|
||||
Buffers: shared hit=5
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.010 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid)))
|
||||
Buffers: shared hit=5
|
||||
Planning Time: 0.043 ms
|
||||
Execution Time: 0.029 ms
|
||||
|
||||
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
QUERY PLAN
|
||||
------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.020..0.155 rows=51 loops=1)
|
||||
Buffers: shared hit=59
|
||||
-> Index Scan using messages_history on messages m (actual time=0.019..0.152 rows=51 loops=1)
|
||||
Index Cond: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid)))
|
||||
Buffers: shared hit=59
|
||||
SubPlan 1
|
||||
-> Sort (actual time=0.002..0.002 rows=0 loops=51)
|
||||
Sort Key: a."position"
|
||||
Sort Method: quicksort Memory: 25kB
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Recheck Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Index Cond: (message_id = m.id)
|
||||
Buffers: shared hit=51
|
||||
Planning:
|
||||
Buffers: shared hit=63
|
||||
Planning Time: 0.205 ms
|
||||
Execution Time: 0.172 ms
|
||||
|
||||
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
QUERY PLAN
|
||||
---------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=0.832..0.838 rows=51 loops=1)
|
||||
Buffers: shared hit=266
|
||||
-> Index Scan using messages_history on messages (actual time=0.007..0.645 rows=10051 loops=1)
|
||||
Index Cond: (chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid)
|
||||
Buffers: shared hit=266
|
||||
Planning Time: 0.041 ms
|
||||
Execution Time: 0.849 ms
|
||||
|
||||
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
QUERY PLAN
|
||||
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
||||
Limit (actual time=1.257..1.261 rows=51 loops=1)
|
||||
Buffers: shared hit=450
|
||||
-> Sort (actual time=1.257..1.258 rows=51 loops=1)
|
||||
Sort Key: created_at, id
|
||||
Sort Method: top-N heapsort Memory: 31kB
|
||||
Buffers: shared hit=450
|
||||
-> Seq Scan on messages (actual time=0.530..0.937 rows=5000 loops=1)
|
||||
Filter: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid)))
|
||||
Rows Removed by Filter: 24169
|
||||
Buffers: shared hit=450
|
||||
Planning:
|
||||
Buffers: shared hit=4
|
||||
Planning Time: 0.082 ms
|
||||
Execution Time: 1.274 ms
|
||||
|
||||
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
explain_users_left
|
||||
--------------------
|
||||
0
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# explain: EXPLAIN of the history page (full plans: explain-history.txt)
|
||||
# date: 2026-10-24 15:41:20, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/explain.py
|
||||
# exit code: 0, 0.7 s
|
||||
|
||||
=== Data volume
|
||||
=== History index
|
||||
=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page
|
||||
Buffers: shared hit=14
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1)
|
||||
Buffers: shared hit=14
|
||||
Buffers: shared hit=6
|
||||
Execution Time: 0.020 ms
|
||||
=== 2. Page after the 10000th message by keyset cursor (this is what the app does)
|
||||
Buffers: shared hit=5
|
||||
-> Index Scan using messages_history on messages (actual time=0.005..0.010 rows=51 loops=1)
|
||||
Buffers: shared hit=5
|
||||
Execution Time: 0.029 ms
|
||||
=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message
|
||||
Buffers: shared hit=59
|
||||
-> Index Scan using messages_history on messages m (actual time=0.019..0.152 rows=51 loops=1)
|
||||
Buffers: shared hit=59
|
||||
Buffers: shared hit=54
|
||||
-> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
-> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51)
|
||||
Buffers: shared hit=51
|
||||
Buffers: shared hit=63
|
||||
Execution Time: 0.172 ms
|
||||
=== 3. The same page with OFFSET 10000 (the app does NOT do this)
|
||||
Buffers: shared hit=266
|
||||
-> Index Scan using messages_history on messages (actual time=0.007..0.645 rows=10051 loops=1)
|
||||
Buffers: shared hit=266
|
||||
Execution Time: 0.849 ms
|
||||
=== 4. Query 2 without the index (index is dropped only inside this transaction)
|
||||
Buffers: shared hit=450
|
||||
Buffers: shared hit=450
|
||||
-> Seq Scan on messages (actual time=0.530..0.937 rows=5000 loops=1)
|
||||
Buffers: shared hit=450
|
||||
Buffers: shared hit=4
|
||||
Execution Time: 1.274 ms
|
||||
=== ROLLBACK: test data removed, index is back
|
||||
PASS: full plans saved to evidence\explain-history.txt
|
||||
@@ -0,0 +1,33 @@
|
||||
# failover: public failover test for every replica, file downloaded via the other one (make test-failover)
|
||||
# date: 2026-10-24 15:42:00, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/run_failover.py
|
||||
# exit code: 0, 40.6 s
|
||||
|
||||
hook: scripts\stop-one.cmd; api replicas: ['5aa14958c864', 'b9ad6d33d190']
|
||||
=== run 1: api containers before
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b9ad6d33d190 Up 44 seconds (healthy)
|
||||
api 5aa14958c864 Up 43 seconds (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
|
||||
{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.406}␍
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b9ad6d33d190 Up About a minute (healthy)
|
||||
api 5aa14958c864 Exited (0) 11 seconds ago
|
||||
the hook stopped 5aa14958c864 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start 5aa14958c864 -> healthy and back in HAProxy
|
||||
=== run 2: api containers before
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b9ad6d33d190 Up About a minute (healthy)
|
||||
api 5aa14958c864 Up 2 seconds (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.␍
|
||||
{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 4.188}␍
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
api containers after
|
||||
SERVICE CONTAINER ID STATUS
|
||||
api b9ad6d33d190 Exited (0) 11 seconds ago
|
||||
api 5aa14958c864 Up 19 seconds (healthy)
|
||||
the hook stopped b9ad6d33d190 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched
|
||||
restored: docker start b9ad6d33d190 -> healthy and back in HAProxy
|
||||
PASS: public failover test passed 2 times; every api replica was stopped once and restored
|
||||
@@ -0,0 +1,29 @@
|
||||
# failures: Redis and PostgreSQL outages via the balancer
|
||||
# date: 2026-10-24 15:43:26, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_failures.py
|
||||
# exit code: 0, 55.0 s
|
||||
|
||||
stopping redis
|
||||
/health/live -> 200, /health/ready -> 503 (8016 ms)
|
||||
valid token: GET /users/me -> 503 unavailable (8015 ms), session cannot be checked
|
||||
unknown token -> 503, not 200: nobody is trusted without a check
|
||||
no token -> 401
|
||||
login -> 503: there is nowhere to store the session
|
||||
registration -> 201: it needs only PostgreSQL
|
||||
starting redis
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: Redis outage gives 503 and does not let anyone in without a session check
|
||||
killing redis with SIGKILL (docker compose kill redis): no graceful shutdown
|
||||
right after the crash: GET /users/me -> 503
|
||||
starting redis
|
||||
after start: the session issued before the crash still works (AOF, appendfsync always)
|
||||
PASS: sessions survive a Redis crash, not only a clean restart
|
||||
stopping postgres
|
||||
/health/live -> 200, /health/ready -> 503 (1016 ms)
|
||||
valid token: users/me, chats, history, new message -> 503 unavailable, no details
|
||||
no token -> 401, unknown token -> 401: anonymous users are not let in
|
||||
login -> 503, registration -> 503
|
||||
starting postgres
|
||||
after start: the old token works, confirmed messages are in place
|
||||
PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out
|
||||
3 passed, 0 failed
|
||||
@@ -0,0 +1,22 @@
|
||||
# idempotency: Idempotency-Key, parallel same key via replicas
|
||||
# date: 2026-10-24 15:41:01, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_idempotency.py
|
||||
# exit code: 0, 2.5 s
|
||||
|
||||
first -> 201; same JSON written differently -> 201; plain retry -> 201
|
||||
PASS: a retry with the same key returns the original 201 and message, no duplicate
|
||||
20 parallel POSTs with one key: statuses {201: 20}, replicas {'5aa14958c864': 10, 'b9ad6d33d190': 10}, different bodies in answers: 1, messages in history: 1
|
||||
PASS: parallel requests with one key through different replicas create exactly one message
|
||||
20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A']
|
||||
PASS: the first body wins, the same body gets 201, the other body gets 409
|
||||
one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages
|
||||
PASS: the key is scoped to the current user and chat; another body with a used key -> 409
|
||||
missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201
|
||||
PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}
|
||||
key used by a 400 and by a 404 first, then valid request -> 201
|
||||
PASS: rejected requests (400, 404) do not store a result for the key
|
||||
same key and body by an outsider -> 404; by alice after logout -> 401
|
||||
PASS: a retry checks the session and chat access again, the saved answer is not leaked
|
||||
POST sent, connection closed without reading the answer; retry -> 201; messages: 1
|
||||
PASS: an unconfirmed POST can be safely retried with the same key
|
||||
8 passed, 0 failed
|
||||
@@ -0,0 +1,8 @@
|
||||
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
|
||||
# date: 2026-10-24 15:37:05, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py concurrency
|
||||
# exit code: 0, 15.1 s
|
||||
|
||||
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
|
||||
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples)
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
|
||||
# date: 2026-10-24 15:35:40, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
|
||||
# exit code: 0, 156.3 s
|
||||
|
||||
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
|
||||
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
|
||||
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
|
||||
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
|
||||
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
|
||||
staging source still present after the crash (worker died before cleanup); waiting 60 s window
|
||||
janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone
|
||||
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
|
||||
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,13 @@
|
||||
# jobs-janitor-replay: lab5: janitor does not delete objects re-written by replay (review fix)
|
||||
# date: 2026-10-26 13:37:58, commit: 9863bb0 + uncommitted changes
|
||||
# command: python.exe student-tests/test_janitor_replay.py
|
||||
# exit code: 0, 186.4 s
|
||||
|
||||
race-replayable: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
race-terminal: failed/retry_exhausted after a crash post-PUT; leftover objects ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
waiting 83 s until the leftovers are older than grace (120 s)
|
||||
janitor pass over the bucket: did not try to delete any object of the replayable failed attachment
|
||||
ops replay -> job 6d86b987-d46f-489a-84db-ea63e1c89579 ready
|
||||
after the janitor pass: original/thumbnail/crop/optimized -> 200, 4 objects in S3; janitor report: orphans 0, deleted 0
|
||||
retention expired (STAGING_RETENTION_SECONDS=1 for this pass): staging source removed, leftovers of the failed attachment swept; replay refused: "source file is no longer in staging (retention expired)"
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,11 @@
|
||||
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
|
||||
# date: 2026-10-24 15:31:54, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py limits
|
||||
# exit code: 0, 73.6 s
|
||||
|
||||
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
|
||||
10 MiB + 8 bytes -> 413 before any job is created
|
||||
imaging.py on 8000x5000 PNG: exit 0, peak RSS 227.2 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
|
||||
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
|
||||
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,9 @@
|
||||
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
|
||||
# date: 2026-10-24 15:33:04, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py outbox-crash
|
||||
# exit code: 0, 15.1 s
|
||||
|
||||
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
|
||||
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action
|
||||
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,16 @@
|
||||
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
|
||||
# date: 2026-10-24 15:32:49, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py worker-down broker-down
|
||||
# exit code: 0, 54.9 s
|
||||
|
||||
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
|
||||
worker stopped
|
||||
upload while worker is down -> 202 queued, attachment cabbd2a3-fa8c-4805-aa76-6d59e2b5d68a
|
||||
message with attachment 201, content 409, other messages 201; job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 published, queue messages/unacked = (1, 0)
|
||||
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
|
||||
worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted
|
||||
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
|
||||
rabbitmq stopped
|
||||
upload -> 202 and /health/ready 200 without broker; outbox row pending for job 17043324-e0c6-4c0c-88df-6ae2a8eca8fd
|
||||
rabbitmq started -> dispatcher published the pending row -> ready
|
||||
2 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
|
||||
# date: 2026-10-24 15:36:50, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq
|
||||
# exit code: 0, 69.7 s
|
||||
|
||||
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
|
||||
2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
|
||||
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
|
||||
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
|
||||
ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
|
||||
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
|
||||
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
|
||||
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
|
||||
schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,14 @@
|
||||
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
|
||||
# date: 2026-10-24 15:30:40, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_jobs.py variants
|
||||
# exit code: 0, 19.5 s
|
||||
|
||||
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
|
||||
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 5,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.047,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 7328,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 243.885,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.0,
|
||||
"p95": 32.0,
|
||||
"p99": 32.0
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# load: public load generator, 30 s, concurrency 4
|
||||
# date: 2026-10-24 15:42:31, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
|
||||
# exit code: 0, 30.3 s
|
||||
|
||||
{
|
||||
"lab": 5,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.047,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 7328,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 243.885,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.0,
|
||||
"p95": 32.0,
|
||||
"p99": 32.0
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids
|
||||
# date: 2026-10-24 14:39:50, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_attachments.py
|
||||
# exit code: 0, 10.5 s
|
||||
|
||||
sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes
|
||||
sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes
|
||||
progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes
|
||||
interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes
|
||||
1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes
|
||||
PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte
|
||||
exactly 10485760 bytes (10 MiB) -> 202, then ready
|
||||
10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body
|
||||
PASS: the 10 MiB limit is exact
|
||||
12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed
|
||||
PASS: oversized streamed body is cut off, the API keeps working
|
||||
8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 4fea67fd
|
||||
8001x5000 = 40 005 000 pixels (38962 bytes) -> 413
|
||||
header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded
|
||||
PASS: pixel limit is checked before decompression
|
||||
PNG cut in half (55908 bytes, image/png) -> 415
|
||||
PNG without IEND (111805 bytes, image/png) -> 415
|
||||
PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415
|
||||
PNG with garbage after IEND (111824 bytes, image/png) -> 415
|
||||
PNG signature only (8 bytes, image/png) -> 415
|
||||
PNG with less pixel data than the header says (68 bytes, image/png) -> 415
|
||||
JPEG cut in half (746 bytes, image/jpeg) -> 415
|
||||
JPEG header only (20 bytes, image/jpeg) -> 415
|
||||
random bytes (1600 bytes, image/png) -> 415
|
||||
plain text (28 bytes, image/jpeg) -> 415
|
||||
empty body -> 400
|
||||
PASS: damaged and foreign files are rejected with 415, never 500
|
||||
PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415
|
||||
PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415
|
||||
empty PLTE (79 bytes, valid CRCs) -> 415
|
||||
palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415
|
||||
palette image without PLTE (67 bytes, valid CRCs) -> 415
|
||||
PLTE in a grayscale image (85 bytes, valid CRCs) -> 415
|
||||
PLTE after IDAT (85 bytes, valid CRCs) -> 415
|
||||
two PLTE chunks (103 bytes, valid CRCs) -> 415
|
||||
tRNS longer than the palette (100 bytes, valid CRCs) -> 415
|
||||
tRNS before PLTE (98 bytes, valid CRCs) -> 415
|
||||
tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415
|
||||
tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415
|
||||
IEND with data (70 bytes, valid CRCs) -> 415
|
||||
unknown critical chunk (81 bytes, valid CRCs) -> 415
|
||||
IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415
|
||||
gAMA after IDAT (85 bytes, valid CRCs) -> 415
|
||||
gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415
|
||||
3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415
|
||||
private chunk after IDAT (181 bytes, valid CRCs) -> 415
|
||||
animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415
|
||||
tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415
|
||||
tEXt without keyword separator (86 bytes, valid CRCs) -> 415
|
||||
zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415
|
||||
zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415
|
||||
palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte
|
||||
palette depth 2 with 4 entries -> 201, downloaded byte for byte
|
||||
RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte
|
||||
compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte
|
||||
PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted
|
||||
PNG bytes sent as image/jpeg -> 415
|
||||
JPEG bytes sent as image/png -> 415
|
||||
GIF bytes sent as image/png -> 415
|
||||
PNG sent as image/gif -> 415
|
||||
PNG sent as text/html -> 415
|
||||
PNG sent as application/octet-stream -> 415
|
||||
PNG sent as multipart/form-data -> 415
|
||||
PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png
|
||||
PASS: the type is taken from the content; a declared type that does not match is refused
|
||||
13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400
|
||||
no filename -> 400
|
||||
Unicode, 255 chars, leading dots, %20 and ? are kept as display names only
|
||||
PASS: filename never becomes a path; dangerous names are refused
|
||||
outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400
|
||||
upload into someone else's / unknown chat -> 404, without token -> 401
|
||||
variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)
|
||||
PASS: only chat members can see an attachment
|
||||
text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member
|
||||
history returns attachment_ids for every message, [] for a message without files; reuse is allowed
|
||||
attachment of another chat (author is a member of both) -> 404
|
||||
own + foreign attachment -> 404
|
||||
unknown attachment_id -> 404
|
||||
outsider referencing an attachment of the chat -> 404
|
||||
duplicate ids -> 400
|
||||
11 ids -> 400
|
||||
bad UUID -> 400
|
||||
empty text without attachments -> 400
|
||||
whitespace text without attachments -> 400
|
||||
PASS: only attachments of the same chat can be linked; rejected requests change nothing
|
||||
same key + same attachments -> same 201; other attachments -> 409; missing field == []
|
||||
PASS: Idempotency-Key covers attachment_ids
|
||||
sent 1.1 MB of a declared 5 MB body and closed the connection
|
||||
the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py)
|
||||
PASS: an interrupted upload is dropped
|
||||
12 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# check: offline contract check (make check)
|
||||
# date: 2026-10-24 14:34:04, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 scripts/check_contract.py
|
||||
# exit code: 0, 0.0 s
|
||||
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,8 @@
|
||||
# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing
|
||||
# date: 2026-10-24 14:39:39, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_jobs.py concurrency
|
||||
# exit code: 0, 7.5 s
|
||||
|
||||
--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.
|
||||
2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples)
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease
|
||||
# date: 2026-10-24 14:38:43, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_jobs.py crash-after-put crash-after-commit staging lease
|
||||
# exit code: 0, 143.5 s
|
||||
|
||||
--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.
|
||||
worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready']
|
||||
--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.
|
||||
worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked)
|
||||
--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).
|
||||
staging source still present after the crash (worker died before cleanup); waiting 60 s window
|
||||
janitor removed the staging source of the ready attachment: 1
|
||||
--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает;
|
||||
frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,11 @@
|
||||
# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)
|
||||
# date: 2026-10-24 14:35:26, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_jobs.py limits
|
||||
# exit code: 0, 69.1 s
|
||||
|
||||
--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.
|
||||
10 MiB + 8 bytes -> 413 before any job is created
|
||||
imaging.py on 8000x5000 PNG: exit 0, peak RSS 223.4 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)
|
||||
memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived
|
||||
imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,9 @@
|
||||
# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm
|
||||
# date: 2026-10-24 14:36:19, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_jobs.py outbox-crash
|
||||
# exit code: 0, 7.9 s
|
||||
|
||||
--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.
|
||||
dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action
|
||||
dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,16 @@
|
||||
# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits
|
||||
# date: 2026-10-24 14:36:11, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_jobs.py worker-down broker-down
|
||||
# exit code: 0, 45.9 s
|
||||
|
||||
--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.
|
||||
worker stopped
|
||||
upload while worker is down -> 202 queued, attachment 5b41ee9a-207b-46d6-a0af-635082295231
|
||||
message with attachment 201, content 409, other messages 201; job d62da8f8-b127-4141-95c3-8f691c39bb72 published, queue messages/unacked = (1, 0)
|
||||
rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued
|
||||
worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted
|
||||
--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.
|
||||
rabbitmq stopped
|
||||
upload -> 202 and /health/ready 200 without broker; outbox row pending for job aab8166b-e551-4ec9-9566-0f8207527e36
|
||||
rabbitmq started -> dispatcher published the pending row -> ready
|
||||
2 passed, 0 failed
|
||||
@@ -0,0 +1,15 @@
|
||||
# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ
|
||||
# date: 2026-10-24 14:39:32, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_jobs.py retry exhausted-replay permanent dlq
|
||||
# exit code: 0, 48.8 s
|
||||
|
||||
--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.
|
||||
2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready
|
||||
--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.
|
||||
5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable)
|
||||
ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs"
|
||||
--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.
|
||||
invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)
|
||||
--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.
|
||||
schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis']
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,14 @@
|
||||
# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped
|
||||
# date: 2026-10-24 14:34:16, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_jobs.py variants
|
||||
# exit code: 0, 11.4 s
|
||||
|
||||
--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.
|
||||
thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg']
|
||||
unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload
|
||||
1 passed, 0 failed
|
||||
@@ -0,0 +1,16 @@
|
||||
# persistence (native sandbox): public tests/resilience.py --mode persistence with scripts/restart.sh
|
||||
# date: 2026-10-24 14:40:30; run_persistence.py (docker inspect proof of restarts) needs real Docker
|
||||
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
# exit code: 0
|
||||
|
||||
# processes after the hook (new PIDs = real restarts):
|
||||
postgres running 2971
|
||||
redis running 2964
|
||||
rabbitmq running 3077
|
||||
s3 running 2988
|
||||
api running 3504
|
||||
dispatcher running 3518
|
||||
worker running 3522
|
||||
janitor running 3508
|
||||
@@ -0,0 +1,10 @@
|
||||
# race: lab2 races via the balancer
|
||||
# date: 2026-10-24 14:39:54, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_race.py
|
||||
# exit code: 0, 1.8 s
|
||||
|
||||
sent 100 messages from 10 clients in 0.50 s; read 100 in 4 pages, unique ids: 100
|
||||
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
|
||||
20 parallel registrations of one username: 201 x1, 409 x19
|
||||
PASS: concurrent registration created exactly one user
|
||||
2 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# smoke: public smoke via the balancer (make test), with attachment
|
||||
# date: 2026-10-24 14:34:05, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
# exit code: 0, 0.8 s
|
||||
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,26 @@
|
||||
# student-tests: lab1/lab2 API tests via the balancer
|
||||
# date: 2026-10-24 14:39:52, commit: 81d8f37 + uncommitted changes
|
||||
# command: python3 student-tests/test_api.py
|
||||
# exit code: 0, 2.6 s
|
||||
|
||||
PASS: duplicate username returns 409
|
||||
PASS: invalid JSON returns 400
|
||||
PASS: wrong and extra fields return 400
|
||||
PASS: foreign chat returns 404
|
||||
PASS: empty and too long messages return 400, 4000 chars is ok
|
||||
PASS: pagination works across multiple pages, foreign cursor returns 400
|
||||
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
|
||||
PASS: chat list pagination
|
||||
PASS: unicode text is stored as is
|
||||
PASS: strings with U+0000 are stored and returned as is
|
||||
PASS: user response has no password or hash
|
||||
PASS: password length 12..128, blank password is rejected
|
||||
PASS: Basic login returns opaque Bearer token with expiry
|
||||
PASS: wrong password and unknown user both return the same 401
|
||||
PASS: malformed Basic header returns 401
|
||||
PASS: password with colon and unicode works
|
||||
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
|
||||
PASS: X-User-Id does not change the current user
|
||||
PASS: 401 comes before 400 and 404
|
||||
PASS: logout revokes only the current session
|
||||
20 passed, 0 failed
|
||||
@@ -0,0 +1,40 @@
|
||||
# Сводка проверок
|
||||
|
||||
| Шаг | Что проверяет | Результат | Время, с | Файл |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| check | offline contract check (make check) | PASS | 0 | evidence/native-sandbox/check.txt |
|
||||
| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/native-sandbox/smoke.txt |
|
||||
| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 11 | evidence/native-sandbox/jobs-variants.txt |
|
||||
| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 69 | evidence/native-sandbox/jobs-limits.txt |
|
||||
| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 46 | evidence/native-sandbox/jobs-recovery.txt |
|
||||
| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 8 | evidence/native-sandbox/jobs-outbox.txt |
|
||||
| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 143 | evidence/native-sandbox/jobs-crash.txt |
|
||||
| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 49 | evidence/native-sandbox/jobs-retry.txt |
|
||||
| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 7 | evidence/native-sandbox/jobs-concurrency.txt |
|
||||
| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 11 | evidence/native-sandbox/attachments.txt |
|
||||
| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/native-sandbox/student-tests.txt |
|
||||
| race | lab2 races via the balancer | PASS | 2 | evidence/native-sandbox/race.txt |
|
||||
| persistence | public persistence test (make test-persistence) on the native stand; docker-inspect proof needs Docker | PASS | 40 | evidence/native-sandbox/persistence.txt |
|
||||
|
||||
# Где прогнано
|
||||
|
||||
Нативный стенд без Docker (в среде сборки нет доступа к Docker Hub): PostgreSQL 16, RabbitMQ 3.12.1, SeaweedFS 3.80, Redis — процессами с той же конфигурацией и секретами, что в compose.yaml; 1 реплика API на 8080 без HAProxy; Python 3.11. Команды `docker compose ...` тестов выполнялись через шим, переводящий их на эти процессы. Для защиты повторить на Docker: `python student-tests/collect_evidence.py` (вывод в evidence/).
|
||||
|
||||
# Отказы lab5
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
|
||||
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
|
||||
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action | без ручного действия |
|
||||
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
|
||||
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
|
||||
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
|
||||
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
|
||||
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready |
|
||||
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
|
||||
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
|
||||
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
|
||||
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
|
||||
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены |
|
||||
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) | - |
|
||||
@@ -0,0 +1,30 @@
|
||||
# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence)
|
||||
# date: 2026-10-24 15:44:25, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/run_persistence.py
|
||||
# exit code: 0, 59.6 s
|
||||
|
||||
before: container, ID, started at
|
||||
messenger-lab5-api-1 b9ad6d33d190 2026-10-24T12:41:59.057757842Z
|
||||
messenger-lab5-api-2 5aa14958c864 2026-10-24T12:41:38.920683556Z
|
||||
messenger-lab5-dispatcher-1 9c5531a487f2 2026-10-24T12:32:58.22608557Z
|
||||
messenger-lab5-janitor-1 c412a9460051 2026-10-24T12:23:20.245209318Z
|
||||
messenger-lab5-lb-1 0760689bc54d 2026-10-24T12:23:27.648064757Z
|
||||
messenger-lab5-postgres-1 10fec15d4ea6 2026-10-24T12:43:25.148756098Z
|
||||
messenger-lab5-rabbitmq-1 95b4214594a8 2026-10-24T12:32:33.167311219Z
|
||||
messenger-lab5-redis-1 e7ed9903c342 2026-10-24T12:43:04.620093059Z
|
||||
messenger-lab5-s3-1 56a0cdfb9bc8 2026-10-24T12:39:05.007533298Z
|
||||
messenger-lab5-worker-1 6e040d05805f 2026-10-24T12:37:00.256852184Z
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.␍
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
after (hook + waiting for readiness took 58.5 s): container, ID, started at, restarted?
|
||||
messenger-lab5-api-1 b9ad6d33d190 2026-10-24T12:44:20.339074469Z same container: True, restarted: True
|
||||
messenger-lab5-api-2 5aa14958c864 2026-10-24T12:44:20.477467705Z same container: True, restarted: True
|
||||
messenger-lab5-dispatcher-1 9c5531a487f2 2026-10-24T12:44:17.516290602Z same container: True, restarted: True
|
||||
messenger-lab5-janitor-1 c412a9460051 2026-10-24T12:44:16.494020858Z same container: True, restarted: True
|
||||
messenger-lab5-lb-1 0760689bc54d 2026-10-24T12:44:23.831977922Z same container: True, restarted: True
|
||||
messenger-lab5-postgres-1 10fec15d4ea6 2026-10-24T12:43:40.513609095Z same container: True, restarted: True
|
||||
messenger-lab5-rabbitmq-1 95b4214594a8 2026-10-24T12:43:41.265182826Z same container: True, restarted: True
|
||||
messenger-lab5-redis-1 e7ed9903c342 2026-10-24T12:43:40.396962627Z same container: True, restarted: True
|
||||
messenger-lab5-s3-1 56a0cdfb9bc8 2026-10-24T12:43:50.186903251Z same container: True, restarted: True
|
||||
messenger-lab5-worker-1 6e040d05805f 2026-10-24T12:44:17.661567668Z same container: True, restarted: True
|
||||
PASS: the hook restarted postgres, redis, s3 (SeaweedFS), rabbitmq, all api replicas, janitor, dispatcher, worker and lb (same containers, new start time); data, the attachment bytes and the old token survived
|
||||
@@ -0,0 +1,10 @@
|
||||
# race: lab2 races via the balancer
|
||||
# date: 2026-10-24 15:41:05, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_race.py
|
||||
# exit code: 0, 1.2 s
|
||||
|
||||
sent 100 messages from 10 clients in 0.20 s; read 100 in 4 pages, unique ids: 100
|
||||
PASS: 100 concurrent messages from 10 clients, no losses and no duplicates
|
||||
20 parallel registrations of one username: 201 x1, 409 x19
|
||||
PASS: concurrent registration created exactly one user
|
||||
2 passed, 0 failed
|
||||
@@ -0,0 +1,14 @@
|
||||
# redis: Redis as the session store
|
||||
# date: 2026-10-24 15:41:19, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/redis_info.py
|
||||
# exit code: 0, 3.1 s
|
||||
|
||||
session key: session:5dc10133a5554a2f...; value is the user id: True
|
||||
TTL of the key, seconds: 3599
|
||||
sessions in Redis: 299
|
||||
config appendonly = yes
|
||||
config appendfsync = always
|
||||
config maxmemory-policy = noeviction
|
||||
aof_enabled:1
|
||||
aof_last_write_status:ok
|
||||
without password: NOAUTH Authentication required.
|
||||
@@ -0,0 +1,47 @@
|
||||
# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap
|
||||
# date: 2026-10-24 15:37:26, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_s3_security.py
|
||||
# exit code: 0, 10.3 s
|
||||
|
||||
anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied
|
||||
anonymous head_object directly to S3 (from the data network) -> 403
|
||||
anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied
|
||||
anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied
|
||||
anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied
|
||||
the same file through the API for a chat member -> 200
|
||||
PASS: the bucket is private: no anonymous read, list, write or policy read
|
||||
API key: list buckets -> ['messenger-attachments']
|
||||
API key: get own object -> allowed
|
||||
API key: put object -> allowed
|
||||
API key: delete object -> allowed
|
||||
API key: list bucket -> allowed
|
||||
API key: create another bucket -> denied: AccessDenied
|
||||
API key: make bucket public (put policy) -> denied: NotImplemented
|
||||
API key: read bucket policy -> denied: NoSuchBucketPolicy
|
||||
API key: delete bucket -> denied: AccessDenied
|
||||
API key: change versioning -> denied: NotImplemented
|
||||
PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied
|
||||
compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand)
|
||||
127.0.0.1:8333 from the host -> connection refused
|
||||
127.0.0.1:9333 from the host -> connection refused
|
||||
127.0.0.1:8888 from the host -> connection refused
|
||||
127.0.0.1:19333 from the host -> connection refused
|
||||
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of dispatcher: ['broker_url_dispatcher', 'database_url']
|
||||
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of lb: -
|
||||
secrets of migrate: ['database_url']
|
||||
secrets of postgres: ['postgres_password']
|
||||
secrets of rabbitmq: ['rabbitmq_definitions']
|
||||
secrets of redis: ['redis_auth']
|
||||
secrets of s3: ['s3_config']
|
||||
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
|
||||
secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init
|
||||
s3-init: s3-init: bucket messenger-attachments exists
|
||||
s3-init: s3-init: api key can put/get/list/delete objects of its bucket
|
||||
s3-init: s3-init: api key cannot create buckets (AccessDenied)
|
||||
s3-init: s3-init: bucket is private (anonymous list/get/put -> [403])
|
||||
after the repeated bootstrap: old file downloads, new upload works
|
||||
PASS: bucket/user/policy bootstrap can be repeated safely
|
||||
4 passed, 0 failed
|
||||
@@ -0,0 +1,6 @@
|
||||
# smoke: public smoke via the balancer (make test), with attachment
|
||||
# date: 2026-10-24 15:30:21, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080
|
||||
# exit code: 0, 1.0 s
|
||||
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,49 @@
|
||||
# stack: stand layout: replicas, ports, limits, networks, secrets, S3
|
||||
# date: 2026-10-24 15:30:20, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/check_stack.py
|
||||
# exit code: 0, 3.0 s
|
||||
|
||||
host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1
|
||||
Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop
|
||||
|
||||
SERVICE CONTAINER ID STATUS PORTS
|
||||
api b9ad6d33d190 Up 4 minutes (healthy) 8080/tcp
|
||||
api 5aa14958c864 Up 4 minutes (healthy) 8080/tcp
|
||||
dispatcher 9c5531a487f2 Up 3 minutes (healthy)
|
||||
janitor c412a9460051 Up 6 minutes (healthy)
|
||||
lb 0760689bc54d Up 6 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
migrate 72d1affe9c8e Exited (0) 3 seconds ago
|
||||
postgres 10fec15d4ea6 Up 7 minutes (healthy)
|
||||
rabbitmq 95b4214594a8 Up 4 minutes (healthy)
|
||||
redis e7ed9903c342 Up 7 minutes (healthy)
|
||||
s3 56a0cdfb9bc8 Up 7 minutes (healthy)
|
||||
s3-init 39d4358fe2ba Exited (0) 2 seconds ago
|
||||
worker 6e040d05805f Up 3 minutes (healthy)
|
||||
|
||||
container image cpus memory host ports networks
|
||||
messenger-lab5-api-1 messenger-lab5-api 1.00 512 MiB - data,edge
|
||||
messenger-lab5-api-2 messenger-lab5-api 1.00 512 MiB - data,edge
|
||||
messenger-lab5-dispatcher-1 messenger-lab5-dispatcher 0.25 128 MiB - data
|
||||
messenger-lab5-janitor-1 messenger-lab5-janitor 0.25 192 MiB - data
|
||||
messenger-lab5-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge
|
||||
messenger-lab5-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data
|
||||
messenger-lab5-rabbitmq-1 rabbitmq:4.1.4-alpine 1.00 512 MiB - data
|
||||
messenger-lab5-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data
|
||||
messenger-lab5-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data
|
||||
messenger-lab5-worker-1 messenger-lab5-worker 2.00 1536 MiB - data
|
||||
|
||||
secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of dispatcher: ['broker_url_dispatcher', 'database_url']
|
||||
secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
secrets of lb: -
|
||||
secrets of migrate: ['database_url']
|
||||
secrets of postgres: ['postgres_password']
|
||||
secrets of rabbitmq: ['rabbitmq_definitions']
|
||||
secrets of redis: ['redis_auth']
|
||||
secrets of s3: ['s3_config']
|
||||
secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key']
|
||||
secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key']
|
||||
|
||||
through http://127.0.0.1:8080 one session is served by 2 replicas: ['5aa14958c864', 'b9ad6d33d190']
|
||||
/health/ready -> 200 from b9ad6d33d190
|
||||
PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor/rabbitmq/worker/dispatcher only in the internal data network, S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker
|
||||
@@ -0,0 +1,26 @@
|
||||
# student-tests: lab1/lab2 API tests via the balancer
|
||||
# date: 2026-10-24 15:41:04, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/test_api.py
|
||||
# exit code: 0, 2.7 s
|
||||
|
||||
PASS: duplicate username returns 409
|
||||
PASS: invalid JSON returns 400
|
||||
PASS: wrong and extra fields return 400
|
||||
PASS: foreign chat returns 404
|
||||
PASS: empty and too long messages return 400, 4000 chars is ok
|
||||
PASS: pagination works across multiple pages, foreign cursor returns 400
|
||||
PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page
|
||||
PASS: chat list pagination
|
||||
PASS: unicode text is stored as is
|
||||
PASS: strings with U+0000 are stored and returned as is
|
||||
PASS: user response has no password or hash
|
||||
PASS: password length 12..128, blank password is rejected
|
||||
PASS: Basic login returns opaque Bearer token with expiry
|
||||
PASS: wrong password and unknown user both return the same 401
|
||||
PASS: malformed Basic header returns 401
|
||||
PASS: password with colon and unicode works
|
||||
PASS: business endpoints need Bearer; Basic and X-User-Id give 401
|
||||
PASS: X-User-Id does not change the current user
|
||||
PASS: 401 comes before 400 and 404
|
||||
PASS: logout revokes only the current session
|
||||
20 passed, 0 failed
|
||||
@@ -0,0 +1,79 @@
|
||||
# Сводка проверок
|
||||
|
||||
| Шаг | Что проверяет | Результат | Время, с | Файл |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| check | offline contract check (make check) | PASS | 0 | evidence/check.txt |
|
||||
| stack | stand layout: replicas, ports, limits, networks, secrets, S3 | PASS | 3 | evidence/stack.txt |
|
||||
| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/smoke.txt |
|
||||
| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 20 | evidence/jobs-variants.txt |
|
||||
| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 74 | evidence/jobs-limits.txt |
|
||||
| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 55 | evidence/jobs-recovery.txt |
|
||||
| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 15 | evidence/jobs-outbox.txt |
|
||||
| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 156 | evidence/jobs-crash.txt |
|
||||
| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 70 | evidence/jobs-retry.txt |
|
||||
| jobs-janitor-replay | lab5: janitor does not delete objects re-written by replay (review fix) | PASS | 186 | evidence/jobs-janitor-replay.txt |
|
||||
| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 15 | evidence/jobs-concurrency.txt |
|
||||
| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 10 | evidence/attachments.txt |
|
||||
| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt |
|
||||
| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 156 | evidence/consistency.txt |
|
||||
| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 54 | evidence/upload-memory.txt |
|
||||
| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt |
|
||||
| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt |
|
||||
| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt |
|
||||
| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt |
|
||||
| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 11 | evidence/db.txt |
|
||||
| redis | Redis as the session store | PASS | 3 | evidence/redis.txt |
|
||||
| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt |
|
||||
| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 41 | evidence/failover.txt |
|
||||
| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt |
|
||||
| failures | Redis and PostgreSQL outages via the balancer | PASS | 55 | evidence/failures.txt |
|
||||
| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 60 | evidence/persistence.txt |
|
||||
|
||||
# Замеры
|
||||
|
||||
Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-24, commit 276dbda + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB.
|
||||
|
||||
Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):
|
||||
|
||||
| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 1 | 5aa14958c864 | 2 | 123 | 0 | 4.406 |
|
||||
| 2 | b9ad6d33d190 | 2 | 125 | 0 | 4.188 |
|
||||
|
||||
Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:
|
||||
|
||||
| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.45 / 1.56 | 5aa14958c864: 5 | 96.0 | 109.7 | 13.7 | 149.6 |
|
||||
| | | | | b9ad6d33d190: 5 | 94.5 | 108.9 | 14.4 | 142.1 |
|
||||
| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.04 / 1.31 | 5aa14958c864: 5 | 96.6 | 97.6 | 1.0 | 89.6 |
|
||||
| | | | | b9ad6d33d190: 5 | 96.2 | 98.6 | 2.3 | 88.9 |
|
||||
| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 4.96 / 5.2 | 5aa14958c864: 10 | 94.7 | 114.9 | 20.3 | 155.4 |
|
||||
| | | | | b9ad6d33d190: 10 | 97.0 | 121.8 | 24.8 | 176.9 |
|
||||
|
||||
Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB.
|
||||
|
||||
Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 243.885 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.406, 4.188 с (hook ждёт остановки ~4 с); потерь нет |
|
||||
| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 58.5 с до готовности; потерь нет |
|
||||
| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19922 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет |
|
||||
| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14860 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |
|
||||
| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает |
|
||||
| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно |
|
||||
| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT |
|
||||
| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет |
|
||||
| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action | без ручного действия |
|
||||
| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов |
|
||||
| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта |
|
||||
| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет |
|
||||
| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан |
|
||||
| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready |
|
||||
| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов |
|
||||
| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - |
|
||||
| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора |
|
||||
| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет |
|
||||
| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены |
|
||||
| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - |
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"date": "2026-10-24 15:40",
|
||||
"api_limit_mib": 512,
|
||||
"api_limit_cpus": 1.0,
|
||||
"runs": [
|
||||
{
|
||||
"scenario": "10 x ~9.5 MiB PNG (noise)",
|
||||
"concurrency": 10,
|
||||
"file_bytes": [
|
||||
9953047
|
||||
],
|
||||
"total_mib": 94.9,
|
||||
"all_202": true,
|
||||
"wall_seconds": 1.56,
|
||||
"latency_s": {
|
||||
"p50": 1.45,
|
||||
"max": 1.56
|
||||
},
|
||||
"sampler_rounds": 9,
|
||||
"replicas": [
|
||||
{
|
||||
"replica": "5aa14958c864",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 96.0,
|
||||
"rss_peak_mib": 109.7,
|
||||
"rss_growth_mib": 13.7,
|
||||
"rss_peak_sampled_mib": 109.7,
|
||||
"cgroup_peak_mib": 149.6,
|
||||
"cgroup_peak_sampled_mib": 145.2
|
||||
},
|
||||
{
|
||||
"replica": "b9ad6d33d190",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 94.5,
|
||||
"rss_peak_mib": 108.9,
|
||||
"rss_growth_mib": 14.4,
|
||||
"rss_peak_sampled_mib": 108.9,
|
||||
"cgroup_peak_mib": 142.1,
|
||||
"cgroup_peak_sampled_mib": 139.3
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)",
|
||||
"concurrency": 10,
|
||||
"file_bytes": [
|
||||
116714
|
||||
],
|
||||
"total_mib": 1.1,
|
||||
"all_202": true,
|
||||
"wall_seconds": 1.31,
|
||||
"latency_s": {
|
||||
"p50": 1.04,
|
||||
"max": 1.31
|
||||
},
|
||||
"sampler_rounds": 8,
|
||||
"replicas": [
|
||||
{
|
||||
"replica": "5aa14958c864",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 96.6,
|
||||
"rss_peak_mib": 97.6,
|
||||
"rss_growth_mib": 1.0,
|
||||
"rss_peak_sampled_mib": 97.6,
|
||||
"cgroup_peak_mib": 89.6,
|
||||
"cgroup_peak_sampled_mib": 89.2
|
||||
},
|
||||
{
|
||||
"replica": "b9ad6d33d190",
|
||||
"uploads_served": 5,
|
||||
"rss_before_mib": 96.2,
|
||||
"rss_peak_mib": 98.6,
|
||||
"rss_growth_mib": 2.3,
|
||||
"rss_peak_sampled_mib": 98.6,
|
||||
"cgroup_peak_mib": 88.9,
|
||||
"cgroup_peak_sampled_mib": 86.2
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"scenario": "20 mixed: large + 40 Mpx + JPEG",
|
||||
"concurrency": 20,
|
||||
"file_bytes": [
|
||||
1492,
|
||||
116714,
|
||||
9953047
|
||||
],
|
||||
"total_mib": 133.2,
|
||||
"all_202": true,
|
||||
"wall_seconds": 5.2,
|
||||
"latency_s": {
|
||||
"p50": 4.96,
|
||||
"max": 5.2
|
||||
},
|
||||
"sampler_rounds": 27,
|
||||
"replicas": [
|
||||
{
|
||||
"replica": "5aa14958c864",
|
||||
"uploads_served": 10,
|
||||
"rss_before_mib": 94.7,
|
||||
"rss_peak_mib": 114.9,
|
||||
"rss_growth_mib": 20.3,
|
||||
"rss_peak_sampled_mib": 114.9,
|
||||
"cgroup_peak_mib": 155.4,
|
||||
"cgroup_peak_sampled_mib": 154.4
|
||||
},
|
||||
{
|
||||
"replica": "b9ad6d33d190",
|
||||
"uploads_served": 10,
|
||||
"rss_before_mib": 97.0,
|
||||
"rss_peak_mib": 121.8,
|
||||
"rss_growth_mib": 24.8,
|
||||
"rss_peak_sampled_mib": 121.8,
|
||||
"cgroup_peak_mib": 176.9,
|
||||
"cgroup_peak_sampled_mib": 173.1
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"single_file_extra_rss_mib": {
|
||||
"naive": 161,
|
||||
"streaming": 0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas
|
||||
# date: 2026-10-24 15:40:56, commit: 276dbda + uncommitted changes
|
||||
# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json
|
||||
# exit code: 0, 53.8 s
|
||||
|
||||
client: Python 3.11.1 on Windows 10
|
||||
preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...
|
||||
large: 9953047 bytes; 40 Mpx: 116714 bytes
|
||||
{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_202": true, "wall_seconds": 1.56, "latency_s": {"p50": 1.45, "max": 1.56}, "sampler_rounds": 9, "replicas": [{"replica": "5aa14958c864", "uploads_served": 5, "rss_before_mib": 96.0, "rss_peak_mib": 109.7, "rss_growth_mib": 13.7, "rss_peak_sampled_mib": 109.7, "cgroup_peak_mib": 149.6, "cgroup_peak_sampled_mib": 145.2}, {"replica": "b9ad6d33d190", "uploads_served": 5, "rss_before_mib": 94.5, "rss_peak_mib": 108.9, "rss_growth_mib": 14.4, "rss_peak_sampled_mib": 108.9, "cgroup_peak_mib": 142.1, "cgroup_peak_sampled_mib": 139.3}]}
|
||||
{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_202": true, "wall_seconds": 1.31, "latency_s": {"p50": 1.04, "max": 1.31}, "sampler_rounds": 8, "replicas": [{"replica": "5aa14958c864", "uploads_served": 5, "rss_before_mib": 96.6, "rss_peak_mib": 97.6, "rss_growth_mib": 1.0, "rss_peak_sampled_mib": 97.6, "cgroup_peak_mib": 89.6, "cgroup_peak_sampled_mib": 89.2}, {"replica": "b9ad6d33d190", "uploads_served": 5, "rss_before_mib": 96.2, "rss_peak_mib": 98.6, "rss_growth_mib": 2.3, "rss_peak_sampled_mib": 98.6, "cgroup_peak_mib": 88.9, "cgroup_peak_sampled_mib": 86.2}]}
|
||||
{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_202": true, "wall_seconds": 5.2, "latency_s": {"p50": 4.96, "max": 5.2}, "sampler_rounds": 27, "replicas": [{"replica": "5aa14958c864", "uploads_served": 10, "rss_before_mib": 94.7, "rss_peak_mib": 114.9, "rss_growth_mib": 20.3, "rss_peak_sampled_mib": 114.9, "cgroup_peak_mib": 155.4, "cgroup_peak_sampled_mib": 154.4}, {"replica": "b9ad6d33d190", "uploads_served": 10, "rss_before_mib": 97.0, "rss_peak_mib": 121.8, "rss_growth_mib": 24.8, "rss_peak_sampled_mib": 121.8, "cgroup_peak_mib": 176.9, "cgroup_peak_sampled_mib": 173.1}]}
|
||||
one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB
|
||||
PASS: all uploads succeeded; largest RSS growth of one replica: 24.8 MiB (limit 512 MiB); results in evidence/upload-memory.json
|
||||
@@ -0,0 +1,51 @@
|
||||
"""Учебное внедрение сбоев для тестов отказов (только при FAULT_INJECTION=1).
|
||||
|
||||
Тест создаёт в общем volume /faults файлы <точка>.<n>, по одному на срабатывание:
|
||||
|
||||
docker compose exec -T worker sh -c 'touch /faults/worker_crash_after_put.1'
|
||||
|
||||
Процесс, дошедший до точки, атомарно удаляет один файл (os.remove успешен ровно у одного процесса)
|
||||
и выполняет сбой. Так сбой случается заданное число раз и не зацикливает перезапуски.
|
||||
|
||||
Точки:
|
||||
dispatcher_crash_before_publish — COMMIT задания уже был, publish ещё нет (окно COMMIT -> publish)
|
||||
dispatcher_crash_after_confirm — брокер подтвердил публикацию, отметка в outbox ещё не записана
|
||||
worker_crash_after_put — объекты записаны в S3, COMMIT результата ещё нет
|
||||
worker_crash_after_commit — COMMIT результата был, ack ещё нет
|
||||
worker_freeze — worker «завис»: не продлевает аренду и спит 3 x lease
|
||||
worker_s3_error — временная ошибка S3 при записи результата
|
||||
imaging_hang / imaging_invalid / imaging_oom — сбой внутри процесса обработки изображения
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import settings
|
||||
|
||||
log = logging.getLogger("messenger.faults")
|
||||
|
||||
|
||||
def hit(point: str) -> bool:
|
||||
if not settings.FAULT_INJECTION:
|
||||
return False
|
||||
directory = Path(settings.FAULTS_DIR)
|
||||
try:
|
||||
candidates = sorted(directory.glob(point + ".*"))
|
||||
except OSError:
|
||||
return False
|
||||
for path in candidates:
|
||||
try:
|
||||
path.unlink()
|
||||
except FileNotFoundError:
|
||||
continue # этот файл забрал другой процесс
|
||||
log.warning("FAULT INJECTION: %s", point)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def crash(point: str) -> None:
|
||||
"""Аварийное завершение всего процесса без очистки (как SIGKILL/OOM): без ack, без закрытия соединений."""
|
||||
if hit(point):
|
||||
sys.stderr.flush()
|
||||
os._exit(137)
|
||||
+209
@@ -0,0 +1,209 @@
|
||||
"""Преобразование изображения, pipeline_version=1. Запускается worker'ом ОТДЕЛЬНЫМ процессом с лимитами.
|
||||
|
||||
python imaging.py <source> <out_dir> <content_type> <max_pixels>
|
||||
|
||||
Почему отдельный процесс: декодер Pillow работает в C, его нельзя прервать изнутри Python и нельзя
|
||||
ограничить по памяти отдельно от остального worker. Процесс сам ставит себе RLIMIT_AS (память,
|
||||
IMAGING_MEMORY_MB), RLIMIT_CPU (процессорное время) и RLIMIT_FSIZE до импорта Pillow, а worker
|
||||
убивает его по таймауту JOB_TIMEOUT_SECONDS.
|
||||
Падение, зависание или нехватка памяти здесь не роняют worker и не ломают подключение к брокеру.
|
||||
|
||||
Результат: файлы thumbnail.jpg, crop.jpg, optimized.webp в out_dir и одна строка JSON в stdout.
|
||||
Код выхода: 0 — успех; 3 — невалидное изображение; 4 — слишком много пикселей; 5 — не хватило памяти;
|
||||
прочее (в том числе убит сигналом) — неожиданная ошибка, worker считает её временной.
|
||||
|
||||
Правила вариантов (contracts/README.md):
|
||||
- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, размеры округляются вниз
|
||||
(минимум 1 px), EXIF orientation применяется, прозрачность накладывается на белый фон. 320x200 -> 256x160.
|
||||
- crop: JPEG 128x128, центрированный квадрат (маленькие исходники увеличиваются).
|
||||
- optimized: WebP, вписан в 1280x1280 без увеличения, quality=80, method=4; прозрачность сохраняется.
|
||||
Во всех вариантах нет EXIF/XMP/комментариев: в save() передаётся только ICC-профиль (цвет), остальное
|
||||
отбрасывается. Pillow при одинаковой версии даёт побайтно одинаковый результат, поэтому повтор job
|
||||
пишет в те же ключи S3 те же байты.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import resource
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
def apply_limits() -> None:
|
||||
"""Лимиты на себя, ДО импорта Pillow: память (RLIMIT_AS), CPU, размер записываемых файлов."""
|
||||
memory = int(os.environ.get("IMAGING_MEMORY_MB", "640")) * 1024 * 1024
|
||||
cpu = int(os.environ.get("IMAGING_CPU_SECONDS", "60"))
|
||||
resource.setrlimit(resource.RLIMIT_AS, (memory, memory))
|
||||
resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5))
|
||||
resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
apply_limits()
|
||||
|
||||
from PIL import Image # noqa: E402 — после apply_limits: память декодера тоже под лимитом
|
||||
|
||||
PIPELINE_VERSION = 1
|
||||
THUMB_BOX = 256
|
||||
CROP_SIZE = 128
|
||||
OPTIMIZED_BOX = 1280
|
||||
THUMB_QUALITY = 85
|
||||
CROP_QUALITY = 85
|
||||
WEBP_QUALITY = 80
|
||||
WEBP_METHOD = 4
|
||||
|
||||
EXIT_INVALID = 3
|
||||
EXIT_TOO_LARGE = 4
|
||||
EXIT_MEMORY = 5
|
||||
|
||||
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
||||
# EXIF orientation -> преобразование (как в PIL.ImageOps.exif_transpose)
|
||||
ORIENTATION = {
|
||||
2: [Image.Transpose.FLIP_LEFT_RIGHT],
|
||||
3: [Image.Transpose.ROTATE_180],
|
||||
4: [Image.Transpose.FLIP_TOP_BOTTOM],
|
||||
5: [Image.Transpose.TRANSPOSE],
|
||||
6: [Image.Transpose.ROTATE_270],
|
||||
7: [Image.Transpose.TRANSVERSE],
|
||||
8: [Image.Transpose.ROTATE_90],
|
||||
}
|
||||
|
||||
|
||||
class TooLarge(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fit(width: int, height: int, box: int) -> tuple[int, int]:
|
||||
"""Вписать в box x box без увеличения; размеры вниз, минимум 1 px. Целочисленно, без float."""
|
||||
if width <= box and height <= box:
|
||||
return width, height
|
||||
if width >= height:
|
||||
return box, max(1, height * box // width)
|
||||
return max(1, width * box // height), box
|
||||
|
||||
|
||||
def orientation_of(image: Image.Image) -> int:
|
||||
try:
|
||||
value = image.getexif().get(0x0112, 1)
|
||||
except Exception: # noqa: BLE001 — битый EXIF не делает картинку невалидной, ориентацию не трогаем
|
||||
return 1
|
||||
return value if value in ORIENTATION else 1
|
||||
|
||||
|
||||
def normalize(image: Image.Image) -> Image.Image:
|
||||
"""Любой режим PNG/JPEG -> L, RGB или RGBA (RGBA только если есть прозрачность)."""
|
||||
mode = image.mode
|
||||
transparent = "transparency" in image.info
|
||||
if mode in ("I;16", "I;16B", "I;16L", "I"):
|
||||
image = image.convert("I").point(lambda value: value * (1 / 256)).convert("L")
|
||||
mode = "L"
|
||||
if mode == "1":
|
||||
return image.convert("L")
|
||||
if mode in ("P", "PA"):
|
||||
return image.convert("RGBA" if transparent or mode == "PA" else "RGB")
|
||||
if mode in ("LA", "La", "RGBa"):
|
||||
return image.convert("RGBA")
|
||||
if mode in ("L", "RGB") and transparent:
|
||||
return image.convert("RGBA")
|
||||
if mode in ("L", "RGB", "RGBA"):
|
||||
return image
|
||||
return image.convert("RGB") # CMYK, YCbCr, LAB, ...
|
||||
|
||||
|
||||
def on_white(image: Image.Image) -> Image.Image:
|
||||
"""Прозрачность -> белый фон (для JPEG)."""
|
||||
if image.mode != "RGBA":
|
||||
return image
|
||||
background = Image.new("RGB", image.size, (255, 255, 255))
|
||||
background.paste(image, mask=image.getchannel("A"))
|
||||
return background
|
||||
|
||||
|
||||
def process(source: str, out_dir: str, content_type: str, max_pixels: int) -> dict:
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
with Image.open(source) as image:
|
||||
if image.format != FORMATS.get(content_type):
|
||||
raise ValueError("format does not match stored content type")
|
||||
width, height = image.size
|
||||
if width * height > max_pixels:
|
||||
raise TooLarge()
|
||||
orientation = orientation_of(image)
|
||||
icc = image.info.get("icc_profile")
|
||||
swap = orientation in (5, 6, 7, 8)
|
||||
oriented = (height, width) if swap else (width, height)
|
||||
|
||||
# Сначала уменьшаем до optimized (<= 1280), потом поворачиваем: квадратные рамки 1280/256/128
|
||||
# не зависят от поворота, а поворот маленькой картинки дешевле, чем копия 40-мегапиксельной.
|
||||
target = fit(*oriented, OPTIMIZED_BOX)
|
||||
stored_target = (target[1], target[0]) if swap else target
|
||||
if image.format == "JPEG":
|
||||
image.draft(None, stored_target) # декодирование сразу в масштабе 1/2..1/8, если хватает
|
||||
image.load()
|
||||
work = normalize(image)
|
||||
if work.size != stored_target:
|
||||
work = work.resize(stored_target, Image.Resampling.LANCZOS, reducing_gap=3.0)
|
||||
elif work is image:
|
||||
work = work.copy()
|
||||
for step in ORIENTATION.get(orientation, []):
|
||||
work = work.transpose(step)
|
||||
|
||||
extra = {"icc_profile": icc} if icc else {}
|
||||
results = []
|
||||
|
||||
optimized = work if work.mode != "L" else work.convert("RGB")
|
||||
path = os.path.join(out_dir, "optimized.webp")
|
||||
optimized.save(path, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, **extra)
|
||||
results.append({"name": "optimized", "content_type": "image/webp", "width": optimized.width,
|
||||
"height": optimized.height, "file": "optimized.webp"})
|
||||
|
||||
thumb_size = fit(*oriented, THUMB_BOX) # от размеров ИСХОДНИКА, а не промежуточной копии
|
||||
thumb = work if work.size == thumb_size else work.resize(thumb_size, Image.Resampling.LANCZOS)
|
||||
thumb = on_white(thumb)
|
||||
path = os.path.join(out_dir, "thumbnail.jpg")
|
||||
thumb.save(path, format="JPEG", quality=THUMB_QUALITY, optimize=True, **extra)
|
||||
results.append({"name": "thumbnail", "content_type": "image/jpeg", "width": thumb.width,
|
||||
"height": thumb.height, "file": "thumbnail.jpg"})
|
||||
|
||||
side = min(work.size)
|
||||
left, top = (work.width - side) // 2, (work.height - side) // 2
|
||||
crop = work.crop((left, top, left + side, top + side))
|
||||
crop = on_white(crop.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS))
|
||||
path = os.path.join(out_dir, "crop.jpg")
|
||||
crop.save(path, format="JPEG", quality=CROP_QUALITY, optimize=True, **extra)
|
||||
results.append({"name": "crop", "content_type": "image/jpeg", "width": CROP_SIZE, "height": CROP_SIZE,
|
||||
"file": "crop.jpg"})
|
||||
|
||||
return {"pipeline_version": PIPELINE_VERSION, "source": {"width": width, "height": height,
|
||||
"orientation": orientation}, "variants": results}
|
||||
|
||||
|
||||
def injected_fault() -> None:
|
||||
"""Учебные сбои (FAULT_IMAGING задаёт worker, только при FAULT_INJECTION=1)."""
|
||||
fault = os.environ.get("FAULT_IMAGING", "")
|
||||
if fault == "hang":
|
||||
time.sleep(3600)
|
||||
elif fault == "invalid":
|
||||
raise ValueError("injected invalid image")
|
||||
elif fault == "oom":
|
||||
blocks = []
|
||||
while True:
|
||||
blocks.append(bytearray(64 * 1024 * 1024))
|
||||
|
||||
|
||||
def main() -> int:
|
||||
source, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4])
|
||||
try:
|
||||
injected_fault()
|
||||
result = process(source, out_dir, content_type, max_pixels)
|
||||
except (TooLarge, Image.DecompressionBombError):
|
||||
return EXIT_TOO_LARGE
|
||||
except MemoryError:
|
||||
return EXIT_MEMORY
|
||||
except (OSError, SyntaxError, ValueError, IndexError, TypeError, EOFError, Image.UnidentifiedImageError):
|
||||
# Pillow сообщает о битом файле разными исключениями; подробности не нужны ни клиенту, ни логу.
|
||||
return EXIT_INVALID
|
||||
print(json.dumps(result), flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,77 @@
|
||||
# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики).
|
||||
# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить
|
||||
# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky).
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
# При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с.
|
||||
hard-stop-after 10s
|
||||
|
||||
# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик.
|
||||
# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска.
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s # перечитывать список реплик раз в 2 с
|
||||
hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с
|
||||
hold nx 2s
|
||||
hold other 2s
|
||||
hold refused 2s
|
||||
hold timeout 2s
|
||||
accepted_payload_size 8192
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlog-normal # в лог только ошибки, а не каждый запрос
|
||||
timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос
|
||||
timeout client 30s
|
||||
# Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка
|
||||
# при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт,
|
||||
# сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT).
|
||||
timeout server 30s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 5s
|
||||
timeout check 1s
|
||||
# Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода.
|
||||
# Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key.
|
||||
retries 2
|
||||
option redispatch 1
|
||||
|
||||
frontend api_in
|
||||
bind :8080
|
||||
option forwardfor
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
# Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis.
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
|
||||
http-check expect status 200
|
||||
# До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT.
|
||||
# Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP.
|
||||
# observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки.
|
||||
server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none
|
||||
# Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis
|
||||
# или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате
|
||||
# контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает.
|
||||
# Пока готова хоть одна реплика, эти записи не используются.
|
||||
# allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend).
|
||||
server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none
|
||||
option allbackups
|
||||
|
||||
# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1).
|
||||
# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда.
|
||||
frontend stats
|
||||
bind :8404
|
||||
monitor-uri /lb-health
|
||||
monitor fail if { nbsrv(api) lt 1 }
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
@@ -0,0 +1,22 @@
|
||||
# RabbitMQ 4.1 (сервис rabbitmq). Топология, пользователи и права — в secrets/rabbitmq_definitions.json
|
||||
# (генерирует scripts/init_secrets.py), файл импортируется при каждом старте узла. Пользователя guest нет:
|
||||
# при импорте определений на старте пользователь по умолчанию не создаётся.
|
||||
definitions.import_backend = local_filesystem
|
||||
definitions.local.path = /run/secrets/rabbitmq_definitions
|
||||
definitions.skip_if_unchanged = true
|
||||
|
||||
listeners.tcp.default = 5672
|
||||
loopback_users = none
|
||||
|
||||
# Ресурсы: при нехватке памяти/диска брокер блокирует издателей (dispatcher ждёт и повторяет),
|
||||
# сообщения не теряются. Абсолютное значение: ниже лимита контейнера 512M (доля считалась бы от RAM хоста).
|
||||
vm_memory_high_watermark.absolute = 320MB
|
||||
disk_free_limit.absolute = 256MB
|
||||
|
||||
# Consumer, не приславший ack за это время, считается зависшим: канал закрывается, сообщение
|
||||
# возвращается в очередь. Должно быть больше JOB_TIMEOUT_SECONDS + времени записи в S3.
|
||||
consumer_timeout = 300000
|
||||
|
||||
log.console = true
|
||||
log.console.level = info
|
||||
log.file = false
|
||||
@@ -0,0 +1,13 @@
|
||||
bind 0.0.0.0
|
||||
port 6379
|
||||
protected-mode yes
|
||||
dir /data
|
||||
|
||||
appendonly yes
|
||||
appendfsync always
|
||||
save ""
|
||||
|
||||
maxmemory 64mb
|
||||
maxmemory-policy noeviction
|
||||
|
||||
include /run/secrets/redis_auth
|
||||
+304
@@ -0,0 +1,304 @@
|
||||
"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor).
|
||||
|
||||
python janitor.py --once [--dry-run] [--grace-seconds N]
|
||||
|
||||
Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock):
|
||||
|
||||
1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен
|
||||
в очередь. Удалить объект staging (идемпотентно), затем строку.
|
||||
2. Staging после безопасной фиксации:
|
||||
- ready: исходник уже опубликован в attachments/<id>/v<N>/original -> удалить staging-объект
|
||||
(обычно это делает worker сразу после ack; здесь — если он не успел);
|
||||
- failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется.
|
||||
queued/processing не трогаются никогда: это текущая работа.
|
||||
3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются.
|
||||
4. Обход bucket по всем prefix (объекты старше grace):
|
||||
- staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key;
|
||||
- attachments/<id>/...: сирота, только если вложение в КОНЕЧНОМ состоянии и ключ ему не нужен:
|
||||
вложения нет; оно failed и replay уже невозможен (source_key IS NULL — исходник удалён по retention);
|
||||
оно ready, но ключ не входит в опубликованный набор (оригинал + варианты). У queued/processing
|
||||
и у failed с исходником в staging объекты не трогаются: replay/следующая попытка перезапишет их
|
||||
теми же ключами, и удаление устаревшим проходом сломало бы новый результат.
|
||||
Повторная проверка и DELETE выполняются в одной транзакции под FOR SHARE на строке вложения:
|
||||
replay (FOR UPDATE) и захват worker'ом (UPDATE) ждут конца DELETE, а не идут параллельно с ним.
|
||||
Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных.
|
||||
|
||||
Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому
|
||||
janitor не удаляет то, что ещё пишется.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import UUID
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import pipeline
|
||||
import settings
|
||||
import storage
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
log = logging.getLogger("messenger.janitor")
|
||||
|
||||
JANITOR_LOCK = 20260404
|
||||
BATCH = 100
|
||||
STOP = threading.Event()
|
||||
ACTOR = "janitor"
|
||||
FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/")
|
||||
|
||||
|
||||
def touch_alive() -> None:
|
||||
try:
|
||||
Path(settings.ALIVE_FILE).touch()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def idle(seconds: float) -> bool:
|
||||
"""Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while (left := deadline - time.monotonic()) > 0:
|
||||
touch_alive()
|
||||
if STOP.wait(min(left, 10)):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def connect():
|
||||
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||
options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC")
|
||||
|
||||
|
||||
def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]:
|
||||
"""Шаг 1. Возвращает (найдено, удалено)."""
|
||||
found = deleted = 0
|
||||
while not STOP.is_set():
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""SELECT id, source_key FROM attachments
|
||||
WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s)
|
||||
ORDER BY created_at LIMIT %s
|
||||
FOR UPDATE SKIP LOCKED""",
|
||||
(grace, BATCH),
|
||||
).fetchall()
|
||||
found += len(rows)
|
||||
if dry_run or not rows:
|
||||
return found, deleted
|
||||
for row in rows:
|
||||
storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится
|
||||
conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],))
|
||||
deleted += 1
|
||||
log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"])
|
||||
if len(rows) < BATCH:
|
||||
return found, deleted
|
||||
return found, deleted
|
||||
|
||||
|
||||
def cleanup_staging(conn, dry_run: bool) -> dict:
|
||||
"""Шаг 2. Исходники ready (сразу) и failed (после retention)."""
|
||||
report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0}
|
||||
while not STOP.is_set():
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a
|
||||
LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version
|
||||
WHERE a.source_key IS NOT NULL
|
||||
AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds')
|
||||
OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s)))
|
||||
ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""",
|
||||
(settings.STAGING_RETENTION_SECONDS, BATCH),
|
||||
).fetchall()
|
||||
if dry_run or not rows:
|
||||
if dry_run:
|
||||
report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows)
|
||||
report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows)
|
||||
break
|
||||
for row in rows:
|
||||
storage.delete(row["source_key"])
|
||||
conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],))
|
||||
if row["state"] == "ready":
|
||||
report["staging_ready_deleted"] += 1
|
||||
else:
|
||||
report["staging_failed_expired"] += 1
|
||||
if row["job_id"] is not None:
|
||||
pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR,
|
||||
"replay no longer possible")
|
||||
if len(rows) < BATCH:
|
||||
break
|
||||
report["staging_kept_for_replay"] = conn.execute(
|
||||
"SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"]
|
||||
return report
|
||||
|
||||
|
||||
def cleanup_outbox(conn, dry_run: bool) -> int:
|
||||
if dry_run:
|
||||
return conn.execute(
|
||||
"""SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
|
||||
(settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"]
|
||||
return conn.execute(
|
||||
"""DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""",
|
||||
(settings.OUTBOX_RETENTION_SECONDS,)).rowcount
|
||||
|
||||
|
||||
def classify_final(conn, keys: list[str]) -> set[str]:
|
||||
"""Какие ключи attachments/<id>/... не принадлежат ни одному живому вложению (кандидаты в сироты)."""
|
||||
ids = {}
|
||||
for key in keys:
|
||||
match = FINAL_KEY_RE.match(key)
|
||||
if match:
|
||||
try:
|
||||
ids[key] = UUID(match.group(1))
|
||||
except ValueError:
|
||||
pass
|
||||
unique = list(set(ids.values()))
|
||||
rows = {r["id"]: r for r in conn.execute(
|
||||
"""SELECT a.id, a.state, a.object_key, a.source_key,
|
||||
ARRAY(SELECT v.object_key FROM attachment_variants v
|
||||
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
|
||||
FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()}
|
||||
orphans = set()
|
||||
for key in keys:
|
||||
attachment_id = ids.get(key)
|
||||
row = rows.get(attachment_id) if attachment_id else None
|
||||
if row is None:
|
||||
orphans.add(key)
|
||||
elif row["state"] == "failed" and row["source_key"] is None:
|
||||
# failed без исходника — конечное состояние: replay отказывает («source file is no longer in
|
||||
# staging»), других переходов из failed нет, поэтому никто больше не запишет эти ключи.
|
||||
orphans.add(key)
|
||||
elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]:
|
||||
orphans.add(key)
|
||||
return orphans
|
||||
|
||||
|
||||
def sweep_objects(conn, grace: int, dry_run: bool) -> dict:
|
||||
"""Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000."""
|
||||
started = datetime.now(timezone.utc)
|
||||
cutoff = started - timedelta(seconds=grace)
|
||||
scanned = orphans = orphans_deleted = 0
|
||||
listed = set()
|
||||
|
||||
def orphan_keys(prefix, items):
|
||||
keys = [key for key, _ in items]
|
||||
if prefix == storage.FINAL_PREFIX:
|
||||
return classify_final(conn, keys)
|
||||
known = {r["source_key"] for r in conn.execute(
|
||||
"SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()}
|
||||
return {key for key in keys if key not in known}
|
||||
|
||||
def delete_if_orphan(prefix, key) -> bool:
|
||||
"""Повторная проверка прямо перед удалением и DELETE — атомарно относительно replay/worker."""
|
||||
if prefix != storage.FINAL_PREFIX:
|
||||
# staging/originals: строка-указатель создаётся ДО PUT, ключ случайный и не переиспользуется,
|
||||
# поэтому «нет строки сейчас» означает «не будет никогда».
|
||||
if conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is not None:
|
||||
return False
|
||||
storage.delete(key)
|
||||
return True
|
||||
match = FINAL_KEY_RE.match(key)
|
||||
try:
|
||||
attachment_id = UUID(match.group(1)) if match else None
|
||||
except ValueError:
|
||||
attachment_id = None
|
||||
with conn.transaction():
|
||||
if attachment_id is not None:
|
||||
# FOR SHARE совместим с FOR KEY SHARE (вставка сообщения со ссылкой на вложение), но
|
||||
# конфликтует с FOR UPDATE replay и с UPDATE worker'а: состояние не изменится до COMMIT.
|
||||
conn.execute("SELECT 1 FROM attachments WHERE id = %s FOR SHARE", (attachment_id,))
|
||||
if key not in classify_final(conn, [key]):
|
||||
return False
|
||||
storage.delete(key) # S3 недоступен -> исключение, транзакция откатится, повтор на след. проходе
|
||||
return True
|
||||
|
||||
def check(prefix, items):
|
||||
nonlocal orphans, orphans_deleted
|
||||
candidates = orphan_keys(prefix, items)
|
||||
for key, modified in items:
|
||||
if key not in candidates or modified >= cutoff:
|
||||
continue
|
||||
orphans += 1
|
||||
if not dry_run and delete_if_orphan(prefix, key):
|
||||
orphans_deleted += 1
|
||||
log.info("deleted orphan object %s", key)
|
||||
|
||||
for prefix in storage.ALL_PREFIXES:
|
||||
page = []
|
||||
for key, modified, _size in storage.list_objects(prefix):
|
||||
scanned += 1
|
||||
listed.add(key)
|
||||
page.append((key, modified))
|
||||
if len(page) >= 1000:
|
||||
check(prefix, page)
|
||||
page = []
|
||||
if page:
|
||||
check(prefix, page)
|
||||
|
||||
missing = []
|
||||
for r in conn.execute(
|
||||
"""SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v
|
||||
WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys
|
||||
FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall():
|
||||
if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]):
|
||||
missing.append(str(r["id"]))
|
||||
if missing:
|
||||
log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s",
|
||||
", ".join(missing[:20]))
|
||||
return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted,
|
||||
"missing_objects": len(missing)}
|
||||
|
||||
|
||||
def run_once(grace: int, dry_run: bool) -> dict | None:
|
||||
started = time.monotonic()
|
||||
with connect() as conn:
|
||||
if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]:
|
||||
log.info("another janitor is running, skip")
|
||||
return None
|
||||
try:
|
||||
found, deleted = reclaim_stale_intents(conn, grace, dry_run)
|
||||
report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found,
|
||||
"stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run),
|
||||
"outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)}
|
||||
finally:
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,))
|
||||
report["seconds"] = round(time.monotonic() - started, 3)
|
||||
return report
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects")
|
||||
parser.add_argument("--once", action="store_true", help="one pass and exit")
|
||||
parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing")
|
||||
parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS)
|
||||
args = parser.parse_args()
|
||||
if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS:
|
||||
parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} "
|
||||
"(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted")
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
signal.signal(sig, lambda *_: STOP.set())
|
||||
while True:
|
||||
touch_alive()
|
||||
try:
|
||||
report = run_once(args.grace_seconds, args.dry_run)
|
||||
if report is not None:
|
||||
print(json.dumps(report), flush=True)
|
||||
failed = False
|
||||
except (psycopg.Error, storage.StorageUnavailable) as exc:
|
||||
log.warning("pass failed, will retry: %s", type(exc).__name__)
|
||||
failed = True
|
||||
if args.once:
|
||||
raise SystemExit(1 if failed else 0)
|
||||
if idle(settings.JANITOR_INTERVAL_SECONDS):
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,603 @@
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI, Header, Query, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, StreamingResponse
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
import attachments
|
||||
import auth
|
||||
import db
|
||||
import settings
|
||||
import storage
|
||||
import uploads
|
||||
from errors import ApiError, bad_request, conflict, not_found, unauthorized, unavailable
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
log = logging.getLogger("messenger")
|
||||
|
||||
DRAINING = threading.Event()
|
||||
|
||||
|
||||
class Server(uvicorn.Server):
|
||||
"""Остановка по SIGTERM в два шага.
|
||||
|
||||
1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS
|
||||
HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы.
|
||||
2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы
|
||||
(не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM,
|
||||
получал бы 502 от балансировщика.
|
||||
Повторный сигнал останавливает сразу.
|
||||
"""
|
||||
|
||||
def handle_exit(self, sig, frame):
|
||||
if DRAINING.is_set():
|
||||
return super().handle_exit(sig, frame)
|
||||
DRAINING.set()
|
||||
log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s",
|
||||
sig, settings.SHUTDOWN_DRAIN_SECONDS)
|
||||
threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
settings.check()
|
||||
# Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему.
|
||||
await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS)
|
||||
# Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3.
|
||||
Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True)
|
||||
db.pool.open()
|
||||
log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s, "
|
||||
"processing via outbox (pipeline v%s)", settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV,
|
||||
settings.SESSION_TTL_SECONDS, settings.S3_BUCKET, settings.PIPELINE_VERSION)
|
||||
yield
|
||||
db.pool.close()
|
||||
auth.close()
|
||||
log.info("Stopped: connections closed, data stays in PostgreSQL and Redis")
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
|
||||
|
||||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH)
|
||||
|
||||
@field_validator("display_name", "password")
|
||||
@classmethod
|
||||
def validate_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("value cannot be blank")
|
||||
return value
|
||||
|
||||
|
||||
class User(BaseModel):
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class Session(BaseModel):
|
||||
token: str
|
||||
token_type: str
|
||||
expires_at: datetime
|
||||
user: User
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
@classmethod
|
||||
def validate_title(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError("title cannot be empty")
|
||||
return value
|
||||
|
||||
|
||||
class Chat(BaseModel):
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: datetime
|
||||
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
items: list[Chat]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
text: str = Field(max_length=4000)
|
||||
# С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением.
|
||||
attachment_ids: list[UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_content(self):
|
||||
if len(set(self.attachment_ids)) != len(self.attachment_ids):
|
||||
raise ValueError("attachment_ids must be unique")
|
||||
if not self.attachment_ids and not self.text.strip():
|
||||
raise ValueError("text cannot be empty without attachments")
|
||||
return self
|
||||
|
||||
|
||||
class Message(BaseModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[UUID]
|
||||
|
||||
|
||||
class Variant(BaseModel):
|
||||
name: Literal["thumbnail", "crop", "optimized"]
|
||||
content_type: Literal["image/jpeg", "image/png", "image/webp"]
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
|
||||
class Attachment(BaseModel):
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
owner_id: UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: Literal["queued", "processing", "ready", "failed"]
|
||||
variants: list[Variant]
|
||||
error_code: str | None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
items: list[Message]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
def error_response(
|
||||
request: Request,
|
||||
code: str,
|
||||
message: str,
|
||||
status_code: int,
|
||||
headers: dict | None = None,
|
||||
):
|
||||
request_id = request.state.request_id
|
||||
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content={
|
||||
"error": {
|
||||
"code": code,
|
||||
"message": message,
|
||||
"request_id": request_id,
|
||||
}
|
||||
},
|
||||
headers={"X-Request-Id": request_id, **(headers or {})},
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
return error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_exception_handler(
|
||||
request: Request,
|
||||
exc: RequestValidationError,
|
||||
):
|
||||
return error_response(
|
||||
request,
|
||||
"invalid_request",
|
||||
"Invalid request data",
|
||||
400,
|
||||
)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
|
||||
if exc.status_code == 404:
|
||||
return error_response(request, "not_found", "Not found", 404)
|
||||
return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def general_exception_handler(
|
||||
request: Request,
|
||||
exc: Exception,
|
||||
):
|
||||
return error_response(
|
||||
request,
|
||||
"internal_error",
|
||||
"Internal server error",
|
||||
500,
|
||||
)
|
||||
|
||||
PUBLIC_ENDPOINTS = {
|
||||
("POST", "/api/v1/users"),
|
||||
("POST", "/api/v1/auth/sessions"),
|
||||
}
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def request_context(request: Request, call_next):
|
||||
request_id = str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
started = time.perf_counter()
|
||||
path = request.url.path
|
||||
|
||||
try:
|
||||
if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS:
|
||||
request.state.session = await run_in_threadpool(
|
||||
auth.check_bearer, request.headers.get("Authorization")
|
||||
)
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers)
|
||||
except Exception:
|
||||
log.exception("Unhandled error, request_id=%s", request_id)
|
||||
response = error_response(request, "internal_error", "Internal server error", 500)
|
||||
|
||||
response.headers["X-Request-Id"] = request_id
|
||||
response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила
|
||||
# Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки.
|
||||
level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO
|
||||
log.log(
|
||||
level,
|
||||
"%s %s -> %s (%.1f ms) request_id=%s",
|
||||
request.method, path, response.status_code,
|
||||
(time.perf_counter() - started) * 1000, request_id,
|
||||
)
|
||||
return response
|
||||
|
||||
|
||||
def current_user_id(request: Request) -> UUID:
|
||||
session = getattr(request.state, "session", None)
|
||||
if session is None:
|
||||
raise unauthorized("Bearer token is required")
|
||||
return session.user_id
|
||||
|
||||
|
||||
def encode_cursor(
|
||||
created_at: datetime,
|
||||
object_id: UUID,
|
||||
*,
|
||||
chat_id: UUID | None = None,
|
||||
user_id: UUID | None = None,
|
||||
) -> str:
|
||||
data = {
|
||||
"created_at": created_at.isoformat(),
|
||||
"id": str(object_id),
|
||||
}
|
||||
|
||||
if chat_id is not None:
|
||||
data["chat_id"] = str(chat_id)
|
||||
|
||||
if user_id is not None:
|
||||
data["user_id"] = str(user_id)
|
||||
|
||||
raw = json.dumps(data).encode("utf-8")
|
||||
return base64.urlsafe_b64encode(raw).decode("ascii")
|
||||
|
||||
|
||||
def decode_cursor(cursor: str):
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor.encode("ascii"))
|
||||
data = json.loads(raw.decode("utf-8"))
|
||||
|
||||
created_at = datetime.fromisoformat(data["created_at"])
|
||||
|
||||
if created_at.tzinfo is None:
|
||||
raise ValueError("Invalid cursor")
|
||||
|
||||
if not isinstance(data["id"], str):
|
||||
raise ValueError("Invalid cursor")
|
||||
|
||||
object_id = UUID(data["id"])
|
||||
|
||||
chat_id = UUID(data["chat_id"]) if "chat_id" in data else None
|
||||
user_id = UUID(data["user_id"]) if "user_id" in data else None
|
||||
|
||||
return created_at, object_id, chat_id, user_id
|
||||
|
||||
except (
|
||||
ValueError,
|
||||
KeyError,
|
||||
TypeError,
|
||||
AttributeError,
|
||||
json.JSONDecodeError,
|
||||
binascii.Error,
|
||||
):
|
||||
raise ValueError("Invalid cursor")
|
||||
|
||||
|
||||
def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None):
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor)
|
||||
except ValueError:
|
||||
raise bad_request("Invalid cursor") from None
|
||||
if cursor_chat_id != chat_id or cursor_user_id != user_id:
|
||||
raise bad_request("Invalid cursor")
|
||||
return created_at, object_id
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
def health_live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
def health_ready():
|
||||
if DRAINING.is_set():
|
||||
raise unavailable("Instance is shutting down")
|
||||
db.ping()
|
||||
auth.ping_redis()
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", response_model=User, status_code=201)
|
||||
def create_user(user_data: CreateUser):
|
||||
password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется
|
||||
return db.create_user(user_data.username, user_data.display_name, password_hash)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201)
|
||||
def create_session(
|
||||
response: Response,
|
||||
authorization: str | None = Header(default=None),
|
||||
):
|
||||
credentials = auth.parse_basic(authorization)
|
||||
if credentials is None:
|
||||
raise auth.login_failed()
|
||||
username, password = credentials
|
||||
|
||||
found = None
|
||||
if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH:
|
||||
found = db.get_user_with_password(username)
|
||||
if not auth.password_matches(found["password_hash"] if found else None, password):
|
||||
raise auth.login_failed()
|
||||
|
||||
token, expires_at = auth.create_session(found["id"])
|
||||
response.headers["Cache-Control"] = "no-store"
|
||||
user = {key: found[key] for key in ("id", "username", "display_name", "created_at")}
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
def delete_session(request: Request):
|
||||
current_user_id(request)
|
||||
auth.delete_session(request.state.session)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me", response_model=User)
|
||||
def get_current_user(request: Request):
|
||||
user = db.get_user(current_user_id(request))
|
||||
if user is None:
|
||||
raise unauthorized("Session is invalid or expired")
|
||||
return user
|
||||
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", response_model=Chat, status_code=201)
|
||||
def create_chat(request: Request, chat_data: CreateChat):
|
||||
user_id = current_user_id(request)
|
||||
|
||||
if user_id not in chat_data.member_ids:
|
||||
raise bad_request("Creator must be a chat member")
|
||||
|
||||
if len(chat_data.member_ids) != len(set(chat_data.member_ids)):
|
||||
raise bad_request("Duplicate member IDs")
|
||||
return db.create_chat(chat_data.title, chat_data.member_ids)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats", response_model=ChatPage)
|
||||
def list_chats(
|
||||
request: Request,
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None),
|
||||
):
|
||||
user_id = current_user_id(request)
|
||||
after = page_after(cursor, user_id=user_id)
|
||||
|
||||
rows = db.list_chats(user_id, after, limit)
|
||||
items = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = items[-1]
|
||||
next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id)
|
||||
|
||||
return ChatPage(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
|
||||
def get_chat(request: Request, chat_id: UUID):
|
||||
return db.get_chat(chat_id, current_user_id(request))
|
||||
|
||||
|
||||
def request_fingerprint(data: BaseModel) -> bytes:
|
||||
"""SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют.
|
||||
|
||||
Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []}
|
||||
равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними.
|
||||
"""
|
||||
body = data.model_dump(mode="json")
|
||||
if not body.get("attachment_ids"):
|
||||
body.pop("attachment_ids", None)
|
||||
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
response_model=Message,
|
||||
status_code=201,
|
||||
)
|
||||
def create_message(
|
||||
request: Request,
|
||||
message_data: CreateMessage,
|
||||
chat_id: UUID,
|
||||
idempotency_key: str | None = Header(default=None),
|
||||
):
|
||||
# С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику)
|
||||
# вернёт то же сообщение, а не создаст второе. См. db.add_message.
|
||||
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||||
raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -")
|
||||
return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids,
|
||||
idempotency_key, request_fingerprint(message_data))
|
||||
|
||||
|
||||
@app.get(
|
||||
"/api/v1/chats/{chat_id}/messages",
|
||||
response_model=MessagePage,
|
||||
)
|
||||
def list_messages(
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None),
|
||||
):
|
||||
user_id = current_user_id(request)
|
||||
after = page_after(cursor, chat_id=chat_id)
|
||||
|
||||
rows = db.list_messages(chat_id, user_id, after, limit)
|
||||
items = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if len(rows) > limit:
|
||||
last = items[-1]
|
||||
next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id)
|
||||
|
||||
return MessagePage(items=items, next_cursor=next_cursor)
|
||||
|
||||
|
||||
@app.post(
|
||||
"/api/v1/chats/{chat_id}/attachments",
|
||||
response_model=Attachment,
|
||||
status_code=202,
|
||||
)
|
||||
async def upload_attachment(
|
||||
request: Request,
|
||||
chat_id: UUID,
|
||||
filename: str | None = Query(default=None),
|
||||
):
|
||||
"""Приём PNG/JPEG сырыми байтами. 202 queued — когда исходник в staging (S3), а задание и сообщение
|
||||
для брокера зафиксированы в PostgreSQL одной транзакцией (attachments.store). Обработку делает worker.
|
||||
|
||||
Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415)
|
||||
-> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413),
|
||||
проверка содержимого (415/413) и запись.
|
||||
"""
|
||||
user_id = current_user_id(request)
|
||||
try:
|
||||
await run_in_threadpool(db.check_member, chat_id, user_id)
|
||||
name = uploads.validate_filename(filename)
|
||||
declared = uploads.declared_type(request.headers.get("content-type"))
|
||||
uploads.declared_length(request.headers.get("content-length"))
|
||||
except ApiError:
|
||||
await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке
|
||||
raise
|
||||
upload = await uploads.receive(request)
|
||||
try:
|
||||
image = await run_in_threadpool(uploads.inspect, upload, declared)
|
||||
row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image)
|
||||
finally:
|
||||
upload.file.close() # временный файл удаляется сразу; исходник остаётся только в staging (S3)
|
||||
return attachments.public(row)
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
|
||||
def get_attachment(request: Request, attachment_id: UUID):
|
||||
return attachments.public(db.get_attachment(attachment_id, current_user_id(request)))
|
||||
|
||||
|
||||
VARIANT_SUFFIX = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp"}
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
def download_attachment(
|
||||
request: Request,
|
||||
attachment_id: UUID,
|
||||
variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(default="original"),
|
||||
):
|
||||
"""Байты оригинала или варианта через API (без redirect): сначала участник чата, потом S3.
|
||||
|
||||
queued/processing/failed -> 409 (ещё/уже нечего отдавать); варианта нет у ready -> 404.
|
||||
"""
|
||||
row = db.get_attachment(attachment_id, current_user_id(request))
|
||||
if row["state"] != "ready":
|
||||
raise conflict("Attachment is not ready")
|
||||
if variant == "original":
|
||||
key, content_type, size, sha256 = row["object_key"], row["content_type"], row["size_bytes"], row["sha256"]
|
||||
name = row["filename"]
|
||||
else:
|
||||
found = [v for v in row["variants"] if v["name"] == variant]
|
||||
if not found:
|
||||
raise not_found("Variant not found")
|
||||
v = found[0]
|
||||
key, content_type, size, sha256 = v["object_key"], v["content_type"], v["size_bytes"], v["sha256"]
|
||||
stem = row["filename"].rsplit(".", 1)[0] or "image"
|
||||
name = f"{stem}.{variant}{VARIANT_SUFFIX[content_type]}"
|
||||
try:
|
||||
stream = storage.open_read(key)
|
||||
except storage.ObjectMissing:
|
||||
log.error("Object %s of ready attachment %s is missing in S3", variant, row["id"])
|
||||
raise unavailable("File is temporarily unavailable") from None
|
||||
except storage.StorageUnavailable:
|
||||
raise unavailable("File storage is temporarily unavailable") from None
|
||||
headers = {
|
||||
"Content-Length": str(size),
|
||||
"Content-Disposition": "inline; filename*=UTF-8''" + quote(name, safe=""),
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
"Cache-Control": "private, no-store",
|
||||
"ETag": '"' + sha256 + '"',
|
||||
}
|
||||
return StreamingResponse(storage.iter_chunks(stream), media_type=content_type, headers=headers)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
Server(uvicorn.Config(
|
||||
app,
|
||||
host=settings.APP_HOST,
|
||||
port=settings.APP_PORT,
|
||||
timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS,
|
||||
access_log=False,
|
||||
)).run()
|
||||
@@ -0,0 +1,8 @@
|
||||
import logging
|
||||
|
||||
import db
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s")
|
||||
|
||||
if __name__ == "__main__":
|
||||
db.wait_and_migrate()
|
||||
@@ -0,0 +1,33 @@
|
||||
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE
|
||||
CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_by_user ON chat_members (user_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE users DROP CONSTRAINT users_display_name_check;
|
||||
ALTER TABLE chats DROP CONSTRAINT chats_title_check;
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
|
||||
ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8');
|
||||
ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8');
|
||||
ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8');
|
||||
|
||||
ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400);
|
||||
ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400);
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000);
|
||||
@@ -0,0 +1,11 @@
|
||||
ALTER TABLE messages ADD COLUMN idempotency_key text
|
||||
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$');
|
||||
|
||||
ALTER TABLE messages ADD COLUMN request_hash bytea
|
||||
CHECK (octet_length(request_hash) = 32);
|
||||
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together
|
||||
CHECK ((idempotency_key IS NULL) = (request_hash IS NULL));
|
||||
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key
|
||||
UNIQUE (chat_id, sender_id, idempotency_key);
|
||||
@@ -0,0 +1,48 @@
|
||||
-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением.
|
||||
-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать.
|
||||
|
||||
-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата.
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL,
|
||||
owner_id uuid NOT NULL,
|
||||
filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width integer NOT NULL CHECK (width >= 1),
|
||||
height integer NOT NULL CHECK (height >= 1),
|
||||
-- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит.
|
||||
object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'),
|
||||
-- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна;
|
||||
-- ready: объект в S3 сохранён, можно читать и прикреплять.
|
||||
state text NOT NULL CHECK (state IN ('uploading', 'ready')),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
CHECK (width::bigint * height <= 40000000),
|
||||
UNIQUE (id, chat_id),
|
||||
-- владелец на момент загрузки был участником именно этого чата
|
||||
FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
|
||||
-- janitor ищет старые незавершённые загрузки
|
||||
CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading';
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, position),
|
||||
UNIQUE (message_id, attachment_id),
|
||||
-- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id),
|
||||
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id);
|
||||
|
||||
-- С lab4 текст может быть пустым, если есть вложение (проверяет API).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000);
|
||||
@@ -0,0 +1,133 @@
|
||||
-- lab5: асинхронная обработка вложений. Задания, outbox, варианты, журнал событий.
|
||||
--
|
||||
-- Состояния attachments.state:
|
||||
-- uploading — строка-намерение до записи в staging (как в lab4), клиенту не видна;
|
||||
-- queued / processing / ready / failed — публичные состояния из контракта.
|
||||
-- source_key — исходник в staging (staging/<hex>; у старых вложений lab4 — originals/<hex>);
|
||||
-- object_key — опубликованный оригинал attachments/<id>/v<N>/original, есть только у ready.
|
||||
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_state_check;
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_check;
|
||||
ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL;
|
||||
ALTER TABLE attachments ADD COLUMN source_key text UNIQUE;
|
||||
ALTER TABLE attachments ADD COLUMN error_code text;
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer;
|
||||
|
||||
-- Вложения lab4 были приняты синхронно и не имеют вариантов: переносим их в staging-модель и ставим
|
||||
-- в очередь (ниже), worker опубликует оригинал в конечный prefix и сделает варианты.
|
||||
UPDATE attachments SET source_key = object_key, object_key = NULL;
|
||||
UPDATE attachments SET state = 'queued', updated_at = now() WHERE state = 'ready';
|
||||
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_state_check
|
||||
CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed'));
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_source_key_check
|
||||
CHECK (source_key ~ '^(staging|originals)/[a-f0-9]{32}$');
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_check
|
||||
CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/original$');
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_error_code_check
|
||||
CHECK (error_code ~ '^[a-z_]{1,64}$');
|
||||
-- ready <=> опубликованный оригинал; failed <=> код ошибки; до ready исходник обязан быть в staging.
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_ready_has_object
|
||||
CHECK ((state = 'ready') = (object_key IS NOT NULL));
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_failed_has_code
|
||||
CHECK ((state = 'failed') = (error_code IS NOT NULL));
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_pending_has_source
|
||||
CHECK (state NOT IN ('uploading', 'queued', 'processing') OR source_key IS NOT NULL);
|
||||
|
||||
-- janitor: исходники ready/failed, которые пора удалить из staging
|
||||
CREATE INDEX attachments_staging ON attachments (updated_at)
|
||||
WHERE source_key IS NOT NULL AND state IN ('ready', 'failed');
|
||||
|
||||
-- Варианты. Первичный ключ = ключ эффекта (attachment_id, pipeline_version, variant):
|
||||
-- повтор job не может создать второй набор вариантов.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes >= 1),
|
||||
width integer NOT NULL CHECK (width >= 1),
|
||||
height integer NOT NULL CHECK (height >= 1),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE
|
||||
CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/[a-z]+\.(jpg|png|webp)$'),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name)
|
||||
);
|
||||
|
||||
-- Задание: одна логическая работа на (вложение, версия pipeline). job_id не меняется при повторах.
|
||||
-- attempt — управляемая попытка (claim worker'ом); lease_* — аренда processing с fencing-токеном.
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL CHECK (pipeline_version >= 1),
|
||||
status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')),
|
||||
attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1),
|
||||
max_attempts integer NOT NULL CHECK (max_attempts >= 1),
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
lease_token uuid,
|
||||
lease_owner text,
|
||||
lease_until timestamptz,
|
||||
error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'),
|
||||
last_error text CHECK (last_error ~ '^[a-z_]{1,64}$'),
|
||||
replays integer NOT NULL DEFAULT 0,
|
||||
traceparent text CHECK (traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
finished_at timestamptz,
|
||||
UNIQUE (attachment_id, pipeline_version),
|
||||
CHECK (attempt <= max_attempts),
|
||||
CHECK ((status = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)),
|
||||
CHECK ((status = 'failed') = (error_code IS NOT NULL))
|
||||
);
|
||||
CREATE INDEX attachment_jobs_leases ON attachment_jobs (lease_until) WHERE status = 'processing';
|
||||
CREATE INDEX attachment_jobs_queued ON attachment_jobs (available_at) WHERE status = 'queued';
|
||||
CREATE INDEX attachment_jobs_failed ON attachment_jobs (finished_at) WHERE status = 'failed';
|
||||
|
||||
-- Журнал (audit): claim, повторы, истечение аренды, дубли, replay. Только коды, без секретов и путей.
|
||||
CREATE TABLE job_events (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id),
|
||||
attempt integer NOT NULL,
|
||||
event text NOT NULL CHECK (event ~ '^[a-z_]{1,64}$'),
|
||||
detail text CHECK (char_length(detail) <= 200),
|
||||
actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 200),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX job_events_by_job ON job_events (job_id, id);
|
||||
|
||||
-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и задание.
|
||||
-- dispatcher публикует строки с available_at <= now() и отмечает published_at после publisher confirm.
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id),
|
||||
attempt integer NOT NULL,
|
||||
payload jsonb NOT NULL,
|
||||
available_at timestamptz NOT NULL DEFAULT now(),
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
published_at timestamptz,
|
||||
publish_failures integer NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
CREATE INDEX outbox_by_job ON outbox (job_id);
|
||||
|
||||
-- Задания для вложений, принятых в lab4 (миграция = их producer). JOB_MAX_ATTEMPTS по умолчанию 5.
|
||||
INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, traceparent)
|
||||
SELECT gen_random_uuid(), id, 1, 'queued', 1, 5,
|
||||
'00-' || md5(random()::text || id::text) || '-' || substr(md5(id::text), 1, 16) || '-01'
|
||||
FROM attachments WHERE state = 'queued';
|
||||
|
||||
INSERT INTO job_events (job_id, attempt, event, detail, actor)
|
||||
SELECT job_id, 1, 'created', 'lab4_backfill', 'migration 005' FROM attachment_jobs;
|
||||
|
||||
INSERT INTO outbox (job_id, attempt, payload)
|
||||
SELECT job_id, 1, jsonb_build_object(
|
||||
'schema_version', 1,
|
||||
'job_id', job_id,
|
||||
'type', 'attachment.process.v1',
|
||||
'attachment_id', attachment_id,
|
||||
'pipeline_version', pipeline_version,
|
||||
'attempt', 1,
|
||||
'created_at', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'),
|
||||
'traceparent', traceparent)
|
||||
FROM attachment_jobs;
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher).
|
||||
|
||||
docker compose exec dispatcher python ops.py status
|
||||
docker compose exec dispatcher python ops.py failed [--limit N]
|
||||
docker compose exec dispatcher python ops.py events <job_id>
|
||||
docker compose exec dispatcher python ops.py replay <job_id> [--reason TEXT]
|
||||
docker compose exec dispatcher python ops.py dlq [--limit N]
|
||||
docker compose exec dispatcher python ops.py dlq-replay [--limit N]
|
||||
|
||||
Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code,
|
||||
число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя
|
||||
обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit.
|
||||
|
||||
replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready,
|
||||
queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка
|
||||
идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant).
|
||||
|
||||
dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины):
|
||||
failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу
|
||||
ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения
|
||||
остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
|
||||
import broker
|
||||
import pipeline
|
||||
import settings
|
||||
|
||||
ACTOR = "ops@" + settings.INSTANCE_ID
|
||||
|
||||
|
||||
def connect_db():
|
||||
return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row,
|
||||
connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS,
|
||||
options="-c TimeZone=UTC")
|
||||
|
||||
|
||||
def text(value):
|
||||
if isinstance(value, (UUID,)):
|
||||
return str(value)
|
||||
if hasattr(value, "isoformat"):
|
||||
return pipeline.utc_text(value)
|
||||
return value
|
||||
|
||||
|
||||
def rows_json(rows):
|
||||
return [{key: text(value) for key, value in row.items()} for row in rows]
|
||||
|
||||
|
||||
def cmd_status(conn, _args):
|
||||
jobs = {r["status"]: r["n"] for r in conn.execute(
|
||||
"SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()}
|
||||
outbox = conn.execute(
|
||||
"""SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending,
|
||||
count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due,
|
||||
coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures,
|
||||
extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL
|
||||
AND available_at <= now())) AS oldest_due_s
|
||||
FROM outbox""").fetchone()
|
||||
leases = conn.execute(
|
||||
"""SELECT count(*) FILTER (WHERE lease_until >= now()) AS active,
|
||||
count(*) FILTER (WHERE lease_until < now()) AS expired
|
||||
FROM attachment_jobs WHERE status = 'processing'""").fetchone()
|
||||
staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"]
|
||||
report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v)
|
||||
for k, v in outbox.items()},
|
||||
"leases": leases, "attachments_with_staging_source": staging}
|
||||
try:
|
||||
connection = broker.connect()
|
||||
try:
|
||||
report["broker"] = broker.check_topology(connection.channel())
|
||||
finally:
|
||||
connection.close()
|
||||
except broker.BrokerError as exc:
|
||||
report["broker"] = {"error": type(exc).__name__}
|
||||
return report
|
||||
|
||||
|
||||
def cmd_failed(conn, args):
|
||||
return rows_json(conn.execute(
|
||||
"""SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code,
|
||||
j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable
|
||||
FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id
|
||||
WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall())
|
||||
|
||||
|
||||
def cmd_events(conn, args):
|
||||
return rows_json(conn.execute(
|
||||
"SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id",
|
||||
(args.job_id,)).fetchall())
|
||||
|
||||
|
||||
def cmd_replay(conn, args):
|
||||
try:
|
||||
return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)}
|
||||
except pipeline.ReplayRefused as exc:
|
||||
return {"refused": str(exc), "job_id": str(args.job_id)}
|
||||
|
||||
|
||||
def dlq_messages(limit, handler):
|
||||
"""Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'."""
|
||||
connection = broker.connect()
|
||||
results = []
|
||||
try:
|
||||
channel = connection.channel()
|
||||
keep = []
|
||||
for _ in range(limit):
|
||||
method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False)
|
||||
if method is None:
|
||||
break
|
||||
death = (properties.headers or {}).get("x-death") or [{}]
|
||||
reason = death[0].get("reason")
|
||||
reason = reason.decode() if isinstance(reason, bytes) else reason
|
||||
try:
|
||||
message = pipeline.parse_message(body)
|
||||
except pipeline.InvalidMessage as exc:
|
||||
message, problem = None, str(exc)
|
||||
else:
|
||||
problem = None
|
||||
verdict, note = handler(message)
|
||||
results.append({"job_id": message["job_id"] if message else None, "invalid": problem,
|
||||
"dead_letter_reason": reason, "action": verdict, "note": note})
|
||||
if verdict == "ack":
|
||||
channel.basic_ack(method.delivery_tag)
|
||||
else:
|
||||
keep.append(method.delivery_tag)
|
||||
for tag in keep: # вернуть непрочитанными в DLQ
|
||||
channel.basic_nack(tag, requeue=True)
|
||||
finally:
|
||||
connection.close()
|
||||
return results
|
||||
|
||||
|
||||
def cmd_dlq(_conn, args):
|
||||
return dlq_messages(args.limit, lambda message: ("keep", "peek"))
|
||||
|
||||
|
||||
def cmd_dlq_replay(conn, args):
|
||||
def handle(message):
|
||||
if message is None:
|
||||
return "keep", "invalid message is kept for analysis"
|
||||
job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s",
|
||||
(UUID(message["job_id"]),)).fetchone()
|
||||
if job is None:
|
||||
return "keep", "unknown job"
|
||||
if job["status"] == "failed":
|
||||
try:
|
||||
pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay")
|
||||
return "ack", "replayed"
|
||||
except pipeline.ReplayRefused as exc:
|
||||
return "keep", str(exc)
|
||||
if job["status"] == "ready":
|
||||
return "ack", "already ready, no-op"
|
||||
# queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново
|
||||
conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s "
|
||||
"AND status = 'queued'", (UUID(message["job_id"]),))
|
||||
return "ack", f"job is {job['status']}, handled by outbox/lease recovery"
|
||||
return dlq_messages(args.limit, handle)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Attachment pipeline operator commands")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
sub.add_parser("status")
|
||||
p = sub.add_parser("failed")
|
||||
p.add_argument("--limit", type=int, default=50)
|
||||
p = sub.add_parser("events")
|
||||
p.add_argument("job_id", type=UUID)
|
||||
p = sub.add_parser("replay")
|
||||
p.add_argument("job_id", type=UUID)
|
||||
p.add_argument("--reason", default="operator replay")
|
||||
p = sub.add_parser("dlq")
|
||||
p.add_argument("--limit", type=int, default=20)
|
||||
p = sub.add_parser("dlq-replay")
|
||||
p.add_argument("--limit", type=int, default=100)
|
||||
args = parser.parse_args()
|
||||
if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()):
|
||||
parser.error("--reason: up to 100 printable characters")
|
||||
commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay,
|
||||
"dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay}
|
||||
with connect_db() as conn:
|
||||
result = commands[args.command](conn, args)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+445
@@ -0,0 +1,445 @@
|
||||
"""Задания обработки вложений: формат сообщения, outbox, переходы состояний.
|
||||
|
||||
Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат)
|
||||
и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции
|
||||
вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT.
|
||||
|
||||
Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать,
|
||||
решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение
|
||||
безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op.
|
||||
|
||||
Переходы (contracts/attachment-states.md):
|
||||
(нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202
|
||||
queued -> processing claim worker, аренда lease_until + fencing-токен lease_token
|
||||
processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter)
|
||||
recover_lease аренда истекла (worker умер/завис), попытка засчитана
|
||||
processing -> ready finish_ready только при совпадении lease_token (fencing)
|
||||
processing -> failed finish_failed постоянная ошибка или попытки исчерпаны
|
||||
failed -> queued replay явная операция оператора (ops.py), с записью в журнал
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import secrets
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from psycopg.types.json import Jsonb
|
||||
|
||||
import settings
|
||||
|
||||
SCHEMA_VERSION = 1
|
||||
JOB_TYPE = "attachment.process.v1"
|
||||
EXCHANGE = "attachments"
|
||||
ROUTING_KEY = "attachment.process.v1"
|
||||
QUEUE = "attachments.process.v1"
|
||||
DLX = "attachments.dlx"
|
||||
DLQ = "attachments.process.v1.dlq"
|
||||
DLQ_ROUTING_KEY = "attachment.process.v1.dlq"
|
||||
|
||||
MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt",
|
||||
"created_at", "traceparent", "tracestate"}
|
||||
REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"}
|
||||
TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$")
|
||||
UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||
CODE_RE = re.compile(r"^[a-z_]{1,64}$")
|
||||
|
||||
# Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES.
|
||||
PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"}
|
||||
FINAL_CODES = PERMANENT | {"retry_exhausted"}
|
||||
|
||||
|
||||
class InvalidMessage(Exception):
|
||||
"""Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ."""
|
||||
|
||||
|
||||
def utc_text(moment: datetime) -> str:
|
||||
return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
|
||||
def new_traceparent() -> str:
|
||||
"""W3C trace context: trace-id на всё задание, span-id новый для каждой публикации."""
|
||||
return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01"
|
||||
|
||||
|
||||
def child_traceparent(parent: str | None) -> str:
|
||||
if parent and TRACEPARENT_RE.match(parent):
|
||||
return f"00-{parent[3:35]}-{secrets.token_hex(8)}-01"
|
||||
return new_traceparent()
|
||||
|
||||
|
||||
def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
|
||||
traceparent: str | None) -> dict:
|
||||
"""Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3."""
|
||||
message = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"job_id": str(job_id),
|
||||
"type": JOB_TYPE,
|
||||
"attachment_id": str(attachment_id),
|
||||
"pipeline_version": pipeline_version,
|
||||
"attempt": attempt,
|
||||
"created_at": utc_text(datetime.now(timezone.utc)),
|
||||
}
|
||||
if traceparent:
|
||||
message["traceparent"] = child_traceparent(traceparent)
|
||||
return message
|
||||
|
||||
|
||||
def parse_message(body: bytes) -> dict:
|
||||
"""Строгая проверка по схеме (additionalProperties: false, const, типы, форматы)."""
|
||||
try:
|
||||
message = json.loads(body.decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise InvalidMessage("not json") from None
|
||||
if not isinstance(message, dict):
|
||||
raise InvalidMessage("not an object")
|
||||
if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message):
|
||||
raise InvalidMessage("fields")
|
||||
if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE:
|
||||
raise InvalidMessage("schema_version/type")
|
||||
for name in ("job_id", "attachment_id"):
|
||||
if not isinstance(message[name], str) or not UUID_RE.match(message[name]):
|
||||
raise InvalidMessage(name)
|
||||
for name in ("pipeline_version", "attempt"):
|
||||
value = message[name]
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||
raise InvalidMessage(name)
|
||||
try:
|
||||
created = message["created_at"]
|
||||
if not isinstance(created, str):
|
||||
raise ValueError
|
||||
datetime.fromisoformat(created.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
raise InvalidMessage("created_at") from None
|
||||
if "traceparent" in message and (not isinstance(message["traceparent"], str)
|
||||
or not TRACEPARENT_RE.match(message["traceparent"])):
|
||||
raise InvalidMessage("traceparent")
|
||||
if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512):
|
||||
raise InvalidMessage("tracestate")
|
||||
return message
|
||||
|
||||
|
||||
def backoff_seconds(failed_attempt: int) -> float:
|
||||
"""Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна.
|
||||
|
||||
Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS.
|
||||
Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу).
|
||||
"""
|
||||
ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS,
|
||||
settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1))
|
||||
return ceiling / 2 + random.uniform(0, ceiling / 2)
|
||||
|
||||
|
||||
def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None:
|
||||
conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)",
|
||||
(job_id, attempt, name, detail[:200] if detail else None, actor[:200]))
|
||||
|
||||
|
||||
def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int,
|
||||
traceparent: str | None, delay_seconds: float = 0) -> None:
|
||||
"""Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT."""
|
||||
payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent)
|
||||
conn.execute(
|
||||
"""INSERT INTO outbox (job_id, attempt, payload, available_at)
|
||||
VALUES (%s, %s, %s, now() + make_interval(secs => %s))""",
|
||||
(job_id, attempt, Jsonb(payload), delay_seconds),
|
||||
)
|
||||
conn.execute("NOTIFY outbox")
|
||||
|
||||
|
||||
def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID:
|
||||
"""Новое задание для вложения, уже переведённого в queued (в транзакции API)."""
|
||||
job_id = uuid4()
|
||||
traceparent = new_traceparent()
|
||||
conn.execute(
|
||||
"""INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts,
|
||||
traceparent)
|
||||
VALUES (%s, %s, %s, 'queued', 1, %s, %s)""",
|
||||
(job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent),
|
||||
)
|
||||
event(conn, job_id, 1, "created", actor)
|
||||
add_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent)
|
||||
return job_id
|
||||
|
||||
|
||||
# ---------- worker: захват и результат ----------
|
||||
|
||||
@dataclass
|
||||
class Claim:
|
||||
job_id: UUID
|
||||
attachment_id: UUID
|
||||
pipeline_version: int
|
||||
attempt: int
|
||||
max_attempts: int
|
||||
lease_token: UUID
|
||||
source_key: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
traceparent: str | None
|
||||
|
||||
|
||||
@dataclass
|
||||
class Skip:
|
||||
"""Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker)."""
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Reject:
|
||||
"""Сообщение не относится ни к одной job -> DLQ."""
|
||||
reason: str
|
||||
|
||||
|
||||
def claim(conn, message: dict, owner: str, redelivered: bool):
|
||||
"""queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject."""
|
||||
job_id = UUID(message["job_id"])
|
||||
with conn.transaction():
|
||||
job = conn.execute(
|
||||
"""SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired
|
||||
FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
if job is None:
|
||||
return Reject("unknown_job")
|
||||
if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]:
|
||||
return Reject("job_mismatch")
|
||||
status = job["status"]
|
||||
if status in ("ready", "failed"):
|
||||
event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}")
|
||||
return Skip(f"already_{status}")
|
||||
attempt = job["attempt"]
|
||||
if status == "queued":
|
||||
if not job["due"]:
|
||||
return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox
|
||||
else: # processing
|
||||
if not job["lease_expired"]:
|
||||
event(conn, job_id, attempt, "duplicate_ignored", owner, "leased")
|
||||
return Skip("leased")
|
||||
# Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана.
|
||||
event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker")
|
||||
if attempt >= job["max_attempts"]:
|
||||
_fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired")
|
||||
return Skip("exhausted")
|
||||
attempt += 1
|
||||
token = uuid4()
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs
|
||||
SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s,
|
||||
lease_until = now() + make_interval(secs => %s), updated_at = now()
|
||||
WHERE job_id = %s""",
|
||||
(attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id),
|
||||
)
|
||||
att = conn.execute(
|
||||
"""UPDATE attachments SET state = 'processing', updated_at = now()
|
||||
WHERE id = %s AND state IN ('queued', 'processing')
|
||||
RETURNING source_key, content_type, size_bytes, sha256""",
|
||||
(job["attachment_id"],),
|
||||
).fetchone()
|
||||
if att is None or att["source_key"] is None:
|
||||
raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат
|
||||
event(conn, job_id, attempt, "claimed", owner,
|
||||
f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}")
|
||||
return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token,
|
||||
att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"])
|
||||
|
||||
|
||||
def renew_lease(conn, c: Claim) -> bool:
|
||||
"""Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить."""
|
||||
row = conn.execute(
|
||||
"""UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now()
|
||||
WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""",
|
||||
(settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _fenced(conn, c: Claim, owner: str) -> bool:
|
||||
"""Fencing: результат принимается, только если lease_token всё ещё наш."""
|
||||
row = conn.execute(
|
||||
"""SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing'
|
||||
FOR UPDATE""",
|
||||
(c.job_id, c.lease_token),
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> bool:
|
||||
"""processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT."""
|
||||
with conn.transaction():
|
||||
if not _fenced(conn, c, owner):
|
||||
stale = True
|
||||
else:
|
||||
stale = False
|
||||
for v in variants:
|
||||
conn.execute(
|
||||
"""INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type,
|
||||
size_bytes, width, height, sha256, object_key)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""",
|
||||
(c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"],
|
||||
v["width"], v["height"], v["sha256"], v["object_key"]),
|
||||
)
|
||||
# Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна).
|
||||
updated = conn.execute(
|
||||
"""UPDATE attachments
|
||||
SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now()
|
||||
WHERE id = %s AND state = 'processing'
|
||||
AND (pipeline_version IS NULL OR pipeline_version <= %s)
|
||||
RETURNING id""",
|
||||
(original_key, c.pipeline_version, c.attachment_id, c.pipeline_version),
|
||||
).fetchone()
|
||||
if updated is None:
|
||||
raise RuntimeError("attachment changed state during processing")
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs
|
||||
SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL,
|
||||
error_code = NULL, finished_at = now(), updated_at = now()
|
||||
WHERE job_id = %s""",
|
||||
(c.job_id,),
|
||||
)
|
||||
event(conn, c.job_id, c.attempt, "ready", owner)
|
||||
if stale:
|
||||
with conn.transaction():
|
||||
event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost")
|
||||
return not stale
|
||||
|
||||
|
||||
def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None):
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs
|
||||
SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL,
|
||||
lease_until = NULL, finished_at = now(), updated_at = now()
|
||||
WHERE job_id = %s""",
|
||||
(code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id),
|
||||
)
|
||||
conn.execute(
|
||||
"""UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now()
|
||||
WHERE id = %s AND state IN ('queued', 'processing')""",
|
||||
(code, attachment_id),
|
||||
)
|
||||
event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}")
|
||||
|
||||
|
||||
def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int,
|
||||
traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None:
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL,
|
||||
lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now()
|
||||
WHERE job_id = %s""",
|
||||
(next_attempt, reason, delay, job_id),
|
||||
)
|
||||
conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'",
|
||||
(attachment_id,))
|
||||
add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay)
|
||||
event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s")
|
||||
|
||||
|
||||
def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str:
|
||||
"""Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'."""
|
||||
with conn.transaction():
|
||||
if not _fenced(conn, c, owner):
|
||||
outcome = "stale"
|
||||
elif permanent:
|
||||
_fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None)
|
||||
outcome = "failed"
|
||||
elif c.attempt >= c.max_attempts:
|
||||
_fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code)
|
||||
outcome = "failed"
|
||||
else:
|
||||
_requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent,
|
||||
backoff_seconds(c.attempt), code, owner, "retry_scheduled")
|
||||
outcome = "retry"
|
||||
if outcome == "stale":
|
||||
with conn.transaction():
|
||||
event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code)
|
||||
return outcome
|
||||
|
||||
|
||||
# ---------- dispatcher: аренды и сверка ----------
|
||||
|
||||
def recover_expired_leases(conn, actor: str, limit: int = 100) -> int:
|
||||
"""processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted)."""
|
||||
recovered = 0
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent
|
||||
FROM attachment_jobs WHERE status = 'processing' AND lease_until < now()
|
||||
ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""",
|
||||
(limit,),
|
||||
).fetchall()
|
||||
for job in rows:
|
||||
if job["attempt"] >= job["max_attempts"]:
|
||||
_fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor,
|
||||
"lease_expired")
|
||||
else:
|
||||
_requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1,
|
||||
job["traceparent"], backoff_seconds(job["attempt"]), "lease_expired", actor,
|
||||
"lease_expired")
|
||||
recovered += 1
|
||||
return recovered
|
||||
|
||||
|
||||
def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int:
|
||||
"""Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет.
|
||||
|
||||
Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его
|
||||
подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет.
|
||||
"""
|
||||
with conn.transaction():
|
||||
rows = conn.execute(
|
||||
"""SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent
|
||||
FROM attachment_jobs j
|
||||
WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s)
|
||||
AND j.updated_at < now() - make_interval(secs => %s)
|
||||
AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND
|
||||
(o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s)))
|
||||
ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""",
|
||||
(older_than, older_than, older_than, limit),
|
||||
).fetchall()
|
||||
for job in rows:
|
||||
add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"],
|
||||
job["traceparent"])
|
||||
conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],))
|
||||
event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight")
|
||||
return len(rows)
|
||||
|
||||
|
||||
# ---------- оператор: replay ----------
|
||||
|
||||
class ReplayRefused(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def replay(conn, job_id: UUID, actor: str, reason: str) -> dict:
|
||||
"""failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются."""
|
||||
with conn.transaction():
|
||||
job = conn.execute(
|
||||
"""SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j
|
||||
JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""",
|
||||
(job_id,),
|
||||
).fetchone()
|
||||
if job is None:
|
||||
raise ReplayRefused("job not found")
|
||||
if job["status"] != "failed":
|
||||
# «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается.
|
||||
raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs")
|
||||
if job["source_key"] is None:
|
||||
raise ReplayRefused("source file is no longer in staging (retention expired)")
|
||||
attempt = job["attempt"] + 1
|
||||
conn.execute(
|
||||
"""UPDATE attachment_jobs
|
||||
SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL,
|
||||
available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now()
|
||||
WHERE job_id = %s""",
|
||||
(attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id),
|
||||
)
|
||||
conn.execute(
|
||||
"UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'",
|
||||
(job["attachment_id"],),
|
||||
)
|
||||
event(conn, job_id, attempt, "replayed", actor, reason or "operator replay")
|
||||
add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, job["traceparent"])
|
||||
return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt}
|
||||
@@ -0,0 +1,30 @@
|
||||
fastapi==0.141.1
|
||||
uvicorn==0.54.0
|
||||
pydantic==2.13.5
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
argon2-cffi==25.1.0
|
||||
redis==6.4.0
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi-bindings==26.1.0
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
pycparser==3.1
|
||||
pydantic_core==2.46.5
|
||||
starlette==1.7.0
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
boto3==1.43.116
|
||||
botocore==1.43.116
|
||||
s3transfer==0.19.3
|
||||
jmespath==1.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
six==1.17.0
|
||||
urllib3==2.8.0
|
||||
pillow==12.3.0
|
||||
pika==1.3.2
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init.
|
||||
|
||||
Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает.
|
||||
Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py):
|
||||
admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init;
|
||||
messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor.
|
||||
Значения ключей не печатаются.
|
||||
"""
|
||||
import logging
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
import settings
|
||||
|
||||
logging.basicConfig(level=logging.WARNING)
|
||||
|
||||
|
||||
def client(key_id: str, secret: str):
|
||||
return boto3.client(
|
||||
"s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION,
|
||||
aws_access_key_id=key_id, aws_secret_access_key=secret,
|
||||
config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2,
|
||||
read_timeout=10, retries={"total_max_attempts": 1},
|
||||
request_checksum_calculation="when_required", response_checksum_validation="when_required"),
|
||||
)
|
||||
|
||||
|
||||
def code(exc: ClientError) -> str:
|
||||
return exc.response.get("Error", {}).get("Code", "?")
|
||||
|
||||
|
||||
def anonymous_status(url: str, method: str = "GET") -> int:
|
||||
try:
|
||||
with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply:
|
||||
return reply.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
|
||||
|
||||
def main() -> int:
|
||||
bucket = settings.S3_BUCKET
|
||||
admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"),
|
||||
settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY"))
|
||||
api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY)
|
||||
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
admin.list_buckets()
|
||||
break
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
if time.monotonic() > deadline:
|
||||
print(f"s3-init: S3 is not ready: {type(exc).__name__}")
|
||||
return 1
|
||||
time.sleep(1)
|
||||
|
||||
try:
|
||||
admin.create_bucket(Bucket=bucket)
|
||||
print(f"s3-init: bucket {bucket} created")
|
||||
except ClientError as exc:
|
||||
if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"):
|
||||
print(f"s3-init: cannot create bucket: {code(exc)}")
|
||||
return 1
|
||||
print(f"s3-init: bucket {bucket} exists")
|
||||
|
||||
# Ключ API работает со своим bucket ...
|
||||
probe = "originals/s3-init-probe-" + uuid.uuid4().hex
|
||||
api.put_object(Bucket=bucket, Key=probe, Body=b"probe")
|
||||
api.get_object(Bucket=bucket, Key=probe)["Body"].read()
|
||||
api.list_objects_v2(Bucket=bucket, MaxKeys=1)
|
||||
api.delete_object(Bucket=bucket, Key=probe)
|
||||
print("s3-init: api key can put/get/list/delete objects of its bucket")
|
||||
# ... и не может администрировать хранилище.
|
||||
intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8]
|
||||
try:
|
||||
api.create_bucket(Bucket=intruder)
|
||||
admin.delete_bucket(Bucket=intruder)
|
||||
print("s3-init: api key must NOT be able to create buckets")
|
||||
return 1
|
||||
except ClientError as exc:
|
||||
print(f"s3-init: api key cannot create buckets ({code(exc)})")
|
||||
|
||||
# Приватность: анонимные запросы без подписи.
|
||||
base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket
|
||||
statuses = {"list": anonymous_status(base + "?list-type=2"),
|
||||
"get": anonymous_status(base + "/originals/any"),
|
||||
"put": anonymous_status(base + "/originals/any", "PUT")}
|
||||
if any(status not in (401, 403) for status in statuses.values()):
|
||||
print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}")
|
||||
return 1
|
||||
print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,153 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SECRETS = ROOT / "secrets"
|
||||
|
||||
|
||||
def write(name: str, value: str) -> None:
|
||||
with open(SECRETS / name, "w", encoding="utf-8", newline="") as file:
|
||||
file.write(value)
|
||||
(SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore
|
||||
|
||||
|
||||
def bucket_name() -> str:
|
||||
"""S3_BUCKET из .env (если задан), иначе значение по умолчанию."""
|
||||
env = ROOT / ".env"
|
||||
if env.exists():
|
||||
for line in env.read_text(encoding="utf-8").splitlines():
|
||||
if line.strip().startswith("S3_BUCKET="):
|
||||
return line.split("=", 1)[1].strip()
|
||||
return "messenger-attachments"
|
||||
|
||||
|
||||
def s3_config(bucket: str) -> str:
|
||||
"""Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket.
|
||||
|
||||
Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects)
|
||||
с суффиксом :<bucket> действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет.
|
||||
Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются.
|
||||
"""
|
||||
def read(name):
|
||||
return (SECRETS / name).read_text(encoding="utf-8").strip()
|
||||
config = {"identities": [
|
||||
{"name": "admin",
|
||||
"credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}],
|
||||
"actions": ["Admin", "Read", "Write", "List", "Tagging"]},
|
||||
{"name": "messenger-api",
|
||||
"credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}],
|
||||
"actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]},
|
||||
]}
|
||||
return json.dumps(config, indent=2)
|
||||
|
||||
|
||||
BROKER_VHOST = "messenger"
|
||||
# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей).
|
||||
EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1"
|
||||
DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq"
|
||||
|
||||
|
||||
def rabbit_hash(password: str) -> str:
|
||||
"""Формат rabbit_password_hashing_sha256: base64(salt(4 байта) + SHA-256(salt + пароль))."""
|
||||
salt = secrets.token_bytes(4)
|
||||
return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii")
|
||||
|
||||
|
||||
def rabbit_definitions(passwords: dict) -> str:
|
||||
"""Топология и пользователи RabbitMQ; брокер загружает файл при каждом старте (infra/rabbitmq.conf).
|
||||
|
||||
Права (configure / write / read, регулярные выражения по именам ресурсов):
|
||||
dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет;
|
||||
worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет.
|
||||
Ни у кого из сервисов нет тега administrator и права создать/удалить очередь.
|
||||
"""
|
||||
queue_args = {
|
||||
"x-queue-type": "quorum",
|
||||
"x-delivery-limit": 20, # ядовитое сообщение не крутится бесконечно -> DLQ
|
||||
"x-dead-letter-exchange": DLX,
|
||||
"x-dead-letter-routing-key": DLQ_ROUTING_KEY,
|
||||
"x-dead-letter-strategy": "at-least-once", # перенос в DLQ без потери (требует reject-publish)
|
||||
"x-overflow": "reject-publish", # переполнение -> nack издателю, outbox повторит
|
||||
"x-max-length": 100000,
|
||||
}
|
||||
definitions = {
|
||||
"users": [{"name": name, "password_hash": rabbit_hash(password),
|
||||
"hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []}
|
||||
for name, password in passwords.items()],
|
||||
"vhosts": [{"name": BROKER_VHOST}],
|
||||
"permissions": [
|
||||
{"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"},
|
||||
{"user": "worker", "vhost": BROKER_VHOST, "configure": "^$",
|
||||
"write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"},
|
||||
],
|
||||
"exchanges": [
|
||||
{"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
{"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False,
|
||||
"internal": False, "arguments": {}},
|
||||
],
|
||||
"queues": [
|
||||
{"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args},
|
||||
# DLQ — classic durable: у quorum в 4.x delivery-limit по умолчанию 20, и многократный
|
||||
# просмотр (basic_get + nack) в ops.py мог бы выбросить сообщение. Здесь потерь от просмотра нет.
|
||||
{"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False,
|
||||
"arguments": {"x-queue-type": "classic"}},
|
||||
],
|
||||
"bindings": [
|
||||
{"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue",
|
||||
"routing_key": ROUTING_KEY, "arguments": {}},
|
||||
{"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue",
|
||||
"routing_key": DLQ_ROUTING_KEY, "arguments": {}},
|
||||
],
|
||||
}
|
||||
return json.dumps(definitions, indent=2)
|
||||
|
||||
|
||||
def broker_secrets(created: list, host: str = "rabbitmq") -> None:
|
||||
if (SECRETS / "rabbitmq_definitions.json").exists():
|
||||
return
|
||||
passwords = {"dispatcher": secrets.token_hex(24), "worker": secrets.token_hex(24)}
|
||||
for name, password in passwords.items():
|
||||
write(f"broker_url_{name}", f"amqp://{name}:{password}@{host}:5672/{BROKER_VHOST}")
|
||||
write("rabbitmq_definitions.json", rabbit_definitions(passwords))
|
||||
created += ["broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions.json"]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
"""Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes)."""
|
||||
SECRETS.mkdir(exist_ok=True)
|
||||
created = []
|
||||
if not (SECRETS / "postgres_password").exists():
|
||||
db_password = secrets.token_hex(24)
|
||||
redis_password = secrets.token_hex(24)
|
||||
write("postgres_password", db_password)
|
||||
write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger")
|
||||
write("cache_url", f"redis://:{redis_password}@redis:6379/0")
|
||||
write("redis_auth", f"requirepass {redis_password}\n")
|
||||
created += ["postgres_password", "database_url", "cache_url", "redis_auth"]
|
||||
if not (SECRETS / "s3_root_password").exists():
|
||||
# admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init
|
||||
write("s3_root_user", "root-" + secrets.token_hex(6))
|
||||
write("s3_root_password", secrets.token_hex(24))
|
||||
created += ["s3_root_user", "s3_root_password"]
|
||||
if not (SECRETS / "s3_secret_access_key").exists():
|
||||
# отдельный ключ API с политикой только на свой bucket (создаёт s3-init)
|
||||
write("s3_access_key_id", "api-" + secrets.token_hex(8))
|
||||
write("s3_secret_access_key", secrets.token_hex(24))
|
||||
created += ["s3_access_key_id", "s3_secret_access_key"]
|
||||
if not (SECRETS / "s3_config.json").exists():
|
||||
write("s3_config.json", s3_config(bucket_name()))
|
||||
created.append("s3_config.json")
|
||||
broker_secrets(created)
|
||||
if created:
|
||||
print("Created in secrets/: " + ", ".join(created) + " (values are not printed).")
|
||||
else:
|
||||
print("secrets/ is complete, nothing changed.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,14 @@
|
||||
@echo off
|
||||
rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ), then api, janitor, dispatcher, worker,
|
||||
rem then the balancer. Volumes are kept.
|
||||
rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
|
||||
rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
|
||||
cd /d "%~dp0.."
|
||||
docker compose restart postgres redis s3 rabbitmq || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq || exit /b 1
|
||||
docker compose restart api janitor dispatcher worker || exit /b 1
|
||||
docker compose restart lb || exit /b 1
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps api lb dispatcher worker || exit /b 1
|
||||
exit /b 0
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
# Restart hook for tests/resilience.py --mode persistence (Linux/macOS).
|
||||
# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ), then api, janitor, dispatcher and
|
||||
# worker, then the balancer. Volumes are kept (no down -v), nothing is re-seeded.
|
||||
# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly
|
||||
# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy).
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart postgres redis s3 rabbitmq
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq
|
||||
docker compose restart api janitor dispatcher worker
|
||||
docker compose restart lb
|
||||
docker compose up -d --wait --no-recreate --no-build --no-deps api lb dispatcher worker
|
||||
@@ -0,0 +1,6 @@
|
||||
@echo off
|
||||
rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh).
|
||||
rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works.
|
||||
rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py.
|
||||
python "%~dp0stop_one.py"
|
||||
exit /b %ERRORLEVEL%
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID.
|
||||
# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd.
|
||||
set -eu
|
||||
exec python3 "$(dirname "$0")/stop_one.py"
|
||||
@@ -0,0 +1,29 @@
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
STOP_TIMEOUT_SECONDS = 8
|
||||
|
||||
|
||||
def main() -> int:
|
||||
target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower()
|
||||
if not re.fullmatch(r"[0-9a-f]{12,64}", target):
|
||||
print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr)
|
||||
return 2
|
||||
running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT,
|
||||
capture_output=True, text=True, check=True).stdout.split()
|
||||
matches = [container for container in running if container.startswith(target)]
|
||||
if len(matches) != 1:
|
||||
print(f"no running api container with id {target}", file=sys.stderr)
|
||||
return 1
|
||||
subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]],
|
||||
cwd=ROOT, check=True, stdout=subprocess.DEVNULL)
|
||||
print(f"stopped api container {matches[0][:12]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept).
|
||||
cd /d "%~dp0.."
|
||||
if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% )
|
||||
if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% )
|
||||
if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% )
|
||||
echo usage: %0 stop^|start^|status
|
||||
exit /b 2
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Demo hook for the defense: stop or start the worker without touching volumes.
|
||||
# scripts/worker.sh stop | start | status
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
case "${1:-}" in
|
||||
stop) docker compose stop worker ;;
|
||||
start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;;
|
||||
status) docker compose ps worker ;;
|
||||
*) echo "usage: $0 stop|start|status" >&2; exit 2 ;;
|
||||
esac
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
import os
|
||||
import socket
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def env_str(name: str, default: str) -> str:
|
||||
return os.environ.get(name, default).strip()
|
||||
|
||||
|
||||
def env_int(name: str, default: int) -> int:
|
||||
value = int(env_str(name, str(default)))
|
||||
if value < 1:
|
||||
raise ValueError(f"{name} must be >= 1")
|
||||
return value
|
||||
|
||||
|
||||
def read_secret(file_variable: str, plain_variable: str) -> str:
|
||||
path = env_str(file_variable, "")
|
||||
if path:
|
||||
return Path(path).read_text(encoding="utf-8").strip()
|
||||
return env_str(plain_variable, "")
|
||||
|
||||
|
||||
APP_HOST = env_str("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = env_int("APP_PORT", 8080)
|
||||
APP_ENV = env_str("APP_ENV", "lab")
|
||||
APP_VERSION = env_str("APP_VERSION", "dev")
|
||||
|
||||
INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname()
|
||||
|
||||
SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_STORE = env_str("SESSION_STORE", "redis")
|
||||
|
||||
DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL")
|
||||
DB_POOL_MIN = env_int("DB_POOL_MIN", 1)
|
||||
DB_POOL_MAX = env_int("DB_POOL_MAX", 10)
|
||||
DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3)
|
||||
DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3)
|
||||
DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
|
||||
REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL")
|
||||
REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20)
|
||||
REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500)
|
||||
|
||||
MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations")))
|
||||
SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60)
|
||||
|
||||
# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket.
|
||||
S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333")
|
||||
S3_REGION = env_str("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY")
|
||||
S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1)
|
||||
S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5)
|
||||
S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2)
|
||||
S3_POOL_MAX = env_int("S3_POOL_MAX", 20)
|
||||
|
||||
# Вложения: лимиты из контракта и защита памяти/времени.
|
||||
MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024)
|
||||
MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000)
|
||||
UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60)
|
||||
UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10)
|
||||
DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2)
|
||||
UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads")
|
||||
|
||||
# Очистка незавершённых загрузок (janitor.py).
|
||||
ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600)
|
||||
JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300)
|
||||
|
||||
SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3)
|
||||
SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10)
|
||||
|
||||
# lab5: брокер (RabbitMQ) и фоновая обработка вложений.
|
||||
# У API адреса брокера нет вообще: API пишет задание в outbox (PostgreSQL), публикует dispatcher.
|
||||
BROKER_URL = read_secret("BROKER_URL_FILE", "BROKER_URL")
|
||||
BROKER_HEARTBEAT_SECONDS = env_int("BROKER_HEARTBEAT_SECONDS", 30)
|
||||
PIPELINE_VERSION = env_int("PIPELINE_VERSION", 1)
|
||||
WORKER_CONCURRENCY = env_int("WORKER_CONCURRENCY", 2) # потоков-потребителей в одном worker
|
||||
WORKER_PREFETCH = env_int("WORKER_PREFETCH", 1) # неподтверждённых сообщений на поток
|
||||
JOB_MAX_ATTEMPTS = env_int("JOB_MAX_ATTEMPTS", 5)
|
||||
JOB_TIMEOUT_SECONDS = env_int("JOB_TIMEOUT_SECONDS", 60) # жёсткий предел одной обработки изображения
|
||||
JOB_LEASE_SECONDS = env_int("JOB_LEASE_SECONDS", 15) # аренда processing, продлевается каждые lease/3
|
||||
JOB_BACKOFF_BASE_SECONDS = env_int("JOB_BACKOFF_BASE_SECONDS", 2)
|
||||
JOB_BACKOFF_MAX_SECONDS = env_int("JOB_BACKOFF_MAX_SECONDS", 60)
|
||||
JOB_RECONCILE_SECONDS = env_int("JOB_RECONCILE_SECONDS", 120) # queued без сообщения дольше -> опубликовать заново
|
||||
WORKER_MEMORY_LIMIT_MB = env_int("WORKER_MEMORY_LIMIT_MB", 640) # RLIMIT_AS процесса обработки
|
||||
WORKER_TMP_DIR = env_str("WORKER_TMP_DIR", "/tmp/work")
|
||||
DISPATCHER_POLL_SECONDS = env_int("DISPATCHER_POLL_SECONDS", 1)
|
||||
DISPATCHER_BATCH = env_int("DISPATCHER_BATCH", 100)
|
||||
STAGING_RETENTION_SECONDS = env_int("STAGING_RETENTION_SECONDS", 86400) # исходник failed-вложения для replay
|
||||
OUTBOX_RETENTION_SECONDS = env_int("OUTBOX_RETENTION_SECONDS", 7 * 86400)
|
||||
ALIVE_FILE = env_str("ALIVE_FILE", "/tmp/alive") # healthcheck фоновых процессов
|
||||
|
||||
# Учебное внедрение сбоев (faults.py). На стенде включено, в эксплуатации FAULT_INJECTION=0.
|
||||
FAULT_INJECTION = env_str("FAULT_INJECTION", "0") == "1"
|
||||
FAULTS_DIR = env_str("FAULTS_DIR", "/faults")
|
||||
|
||||
|
||||
def check() -> None:
|
||||
"""Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе."""
|
||||
if not DATABASE_URL:
|
||||
raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE")
|
||||
if not REDIS_URL:
|
||||
raise RuntimeError("Redis address is not set: CACHE_URL_FILE")
|
||||
if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY:
|
||||
raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE")
|
||||
if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS:
|
||||
raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS")
|
||||
if SESSION_STORE != "redis":
|
||||
raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution")
|
||||
+183
@@ -0,0 +1,183 @@
|
||||
"""Приватное S3-хранилище (SeaweedFS) для байтов вложений.
|
||||
|
||||
Раскладка ключей (lab5):
|
||||
staging/<32 hex> — исходник, принятый API до ответа 202 (устойчивый staging);
|
||||
attachments/<id>/v<N>/original — опубликованный worker'ом оригинал (побайтно исходный файл);
|
||||
attachments/<id>/v<N>/thumbnail.jpg, crop.jpg, optimized.webp — варианты pipeline версии N;
|
||||
originals/<32 hex> — исходники вложений lab4, worker обрабатывает их как staging.
|
||||
|
||||
API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket
|
||||
(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер,
|
||||
имя файла пользователя в ключ не попадает.
|
||||
"""
|
||||
import base64
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import BinaryIO, Iterator
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
import settings
|
||||
|
||||
log = logging.getLogger("messenger.storage")
|
||||
|
||||
ORIGINALS_PREFIX = "originals/" # lab4
|
||||
STAGING_PREFIX = "staging/"
|
||||
FINAL_PREFIX = "attachments/"
|
||||
ALL_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, ORIGINALS_PREFIX)
|
||||
READ_CHUNK = 64 * 1024
|
||||
|
||||
|
||||
class StorageUnavailable(Exception):
|
||||
"""S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей."""
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""Объекта нет в bucket (NoSuchKey / 404)."""
|
||||
|
||||
|
||||
def _make_client(connect_timeout: float, read_timeout: float, attempts: int):
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.S3_ENDPOINT,
|
||||
region_name=settings.S3_REGION,
|
||||
aws_access_key_id=settings.S3_ACCESS_KEY_ID or None,
|
||||
aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"}, # http://s3:8333/<bucket>/<key>
|
||||
connect_timeout=connect_timeout,
|
||||
read_timeout=read_timeout,
|
||||
retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую
|
||||
max_pool_connections=settings.S3_POOL_MAX,
|
||||
# Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно.
|
||||
request_checksum_calculation="when_required",
|
||||
response_checksum_validation="when_required",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS)
|
||||
# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor.
|
||||
_quick_client = _make_client(1, 2, 1)
|
||||
|
||||
|
||||
def _is_missing(exc: ClientError) -> bool:
|
||||
code = exc.response.get("Error", {}).get("Code", "")
|
||||
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
|
||||
return code in ("NoSuchKey", "404", "NotFound") or status == 404
|
||||
|
||||
|
||||
def _unavailable(action: str, exc: Exception) -> StorageUnavailable:
|
||||
detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else ""
|
||||
log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail)
|
||||
return StorageUnavailable()
|
||||
|
||||
|
||||
def new_key(object_id: str) -> str:
|
||||
"""Ключ staging для нового исходника (имя файла пользователя в ключ не попадает)."""
|
||||
return STAGING_PREFIX + object_id
|
||||
|
||||
|
||||
def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None:
|
||||
"""Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект."""
|
||||
body.seek(0)
|
||||
try:
|
||||
_client.put_object(
|
||||
Bucket=settings.S3_BUCKET,
|
||||
Key=key,
|
||||
Body=body,
|
||||
ContentLength=size,
|
||||
ContentType=content_type,
|
||||
ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"),
|
||||
)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("put", exc) from None
|
||||
|
||||
|
||||
def open_read(key: str):
|
||||
"""GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после."""
|
||||
try:
|
||||
return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"]
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
raise ObjectMissing(key) from None
|
||||
raise _unavailable("get", exc) from None
|
||||
except BotoCoreError as exc:
|
||||
raise _unavailable("get", exc) from None
|
||||
|
||||
|
||||
def download(key: str, path, max_bytes: int) -> int:
|
||||
"""GET объекта в файл кусками. Дальше max_bytes не читаем: размер не совпадёт, worker отклонит исходник."""
|
||||
stream = open_read(key)
|
||||
size = 0
|
||||
try:
|
||||
with open(path, "wb") as file:
|
||||
while True:
|
||||
chunk = stream.read(READ_CHUNK)
|
||||
if not chunk:
|
||||
return size
|
||||
size += len(chunk)
|
||||
file.write(chunk)
|
||||
if size > max_bytes:
|
||||
return size
|
||||
except (BotoCoreError, OSError) as exc:
|
||||
raise _unavailable("get", exc) from None
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
|
||||
def iter_chunks(stream) -> Iterator[bytes]:
|
||||
"""Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание."""
|
||||
try:
|
||||
while True:
|
||||
chunk = stream.read(READ_CHUNK)
|
||||
if not chunk:
|
||||
return
|
||||
yield chunk
|
||||
except (BotoCoreError, OSError) as exc:
|
||||
# Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length).
|
||||
log.warning("S3 stream interrupted: %s", type(exc).__name__)
|
||||
raise
|
||||
finally:
|
||||
stream.close()
|
||||
|
||||
|
||||
def delete(key: str, quick: bool = False) -> None:
|
||||
"""Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа."""
|
||||
try:
|
||||
(_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("delete", exc) from None
|
||||
|
||||
|
||||
def exists(key: str) -> bool:
|
||||
try:
|
||||
_client.head_object(Bucket=settings.S3_BUCKET, Key=key)
|
||||
return True
|
||||
except ClientError as exc:
|
||||
if _is_missing(exc):
|
||||
return False
|
||||
raise _unavailable("head", exc) from None
|
||||
except BotoCoreError as exc:
|
||||
raise _unavailable("head", exc) from None
|
||||
|
||||
|
||||
def list_objects(prefix: str) -> Iterator[tuple[str, datetime, int]]:
|
||||
"""Все объекты под prefix: (key, last_modified, size). Постранично, по 1000."""
|
||||
paginator = _client.get_paginator("list_objects_v2")
|
||||
try:
|
||||
for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix):
|
||||
for item in page.get("Contents", []):
|
||||
yield item["Key"], item["LastModified"], item["Size"]
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("list", exc) from None
|
||||
|
||||
|
||||
def ping() -> None:
|
||||
try:
|
||||
_client.head_bucket(Bucket=settings.S3_BUCKET)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
raise _unavailable("head_bucket", exc) from None
|
||||
@@ -0,0 +1,91 @@
|
||||
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
|
||||
from common import call, check, compose_output, docker, new_user, seen_instances
|
||||
|
||||
|
||||
def main():
|
||||
info = json.loads(docker("info", "--format", "{{json .}}"))
|
||||
print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
|
||||
f"client Python {platform.python_version()}")
|
||||
print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, "
|
||||
f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, "
|
||||
f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}")
|
||||
|
||||
print()
|
||||
print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip())
|
||||
for one_shot in ("migrate", "s3-init"):
|
||||
status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip()
|
||||
check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0")
|
||||
|
||||
config = json.loads(compose_output("config", "--format", "json"))
|
||||
services = config["services"]
|
||||
containers = compose_output("ps", "-q").split()
|
||||
rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()]
|
||||
print()
|
||||
print("container image cpus memory host ports networks")
|
||||
api_count = 0
|
||||
for row in sorted(rows, key=lambda r: r["Name"]):
|
||||
labels = row["Config"]["Labels"]
|
||||
service = labels.get("com.docker.compose.service")
|
||||
limits = row["HostConfig"]
|
||||
cpus = limits.get("NanoCpus", 0) / 1e9
|
||||
memory = limits.get("Memory", 0) // 2**20
|
||||
ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values()
|
||||
for b in (bindings or [])})
|
||||
networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"])
|
||||
print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB "
|
||||
f"{','.join(ports) or '-':<23} {','.join(networks)}")
|
||||
check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit")
|
||||
if service == "api":
|
||||
api_count += 1
|
||||
check(not ports, "api must not publish ports: everything goes through the balancer")
|
||||
if service == "lb":
|
||||
check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only")
|
||||
if service in ("postgres", "redis", "s3", "janitor", "rabbitmq", "worker", "dispatcher"):
|
||||
check(not ports and networks == ["data"], f"{service} must be only in the internal data network")
|
||||
check(api_count >= 2, "need at least 2 api replicas")
|
||||
running = {r["Config"]["Labels"].get("com.docker.compose.service") for r in rows if r["State"]["Running"]}
|
||||
check({"rabbitmq", "worker", "dispatcher"} <= running, "lab5: broker, worker and dispatcher must be running")
|
||||
check(any(v.get("target") == "/var/lib/rabbitmq" and v.get("type") == "volume"
|
||||
for v in services["rabbitmq"].get("volumes", [])), "rabbitmq needs a persistent volume")
|
||||
check(services["rabbitmq"].get("hostname") == "rabbitmq", "rabbitmq needs a fixed hostname (node data dir)")
|
||||
check(all("container_name" not in s for s in services.values()), "container_name prevents scaling")
|
||||
check(config["networks"]["data"].get("internal") is True, "data network must be internal")
|
||||
check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis")
|
||||
|
||||
print()
|
||||
for name, service in sorted(services.items()):
|
||||
secrets = sorted(s["source"] for s in service.get("secrets", []))
|
||||
print(f"secrets of {name}: {secrets or '-'}")
|
||||
check(not services["lb"].get("secrets"), "lb gets no secrets")
|
||||
check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url")
|
||||
for name in ("api", "janitor", "worker"):
|
||||
names = {s["source"] for s in services[name]["secrets"]}
|
||||
check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys")
|
||||
check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key")
|
||||
broker_secrets = {"broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions"}
|
||||
for name in ("api", "janitor", "lb", "migrate"):
|
||||
names = {s["source"] for s in services[name].get("secrets", [])}
|
||||
check(not names & broker_secrets, f"{name} must not get broker credentials (API uses the outbox)")
|
||||
check({s["source"] for s in services["dispatcher"]["secrets"]} == {"database_url", "broker_url_dispatcher"},
|
||||
"dispatcher: only database_url and its own broker user")
|
||||
check(not {s["source"] for s in services["dispatcher"]["secrets"]} & {"s3_access_key_id", "s3_secret_access_key"},
|
||||
"dispatcher needs no S3 key")
|
||||
|
||||
user = new_user("stack")
|
||||
served = seen_instances(user["token"])
|
||||
print()
|
||||
print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}")
|
||||
check(len(served) >= 2, "requests are not balanced")
|
||||
health = call("GET", "/health/ready", 200)
|
||||
print(f"/health/ready -> {health.status} from {health.instance}")
|
||||
print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every "
|
||||
f"container, postgres/redis/s3/janitor/rabbitmq/worker/dispatcher only in the internal data network, "
|
||||
f"S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,361 @@
|
||||
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence") # по умолчанию evidence/
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
PY = sys.executable
|
||||
|
||||
STEPS = [
|
||||
("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]),
|
||||
("stack", "stand layout: replicas, ports, limits, networks, secrets, S3", [PY, "student-tests/check_stack.py"]),
|
||||
("smoke", "public smoke via the balancer (make test), with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]),
|
||||
("jobs-variants", "lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped",
|
||||
[PY, "student-tests/test_jobs.py", "variants"]),
|
||||
("jobs-limits", "lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)",
|
||||
[PY, "student-tests/test_jobs.py", "limits"]),
|
||||
("jobs-recovery", "lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits",
|
||||
[PY, "student-tests/test_jobs.py", "worker-down", "broker-down"]),
|
||||
("jobs-outbox", "lab5: dispatcher crash between COMMIT and publish / after confirm",
|
||||
[PY, "student-tests/test_jobs.py", "outbox-crash"]),
|
||||
("jobs-crash", "lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease",
|
||||
[PY, "student-tests/test_jobs.py", "crash-after-put", "crash-after-commit", "staging", "lease"]),
|
||||
("jobs-retry", "lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ",
|
||||
[PY, "student-tests/test_jobs.py", "retry", "exhausted-replay", "permanent", "dlq"]),
|
||||
("jobs-janitor-replay", "lab5: janitor does not delete objects re-written by replay (review fix)",
|
||||
[PY, "student-tests/test_janitor_replay.py"]),
|
||||
("jobs-concurrency", "lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing",
|
||||
[PY, "student-tests/test_jobs.py", "concurrency"]),
|
||||
("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids",
|
||||
[PY, "student-tests/test_attachments.py"]),
|
||||
("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap",
|
||||
[PY, "student-tests/test_s3_security.py"]),
|
||||
("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage",
|
||||
[PY, "student-tests/test_consistency.py"]),
|
||||
("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas",
|
||||
[PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]),
|
||||
("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]),
|
||||
("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]),
|
||||
("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]),
|
||||
("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]),
|
||||
("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]),
|
||||
("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]),
|
||||
("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]),
|
||||
("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)",
|
||||
[PY, "student-tests/run_failover.py"]),
|
||||
("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL,
|
||||
"--duration", "30", "--concurrency", "4",
|
||||
"--output", "evidence/load.json"]),
|
||||
("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]),
|
||||
("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)",
|
||||
[PY, "student-tests/run_persistence.py"]),
|
||||
]
|
||||
|
||||
|
||||
def git(*args):
|
||||
try:
|
||||
return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip()
|
||||
except (OSError, subprocess.CalledProcessError):
|
||||
return ""
|
||||
|
||||
|
||||
def commit():
|
||||
head = git("rev-parse", "--short", "HEAD") or "unknown"
|
||||
changed = [line for line in git("status", "--porcelain").splitlines()
|
||||
if not line[3:].startswith(("evidence/", "REPORT.md"))]
|
||||
return head + (" + uncommitted changes" if changed else "")
|
||||
|
||||
|
||||
def run_step(name, description, command):
|
||||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL)
|
||||
print(f"=== {name}: {description}", flush=True)
|
||||
started = time.monotonic()
|
||||
result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
||||
seconds = time.monotonic() - started
|
||||
output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n")
|
||||
shown = " ".join(Path(part).name if part == PY else part for part in command)
|
||||
header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n"
|
||||
f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n")
|
||||
(EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8")
|
||||
tail = [line for line in output.splitlines() if line.strip()][-3:]
|
||||
for line in tail:
|
||||
print(" " + line)
|
||||
print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True)
|
||||
return result.returncode == 0, seconds
|
||||
|
||||
|
||||
def load_json(name):
|
||||
path = EVIDENCE / name
|
||||
return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None
|
||||
|
||||
|
||||
def measurements():
|
||||
"""Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md)."""
|
||||
lines = []
|
||||
try:
|
||||
info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True,
|
||||
text=True, check=True).stdout)
|
||||
docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM"
|
||||
except (OSError, ValueError, subprocess.CalledProcessError):
|
||||
docker_vm = "unknown"
|
||||
lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; "
|
||||
f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона "
|
||||
f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB "
|
||||
f"каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, "
|
||||
f"janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB.")
|
||||
lines.append("")
|
||||
|
||||
failover = EVIDENCE / "failover.txt"
|
||||
results = []
|
||||
if failover.exists():
|
||||
text = failover.read_text(encoding="utf-8")
|
||||
stopped = re.findall(r"the hook stopped (\w+)", text)
|
||||
results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)]
|
||||
if results:
|
||||
lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):")
|
||||
lines.append("")
|
||||
lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток "
|
||||
"(повторены с тем же ключом) | Первая запись после hook, с |")
|
||||
lines.append("| --- | --- | --- | --- | --- | --- |")
|
||||
for number, (result, container) in enumerate(zip(results, stopped), start=1):
|
||||
lines.append(f"| {number} | {container} | {result['observed_instances_before']} | "
|
||||
f"{result['successful_writes']} | {result['failed_attempts']} | "
|
||||
f"{result['first_recovery_seconds']} |")
|
||||
lines.append("")
|
||||
|
||||
memory = load_json("upload-memory.json")
|
||||
if memory:
|
||||
lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики "
|
||||
f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). "
|
||||
f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:")
|
||||
lines.append("")
|
||||
lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | "
|
||||
"Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |")
|
||||
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |")
|
||||
for run in memory["runs"]:
|
||||
for n, r in enumerate(run["replicas"]):
|
||||
head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | "
|
||||
f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |"
|
||||
cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-"
|
||||
lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | "
|
||||
f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |")
|
||||
naive = memory["single_file_extra_rss_mib"]
|
||||
lines.append("")
|
||||
lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в "
|
||||
f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка "
|
||||
f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в "
|
||||
f"{memory['api_limit_mib']} MiB.")
|
||||
lines.append("")
|
||||
|
||||
metrics = load_json("failover-metrics.json")
|
||||
if metrics:
|
||||
lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи "
|
||||
"без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):")
|
||||
lines.append("")
|
||||
lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок "
|
||||
"(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | "
|
||||
"Вернулась в балансировку после docker start, с |")
|
||||
lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |")
|
||||
for r in metrics["runs"]:
|
||||
how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)"
|
||||
codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-"
|
||||
lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | "
|
||||
f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | "
|
||||
f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |")
|
||||
lines.append("")
|
||||
lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика "
|
||||
"после остановки не ответила ни разу.")
|
||||
lines.append("")
|
||||
|
||||
rolling = load_json("rolling-update.json")
|
||||
if rolling:
|
||||
during = rolling["during"]
|
||||
codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-"
|
||||
lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> "
|
||||
f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за "
|
||||
f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, "
|
||||
f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); "
|
||||
f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено "
|
||||
f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.")
|
||||
lines.append("")
|
||||
|
||||
load = load_json("load.json")
|
||||
if load:
|
||||
latency = load["successful_latency_ms"]
|
||||
lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, "
|
||||
f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных "
|
||||
f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных "
|
||||
f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.")
|
||||
lines.append("")
|
||||
|
||||
persistence = EVIDENCE / "persistence.txt"
|
||||
restart_seconds = None
|
||||
if persistence.exists():
|
||||
found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8"))
|
||||
restart_seconds = found.group(1) if found else None
|
||||
|
||||
impacts = []
|
||||
if metrics:
|
||||
for mode, what, expected in (
|
||||
("stop", "`docker stop` реплики (SIGTERM) под нагрузкой",
|
||||
"ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"),
|
||||
("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой",
|
||||
"единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")):
|
||||
runs = [r for r in metrics["runs"] if r["mode"] == mode]
|
||||
if runs:
|
||||
impacts.append(
|
||||
f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); "
|
||||
f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | "
|
||||
f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; "
|
||||
f"потерь 0, дублей 0; реплика вернулась через "
|
||||
f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |")
|
||||
if failover.exists() and results:
|
||||
impacts.append(
|
||||
f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, "
|
||||
f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: "
|
||||
f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через "
|
||||
f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |")
|
||||
if rolling:
|
||||
impacts.append(
|
||||
f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | "
|
||||
f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |")
|
||||
if restart_seconds:
|
||||
impacts.append(
|
||||
f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и "
|
||||
f"прежний токен живы | "
|
||||
f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |")
|
||||
consistency = EVIDENCE / "consistency.txt"
|
||||
if consistency.exists():
|
||||
text = consistency.read_text(encoding="utf-8")
|
||||
timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text)
|
||||
if len(timings) >= 2:
|
||||
impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, "
|
||||
f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | "
|
||||
f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |")
|
||||
impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за "
|
||||
f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |")
|
||||
if "PASS: S3 written but DB not committed" in text:
|
||||
impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | "
|
||||
"вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | "
|
||||
"GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | "
|
||||
"мусора не осталось; повторный проход ничего не делает |")
|
||||
if "PASS: an interrupted upload leaves no row" in text:
|
||||
impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | "
|
||||
"строк и объектов не прибавилось, временных файлов 0 | мгновенно |")
|
||||
impacts += lab5_impacts()
|
||||
if impacts:
|
||||
lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |")
|
||||
lines.append("| --- | --- | --- | --- |")
|
||||
lines.extend(impacts)
|
||||
lines.append("")
|
||||
return "\n".join(lines).strip()
|
||||
|
||||
|
||||
def lab5_line(name, pattern):
|
||||
"""Строка вывода test_jobs.py из evidence/<name>.txt, содержащая pattern (или None)."""
|
||||
path = EVIDENCE / f"{name}.txt"
|
||||
if not path.exists():
|
||||
return None
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if pattern in line and not line.startswith(("#", "---")):
|
||||
return line.strip()
|
||||
return None
|
||||
|
||||
|
||||
def lab5_impacts():
|
||||
"""Строки таблицы отказов lab5: наблюдение — дословно строка вывода теста (числа из прогона)."""
|
||||
rows = [
|
||||
("jobs-recovery", "worker started", "worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker",
|
||||
"202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready",
|
||||
"потерь нет; staging удалён после COMMIT"),
|
||||
("jobs-recovery", "rabbitmq started", "rabbitmq остановлен во время upload",
|
||||
"202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется", "потерь нет"),
|
||||
("jobs-outbox", "killed between COMMIT and publish", "crash dispatcher между COMMIT задания и publish",
|
||||
"строка outbox не отмечена -> публикуется после рестарта процесса", "без ручного действия"),
|
||||
("jobs-outbox", "after publisher confirm", "crash dispatcher после confirm, до отметки outbox",
|
||||
"дубль сообщения, worker распознаёт по состоянию job", "один набор вариантов"),
|
||||
("jobs-crash", "killed after S3 PUT", "crash worker после PUT в S3, до COMMIT и ack",
|
||||
"повтор после истечения аренды перезаписывает те же ключи", "один набор, 4 объекта"),
|
||||
("jobs-crash", "after COMMIT, before ack", "crash worker после COMMIT, до ack",
|
||||
"redelivery ready-задачи -> no-op ack", "очередь пуста, дублей нет"),
|
||||
("jobs-crash", "frozen attempt", "worker завис (не продлевает аренду)",
|
||||
"dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing", "ready не откатан"),
|
||||
("jobs-limits", "imaging hang", "зависание обработки изображения",
|
||||
"процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась", "повтор -> ready"),
|
||||
("jobs-limits", "memory bomb", "нехватка памяти в процессе обработки (RLIMIT_AS)",
|
||||
"failed/resource_limit, worker жив", "постоянная ошибка, без повторов"),
|
||||
("jobs-retry", "storage_unavailable ->", "2 временные ошибки S3", "backoff с jitter, затем ready", "-"),
|
||||
("jobs-retry", "retry_exhausted (public", "5 временных ошибок подряд",
|
||||
"failed/retry_exhausted, реестр failed, content 409", "replay оператора"),
|
||||
("jobs-retry", "ops replay", "replay failed и повторный replay ready", "тот же job_id -> ready; повтор отклонён",
|
||||
"дубля нет"),
|
||||
("jobs-retry", "-> reject -> DLQ", "невалидное сообщение и неизвестный job_id", "reject -> DLQ, dlq-replay не исполняет",
|
||||
"сообщения сохранены"),
|
||||
("jobs-concurrency", "peak processing", "8 заданий в очереди, старт worker",
|
||||
"WORKER_CONCURRENCY=2, prefetch 1", "-"),
|
||||
]
|
||||
out = []
|
||||
for name, pattern, what, expected, recovery in rows:
|
||||
line = lab5_line(name, pattern)
|
||||
if line:
|
||||
out.append(f"| {what} | {expected} | {line.replace('|', '/')} | {recovery} |")
|
||||
return out
|
||||
|
||||
|
||||
def update_report(block):
|
||||
report = ROOT / "REPORT.md"
|
||||
text = report.read_text(encoding="utf-8")
|
||||
begin, end = "<!-- measurements:begin -->", "<!-- measurements:end -->"
|
||||
if begin not in text or end not in text:
|
||||
print("REPORT.md has no measurement markers, only evidence/summary.md was written")
|
||||
return
|
||||
head, rest = text.split(begin, 1)
|
||||
_, tail = rest.split(end, 1)
|
||||
report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8")
|
||||
print("REPORT.md: measurement tables updated")
|
||||
|
||||
|
||||
def main():
|
||||
args = sys.argv[1:]
|
||||
names = [name for name, _, _ in STEPS]
|
||||
if args[:1] == ["--from"]:
|
||||
if len(args) < 2 or args[1] not in names:
|
||||
raise SystemExit("usage: --from <step>; steps: " + ", ".join(names))
|
||||
selected = names[names.index(args[1]):]
|
||||
else:
|
||||
unknown = [a for a in args if a not in names]
|
||||
if unknown:
|
||||
raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names))
|
||||
selected = args or names
|
||||
EVIDENCE.mkdir(exist_ok=True)
|
||||
results = []
|
||||
for name, description, command in STEPS:
|
||||
if name not in selected:
|
||||
continue
|
||||
ok, seconds = run_step(name, description, command)
|
||||
results.append((name, description, ok, seconds))
|
||||
if not ok:
|
||||
print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.")
|
||||
print(f"After fixing: python student-tests/collect_evidence.py --from {name}")
|
||||
break
|
||||
block = measurements()
|
||||
summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |",
|
||||
"| --- | --- | --- | --- | --- |"]
|
||||
summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results]
|
||||
(EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8")
|
||||
print("evidence/summary.md written")
|
||||
passed = all(ok for _, _, ok, _ in results)
|
||||
if passed:
|
||||
update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков
|
||||
raise SystemExit(0 if passed else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,347 @@
|
||||
|
||||
import base64
|
||||
import csv
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
|
||||
LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv")
|
||||
PASSWORD = "Test-password-123"
|
||||
|
||||
|
||||
class Response:
|
||||
def __init__(self, status, headers, data):
|
||||
self.status = status
|
||||
self.headers = {name.lower(): value for name, value in headers.items()}
|
||||
self.data = data
|
||||
try:
|
||||
self.json = json.loads(data.decode("utf-8")) if data else None
|
||||
except ValueError:
|
||||
self.json = None
|
||||
|
||||
@property
|
||||
def instance(self):
|
||||
"""X-Instance-Id: какая реплика API ответила (ID её контейнера)."""
|
||||
return self.headers.get("x-instance-id")
|
||||
|
||||
|
||||
def check(condition, message):
|
||||
if not condition:
|
||||
raise AssertionError(message)
|
||||
|
||||
|
||||
def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30):
|
||||
"""HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError.
|
||||
|
||||
Заодно проверяет то, что контракт требует от любого ответа:
|
||||
X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401.
|
||||
"""
|
||||
all_headers = dict(headers or {})
|
||||
data = raw
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
if data is not None:
|
||||
all_headers.setdefault("Content-Type", "application/json")
|
||||
if token is not None:
|
||||
all_headers["Authorization"] = "Bearer " + token
|
||||
|
||||
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as reply:
|
||||
response = Response(reply.status, reply.headers, reply.read())
|
||||
except urllib.error.HTTPError as error:
|
||||
response = Response(error.code, error.headers, error.read())
|
||||
|
||||
expected = expected if isinstance(expected, tuple) else (expected,)
|
||||
check(response.status in expected,
|
||||
f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}")
|
||||
request_id = response.headers.get("x-request-id")
|
||||
check(request_id, f"{method} {path}: no X-Request-Id header")
|
||||
check(response.instance, f"{method} {path}: no X-Instance-Id header")
|
||||
if response.status >= 400:
|
||||
error = (response.json or {}).get("error") or {}
|
||||
check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format")
|
||||
check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header")
|
||||
if response.status == 401:
|
||||
check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate")
|
||||
return response
|
||||
|
||||
|
||||
def try_call(method, path, *, body=None, token=None, headers=None, timeout=2):
|
||||
"""Запрос без проверок и без исключений — для нагрузки во время отказа.
|
||||
|
||||
Возвращает Response или None, если соединение оборвалось / вышел timeout.
|
||||
"""
|
||||
all_headers = dict(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode("utf-8")
|
||||
all_headers["Content-Type"] = "application/json"
|
||||
if token is not None:
|
||||
all_headers["Authorization"] = "Bearer " + token
|
||||
request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=timeout) as reply:
|
||||
return Response(reply.status, reply.headers, reply.read())
|
||||
except urllib.error.HTTPError as error:
|
||||
try:
|
||||
return Response(error.code, error.headers, error.read())
|
||||
except OSError:
|
||||
return None
|
||||
except (urllib.error.URLError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
"""Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8."""
|
||||
return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii")
|
||||
|
||||
|
||||
def login(username, password, expected=201):
|
||||
return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)})
|
||||
|
||||
|
||||
def new_user(prefix="user", password=PASSWORD):
|
||||
"""Регистрирует пользователя и входит. Возвращает dict: id, username, password, token."""
|
||||
username = f"{prefix}_{uuid.uuid4().hex[:10]}"
|
||||
user = call("POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": prefix.title(), "password": password}).json
|
||||
session = login(username, password).json
|
||||
return {**user, "password": password, "token": session["token"]}
|
||||
|
||||
|
||||
def new_chat(owner, *others, title="Test chat"):
|
||||
member_ids = [owner["id"]] + [user["id"] for user in others]
|
||||
return call("POST", "/api/v1/chats", 201, token=owner["token"],
|
||||
body={"title": title, "member_ids": member_ids}).json
|
||||
|
||||
|
||||
def new_key():
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def send(user, chat_id, text, expected=201, key=None):
|
||||
"""POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный."""
|
||||
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
|
||||
body={"text": text}, headers={"Idempotency-Key": key or new_key()})
|
||||
|
||||
|
||||
def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None):
|
||||
"""POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся."""
|
||||
body = {"text": text}
|
||||
if attachment_ids is not None:
|
||||
body["attachment_ids"] = attachment_ids
|
||||
return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"],
|
||||
body=body, headers={"Idempotency-Key": key or new_key()})
|
||||
|
||||
|
||||
def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=202, timeout=60,
|
||||
wait=True):
|
||||
"""POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра.
|
||||
|
||||
Lab5: успешный приём — 202 со снимком queued (переход 201 -> 202 из COURSE.md). При wait=True
|
||||
ответ дополняется ожиданием готовности: .json — метаданные после GET в состоянии ready/failed,
|
||||
.accepted — исходный снимок 202. Так проверки lab4 («файл скачивается побайтно») сохраняются.
|
||||
"""
|
||||
query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="")
|
||||
headers = {"Content-Type": content_type} if content_type is not None else {}
|
||||
reply = call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data,
|
||||
token=user["token"] if user else None, headers=headers, timeout=timeout)
|
||||
if reply.status == 202:
|
||||
check(reply.json["status"] == "queued" and reply.json["variants"] == [], "202 must be a queued snapshot")
|
||||
reply.accepted = reply.json
|
||||
if wait:
|
||||
reply.json = wait_attachment(user, reply.json["id"])
|
||||
return reply
|
||||
|
||||
|
||||
def wait_attachment(user, attachment_id, statuses=("ready", "failed"), seconds=60):
|
||||
"""GET метаданных, пока вложение не перейдёт в одно из statuses (smoke ждёт ready до 60 с)."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
att = call("GET", f"/api/v1/attachments/{attachment_id}", 200, token=user["token"]).json
|
||||
if att["status"] in statuses:
|
||||
return att
|
||||
check(time.monotonic() < deadline, f"attachment {attachment_id} is {att['status']} after {seconds} s")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def read_history(user, chat_id, limit=100):
|
||||
"""Вся история чата по страницам. Возвращает (сообщения, число_страниц)."""
|
||||
items, pages, cursor = [], 0, None
|
||||
while True:
|
||||
query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "")
|
||||
page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json
|
||||
items += page["items"]
|
||||
pages += 1
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
return items, pages
|
||||
|
||||
|
||||
def wait_ready(seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply:
|
||||
if reply.status == 200:
|
||||
return
|
||||
except (urllib.error.URLError, OSError):
|
||||
pass
|
||||
check(time.monotonic() < deadline, f"API is not ready after {seconds} s")
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def compose(*args, env=None):
|
||||
"""docker compose <args> в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам."""
|
||||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def compose_output(*args, env=None):
|
||||
"""docker compose <args> и его вывод (для logs, ps)."""
|
||||
result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env)
|
||||
return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def docker(*args, check_exit=True):
|
||||
"""docker <args>, возвращает stdout."""
|
||||
result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit)
|
||||
return result.stdout.decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def api_containers(include_stopped=False):
|
||||
"""Полные ID контейнеров сервиса api (по умолчанию только запущенные)."""
|
||||
args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"]
|
||||
return sorted(compose_output(*args).split())
|
||||
|
||||
|
||||
def health(container):
|
||||
"""Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''."""
|
||||
return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container,
|
||||
check_exit=False)
|
||||
|
||||
|
||||
def lb_servers():
|
||||
"""Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}."""
|
||||
with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply:
|
||||
rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace"))))
|
||||
header = [name.lstrip("# ") for name in rows[0]]
|
||||
servers = {}
|
||||
for row in rows[1:]:
|
||||
item = dict(zip(header, row))
|
||||
if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"):
|
||||
servers[item["svname"]] = item.get("status", "")
|
||||
return servers
|
||||
|
||||
|
||||
def lb_up_count():
|
||||
"""Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP)."""
|
||||
return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP"))
|
||||
|
||||
|
||||
def seen_instances(token, requests=20):
|
||||
"""Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me."""
|
||||
return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)}
|
||||
|
||||
|
||||
def wait_balanced(replicas, seconds=60):
|
||||
"""Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
running = api_containers()
|
||||
healthy = [container for container in running if health(container) == "healthy"]
|
||||
try:
|
||||
up = lb_up_count()
|
||||
except OSError:
|
||||
up = 0
|
||||
if len(healthy) >= replicas and up >= replicas:
|
||||
return running
|
||||
check(time.monotonic() < deadline,
|
||||
f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}")
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def redis_cli(*args):
|
||||
"""redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод."""
|
||||
command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args)
|
||||
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command],
|
||||
cwd=ROOT, capture_output=True, check=True)
|
||||
return result.stdout.decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def session_key(token):
|
||||
"""Ключ сессии в Redis, как его считает auth.py: session:<SHA-256 токена>."""
|
||||
return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def psql(sql):
|
||||
"""Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки).
|
||||
|
||||
Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя.
|
||||
"""
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "postgres",
|
||||
"psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql],
|
||||
cwd=ROOT, capture_output=True, check=False,
|
||||
)
|
||||
output = result.stdout.decode("utf-8", "replace").strip()
|
||||
error = result.stderr.decode("utf-8", "replace").strip()
|
||||
return result.returncode, output, error
|
||||
|
||||
|
||||
def app_python(code, *args, service="api", env=None):
|
||||
"""Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API).
|
||||
|
||||
Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON).
|
||||
"""
|
||||
command = ["docker", "compose", "exec", "-T"]
|
||||
for name, value in (env or {}).items():
|
||||
command += ["-e", f"{name}={value}"]
|
||||
command += [service, "python", "-", *map(str, args)]
|
||||
result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False)
|
||||
output = result.stdout.decode("utf-8", "replace").strip()
|
||||
check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
|
||||
return output
|
||||
|
||||
|
||||
def janitor(*args, env=None):
|
||||
"""Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт."""
|
||||
command = ["docker", "compose", "exec", "-T"]
|
||||
for name, value in (env or {}).items():
|
||||
command += ["-e", f"{name}={value}"]
|
||||
command += ["janitor", "python", "janitor.py", "--once", *map(str, args)]
|
||||
result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False)
|
||||
output = result.stdout.decode("utf-8", "replace").strip()
|
||||
check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}")
|
||||
return json.loads(output.splitlines()[-1])
|
||||
|
||||
|
||||
def sql_value(query):
|
||||
"""Одно значение из PostgreSQL (psql -A -t)."""
|
||||
code, output, error = psql(query)
|
||||
check(code == 0, f"psql failed: {error[-300:]}")
|
||||
return output
|
||||
|
||||
|
||||
def run_tests(tests):
|
||||
"""Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал."""
|
||||
failed = 0
|
||||
for test in tests:
|
||||
try:
|
||||
test()
|
||||
except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста
|
||||
failed += 1
|
||||
print(f"FAIL: {test.__name__}: {exc}")
|
||||
print(f"{len(tests) - failed} passed, {failed} failed")
|
||||
raise SystemExit(1 if failed else 0)
|
||||
@@ -0,0 +1,16 @@
|
||||
|
||||
from common import call, compose_output, new_user
|
||||
|
||||
|
||||
def main():
|
||||
user = new_user("demo")
|
||||
print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:")
|
||||
for number in range(1, 9):
|
||||
reply = call("GET", "/api/v1/users/me", 200, token=user["token"])
|
||||
print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}")
|
||||
print()
|
||||
print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,28 @@
|
||||
|
||||
import subprocess
|
||||
|
||||
from common import ROOT
|
||||
|
||||
OUTPUT = ROOT / "evidence" / "explain-history.txt"
|
||||
|
||||
|
||||
def main():
|
||||
sql = (ROOT / "student-tests" / "explain.sql").read_bytes()
|
||||
result = subprocess.run(
|
||||
["docker", "compose", "exec", "-T", "postgres",
|
||||
"psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"],
|
||||
cwd=ROOT, input=sql, capture_output=True, check=False,
|
||||
)
|
||||
OUTPUT.parent.mkdir(exist_ok=True)
|
||||
OUTPUT.write_bytes(result.stdout + result.stderr)
|
||||
if result.returncode != 0:
|
||||
print(result.stderr.decode("utf-8", "replace"))
|
||||
raise SystemExit("FAIL: psql returned an error, see output above")
|
||||
for line in result.stdout.decode("utf-8", "replace").splitlines():
|
||||
if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line:
|
||||
print(line)
|
||||
print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,76 @@
|
||||
|
||||
\pset footer off
|
||||
BEGIN;
|
||||
|
||||
CREATE TEMP TABLE seed_chats AS
|
||||
SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size
|
||||
FROM generate_series(0, 50) AS n;
|
||||
CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id;
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash)
|
||||
SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash'
|
||||
FROM seed_user;
|
||||
INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats;
|
||||
INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u;
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'),
|
||||
timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second'
|
||||
FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g;
|
||||
ANALYZE messages;
|
||||
|
||||
SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset
|
||||
SELECT created_at AS cursor_time, id AS cursor_id FROM messages
|
||||
WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
|
||||
|
||||
\echo '=== Data volume'
|
||||
SELECT count(*) AS messages_total,
|
||||
count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat
|
||||
FROM messages;
|
||||
|
||||
\echo ''
|
||||
\echo '=== History index'
|
||||
SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history';
|
||||
|
||||
\echo ''
|
||||
\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat'
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at,
|
||||
ARRAY(SELECT a.attachment_id FROM message_attachments a
|
||||
WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids
|
||||
FROM messages m
|
||||
WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY m.created_at, m.id LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)'
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat'
|
||||
ORDER BY created_at, id OFFSET 10000 LIMIT 51;
|
||||
|
||||
\echo ''
|
||||
\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)'
|
||||
DROP INDEX messages_history;
|
||||
EXPLAIN (ANALYZE, BUFFERS, COSTS OFF)
|
||||
SELECT id, chat_id, sender_id, text, created_at FROM messages
|
||||
WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid)
|
||||
ORDER BY created_at, id LIMIT 51;
|
||||
|
||||
ROLLBACK;
|
||||
\echo ''
|
||||
\echo '=== ROLLBACK: test data removed, index is back'
|
||||
SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%';
|
||||
@@ -0,0 +1,129 @@
|
||||
import json
|
||||
import platform
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced
|
||||
from loadgen import Load, summary
|
||||
|
||||
WARMUP_SECONDS = 3
|
||||
AFTER_SECONDS = 10
|
||||
OUTPUT = ROOT / "evidence" / "failover-metrics.json"
|
||||
|
||||
|
||||
class LbWatch:
|
||||
"""Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP)."""
|
||||
|
||||
def __init__(self):
|
||||
self.points, self.stopping = [], threading.Event()
|
||||
self.thread = threading.Thread(target=self._run, daemon=True)
|
||||
|
||||
def _run(self):
|
||||
while not self.stopping.is_set():
|
||||
try:
|
||||
self.points.append((time.monotonic(), lb_up_count()))
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
|
||||
def start(self):
|
||||
self.thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.stopping.set()
|
||||
self.thread.join(timeout=3)
|
||||
|
||||
def first_drop(self, after, replicas):
|
||||
return next((t for t, up in self.points if t >= after and up < replicas), None)
|
||||
|
||||
|
||||
def measure(container, mode, replicas):
|
||||
wait_balanced(replicas)
|
||||
load, watch = Load(), LbWatch()
|
||||
watch.start()
|
||||
load.start()
|
||||
time.sleep(WARMUP_SECONDS)
|
||||
t0 = time.monotonic()
|
||||
if mode == "stop":
|
||||
docker("stop", "--time", "8", container)
|
||||
else:
|
||||
docker("kill", container)
|
||||
command_seconds = time.monotonic() - t0
|
||||
time.sleep(max(0.0, AFTER_SECONDS - command_seconds))
|
||||
end = t0 + AFTER_SECONDS
|
||||
load.stop()
|
||||
watch.stop()
|
||||
|
||||
errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]]
|
||||
errors_end = max((e[0] + e[5] for e in errors), default=None)
|
||||
drop = watch.first_drop(t0, replicas)
|
||||
served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]]
|
||||
target_hits = sum(1 for instance in served_after if instance and container.startswith(instance))
|
||||
integrity = load.verify()
|
||||
|
||||
started = time.monotonic()
|
||||
docker("start", container)
|
||||
wait_balanced(replicas)
|
||||
rejoin_seconds = time.monotonic() - started
|
||||
|
||||
return {
|
||||
"replica": container[:12],
|
||||
"mode": mode,
|
||||
"command_seconds": round(command_seconds, 2),
|
||||
"lb_excluded_after_seconds": round(drop - t0, 2) if drop else None,
|
||||
"errors": len(errors),
|
||||
"error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"],
|
||||
"errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0,
|
||||
"before": summary(load, t0 - WARMUP_SECONDS, t0),
|
||||
"after": summary(load, t0, end),
|
||||
"stopped_replica_answers_after_stop": target_hits,
|
||||
"integrity": integrity,
|
||||
"rejoin_seconds": round(rejoin_seconds, 2),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
modes = sys.argv[1:] or ["stop", "kill"]
|
||||
check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]")
|
||||
containers = api_containers()
|
||||
replicas = len(containers)
|
||||
check(replicas >= 2, "need at least 2 running api replicas")
|
||||
print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, "
|
||||
f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and "
|
||||
f"{AFTER_SECONDS} s after the failure")
|
||||
runs = []
|
||||
for container in containers:
|
||||
for mode in modes:
|
||||
print(f"--- {mode} {container[:12]} ...")
|
||||
run = measure(container, mode, replicas)
|
||||
runs.append(run)
|
||||
print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after "
|
||||
f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, "
|
||||
f"stopped after {run['errors_stopped_after_seconds']} s")
|
||||
print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; "
|
||||
f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; "
|
||||
f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}")
|
||||
print(f" confirmed {run['integrity']['confirmed_writes']}, in history "
|
||||
f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; "
|
||||
f"back in the balancer {run['rejoin_seconds']} s after docker start")
|
||||
check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered")
|
||||
check(run["after"]["write_rps"] > 0, "no successful writes after the failure")
|
||||
print()
|
||||
print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s")
|
||||
for r in runs:
|
||||
print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} "
|
||||
f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} "
|
||||
f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}")
|
||||
OUTPUT.parent.mkdir(exist_ok=True)
|
||||
OUTPUT.write_text(json.dumps({
|
||||
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS,
|
||||
"after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()},
|
||||
"runs": runs,
|
||||
}, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; "
|
||||
f"saved {OUTPUT.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.5 KiB |
@@ -0,0 +1,93 @@
|
||||
"""Синтетические изображения для тестов вложений: только стандартная библиотека.
|
||||
|
||||
PNG собирается вручную (сигнатура, IHDR, IDAT, IEND), поэтому можно получить любой размер
|
||||
в пикселях и байтах, а также испортить конкретный байт. JPEG берётся из fixtures/.
|
||||
"""
|
||||
import os
|
||||
import struct
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parent / "fixtures"
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
MAX_UPLOAD_BYTES = 10 * 1024 * 1024
|
||||
MAX_PIXELS = 40_000_000
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
|
||||
|
||||
def chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes:
|
||||
"""PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой).
|
||||
|
||||
pad_to — дополнить файл текстовым чанком tEXt после IDAT до точного размера в байтах.
|
||||
idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое).
|
||||
"""
|
||||
channels = 1 if gray else 3
|
||||
header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace)
|
||||
if idat is None:
|
||||
compressor = zlib.compressobj(6)
|
||||
parts = []
|
||||
# Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход.
|
||||
passes = [(width, height)] if not interlace else [
|
||||
((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7]
|
||||
for pass_width, pass_height in passes:
|
||||
if pass_width <= 0 or pass_height <= 0:
|
||||
continue
|
||||
row = b"\x00" + bytes(pass_width * channels)
|
||||
for _ in range(pass_height):
|
||||
parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row))
|
||||
parts.append(compressor.flush())
|
||||
idat = b"".join(parts)
|
||||
body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat)
|
||||
if pad_to is not None:
|
||||
need = pad_to - len(body) - 12 - 12 - 8 # заголовок чанка-заполнителя, IEND, ключ "Comment\0"
|
||||
assert need >= 0, "image is already larger than pad_to"
|
||||
body += chunk(b"tEXt", b"Comment\x00" + b"a" * need)
|
||||
return body + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def noise_png(target_bytes: int) -> bytes:
|
||||
"""Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями)."""
|
||||
side = int(((target_bytes - 4096) / 3) ** 0.5)
|
||||
return png(side, side, noise=True)
|
||||
|
||||
|
||||
def sample_png() -> bytes:
|
||||
return (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def sample_jpeg() -> bytes:
|
||||
return (FIXTURES / "sample.jpg").read_bytes()
|
||||
|
||||
|
||||
def progressive_jpeg() -> bytes:
|
||||
return (FIXTURES / "progressive.jpg").read_bytes()
|
||||
|
||||
|
||||
def flip_byte(data: bytes, position: int) -> bytes:
|
||||
return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:]
|
||||
|
||||
|
||||
GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;"
|
||||
|
||||
|
||||
# --- сборка PNG из произвольных чанков (для проверок смысла чанков при верных CRC) ---
|
||||
|
||||
def ihdr(width, height, depth=8, color=2, interlace=0):
|
||||
return b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace)
|
||||
|
||||
|
||||
def zero_idat(width, height, depth=8, channels=3):
|
||||
"""Сжатые пиксели: каждая строка = байт фильтра 0 + нули (индекс палитры 0 / чёрный)."""
|
||||
row_bytes = (width * depth * channels + 7) // 8
|
||||
return b"IDAT", zlib.compress((b"\x00" + bytes(row_bytes)) * height)
|
||||
|
||||
|
||||
def assemble(*chunks):
|
||||
"""PNG из чанков (kind, data) с ВЕРНЫМИ CRC; IEND добавляется, если не передан."""
|
||||
if not chunks or chunks[-1][0] != b"IEND":
|
||||
chunks = chunks + ((b"IEND", b""),)
|
||||
return b"\x89PNG\r\n\x1a\n" + b"".join(chunk(kind, data) for kind, data in chunks)
|
||||
@@ -0,0 +1,102 @@
|
||||
import math
|
||||
import threading
|
||||
import time
|
||||
|
||||
from common import check, new_chat, new_key, new_user, read_history, try_call
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, writers=4, timeout=2):
|
||||
self.alice, self.bob = new_user("load_a"), new_user("load_b")
|
||||
self.chat = new_chat(self.alice, self.bob, title="load")
|
||||
self.writers, self.timeout = writers, timeout
|
||||
self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с)
|
||||
self.confirmed = [] # сообщения, на которые пришёл 201
|
||||
self.lock = threading.Lock()
|
||||
self.stopping = threading.Event()
|
||||
self.threads = []
|
||||
|
||||
def _record(self, kind, started, reply, ok):
|
||||
code = reply.status if reply is not None else "conn"
|
||||
instance = reply.instance if reply is not None else None
|
||||
with self.lock:
|
||||
self.events.append((started, kind, ok, code, instance, time.monotonic() - started))
|
||||
|
||||
def _writer(self, number):
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages"
|
||||
sent = 0
|
||||
key, text = new_key(), f"load-{number}-{sent}"
|
||||
while not self.stopping.is_set():
|
||||
started = time.monotonic()
|
||||
reply = try_call("POST", path, body={"text": text}, token=self.alice["token"],
|
||||
headers={"Idempotency-Key": key}, timeout=self.timeout)
|
||||
ok = reply is not None and reply.status == 201
|
||||
self._record("write", started, reply, ok)
|
||||
if ok:
|
||||
with self.lock:
|
||||
self.confirmed.append(reply.json)
|
||||
sent += 1
|
||||
key, text = new_key(), f"load-{number}-{sent}"
|
||||
else:
|
||||
time.sleep(0.05) # и повторить тот же text с тем же ключом
|
||||
|
||||
def _reader(self):
|
||||
path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20"
|
||||
while not self.stopping.is_set():
|
||||
started = time.monotonic()
|
||||
reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout)
|
||||
self._record("read", started, reply, reply is not None and reply.status == 200)
|
||||
time.sleep(0.05)
|
||||
|
||||
def start(self):
|
||||
self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)]
|
||||
self.threads.append(threading.Thread(target=self._reader, daemon=True))
|
||||
for thread in self.threads:
|
||||
thread.start()
|
||||
|
||||
def stop(self):
|
||||
self.stopping.set()
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=self.timeout + 5)
|
||||
|
||||
def window(self, start, end, kind=None):
|
||||
with self.lock:
|
||||
return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)]
|
||||
|
||||
def verify(self):
|
||||
"""Все подтверждённые сообщения есть в истории, id и тексты не повторяются."""
|
||||
history, _ = read_history(self.bob, self.chat["id"])
|
||||
ids = [message["id"] for message in history]
|
||||
texts = [message["text"] for message in history]
|
||||
by_id = {message["id"]: message for message in history}
|
||||
check(len(ids) == len(set(ids)), "duplicated message ids in history")
|
||||
check(len(texts) == len(set(texts)), "a retried request created a duplicate message")
|
||||
lost = [m for m in self.confirmed if by_id.get(m["id"]) != m]
|
||||
check(not lost, f"{len(lost)} confirmed messages are lost or changed")
|
||||
return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0}
|
||||
|
||||
|
||||
def summary(load, start, end):
|
||||
"""Успешные RPS, ошибки и задержки на отрезке [start, end)."""
|
||||
events = load.window(start, end)
|
||||
writes = [e for e in events if e[1] == "write"]
|
||||
ok_writes = sorted(e[5] for e in writes if e[2])
|
||||
errors = [e for e in events if not e[2]]
|
||||
seconds = max(end - start, 1e-6)
|
||||
|
||||
def percentile(p):
|
||||
return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None
|
||||
|
||||
codes = {}
|
||||
for e in errors:
|
||||
codes[str(e[3])] = codes.get(str(e[3]), 0) + 1
|
||||
return {
|
||||
"seconds": round(seconds, 2),
|
||||
"write_rps": round(len(ok_writes) / seconds, 1),
|
||||
"reads": len([e for e in events if e[1] == "read"]),
|
||||
"errors": len(errors),
|
||||
"error_codes": codes,
|
||||
"write_p50_ms": percentile(0.50),
|
||||
"write_p95_ms": percentile(0.95),
|
||||
"write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None,
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import subprocess
|
||||
|
||||
from common import ROOT, login, new_user, redis_cli, session_key
|
||||
|
||||
|
||||
def without_password():
|
||||
result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"],
|
||||
cwd=ROOT, capture_output=True, check=False)
|
||||
return (result.stdout + result.stderr).decode("utf-8", "replace").strip()
|
||||
|
||||
|
||||
def main():
|
||||
user = new_user("info")
|
||||
token = login(user["username"], user["password"]).json["token"]
|
||||
key = session_key(token)
|
||||
print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}")
|
||||
print(f"TTL of the key, seconds: {redis_cli('ttl', key)}")
|
||||
keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines()
|
||||
print(f"sessions in Redis: {len(keys)}")
|
||||
for name in ("appendonly", "appendfsync", "maxmemory-policy"):
|
||||
print("config", " = ".join(redis_cli("config", "get", name).split()))
|
||||
for line in redis_cli("info", "persistence").splitlines():
|
||||
if line.startswith(("aof_enabled", "aof_last_write_status")):
|
||||
print(line)
|
||||
print(f"without password: {without_password()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,88 @@
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced
|
||||
from loadgen import Load, summary
|
||||
|
||||
OUTPUT = ROOT / "evidence" / "rolling-update.json"
|
||||
|
||||
|
||||
def env_of(container):
|
||||
lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines()
|
||||
return dict(line.split("=", 1) for line in lines if "=" in line)
|
||||
|
||||
|
||||
def wait_new_replica(container, up_needed, seconds=90):
|
||||
deadline = time.monotonic() + seconds
|
||||
while health(container) != "healthy" or lb_up_count() < up_needed:
|
||||
check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s")
|
||||
time.sleep(0.2)
|
||||
|
||||
|
||||
def main():
|
||||
old = api_containers()
|
||||
replicas = len(old)
|
||||
check(replicas >= 2, "need at least 2 running api replicas")
|
||||
wait_balanced(replicas)
|
||||
old_version = env_of(old[0]).get("APP_VERSION")
|
||||
version = time.strftime("rolling-%Y%m%d-%H%M%S")
|
||||
env = dict(os.environ, APP_VERSION=version)
|
||||
|
||||
print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}")
|
||||
started = time.monotonic()
|
||||
compose("build", "api", "migrate", env=env)
|
||||
print(f"image built in {time.monotonic() - started:.1f} s")
|
||||
migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env)
|
||||
print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1]
|
||||
.split("messenger.db: ")[-1])
|
||||
|
||||
load = Load()
|
||||
load.start()
|
||||
time.sleep(3)
|
||||
t0 = time.monotonic()
|
||||
steps = []
|
||||
for container in old:
|
||||
step_start = time.monotonic()
|
||||
before = set(api_containers())
|
||||
compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env)
|
||||
new = sorted(set(api_containers()) - before)
|
||||
check(len(new) == 1, f"expected one new container, got {len(new)}")
|
||||
wait_new_replica(new[0], replicas + 1)
|
||||
ready = time.monotonic() - step_start
|
||||
docker("stop", "--time", "15", container)
|
||||
docker("rm", container)
|
||||
steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2),
|
||||
"step_seconds": round(time.monotonic() - step_start, 2)})
|
||||
print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, "
|
||||
f"step {steps[-1]['step_seconds']} s")
|
||||
t1 = time.monotonic()
|
||||
time.sleep(3)
|
||||
load.stop()
|
||||
|
||||
final = api_containers()
|
||||
versions = sorted({env_of(c).get("APP_VERSION") for c in final})
|
||||
integrity = load.verify()
|
||||
during = summary(load, t0, t1)
|
||||
before_stats = summary(load, t0 - 3, t0)
|
||||
print(f"now: replicas {[c[:12] for c in final]}, versions {versions}")
|
||||
print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; "
|
||||
f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, "
|
||||
f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s")
|
||||
print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, "
|
||||
f"lost 0, duplicates 0")
|
||||
check(len(final) == replicas and not set(final) & set(old), "old replicas are still running")
|
||||
check(versions == [version], "not every replica runs the new version")
|
||||
OUTPUT.parent.mkdir(exist_ok=True)
|
||||
OUTPUT.write_text(json.dumps({
|
||||
"from_version": old_version, "to_version": version, "replicas": replicas,
|
||||
"profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2},
|
||||
"steps": steps, "update_seconds": round(t1 - t0, 2),
|
||||
"before": before_stats, "during": during, "integrity": integrity,
|
||||
}, indent=2) + "\n", encoding="utf-8")
|
||||
print(f"PASS: all {replicas} replicas replaced one by one under load, "
|
||||
f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced
|
||||
|
||||
HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh")
|
||||
|
||||
|
||||
def ps():
|
||||
return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()
|
||||
|
||||
|
||||
def aim(wanted):
|
||||
"""Для 2 реплик: последним перед тестом должна ответить реплика из wanted."""
|
||||
probe = call("GET", "/health/live", 200).instance
|
||||
if not any(container.startswith(probe) for container in wanted):
|
||||
call("GET", "/health/live", 200) # теперь последней ответила другая реплика
|
||||
|
||||
|
||||
def main():
|
||||
containers = api_containers()
|
||||
check(len(containers) >= 2, "need at least 2 running api replicas")
|
||||
print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}")
|
||||
stopped_once = set()
|
||||
for attempt in range(1, 7):
|
||||
wait_balanced(len(containers))
|
||||
if len(containers) == 2:
|
||||
aim(set(containers) - stopped_once)
|
||||
print(f"=== run {attempt}: api containers before")
|
||||
print(ps())
|
||||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
|
||||
result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover",
|
||||
"--action-script", str(HOOK), "--base-url", BASE_URL],
|
||||
cwd=ROOT, capture_output=True, env=env)
|
||||
print((result.stdout + result.stderr).decode("utf-8", "replace").strip())
|
||||
check(result.returncode == 0, "public failover test failed")
|
||||
print("api containers after")
|
||||
print(ps())
|
||||
stopped = [container for container in containers if container not in api_containers()]
|
||||
check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}")
|
||||
print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); "
|
||||
f"other api containers kept running, postgres/redis/lb were not touched")
|
||||
stopped_once.add(stopped[0])
|
||||
docker("start", stopped[0])
|
||||
wait_balanced(len(containers))
|
||||
print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy")
|
||||
if stopped_once == set(containers):
|
||||
break
|
||||
check(stopped_once == set(containers), "not every replica was stopped by the public test")
|
||||
print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from common import BASE_URL, ROOT, check, compose_output, docker
|
||||
|
||||
HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh")
|
||||
|
||||
|
||||
def started():
|
||||
containers = compose_output("ps", "-q").split()
|
||||
lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines()
|
||||
return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)}
|
||||
|
||||
|
||||
def main():
|
||||
before = started()
|
||||
print("before: container, ID, started at")
|
||||
for name, (cid, at) in sorted(before.items()):
|
||||
print(f" {name:<28} {cid[:12]} {at}")
|
||||
env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8")
|
||||
begin = time.monotonic()
|
||||
result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence",
|
||||
"--action-script", str(HOOK), "--base-url", BASE_URL],
|
||||
cwd=ROOT, capture_output=True, env=env)
|
||||
seconds = time.monotonic() - begin
|
||||
print((result.stdout + result.stderr).decode("utf-8", "replace").strip())
|
||||
check(result.returncode == 0, "public persistence test failed")
|
||||
after = started()
|
||||
print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?")
|
||||
restarted = []
|
||||
for name, (cid, at) in sorted(after.items()):
|
||||
same_id = before.get(name, ("", ""))[0] == cid
|
||||
new_start = before.get(name, ("", ""))[1] != at
|
||||
print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}")
|
||||
check(same_id, f"{name}: container was recreated, volumes/IDs must stay")
|
||||
if new_start:
|
||||
restarted.append(name.split("-")[-2])
|
||||
for service in ("postgres", "redis", "s3", "rabbitmq", "api", "janitor", "dispatcher", "worker", "lb"):
|
||||
check(service in restarted, f"{service} was not restarted by the hook")
|
||||
print("PASS: the hook restarted postgres, redis, s3 (SeaweedFS), rabbitmq, all api replicas, janitor, dispatcher, worker and lb (same containers, "
|
||||
"new start time); data, the attachment bytes and the old token survived")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,258 @@
|
||||
import base64
|
||||
import datetime as dt
|
||||
import uuid
|
||||
|
||||
from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send
|
||||
|
||||
|
||||
def parse_time(value):
|
||||
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
|
||||
def test_duplicate_username():
|
||||
body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD}
|
||||
call("POST", "/api/v1/users", 201, body=body)
|
||||
call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user"))
|
||||
print("PASS: duplicate username returns 409")
|
||||
|
||||
|
||||
def test_invalid_json():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
call("POST", "/api/v1/users", 400, raw=b'{"username":')
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json",
|
||||
headers={"Idempotency-Key": new_key()})
|
||||
print("PASS: invalid JSON returns 400")
|
||||
|
||||
|
||||
def test_wrong_fields():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD})
|
||||
call("POST", "/api/v1/users", 400,
|
||||
body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True})
|
||||
# Отправителя нельзя подставить: он берётся из сессии.
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"],
|
||||
body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()})
|
||||
print("PASS: wrong and extra fields return 400")
|
||||
|
||||
|
||||
def test_foreign_chat():
|
||||
alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve")
|
||||
chat = new_chat(alice, bob)
|
||||
call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"])
|
||||
call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"])
|
||||
send(eve, chat["id"], "not my chat", expected=404)
|
||||
# Несуществующий чат выглядит так же, как чужой.
|
||||
call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"])
|
||||
print("PASS: foreign chat returns 404")
|
||||
|
||||
|
||||
def test_empty_message():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
for text in ("", " ", "\n\t"):
|
||||
send(alice, chat["id"], text, expected=400)
|
||||
send(alice, chat["id"], "я" * 4001, expected=400)
|
||||
send(alice, chat["id"], "я" * 4000)
|
||||
print("PASS: empty and too long messages return 400, 4000 chars is ok")
|
||||
|
||||
|
||||
def test_pagination():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob)
|
||||
sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)]
|
||||
|
||||
history, pages = read_history(bob, chat["id"], limit=2)
|
||||
check(pages == 4, f"expected 4 pages, got {pages}")
|
||||
check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages")
|
||||
|
||||
other = new_chat(alice)
|
||||
for i in range(3):
|
||||
send(alice, other["id"], f"Other {i}")
|
||||
first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json
|
||||
call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"])
|
||||
call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"])
|
||||
print("PASS: pagination works across multiple pages, foreign cursor returns 400")
|
||||
|
||||
|
||||
def test_empty_cursor():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
send(alice, chat["id"], "one message")
|
||||
for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"):
|
||||
for query in ("?cursor=", "?cursor", "?limit=10&cursor="):
|
||||
call("GET", path + query, 400, token=alice["token"])
|
||||
call("GET", path, 200, token=alice["token"])
|
||||
print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page")
|
||||
|
||||
|
||||
def test_chat_list_pagination():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)]
|
||||
seen, cursor = [], None
|
||||
while True:
|
||||
query = "?limit=1" + (f"&cursor={cursor}" if cursor else "")
|
||||
page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json
|
||||
seen += [chat["id"] for chat in page["items"]]
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
check(seen == created, "chat list pages are wrong")
|
||||
print("PASS: chat list pagination")
|
||||
|
||||
|
||||
def test_unicode():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
chat = new_chat(alice, bob, title="Чат 👋")
|
||||
text = "Привет 👋 你好"
|
||||
check(send(alice, chat["id"], text).json["text"] == text, "text changed on write")
|
||||
history, _ = read_history(bob, chat["id"])
|
||||
check(history[-1]["text"] == text, "text changed on read")
|
||||
print("PASS: unicode text is stored as is")
|
||||
|
||||
|
||||
def test_nul_character_is_stored():
|
||||
name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world"
|
||||
body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"}
|
||||
user = call("POST", "/api/v1/users", 201, body=body).json
|
||||
check(user["display_name"] == name, "display_name with U+0000 changed")
|
||||
token = login(body["username"], body["password"]).json["token"]
|
||||
check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read")
|
||||
chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json
|
||||
check(chat["title"] == title, "title with U+0000 changed")
|
||||
check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read")
|
||||
sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text},
|
||||
headers={"Idempotency-Key": new_key()}).json
|
||||
history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"]
|
||||
check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed")
|
||||
login("nul\u0000user", PASSWORD, expected=401)
|
||||
print("PASS: strings with U+0000 are stored and returned as is")
|
||||
|
||||
|
||||
|
||||
def test_user_has_no_password():
|
||||
body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD}
|
||||
user = call("POST", "/api/v1/users", 201, body=body).json
|
||||
check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}")
|
||||
print("PASS: user response has no password or hash")
|
||||
|
||||
|
||||
def test_password_rules():
|
||||
def register(password, expected):
|
||||
body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password}
|
||||
call("POST", "/api/v1/users", expected, body=body)
|
||||
|
||||
register("x" * 11, 400)
|
||||
register("x" * 129, 400)
|
||||
register(" " * 12, 400)
|
||||
register("x" * 12, 201)
|
||||
register("я" * 128, 201)
|
||||
print("PASS: password length 12..128, blank password is rejected")
|
||||
|
||||
|
||||
def test_login_gives_token():
|
||||
username = "lg_" + uuid.uuid4().hex[:10]
|
||||
user = call("POST", "/api/v1/users", 201,
|
||||
body={"username": username, "display_name": "Login", "password": PASSWORD}).json
|
||||
reply = login(username, PASSWORD)
|
||||
session = reply.json
|
||||
check(session["token_type"] == "Bearer", "token_type must be Bearer")
|
||||
check(session["user"] == user, "session user differs from registered user")
|
||||
check(len(session["token"]) >= 43, "token is too short")
|
||||
check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque")
|
||||
check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached")
|
||||
left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds()
|
||||
check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s")
|
||||
me = call("GET", "/api/v1/users/me", token=session["token"]).json
|
||||
check(me == user, "/users/me returned another user")
|
||||
print("PASS: Basic login returns opaque Bearer token with expiry")
|
||||
|
||||
|
||||
def test_wrong_password():
|
||||
user = new_user("wp")
|
||||
wrong = login(user["username"], "Wrong-password-000", expected=401)
|
||||
unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401)
|
||||
check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic")
|
||||
check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames")
|
||||
print("PASS: wrong password and unknown user both return the same 401")
|
||||
|
||||
|
||||
def test_malformed_basic():
|
||||
no_colon = base64.b64encode(b"just_username").decode()
|
||||
for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"):
|
||||
headers = {} if header is None else {"Authorization": header}
|
||||
call("POST", "/api/v1/auth/sessions", 401, headers=headers)
|
||||
print("PASS: malformed Basic header returns 401")
|
||||
|
||||
|
||||
def test_password_with_colon_and_unicode():
|
||||
password = "пароль:с:двоеточием 👋"
|
||||
user = new_user("colon", password=password)
|
||||
call("GET", "/api/v1/users/me", token=user["token"])
|
||||
print("PASS: password with colon and unicode works")
|
||||
|
||||
|
||||
def test_bearer_required():
|
||||
user = new_user("br")
|
||||
for headers in ({}, {"Authorization": basic(user["username"], user["password"])},
|
||||
{"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43},
|
||||
{"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}):
|
||||
reply = call("GET", "/api/v1/users/me", 401, headers=headers)
|
||||
check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer")
|
||||
call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]})
|
||||
print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401")
|
||||
|
||||
|
||||
def test_x_user_id_ignored():
|
||||
alice, bob = new_user("alice"), new_user("bob")
|
||||
me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json
|
||||
check(me["id"] == alice["id"], "X-User-Id changed the current user")
|
||||
print("PASS: X-User-Id does not change the current user")
|
||||
|
||||
|
||||
def test_auth_before_body():
|
||||
alice = new_user("alice")
|
||||
chat = new_chat(alice)
|
||||
call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json")
|
||||
call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401)
|
||||
print("PASS: 401 comes before 400 and 404")
|
||||
|
||||
|
||||
def test_logout():
|
||||
user = new_user("lo")
|
||||
second = login(user["username"], user["password"]).json["token"]
|
||||
call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"])
|
||||
call("GET", "/api/v1/users/me", 401, token=user["token"])
|
||||
call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"])
|
||||
call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива
|
||||
again = login(user["username"], user["password"]).json["token"]
|
||||
check(again != user["token"], "new login must give a new token")
|
||||
call("GET", "/api/v1/users/me", 200, token=again)
|
||||
print("PASS: logout revokes only the current session")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([
|
||||
test_duplicate_username,
|
||||
test_invalid_json,
|
||||
test_wrong_fields,
|
||||
test_foreign_chat,
|
||||
test_empty_message,
|
||||
test_pagination,
|
||||
test_empty_cursor,
|
||||
test_chat_list_pagination,
|
||||
test_unicode,
|
||||
test_nul_character_is_stored,
|
||||
test_user_has_no_password,
|
||||
test_password_rules,
|
||||
test_login_gives_token,
|
||||
test_wrong_password,
|
||||
test_malformed_basic,
|
||||
test_password_with_colon_and_unicode,
|
||||
test_bearer_required,
|
||||
test_x_user_id_ignored,
|
||||
test_auth_before_body,
|
||||
test_logout,
|
||||
])
|
||||
@@ -0,0 +1,338 @@
|
||||
"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ.
|
||||
|
||||
Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа:
|
||||
пользовательская ошибка не должна становиться 500.
|
||||
"""
|
||||
import hashlib
|
||||
import http.client
|
||||
import socket
|
||||
import struct
|
||||
import urllib.parse
|
||||
import uuid
|
||||
import zlib
|
||||
|
||||
import images
|
||||
from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload
|
||||
|
||||
URL = urllib.parse.urlsplit(BASE_URL)
|
||||
|
||||
|
||||
def setup():
|
||||
alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve")
|
||||
return alice, bob, eve, new_chat(alice, bob)
|
||||
|
||||
|
||||
def sha(data):
|
||||
return hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def check_error(reply, code):
|
||||
check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}")
|
||||
|
||||
|
||||
def test_png_and_jpeg_roundtrip():
|
||||
alice, bob, _, chat = setup()
|
||||
for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"),
|
||||
(images.sample_jpeg(), "image/jpeg", "sample.jpg"),
|
||||
(images.progressive_jpeg(), "image/jpeg", "progressive.jpg"),
|
||||
(images.png(33, 17, interlace=1), "image/png", "interlaced.png"),
|
||||
(images.png(1, 1, gray=True), "image/png", "1x1.png")):
|
||||
att = upload(alice, chat["id"], data, ctype, name).json
|
||||
check(att["status"] == "ready" and att["error_code"] is None, "not ready")
|
||||
check([v["name"] for v in att["variants"]] == ["thumbnail", "crop", "optimized"], "lab5 variants missing")
|
||||
check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ")
|
||||
check(att["content_type"] == ctype and att["filename"] == name, "metadata differs")
|
||||
meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json
|
||||
check(meta == att, "GET metadata differs from POST response")
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
check(got.data == data, "downloaded bytes differ")
|
||||
check(got.headers["content-type"] == ctype, "wrong Content-Type on download")
|
||||
check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length")
|
||||
check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing")
|
||||
print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes")
|
||||
print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte")
|
||||
|
||||
|
||||
def test_size_limit_exact_boundary():
|
||||
alice, _, _, chat = setup()
|
||||
exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES)
|
||||
check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB")
|
||||
att = upload(alice, chat["id"], exact).json
|
||||
check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed")
|
||||
print(f"exactly {len(exact)} bytes (10 MiB) -> 202, then ready")
|
||||
over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1)
|
||||
reply = upload(alice, chat["id"], over, expected=413)
|
||||
check_error(reply, "payload_too_large")
|
||||
print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body")
|
||||
print("PASS: the 10 MiB limit is exact")
|
||||
|
||||
|
||||
def chunked_upload(user, chat_id, total, piece=64 * 1024):
|
||||
"""Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении."""
|
||||
conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60)
|
||||
|
||||
def body():
|
||||
sent = 0
|
||||
head = images.sample_png()[:64]
|
||||
yield head
|
||||
sent += len(head)
|
||||
while sent < total:
|
||||
part = bytes(min(piece, total - sent))
|
||||
sent += len(part)
|
||||
yield part
|
||||
try:
|
||||
conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(),
|
||||
headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]},
|
||||
encode_chunked=True)
|
||||
reply = conn.getresponse()
|
||||
return reply.status, reply.read()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def test_size_limit_streaming():
|
||||
alice, _, _, chat = setup()
|
||||
status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024)
|
||||
check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}")
|
||||
print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed")
|
||||
call("GET", "/health/ready", 200)
|
||||
print("PASS: oversized streamed body is cut off, the API keeps working")
|
||||
|
||||
|
||||
def test_pixel_limit():
|
||||
alice, _, _, chat = setup()
|
||||
at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей
|
||||
att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json
|
||||
print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 202, ready {att['id'][:8]}")
|
||||
over = images.png(8001, 5000, gray=True)
|
||||
check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large")
|
||||
print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413")
|
||||
bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01")
|
||||
check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large")
|
||||
print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded")
|
||||
print("PASS: pixel limit is checked before decompression")
|
||||
|
||||
|
||||
def test_damaged_images():
|
||||
alice, _, _, chat = setup()
|
||||
png, jpeg = images.sample_png(), images.sample_jpeg()
|
||||
cases = [
|
||||
("PNG cut in half", png[:len(png) // 2], "image/png"),
|
||||
("PNG without IEND", png[:-12], "image/png"),
|
||||
("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"),
|
||||
("PNG with garbage after IEND", png + b"garbage", "image/png"),
|
||||
("PNG signature only", png[:8], "image/png"),
|
||||
("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"),
|
||||
("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"),
|
||||
("JPEG header only", jpeg[:20], "image/jpeg"),
|
||||
("random bytes", uuid.uuid4().bytes * 100, "image/png"),
|
||||
("plain text", "это не картинка".encode(), "image/jpeg"),
|
||||
]
|
||||
for name, data, ctype in cases:
|
||||
reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415)
|
||||
check_error(reply, "unsupported_media_type")
|
||||
print(f"{name} ({len(data)} bytes, {ctype}) -> 415")
|
||||
check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request")
|
||||
print("empty body -> 400")
|
||||
print("PASS: damaged and foreign files are rejected with 415, never 500")
|
||||
|
||||
|
||||
def test_png_chunk_semantics():
|
||||
"""CRC у всех чанков верные, но содержимое нарушает спецификацию PNG -> 415, а не 201.
|
||||
|
||||
Регрессия: PNG 1x1, глубина 8, палитровый, PLTE из 771 байта (257 записей) раньше получал 201 ready,
|
||||
хотя Pillow его отклоняет (invalid palette size).
|
||||
"""
|
||||
alice, bob, _, chat = setup()
|
||||
from images import assemble, ihdr, zero_idat
|
||||
pal = lambda entries: (b"PLTE", bytes(3 * entries))
|
||||
idat_p = zero_idat(1, 1, 8, 1)
|
||||
bad = [
|
||||
("PLTE 771 bytes = 257 entries (review case)",
|
||||
assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(771)), (b"IDAT", zlib.compress(b"\0\0")))),
|
||||
("PLTE length not a multiple of 3", assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(4)), idat_p)),
|
||||
("empty PLTE", assemble(ihdr(1, 1, 8, 3), (b"PLTE", b""), idat_p)),
|
||||
("palette larger than bit depth allows (depth 1, 3 entries)",
|
||||
assemble(ihdr(1, 1, 1, 3), pal(3), zero_idat(1, 1, 1, 1))),
|
||||
("palette image without PLTE", assemble(ihdr(1, 1, 8, 3), idat_p)),
|
||||
("PLTE in a grayscale image", assemble(ihdr(1, 1, 8, 0), pal(2), zero_idat(1, 1, 8, 1))),
|
||||
("PLTE after IDAT", assemble(ihdr(1, 1, 8, 3), idat_p, pal(2))),
|
||||
("two PLTE chunks", assemble(ihdr(1, 1, 8, 3), pal(2), pal(2), idat_p)),
|
||||
("tRNS longer than the palette", assemble(ihdr(1, 1, 8, 3), pal(2), (b"tRNS", bytes(3)), idat_p)),
|
||||
("tRNS before PLTE", assemble(ihdr(1, 1, 8, 3), (b"tRNS", bytes(1)), pal(2), idat_p)),
|
||||
("tRNS of wrong size for RGB (4 bytes instead of 6)",
|
||||
assemble(ihdr(1, 1, 8, 2), (b"tRNS", bytes(4)), zero_idat(1, 1, 8, 3))),
|
||||
("tRNS in an image with alpha channel",
|
||||
assemble(ihdr(1, 1, 8, 6), (b"tRNS", bytes(2)), zero_idat(1, 1, 8, 4))),
|
||||
("IEND with data", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"IEND", b"x"))),
|
||||
("unknown critical chunk", assemble(ihdr(1, 1, 8, 2), (b"ABCD", b""), zero_idat(1, 1, 8, 3))),
|
||||
("IDAT chunks not consecutive",
|
||||
assemble(ihdr(1, 2, 8, 2), (b"IDAT", zero_idat(1, 2, 8, 3)[1][:5]), (b"tEXt", b"k\0v"),
|
||||
(b"IDAT", zero_idat(1, 2, 8, 3)[1][5:]))),
|
||||
("gAMA after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"gAMA", bytes(4)))),
|
||||
("gAMA of 2 bytes instead of 4", assemble(ihdr(1, 1, 8, 2), (b"gAMA", bytes(2)), zero_idat(1, 1, 8, 3))),
|
||||
("3 MiB metadata chunk before IDAT", assemble(ihdr(1, 1, 8, 2), (b"prVt", bytes(3 * 1024 * 1024)),
|
||||
zero_idat(1, 1, 8, 3))),
|
||||
("private chunk after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"prVt", bytes(100)))),
|
||||
("animated PNG frame (fdAT)", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"fdAT", bytes(10)))),
|
||||
("tIME of 2 bytes instead of 7", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tIME", bytes(2)))),
|
||||
("tEXt without keyword separator", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tEXt", b"nokey"))),
|
||||
("zTXt with broken zlib after IDAT",
|
||||
assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0garbage"))),
|
||||
("zTXt text bomb: 50 MB of text in ~50 KB after IDAT",
|
||||
assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0" + zlib.compress(bytes(50_000_000))))),
|
||||
]
|
||||
for name, data in bad:
|
||||
check_error(upload(alice, chat["id"], data, filename="bad.png", expected=415), "unsupported_media_type")
|
||||
print(f"{name} ({len(data)} bytes, valid CRCs) -> 415")
|
||||
good = [
|
||||
("palette 256 entries + tRNS + tEXt and tIME after IDAT",
|
||||
assemble(ihdr(2, 2, 8, 3), pal(256), (b"tRNS", bytes(256)), zero_idat(2, 2, 8, 1),
|
||||
(b"tEXt", b"Comment\0ok"), (b"tIME", struct.pack(">HBBBBB", 2026, 10, 22, 12, 0, 0)))),
|
||||
("palette depth 2 with 4 entries", assemble(ihdr(3, 1, 2, 3), pal(4), zero_idat(3, 1, 2, 1))),
|
||||
("RGB with 6-byte tRNS and gAMA", assemble(ihdr(1, 1, 8, 2), (b"gAMA", struct.pack(">I", 45455)),
|
||||
(b"tRNS", bytes(6)), zero_idat(1, 1, 8, 3))),
|
||||
("compressed zTXt (100 KB of text) and iTXt after IDAT",
|
||||
assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"Comment\0\0" + zlib.compress(b"a" * 100_000)),
|
||||
(b"iTXt", b"Title\0\1\0ru\0\0" + zlib.compress("Привет".encode())))),
|
||||
]
|
||||
for name, data in good:
|
||||
att = upload(alice, chat["id"], data, filename="good.png").json
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"])
|
||||
check(got.data == data, "valid PNG changed")
|
||||
print(f"{name} -> 201, downloaded byte for byte")
|
||||
print("PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted")
|
||||
|
||||
|
||||
def test_mime_substitution():
|
||||
alice, _, _, chat = setup()
|
||||
cases = [
|
||||
("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"),
|
||||
("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"),
|
||||
("GIF bytes sent as image/png", images.GIF, "image/png"),
|
||||
("PNG sent as image/gif", images.sample_png(), "image/gif"),
|
||||
("PNG sent as text/html", images.sample_png(), "text/html"),
|
||||
("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"),
|
||||
("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"),
|
||||
]
|
||||
for name, data, ctype in cases:
|
||||
check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type")
|
||||
print(f"{name} -> 415")
|
||||
att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json
|
||||
check(att["content_type"] == "image/png", "content type must be normalised")
|
||||
print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png")
|
||||
print("PASS: the type is taken from the content; a declared type that does not match is refused")
|
||||
|
||||
|
||||
def test_filenames():
|
||||
alice, _, _, chat = setup()
|
||||
bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png",
|
||||
"bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256]
|
||||
for name in bad:
|
||||
check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request")
|
||||
print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400")
|
||||
check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request")
|
||||
print("no filename -> 400")
|
||||
for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"):
|
||||
att = upload(alice, chat["id"], images.sample_png(), filename=name).json
|
||||
check(att["filename"] == name, "display name changed")
|
||||
got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"])
|
||||
check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded")
|
||||
print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only")
|
||||
print("PASS: filename never becomes a path; dangerous names are refused")
|
||||
|
||||
|
||||
def test_access_control():
|
||||
alice, bob, eve, chat = setup()
|
||||
att = upload(alice, chat["id"], images.sample_png()).json
|
||||
for suffix in ("", "/content"):
|
||||
reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"])
|
||||
check_error(reply, "not_found")
|
||||
call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401)
|
||||
call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"])
|
||||
call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"])
|
||||
print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400")
|
||||
check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found")
|
||||
check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found")
|
||||
upload(None, chat["id"], images.sample_png(), expected=401)
|
||||
print("upload into someone else's / unknown chat -> 404, without token -> 401")
|
||||
call("GET", f"/api/v1/attachments/{att['id']}/content?variant=preview", 400, token=bob["token"])
|
||||
for variant in ("original", "thumbnail", "crop", "optimized"):
|
||||
call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}", 200, token=bob["token"])
|
||||
print("variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)")
|
||||
print("PASS: only chat members can see an attachment")
|
||||
|
||||
|
||||
def test_message_attachments():
|
||||
alice, bob, eve, chat = setup()
|
||||
first = upload(alice, chat["id"], images.sample_png()).json
|
||||
second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json
|
||||
msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json
|
||||
check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed")
|
||||
print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member")
|
||||
plain = post_message(bob, chat["id"], "без вложений").json
|
||||
check(plain["attachment_ids"] == [], "default attachment_ids must be []")
|
||||
reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json
|
||||
history, _ = read_history(bob, chat["id"])
|
||||
check(history == [msg, plain, reused], "history differs from the confirmed messages")
|
||||
print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed")
|
||||
|
||||
other_chat = new_chat(alice, eve)
|
||||
foreign = upload(alice, other_chat["id"], images.sample_png()).json
|
||||
for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"),
|
||||
([first["id"], foreign["id"]], "own + foreign attachment"),
|
||||
([str(uuid.uuid4())], "unknown attachment_id")):
|
||||
check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found")
|
||||
print(f"{what} -> 404")
|
||||
check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found")
|
||||
print("outsider referencing an attachment of the chat -> 404")
|
||||
for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"),
|
||||
([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"),
|
||||
(["not-a-uuid"], "x", "bad UUID"),
|
||||
([], "", "empty text without attachments"),
|
||||
([], " ", "whitespace text without attachments")):
|
||||
check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request")
|
||||
print(f"{what} -> 400")
|
||||
history_after, _ = read_history(bob, chat["id"])
|
||||
check(history_after == history, "a rejected message was stored")
|
||||
print("PASS: only attachments of the same chat can be linked; rejected requests change nothing")
|
||||
|
||||
|
||||
def test_idempotency_with_attachments():
|
||||
alice, bob, _, chat = setup()
|
||||
att = upload(alice, chat["id"], images.sample_png()).json
|
||||
other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json
|
||||
key = new_key()
|
||||
first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json
|
||||
again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json
|
||||
check(again == first, "replay returned a different message")
|
||||
check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict")
|
||||
check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict")
|
||||
key2 = new_key()
|
||||
plain = post_message(alice, chat["id"], "текст", None, key=key2).json
|
||||
check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field")
|
||||
history, _ = read_history(bob, chat["id"])
|
||||
check(len(history) == 2, "replays created extra messages")
|
||||
print("same key + same attachments -> same 201; other attachments -> 409; missing field == []")
|
||||
print("PASS: Idempotency-Key covers attachment_ids")
|
||||
|
||||
|
||||
def test_aborted_upload():
|
||||
alice, _, _, chat = setup()
|
||||
sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10)
|
||||
head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n"
|
||||
f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n")
|
||||
sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000))
|
||||
sock.close() # клиент пропал на середине тела
|
||||
print("sent 1.1 MB of a declared 5 MB body and closed the connection")
|
||||
att = upload(alice, chat["id"], images.sample_png()).json
|
||||
call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"])
|
||||
print("the API keeps serving uploads; nothing of the aborted request is visible "
|
||||
"(DB/S3 side is checked by test_consistency.py)")
|
||||
print("PASS: an interrupted upload is dropped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming,
|
||||
test_pixel_limit, test_damaged_images, test_png_chunk_semantics, test_mime_substitution, test_filenames, test_access_control,
|
||||
test_message_attachments, test_idempotency_with_attachments, test_aborted_upload])
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user