forked from mtusi2026/mtusi2026lab5
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
43746f7a00 | ||
|
|
0a221b0675 |
+44
-1
@@ -7,7 +7,9 @@ SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
# postgres - sessions only in PostgreSQL
|
||||
# postgres+valkey - PostgreSQL is the source of truth, Valkey is a shared read-through session cache with TTL
|
||||
SESSION_STORE=postgres+valkey
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
@@ -23,3 +25,44 @@ WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
JOB_TIMEOUT_SECONDS=60
|
||||
STAGING_RETENTION_SECONDS=86400
|
||||
# --- solution-specific (lab3) ---
|
||||
# Max lifetime of a cached session copy (also the cache-distrust window after a Valkey error)
|
||||
SESSION_CACHE_TTL_SECONDS=60
|
||||
# On SIGTERM a replica keeps serving but reports not-ready this long, so the LB drains it
|
||||
SHUTDOWN_DRAIN_MS=3000
|
||||
# Bounded pool and timeouts, per replica
|
||||
DB_POOL_MAX=10
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
DB_ACQUIRE_TIMEOUT_MS=2000
|
||||
CACHE_TIMEOUT_MS=300
|
||||
# Parallel Argon2id computations per replica (each uses 64 MiB)
|
||||
PASSWORD_HASH_WORKERS=2
|
||||
# --- solution-specific (lab4) ---
|
||||
# Deadline of one S3 operation (connect 2 s, no retries); after it the API answers 503
|
||||
S3_TIMEOUT_MS=5000
|
||||
# Parallel image validations per replica (CPU/RAM bound)
|
||||
IMAGE_CHECK_WORKERS=2
|
||||
# --- solution-specific (lab5) ---
|
||||
# Unacked messages RabbitMQ hands to one worker (defaults to WORKER_CONCURRENCY)
|
||||
WORKER_PREFETCH=2
|
||||
# Processing lease, extended by worker heartbeat every lease/3; expired lease -> job re-queued by reaper
|
||||
JOB_LEASE_SECONDS=15
|
||||
# Retry backoff: min(RETRY_MAX_MS, RETRY_BASE_MS * 2^(attempt-1)), equal jitter
|
||||
RETRY_BASE_MS=1000
|
||||
RETRY_MAX_MS=60000
|
||||
# Backpressure: uploads get 503 + Retry-After when queued+processing jobs reach this number
|
||||
MAX_PENDING_JOBS=200
|
||||
# Outbox relay poll interval (relay is also woken right after each upload COMMIT)
|
||||
OUTBOX_POLL_MS=500
|
||||
# Address-space limit of one image-processing child process
|
||||
WORKER_TASK_MEMORY_MB=640
|
||||
WORKER_REPLICAS=1
|
||||
WORKER_CPUS=2.0
|
||||
WORKER_MEMORY=1536M
|
||||
# Per-replica limits
|
||||
API_CPUS=1.0
|
||||
API_MEMORY=384M
|
||||
PG_CPUS=2.0
|
||||
# Host ports bound to 127.0.0.1: LB entrypoint and LB stats page
|
||||
LB_PUBLISHED_PORT=8080
|
||||
LB_STATS_PORT=8404
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Shell-скрипты выполняются в Linux-контейнерах: всегда LF, даже при checkout на Windows.
|
||||
*.sh text eol=lf
|
||||
*.sql text eol=lf
|
||||
*.conf text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
*.Dockerfile text eol=lf
|
||||
@@ -30,3 +30,4 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
student-tests/.generated/
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
# API и worker мессенджера lab5: один образ, разные команды (python -m app | python -m app.worker).
|
||||
FROM python:3.12.12-slim-bookworm
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1
|
||||
|
||||
WORKDIR /srv
|
||||
|
||||
# Сначала зависимости (кэшируемый слой), потом код.
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-deps -r requirements.txt
|
||||
|
||||
COPY app ./app
|
||||
COPY migrations ./migrations
|
||||
COPY contracts/jobs.schema.json ./contracts/jobs.schema.json
|
||||
|
||||
RUN useradd --system --uid 10001 --no-create-home appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"]
|
||||
|
||||
# Метка сборки (последний слой, кэш не ломает): rolling update показывает, что реплики заменены новым образом.
|
||||
ARG BUILD_ID=dev
|
||||
LABEL org.opencontainers.image.revision=$BUILD_ID
|
||||
|
||||
# exec-форма: python — PID 1 и сам получает SIGTERM от docker stop (drain -> graceful shutdown).
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -1,37 +1,191 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Запорожец Михаил, БВТ2304
|
||||
- Целевая оценка: **5**, трек **1 «Очередь под перегрузкой»**
|
||||
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab5/pulls/88 (ветка `solution/bvt2304-zaporozhets`, коммит 0a221b0)
|
||||
- Использованные источники, библиотеки и помощь ИИ: RabbitMQ docs (Reliability Guide, Consumer Acknowledgements and Publisher Confirms, Quorum Queues, Dead Lettering, Consumer Prefetch), паттерн Transactional Outbox, AWS Architecture Blog «Exponential Backoff And Jitter», Python `multiprocessing` (forkserver, preload), Pillow (`draft`, `ImageOps.exif_transpose`, WebP), PostgreSQL (`FOR UPDATE SKIP LOCKED`, advisory locks). Библиотеки: FastAPI 0.115, uvicorn 0.34, asyncpg 0.30, argon2-cffi 23.1, redis-py 5.2, minio-py 7.2.18, Pillow 11.3.0, **aio-pika 9.5.7**, **jsonschema 4.25.1**. Код, конфигурации и тесты написаны с помощью ИИ-ассистента (Claude); решения разобраны и проверены автором.
|
||||
|
||||
## Исправления по замечаниям преподавателя (перенесены из lab3/lab4)
|
||||
|
||||
| Замечание | Исправление | Проверка |
|
||||
| --- | --- | --- |
|
||||
| PR lab3 #99: `{"text":"abc\u0000def"}` возвращает 500 | PostgreSQL `text` не хранит 0x00. Миграция `0005_nul_safe_text.sql` (expand) добавляет `*_raw bytea` для text, display_name, title: при наличии U+0000 там хранятся точные UTF-8 байты, в основном столбце — строка с U+FFFD (длина та же, CHECK работают); API отдаёт исходную строку (`app/pgtext.py`). Исправлено и запушено также в PR lab3 #99 и lab4 #115 | `test_nul_in_text_roundtrip` (регрессия lab3): 201, точный текст в ответе, истории и идемпотентном повторе, 409 на другой текст с тем же ключом (`evidence/student-tests.txt`) |
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
**Нужно:** Docker Engine/Desktop с Compose v2.20+ (проверено: Docker 29.8.2, Compose 5.6.0, Windows 11 + Docker Desktop/WSL2, VM 8 vCPU / 4 GiB, Intel i5-1135G7) и POSIX-shell (Git Bash). Python/make на хосте не нужны: проверки идут в сервисе `tester`.
|
||||
|
||||
| Роль | Сервис | Образ | Health/readiness | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| Балансировщик | `lb` | `haproxy:3.2.8-alpine` | `/stats`; API по `/health/ready` | — | edge; **единственная** публикация `127.0.0.1:8080` (+8404 stats) | — |
|
||||
| API × N | `api` (`API_REPLICAS=2`) | сборка `Dockerfile` | `/health/ready` (критична только PostgreSQL) | нет; rootfs read-only | edge + data | secrets `database_url`, `cache_url`, `s3_*`, `broker_url` |
|
||||
| **Worker** | `worker` (`WORKER_REPLICAS=1`) | тот же образ, `python -m app.worker` | пульс `/tmp/worker-alive` моложе 20 с | нет; rootfs read-only | только data | `database_url`, `s3_*`, `broker_url` |
|
||||
| **Брокер** | `rabbitmq` | `rabbitmq:4.3.6-alpine@sha256:6931816a…f5` | `rabbitmq-diagnostics check_port_connectivity` | **`rabbitdata`** | data (internal); порты не публикуются, management UI не включён | `secrets/rabbitmq.conf` → `/etc/rabbitmq/conf.d/90-credentials.conf` (отдельный пользователь, не guest) |
|
||||
| S3 | `s3` | `rustfs/rustfs:1.0.4@sha256:ed9210d6…dcc2` | `/health` | `s3data` | data | `RUSTFS_*_FILE` |
|
||||
| PostgreSQL | `postgres` | `postgres:17.7-bookworm` | `pg_isready` | `pgdata` | data | `POSTGRES_PASSWORD_FILE` |
|
||||
| Кэш сессий | `valkey` | `valkey/valkey:8.1.4` | ping | — | data | `valkey_conf` |
|
||||
| Bootstrap | `migrate`, `s3-init` (one-shot) | образ API | exit 0 | — | data | как у API |
|
||||
| Тесты | `tester` (profile tools), `api-ttl` (profile test) | python:3.12.12-alpine3.22 / образ API | — | bind `.` | edge | docker.sock |
|
||||
|
||||
```sh
|
||||
git clone <repo> && cd <repo>
|
||||
sh scripts/init-secrets.sh # ./secrets/* (БД, Valkey, S3, RabbitMQ) и .env; значения не печатает
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
# postgres, s3, rabbitmq, valkey -> migrate (схема, 0006_jobs) -> s3-init (bucket) -> api x2, worker -> lb
|
||||
|
||||
docker compose --env-file .env --profile tools run --rm tester make check
|
||||
docker compose --env-file .env --profile tools run --rm tester make test
|
||||
docker compose --env-file .env --profile tools run --rm tester make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
docker compose --env-file .env --profile tools run --rm tester sh scripts/restore.sh
|
||||
docker compose --env-file .env --profile tools run --rm tester make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
|
||||
sh scripts/test-lab5.sh # собственные тесты lab5 (уровни 3–4) + регрессия lab4/lab3/lab2
|
||||
sh scripts/collect-evidence.sh # доказательства evidence/*.txt
|
||||
sh scripts/track1.sh # трек 1 (~25 мин): evidence/track1/*.json
|
||||
python student-tests/track1_report.py # сводные таблицы -> evidence/track1/report.md
|
||||
|
||||
# Оператор: реестр failed и безопасный replay
|
||||
docker compose --env-file .env exec worker python -m app.replay list
|
||||
docker compose --env-file .env exec worker python -m app.replay replay --attachment <uuid> --reason "..." [--dry-run]
|
||||
docker compose --env-file .env exec worker python -m app.replay replay --all-failed --reason "..." --dry-run
|
||||
```
|
||||
|
||||
- **Bootstrap:** `migrate` применяет `migrations/*.sql` (новая — `0006_jobs.sql`, только expand). Топологию RabbitMQ (exchange, quorum queue, DLX/DLQ) идемпотентно объявляют и API (relay), и worker при подключении. Bucket создаёт `s3-init`, как в lab4.
|
||||
- **Действий оператора**, кроме `init-secrets.sh`, нет. Если на 8080 работает стенд lab3/lab4 — остановить (`docker compose -p mtusi-lab4 stop`).
|
||||
- **Безопасная остановка:** `docker compose --env-file .env stop` (или `down`) — volumes сохраняются, принятые задания остаются в PostgreSQL/RabbitMQ.
|
||||
- **Удаление учебных данных:** `docker compose --env-file .env down -v` (volumes `mtusi-lab5_pgdata`, `_s3data`, `_rabbitdata`); секреты — `secrets/`.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ─HTTP─> lb ─> api x2 ──(1) PUT staging/<id> ─────────────────────────────────> S3 (RustFS)
|
||||
│ (2) BEGIN; INSERT attachment(queued) + job + outbox; COMMIT ▲
|
||||
│ (3) 202 queued │
|
||||
▼ │
|
||||
PostgreSQL ◄── relay (в каждой API): outbox FOR UPDATE SKIP LOCKED │
|
||||
attachments, ── publish persistent + publisher confirm ──> RabbitMQ │
|
||||
attachment_jobs, quorum queue attachment.process.v1 │
|
||||
attachment_variants, (DLX -> attachment.process.v1.dlq) │
|
||||
outbox, job_audit │ prefetch=N, manual ack │
|
||||
▲ ▼ │
|
||||
└──── claim / lease+heartbeat / COMMIT ── worker ── forkserver-процессы ──┘
|
||||
(Pillow, RLIMIT_AS, timeout)
|
||||
originals/<id>, variants/<id>/v1/*
|
||||
```
|
||||
|
||||
**Перенесено из lab4 без изменений:** HAProxy, реплики API, Bearer-сессии, идемпотентность сообщений, приватный bucket, синхронная валидация upload (10 MiB / 40 Мпикс до декодирования, MIME по содержимому, filename), авторизованный download без redirect. **Изменено по объявленному переходу:** upload → `202 queued`, готовность — через `GET /attachments/{id}`; `variant` = original/thumbnail/crop/optimized; content для queued/processing/failed — 409; failed-вложение в сообщение — 409, queued/processing — можно.
|
||||
|
||||
**Что именно сохранено в момент 202.** (1) Все байты проверенного оригинала — в устойчивом staging (приватный prefix `staging/` S3, а не память/диск API). (2) Одной транзакцией PostgreSQL — метаданные вложения (`queued`, size, sha256), логическое задание `attachment_jobs` и **намерение публикации** (строка `outbox`). Публикация в брокер к этому моменту могла ещё не произойти — это допустимо: после COMMIT работу восстановит relay. Граница честная: staging уже в S3, worker переносит оригинал server-side copy `staging/<id> → originals/<id>`.
|
||||
|
||||
**Transactional outbox.** Разрыв «COMMIT — publish» закрыт тем, что публикация — отдельный повторяемый шаг над строкой outbox. Relay (фоновая задача каждой реплики API) берёт строки `published_at IS NULL AND available_at <= now()` через `FOR UPDATE SKIP LOCKED`, публикует persistent-сообщение с `mandatory`, ждёт **publisher confirm** и только потом ставит `published_at`. Падение между confirm и UPDATE даёт повторную публикацию — допустимый дубль (at-least-once). `available_at` в будущем используется для retry с backoff без плагинов брокера. Сразу после COMMIT upload будит relay (`kick`), поэтому задержка до брокера — миллисекунды, а не период опроса.
|
||||
|
||||
**Publisher confirm и consumer ack — разные границы.** Confirm — брокер принял и записал сообщение (producer→broker; после него можно отметить outbox). Ack — consumer устойчиво обработал сообщение (broker→worker); worker шлёт ack только после COMMIT результата или после того, как убедился, что работа уже сделана/не его. Без ack сообщение вернётся при обрыве канала.
|
||||
|
||||
**Брокер.** RabbitMQ 4.3, quorum queue (Raft-журнал на диске, volume `rabbitdata`), сообщения persistent. `x-delivery-limit=20` + DLX: poison message, который валит worker до ack, или сообщение, не прошедшее `contracts/jobs.schema.json`, уходит в `attachment.process.v1.dlq`. В сообщении только `schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at` — без bytes, URL, ключей S3 и credentials; worker берёт доверенные метаданные (staging_key, size, sha256) из БД.
|
||||
|
||||
**Ключ идемпотентности job** — `job_id` (одна логическая работа на `(attachment_id, pipeline_version)`, UNIQUE); не меняется при redelivery, retry и replay. **Ключ эффекта** — `(attachment_id, pipeline_version, variant)`: PK `attachment_variants` и детерминированные S3-ключи `variants/<id>/v1/<name>.<ext>`. Повтор после сбоя перезаписывает те же объекты тем же содержимым и вставляет варианты `ON CONFLICT DO NOTHING` — второго набора нет. Повторная доставка уже `done` job — no-op + ack. `ready` никогда не откатывается: все переходы назад ограничены `WHERE status IN ('queued','processing')`. **Почему повторная доставка нормальна:** at-least-once неизбежно даёт дубли (обрыв до ack, повтор publish после confirm), поэтому корректность обеспечивает идемпотентный эффект, а не «exactly-once доставка».
|
||||
|
||||
**Захват, аренда, fencing — как отличить зависший worker от долгого и не запустить два неконтролируемых дубля.** Claim — транзакция `SELECT … FOR UPDATE` строки job: queued (или processing с **истёкшей** арендой) → processing, `attempt+1`, `worker_id`, `lease_until = now + JOB_LEASE_SECONDS (15)`. Пока worker жив, **heartbeat** каждые lease/3 продлевает аренду: долгая обработка аренду не теряет, а мёртвый или зависший процесс перестаёт её продлевать. Истёкшую аренду находит reaper в relay (раз в ~1 с, `pg_try_advisory_xact_lock` — одна реплика) и возвращает задание в queued с новой строкой outbox (или failed, если попытки исчерпаны). Дубль доставки при живой аренде — no-op. Чтобы «воскресший» старый исполнитель не перезаписал результат, фиксация результата/ошибки фенсится токеном `(worker_id, attempt)`: `UPDATE … WHERE status='processing' AND worker_id=$me AND attempt=$n` — при перехваченной аренде COMMIT не проходит. Отдельно — жёсткий лимит одной попытки `JOB_TIMEOUT_SECONDS=60` (дочерний процесс убивается).
|
||||
|
||||
**Retry.** **Постоянные** ошибки (`invalid_image`, `image_too_large`, `resource_limit`, `source_missing`, `source_corrupted`, `processing_failed`) — сразу `failed`, без повторов. **Временные** (S3/БД недоступны, `processing_timeout`) — повтор с задержкой `min(RETRY_MAX_MS, RETRY_BASE_MS·2^(attempt−1))` с equal jitter (половина фиксирована, половина случайна), не более `JOB_MAX_ATTEMPTS=5`, затем `failed` / `retry_exhausted`. Реестр failed — `attachment_jobs` (`status='failed'`, `error_code`, `last_error`, время) + DLQ для непарсящихся сообщений. **Replay** (`python -m app.replay`) — только для `failed` (строка под `FOR UPDATE`), есть `--dry-run`, запись в `job_audit` (кто, почему, прежняя ошибка), тот же `job_id`; для `done` — отказ: «повторить» не исполняет дубль успешной работы.
|
||||
|
||||
**Варианты** (`app/imaging.py`). thumbnail — JPEG q85, вписан в 256×256 без увеличения, размеры целочисленно вниз, min 1 px (320×200 → 256×160); crop — JPEG q85, центральный квадрат → 128×128 (маленькие увеличиваются); optimized — WebP q80 method 4, вписан в 1280×1280 без увеличения (маленький файл может не уменьшиться — это не обещается). EXIF orientation применяется (`exif_transpose`), alpha компонуется на белый, EXIF/ICC/XMP не записываются. **Ограничения worker:** bytes (size и sha256 сверяются с БД), пиксели (по заголовку до декодирования, ≤ 40 Мпикс), JPEG декодируется в draft-масштабе ≥ 1280, обработка — в отдельном процессе (`forkserver`) с `RLIMIT_AS = WORKER_TASK_MEMORY_MB` (640 MiB) и таймаутом `JOB_TIMEOUT_SECONDS` (kill), конкуренция `WORKER_CONCURRENCY` (семафор) и `prefetch_count = WORKER_PREFETCH`.
|
||||
|
||||
**Staging cleanup.** Staging удаляется worker-ом сразу после COMMIT результата (до ack). Если удаление не удалось — retention sweeper в worker (раз в 5 мин, advisory lock) удаляет `staging/*` старше `STAGING_RETENTION_SECONDS` (24 ч), **по которым нет queued/processing задания**. Staging failed-вложений хранится до retention, чтобы был возможен replay.
|
||||
|
||||
**Backpressure.** Перед чтением тела upload API считает незавершённые задания (`queued + processing`, частичный индекс). При `>= MAX_PENDING_JOBS` — `503 unavailable` + `Retry-After: 5`, байты не принимаются, в S3/БД ничего не пишется. Счётчик без блокировки: при одновременных upload порог можно превысить на число параллельных запросов (измерено: 22 при пороге 20).
|
||||
|
||||
**Готовность:** `/health/ready` — только PostgreSQL. Брокер и worker не критичны для приёма: при недоступном RabbitMQ API продолжает отвечать 202 (намерение в outbox), при остановленном worker — тоже. Resize синхронно в API не выполняется никогда.
|
||||
|
||||
**Соответствие `.env.example`:** `BROKER_URL_FILE` → secret `broker_url`; `WORKER_CONCURRENCY` → семафор worker; `JOB_MAX_ATTEMPTS` → `attachment_jobs.max_attempts`; `JOB_TIMEOUT_SECONDS` → таймаут дочернего процесса; `STAGING_RETENTION_SECONDS` → sweeper. Добавлены решением: `WORKER_PREFETCH`, `JOB_LEASE_SECONDS`, `RETRY_BASE_MS`, `RETRY_MAX_MS`, `MAX_PENDING_JOBS`, `OUTBOX_POLL_MS`, `WORKER_TASK_MEMORY_MB`, `WORKER_REPLICAS/CPUS/MEMORY`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
Текстовые доказательства собраны `sh scripts/collect-evidence.sh` 17.11.2026; трек 1 — `sh scripts/track1.sh`.
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `make check`, `make validate`, `make test` | `PASS: lab5 public API scenario` (202, ready, оригинал, настоящий JPEG 256×160) | `evidence/public-tests.txt` |
|
||||
| Минимум lab4 сохранён (201→202) | регрессия `test_lab4.py`, `make test-failover` ×2, `make test-persistence` | 9/9; failover без потерь; после рестарта всего стенда оригинал скачивается | `evidence/student-tests.txt`, `failover-hook.txt`, `persistence.txt` |
|
||||
| 202 только после устойчивого приёма; остановка API не теряет работу | `test_worker_down_demo`, `test_crash_between_commit_and_publish` | API перезапущен после 202 → ready; API убит сразу после COMMIT (клиент видит обрыв) → вложение всё равно ready через outbox | `evidence/student-tests.txt` |
|
||||
| Отдельные broker и worker, durable, at-least-once, ack после эффекта | `evidence/stand.txt` (quorum, durable, consumer `ack_required`), `test_worker_crash_after_commit_before_ack` | очередь `quorum true`, consumer с prefetch 2; падение после COMMIT до ack → redelivery → `no-op, already done` | `evidence/stand.txt`, `student-tests.txt` |
|
||||
| Worker недоступен — API не делает resize | `test_worker_down_demo` | upload 202 и запись `queued` при остановленном worker; content 409 | `evidence/student-tests.txt` |
|
||||
| Состояния queued/processing/ready/failed в API | `test_worker_down_demo`, `test_expired_lease_takeover`, `test_failed_state_and_409` | все 4 состояния видны в `GET /attachments/{id}` | там же |
|
||||
| Оригинал неизменен, thumbnail по контракту | `test_variants_contract` | оригинал побайтно; thumbnail JPEG 256×160 (реальные размеры), 160×100 без увеличения для маленького | там же |
|
||||
| Ограниченный retry, безопасный error_code, нет второго набора | `test_retry_bounded_then_replay`, `test_duplicate_delivery_is_noop` | S3 недоступен → 5 попыток, задержки 0.6 / 1.5 / 2.9 / 7.7 с (экспонента с jitter) → `failed retry_exhausted`; 3 дубля сообщения → `attempt=1`, 3 варианта | там же |
|
||||
| Демо: worker стоп → 202 → message → рестарт API → рестарт broker → worker → ready | `test_worker_down_demo` | глубина очереди до/после рестарта RabbitMQ: 1/1; после старта worker ready и thumbnail | там же |
|
||||
| **4:** crop 128×128, optimized WebP | `test_variants_contract` | crop JPEG 128×128, optimized WebP 320×200 (реальные размеры из файла) | там же |
|
||||
| **4:** лимиты worker (bytes/пиксели/время/память) | `test_worker_resource_limits` | 6000×6000 при RLIMIT 96 MiB → `resource_limit`; таймаут 0.2 с → `processing_timeout`; alpha → белый; EXIF 6 → 85×256, EXIF удалён | там же |
|
||||
| **4:** outbox, crash между COMMIT и publish | `test_crash_between_commit_and_publish` | `faults: api_crash_after_commit` → процесс API завершён до publish → ready без ручного создания задания | там же |
|
||||
| **4:** backoff + jitter, постоянные/временные, DLQ/реестр, безопасный replay | `test_retry_bounded_then_replay`, `test_failed_state_and_409`, `test_poison_message_to_dlq` | sha-mismatch → `failed source_corrupted` с 1 попыткой; replay `--dry-run` ничего не меняет, replay → ready, audit 1 строка, повторный replay `done` отклонён; невалидное сообщение → DLQ | там же |
|
||||
| **4:** остановка worker после записи объекта, до ack | `test_worker_crash_after_s3_before_commit` | `worker_crash_after_s3` → аренда истекла → попытка 2 → ready, в S3 ровно 3 объекта варианта | там же |
|
||||
| **4:** lease зависшего processing, prefetch, очистка staging | `test_expired_lease_takeover`, `test_prefetch_limited`, `test_staging_cleanup_after_commit` | processing с истёкшей арендой перехвачен (attempt 2); `prefetch_count=2`; после ready `staging/<id>` нет, `originals/<id>` есть | там же |
|
||||
| **5, трек 1** | `scripts/track1.sh`, `student-tests/track1_report.py` | см. раздел ниже | `evidence/track1/` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Отказы (все — автоматические тесты, `evidence/student-tests.txt`)
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| worker остановлен, upload, рестарт API и RabbitMQ | задание не теряется | queue depth 1 → 1 после рестарта брокера | ready через ~3 с после старта worker; потерь нет |
|
||||
| API убит после COMMIT, до publish | outbox допубликует | клиент получил обрыв соединения; запись в БД есть | ready через ~1–3 с (relay другой реплики); потерь нет |
|
||||
| worker убит после записи S3, до COMMIT | повтор тем же результатом | attempt 2, 3 варианта, 3 объекта в S3 | ~18 с (аренда 15 с + повтор); дублей нет |
|
||||
| worker убит после COMMIT, до ack | redelivery — no-op | `no-op, already done`, attempt 1 | сразу после рестарта; дублей нет |
|
||||
| S3 недоступен для worker | ограниченные повторы | 5 попыток за ~14 с → `retry_exhausted` | replay после возврата S3 → ready |
|
||||
| staging подменён (sha256 не совпал) | постоянная ошибка без повторов | `failed source_corrupted`, attempt 1 | content/attach → 409 |
|
||||
| poison message | не блокирует очередь | уходит в DLQ | — |
|
||||
|
||||
### Трек 1 «Очередь под перегрузкой»
|
||||
|
||||
**Гипотеза.** Пропускная способность worker растёт с `WORKER_CONCURRENCY` почти линейно, пока не упрётся в CPU-лимит контейнера (2 vCPU), а большой prefetch при той же concurrency не ускоряет обработку, но ухудшает справедливость и увеличивает объём «невидимой» для других consumers работы. Без ограничения принятой работы время до ready при перегрузке растёт неограниченно (вместе с очередью); с `MAX_PENDING_JOBS` оно ограничено ценой 503 для части клиентов.
|
||||
|
||||
**Методика** (`student-tests/overload.py`, драйвер `scripts/track1.sh`):
|
||||
- Стенд: Docker Desktop/WSL2, VM 8 vCPU / 4 GiB, Intel i5-1135G7; worker — 1 реплика, лимит 2 CPU / 1536 MiB; API ×2 по 1 CPU.
|
||||
- Изображения: **small** — `tests/fixtures/sample.png` (320×200, 112 KB); **large** — синтетический JPEG 4000×3000 (12 Мпикс, 2.9 MB, генерируется скриптом, в Git не хранится).
|
||||
- **Open-loop** поступление (расписание отправки не зависит от ответов — нет coordinated omission генератора; опоздание отправки фиксируется, максимум в прогонах < 0.2 с). Поток заведомо выше пропускной способности: small — 200 файлов по 40/с, large — 60 файлов по 8/с.
|
||||
- Конфигурации worker `concurrency/prefetch`: 1/1, 2/2, 4/4, 2/16; каждая — 2 повтора на каждом размере (16 прогонов).
|
||||
- Метрики: **время до ready** — по серверным меткам PostgreSQL (`created_at` COMMIT приёма → `updated_at` перехода в ready), p50/p95/p99; **throughput** — ready / (последний ready − первый приём); **queue lag** — число незавершённых заданий (queued+processing, раз в 0.5 с) и глубина очереди RabbitMQ; **память** — `docker stats` контейнера worker (процесс + дочерние процессы обработки); время полной разгрузки.
|
||||
- Backpressure: `MAX_PENDING_JOBS=20`, 120 больших по 15/с; затем **восстановление** — 60 маленьких по 5/с (ниже пропускной способности), сравнение с таким же прогоном **до** перегрузки.
|
||||
|
||||
**Итерация 1 — эксперимент нашёл узкое место** (`evidence/track1/iter1-before-fix/report.md`). Throughput был ≈ 1 задание/с **при любой concurrency и для обоих размеров**: small 0.78–1.14/s, large 0.74–1.01/s, хотя 320×200 обрабатывается в ~100 раз быстрее 12 Мпикс. Значит, время уходит не на изображение. Поэтапные замеры worker (`fetch/build/store/commit/cleanup` в логе) показали `build` 0.7–2.3 с при 0.07 с полезной работы: каждый дочерний процесс `forkserver` заново импортировал главный модуль `app.worker` со всеми зависимостями (при запуске `python -m` multiprocessing восстанавливает `__main__` по имени, а preload `'__main__'` при `-m` не срабатывает). **Исправление:** `set_forkserver_preload(['app.worker', 'app.imaging'])` — зависимости импортируются один раз в forkserver. Задание small: **1.8 с → 0.15 с**.
|
||||
|
||||
**Итерация 2 — после исправления** (`evidence/track1/report.md`, среднее по 2 повторам, в скобках min..max):
|
||||
|
||||
| Размер | conc / prefetch | Принято / ready / потеряно | Throughput, ready/s | p50 до ready, с | p95 до ready, с | Макс. незавершённых | Пик памяти worker, MiB | Разгрузка, с |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| small | 1 / 1 | 400 / 400 / 0 | 4.50 [3.75..5.26] | 19.7 | 38.2 [30.8..45.6] | 179 | 120 | 46.4 |
|
||||
| small | 2 / 2 | 400 / 400 / 0 | 5.69 [5.29..6.08] | 18.0 | 29.4 [27.0..31.8] | 170 | 133 | 36.2 |
|
||||
| small | 2 / 16 | 400 / 400 / 0 | 9.81 [9.60..10.02] | 7.8 | 14.7 [14.3..15.2] | 156 | 133 | 21.0 |
|
||||
| small | 4 / 4 | 400 / 400 / 0 | 12.85 [12.52..13.17] | 5.7 | 10.2 [9.9..10.6] | 134 | 142 | 16.1 |
|
||||
| large | 1 / 1 | 120 / 120 / 0 | 1.26 [1.13..1.39] | 19.4 | 39.7 [34.8..44.5] | 50 | 179 | 49.4 |
|
||||
| large | 2 / 2 | 120 / 120 / 0 | 2.43 [2.25..2.62] | 10.1 | 17.3 [15.3..19.3] | 46 | 246 | 25.6 |
|
||||
| large | 2 / 16 | 120 / 120 / 0 | 3.23 [2.98..3.48] | 7.1 | 11.1 [9.7..12.5] | 40 | 222 | 19.7 |
|
||||
| large | 4 / 4 | 120 / 120 / 0 | 3.81 [3.73..3.88] | 5.0 | 8.4 [8.1..8.8] | 33 | 294 | 16.4 |
|
||||
|
||||
Для сравнения итерация 1 (до исправления): large 4/4 — 1.01 ready/s, p95 30.0 с; small 4/4 — 1.13 ready/s, p95 46.8 с.
|
||||
|
||||
| Прогон | MAX_PENDING_JOBS | Поток | Ответы upload | Ready / failed / потеряно | Макс. незавершённых | p50 / p95 до ready, с |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| baseline-small (до перегрузки) | 200 | 60 small @ 5/s | 202: 60 | 60 / 0 / 0 | 1 | 0.115 / 1.171 |
|
||||
| backpressure-large | 20 | 120 large @ 15/s | **202: 48, 503: 72** | 48 / 0 / 0 | **22** | 4.94 / 5.65 |
|
||||
| recovery-small (сразу после) | 20 | 60 small @ 5/s | 202: 60 | 60 / 0 / 0 | 1 | 0.113 / 0.145 |
|
||||
|
||||
**Результаты и выводы.**
|
||||
1. **Concurrency.** С 1 до 4 throughput large вырос в 3.0 раза (1.26 → 3.81/s), p95 до ready упал с 39.7 до 8.4 с; small — в 2.9 раза. Рост сублинейный: с 2 до 4 large прибавил лишь 57 % — это насыщение лимита 2 CPU worker (обработка 12 Мпикс CPU-bound; I/O S3/БД перекрывается, поэтому 4 > 2 при 2 CPU). Цена — память: пик worker 179 → 294 MiB (каждый параллельный дочерний процесс держит декодированное изображение); при 4 и `WORKER_TASK_MEMORY_MB=640` наихудший случай (40 Мпикс PNG) упирается в лимит контейнера 1536 MiB — поэтому по умолчанию оставлено 2.
|
||||
2. **Prefetch — гипотеза опровергнута частично.** При той же concurrency 2 prefetch 16 дал +33 % (large) и +72 % (small) throughput относительно prefetch 2. Объяснение (подтверждено характером эффекта, не профилированием): при prefetch = concurrency слот после ack простаивает на время доставки следующего сообщения и его проверки, а при коротких заданиях (small ~0.15 с) эта пауза — заметная доля цикла; запас сообщений внутри worker её скрывает. Цена большого prefetch: до 16 сообщений «заперты» в одном worker (вторая реплика их не получит — хуже справедливость и масштабирование), при падении worker все они возвращаются в очередь разом, а очередь брокера перестаёт отражать реальный lag (сообщения unacked). Компромисс для учебного стенда: prefetch ≈ 2–4 × concurrency; оставлено по умолчанию = concurrency, параметр вынесен в `WORKER_PREFETCH`.
|
||||
3. **Queue lag и latency.** Без ограничения принятой работы при потоке выше пропускной способности число незавершённых заданий росло до 134–179 (small) и 33–50 (large), а p95 до ready — пропорционально очереди (до 40 с). Время ожидания = очередь / throughput — классический компромисс latency/ресурсы: больше concurrency — быстрее разгрузка, но больше RAM.
|
||||
4. **Backpressure.** С `MAX_PENDING_JOBS=20` система приняла 48 из 120 больших файлов, остальным ответила `503` + `Retry-After` **до чтения тела** (upload p50 7 мс); незавершённых заданий — не больше 22 (перебор на 2 из-за неблокирующего счётчика, см. ограничения). p95 до ready принятых — 5.6 с против 17.3 с без ограничения при меньшем потоке (8/с, конфигурация 2/2): принятая работа выполняется с ограниченной задержкой, лишняя отклоняется явно, а не копится.
|
||||
5. **Восстановление после перегрузки.** Сразу после backpressure-прогона лёгкая нагрузка обслужена так же, как до перегрузки: p50 0.113 с против 0.115 с, очередь ≤ 1, все 60 приняты. (p95 базового прогона 1.17 с — холодный старт forkserver только что пересозданного worker в первых заданиях; в recovery forkserver уже «тёплый».) Во всех 19 прогонах итерации 2 — 0 потерянных и 0 failed среди принятых.
|
||||
|
||||
**Ограничения измерения.** Один хост: генератор, API, worker, S3 и БД делят 8 vCPU VM, поэтому абсолютные значения не переносимы на другую машину — сравнимы только конфигурации между собой. 2 повтора дают видимый разброс (до ±20 % по p95 у c=1), выводы опираются на различия больше разброса. Память — `docker stats` контейнера (RSS + page cache), снимок раз в ~1 с, короткие пики дочерних процессов могли не попасть в выборку. Throughput считается по серверным меткам и не включает время опроса клиента. Large-изображение одно (синтетическое, шум + градиент); реальные фото дают другое соотношение decode/encode.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Backpressure-счётчик неблокирующий** — при N одновременных upload порог превышается до N (измерено 22/20). Строгое ограничение — семафор в БД/Valkey или `SELECT … FOR UPDATE` счётчика; не сделано, чтобы не сериализовать все upload.
|
||||
- **Prefetch по умолчанию = concurrency** — по результатам трека больший prefetch быстрее на коротких заданиях; выбор оставлен консервативным (справедливость между репликами worker, честный lag в брокере).
|
||||
- **Takeover только после истечения аренды** (до 15 с задержки после смерти worker в середине задания) — цена простого и безопасного fencing без отдельного реестра живых worker.
|
||||
- **Один узел RabbitMQ и RustFS** — quorum queue на одном узле защищает от рестарта, но не от потери диска; HA брокера/S3 вне рамок работы.
|
||||
- **Staging failed-вложений** удаляется через 24 ч — replay после этого вернёт `source_missing`.
|
||||
- Один worker-контейнер; горизонтальное масштабирование (`WORKER_REPLICAS`) поддержано конфигурацией, но в треке не измерялось.
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Запуск: python -m app (HTTP-процесс — PID 1 в контейнере).
|
||||
|
||||
SIGTERM (docker stop) обрабатывается в два этапа:
|
||||
1. drain: SHUTDOWN_DRAIN_MS процесс продолжает обслуживать запросы, но /health/ready отвечает 503 —
|
||||
балансировщик за это время исключает реплику по активной проверке и перестаёт слать новые запросы;
|
||||
2. обычный graceful shutdown uvicorn: новые соединения не принимаются, начатые запросы
|
||||
дообрабатываются до timeout_graceful_shutdown.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from . import lifecycle
|
||||
|
||||
log = logging.getLogger('messenger')
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
drain_seconds = 0.0
|
||||
|
||||
def handle_exit(self, sig, frame):
|
||||
if lifecycle.draining or self.drain_seconds <= 0:
|
||||
# Повторный сигнал (или drain выключен) — сразу штатное завершение uvicorn.
|
||||
return super().handle_exit(sig, frame)
|
||||
lifecycle.start_draining()
|
||||
log.info('signal %s: draining for %.1fs before shutdown, instance=%s',
|
||||
sig, self.drain_seconds, lifecycle.INSTANCE_ID)
|
||||
timer = threading.Timer(self.drain_seconds, super().handle_exit, (sig, frame))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s %(message)s')
|
||||
config = uvicorn.Config('app.main:app', host=os.environ.get('APP_HOST', '0.0.0.0'),
|
||||
port=int(os.environ.get('APP_PORT', '8080')), proxy_headers=False,
|
||||
server_header=False, timeout_graceful_shutdown=10,
|
||||
timeout_keep_alive=30) # дольше, чем держит idle-соединение HAProxy
|
||||
server = DrainingServer(config)
|
||||
server.drain_seconds = int(os.environ.get('SHUTDOWN_DRAIN_MS', '3000')) / 1000
|
||||
log.info('starting instance %s', lifecycle.INSTANCE_ID)
|
||||
server.run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,39 @@
|
||||
"""RabbitMQ: топология очередей заданий. Объявление идемпотентно — его выполняют и API (relay), и worker.
|
||||
|
||||
attachments (direct, durable) --attachment.process.v1--> quorum queue attachment.process.v1
|
||||
отклонённые/непарсящиеся и превысившие x-delivery-limit сообщения
|
||||
--> attachments.dlx --> quorum queue attachment.process.v1.dlq (просмотр оператором)
|
||||
|
||||
Quorum queue: сообщения реплицируются в Raft-журнал на диске (volume rabbitdata) и переживают рестарт
|
||||
брокера; persistent delivery_mode + publisher confirm у publisher; manual ack у consumer.
|
||||
"""
|
||||
import aio_pika
|
||||
from aio_pika import ExchangeType
|
||||
|
||||
EXCHANGE = 'attachments'
|
||||
DLX = 'attachments.dlx'
|
||||
QUEUE = 'attachment.process.v1'
|
||||
DLQ = 'attachment.process.v1.dlq'
|
||||
ROUTING_KEY = 'attachment.process.v1'
|
||||
# Защита от poison message, который валит worker до ack: после N доставок — в DLQ.
|
||||
# Состояние в БД при этом не теряется: reaper/sweeper вернёт работу по lease (см. app/outbox.py).
|
||||
DELIVERY_LIMIT = 20
|
||||
|
||||
|
||||
async def connect(url: str) -> aio_pika.abc.AbstractRobustConnection:
|
||||
return await aio_pika.connect_robust(url, timeout=5, client_properties={'connection_name': 'messenger'})
|
||||
|
||||
|
||||
async def declare(channel: aio_pika.abc.AbstractChannel):
|
||||
dlx = await channel.declare_exchange(DLX, ExchangeType.DIRECT, durable=True)
|
||||
dlq = await channel.declare_queue(DLQ, durable=True, arguments={'x-queue-type': 'quorum'})
|
||||
await dlq.bind(dlx, ROUTING_KEY)
|
||||
exchange = await channel.declare_exchange(EXCHANGE, ExchangeType.DIRECT, durable=True)
|
||||
queue = await channel.declare_queue(QUEUE, durable=True, arguments={
|
||||
'x-queue-type': 'quorum',
|
||||
'x-dead-letter-exchange': DLX,
|
||||
'x-dead-letter-routing-key': ROUTING_KEY,
|
||||
'x-delivery-limit': DELIVERY_LIMIT,
|
||||
})
|
||||
await queue.bind(exchange, ROUTING_KEY)
|
||||
return exchange, queue
|
||||
+116
@@ -0,0 +1,116 @@
|
||||
"""Настройки из переменных окружения. Секреты читаются только из файлов (*_FILE)."""
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def _int(name: str, default: int, low: int, high: int) -> int:
|
||||
value = int(os.environ.get(name, default))
|
||||
if not low <= value <= high:
|
||||
raise ValueError(f'{name} must be in [{low}, {high}]')
|
||||
return value
|
||||
|
||||
|
||||
def _secret(name: str) -> str | None:
|
||||
"""Значение из файла, путь к которому лежит в переменной NAME_FILE."""
|
||||
path = os.environ.get(name + '_FILE')
|
||||
if not path:
|
||||
return None
|
||||
return Path(path).read_text(encoding='utf-8').strip()
|
||||
|
||||
|
||||
def _required_secret(name: str) -> str:
|
||||
value = _secret(name)
|
||||
if not value:
|
||||
raise RuntimeError(f'{name}_FILE is not configured')
|
||||
return value
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
host: str
|
||||
port: int
|
||||
session_ttl: int
|
||||
session_store: str
|
||||
session_cache_ttl: int
|
||||
database_url: str
|
||||
cache_url: str | None
|
||||
db_pool_max: int
|
||||
db_statement_timeout_ms: int
|
||||
db_acquire_timeout: float
|
||||
cache_timeout: float
|
||||
hash_workers: int
|
||||
idempotency_ttl: int
|
||||
drain_seconds: float
|
||||
s3_endpoint: str
|
||||
s3_region: str
|
||||
s3_bucket: str
|
||||
s3_access_key: str
|
||||
s3_secret_key: str
|
||||
s3_timeout: float
|
||||
max_upload_bytes: int
|
||||
max_image_pixels: int
|
||||
image_workers: int
|
||||
broker_url: str
|
||||
worker_concurrency: int
|
||||
worker_prefetch: int
|
||||
job_max_attempts: int
|
||||
job_timeout: int
|
||||
job_lease: int
|
||||
retry_base_ms: int
|
||||
retry_max_ms: int
|
||||
staging_retention: int
|
||||
max_pending_jobs: int
|
||||
outbox_poll_ms: int
|
||||
worker_memory_mb: int
|
||||
worker_max_pixels: int
|
||||
|
||||
|
||||
def load() -> Settings:
|
||||
database_url = _secret('DATABASE_URL') or os.environ.get('DATABASE_URL')
|
||||
if not database_url:
|
||||
raise RuntimeError('DATABASE_URL_FILE is not configured')
|
||||
store = os.environ.get('SESSION_STORE', 'postgres')
|
||||
if store not in ('postgres', 'postgres+valkey'):
|
||||
raise ValueError('SESSION_STORE must be postgres or postgres+valkey')
|
||||
cache_url = _secret('CACHE_URL') if store == 'postgres+valkey' else None
|
||||
return Settings(
|
||||
host=os.environ.get('APP_HOST', '0.0.0.0'),
|
||||
port=_int('APP_PORT', 8080, 1, 65535),
|
||||
session_ttl=_int('SESSION_TTL_SECONDS', 3600, 1, 30 * 24 * 3600),
|
||||
session_store=store,
|
||||
session_cache_ttl=_int('SESSION_CACHE_TTL_SECONDS', 60, 1, 3600),
|
||||
database_url=database_url,
|
||||
cache_url=cache_url,
|
||||
db_pool_max=_int('DB_POOL_MAX', 10, 1, 100),
|
||||
db_statement_timeout_ms=_int('DB_STATEMENT_TIMEOUT_MS', 5000, 100, 600000),
|
||||
db_acquire_timeout=_int('DB_ACQUIRE_TIMEOUT_MS', 2000, 100, 60000) / 1000,
|
||||
cache_timeout=_int('CACHE_TIMEOUT_MS', 300, 10, 10000) / 1000,
|
||||
hash_workers=_int('PASSWORD_HASH_WORKERS', 4, 1, 32),
|
||||
idempotency_ttl=_int('IDEMPOTENCY_TTL_SECONDS', 86400, 86400, 30 * 24 * 3600), # контракт: не менее 24 ч
|
||||
drain_seconds=_int('SHUTDOWN_DRAIN_MS', 3000, 0, 60000) / 1000,
|
||||
s3_endpoint=os.environ.get('S3_ENDPOINT', 'http://s3:9000'),
|
||||
s3_region=os.environ.get('S3_REGION', 'us-east-1'),
|
||||
s3_bucket=os.environ.get('S3_BUCKET', 'messenger-attachments'),
|
||||
s3_access_key=_required_secret('S3_ACCESS_KEY_ID'),
|
||||
s3_secret_key=_required_secret('S3_SECRET_ACCESS_KEY'),
|
||||
s3_timeout=_int('S3_TIMEOUT_MS', 5000, 100, 12000) / 1000, # < timeout server HAProxy (15 с)
|
||||
# Контракт: не более 10 MiB и 40 млн пикселей; значения можно только уменьшить.
|
||||
max_upload_bytes=_int('MAX_UPLOAD_BYTES', 10485760, 1, 10485760),
|
||||
max_image_pixels=_int('MAX_IMAGE_PIXELS', 40_000_000, 1, 40_000_000),
|
||||
image_workers=_int('IMAGE_CHECK_WORKERS', 2, 1, 16),
|
||||
# lab5: брокер, worker, повторы, backpressure
|
||||
broker_url=_required_secret('BROKER_URL'),
|
||||
worker_concurrency=_int('WORKER_CONCURRENCY', 2, 1, 32),
|
||||
worker_prefetch=_int('WORKER_PREFETCH', int(os.environ.get('WORKER_CONCURRENCY', 2)), 1, 256),
|
||||
job_max_attempts=_int('JOB_MAX_ATTEMPTS', 5, 1, 50),
|
||||
job_timeout=_int('JOB_TIMEOUT_SECONDS', 60, 5, 3600), # жёсткий лимит одной попытки
|
||||
job_lease=_int('JOB_LEASE_SECONDS', 15, 3, 600), # аренда, продлевается heartbeat
|
||||
retry_base_ms=_int('RETRY_BASE_MS', 1000, 10, 600000),
|
||||
retry_max_ms=_int('RETRY_MAX_MS', 60000, 10, 3600000),
|
||||
staging_retention=_int('STAGING_RETENTION_SECONDS', 86400, 60, 30 * 86400),
|
||||
max_pending_jobs=_int('MAX_PENDING_JOBS', 200, 1, 1000000),
|
||||
outbox_poll_ms=_int('OUTBOX_POLL_MS', 500, 50, 60000),
|
||||
worker_memory_mb=_int('WORKER_TASK_MEMORY_MB', 512, 64, 8192),
|
||||
worker_max_pixels=_int('MAX_IMAGE_PIXELS', 40_000_000, 1, 40_000_000),
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
"""Ограниченный пул соединений asyncpg и перевод сбоев БД в 503."""
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import asyncpg
|
||||
|
||||
from .config import Settings
|
||||
from .errors import unavailable
|
||||
|
||||
# Ошибки "БД недоступна/перегружена" -> 503, а не 500 и тем более не пропуск запроса.
|
||||
DB_UNAVAILABLE = (
|
||||
OSError,
|
||||
asyncio.TimeoutError,
|
||||
asyncpg.PostgresConnectionError,
|
||||
asyncpg.InterfaceError,
|
||||
asyncpg.CannotConnectNowError,
|
||||
asyncpg.TooManyConnectionsError,
|
||||
asyncpg.QueryCanceledError, # statement_timeout
|
||||
asyncpg.AdminShutdownError,
|
||||
)
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self, settings: Settings):
|
||||
self.settings = settings
|
||||
self.pool: asyncpg.Pool | None = None
|
||||
|
||||
async def start(self) -> None:
|
||||
s = self.settings
|
||||
# min_size соединений создаётся лениво (connect при первом acquire), поэтому API стартует
|
||||
# даже при недоступной БД и честно отвечает 503 в /health/ready.
|
||||
self.pool = asyncpg.create_pool(
|
||||
s.database_url,
|
||||
min_size=0,
|
||||
max_size=s.db_pool_max,
|
||||
timeout=3, # таймаут установления соединения
|
||||
command_timeout=s.db_statement_timeout_ms / 1000 + 1,
|
||||
max_inactive_connection_lifetime=300,
|
||||
server_settings={
|
||||
'application_name': 'messenger-api',
|
||||
'statement_timeout': str(s.db_statement_timeout_ms),
|
||||
'idle_in_transaction_session_timeout': '10000',
|
||||
'timezone': 'UTC',
|
||||
},
|
||||
)
|
||||
await self.pool
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.pool is not None:
|
||||
await self.pool.close()
|
||||
|
||||
@asynccontextmanager
|
||||
async def connection(self):
|
||||
try:
|
||||
async with self.pool.acquire(timeout=self.settings.db_acquire_timeout) as conn:
|
||||
yield conn
|
||||
except DB_UNAVAILABLE:
|
||||
raise unavailable() from None
|
||||
|
||||
async def ping(self) -> bool:
|
||||
try:
|
||||
async with self.pool.acquire(timeout=1) as conn:
|
||||
return await conn.fetchval('SELECT 1', timeout=1) == 1
|
||||
except DB_UNAVAILABLE:
|
||||
return False
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Единый формат ошибок {"error":{"code","message","request_id"}}."""
|
||||
from contextvars import ContextVar
|
||||
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
request_id_var: ContextVar[str] = ContextVar('request_id', default='-')
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
def invalid(message: str = 'Некорректный запрос') -> ApiError:
|
||||
return ApiError(400, 'invalid_request', message)
|
||||
|
||||
|
||||
def not_found() -> ApiError:
|
||||
# Одинаковый ответ для "нет такого" и "чужое": существование чужого чата не раскрывается.
|
||||
return ApiError(404, 'not_found', 'Объект не найден')
|
||||
|
||||
|
||||
def unauthorized_bearer() -> ApiError:
|
||||
return ApiError(401, 'unauthorized', 'Требуется действительная сессия', {'WWW-Authenticate': BEARER_CHALLENGE})
|
||||
|
||||
|
||||
def unauthorized_basic() -> ApiError:
|
||||
return ApiError(401, 'unauthorized', 'Неверные учётные данные', {'WWW-Authenticate': BASIC_CHALLENGE})
|
||||
|
||||
|
||||
def unavailable() -> ApiError:
|
||||
return ApiError(503, 'unavailable', 'Сервис временно недоступен')
|
||||
|
||||
|
||||
def error_response(status: int, code: str, message: str, headers: dict | None = None) -> JSONResponse:
|
||||
rid = request_id_var.get()
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={'error': {'code': code, 'message': message, 'request_id': rid}},
|
||||
headers=headers,
|
||||
)
|
||||
@@ -0,0 +1,28 @@
|
||||
"""Точки внедрения отказов для тестов окон отказа (contracts/attachment-states.md).
|
||||
|
||||
Точка активируется файлом /tmp/fault-<name> внутри контейнера (tmpfs; создаётся тестом через
|
||||
`docker exec -u 10001 <c> touch /tmp/fault-<name>`). Срабатывает один раз: файл удаляется, процесс
|
||||
завершается os._exit — без finally, ack, закрытия соединений, как при kill -9. Docker перезапускает
|
||||
контейнер (restart: unless-stopped), tmpfs при этом пустой. В обычной работе файлов нет — это no-op.
|
||||
|
||||
Точки:
|
||||
api_crash_after_commit API: после COMMIT вложения+job+outbox, до ответа 202 и до publish;
|
||||
worker_crash_after_s3 worker: варианты и оригинал записаны в S3, COMMIT результата ещё нет;
|
||||
worker_crash_after_commit worker: результат зафиксирован (ready), ack брокеру ещё не отправлен.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
|
||||
log = logging.getLogger('messenger.faults')
|
||||
|
||||
|
||||
def hit(name: str) -> None:
|
||||
path = f'/tmp/fault-{name}'
|
||||
if not os.path.exists(path):
|
||||
return
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
log.error('FAULT INJECTION %s: hard exit', name)
|
||||
os._exit(86)
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
"""Приём изображения: ограниченное чтение тела и проверка PNG/JPEG по содержимому.
|
||||
|
||||
Порядок выбран так, чтобы ни один лимит не проверялся ПОСЛЕ дорогой операции:
|
||||
1. Content-Length больше лимита -> 413 сразу, тело не читается;
|
||||
2. чтение потоком с обрывом на MAX_UPLOAD_BYTES + 1 байте (chunked без длины тоже ограничен);
|
||||
3. сигнатура (magic bytes) -> тип; несовпадение с Content-Type или чужой формат -> 415;
|
||||
4. размеры из заголовка изображения (Pillow читает только header) -> > MAX_IMAGE_PIXELS -> 413
|
||||
ДО распаковки пикселей (защита от decompression bomb);
|
||||
5. только потом проверка целостности: PNG verify() (CRC чанков), JPEG — декодирование в draft-режиме 1/8.
|
||||
"""
|
||||
import asyncio
|
||||
import io
|
||||
import warnings
|
||||
|
||||
from fastapi import Request
|
||||
from PIL import Image
|
||||
|
||||
from .errors import ApiError
|
||||
|
||||
# Лимит пикселей проверяем сами (п.4) с нужным кодом ответа; встроенный в Pillow оставляем как страховку.
|
||||
Image.MAX_IMAGE_PIXELS = 40_000_000
|
||||
Image.LOAD_TRUNCATED_IMAGES = False
|
||||
|
||||
SIGNATURES = {
|
||||
'image/png': b'\x89PNG\r\n\x1a\n',
|
||||
'image/jpeg': b'\xff\xd8\xff',
|
||||
}
|
||||
PIL_FORMATS = {'image/png': 'PNG', 'image/jpeg': 'JPEG'}
|
||||
|
||||
|
||||
def too_large(message: str = 'Файл превышает лимит') -> ApiError:
|
||||
return ApiError(413, 'payload_too_large', message)
|
||||
|
||||
|
||||
def unsupported(message: str = 'Ожидается корректное изображение PNG или JPEG') -> ApiError:
|
||||
return ApiError(415, 'unsupported_media_type', message)
|
||||
|
||||
|
||||
def declared_type(request: Request) -> str:
|
||||
media_type = request.headers.get('content-type', '').split(';')[0].strip().lower()
|
||||
if media_type not in SIGNATURES:
|
||||
raise unsupported('Content-Type должен быть image/png или image/jpeg')
|
||||
return media_type
|
||||
|
||||
|
||||
async def read_limited(request: Request, limit: int) -> bytes:
|
||||
length = request.headers.get('content-length')
|
||||
if length is not None and length.isdigit() and int(length) > limit:
|
||||
raise too_large()
|
||||
buf = bytearray()
|
||||
async for chunk in request.stream():
|
||||
buf += chunk
|
||||
if len(buf) > limit:
|
||||
raise too_large()
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
class ImageChecker:
|
||||
def __init__(self, max_pixels: int, workers: int):
|
||||
self.max_pixels = max_pixels
|
||||
# Не больше N одновременных проверок на реплику: декодирование занимает CPU и память.
|
||||
self.slots = asyncio.Semaphore(workers)
|
||||
|
||||
async def inspect(self, data: bytes, declared: str) -> tuple[str, int, int]:
|
||||
detected = next((t for t, sig in SIGNATURES.items() if data.startswith(sig)), None)
|
||||
if detected is None or detected != declared:
|
||||
raise unsupported()
|
||||
async with self.slots:
|
||||
return await asyncio.to_thread(self._inspect, data, detected)
|
||||
|
||||
def _inspect(self, data: bytes, content_type: str) -> tuple[str, int, int]:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data)) # ленивый: читает только заголовок
|
||||
width, height = img.size
|
||||
if img.format != PIL_FORMATS[content_type]:
|
||||
raise unsupported()
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise too_large('Изображение превышает лимит пикселей') from None
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception:
|
||||
raise unsupported() from None
|
||||
if width <= 0 or height <= 0:
|
||||
raise unsupported()
|
||||
if width * height > self.max_pixels:
|
||||
raise too_large('Изображение превышает лимит пикселей')
|
||||
try:
|
||||
if content_type == 'image/png':
|
||||
img.verify() # CRC и структура чанков без хранения пикселей
|
||||
check_png_pixel_stream(data) # объём пиксельных данных совпадает с IHDR
|
||||
else:
|
||||
img.draft('RGB', (max(1, width // 8), max(1, height // 8)))
|
||||
img.load() # обрезанный/битый JPEG -> исключение
|
||||
except Exception:
|
||||
raise unsupported() from None
|
||||
finally:
|
||||
img.close()
|
||||
return content_type, width, height
|
||||
|
||||
|
||||
# ---------- PNG: потоковая проверка пиксельных данных ----------
|
||||
# verify() Pillow проверяет только CRC чанков. Файл с честным IHDR 6000x6000 и почти пустым IDAT он
|
||||
# пропустит. Полное декодирование 40 Мпикс заняло бы 160 MiB на запрос, поэтому IDAT распаковывается
|
||||
# потоково кусками по 64 KiB: считаются только байты, пиксели не хранятся. Распакованный объём должен
|
||||
# точно совпасть с вычисленным по IHDR, иначе файл повреждён (или это zip-бомба сверх заявленного).
|
||||
|
||||
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
|
||||
|
||||
def _row_bytes(width: int, bits_per_pixel: int) -> int:
|
||||
return (width * bits_per_pixel + 7) // 8
|
||||
|
||||
|
||||
def expected_png_raw_size(width: int, height: int, bit_depth: int, color_type: int, interlace: int) -> int:
|
||||
bpp = CHANNELS[color_type] * bit_depth
|
||||
if not interlace:
|
||||
return height * (1 + _row_bytes(width, bpp))
|
||||
total = 0
|
||||
for xs, ys, dx, dy in ADAM7:
|
||||
pw = (width - xs + dx - 1) // dx if width > xs else 0
|
||||
ph = (height - ys + dy - 1) // dy if height > ys else 0
|
||||
if pw and ph:
|
||||
total += ph * (1 + _row_bytes(pw, bpp))
|
||||
return total
|
||||
|
||||
|
||||
def check_png_pixel_stream(data: bytes) -> None:
|
||||
import struct
|
||||
import zlib
|
||||
pos, expected, produced = 8, None, 0
|
||||
inflater = zlib.decompressobj()
|
||||
while pos + 8 <= len(data):
|
||||
length, kind = struct.unpack('>I4s', data[pos:pos + 8])
|
||||
body = data[pos + 8:pos + 8 + length]
|
||||
pos += 12 + length
|
||||
if kind == b'IHDR':
|
||||
width, height, bit_depth, color_type, _c, _f, interlace = struct.unpack('>IIBBBBB', body)
|
||||
expected = expected_png_raw_size(width, height, bit_depth, color_type, interlace)
|
||||
elif kind == b'IDAT':
|
||||
if expected is None:
|
||||
raise ValueError('IDAT before IHDR')
|
||||
chunk = body
|
||||
while chunk:
|
||||
out = inflater.decompress(chunk, 64 * 1024)
|
||||
produced += len(out)
|
||||
if produced > expected:
|
||||
raise ValueError('pixel data larger than IHDR')
|
||||
chunk = inflater.unconsumed_tail
|
||||
elif kind == b'IEND':
|
||||
break
|
||||
produced += len(inflater.flush())
|
||||
if expected is None or produced != expected or not inflater.eof:
|
||||
raise ValueError('pixel data does not match IHDR')
|
||||
+184
@@ -0,0 +1,184 @@
|
||||
"""Построение вариантов изображения (выполняется worker-ом в отдельном дочернем процессе).
|
||||
|
||||
Контракт (contracts/README.md, «Фоновая обработка»):
|
||||
- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, EXIF orientation применяется,
|
||||
alpha компонуется на белый, размеры округляются вниз, минимум 1 px (320x200 -> 256x160);
|
||||
- crop: JPEG, центрированный квадрат 128x128 (маленький исходник увеличивается);
|
||||
- optimized: WebP, вписан в 1280x1280 без увеличения.
|
||||
|
||||
Параметры: JPEG quality=85 (thumbnail, crop), optimize=True, progressive=False; WebP quality=80, method=4.
|
||||
Метаданные удаляются: EXIF/ICC/XMP не передаются в save() (Pillow не копирует их без явного exif=/icc_profile=),
|
||||
ориентация EXIF уже применена к пикселям. Все варианты — 8-bit sRGB без alpha.
|
||||
|
||||
Ограничения ресурсов: лимит пикселей проверяется по заголовку ДО декодирования; JPEG декодируется в draft-
|
||||
режиме с уменьшением до ближайшего масштаба >= 1280 (меньше памяти и CPU); процесс запускается с RLIMIT_AS
|
||||
(память) и убивается по таймауту (время) — см. run_isolated().
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import io
|
||||
import multiprocessing
|
||||
import resource
|
||||
import warnings
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
THUMB = 256
|
||||
CROP = 128
|
||||
OPTIMIZED = 1280
|
||||
|
||||
|
||||
class PermanentImageError(Exception):
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
class TransientImageError(Exception):
|
||||
def __init__(self, code: str):
|
||||
super().__init__(code)
|
||||
self.code = code
|
||||
|
||||
|
||||
def fit(width: int, height: int, box: int) -> tuple[int, int]:
|
||||
"""Вписать в box x box без увеличения; целочисленно, округление вниз, минимум 1 px."""
|
||||
m = max(width, height)
|
||||
if m <= box:
|
||||
return width, height
|
||||
return max(1, width * box // m), max(1, height * box // m)
|
||||
|
||||
|
||||
def flatten(img: Image.Image) -> Image.Image:
|
||||
"""Alpha (RGBA/LA/P с прозрачностью) компонуется на белый фон; результат RGB."""
|
||||
if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info):
|
||||
rgba = img.convert('RGBA')
|
||||
background = Image.new('RGB', rgba.size, (255, 255, 255))
|
||||
background.paste(rgba, mask=rgba.getchannel('A'))
|
||||
return background
|
||||
if img.mode in ('I;16', 'I;16B', 'I;16L', 'I'):
|
||||
img = img.point(lambda v: v / 256).convert('L') # 16-bit серый -> 8-bit
|
||||
return img.convert('RGB')
|
||||
|
||||
|
||||
def encode(name: str, img: Image.Image, fmt: str, content_type: str, **params) -> dict:
|
||||
buf = io.BytesIO()
|
||||
img.save(buf, fmt, **params)
|
||||
data = buf.getvalue()
|
||||
return {'name': name, 'content_type': content_type, 'width': img.width, 'height': img.height,
|
||||
'size_bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'data': data}
|
||||
|
||||
|
||||
def build_variants(data: bytes, max_pixels: int) -> list[dict]:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
Image.MAX_IMAGE_PIXELS = max_pixels
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
width, height = img.size
|
||||
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
||||
raise PermanentImageError('image_too_large') from None
|
||||
except Exception:
|
||||
raise PermanentImageError('invalid_image') from None
|
||||
if img.format not in ('PNG', 'JPEG'):
|
||||
raise PermanentImageError('invalid_image')
|
||||
if width * height > max_pixels:
|
||||
raise PermanentImageError('image_too_large')
|
||||
try:
|
||||
if img.format == 'JPEG':
|
||||
img.draft('RGB', (OPTIMIZED, OPTIMIZED)) # декодер сам уменьшает в 2/4/8 раз, если можно
|
||||
img.load()
|
||||
img = ImageOps.exif_transpose(img)
|
||||
rgb = flatten(img)
|
||||
except MemoryError:
|
||||
raise
|
||||
except Exception:
|
||||
raise PermanentImageError('invalid_image') from None
|
||||
|
||||
# Размеры считаются от ориентированного ОРИГИНАЛА (не от draft-копии), иначе draft менял бы результат.
|
||||
ow, oh = _oriented_size(data, width, height)
|
||||
variants = []
|
||||
tw, th = fit(ow, oh, THUMB)
|
||||
variants.append(encode('thumbnail', rgb.resize((tw, th), Image.LANCZOS) if (tw, th) != rgb.size else rgb,
|
||||
'JPEG', 'image/jpeg', quality=85, optimize=True))
|
||||
side = min(rgb.size)
|
||||
left, top = (rgb.width - side) // 2, (rgb.height - side) // 2
|
||||
square = rgb.crop((left, top, left + side, top + side))
|
||||
variants.append(encode('crop', square.resize((CROP, CROP), Image.LANCZOS), 'JPEG', 'image/jpeg',
|
||||
quality=85, optimize=True))
|
||||
pw, ph = fit(ow, oh, OPTIMIZED)
|
||||
variants.append(encode('optimized', rgb.resize((pw, ph), Image.LANCZOS) if (pw, ph) != rgb.size else rgb,
|
||||
'WEBP', 'image/webp', quality=80, method=4))
|
||||
return variants
|
||||
|
||||
|
||||
def _orientation(data: bytes) -> int:
|
||||
try:
|
||||
return Image.open(io.BytesIO(data)).getexif().get(0x0112, 1)
|
||||
except Exception:
|
||||
return 1
|
||||
|
||||
|
||||
def _rotated(data: bytes) -> bool:
|
||||
return _orientation(data) in (5, 6, 7, 8)
|
||||
|
||||
|
||||
def _oriented_size(data: bytes, width: int, height: int) -> tuple[int, int]:
|
||||
"""Размер оригинала после применения EXIF orientation (5–8 меняют местами ширину и высоту)."""
|
||||
return (height, width) if _rotated(data) else (width, height)
|
||||
|
||||
|
||||
# ---------- изоляция: отдельный процесс с лимитом памяти и времени ----------
|
||||
|
||||
def _child(conn, data: bytes, max_pixels: int, memory_mb: int) -> None:
|
||||
limit = memory_mb * 1024 * 1024
|
||||
resource.setrlimit(resource.RLIMIT_AS, (limit, limit))
|
||||
try:
|
||||
conn.send(('ok', build_variants(data, max_pixels)))
|
||||
except PermanentImageError as exc:
|
||||
conn.send(('permanent', exc.code))
|
||||
except MemoryError:
|
||||
conn.send(('permanent', 'resource_limit'))
|
||||
except Exception:
|
||||
conn.send(('permanent', 'processing_failed'))
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
_ctx = multiprocessing.get_context('forkserver')
|
||||
# Preload app.worker: forkserver один раз импортирует главный модуль worker со всеми зависимостями
|
||||
# (aio-pika, jsonschema, minio, asyncpg, Pillow), дочерние процессы получают их готовыми через fork.
|
||||
# Без этого каждый запуск обработки заново импортировал app.worker (при `python -m` multiprocessing
|
||||
# восстанавливает __main__ по имени модуля): 0.7–2.3 с накладных расходов на задание при 0.07 с полезной
|
||||
# работы для 320x200 — найдено в эксперименте трека 1 (evidence/track1/, раздел «Итерация 1» в REPORT.md).
|
||||
# '__main__' по пути здесь не работает: при запуске через -m пути к главному модулю нет.
|
||||
_ctx.set_forkserver_preload(['app.worker', 'app.imaging'])
|
||||
|
||||
|
||||
async def run_isolated(data: bytes, max_pixels: int, memory_mb: int, timeout: float) -> list[dict]:
|
||||
"""Варианты строятся в дочернем процессе: зависание/бомба памяти не роняют и не блокируют worker.
|
||||
|
||||
Таймаут -> процесс убивается, TransientImageError('processing_timeout') (мог помешать перегруз CPU —
|
||||
повтор с backoff; при исчерпании попыток — failed retry_exhausted).
|
||||
RLIMIT_AS/смерть процесса -> PermanentImageError('resource_limit').
|
||||
"""
|
||||
parent, child = _ctx.Pipe(duplex=False)
|
||||
proc = _ctx.Process(target=_child, args=(child, data, max_pixels, memory_mb), daemon=True)
|
||||
proc.start()
|
||||
child.close()
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
ready = await loop.run_in_executor(None, parent.poll, timeout)
|
||||
if not ready:
|
||||
raise TransientImageError('processing_timeout')
|
||||
try:
|
||||
status, value = await loop.run_in_executor(None, parent.recv)
|
||||
except EOFError:
|
||||
raise PermanentImageError('resource_limit') from None
|
||||
finally:
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
await loop.run_in_executor(None, proc.join, 5)
|
||||
parent.close()
|
||||
if status == 'ok':
|
||||
return value
|
||||
raise PermanentImageError(value)
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
"""Общие операции над заданиями обработки вложений (API, relay, worker, replay).
|
||||
|
||||
Все переходы состояний выполняются SQL-транзакциями в PostgreSQL — это источник истины о работе.
|
||||
Брокер только доставляет "напоминание" (job_id), поэтому потеря/дубль сообщения не теряет и не
|
||||
дублирует эффект: worker всегда сверяется с attachment_jobs.
|
||||
"""
|
||||
import json
|
||||
import random
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
PIPELINE_VERSION = 1
|
||||
JOB_TYPE = 'attachment.process.v1'
|
||||
|
||||
|
||||
def payload(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int) -> dict:
|
||||
"""Сообщение по contracts/jobs.schema.json: только ID и служебные поля — без bytes, URL и credentials."""
|
||||
return {
|
||||
'schema_version': 1,
|
||||
'job_id': str(job_id),
|
||||
'type': JOB_TYPE,
|
||||
'attachment_id': str(attachment_id),
|
||||
'pipeline_version': PIPELINE_VERSION,
|
||||
'attempt': attempt,
|
||||
'created_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ'),
|
||||
}
|
||||
|
||||
|
||||
def backoff_seconds(attempt: int, base_ms: int, max_ms: int) -> float:
|
||||
"""Экспоненциальный backoff с "equal jitter": половина задержки фиксирована, половина случайна.
|
||||
|
||||
Фиксированная половина не даёт повтору прийти мгновенно (full jitter может дать ~0),
|
||||
случайная — разводит повторы разных заданий после общего сбоя (нет "стада").
|
||||
"""
|
||||
cap = min(max_ms, base_ms * 2 ** max(0, attempt - 1))
|
||||
return (cap / 2 + random.uniform(0, cap / 2)) / 1000
|
||||
|
||||
|
||||
async def enqueue(conn, job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int, delay_s: float = 0.0) -> None:
|
||||
"""Намерение опубликовать job — строка outbox в ТЕКУЩЕЙ транзакции вызывающего."""
|
||||
await conn.execute(
|
||||
'INSERT INTO outbox (job_id, payload, available_at) '
|
||||
'VALUES ($1, $2::jsonb, clock_timestamp() + make_interval(secs => $3))',
|
||||
job_id, json.dumps(payload(job_id, attachment_id, attempt)), delay_s)
|
||||
|
||||
|
||||
async def create(conn, attachment_id: uuid.UUID, max_attempts: int) -> uuid.UUID:
|
||||
job_id = uuid.uuid4()
|
||||
await conn.execute(
|
||||
'INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, max_attempts) VALUES ($1, $2, $3, $4)',
|
||||
job_id, attachment_id, PIPELINE_VERSION, max_attempts)
|
||||
await enqueue(conn, job_id, attachment_id, attempt=1)
|
||||
return job_id
|
||||
|
||||
|
||||
async def requeue(conn, job_id, attachment_id, next_attempt: int, delay_s: float, reason: str,
|
||||
worker_id: str | None = None, attempt: int | None = None) -> bool:
|
||||
"""processing -> queued: временная ошибка (worker) или истёкшая аренда (reaper).
|
||||
|
||||
worker_id/attempt заданы — фиксация от имени владельца аренды (fencing token = worker_id + номер попытки):
|
||||
чужую или уже перехваченную работу не трогаем.
|
||||
"""
|
||||
fence = 'AND worker_id = $4 AND attempt = $5' if worker_id else ''
|
||||
args = [job_id, delay_s, reason[:200]] + ([worker_id, attempt] if worker_id else [])
|
||||
updated = await conn.fetchval(
|
||||
"UPDATE attachment_jobs SET status = 'queued', lease_until = NULL, worker_id = NULL, "
|
||||
"next_attempt_at = clock_timestamp() + make_interval(secs => $2), last_error = $3, "
|
||||
f"updated_at = clock_timestamp() WHERE job_id = $1 AND status = 'processing' {fence} RETURNING 1",
|
||||
*args)
|
||||
if not updated:
|
||||
return False
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() "
|
||||
"WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id)
|
||||
await enqueue(conn, job_id, attachment_id, next_attempt, delay_s)
|
||||
return True
|
||||
|
||||
|
||||
async def fail(conn, job_id, attachment_id, code: str, reason: str, worker_id: str | None = None,
|
||||
attempt: int | None = None) -> bool:
|
||||
"""-> failed: постоянная ошибка или исчерпаны попытки. Код машинный и безопасный для клиента."""
|
||||
fence = "AND worker_id = $5 AND attempt = $6 AND status = 'processing'" if worker_id else ''
|
||||
args = [job_id, code, reason[:200], attachment_id] + ([worker_id, attempt] if worker_id else [])
|
||||
updated = await conn.fetchval(
|
||||
"UPDATE attachment_jobs SET status = 'failed', lease_until = NULL, error_code = $2, last_error = $3, "
|
||||
"finished_at = clock_timestamp(), updated_at = clock_timestamp() "
|
||||
f"WHERE job_id = $1 AND attachment_id = $4 AND status IN ('queued', 'processing') {fence} RETURNING 1",
|
||||
*args)
|
||||
if not updated:
|
||||
return False
|
||||
# ready никогда не откатывается: условие по статусу.
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = $2, updated_at = clock_timestamp() "
|
||||
"WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id, code)
|
||||
return True
|
||||
@@ -0,0 +1,16 @@
|
||||
"""Состояние процесса реплики: идентификатор для X-Instance-Id и флаг drain при SIGTERM."""
|
||||
import os
|
||||
import secrets
|
||||
import socket
|
||||
|
||||
# Стабилен в пределах жизни процесса, новый после перезапуска. Префикс — hostname контейнера
|
||||
# (= короткий ID контейнера в Docker), по нему scripts/stop-one.sh находит именно эту реплику.
|
||||
# В lab7 по контракту понадобится непрозрачный alias без hostname.
|
||||
INSTANCE_ID = os.environ.get('INSTANCE_ID') or f'{socket.gethostname()}-{secrets.token_hex(3)}'
|
||||
|
||||
draining = False
|
||||
|
||||
|
||||
def start_draining() -> None:
|
||||
global draining
|
||||
draining = True
|
||||
+588
@@ -0,0 +1,588 @@
|
||||
"""HTTP API мессенджера lab4 по contracts/openapi.json (stateless-реплика за балансировщиком).
|
||||
|
||||
lab4: вложения. Метаданные — PostgreSQL, байты — приватный S3-bucket (RustFS). Локальных файлов у API нет.
|
||||
lab5: upload -> устойчивый staging в S3 + вложение/job/outbox одной транзакцией -> 202 queued;
|
||||
обработку выполняет отдельный worker через RabbitMQ (app/worker.py), публикацию — relay outbox (app/outbox.py).
|
||||
"""
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import quote
|
||||
from uuid import UUID
|
||||
|
||||
import asyncpg
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from . import config, faults, images, jobs, lifecycle, pgtext, validation as v
|
||||
from .db import DB_UNAVAILABLE, Database
|
||||
from .errors import (ApiError, error_response, invalid, not_found, request_id_var,
|
||||
unauthorized_basic, unauthorized_bearer, unavailable)
|
||||
from .security import PasswordService, new_token
|
||||
from .outbox import Relay
|
||||
from .sessions import SessionStore
|
||||
from .storage import Storage
|
||||
|
||||
log = logging.getLogger('messenger')
|
||||
settings = config.load()
|
||||
db = Database(settings)
|
||||
sessions = SessionStore(settings, db)
|
||||
passwords = PasswordService(settings.hash_workers)
|
||||
storage = Storage(settings)
|
||||
image_checker = images.ImageChecker(settings.max_image_pixels, settings.image_workers)
|
||||
relay = Relay(settings, db)
|
||||
|
||||
|
||||
async def purge_loop() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(3600)
|
||||
try:
|
||||
removed = await sessions.purge_expired()
|
||||
keys = await purge_idempotency_keys()
|
||||
log.info('purged %d expired sessions, %d idempotency keys', removed, keys)
|
||||
except Exception: # очистка — фоновая гигиена, её сбой не должен ронять процесс
|
||||
log.warning('purge failed', exc_info=True)
|
||||
|
||||
|
||||
async def purge_idempotency_keys() -> int:
|
||||
# Удаление по одному ключу индекса created_at; параллельный запуск на нескольких репликах безопасен.
|
||||
async with db.connection() as conn:
|
||||
status = await conn.execute(
|
||||
'DELETE FROM idempotency_keys WHERE created_at < clock_timestamp() - make_interval(secs => $1)',
|
||||
settings.idempotency_ttl)
|
||||
return int(status.split()[-1])
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
await db.start()
|
||||
task = asyncio.create_task(purge_loop())
|
||||
relay_task = asyncio.create_task(relay.run())
|
||||
yield
|
||||
task.cancel()
|
||||
relay_task.cancel()
|
||||
await relay.close()
|
||||
await sessions.cache.close()
|
||||
await db.close()
|
||||
passwords.close()
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, openapi_url=None, docs_url=None, redoc_url=None)
|
||||
|
||||
# ---------- request id, ошибки ----------
|
||||
|
||||
REQUEST_ID_RE = re.compile(r'[A-Za-z0-9._:-]{1,128}')
|
||||
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_id_middleware(request: Request, call_next):
|
||||
incoming = request.headers.get('x-request-id', '')
|
||||
rid = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
|
||||
request_id_var.set(rid)
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
log.exception('unhandled error, request_id=%s', rid)
|
||||
response = error_response(500, 'internal_error', 'Внутренняя ошибка сервера')
|
||||
response.headers['X-Request-Id'] = rid
|
||||
response.headers['X-Instance-Id'] = lifecycle.INSTANCE_ID
|
||||
if lifecycle.draining:
|
||||
# Во время drain закрываем keep-alive: следующий запрос LB откроет соединение уже к живой реплике.
|
||||
response.headers['Connection'] = 'close'
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(_request: Request, exc: ApiError):
|
||||
return error_response(exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_error_handler(_request: Request, exc: StarletteHTTPException):
|
||||
if exc.status_code == 404:
|
||||
return error_response(404, 'not_found', 'Объект не найден')
|
||||
if exc.status_code == 405:
|
||||
return error_response(405, 'invalid_request', 'Метод не поддерживается', dict(exc.headers or {}))
|
||||
return error_response(exc.status_code, 'invalid_request', 'Некорректный запрос')
|
||||
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(_request: Request, _exc: RequestValidationError):
|
||||
return error_response(400, 'invalid_request', 'Некорректный запрос')
|
||||
|
||||
|
||||
async def db_unavailable_handler(_request: Request, _exc: Exception):
|
||||
# Страховка: сбой связи с БД вне db.connection() тоже даёт ограниченный 503, а не 500.
|
||||
return error_response(503, 'unavailable', 'Сервис временно недоступен')
|
||||
|
||||
|
||||
for _exc_type in DB_UNAVAILABLE:
|
||||
app.add_exception_handler(_exc_type, db_unavailable_handler)
|
||||
|
||||
# ---------- представление ----------
|
||||
|
||||
|
||||
def ts(value: datetime) -> str:
|
||||
# RFC 3339 UTC с микросекундами: точность совпадает с timestamptz и с ключом пагинации.
|
||||
return value.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
|
||||
|
||||
|
||||
def user_json(row) -> dict:
|
||||
return {'id': str(row['id']), 'username': row['username'],
|
||||
'display_name': pgtext.join(row['display_name'], row['display_name_raw']), 'created_at': ts(row['created_at'])}
|
||||
|
||||
|
||||
def chat_json(row) -> dict:
|
||||
return {'id': str(row['id']), 'title': pgtext.join(row['title'], row['title_raw']),
|
||||
'member_ids': [str(m) for m in row['member_ids']], 'created_at': ts(row['created_at'])}
|
||||
|
||||
|
||||
def message_json(row) -> dict:
|
||||
return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'sender_id': str(row['sender_id']),
|
||||
'text': pgtext.join(row['text'], row['text_raw']), 'created_at': ts(row['created_at']),
|
||||
'attachment_ids': [str(a) for a in row['attachment_ids']]}
|
||||
|
||||
|
||||
def attachment_json(row, variants=()) -> dict:
|
||||
# object_key/staging_key и внутренние адреса S3 в ответ не попадают.
|
||||
return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']),
|
||||
'filename': row['filename'], 'content_type': row['content_type'],
|
||||
'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'],
|
||||
'variants': [{'name': x['name'], 'content_type': x['content_type'], 'size_bytes': x['size_bytes'],
|
||||
'width': x['width'], 'height': x['height'], 'sha256': x['sha256']} for x in variants],
|
||||
'error_code': row['error_code'],
|
||||
'created_at': ts(row['created_at']), 'updated_at': ts(row['updated_at'])}
|
||||
|
||||
|
||||
USER_COLUMNS = 'id, username, display_name, display_name_raw, created_at'
|
||||
CHAT_COLUMNS = ('c.id, c.title, c.title_raw, c.created_at, '
|
||||
'ARRAY(SELECT cm.user_id FROM chat_members cm WHERE cm.chat_id = c.id ORDER BY cm.user_id) AS member_ids')
|
||||
MESSAGE_ATTACHMENTS = ('ARRAY(SELECT ma.attachment_id FROM message_attachments ma '
|
||||
'WHERE ma.message_id = {m}.id ORDER BY ma.position) AS attachment_ids')
|
||||
MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, text_raw, created_at, ' + MESSAGE_ATTACHMENTS.format(m='messages')
|
||||
ATTACHMENT_COLUMNS = ('a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, '
|
||||
'a.status, a.error_code, a.object_key, a.pipeline_version, a.created_at, a.updated_at')
|
||||
|
||||
# ---------- аутентификация ----------
|
||||
|
||||
TOKEN_RE = re.compile(r'[A-Za-z0-9_-]{32,128}')
|
||||
|
||||
|
||||
def bearer_token(request: Request) -> str:
|
||||
"""Только Authorization: Bearer. X-User-Id и Basic здесь игнорируются/отклоняются."""
|
||||
scheme, _, token = request.headers.get('authorization', '').partition(' ')
|
||||
token = token.strip()
|
||||
if scheme.lower() != 'bearer' or not TOKEN_RE.fullmatch(token):
|
||||
raise unauthorized_bearer()
|
||||
return token
|
||||
|
||||
|
||||
async def current_user(request: Request) -> UUID:
|
||||
session = await sessions.resolve(bearer_token(request))
|
||||
if session is None:
|
||||
raise unauthorized_bearer()
|
||||
return session.user_id
|
||||
|
||||
|
||||
def basic_credentials(request: Request) -> tuple[str, str]:
|
||||
scheme, _, encoded = request.headers.get('authorization', '').partition(' ')
|
||||
if scheme.lower() != 'basic':
|
||||
raise unauthorized_basic()
|
||||
try:
|
||||
decoded = base64.b64decode(encoded.strip(), validate=True)
|
||||
username, sep, password = decoded.partition(b':') # двоеточие допустимо внутри пароля
|
||||
if not sep:
|
||||
raise ValueError
|
||||
return username.decode('ascii'), password.decode('utf-8')
|
||||
except (ValueError, binascii.Error, UnicodeDecodeError):
|
||||
raise unauthorized_basic() from None
|
||||
|
||||
# ---------- health ----------
|
||||
|
||||
|
||||
@app.get('/health/live')
|
||||
async def live():
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
@app.get('/health/ready')
|
||||
async def ready():
|
||||
# Критична только PostgreSQL (источник истины). Valkey — кэш: без него API работает медленнее, но корректно.
|
||||
# Во время drain (получен SIGTERM) реплика честно сообщает LB, что новых запросов ей давать не надо.
|
||||
if lifecycle.draining or not await db.ping():
|
||||
raise unavailable()
|
||||
return {'status': 'ok'}
|
||||
|
||||
# ---------- пользователи и сессии ----------
|
||||
|
||||
|
||||
@app.post('/api/v1/users', status_code=201)
|
||||
async def create_user(request: Request):
|
||||
data = await v.read_json(request)
|
||||
v.fields(data, {'username', 'display_name', 'password'})
|
||||
username = v.username(data)
|
||||
display_name = v.text_field(data, 'display_name', 1, 100)
|
||||
password = v.text_field(data, 'password', 12, 128)
|
||||
password_hash = await passwords.hash(password)
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'INSERT INTO users (username, display_name, display_name_raw, password_hash) '
|
||||
f'VALUES ($1, $2, $3, $4) RETURNING {USER_COLUMNS}',
|
||||
username, *pgtext.split(display_name), password_hash)
|
||||
except asyncpg.UniqueViolationError:
|
||||
# Гонка двух одинаковых регистраций решается UNIQUE-индексом: COMMIT пройдёт только у одной.
|
||||
raise ApiError(409, 'conflict', 'username уже занят') from None
|
||||
return JSONResponse(user_json(row), status_code=201)
|
||||
|
||||
|
||||
@app.post('/api/v1/auth/sessions', status_code=201)
|
||||
async def create_session(request: Request):
|
||||
if await request.body():
|
||||
raise invalid('Тело запроса не допускается; используйте Authorization: Basic')
|
||||
username, password = basic_credentials(request)
|
||||
row = None
|
||||
if v.USERNAME_RE.fullmatch(username) and 12 <= len(password) <= 128:
|
||||
async with db.connection() as conn:
|
||||
row = await conn.fetchrow(
|
||||
f'SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = $1', username)
|
||||
# verify выполняется всегда (и для несуществующего пользователя) — одинаковое время ответа.
|
||||
if not await passwords.verify(row['password_hash'] if row else None, password):
|
||||
raise unauthorized_basic()
|
||||
token = new_token()
|
||||
session = await sessions.create(row['id'], token)
|
||||
return JSONResponse({'token': token, 'token_type': 'Bearer', 'expires_at': ts(session.expires_at),
|
||||
'user': user_json(row)}, status_code=201, headers={'Cache-Control': 'no-store'})
|
||||
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
async def delete_session(request: Request):
|
||||
token = bearer_token(request)
|
||||
if not await sessions.revoke(token):
|
||||
raise unauthorized_bearer()
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get('/api/v1/users/me')
|
||||
async def me(request: Request):
|
||||
user_id = await current_user(request)
|
||||
async with db.connection() as conn:
|
||||
row = await conn.fetchrow(f'SELECT {USER_COLUMNS} FROM users WHERE id = $1', user_id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return user_json(row)
|
||||
|
||||
# ---------- чаты ----------
|
||||
|
||||
|
||||
@app.post('/api/v1/chats', status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
user_id = await current_user(request)
|
||||
data = await v.read_json(request)
|
||||
v.fields(data, {'title', 'member_ids'})
|
||||
title = v.text_field(data, 'title', 1, 100)
|
||||
raw_members = data['member_ids']
|
||||
if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100:
|
||||
raise invalid('member_ids должен быть массивом из 1..100 UUID')
|
||||
members = [v.uuid_value(m, 'member_id') for m in raw_members]
|
||||
if len(set(members)) != len(members):
|
||||
raise invalid('member_ids содержит дубликаты')
|
||||
if user_id not in members:
|
||||
raise invalid('Создатель должен входить в member_ids')
|
||||
|
||||
async with db.connection() as conn:
|
||||
# Одна транзакция: чат и все участники появляются вместе или не появляются вовсе.
|
||||
async with conn.transaction():
|
||||
chat = await conn.fetchrow(
|
||||
'INSERT INTO chats (title, title_raw, created_by) VALUES ($1, $2, $3) RETURNING id, created_at',
|
||||
*pgtext.split(title), user_id)
|
||||
added = await conn.fetch(
|
||||
'INSERT INTO chat_members (chat_id, user_id) '
|
||||
'SELECT $1, u.id FROM users u WHERE u.id = ANY($2::uuid[]) RETURNING user_id',
|
||||
chat['id'], members)
|
||||
if len(added) != len(members):
|
||||
# Исключение внутри conn.transaction() -> ROLLBACK, строка chats тоже откатывается.
|
||||
raise invalid('member_ids содержит неизвестных пользователей')
|
||||
return JSONResponse({'id': str(chat['id']), 'title': title,
|
||||
'member_ids': sorted(str(m) for m in members),
|
||||
'created_at': ts(chat['created_at'])}, status_code=201)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats')
|
||||
async def list_chats(request: Request):
|
||||
user_id = await current_user(request)
|
||||
limit = v.limit_param(request)
|
||||
after = v.decode_cursor(request, 'chats')
|
||||
query = (f'SELECT {CHAT_COLUMNS} FROM chats c '
|
||||
'JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $1 ')
|
||||
params = [user_id, limit + 1]
|
||||
if after:
|
||||
query += 'WHERE (c.created_at, c.id) > ($3, $4) '
|
||||
params += after
|
||||
query += 'ORDER BY c.created_at, c.id LIMIT $2'
|
||||
async with db.connection() as conn:
|
||||
rows = await conn.fetch(query, *params)
|
||||
return page(rows, limit, 'chats', chat_json)
|
||||
|
||||
|
||||
def page(rows, limit: int, scope: str, render) -> dict:
|
||||
# Читаем limit+1 строку: если лишняя есть — следующая страница существует.
|
||||
items = rows[:limit]
|
||||
cursor = (v.encode_cursor(scope, items[-1]['created_at'], items[-1]['id'])
|
||||
if len(rows) > limit else None)
|
||||
return {'items': [render(r) for r in items], 'next_cursor': cursor}
|
||||
|
||||
|
||||
async def chat_for_member(conn, chat_id: UUID, user_id: UUID):
|
||||
return await conn.fetchrow(
|
||||
f'SELECT {CHAT_COLUMNS} FROM chats c '
|
||||
'WHERE c.id = $1 AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = $2)',
|
||||
chat_id, user_id)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}')
|
||||
async def get_chat(chat_id: str, request: Request):
|
||||
user_id = await current_user(request)
|
||||
cid = v.uuid_value(chat_id, 'chat_id')
|
||||
async with db.connection() as conn:
|
||||
row = await chat_for_member(conn, cid, user_id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
return chat_json(row)
|
||||
|
||||
# ---------- сообщения ----------
|
||||
|
||||
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r'[A-Za-z0-9._:-]{1,128}')
|
||||
|
||||
|
||||
def idempotency_key(request: Request) -> str:
|
||||
key = request.headers.get('idempotency-key')
|
||||
if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key):
|
||||
raise invalid('Требуется заголовок Idempotency-Key: [A-Za-z0-9._:-]{1,128}')
|
||||
return key
|
||||
|
||||
|
||||
def request_fingerprint(text: str, attachments: list[UUID]) -> bytes:
|
||||
# Хешируем разобранное тело, а не байты: порядок ключей и пробелы JSON не делают запрос "другим",
|
||||
# отсутствующий attachment_ids равен []. Порядок вложений значим — он сохраняется в сообщении.
|
||||
canonical = json.dumps({'text': text, 'attachment_ids': [str(a) for a in attachments]},
|
||||
ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||||
return hashlib.sha256(canonical.encode('utf-8')).digest()
|
||||
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
|
||||
async def create_message(chat_id: str, request: Request):
|
||||
# Повтор заново проверяет сессию и членство: сохранённый ответ не выдаётся без текущего доступа.
|
||||
user_id = await current_user(request)
|
||||
cid = v.uuid_value(chat_id, 'chat_id')
|
||||
key = idempotency_key(request)
|
||||
data = await v.read_json(request)
|
||||
# sender_id в запросе не принимается — отправитель берётся из сессии
|
||||
v.fields(data, {'text'}, {'attachment_ids'})
|
||||
text = v.text_field(data, 'text', 0, 4000, allow_blank=True)
|
||||
attachments = v.attachment_ids(data)
|
||||
if not attachments and not text.strip():
|
||||
raise invalid('Нужен непустой текст или хотя бы одно вложение')
|
||||
fingerprint = request_fingerprint(text, attachments)
|
||||
async with db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
member = await conn.fetchval(
|
||||
'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id)
|
||||
if member is None:
|
||||
raise not_found()
|
||||
# Ключ — "замок": параллельный INSERT того же PK с другой реплики ждёт COMMIT/ROLLBACK этой
|
||||
# транзакции. Просроченный (старше TTL) ключ атомарно переиспользуется как новый.
|
||||
message_id = await conn.fetchval(
|
||||
'INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) '
|
||||
'VALUES ($1, $2, $3, $4, gen_random_uuid()) '
|
||||
'ON CONFLICT (user_id, chat_id, key) DO UPDATE '
|
||||
'SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, '
|
||||
' created_at = clock_timestamp() '
|
||||
'WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => $5) '
|
||||
'RETURNING message_id',
|
||||
user_id, cid, key, fingerprint, settings.idempotency_ttl)
|
||||
if message_id is not None:
|
||||
# Ключ и сообщение фиксируются одним COMMIT: при сбое не остаётся ни того, ни другого,
|
||||
# поэтому транзиентная 5xx не становится "окончательным" результатом ключа.
|
||||
if attachments:
|
||||
# Все вложения — из ЭТОГО чата (иначе 404, как для чужого объекта). FOR SHARE
|
||||
# фиксирует строки до COMMIT; составной FK (attachment_id, chat_id) страхует на уровне БД.
|
||||
# lab5: queued/processing прикреплять можно, failed — 409 (контракт).
|
||||
statuses = await conn.fetch(
|
||||
'SELECT status FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[]) FOR SHARE',
|
||||
cid, attachments)
|
||||
if len(statuses) != len(attachments):
|
||||
raise not_found()
|
||||
if any(r['status'] == 'failed' for r in statuses):
|
||||
raise ApiError(409, 'conflict', 'Вложение не обработано (failed)')
|
||||
row = await conn.fetchrow(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text, text_raw) VALUES ($1, $2, $3, $4, $5) '
|
||||
'RETURNING id, chat_id, sender_id, text, text_raw, created_at',
|
||||
message_id, cid, user_id, *pgtext.split(text))
|
||||
if attachments:
|
||||
await conn.execute(
|
||||
'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) '
|
||||
'SELECT $1, a.id, $2, a.ord - 1 FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord)',
|
||||
message_id, cid, attachments)
|
||||
row = {**dict(row), 'attachment_ids': attachments}
|
||||
else:
|
||||
row = await conn.fetchrow(
|
||||
'SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.text_raw, m.created_at, '
|
||||
+ MESSAGE_ATTACHMENTS.format(m='m') + ' '
|
||||
'FROM idempotency_keys k JOIN messages m ON m.id = k.message_id '
|
||||
'WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3', user_id, cid, key)
|
||||
if row is None:
|
||||
raise unavailable() # не должно случаться: ключ без сообщения не коммитится
|
||||
if row['request_hash'] != fingerprint:
|
||||
raise ApiError(409, 'conflict', 'Idempotency-Key уже использован с другим сообщением')
|
||||
return JSONResponse(message_json(row), status_code=201)
|
||||
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}/messages')
|
||||
async def list_messages(chat_id: str, request: Request):
|
||||
user_id = await current_user(request)
|
||||
cid = v.uuid_value(chat_id, 'chat_id')
|
||||
limit = v.limit_param(request)
|
||||
after = v.decode_cursor(request, 'messages:' + str(cid))
|
||||
async with db.connection() as conn:
|
||||
member = await conn.fetchval(
|
||||
'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id)
|
||||
if member is None:
|
||||
raise not_found()
|
||||
if after:
|
||||
# Keyset: продолжаем строго после последнего элемента прошлой страницы (без OFFSET).
|
||||
rows = await conn.fetch(
|
||||
f'SELECT {MESSAGE_COLUMNS} FROM messages WHERE chat_id = $1 AND (created_at, id) > ($3, $4) '
|
||||
'ORDER BY created_at, id LIMIT $2', cid, limit + 1, *after)
|
||||
else:
|
||||
rows = await conn.fetch(
|
||||
f'SELECT {MESSAGE_COLUMNS} FROM messages WHERE chat_id = $1 '
|
||||
'ORDER BY created_at, id LIMIT $2', cid, limit + 1)
|
||||
return page(rows, limit, 'messages:' + str(cid), message_json)
|
||||
|
||||
# ---------- вложения (lab4, асинхронная обработка с lab5) ----------
|
||||
|
||||
|
||||
async def attachment_for_member(conn, attachment_id: UUID, user_id: UUID):
|
||||
# Доступ = членство в чате вложения. Чужое и несуществующее неразличимы (404).
|
||||
return await conn.fetchrow(
|
||||
f'SELECT {ATTACHMENT_COLUMNS} FROM attachments a '
|
||||
'JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 WHERE a.id = $1',
|
||||
attachment_id, user_id)
|
||||
|
||||
|
||||
async def variants_of(conn, row) -> list:
|
||||
if row['status'] != 'ready':
|
||||
return [] # варианты объявляются только вместе с ready
|
||||
return await conn.fetch(
|
||||
'SELECT name, content_type, size_bytes, width, height, sha256, object_key FROM attachment_variants '
|
||||
'WHERE attachment_id = $1 AND pipeline_version = $2 ORDER BY name', row['id'], row['pipeline_version'])
|
||||
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202)
|
||||
async def upload_attachment(chat_id: str, request: Request):
|
||||
user_id = await current_user(request)
|
||||
cid = v.uuid_value(chat_id, 'chat_id')
|
||||
async with db.connection() as conn:
|
||||
member = await conn.fetchval(
|
||||
'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id)
|
||||
if member is None:
|
||||
raise not_found() # посторонний не загружает ни байта в чужой чат
|
||||
# Backpressure: ограничение принятой, но не обработанной работы. При переполнении — 503 до
|
||||
# чтения тела, клиент повторит позже (Retry-After). Счётчик — по частичному индексу.
|
||||
pending = await conn.fetchval(
|
||||
"SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')")
|
||||
if pending >= settings.max_pending_jobs:
|
||||
raise ApiError(503, 'unavailable', 'Очередь обработки переполнена, повторите позже',
|
||||
{'Retry-After': '5'})
|
||||
filename = v.filename_param(request)
|
||||
declared = images.declared_type(request)
|
||||
data = await images.read_limited(request, settings.max_upload_bytes)
|
||||
# Дешёвые синхронные проверки (сигнатура, лимит пикселей по заголовку, целостность) остаются в API:
|
||||
# заведомо битый/огромный файл получает 413/415 сразу. Декодирование и resize — только в worker.
|
||||
content_type, width, height = await image_checker.inspect(data, declared)
|
||||
sha256 = hashlib.sha256(data).hexdigest()
|
||||
|
||||
attachment_id = uuid.uuid4()
|
||||
staging_key = f'staging/{attachment_id}' # устойчивый staging: приватный prefix S3
|
||||
object_key = f'originals/{attachment_id}' # конечный ключ, его публикует worker
|
||||
# 1) все байты — в устойчивый staging. Сбой -> 503, в БД ничего нет.
|
||||
await storage.put(staging_key, data, content_type)
|
||||
# 2) вложение (queued) + job + outbox — ОДНОЙ транзакцией. После COMMIT работа не теряется:
|
||||
# даже если процесс умрёт до publish, relay любой реплики опубликует job из outbox.
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
row = await conn.fetchrow(
|
||||
'INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, '
|
||||
"width, height, object_key, staging_key, status) "
|
||||
"VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'queued') "
|
||||
'RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, '
|
||||
'error_code, created_at, updated_at',
|
||||
attachment_id, cid, user_id, filename, content_type, len(data), sha256, width, height,
|
||||
object_key, staging_key)
|
||||
await jobs.create(conn, attachment_id, settings.job_max_attempts)
|
||||
except BaseException:
|
||||
await storage.delete_quietly(staging_key) # COMMIT не прошёл: staging без метаданных не нужен
|
||||
raise
|
||||
faults.hit('api_crash_after_commit') # окно отказа «после COMMIT до publish» (тесты)
|
||||
relay.kick()
|
||||
return JSONResponse(attachment_json(row), status_code=202)
|
||||
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}')
|
||||
async def get_attachment(attachment_id: str, request: Request):
|
||||
user_id = await current_user(request)
|
||||
aid = v.uuid_value(attachment_id, 'attachment_id')
|
||||
async with db.connection() as conn:
|
||||
row = await attachment_for_member(conn, aid, user_id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
variants = await variants_of(conn, row)
|
||||
return attachment_json(row, variants)
|
||||
|
||||
|
||||
VARIANTS = {'original', 'thumbnail', 'crop', 'optimized'}
|
||||
|
||||
|
||||
@app.get('/api/v1/attachments/{attachment_id}/content')
|
||||
async def download_attachment(attachment_id: str, request: Request):
|
||||
user_id = await current_user(request)
|
||||
aid = v.uuid_value(attachment_id, 'attachment_id')
|
||||
variant = request.query_params.get('variant', 'original')
|
||||
if variant not in VARIANTS:
|
||||
raise invalid('Неизвестный variant')
|
||||
async with db.connection() as conn:
|
||||
row = await attachment_for_member(conn, aid, user_id)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
if row['status'] != 'ready':
|
||||
# queued/processing/failed: содержимое ещё (или уже) недоступно.
|
||||
raise ApiError(409, 'conflict', 'Вложение ещё не готово' if row['status'] != 'failed'
|
||||
else 'Вложение не обработано')
|
||||
if variant == 'original':
|
||||
key, content_type, digest, name = row['object_key'], row['content_type'], row['sha256'], row['filename']
|
||||
else:
|
||||
found = [x for x in await variants_of(conn, row) if x['name'] == variant]
|
||||
if not found:
|
||||
raise not_found() # отсутствующий вариант ready-объекта
|
||||
x = found[0]
|
||||
key, content_type, digest, name = x['object_key'], x['content_type'], x['sha256'], variant
|
||||
# Байты идут через API после проверки доступа, без redirect на S3.
|
||||
data = await storage.get(key)
|
||||
return Response(data, media_type=content_type, headers={
|
||||
'Cache-Control': 'private, no-store',
|
||||
'X-Content-Type-Options': 'nosniff',
|
||||
'Content-Disposition': "inline; filename*=UTF-8''" + quote(name, safe=''),
|
||||
'ETag': '"' + digest + '"',
|
||||
})
|
||||
@@ -0,0 +1,60 @@
|
||||
"""Версионированные миграции: migrations/NNNN_name.sql применяются по порядку, каждая в своей транзакции.
|
||||
|
||||
Повторный запуск безопасен: применённые версии записаны в schema_migrations и пропускаются,
|
||||
изменённый задним числом файл (другая контрольная сумма) останавливает запуск.
|
||||
Advisory lock не даёт двум параллельным запускам применить одну миграцию дважды.
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
from . import config
|
||||
|
||||
MIGRATIONS = Path(__file__).resolve().parent.parent / 'migrations'
|
||||
LOCK_ID = 2026_0002
|
||||
|
||||
|
||||
async def connect(url: str, attempts: int = 30) -> asyncpg.Connection:
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
return await asyncpg.connect(url, timeout=3)
|
||||
except (OSError, asyncpg.CannotConnectNowError, asyncpg.PostgresConnectionError):
|
||||
if attempt == attempts - 1:
|
||||
raise
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
async def migrate() -> None:
|
||||
# Только DATABASE_URL: сервису migrate не нужны (и не выдаются) S3-credentials API.
|
||||
database_url = config._secret('DATABASE_URL')
|
||||
if not database_url:
|
||||
raise SystemExit('DATABASE_URL_FILE is not configured')
|
||||
conn = await connect(database_url)
|
||||
try:
|
||||
await conn.execute('SELECT pg_advisory_lock($1)', LOCK_ID)
|
||||
await conn.execute(
|
||||
'CREATE TABLE IF NOT EXISTS schema_migrations ('
|
||||
' version text PRIMARY KEY, checksum text NOT NULL,'
|
||||
' applied_at timestamptz NOT NULL DEFAULT clock_timestamp())')
|
||||
applied = {r['version']: r['checksum'] for r in await conn.fetch('SELECT version, checksum FROM schema_migrations')}
|
||||
for path in sorted(MIGRATIONS.glob('[0-9][0-9][0-9][0-9]_*.sql')):
|
||||
version = path.stem
|
||||
sql = path.read_text(encoding='utf-8')
|
||||
checksum = hashlib.sha256(sql.encode()).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
raise SystemExit(f'migration {version} was modified after being applied')
|
||||
print(f'skip {version}')
|
||||
continue
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute('INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)', version, checksum)
|
||||
print(f'apply {version}')
|
||||
finally:
|
||||
await conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(migrate())
|
||||
+149
@@ -0,0 +1,149 @@
|
||||
"""Transactional outbox relay + восстановление работы. Фоновая задача в каждой реплике API.
|
||||
|
||||
1. publish: строки outbox (published_at IS NULL, available_at <= now) берутся FOR UPDATE SKIP LOCKED —
|
||||
две реплики не публикуют одну строку одновременно. Каждое сообщение публикуется persistent с
|
||||
publisher confirm; published_at ставится только после confirm. Падение между confirm и UPDATE даёт
|
||||
повторную публикацию — это at-least-once, worker идемпотентен.
|
||||
2. reap: processing с истёкшей арендой (worker умер, heartbeat прекратился) -> queued + новая строка outbox
|
||||
(или failed, если попытки исчерпаны).
|
||||
3. sweep: queued-задания без неопубликованной строки outbox, "застрявшие" дольше STALE_QUEUED_SECONDS
|
||||
(сообщение потеряно в DLQ по delivery-limit и т.п.), получают новую строку outbox.
|
||||
|
||||
Брокер недоступен -> API продолжает отвечать 202: намерение уже в PostgreSQL, relay допубликует позже.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
|
||||
import aio_pika
|
||||
|
||||
from . import broker, jobs
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
|
||||
log = logging.getLogger('messenger.outbox')
|
||||
BATCH = 100
|
||||
STALE_QUEUED_SECONDS = 120
|
||||
|
||||
|
||||
class Relay:
|
||||
def __init__(self, settings: Settings, db: Database):
|
||||
self.settings = settings
|
||||
self.db = db
|
||||
self.wake = asyncio.Event()
|
||||
self.connection = None
|
||||
self.channel = None
|
||||
self.exchange = None
|
||||
self.broker_ok = False
|
||||
self._last_maintenance = 0.0
|
||||
|
||||
def kick(self) -> None:
|
||||
"""Сразу после COMMIT upload: не ждать следующего опроса (низкая задержка до worker)."""
|
||||
self.wake.set()
|
||||
|
||||
async def _ensure_channel(self) -> None:
|
||||
if self.channel is not None and not self.channel.is_closed:
|
||||
return
|
||||
if self.connection is None:
|
||||
self.connection = await broker.connect(self.settings.broker_url)
|
||||
self.channel = await self.connection.channel(publisher_confirms=True)
|
||||
self.exchange, _queue = await broker.declare(self.channel)
|
||||
|
||||
async def publish_batch(self) -> int:
|
||||
await self._ensure_channel()
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
rows = await conn.fetch(
|
||||
'SELECT id, job_id, payload FROM outbox '
|
||||
'WHERE published_at IS NULL AND available_at <= clock_timestamp() '
|
||||
'ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED', BATCH)
|
||||
published, error = [], None
|
||||
for row in rows:
|
||||
body = row['payload'] if isinstance(row['payload'], str) else json.dumps(row['payload'])
|
||||
try:
|
||||
# await = ожидание publisher confirm; mandatory=True — неприроутированное сообщение
|
||||
# вернётся ошибкой, а не исчезнет молча.
|
||||
await self.exchange.publish(
|
||||
aio_pika.Message(body.encode(), content_type='application/json',
|
||||
delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
|
||||
message_id=str(row['job_id']), type=jobs.JOB_TYPE),
|
||||
routing_key=broker.ROUTING_KEY, mandatory=True, timeout=5)
|
||||
except Exception as exc:
|
||||
error = exc
|
||||
break
|
||||
published.append(row['id'])
|
||||
# Подтверждённые брокером отмечаем даже при ошибке на середине пачки (COMMIT ниже),
|
||||
# неподтверждённые остаются для следующего прохода.
|
||||
if published:
|
||||
await conn.execute('UPDATE outbox SET published_at = clock_timestamp(), '
|
||||
'publish_attempts = publish_attempts + 1 WHERE id = ANY($1::bigint[])',
|
||||
published)
|
||||
if error is not None:
|
||||
raise error
|
||||
return len(rows)
|
||||
|
||||
async def maintenance(self) -> None:
|
||||
"""reap + sweep раз в ~1 с; pg_try_advisory_xact_lock — одна реплика за раз."""
|
||||
loop = asyncio.get_running_loop()
|
||||
if loop.time() - self._last_maintenance < 1.0:
|
||||
return
|
||||
self._last_maintenance = loop.time()
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
if not await conn.fetchval('SELECT pg_try_advisory_xact_lock(20260005)'):
|
||||
return
|
||||
expired = await conn.fetch(
|
||||
"SELECT job_id, attachment_id, attempt, max_attempts FROM attachment_jobs "
|
||||
"WHERE status = 'processing' AND lease_until < clock_timestamp() "
|
||||
"ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED", BATCH)
|
||||
for j in expired:
|
||||
if j['attempt'] >= j['max_attempts']:
|
||||
await jobs.fail(conn, j['job_id'], j['attachment_id'], 'retry_exhausted',
|
||||
'lease expired on last attempt')
|
||||
else:
|
||||
await jobs.requeue(conn, j['job_id'], j['attachment_id'], j['attempt'] + 1, 0.0,
|
||||
'lease expired (worker died or hung)')
|
||||
log.warning('lease expired: job %s attempt %s', j['job_id'], j['attempt'])
|
||||
stale = await conn.fetch(
|
||||
"SELECT j.job_id, j.attachment_id, j.attempt FROM attachment_jobs j "
|
||||
"WHERE j.status = 'queued' AND j.updated_at < clock_timestamp() - make_interval(secs => $1) "
|
||||
"AND j.next_attempt_at < clock_timestamp() - make_interval(secs => $1) "
|
||||
"AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL) "
|
||||
"LIMIT $2 FOR UPDATE OF j SKIP LOCKED", STALE_QUEUED_SECONDS, BATCH)
|
||||
for j in stale:
|
||||
await jobs.enqueue(conn, j['job_id'], j['attachment_id'], j['attempt'] + 1)
|
||||
await conn.execute('UPDATE attachment_jobs SET updated_at = clock_timestamp() WHERE job_id = $1',
|
||||
j['job_id'])
|
||||
log.warning('stale queued job %s re-enqueued', j['job_id'])
|
||||
|
||||
async def run(self) -> None:
|
||||
delay = self.settings.outbox_poll_ms / 1000
|
||||
while True:
|
||||
try:
|
||||
n = await self.publish_batch()
|
||||
self.broker_ok = True
|
||||
await self.maintenance()
|
||||
if n == BATCH:
|
||||
continue # есть ещё — без паузы
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
if self.broker_ok:
|
||||
log.warning('outbox relay error (%s); will retry', type(exc).__name__)
|
||||
self.broker_ok = False
|
||||
self.channel = None
|
||||
try:
|
||||
await self.maintenance() # reap/sweep не зависят от брокера
|
||||
except Exception:
|
||||
pass
|
||||
await asyncio.sleep(1.0)
|
||||
continue
|
||||
self.wake.clear()
|
||||
try:
|
||||
await asyncio.wait_for(self.wake.wait(), delay)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.connection is not None:
|
||||
await self.connection.close()
|
||||
@@ -0,0 +1,22 @@
|
||||
"""Точное хранение строк с U+0000 в PostgreSQL.
|
||||
|
||||
Тип text в PostgreSQL не может содержать байт 0x00, а JSON-контракт допускает "\\u0000" в тексте
|
||||
сообщения, display_name и title. Раньше такой INSERT падал с 500 (замечание к lab3, PR #99).
|
||||
|
||||
Схема (только expand, совместима с предыдущей версией при rolling update):
|
||||
- столбец text/display_name/title получает ту же строку, где U+0000 заменён на U+FFFD: длина в code points
|
||||
не меняется, поэтому CHECK char_length и прежние реплики продолжают работать;
|
||||
- столбец *_raw bytea получает точные UTF-8 байты исходной строки, но ТОЛЬКО если в ней был U+0000
|
||||
(иначе NULL, лишнего места не занимает);
|
||||
- при чтении API возвращает декодированный *_raw, если он есть, иначе обычный столбец.
|
||||
"""
|
||||
|
||||
|
||||
def split(value: str) -> tuple[str, bytes | None]:
|
||||
if '\x00' in value:
|
||||
return value.replace('\x00', '\ufffd'), value.encode('utf-8')
|
||||
return value, None
|
||||
|
||||
|
||||
def join(text: str, raw: bytes | None) -> str:
|
||||
return bytes(raw).decode('utf-8') if raw is not None else text
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Явный операторский replay failed-заданий (failed -> queued) и просмотр реестра failed/DLQ.
|
||||
|
||||
docker compose --env-file .env run --rm --no-deps worker python -m app.replay list
|
||||
docker compose --env-file .env run --rm --no-deps worker python -m app.replay replay --attachment <uuid> --reason "..." [--dry-run]
|
||||
docker compose --env-file .env run --rm --no-deps worker python -m app.replay replay --all-failed --reason "..." [--dry-run]
|
||||
|
||||
Безопасность: переводится только задание в статусе failed (строка блокируется FOR UPDATE). Для ready/done
|
||||
replay — отказ: "повторить" не означает исполнить дубль успешной работы. Каждое действие пишется в
|
||||
job_audit (прежний статус и ошибка, actor, причина), счётчик попыток начинается заново, job_id тот же —
|
||||
логический эффект (attachment_id, pipeline_version, variant) защищён от дубля так же, как при redelivery.
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import getpass
|
||||
import os
|
||||
import sys
|
||||
from uuid import UUID
|
||||
|
||||
from . import config, jobs
|
||||
from .db import Database
|
||||
|
||||
|
||||
async def list_failed(db: Database) -> None:
|
||||
async with db.connection() as conn:
|
||||
rows = await conn.fetch(
|
||||
"SELECT j.job_id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.finished_at "
|
||||
"FROM attachment_jobs j WHERE j.status = 'failed' ORDER BY j.finished_at")
|
||||
print(f'failed jobs: {len(rows)}')
|
||||
for r in rows:
|
||||
print(f" attachment={r['attachment_id']} job={r['job_id']} attempts={r['attempt']}/{r['max_attempts']} "
|
||||
f"code={r['error_code']} reason={r['last_error']!r} at={r['finished_at']:%Y-%m-%dT%H:%M:%SZ}")
|
||||
|
||||
|
||||
async def replay(db: Database, attachment: UUID | None, all_failed: bool, reason: str, dry_run: bool) -> int:
|
||||
actor = os.environ.get('REPLAY_ACTOR') or getpass.getuser()
|
||||
done = 0
|
||||
async with db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
if all_failed:
|
||||
targets = await conn.fetch("SELECT job_id, attachment_id, status, error_code FROM attachment_jobs "
|
||||
"WHERE status = 'failed' ORDER BY finished_at FOR UPDATE")
|
||||
else:
|
||||
targets = await conn.fetch('SELECT job_id, attachment_id, status, error_code FROM attachment_jobs '
|
||||
'WHERE attachment_id = $1 FOR UPDATE', attachment)
|
||||
if not targets:
|
||||
print('no such job')
|
||||
return 1
|
||||
for t in targets:
|
||||
if t['status'] != 'failed':
|
||||
print(f"skip attachment={t['attachment_id']}: status={t['status']} — replay only for failed")
|
||||
continue
|
||||
print(f"{'would replay' if dry_run else 'replay'} attachment={t['attachment_id']} "
|
||||
f"job={t['job_id']} previous={t['error_code']}")
|
||||
if dry_run:
|
||||
continue
|
||||
await conn.execute(
|
||||
'INSERT INTO job_audit (job_id, action, previous_status, previous_error, actor, reason) '
|
||||
"VALUES ($1, 'replay', $2, $3, $4, $5)", t['job_id'], t['status'], t['error_code'], actor, reason)
|
||||
await conn.execute(
|
||||
"UPDATE attachment_jobs SET status = 'queued', attempt = 0, error_code = NULL, last_error = NULL, "
|
||||
"next_attempt_at = clock_timestamp(), finished_at = NULL, worker_id = NULL, "
|
||||
"updated_at = clock_timestamp() WHERE job_id = $1 AND status = 'failed'", t['job_id'])
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = $1 AND status = 'failed'", t['attachment_id'])
|
||||
await jobs.enqueue(conn, t['job_id'], t['attachment_id'], attempt=1)
|
||||
done += 1
|
||||
print(f'replayed: {done}' + (' (dry run, nothing changed)' if dry_run else ''))
|
||||
return 0
|
||||
|
||||
|
||||
async def main_async(args) -> int:
|
||||
db = Database(config.load())
|
||||
await db.start()
|
||||
try:
|
||||
if args.command == 'list':
|
||||
await list_failed(db)
|
||||
return 0
|
||||
return await replay(db, args.attachment, args.all_failed, args.reason, args.dry_run)
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
||||
sub = p.add_subparsers(dest='command', required=True)
|
||||
sub.add_parser('list')
|
||||
r = sub.add_parser('replay')
|
||||
target = r.add_mutually_exclusive_group(required=True)
|
||||
target.add_argument('--attachment', type=UUID)
|
||||
target.add_argument('--all-failed', action='store_true')
|
||||
r.add_argument('--reason', required=True)
|
||||
r.add_argument('--dry-run', action='store_true')
|
||||
sys.exit(asyncio.run(main_async(p.parse_args())))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Bootstrap S3: создаёт приватный bucket для вложений. Запуск: python -m app.s3_bootstrap (сервис s3-init).
|
||||
|
||||
Идемпотентно: повторный запуск не трогает существующий bucket и данные в нём.
|
||||
"Приватный" = у bucket нет политики анонимного доступа. Если политика есть (например, кто-то включил
|
||||
public read руками), она удаляется. В конце выполняется проверка снаружи API: анонимный GET объекта
|
||||
должен получить отказ (403), иначе bootstrap завершается ошибкой и API не стартует.
|
||||
"""
|
||||
import io
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from minio import Minio
|
||||
from minio.error import S3Error
|
||||
|
||||
|
||||
def secret(name: str) -> str:
|
||||
return Path(os.environ[name + '_FILE']).read_text(encoding='utf-8').strip()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
endpoint = os.environ.get('S3_ENDPOINT', 'http://s3:9000')
|
||||
bucket = os.environ.get('S3_BUCKET', 'messenger-attachments')
|
||||
region = os.environ.get('S3_REGION', 'us-east-1')
|
||||
url = urlparse(endpoint)
|
||||
client = Minio(url.netloc, access_key=secret('S3_ACCESS_KEY_ID'), secret_key=secret('S3_SECRET_ACCESS_KEY'),
|
||||
secure=url.scheme == 'https', region=region)
|
||||
|
||||
for attempt in range(30):
|
||||
try:
|
||||
exists = client.bucket_exists(bucket)
|
||||
break
|
||||
except Exception as exc: # S3 ещё стартует
|
||||
if attempt == 29:
|
||||
raise SystemExit(f's3 not reachable: {type(exc).__name__}')
|
||||
time.sleep(1)
|
||||
if exists:
|
||||
print(f'bucket {bucket}: exists')
|
||||
else:
|
||||
client.make_bucket(bucket, location=region)
|
||||
print(f'bucket {bucket}: created')
|
||||
|
||||
try:
|
||||
client.get_bucket_policy(bucket)
|
||||
client.delete_bucket_policy(bucket)
|
||||
print(f'bucket {bucket}: anonymous policy removed')
|
||||
except S3Error as exc:
|
||||
if exc.code not in ('NoSuchBucketPolicy', 'NoSuchPolicy'):
|
||||
raise
|
||||
print(f'bucket {bucket}: no bucket policy (private)')
|
||||
|
||||
# Проверка приватности без credentials: служебный объект, анонимный GET, ожидаем 403.
|
||||
probe = '_bootstrap/private-check'
|
||||
client.put_object(bucket, probe, io.BytesIO(b'private'), 7, content_type='text/plain')
|
||||
try:
|
||||
urllib.request.urlopen(f'{endpoint}/{bucket}/{probe}', timeout=5)
|
||||
print('ERROR: anonymous GET returned object, bucket is public', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except urllib.error.HTTPError as exc:
|
||||
print(f'anonymous GET {bucket}/{probe}: HTTP {exc.code} (denied)')
|
||||
if exc.code not in (401, 403):
|
||||
sys.exit(1)
|
||||
finally:
|
||||
client.remove_object(bucket, probe)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,47 @@
|
||||
"""Пароли (Argon2id) и непрозрачные токены сессий."""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import secrets
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
|
||||
# Параметры по умолчанию argon2-cffi = профиль RFC 9106 "low memory": Argon2id, m=64 MiB, t=3, p=4.
|
||||
_hasher = PasswordHasher()
|
||||
# Хеш для несуществующего пользователя: вход выполняет ту же работу и не выдаёт по времени,
|
||||
# что такого username нет.
|
||||
_DUMMY_HASH = _hasher.hash('dummy-password-for-timing')
|
||||
|
||||
|
||||
class PasswordService:
|
||||
"""KDF дорогой по CPU и памяти: считаем в отдельном ограниченном пуле потоков,
|
||||
чтобы не блокировать event loop и не съесть всю память параллельными входами."""
|
||||
|
||||
def __init__(self, workers: int):
|
||||
self._pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix='argon2')
|
||||
|
||||
async def hash(self, password: str) -> str:
|
||||
return await asyncio.get_running_loop().run_in_executor(self._pool, _hasher.hash, password)
|
||||
|
||||
async def verify(self, password_hash: str | None, password: str) -> bool:
|
||||
def check() -> bool:
|
||||
try:
|
||||
return _hasher.verify(password_hash or _DUMMY_HASH, password) and password_hash is not None
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
return await asyncio.get_running_loop().run_in_executor(self._pool, check)
|
||||
|
||||
def close(self) -> None:
|
||||
self._pool.shutdown(wait=False)
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
# 32 случайных байта = 256 бит энтропии из CSPRNG ОС; base64url -> 43 символа.
|
||||
return secrets.token_urlsafe(32)
|
||||
|
||||
|
||||
def token_hash(token: str) -> bytes:
|
||||
# Для токена с 256 битами энтропии быстрый SHA-256 достаточен: перебор невозможен.
|
||||
# Для паролей (низкая энтропия) так нельзя — там нужен медленный KDF.
|
||||
return hashlib.sha256(token.encode('ascii')).digest()
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
"""Сессии: PostgreSQL — источник истины, Valkey — read-through кэш с TTL.
|
||||
|
||||
Политика:
|
||||
- создание: INSERT в PostgreSQL (COMMIT), затем SET в кэш с TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS);
|
||||
- проверка: кэш -> при промахе PostgreSQL -> заполнить кэш;
|
||||
- logout: UPDATE revoked_at в PostgreSQL (COMMIT), затем DEL из кэша;
|
||||
- любой сбой Valkey: работаем только с PostgreSQL и SESSION_CACHE_TTL_SECONDS не доверяем кэшу
|
||||
(все записи, сделанные до сбоя, за это время гарантированно истекут — например, не удалённая
|
||||
при logout копия отозванной сессии). Flush/рестарт Valkey = пустой кэш, сессии не теряются.
|
||||
- lab3, несколько реплик: окно недоверия ОБЩЕЕ. Реплика, получившая ошибку Valkey, записывает его
|
||||
в PostgreSQL (cache_state), а попадание в кэш принимается, только если общий признак проверен
|
||||
не раньше SHARED_STATE_MAX_AGE назад. Иначе реплика, которая сама сбоя не видела, могла бы
|
||||
отдать копию сессии, отозванной на другой реплике во время разрыва с Valkey.
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from uuid import UUID
|
||||
|
||||
import redis.asyncio as redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from .config import Settings
|
||||
from .db import Database
|
||||
from .security import token_hash
|
||||
|
||||
log = logging.getLogger('messenger.sessions')
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Session:
|
||||
user_id: UUID
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
# Насколько устаревшим может быть знание реплики об общем окне недоверия (не чаще 2 запросов/с к БД).
|
||||
SHARED_STATE_MAX_AGE = 0.5
|
||||
|
||||
|
||||
class SessionCache:
|
||||
def __init__(self, url: str | None, ttl: int, timeout: float, db: Database):
|
||||
self.ttl = ttl
|
||||
self.db = db
|
||||
self.client = redis.Redis.from_url(
|
||||
url, socket_timeout=timeout, socket_connect_timeout=timeout,
|
||||
retry_on_timeout=False, health_check_interval=30,
|
||||
) if url else None
|
||||
self._distrust_until = 0.0 # локально (monotonic): своя ошибка действует сразу
|
||||
self._shared_trusted = False # результат последней проверки общего признака
|
||||
self._shared_checked = -1e9
|
||||
self._lock = asyncio.Lock()
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return 'session:' + digest.hex()
|
||||
|
||||
def usable(self) -> bool:
|
||||
return self.client is not None and time.monotonic() >= self._distrust_until
|
||||
|
||||
async def trusted(self) -> bool:
|
||||
"""Можно ли принять попадание в кэш: нет ни своего, ни общего (с других реплик) окна недоверия."""
|
||||
if not self.usable():
|
||||
return False
|
||||
if time.monotonic() - self._shared_checked > SHARED_STATE_MAX_AGE:
|
||||
async with self._lock:
|
||||
if time.monotonic() - self._shared_checked > SHARED_STATE_MAX_AGE:
|
||||
try:
|
||||
async with self.db.pool.acquire(timeout=1) as conn:
|
||||
self._shared_trusted = await conn.fetchval(
|
||||
'SELECT distrust_until <= clock_timestamp() FROM cache_state WHERE id = 1',
|
||||
timeout=1) is True
|
||||
except Exception: # без БД сессию всё равно не проверить; кэшу не доверяем
|
||||
self._shared_trusted = False
|
||||
self._shared_checked = time.monotonic()
|
||||
return self._shared_trusted
|
||||
|
||||
async def _failed(self, op: str) -> None:
|
||||
# Не знаем, что успело/не успело записаться: перестаём читать кэш на один максимальный TTL —
|
||||
# сами сразу и все остальные реплики через общий признак в PostgreSQL.
|
||||
self._distrust_until = time.monotonic() + self.ttl
|
||||
self._shared_checked = -1e9
|
||||
log.warning('valkey %s failed; bypassing cache for %ss on all replicas', op, self.ttl)
|
||||
try:
|
||||
async with self.db.pool.acquire(timeout=1) as conn:
|
||||
await conn.execute(
|
||||
'UPDATE cache_state SET distrust_until = greatest(distrust_until, '
|
||||
'clock_timestamp() + make_interval(secs => $1)) WHERE id = 1', self.ttl, timeout=1)
|
||||
except Exception: # БД недоступна — и кэш, и сессии всё равно проверить нельзя
|
||||
log.warning('could not publish cache distrust window', exc_info=True)
|
||||
|
||||
async def get(self, digest: bytes) -> Session | None:
|
||||
if not await self.trusted():
|
||||
return None
|
||||
try:
|
||||
raw = await self.client.get(self.key(digest))
|
||||
except (RedisError, OSError):
|
||||
await self._failed('get')
|
||||
return None
|
||||
if raw is None:
|
||||
return None
|
||||
data = json.loads(raw)
|
||||
return Session(UUID(data['u']), datetime.fromtimestamp(data['e'], timezone.utc))
|
||||
|
||||
async def put(self, digest: bytes, session: Session) -> None:
|
||||
if not self.usable():
|
||||
return
|
||||
remaining = int((session.expires_at - datetime.now(timezone.utc)).total_seconds())
|
||||
ttl = min(self.ttl, remaining)
|
||||
if ttl <= 0:
|
||||
return
|
||||
value = json.dumps({'u': str(session.user_id), 'e': session.expires_at.timestamp()})
|
||||
try:
|
||||
await self.client.set(self.key(digest), value, ex=ttl)
|
||||
except (RedisError, OSError):
|
||||
await self._failed('set')
|
||||
|
||||
async def delete(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.delete(self.key(digest))
|
||||
except (RedisError, OSError):
|
||||
await self._failed('delete')
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, settings: Settings, db: Database):
|
||||
self.ttl = settings.session_ttl
|
||||
self.db = db
|
||||
self.cache = SessionCache(settings.cache_url, settings.session_cache_ttl, settings.cache_timeout, db)
|
||||
|
||||
async def create(self, user_id: UUID, token: str) -> Session:
|
||||
digest = token_hash(token)
|
||||
async with self.db.connection() as conn:
|
||||
# Время берём из БД, чтобы срок не зависел от часов конкретной реплики API.
|
||||
expires_at = await conn.fetchval(
|
||||
'INSERT INTO sessions (token_hash, user_id, expires_at) '
|
||||
'VALUES ($1, $2, clock_timestamp() + make_interval(secs => $3)) RETURNING expires_at',
|
||||
digest, user_id, self.ttl,
|
||||
)
|
||||
session = Session(user_id, expires_at)
|
||||
await self.cache.put(digest, session)
|
||||
return session
|
||||
|
||||
async def resolve(self, token: str) -> Session | None:
|
||||
digest = token_hash(token)
|
||||
now = datetime.now(timezone.utc)
|
||||
cached = await self.cache.get(digest)
|
||||
if cached is not None and cached.expires_at > now:
|
||||
return cached
|
||||
async with self.db.connection() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'SELECT user_id, expires_at FROM sessions '
|
||||
'WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > clock_timestamp()',
|
||||
digest,
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
session = Session(row['user_id'], row['expires_at'])
|
||||
await self.cache.put(digest, session)
|
||||
return session
|
||||
|
||||
async def revoke(self, token: str) -> bool:
|
||||
digest = token_hash(token)
|
||||
async with self.db.connection() as conn:
|
||||
status = await conn.execute(
|
||||
'UPDATE sessions SET revoked_at = clock_timestamp() '
|
||||
'WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > clock_timestamp()',
|
||||
digest,
|
||||
)
|
||||
# Инвалидация кэша строго после COMMIT в источнике истины.
|
||||
await self.cache.delete(digest)
|
||||
return status == 'UPDATE 1'
|
||||
|
||||
async def purge_expired(self) -> int:
|
||||
async with self.db.connection() as conn:
|
||||
status = await conn.execute(
|
||||
"DELETE FROM sessions WHERE expires_at < clock_timestamp() - interval '1 day'")
|
||||
return int(status.split()[-1])
|
||||
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
"""Приватное S3-хранилище оригиналов вложений (MinIO, S3 API).
|
||||
|
||||
Клиент minio-py синхронный, поэтому вызовы уходят в пул потоков (asyncio.to_thread) и не блокируют
|
||||
event loop. Таймауты и число повторов ограничены: недоступный S3 даёт 503 за секунды, а не зависание.
|
||||
Ключи объектов назначает сервер; bucket приватный, байты отдаёт только API после проверки доступа.
|
||||
"""
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import urllib3
|
||||
from minio import Minio
|
||||
from minio.error import MinioException, S3Error
|
||||
|
||||
from .config import Settings
|
||||
from .errors import ApiError, unavailable
|
||||
|
||||
log = logging.getLogger('messenger.storage')
|
||||
|
||||
|
||||
class ObjectMissing(Exception):
|
||||
"""Метаданные в БД есть, а объекта в bucket нет (рассогласование хранилищ)."""
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, settings: Settings):
|
||||
endpoint = urlparse(settings.s3_endpoint)
|
||||
self.bucket = settings.s3_bucket
|
||||
# Без повторов на уровне HTTP-клиента: повтор удваивал бы время отказа. Общий дедлайн операции —
|
||||
# в _call: getaddrinfo (DNS остановленного контейнера ~8 с) таймаутами urllib3 не ограничен.
|
||||
self.deadline = settings.s3_timeout
|
||||
http = urllib3.PoolManager(
|
||||
timeout=urllib3.Timeout(connect=2, read=settings.s3_timeout),
|
||||
retries=urllib3.Retry(total=0, redirect=0),
|
||||
maxsize=16,
|
||||
)
|
||||
self.client = Minio(endpoint.netloc, access_key=settings.s3_access_key,
|
||||
secret_key=settings.s3_secret_key, secure=endpoint.scheme == 'https',
|
||||
region=settings.s3_region, http_client=http)
|
||||
|
||||
async def _call(self, fn, *args, **kwargs):
|
||||
try:
|
||||
return await asyncio.wait_for(asyncio.to_thread(fn, *args, **kwargs), self.deadline)
|
||||
except asyncio.TimeoutError:
|
||||
# Поток дорабатывает в фоне (его нельзя прервать), но клиент получает 503 в пределах дедлайна.
|
||||
log.warning('S3 deadline %.1fs exceeded on %s', self.deadline, fn.__name__)
|
||||
raise unavailable() from None
|
||||
except S3Error as exc:
|
||||
if exc.code in ('NoSuchKey', 'NoSuchObject'):
|
||||
raise ObjectMissing() from None
|
||||
log.warning('S3 error %s on %s', exc.code, fn.__name__)
|
||||
raise unavailable() from None
|
||||
except (MinioException, urllib3.exceptions.HTTPError, OSError, ValueError):
|
||||
# Нет связи, таймаут, 5xx после повтора — для клиента это временная недоступность.
|
||||
log.warning('S3 unavailable on %s', fn.__name__, exc_info=True)
|
||||
raise unavailable() from None
|
||||
|
||||
async def put(self, key: str, data: bytes, content_type: str) -> None:
|
||||
await self._call(self.client.put_object, self.bucket, key, io.BytesIO(data), len(data),
|
||||
content_type=content_type)
|
||||
|
||||
async def get(self, key: str) -> bytes:
|
||||
def read() -> bytes:
|
||||
response = self.client.get_object(self.bucket, key)
|
||||
try:
|
||||
return response.read()
|
||||
finally:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
try:
|
||||
return await self._call(read)
|
||||
except ObjectMissing:
|
||||
log.error('object %s is missing in bucket %s', key, self.bucket)
|
||||
raise ApiError(500, 'internal_error', 'Внутренняя ошибка сервера') from None
|
||||
|
||||
async def delete(self, key: str) -> None:
|
||||
await self._call(self.client.remove_object, self.bucket, key)
|
||||
|
||||
async def delete_quietly(self, key: str) -> None:
|
||||
"""Компенсация после неудачной записи метаданных. Сбой здесь не меняет ответ клиенту."""
|
||||
try:
|
||||
await asyncio.shield(self.delete(key))
|
||||
except Exception:
|
||||
log.error('compensation failed: orphan object %s left in bucket', key)
|
||||
|
||||
# ---------- lab5: операции worker ----------
|
||||
|
||||
async def get_raw(self, key: str) -> bytes:
|
||||
"""Как get(), но отсутствие объекта — ObjectMissing (worker решает сам: это постоянная ошибка)."""
|
||||
def read() -> bytes:
|
||||
response = self.client.get_object(self.bucket, key)
|
||||
try:
|
||||
return response.read()
|
||||
finally:
|
||||
response.close()
|
||||
response.release_conn()
|
||||
return await self._call(read)
|
||||
|
||||
async def copy(self, source_key: str, target_key: str) -> None:
|
||||
"""Server-side copy внутри bucket: байты оригинала не гоняются через worker повторно."""
|
||||
from minio.commonconfig import CopySource
|
||||
await self._call(self.client.copy_object, self.bucket, target_key, CopySource(self.bucket, source_key))
|
||||
|
||||
async def exists(self, key: str) -> bool:
|
||||
try:
|
||||
await self._call(self.client.stat_object, self.bucket, key)
|
||||
return True
|
||||
except ObjectMissing:
|
||||
return False
|
||||
|
||||
async def list_older(self, prefix: str, older_than_seconds: int) -> list[str]:
|
||||
"""Ключи под prefix старше заданного возраста (для retention staging)."""
|
||||
import datetime as dt
|
||||
|
||||
def scan() -> list[str]:
|
||||
border = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=older_than_seconds)
|
||||
return [o.object_name for o in self.client.list_objects(self.bucket, prefix=prefix, recursive=True)
|
||||
if o.last_modified is not None and o.last_modified < border]
|
||||
return await self._call(scan)
|
||||
@@ -0,0 +1,138 @@
|
||||
"""Строгий разбор JSON и параметров: лишние поля, неверные типы, malformed JSON -> 400."""
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from .errors import ApiError, invalid
|
||||
|
||||
MAX_BODY_BYTES = 64 * 1024
|
||||
USERNAME_RE = re.compile(r'[a-z0-9_]{3,32}')
|
||||
|
||||
|
||||
async def read_json(request: Request) -> dict:
|
||||
media_type = request.headers.get('content-type', '').split(';')[0].strip().lower()
|
||||
if media_type != 'application/json':
|
||||
raise ApiError(415, 'unsupported_media_type', 'Ожидается Content-Type: application/json')
|
||||
body = await request.body()
|
||||
if len(body) > MAX_BODY_BYTES:
|
||||
raise ApiError(413, 'payload_too_large', 'Слишком большое тело запроса')
|
||||
try:
|
||||
data = json.loads(body.decode('utf-8'))
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise invalid('Некорректный JSON') from None
|
||||
if not isinstance(data, dict):
|
||||
raise invalid('Ожидается JSON-объект')
|
||||
return data
|
||||
|
||||
|
||||
def fields(data: dict, required: set[str], optional: set[str] = frozenset()) -> None:
|
||||
extra = set(data) - required - optional
|
||||
if extra:
|
||||
raise invalid('Неизвестные поля: ' + ', '.join(sorted(extra)))
|
||||
missing = required - set(data)
|
||||
if missing:
|
||||
raise invalid('Отсутствуют поля: ' + ', '.join(sorted(missing)))
|
||||
|
||||
|
||||
def text_field(data: dict, name: str, min_len: int, max_len: int, *, allow_blank: bool = False) -> str:
|
||||
value = data[name]
|
||||
if not isinstance(value, str):
|
||||
raise invalid(f'Поле {name} должно быть строкой')
|
||||
try:
|
||||
value.encode('utf-8') # отсекает одиночные суррогаты из \uD800-экранирования
|
||||
except UnicodeEncodeError:
|
||||
raise invalid(f'Поле {name} содержит некорректный Unicode') from None
|
||||
if not min_len <= len(value) <= max_len: # len(str) в Python = число code points
|
||||
raise invalid(f'Длина поля {name} должна быть {min_len}..{max_len} символов')
|
||||
if not allow_blank and not value.strip():
|
||||
raise invalid(f'Поле {name} не может состоять из пробелов')
|
||||
return value
|
||||
|
||||
|
||||
def username(data: dict) -> str:
|
||||
value = data['username']
|
||||
if not isinstance(value, str) or not USERNAME_RE.fullmatch(value):
|
||||
raise invalid('username должен соответствовать [a-z0-9_]{3,32}')
|
||||
return value
|
||||
|
||||
|
||||
def uuid_value(value, what: str = 'идентификатор') -> UUID:
|
||||
if not isinstance(value, str):
|
||||
raise invalid(f'Некорректный {what}')
|
||||
try:
|
||||
parsed = UUID(value)
|
||||
except ValueError:
|
||||
raise invalid(f'Некорректный {what}') from None
|
||||
if str(parsed) != value.lower(): # только каноническая форма 8-4-4-4-12
|
||||
raise invalid(f'Некорректный {what}')
|
||||
return parsed
|
||||
|
||||
|
||||
def limit_param(request: Request) -> int:
|
||||
raw = request.query_params.get('limit')
|
||||
if raw is None:
|
||||
return 50
|
||||
if not re.fullmatch(r'[0-9]{1,3}', raw) or not 1 <= int(raw) <= 100:
|
||||
raise invalid('limit должен быть целым 1..100')
|
||||
return int(raw)
|
||||
|
||||
|
||||
# Курсор: base64url(JSON {"s": область, "t": created_at, "i": id}). Для клиента это непрозрачная строка.
|
||||
# Подделка курсора ничего не даёт: доступ к чату проверяется отдельно, а область привязывает
|
||||
# курсор к конкретному чату/списку — курсор другого чата отклоняется с 400.
|
||||
|
||||
def encode_cursor(scope: str, created_at: datetime, item_id: UUID) -> str:
|
||||
raw = json.dumps({'s': scope, 't': created_at.isoformat(), 'i': str(item_id)}, separators=(',', ':'))
|
||||
return base64.urlsafe_b64encode(raw.encode()).decode().rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(request: Request, scope: str) -> tuple[datetime, UUID] | None:
|
||||
raw = request.query_params.get('cursor')
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
if len(raw) > 512:
|
||||
raise ValueError
|
||||
data = json.loads(base64.urlsafe_b64decode(raw + '=' * (-len(raw) % 4)))
|
||||
if not isinstance(data, dict) or set(data) != {'s', 't', 'i'} or data['s'] != scope:
|
||||
raise ValueError
|
||||
created_at = datetime.fromisoformat(data['t'])
|
||||
if created_at.tzinfo is None:
|
||||
raise ValueError
|
||||
return created_at, UUID(data['i'])
|
||||
except (ValueError, TypeError, UnicodeDecodeError):
|
||||
raise invalid('Некорректный cursor') from None
|
||||
|
||||
|
||||
# Имя файла — только для отображения: в ключ S3 и пути оно не попадает. Разделители путей,
|
||||
# NUL и управляющие символы (C0, DEL, C1) отклоняются, чтобы имя нельзя было выдать за путь.
|
||||
FILENAME_FORBIDDEN_RE = re.compile(r'[/\\\x00-\x1f\x7f-\x9f]')
|
||||
|
||||
|
||||
def filename_param(request: Request) -> str:
|
||||
values = request.query_params.getlist('filename')
|
||||
if len(values) != 1:
|
||||
raise invalid('Требуется ровно один параметр filename')
|
||||
value = values[0]
|
||||
try:
|
||||
value.encode('utf-8')
|
||||
except UnicodeEncodeError:
|
||||
raise invalid('filename содержит некорректный Unicode') from None
|
||||
if not 1 <= len(value) <= 255 or not value.strip() or value in ('.', '..') \
|
||||
or FILENAME_FORBIDDEN_RE.search(value):
|
||||
raise invalid('Недопустимое имя файла')
|
||||
return value
|
||||
|
||||
|
||||
def attachment_ids(data: dict) -> list[UUID]:
|
||||
raw = data.get('attachment_ids', [])
|
||||
if not isinstance(raw, list) or len(raw) > 10:
|
||||
raise invalid('attachment_ids должен быть массивом из 0..10 UUID')
|
||||
ids = [uuid_value(x, 'attachment_id') for x in raw]
|
||||
if len(set(ids)) != len(ids):
|
||||
raise invalid('attachment_ids содержит дубликаты')
|
||||
return ids
|
||||
+307
@@ -0,0 +1,307 @@
|
||||
"""Worker обработки вложений: python -m app.worker (отдельный сервис, не API).
|
||||
|
||||
Одна доставка сообщения:
|
||||
1. проверить сообщение по contracts/jobs.schema.json (иначе reject без requeue -> DLQ);
|
||||
2. claim в PostgreSQL: queued (или processing с истёкшей арендой) -> processing, attempt+1, lease;
|
||||
done/failed/чужая живая аренда/ещё не время retry -> no-op + ack;
|
||||
3. heartbeat продлевает lease, пока идёт работа (отличает долгую обработку от зависшей/мёртвой);
|
||||
4. оригинал из staging, сверка size/sha256 с доверенными метаданными из БД;
|
||||
5. варианты в дочернем процессе (лимиты памяти/времени), запись в S3 по ДЕТЕРМИНИРОВАННЫМ ключам
|
||||
variants/<id>/v<pipeline>/<name> + server-side copy staging -> originals/<id>;
|
||||
6. COMMIT результата с fencing по (worker_id, attempt): variants (ON CONFLICT DO NOTHING) + ready;
|
||||
7. удалить staging (после безопасной фиксации), затем ack.
|
||||
Сбой на любом шаге до ack -> redelivery/reaper -> повтор шагов 4–6 перезаписывает те же ключи тем же
|
||||
содержимым и не создаёт второй набор вариантов (ключ эффекта (attachment_id, pipeline_version, variant)).
|
||||
"""
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import secrets
|
||||
import signal
|
||||
import socket
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import jsonschema
|
||||
|
||||
from . import broker, config, faults, imaging, jobs
|
||||
from .db import Database
|
||||
from .errors import ApiError
|
||||
from .storage import ObjectMissing, Storage
|
||||
|
||||
log = logging.getLogger('messenger.worker')
|
||||
SCHEMA = json.loads((Path(__file__).resolve().parent.parent / 'contracts/jobs.schema.json').read_text(encoding='utf-8'))
|
||||
VALIDATOR = jsonschema.Draft202012Validator(SCHEMA, format_checker=jsonschema.FormatChecker())
|
||||
ALIVE_FILE = Path('/tmp/worker-alive')
|
||||
|
||||
|
||||
class LostLease(Exception):
|
||||
"""Аренду перехватили (истекла): результат фиксирует новый владелец, этот — только ack."""
|
||||
|
||||
|
||||
class Worker:
|
||||
def __init__(self):
|
||||
self.settings = config.load()
|
||||
self.db = Database(self.settings)
|
||||
self.storage = Storage(self.settings)
|
||||
self.worker_id = f'{socket.gethostname()}-{secrets.token_hex(3)}'
|
||||
self.slots = asyncio.Semaphore(self.settings.worker_concurrency)
|
||||
self.inflight: set[asyncio.Task] = set()
|
||||
self.stopping = asyncio.Event()
|
||||
self.connection = None
|
||||
|
||||
# ---------- приём сообщения ----------
|
||||
|
||||
async def on_message(self, message) -> None:
|
||||
task = asyncio.current_task()
|
||||
self.inflight.add(task)
|
||||
try:
|
||||
try:
|
||||
payload = json.loads(message.body)
|
||||
VALIDATOR.validate(payload)
|
||||
except (ValueError, jsonschema.ValidationError):
|
||||
log.error('invalid job message %s -> DLQ', message.message_id)
|
||||
await message.reject(requeue=False)
|
||||
return
|
||||
async with self.slots:
|
||||
try:
|
||||
await self.process(payload)
|
||||
except Exception:
|
||||
# Инфраструктура (БД/S3) недоступна настолько, что даже состояние не записать:
|
||||
# вернуть сообщение в очередь с паузой; delivery-limit страхует от бесконечного цикла.
|
||||
log.exception('job %s: infrastructure error, requeue', payload['job_id'])
|
||||
await asyncio.sleep(2)
|
||||
await message.nack(requeue=True)
|
||||
return
|
||||
await message.ack() # только после устойчивого эффекта (или no-op)
|
||||
finally:
|
||||
self.inflight.discard(task)
|
||||
|
||||
# ---------- обработка ----------
|
||||
|
||||
async def claim(self, job_id: str, attachment_id: str):
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
j = await conn.fetchrow(
|
||||
'SELECT j.status, j.attempt, j.max_attempts, '
|
||||
' coalesce(j.lease_until > clock_timestamp(), false) AS leased, '
|
||||
' j.next_attempt_at > clock_timestamp() AS not_yet, '
|
||||
' a.staging_key, a.object_key, a.size_bytes, a.sha256, a.pipeline_version '
|
||||
'FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id '
|
||||
'WHERE j.job_id = $1 AND j.attachment_id = $2 FOR UPDATE OF j', job_id, attachment_id)
|
||||
if j is None:
|
||||
return None, 'unknown job'
|
||||
if j['status'] in ('done', 'failed'):
|
||||
return None, f"already {j['status']} (duplicate delivery)"
|
||||
if j['status'] == 'processing' and j['leased']:
|
||||
return None, 'leased by another live worker (duplicate delivery)'
|
||||
if j['status'] == 'queued' and j['not_yet']:
|
||||
return None, 'retry is scheduled later (stale message)'
|
||||
if j['attempt'] >= j['max_attempts']:
|
||||
await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', 'no attempts left')
|
||||
return None, 'attempts exhausted'
|
||||
attempt = await conn.fetchval(
|
||||
"UPDATE attachment_jobs SET status = 'processing', attempt = attempt + 1, worker_id = $2, "
|
||||
"lease_until = clock_timestamp() + make_interval(secs => $3), started_at = clock_timestamp(), "
|
||||
"updated_at = clock_timestamp() WHERE job_id = $1 RETURNING attempt",
|
||||
job_id, self.worker_id, self.settings.job_lease)
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() "
|
||||
"WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id)
|
||||
return {**dict(j), 'attempt': attempt, 'max_attempts': j['max_attempts']}, None
|
||||
|
||||
async def heartbeat(self, job_id: str, attempt: int) -> None:
|
||||
interval = max(1.0, self.settings.job_lease / 3)
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
async with self.db.connection() as conn:
|
||||
ok = await conn.fetchval(
|
||||
"UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => $3) "
|
||||
"WHERE job_id = $1 AND worker_id = $2 AND attempt = $4 AND status = 'processing' RETURNING 1",
|
||||
job_id, self.worker_id, self.settings.job_lease, attempt)
|
||||
if not ok:
|
||||
log.warning('job %s: lease lost', job_id)
|
||||
return
|
||||
except Exception:
|
||||
log.warning('job %s: heartbeat failed', job_id)
|
||||
|
||||
async def process(self, payload: dict) -> None:
|
||||
job_id, attachment_id = payload['job_id'], payload['attachment_id']
|
||||
job, reason = await self.claim(job_id, attachment_id)
|
||||
if job is None:
|
||||
log.info('job %s: no-op, %s', job_id, reason)
|
||||
return
|
||||
started = time.monotonic()
|
||||
beat = asyncio.create_task(self.heartbeat(job_id, job['attempt']))
|
||||
try:
|
||||
stages = await self.execute(job_id, attachment_id, job)
|
||||
log.info('job %s: ready in %.2fs (attempt %d) %s', job_id, time.monotonic() - started, job['attempt'],
|
||||
' '.join(f'{k}={v:.3f}' for k, v in stages.items()))
|
||||
except imaging.PermanentImageError as exc:
|
||||
await self.finish_failed(job_id, attachment_id, job['attempt'], exc.code, exc.code)
|
||||
except LostLease:
|
||||
log.warning('job %s: result not committed, lease taken over', job_id)
|
||||
except (imaging.TransientImageError, ApiError) as exc:
|
||||
code = getattr(exc, 'code', 'unavailable')
|
||||
await self.retry_or_fail(job_id, attachment_id, job, f'transient: {code}')
|
||||
finally:
|
||||
beat.cancel()
|
||||
|
||||
async def execute(self, job_id: str, attachment_id: str, job) -> None:
|
||||
version = job['pipeline_version']
|
||||
stages, mark = {}, time.monotonic()
|
||||
|
||||
def lap(name):
|
||||
nonlocal mark
|
||||
now = time.monotonic()
|
||||
stages[name] = now - mark
|
||||
mark = now
|
||||
try:
|
||||
data = await self.storage.get_raw(job['staging_key'])
|
||||
except ObjectMissing:
|
||||
raise imaging.PermanentImageError('source_missing') from None
|
||||
# Доверенные метаданные — из БД, а не из сообщения брокера.
|
||||
if len(data) != job['size_bytes'] or hashlib.sha256(data).hexdigest() != job['sha256']:
|
||||
raise imaging.PermanentImageError('source_corrupted')
|
||||
lap('fetch')
|
||||
variants = await imaging.run_isolated(data, self.settings.worker_max_pixels,
|
||||
self.settings.worker_memory_mb, self.settings.job_timeout)
|
||||
lap('build')
|
||||
ext = {'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/png': 'png'}
|
||||
for v in variants:
|
||||
v['object_key'] = f"variants/{attachment_id}/v{version}/{v['name']}.{ext[v['content_type']]}"
|
||||
await self.storage.put(v['object_key'], v.pop('data'), v['content_type'])
|
||||
await self.storage.copy(job['staging_key'], job['object_key'])
|
||||
lap('store')
|
||||
faults.hit('worker_crash_after_s3')
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
owned = await conn.fetchval(
|
||||
"UPDATE attachment_jobs SET status = 'done', lease_until = NULL, error_code = NULL, "
|
||||
"last_error = NULL, finished_at = clock_timestamp(), updated_at = clock_timestamp() "
|
||||
"WHERE job_id = $1 AND status = 'processing' AND worker_id = $2 AND attempt = $3 RETURNING 1",
|
||||
job_id, self.worker_id, job['attempt'])
|
||||
if not owned:
|
||||
raise LostLease()
|
||||
for v in variants:
|
||||
await conn.execute(
|
||||
'INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, '
|
||||
'size_bytes, width, height, sha256, object_key) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) '
|
||||
'ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING',
|
||||
attachment_id, version, v['name'], v['content_type'], v['size_bytes'],
|
||||
v['width'], v['height'], v['sha256'], v['object_key'])
|
||||
await conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() "
|
||||
"WHERE id = $1 AND status = 'processing'", attachment_id)
|
||||
lap('commit')
|
||||
faults.hit('worker_crash_after_commit')
|
||||
try:
|
||||
await self.storage.delete(job['staging_key']) # staging больше не нужен: результат зафиксирован
|
||||
except ApiError:
|
||||
log.warning('job %s: staging cleanup deferred to retention sweeper', job_id)
|
||||
lap('cleanup')
|
||||
return stages
|
||||
|
||||
async def finish_failed(self, job_id, attachment_id, attempt: int, code: str, reason: str) -> None:
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
await jobs.fail(conn, job_id, attachment_id, code, reason, worker_id=self.worker_id, attempt=attempt)
|
||||
log.warning('job %s: failed (%s)', job_id, code)
|
||||
|
||||
async def retry_or_fail(self, job_id, attachment_id, job, reason: str) -> None:
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
if job['attempt'] >= job['max_attempts']:
|
||||
await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', reason,
|
||||
worker_id=self.worker_id, attempt=job['attempt'])
|
||||
log.warning('job %s: failed, retry_exhausted after %d attempts', job_id, job['attempt'])
|
||||
return
|
||||
delay = jobs.backoff_seconds(job['attempt'], self.settings.retry_base_ms, self.settings.retry_max_ms)
|
||||
await jobs.requeue(conn, job_id, attachment_id, job['attempt'] + 1, delay, reason,
|
||||
worker_id=self.worker_id, attempt=job['attempt'])
|
||||
log.warning('job %s: %s, retry in %.1fs (attempt %d/%d)', job_id, reason, delay,
|
||||
job['attempt'], job['max_attempts'])
|
||||
|
||||
# ---------- обслуживание ----------
|
||||
|
||||
async def maintenance(self) -> None:
|
||||
"""Пульс для healthcheck + retention staging (одна реплика за раз через advisory lock)."""
|
||||
last_sweep = 0.0
|
||||
while not self.stopping.is_set():
|
||||
if self.connection is not None and not self.connection.is_closed:
|
||||
ALIVE_FILE.touch()
|
||||
if time.monotonic() - last_sweep > 300:
|
||||
last_sweep = time.monotonic()
|
||||
try:
|
||||
await self.sweep_staging()
|
||||
except Exception:
|
||||
log.warning('staging sweep failed', exc_info=True)
|
||||
try:
|
||||
await asyncio.wait_for(self.stopping.wait(), 5)
|
||||
except asyncio.TimeoutError:
|
||||
pass
|
||||
|
||||
async def sweep_staging(self) -> None:
|
||||
old = await self.storage.list_older('staging/', self.settings.staging_retention)
|
||||
if not old:
|
||||
return
|
||||
async with self.db.connection() as conn:
|
||||
async with conn.transaction():
|
||||
if not await conn.fetchval('SELECT pg_try_advisory_xact_lock(20260006)'):
|
||||
return
|
||||
# Удаляем только staging, по которому НЕТ текущей работы (queued/processing).
|
||||
active = {r['staging_key'] for r in await conn.fetch(
|
||||
"SELECT staging_key FROM attachments WHERE staging_key = ANY($1::text[]) "
|
||||
"AND status IN ('queued', 'processing')", old)}
|
||||
removed = 0
|
||||
for key in old:
|
||||
if key not in active:
|
||||
await self.storage.delete(key)
|
||||
removed += 1
|
||||
log.info('staging retention: removed %d objects older than %ds', removed, self.settings.staging_retention)
|
||||
|
||||
# ---------- жизненный цикл ----------
|
||||
|
||||
async def run(self) -> None:
|
||||
await self.db.start()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
loop.add_signal_handler(sig, self.stopping.set)
|
||||
housekeeping = asyncio.create_task(self.maintenance())
|
||||
while True:
|
||||
try:
|
||||
self.connection = await broker.connect(self.settings.broker_url)
|
||||
break
|
||||
except Exception as exc:
|
||||
if self.stopping.is_set():
|
||||
return
|
||||
log.warning('broker not reachable (%s), retrying', type(exc).__name__)
|
||||
await asyncio.sleep(2)
|
||||
channel = await self.connection.channel()
|
||||
# prefetch = сколько неподтверждённых сообщений брокер отдаёт этому worker (ограниченная конкуренция).
|
||||
await channel.set_qos(prefetch_count=self.settings.worker_prefetch)
|
||||
_exchange, queue = await broker.declare(channel)
|
||||
tag = await queue.consume(self.on_message, no_ack=False)
|
||||
log.info('worker %s consuming %s, concurrency=%d prefetch=%d', self.worker_id, broker.QUEUE,
|
||||
self.settings.worker_concurrency, self.settings.worker_prefetch)
|
||||
await self.stopping.wait()
|
||||
# Graceful: перестать брать новые, дождаться текущих (неподтверждённые prefetch вернутся в очередь).
|
||||
await queue.cancel(tag)
|
||||
if self.inflight:
|
||||
log.info('draining %d in-flight jobs', len(self.inflight))
|
||||
await asyncio.wait(self.inflight, timeout=self.settings.job_timeout)
|
||||
housekeeping.cancel()
|
||||
await self.connection.close()
|
||||
await self.db.close()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s %(message)s')
|
||||
asyncio.run(Worker().run())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+334
@@ -0,0 +1,334 @@
|
||||
# Стенд lab5: lb (HAProxy) -> api x N -> postgres (метаданные, jobs, outbox) + s3 (RustFS: staging/, originals/,
|
||||
# variants/) + rabbitmq (очередь заданий) <- worker (обработка изображений) ; valkey — кэш сессий.
|
||||
# migrate и s3-init — одноразовый bootstrap схемы БД и приватного bucket.
|
||||
# Подготовка: sh scripts/init-secrets.sh
|
||||
# Запуск: docker compose --env-file .env up -d --build --wait
|
||||
name: mtusi-lab5
|
||||
|
||||
x-api-env: &api-env
|
||||
APP_HOST: ${APP_HOST:-0.0.0.0}
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
SESSION_STORE: ${SESSION_STORE:-postgres+valkey}
|
||||
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000}
|
||||
DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url}
|
||||
CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000}
|
||||
DB_ACQUIRE_TIMEOUT_MS: ${DB_ACQUIRE_TIMEOUT_MS:-2000}
|
||||
CACHE_TIMEOUT_MS: ${CACHE_TIMEOUT_MS:-300}
|
||||
PASSWORD_HASH_WORKERS: ${PASSWORD_HASH_WORKERS:-2}
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id}
|
||||
S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key}
|
||||
S3_TIMEOUT_MS: ${S3_TIMEOUT_MS:-5000}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
IMAGE_CHECK_WORKERS: ${IMAGE_CHECK_WORKERS:-2}
|
||||
BROKER_URL_FILE: ${BROKER_URL_FILE:-/run/secrets/broker_url}
|
||||
WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2}
|
||||
WORKER_PREFETCH: ${WORKER_PREFETCH:-2}
|
||||
JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5}
|
||||
JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60}
|
||||
JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15}
|
||||
RETRY_BASE_MS: ${RETRY_BASE_MS:-1000}
|
||||
RETRY_MAX_MS: ${RETRY_MAX_MS:-60000}
|
||||
STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400}
|
||||
MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200}
|
||||
OUTBOX_POLL_MS: ${OUTBOX_POLL_MS:-500}
|
||||
WORKER_TASK_MEMORY_MB: ${WORKER_TASK_MEMORY_MB:-640}
|
||||
|
||||
x-api-depends: &api-depends
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
# bucket создан и проверен на приватность до старта API
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
# Брокер не критичен для приёма: API пишет намерение в outbox и допубликует позже.
|
||||
condition: service_started
|
||||
required: false
|
||||
valkey:
|
||||
# Valkey — кэш, не источник истины: API должен стартовать и без него.
|
||||
condition: service_started
|
||||
required: false
|
||||
|
||||
x-api-limits: &api-limits
|
||||
limits:
|
||||
cpus: "${API_CPUS:-1.0}"
|
||||
memory: ${API_MEMORY:-384M} # 2 Argon2id x 64 MiB + 2 загрузки x (10 MiB тело + проверка) + интерпретатор
|
||||
|
||||
services:
|
||||
lb:
|
||||
image: haproxy:3.2.8-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
# Единственная публичная точка входа API (только loopback хоста) + страница состояния LB.
|
||||
- "127.0.0.1:${LB_PUBLISHED_PORT:-8080}:8080"
|
||||
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404"
|
||||
networks: [edge]
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/stats"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 5
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 128M}
|
||||
restart: unless-stopped
|
||||
|
||||
api:
|
||||
image: mtusi-lab5-api:local
|
||||
build: .
|
||||
# Нет container_name и ports: реплики взаимозаменяемы, доступны только через lb.
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources: *api-limits
|
||||
environment: *api-env
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [edge, data]
|
||||
# Корневая ФС только для чтения, tmpfs /tmp — у API нет места для локальной копии оригинала.
|
||||
read_only: true
|
||||
tmpfs: [/tmp]
|
||||
stop_grace_period: 20s # drain 3 с + graceful shutdown uvicorn до 10 с, с запасом
|
||||
depends_on: *api-depends
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
# Один запуск на весь стенд (а не в каждой реплике): pg_advisory_lock + schema_migrations.
|
||||
build: .
|
||||
image: mtusi-lab5-api:local
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
SESSION_STORE: postgres
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
restart: "no"
|
||||
|
||||
# S3-совместимое хранилище оригиналов. Порты не публикуются на хост: доступ только из сети data
|
||||
# (API и s3-init). Веб-консоль RustFS выключена (по умолчанию, RUSTFS_CONSOLE_ENABLE не задан).
|
||||
s3:
|
||||
image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2
|
||||
environment:
|
||||
RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id
|
||||
RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
RUSTFS_REGION: ${S3_REGION:-us-east-1}
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_OBS_LOG_DIRECTORY: "" # логи в stdout контейнера
|
||||
RUSTFS_OBS_LOGGER_LEVEL: warn
|
||||
secrets: [s3_access_key_id, s3_secret_access_key]
|
||||
volumes:
|
||||
- s3data:/data
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-fs", "-o", "/dev/null", "http://127.0.0.1:9000/health"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
stop_grace_period: 20s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
restart: unless-stopped
|
||||
|
||||
s3-init:
|
||||
# Идемпотентный bootstrap: bucket создаётся, если его нет; анонимная политика удаляется;
|
||||
# анонимный GET проверяется на отказ. Повторный запуск данные не трогает.
|
||||
image: mtusi-lab5-api:local
|
||||
command: ["python", "-m", "app.s3_bootstrap"]
|
||||
environment:
|
||||
S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000}
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
secrets: [s3_access_key_id, s3_secret_access_key]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
s3:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully # образ api уже собран сервисом migrate
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 128M}
|
||||
restart: "no"
|
||||
|
||||
# Брокер заданий. Quorum queue + persistent messages на volume rabbitdata переживают рестарт.
|
||||
# hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume.
|
||||
# Порты не публикуются; management UI не включён (только встроенный prometheus на 15692 в сети data).
|
||||
rabbitmq:
|
||||
image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5
|
||||
hostname: rabbitmq
|
||||
secrets:
|
||||
- source: rabbitmq_conf
|
||||
target: /etc/rabbitmq/conf.d/90-credentials.conf
|
||||
volumes:
|
||||
- rabbitdata:/var/lib/rabbitmq
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"]
|
||||
interval: 5s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
start_period: 10s
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "1.0", memory: 512M}
|
||||
restart: unless-stopped
|
||||
|
||||
# Worker: отдельный процесс обработки (тот же образ, другая команда). Не в сети edge, портов нет.
|
||||
worker:
|
||||
image: mtusi-lab5-api:local
|
||||
command: ["python", "-m", "app.worker"]
|
||||
environment:
|
||||
<<: *api-env
|
||||
SESSION_STORE: postgres # worker не работает с сессиями
|
||||
secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [data]
|
||||
read_only: true
|
||||
tmpfs: [/tmp]
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-alive')<20 else 1)"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
stop_grace_period: 70s # дождаться текущих заданий (JOB_TIMEOUT_SECONDS)
|
||||
deploy:
|
||||
replicas: ${WORKER_REPLICAS:-1}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${WORKER_CPUS:-2.0}"
|
||||
memory: ${WORKER_MEMORY:-1536M} # WORKER_CONCURRENCY x WORKER_TASK_MEMORY_MB + процесс worker
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
s3-init:
|
||||
condition: service_completed_successfully
|
||||
rabbitmq:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
|
||||
postgres:
|
||||
image: postgres:17.7-bookworm
|
||||
environment:
|
||||
POSTGRES_USER: app
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD", "pg_isready", "-U", "app", "-d", "messenger"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
shm_size: 128m
|
||||
stop_grace_period: 30s # даём PostgreSQL завершиться с checkpoint, без recovery при старте
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "${PG_CPUS:-2.0}", memory: 768M}
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.4
|
||||
command: ["valkey-server", "/run/secrets/valkey_conf"]
|
||||
secrets: [valkey_conf]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_conf)\" ping | grep -q PONG"]
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits: {cpus: "0.5", memory: 96M}
|
||||
restart: unless-stopped
|
||||
|
||||
# Отдельная реплика с коротким TTL только для автотеста истечения сессии (lab2, уровень 4).
|
||||
# Порт не публикуется: tester обращается к ней по внутренней сети.
|
||||
api-ttl:
|
||||
profiles: [test]
|
||||
image: mtusi-lab5-api:local
|
||||
environment:
|
||||
<<: *api-env
|
||||
SESSION_TTL_SECONDS: "2"
|
||||
secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url]
|
||||
networks: [edge, data]
|
||||
read_only: true
|
||||
tmpfs: [/tmp]
|
||||
deploy:
|
||||
resources: *api-limits
|
||||
depends_on: *api-depends
|
||||
|
||||
# Linux-окружение для make/тестов/hook-скриптов (нужно на Windows без make и WSL).
|
||||
tester:
|
||||
profiles: [tools]
|
||||
build:
|
||||
context: .
|
||||
dockerfile: tools/tester.Dockerfile
|
||||
image: mtusi-lab5-tester:local
|
||||
working_dir: /work
|
||||
environment:
|
||||
BASE_URL: http://lb:8080
|
||||
TTL_BASE_URL: http://api-ttl:8080
|
||||
LB_STATS_URL: http://lb:8404/stats
|
||||
PYTHONUTF8: "1"
|
||||
PYTHONDONTWRITEBYTECODE: "1"
|
||||
networks: [edge]
|
||||
volumes:
|
||||
- .:/work
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
|
||||
networks:
|
||||
# edge: lb <-> api (+ tester). data: api/migrate <-> хранилища; lb в неё не входит.
|
||||
edge:
|
||||
data:
|
||||
internal: true
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
s3data:
|
||||
rabbitdata:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
valkey_conf:
|
||||
file: ./secrets/valkey.conf
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
broker_url:
|
||||
file: ./secrets/broker_url
|
||||
rabbitmq_conf:
|
||||
file: ./secrets/rabbitmq.conf
|
||||
@@ -0,0 +1,17 @@
|
||||
# 2026-11-17T15:27:35Z, commit 276dbda
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 137, "failed_attempts": 0, "first_recovery_seconds": 4.458}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
mtusi-lab5-api-2 Up 5 seconds (healthy)
|
||||
mtusi-lab5-api-1 Up About a minute (healthy)
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 137, "failed_attempts": 0, "first_recovery_seconds": 4.163}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
mtusi-lab5-api-2 Up 31 seconds (healthy)
|
||||
mtusi-lab5-api-1 Up 5 seconds (healthy)
|
||||
|
||||
# evidence/stop-one.log (what the hook really stopped):
|
||||
2026-11-17T15:27:38Z mode=stop instance=cdb3725c641e-b9c808 container=mtusi-lab5-api-2 state=exited exit=0 finished=2026-11-17T15:27:42.125871236Z
|
||||
2026-11-17T15:28:04Z mode=stop instance=824c92061861-f52bc3 container=mtusi-lab5-api-1 state=exited exit=0 finished=2026-11-17T15:28:07.582211619Z
|
||||
@@ -0,0 +1,25 @@
|
||||
# 2026-11-17T15:28:26Z, commit 276dbda
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
# hook scope (StartedAt before/after of every process):
|
||||
before:
|
||||
/mtusi-lab5-api-1 2026-11-17T15:28:44.882592618Z
|
||||
/mtusi-lab5-api-2 2026-11-17T15:28:44.5554688Z
|
||||
/mtusi-lab5-lb-1 2026-11-17T15:28:50.680908369Z
|
||||
/mtusi-lab5-postgres-1 2026-11-17T15:28:36.844686149Z
|
||||
/mtusi-lab5-rabbitmq-1 2026-11-17T15:28:37.794780107Z
|
||||
/mtusi-lab5-s3-1 2026-11-17T15:28:36.68421067Z
|
||||
/mtusi-lab5-valkey-1 2026-11-17T15:28:36.995474977Z
|
||||
/mtusi-lab5-worker-1 2026-11-17T15:28:45.270340876Z
|
||||
after:
|
||||
/mtusi-lab5-api-1 2026-11-17T15:29:08.027213384Z
|
||||
/mtusi-lab5-api-2 2026-11-17T15:29:07.790819877Z
|
||||
/mtusi-lab5-lb-1 2026-11-17T15:29:13.806020318Z
|
||||
/mtusi-lab5-postgres-1 2026-11-17T15:28:59.585810192Z
|
||||
/mtusi-lab5-rabbitmq-1 2026-11-17T15:29:00.317641583Z
|
||||
/mtusi-lab5-s3-1 2026-11-17T15:28:59.897534526Z
|
||||
/mtusi-lab5-valkey-1 2026-11-17T15:28:59.750297747Z
|
||||
/mtusi-lab5-worker-1 2026-11-17T15:29:07.569183681Z
|
||||
ready
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-11-17T15:27:18Z, commit 276dbda
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://lb:8080"
|
||||
PASS: lab5 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,35 @@
|
||||
# 2026-11-17T15:27:09Z, commit 276dbda
|
||||
NAME IMAGE STATUS PORTS
|
||||
mtusi-lab5-api-1 mtusi-lab5-api:local Up 37 seconds (healthy) 8080/tcp
|
||||
mtusi-lab5-api-2 mtusi-lab5-api:local Up 38 seconds (healthy) 8080/tcp
|
||||
mtusi-lab5-api-ttl-1 sha256:d872bf2ffa4ed28b4db76ab08df13745276eff2c8b4e56b54531370f3eb6f2fb Exited (137) 37 minutes ago
|
||||
mtusi-lab5-lb-1 haproxy:3.2.8-alpine Up 6 hours (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp
|
||||
mtusi-lab5-migrate-1 mtusi-lab5-api:local Exited (0) 12 minutes ago
|
||||
mtusi-lab5-postgres-1 postgres:17.7-bookworm Up 37 minutes (healthy) 5432/tcp
|
||||
mtusi-lab5-rabbitmq-1 rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5 Up 36 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
mtusi-lab5-s3-1 rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 Up 35 minutes (healthy) 9000-9001/tcp
|
||||
mtusi-lab5-s3-init-1 mtusi-lab5-api:local Exited (0) 12 minutes ago
|
||||
mtusi-lab5-valkey-1 valkey/valkey:8.1.4 Up 37 minutes (healthy) 6379/tcp
|
||||
mtusi-lab5-worker-1 mtusi-lab5-api:local Up About a minute (healthy) 8080/tcp
|
||||
|
||||
# s3-init (idempotent bucket bootstrap + anonymous access check):
|
||||
bucket messenger-attachments: exists
|
||||
bucket messenger-attachments: no bucket policy (private)
|
||||
anonymous GET messenger-attachments/_bootstrap/private-check: HTTP 403 (denied)
|
||||
|
||||
# repeated bootstrap run (must not touch existing bucket):
|
||||
bucket messenger-attachments: exists
|
||||
bucket messenger-attachments: no bucket policy (private)
|
||||
anonymous GET messenger-attachments/_bootstrap/private-check: HTTP 403 (denied)
|
||||
|
||||
# volumes:
|
||||
mtusi-lab5_pgdata
|
||||
mtusi-lab5_rabbitdata
|
||||
mtusi-lab5_s3data
|
||||
|
||||
# RabbitMQ: очереди (quorum), consumers и prefetch:
|
||||
name type durable messages consumers
|
||||
attachment.process.v1.dlq quorum true 2 0
|
||||
attachment.process.v1 quorum true 0 1
|
||||
queue_name ack_required prefetch_count
|
||||
attachment.process.v1 true 2
|
||||
@@ -0,0 +1,57 @@
|
||||
# 2026-11-17T15:29:15Z, commit 276dbda
|
||||
PASS test_variants_contract (3.2s)
|
||||
worker stopped; upload 202 attachment=89ff98d0-59f5-4296-a4ac-455e23e234a0
|
||||
api and rabbitmq restarted; queue depth before/after broker restart: 1/1
|
||||
worker started -> ready in 2.6s, thumbnail OK
|
||||
PASS test_worker_down_demo (24.4s)
|
||||
PASS test_failed_state_and_409 (4.7s)
|
||||
PASS test_duplicate_delivery_is_noop (7.0s)
|
||||
S3 down: failed retry_exhausted after 5 attempts, 15s
|
||||
backoff delays (s): 0.6, 1.5, 2.9, 7.7
|
||||
PASS test_retry_bounded_then_replay (28.8s)
|
||||
client saw: connection error (API process exited after COMMIT)
|
||||
attachment e6e30a4d-1e9e-47b2-8ef4-8976af20b767 reached ready via outbox relay
|
||||
PASS test_crash_between_commit_and_publish (2.3s)
|
||||
crash after S3 writes -> ready on attempt 2 in 17.8s, 3 objects in S3
|
||||
PASS test_worker_crash_after_s3_before_commit (20.5s)
|
||||
PASS test_worker_crash_after_commit_before_ack (14.6s)
|
||||
PASS test_expired_lease_takeover (5.5s)
|
||||
PASS test_staging_cleanup_after_commit (2.2s)
|
||||
PASS test_poison_message_to_dlq (3.8s)
|
||||
PASS test_prefetch_limited (1.4s)
|
||||
mem: resource_limit; time: processing_timeout; alpha: (255, 255, 255); exif: (85, 256) None
|
||||
PASS test_worker_resource_limits (2.3s)
|
||||
13/13 passed
|
||||
PASS test_png_and_jpeg_roundtrip (1.9s)
|
||||
PASS test_message_with_attachments (1.9s)
|
||||
PASS test_foreign_attachment_rejected (1.1s)
|
||||
PASS test_limits_and_mime (1.2s)
|
||||
PASS test_dangerous_filenames (1.9s)
|
||||
anonymous GET object/list -> 403
|
||||
PASS test_bucket_private_outside_api (2.2s)
|
||||
PASS test_no_local_copy_on_api (1.9s)
|
||||
uploaded via cdb3725c641e-42f61c, downloaded via 824c92061861-21c48d
|
||||
PASS test_failover_of_uploading_replica (8.0s)
|
||||
PASS test_s3_unavailable (14.8s)
|
||||
9/9 passed
|
||||
PASS test_session_across_replicas (0.6s)
|
||||
PASS test_same_key_concurrent_across_replicas (2.6s)
|
||||
PASS test_same_key_concurrent_different_bodies (1.7s)
|
||||
PASS test_key_scope_and_format (1.5s)
|
||||
PASS test_replay_rechecks_session (0.8s)
|
||||
PASS test_transient_failure_not_recorded (6.0s)
|
||||
PASS test_sigterm_drain (10.3s)
|
||||
PASS test_infrastructure_rules (0.5s)
|
||||
PASS test_nul_in_text_roundtrip (1.3s)
|
||||
9/9 passed
|
||||
PASS test_wrong_password (0.6s)
|
||||
PASS test_logout (0.5s)
|
||||
PASS test_short_ttl (3.4s)
|
||||
PASS test_username_race (3.8s)
|
||||
PASS test_chat_rollback (0.9s)
|
||||
PASS test_cursor_rules (1.3s)
|
||||
PASS test_valkey_cache_role (1.0s)
|
||||
PASS test_revoke_during_cache_partition (1.6s)
|
||||
PASS test_valkey_down (4.5s)
|
||||
PASS test_postgres_down (10.8s)
|
||||
10/10 passed
|
||||
@@ -0,0 +1,97 @@
|
||||
{
|
||||
"label": "backpressure-large",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 120,
|
||||
"rate_per_s": 15.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "20",
|
||||
"upload_status": {
|
||||
"202": 48,
|
||||
"503": 72
|
||||
},
|
||||
"accepted": 48,
|
||||
"ready": 48,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.94,
|
||||
"drain_seconds": 13.33,
|
||||
"throughput_ready_per_s": 3.691,
|
||||
"time_to_ready_s": {
|
||||
"p50": 4.937,
|
||||
"p95": 5.646,
|
||||
"p99": 5.682,
|
||||
"max": 5.682,
|
||||
"mean": 4.161
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.007,
|
||||
"p95": 0.142
|
||||
},
|
||||
"generator_max_late_s": 0.004,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 22,
|
||||
"max_broker_messages": 20
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 236.9,
|
||||
"idle_first": 110.5
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
2
|
||||
],
|
||||
[
|
||||
4.58,
|
||||
19
|
||||
],
|
||||
[
|
||||
5.34,
|
||||
21
|
||||
],
|
||||
[
|
||||
7.73,
|
||||
22
|
||||
],
|
||||
[
|
||||
8.5,
|
||||
18
|
||||
],
|
||||
[
|
||||
13.21,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
20
|
||||
],
|
||||
[
|
||||
8.5,
|
||||
19
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
110.5
|
||||
],
|
||||
[
|
||||
5.34,
|
||||
236.9
|
||||
],
|
||||
[
|
||||
8.5,
|
||||
236.0
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"label": "baseline-small",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 5.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 11.82,
|
||||
"drain_seconds": 12.1,
|
||||
"throughput_ready_per_s": 5.031,
|
||||
"time_to_ready_s": {
|
||||
"p50": 0.115,
|
||||
"p95": 1.171,
|
||||
"p99": 1.377,
|
||||
"max": 1.377,
|
||||
"mean": 0.248
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.024,
|
||||
"p95": 0.035
|
||||
},
|
||||
"generator_max_late_s": 0.003,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 1,
|
||||
"max_broker_messages": 4
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 109.7,
|
||||
"idle_first": 101.1
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
3.67,
|
||||
1
|
||||
],
|
||||
[
|
||||
4.4,
|
||||
0
|
||||
],
|
||||
[
|
||||
6.68,
|
||||
1
|
||||
],
|
||||
[
|
||||
7.43,
|
||||
1
|
||||
],
|
||||
[
|
||||
11.08,
|
||||
1
|
||||
],
|
||||
[
|
||||
11.81,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
4
|
||||
],
|
||||
[
|
||||
7.43,
|
||||
1
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
101.1
|
||||
],
|
||||
[
|
||||
4.4,
|
||||
109.7
|
||||
],
|
||||
[
|
||||
7.43,
|
||||
98.9
|
||||
],
|
||||
[
|
||||
11.81,
|
||||
98.9
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
{
|
||||
"label": "c1-p1-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.57,
|
||||
"drain_seconds": 54.51,
|
||||
"throughput_ready_per_s": 1.131,
|
||||
"time_to_ready_s": {
|
||||
"p50": 20.214,
|
||||
"p95": 44.534,
|
||||
"p99": 45.674,
|
||||
"max": 45.674,
|
||||
"mean": 21.052
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.143,
|
||||
"p95": 0.209
|
||||
},
|
||||
"generator_max_late_s": 0.002,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 50,
|
||||
"max_broker_messages": 43
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 172.9,
|
||||
"idle_first": 105.8
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
6.22,
|
||||
44
|
||||
],
|
||||
[
|
||||
7.27,
|
||||
50
|
||||
],
|
||||
[
|
||||
9.82,
|
||||
46
|
||||
],
|
||||
[
|
||||
10.57,
|
||||
44
|
||||
],
|
||||
[
|
||||
14.83,
|
||||
37
|
||||
],
|
||||
[
|
||||
15.67,
|
||||
36
|
||||
],
|
||||
[
|
||||
18.82,
|
||||
34
|
||||
],
|
||||
[
|
||||
20.43,
|
||||
33
|
||||
],
|
||||
[
|
||||
27.3,
|
||||
27
|
||||
],
|
||||
[
|
||||
28.37,
|
||||
26
|
||||
],
|
||||
[
|
||||
31.19,
|
||||
23
|
||||
],
|
||||
[
|
||||
32.43,
|
||||
22
|
||||
],
|
||||
[
|
||||
39.52,
|
||||
13
|
||||
],
|
||||
[
|
||||
40.41,
|
||||
12
|
||||
],
|
||||
[
|
||||
42.94,
|
||||
9
|
||||
],
|
||||
[
|
||||
43.8,
|
||||
7
|
||||
],
|
||||
[
|
||||
51.79,
|
||||
2
|
||||
],
|
||||
[
|
||||
52.68,
|
||||
1
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
11
|
||||
],
|
||||
[
|
||||
10.57,
|
||||
43
|
||||
],
|
||||
[
|
||||
20.43,
|
||||
32
|
||||
],
|
||||
[
|
||||
32.43,
|
||||
18
|
||||
],
|
||||
[
|
||||
43.8,
|
||||
6
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
105.8
|
||||
],
|
||||
[
|
||||
7.27,
|
||||
115.9
|
||||
],
|
||||
[
|
||||
10.57,
|
||||
120.7
|
||||
],
|
||||
[
|
||||
15.67,
|
||||
146.7
|
||||
],
|
||||
[
|
||||
20.43,
|
||||
172.7
|
||||
],
|
||||
[
|
||||
28.37,
|
||||
172.9
|
||||
],
|
||||
[
|
||||
32.43,
|
||||
116.0
|
||||
],
|
||||
[
|
||||
40.41,
|
||||
130.2
|
||||
],
|
||||
[
|
||||
43.8,
|
||||
116.2
|
||||
],
|
||||
[
|
||||
52.68,
|
||||
119.1
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
{
|
||||
"label": "c1-p1-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.53,
|
||||
"drain_seconds": 44.28,
|
||||
"throughput_ready_per_s": 1.385,
|
||||
"time_to_ready_s": {
|
||||
"p50": 18.493,
|
||||
"p95": 34.835,
|
||||
"p99": 36.043,
|
||||
"max": 36.043,
|
||||
"mean": 18.892
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.166,
|
||||
"p95": 0.473
|
||||
},
|
||||
"generator_max_late_s": 0.004,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 50,
|
||||
"max_broker_messages": 47
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 185.6,
|
||||
"idle_first": 176.1
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
2
|
||||
],
|
||||
[
|
||||
5.13,
|
||||
39
|
||||
],
|
||||
[
|
||||
6.05,
|
||||
45
|
||||
],
|
||||
[
|
||||
8.57,
|
||||
50
|
||||
],
|
||||
[
|
||||
9.46,
|
||||
49
|
||||
],
|
||||
[
|
||||
13.98,
|
||||
43
|
||||
],
|
||||
[
|
||||
14.86,
|
||||
41
|
||||
],
|
||||
[
|
||||
17.29,
|
||||
37
|
||||
],
|
||||
[
|
||||
18.14,
|
||||
36
|
||||
],
|
||||
[
|
||||
23.74,
|
||||
28
|
||||
],
|
||||
[
|
||||
24.69,
|
||||
27
|
||||
],
|
||||
[
|
||||
27.22,
|
||||
23
|
||||
],
|
||||
[
|
||||
28.12,
|
||||
22
|
||||
],
|
||||
[
|
||||
34.56,
|
||||
13
|
||||
],
|
||||
[
|
||||
35.4,
|
||||
11
|
||||
],
|
||||
[
|
||||
37.92,
|
||||
8
|
||||
],
|
||||
[
|
||||
38.79,
|
||||
6
|
||||
],
|
||||
[
|
||||
43.93,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
10
|
||||
],
|
||||
[
|
||||
9.46,
|
||||
47
|
||||
],
|
||||
[
|
||||
18.14,
|
||||
32
|
||||
],
|
||||
[
|
||||
28.12,
|
||||
18
|
||||
],
|
||||
[
|
||||
38.79,
|
||||
3
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
176.1
|
||||
],
|
||||
[
|
||||
6.05,
|
||||
141.2
|
||||
],
|
||||
[
|
||||
9.46,
|
||||
128.0
|
||||
],
|
||||
[
|
||||
14.86,
|
||||
128.0
|
||||
],
|
||||
[
|
||||
18.14,
|
||||
130.1
|
||||
],
|
||||
[
|
||||
24.69,
|
||||
132.4
|
||||
],
|
||||
[
|
||||
28.12,
|
||||
170.3
|
||||
],
|
||||
[
|
||||
35.4,
|
||||
185.6
|
||||
],
|
||||
[
|
||||
38.79,
|
||||
185.2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
{
|
||||
"label": "c1-p1-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.0,
|
||||
"drain_seconds": 39.01,
|
||||
"throughput_ready_per_s": 5.256,
|
||||
"time_to_ready_s": {
|
||||
"p50": 14.65,
|
||||
"p95": 30.791,
|
||||
"p99": 32.731,
|
||||
"max": 33.144,
|
||||
"mean": 15.373
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.023,
|
||||
"p95": 0.037
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 177,
|
||||
"max_broker_messages": 141
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 109.7,
|
||||
"idle_first": 91.2
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
10
|
||||
],
|
||||
[
|
||||
3.85,
|
||||
151
|
||||
],
|
||||
[
|
||||
4.61,
|
||||
177
|
||||
],
|
||||
[
|
||||
6.95,
|
||||
167
|
||||
],
|
||||
[
|
||||
7.67,
|
||||
162
|
||||
],
|
||||
[
|
||||
12.33,
|
||||
133
|
||||
],
|
||||
[
|
||||
13.1,
|
||||
128
|
||||
],
|
||||
[
|
||||
15.38,
|
||||
111
|
||||
],
|
||||
[
|
||||
16.12,
|
||||
106
|
||||
],
|
||||
[
|
||||
20.66,
|
||||
78
|
||||
],
|
||||
[
|
||||
21.39,
|
||||
73
|
||||
],
|
||||
[
|
||||
23.68,
|
||||
57
|
||||
],
|
||||
[
|
||||
24.43,
|
||||
52
|
||||
],
|
||||
[
|
||||
28.12,
|
||||
28
|
||||
],
|
||||
[
|
||||
29.15,
|
||||
25
|
||||
],
|
||||
[
|
||||
31.87,
|
||||
21
|
||||
],
|
||||
[
|
||||
33.11,
|
||||
18
|
||||
],
|
||||
[
|
||||
38.16,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
41
|
||||
],
|
||||
[
|
||||
7.67,
|
||||
141
|
||||
],
|
||||
[
|
||||
16.12,
|
||||
108
|
||||
],
|
||||
[
|
||||
24.43,
|
||||
43
|
||||
],
|
||||
[
|
||||
33.11,
|
||||
8
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
91.2
|
||||
],
|
||||
[
|
||||
4.61,
|
||||
98.0
|
||||
],
|
||||
[
|
||||
7.67,
|
||||
97.9
|
||||
],
|
||||
[
|
||||
13.1,
|
||||
103.0
|
||||
],
|
||||
[
|
||||
16.12,
|
||||
98.1
|
||||
],
|
||||
[
|
||||
21.39,
|
||||
98.5
|
||||
],
|
||||
[
|
||||
24.43,
|
||||
98.6
|
||||
],
|
||||
[
|
||||
29.15,
|
||||
98.1
|
||||
],
|
||||
[
|
||||
33.11,
|
||||
109.7
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"label": "c1-p1-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.35,
|
||||
"drain_seconds": 53.7,
|
||||
"throughput_ready_per_s": 3.745,
|
||||
"time_to_ready_s": {
|
||||
"p50": 24.775,
|
||||
"p95": 45.564,
|
||||
"p99": 47.589,
|
||||
"max": 48.149,
|
||||
"mean": 24.871
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.04,
|
||||
"p95": 0.62
|
||||
},
|
||||
"generator_max_late_s": 0.11,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 181,
|
||||
"max_broker_messages": 156
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 130.1,
|
||||
"idle_first": 123.8
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
9
|
||||
],
|
||||
[
|
||||
4.82,
|
||||
161
|
||||
],
|
||||
[
|
||||
5.66,
|
||||
181
|
||||
],
|
||||
[
|
||||
8.29,
|
||||
171
|
||||
],
|
||||
[
|
||||
9.15,
|
||||
168
|
||||
],
|
||||
[
|
||||
13.92,
|
||||
149
|
||||
],
|
||||
[
|
||||
14.76,
|
||||
145
|
||||
],
|
||||
[
|
||||
17.23,
|
||||
135
|
||||
],
|
||||
[
|
||||
18.13,
|
||||
132
|
||||
],
|
||||
[
|
||||
24.55,
|
||||
110
|
||||
],
|
||||
[
|
||||
25.46,
|
||||
107
|
||||
],
|
||||
[
|
||||
27.93,
|
||||
97
|
||||
],
|
||||
[
|
||||
28.88,
|
||||
94
|
||||
],
|
||||
[
|
||||
35.24,
|
||||
72
|
||||
],
|
||||
[
|
||||
36.13,
|
||||
68
|
||||
],
|
||||
[
|
||||
38.69,
|
||||
58
|
||||
],
|
||||
[
|
||||
39.64,
|
||||
54
|
||||
],
|
||||
[
|
||||
44.54,
|
||||
34
|
||||
],
|
||||
[
|
||||
45.38,
|
||||
31
|
||||
],
|
||||
[
|
||||
47.97,
|
||||
21
|
||||
],
|
||||
[
|
||||
48.86,
|
||||
17
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
86
|
||||
],
|
||||
[
|
||||
9.15,
|
||||
156
|
||||
],
|
||||
[
|
||||
18.13,
|
||||
118
|
||||
],
|
||||
[
|
||||
28.88,
|
||||
85
|
||||
],
|
||||
[
|
||||
39.64,
|
||||
46
|
||||
],
|
||||
[
|
||||
48.86,
|
||||
5
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
123.8
|
||||
],
|
||||
[
|
||||
5.66,
|
||||
119.3
|
||||
],
|
||||
[
|
||||
9.15,
|
||||
119.2
|
||||
],
|
||||
[
|
||||
14.76,
|
||||
119.4
|
||||
],
|
||||
[
|
||||
18.13,
|
||||
119.7
|
||||
],
|
||||
[
|
||||
25.46,
|
||||
126.6
|
||||
],
|
||||
[
|
||||
28.88,
|
||||
129.2
|
||||
],
|
||||
[
|
||||
36.13,
|
||||
130.0
|
||||
],
|
||||
[
|
||||
39.64,
|
||||
119.5
|
||||
],
|
||||
[
|
||||
45.38,
|
||||
130.1
|
||||
],
|
||||
[
|
||||
48.86,
|
||||
119.6
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"label": "c2-p16-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.51,
|
||||
"drain_seconds": 21.1,
|
||||
"throughput_ready_per_s": 2.98,
|
||||
"time_to_ready_s": {
|
||||
"p50": 8.645,
|
||||
"p95": 12.515,
|
||||
"p99": 12.789,
|
||||
"max": 12.789,
|
||||
"mean": 7.964
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.167,
|
||||
"p95": 1.092
|
||||
},
|
||||
"generator_max_late_s": 0.01,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 43,
|
||||
"max_broker_messages": 40
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 238.7,
|
||||
"idle_first": 117.7
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
2
|
||||
],
|
||||
[
|
||||
5.13,
|
||||
29
|
||||
],
|
||||
[
|
||||
6.27,
|
||||
43
|
||||
],
|
||||
[
|
||||
9.23,
|
||||
42
|
||||
],
|
||||
[
|
||||
10.21,
|
||||
38
|
||||
],
|
||||
[
|
||||
14.18,
|
||||
23
|
||||
],
|
||||
[
|
||||
14.98,
|
||||
21
|
||||
],
|
||||
[
|
||||
17.39,
|
||||
11
|
||||
],
|
||||
[
|
||||
18.21,
|
||||
8
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
10.21,
|
||||
40
|
||||
],
|
||||
[
|
||||
18.21,
|
||||
1
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
117.7
|
||||
],
|
||||
[
|
||||
6.27,
|
||||
136.3
|
||||
],
|
||||
[
|
||||
10.21,
|
||||
238.7
|
||||
],
|
||||
[
|
||||
14.98,
|
||||
194.3
|
||||
],
|
||||
[
|
||||
18.21,
|
||||
195.0
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"label": "c2-p16-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.47,
|
||||
"drain_seconds": 18.25,
|
||||
"throughput_ready_per_s": 3.475,
|
||||
"time_to_ready_s": {
|
||||
"p50": 5.555,
|
||||
"p95": 9.662,
|
||||
"p99": 9.912,
|
||||
"max": 9.912,
|
||||
"mean": 5.505
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.11,
|
||||
"p95": 0.168
|
||||
},
|
||||
"generator_max_late_s": 0.006,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 36,
|
||||
"max_broker_messages": 34
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 204.8,
|
||||
"idle_first": 138.9
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
4.2,
|
||||
24
|
||||
],
|
||||
[
|
||||
5.04,
|
||||
27
|
||||
],
|
||||
[
|
||||
7.44,
|
||||
36
|
||||
],
|
||||
[
|
||||
8.25,
|
||||
34
|
||||
],
|
||||
[
|
||||
13.41,
|
||||
15
|
||||
],
|
||||
[
|
||||
14.19,
|
||||
12
|
||||
],
|
||||
[
|
||||
16.58,
|
||||
3
|
||||
],
|
||||
[
|
||||
17.36,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
16
|
||||
],
|
||||
[
|
||||
8.25,
|
||||
34
|
||||
],
|
||||
[
|
||||
17.36,
|
||||
0
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
138.9
|
||||
],
|
||||
[
|
||||
5.04,
|
||||
204.8
|
||||
],
|
||||
[
|
||||
8.25,
|
||||
190.6
|
||||
],
|
||||
[
|
||||
14.19,
|
||||
181.0
|
||||
],
|
||||
[
|
||||
17.36,
|
||||
147.6
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"label": "c2-p16-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.01,
|
||||
"drain_seconds": 20.95,
|
||||
"throughput_ready_per_s": 9.604,
|
||||
"time_to_ready_s": {
|
||||
"p50": 8.479,
|
||||
"p95": 15.174,
|
||||
"p99": 15.711,
|
||||
"max": 15.851,
|
||||
"mean": 8.697
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.025,
|
||||
"p95": 0.035
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 165,
|
||||
"max_broker_messages": 107
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 117.2,
|
||||
"idle_first": 100.4
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
9
|
||||
],
|
||||
[
|
||||
3.99,
|
||||
143
|
||||
],
|
||||
[
|
||||
4.77,
|
||||
165
|
||||
],
|
||||
[
|
||||
7.27,
|
||||
138
|
||||
],
|
||||
[
|
||||
8.0,
|
||||
129
|
||||
],
|
||||
[
|
||||
13.91,
|
||||
70
|
||||
],
|
||||
[
|
||||
14.78,
|
||||
62
|
||||
],
|
||||
[
|
||||
17.12,
|
||||
36
|
||||
],
|
||||
[
|
||||
17.9,
|
||||
28
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
15
|
||||
],
|
||||
[
|
||||
8.0,
|
||||
107
|
||||
],
|
||||
[
|
||||
17.9,
|
||||
6
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
100.4
|
||||
],
|
||||
[
|
||||
4.77,
|
||||
112.8
|
||||
],
|
||||
[
|
||||
8.0,
|
||||
110.4
|
||||
],
|
||||
[
|
||||
14.78,
|
||||
117.0
|
||||
],
|
||||
[
|
||||
17.9,
|
||||
117.2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"label": "c2-p16-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.0,
|
||||
"drain_seconds": 20.97,
|
||||
"throughput_ready_per_s": 10.022,
|
||||
"time_to_ready_s": {
|
||||
"p50": 7.194,
|
||||
"p95": 14.257,
|
||||
"p99": 14.841,
|
||||
"max": 14.998,
|
||||
"mean": 7.499
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.025,
|
||||
"p95": 0.035
|
||||
},
|
||||
"generator_max_late_s": 0.002,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 148,
|
||||
"max_broker_messages": 120
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 149.3,
|
||||
"idle_first": 144.1
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
7
|
||||
],
|
||||
[
|
||||
5.09,
|
||||
148
|
||||
],
|
||||
[
|
||||
5.89,
|
||||
139
|
||||
],
|
||||
[
|
||||
8.23,
|
||||
113
|
||||
],
|
||||
[
|
||||
8.98,
|
||||
104
|
||||
],
|
||||
[
|
||||
13.3,
|
||||
63
|
||||
],
|
||||
[
|
||||
14.15,
|
||||
56
|
||||
],
|
||||
[
|
||||
16.59,
|
||||
31
|
||||
],
|
||||
[
|
||||
17.46,
|
||||
25
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
88
|
||||
],
|
||||
[
|
||||
8.98,
|
||||
120
|
||||
],
|
||||
[
|
||||
17.46,
|
||||
24
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
144.1
|
||||
],
|
||||
[
|
||||
5.89,
|
||||
149.3
|
||||
],
|
||||
[
|
||||
8.98,
|
||||
148.1
|
||||
],
|
||||
[
|
||||
14.15,
|
||||
139.2
|
||||
],
|
||||
[
|
||||
17.46,
|
||||
143.6
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,120 @@
|
||||
{
|
||||
"label": "c2-p2-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.53,
|
||||
"drain_seconds": 23.59,
|
||||
"throughput_ready_per_s": 2.615,
|
||||
"time_to_ready_s": {
|
||||
"p50": 8.955,
|
||||
"p95": 15.298,
|
||||
"p99": 15.667,
|
||||
"max": 15.667,
|
||||
"mean": 8.775
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.167,
|
||||
"p95": 0.332
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 41,
|
||||
"max_broker_messages": 30
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 239.2,
|
||||
"idle_first": 138.2
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
2
|
||||
],
|
||||
[
|
||||
4.95,
|
||||
32
|
||||
],
|
||||
[
|
||||
5.97,
|
||||
37
|
||||
],
|
||||
[
|
||||
8.59,
|
||||
41
|
||||
],
|
||||
[
|
||||
9.57,
|
||||
38
|
||||
],
|
||||
[
|
||||
14.5,
|
||||
25
|
||||
],
|
||||
[
|
||||
15.36,
|
||||
22
|
||||
],
|
||||
[
|
||||
17.83,
|
||||
15
|
||||
],
|
||||
[
|
||||
18.69,
|
||||
12
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
18
|
||||
],
|
||||
[
|
||||
9.57,
|
||||
30
|
||||
],
|
||||
[
|
||||
18.69,
|
||||
2
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
138.2
|
||||
],
|
||||
[
|
||||
5.97,
|
||||
187.7
|
||||
],
|
||||
[
|
||||
9.57,
|
||||
222.7
|
||||
],
|
||||
[
|
||||
15.36,
|
||||
229.2
|
||||
],
|
||||
[
|
||||
18.69,
|
||||
239.2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"label": "c2-p2-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.51,
|
||||
"drain_seconds": 27.53,
|
||||
"throughput_ready_per_s": 2.25,
|
||||
"time_to_ready_s": {
|
||||
"p50": 11.312,
|
||||
"p95": 19.302,
|
||||
"p99": 19.573,
|
||||
"max": 19.573,
|
||||
"mean": 11.959
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.871,
|
||||
"p95": 3.157
|
||||
},
|
||||
"generator_max_late_s": 0.019,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 51,
|
||||
"max_broker_messages": 50
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 253.8,
|
||||
"idle_first": 253.8
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
9.14,
|
||||
51
|
||||
],
|
||||
[
|
||||
10.03,
|
||||
48
|
||||
],
|
||||
[
|
||||
12.53,
|
||||
42
|
||||
],
|
||||
[
|
||||
13.41,
|
||||
39
|
||||
],
|
||||
[
|
||||
18.27,
|
||||
25
|
||||
],
|
||||
[
|
||||
19.16,
|
||||
22
|
||||
],
|
||||
[
|
||||
21.63,
|
||||
16
|
||||
],
|
||||
[
|
||||
22.56,
|
||||
13
|
||||
],
|
||||
[
|
||||
27.16,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
50
|
||||
],
|
||||
[
|
||||
13.41,
|
||||
30
|
||||
],
|
||||
[
|
||||
22.56,
|
||||
16
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
253.8
|
||||
],
|
||||
[
|
||||
10.03,
|
||||
200.2
|
||||
],
|
||||
[
|
||||
13.41,
|
||||
192.3
|
||||
],
|
||||
[
|
||||
19.16,
|
||||
200.9
|
||||
],
|
||||
[
|
||||
22.56,
|
||||
196.6
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
{
|
||||
"label": "c2-p2-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.03,
|
||||
"drain_seconds": 34.07,
|
||||
"throughput_ready_per_s": 6.085,
|
||||
"time_to_ready_s": {
|
||||
"p50": 16.317,
|
||||
"p95": 26.997,
|
||||
"p99": 27.79,
|
||||
"max": 27.979,
|
||||
"mean": 16.313
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.049,
|
||||
"p95": 0.112
|
||||
},
|
||||
"generator_max_late_s": 0.009,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 170,
|
||||
"max_broker_messages": 180
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 113.1,
|
||||
"idle_first": 78.4
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
19
|
||||
],
|
||||
[
|
||||
8.28,
|
||||
170
|
||||
],
|
||||
[
|
||||
9.17,
|
||||
164
|
||||
],
|
||||
[
|
||||
11.74,
|
||||
146
|
||||
],
|
||||
[
|
||||
12.62,
|
||||
140
|
||||
],
|
||||
[
|
||||
18.17,
|
||||
103
|
||||
],
|
||||
[
|
||||
19.05,
|
||||
96
|
||||
],
|
||||
[
|
||||
21.64,
|
||||
79
|
||||
],
|
||||
[
|
||||
22.57,
|
||||
73
|
||||
],
|
||||
[
|
||||
27.7,
|
||||
37
|
||||
],
|
||||
[
|
||||
28.69,
|
||||
31
|
||||
],
|
||||
[
|
||||
31.14,
|
||||
13
|
||||
],
|
||||
[
|
||||
32.11,
|
||||
5
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
180
|
||||
],
|
||||
[
|
||||
12.62,
|
||||
115
|
||||
],
|
||||
[
|
||||
22.57,
|
||||
45
|
||||
],
|
||||
[
|
||||
32.11,
|
||||
0
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
78.4
|
||||
],
|
||||
[
|
||||
9.17,
|
||||
110.6
|
||||
],
|
||||
[
|
||||
12.62,
|
||||
106.4
|
||||
],
|
||||
[
|
||||
19.05,
|
||||
108.2
|
||||
],
|
||||
[
|
||||
22.57,
|
||||
113.1
|
||||
],
|
||||
[
|
||||
28.69,
|
||||
113.0
|
||||
],
|
||||
[
|
||||
32.11,
|
||||
104.6
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
{
|
||||
"label": "c2-p2-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.01,
|
||||
"drain_seconds": 38.38,
|
||||
"throughput_ready_per_s": 5.29,
|
||||
"time_to_ready_s": {
|
||||
"p50": 19.744,
|
||||
"p95": 31.828,
|
||||
"p99": 32.749,
|
||||
"max": 32.946,
|
||||
"mean": 18.353
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.053,
|
||||
"p95": 0.154
|
||||
},
|
||||
"generator_max_late_s": 0.014,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 170,
|
||||
"max_broker_messages": 131
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 153.1,
|
||||
"idle_first": 139.4
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
18
|
||||
],
|
||||
[
|
||||
5.62,
|
||||
170
|
||||
],
|
||||
[
|
||||
6.77,
|
||||
165
|
||||
],
|
||||
[
|
||||
9.59,
|
||||
156
|
||||
],
|
||||
[
|
||||
11.0,
|
||||
151
|
||||
],
|
||||
[
|
||||
17.53,
|
||||
121
|
||||
],
|
||||
[
|
||||
18.48,
|
||||
117
|
||||
],
|
||||
[
|
||||
21.02,
|
||||
105
|
||||
],
|
||||
[
|
||||
21.94,
|
||||
100
|
||||
],
|
||||
[
|
||||
28.43,
|
||||
62
|
||||
],
|
||||
[
|
||||
29.36,
|
||||
56
|
||||
],
|
||||
[
|
||||
31.88,
|
||||
38
|
||||
],
|
||||
[
|
||||
32.85,
|
||||
32
|
||||
],
|
||||
[
|
||||
38.04,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
38
|
||||
],
|
||||
[
|
||||
11.0,
|
||||
131
|
||||
],
|
||||
[
|
||||
21.94,
|
||||
78
|
||||
],
|
||||
[
|
||||
32.85,
|
||||
14
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
139.4
|
||||
],
|
||||
[
|
||||
6.77,
|
||||
148.2
|
||||
],
|
||||
[
|
||||
11.0,
|
||||
153.1
|
||||
],
|
||||
[
|
||||
18.48,
|
||||
148.5
|
||||
],
|
||||
[
|
||||
21.94,
|
||||
139.8
|
||||
],
|
||||
[
|
||||
29.36,
|
||||
150.5
|
||||
],
|
||||
[
|
||||
32.85,
|
||||
149.4
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"label": "c4-p4-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.47,
|
||||
"drain_seconds": 16.84,
|
||||
"throughput_ready_per_s": 3.729,
|
||||
"time_to_ready_s": {
|
||||
"p50": 5.538,
|
||||
"p95": 8.81,
|
||||
"p99": 8.9,
|
||||
"max": 8.9,
|
||||
"mean": 5.31
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.131,
|
||||
"p95": 0.237
|
||||
},
|
||||
"generator_max_late_s": 0.008,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 36,
|
||||
"max_broker_messages": 19
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 248.5,
|
||||
"idle_first": 174.6
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
6.89,
|
||||
36
|
||||
],
|
||||
[
|
||||
7.83,
|
||||
35
|
||||
],
|
||||
[
|
||||
10.29,
|
||||
25
|
||||
],
|
||||
[
|
||||
11.07,
|
||||
21
|
||||
],
|
||||
[
|
||||
15.37,
|
||||
4
|
||||
],
|
||||
[
|
||||
16.26,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
11
|
||||
],
|
||||
[
|
||||
11.07,
|
||||
19
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
174.6
|
||||
],
|
||||
[
|
||||
7.83,
|
||||
196.0
|
||||
],
|
||||
[
|
||||
11.07,
|
||||
248.5
|
||||
],
|
||||
[
|
||||
16.26,
|
||||
159.9
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"label": "c4-p4-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 60,
|
||||
"rate_per_s": 8.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.49,
|
||||
"drain_seconds": 15.93,
|
||||
"throughput_ready_per_s": 3.883,
|
||||
"time_to_ready_s": {
|
||||
"p50": 4.504,
|
||||
"p95": 8.075,
|
||||
"p99": 8.152,
|
||||
"max": 8.152,
|
||||
"mean": 4.407
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.114,
|
||||
"p95": 0.225
|
||||
},
|
||||
"generator_max_late_s": 0.006,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 30,
|
||||
"max_broker_messages": 17
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 338.9,
|
||||
"idle_first": 286.2
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
5.59,
|
||||
27
|
||||
],
|
||||
[
|
||||
6.5,
|
||||
30
|
||||
],
|
||||
[
|
||||
8.92,
|
||||
27
|
||||
],
|
||||
[
|
||||
9.77,
|
||||
23
|
||||
],
|
||||
[
|
||||
14.76,
|
||||
3
|
||||
],
|
||||
[
|
||||
15.6,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
17
|
||||
],
|
||||
[
|
||||
9.77,
|
||||
9
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
286.2
|
||||
],
|
||||
[
|
||||
6.5,
|
||||
338.9
|
||||
],
|
||||
[
|
||||
9.77,
|
||||
324.9
|
||||
],
|
||||
[
|
||||
15.6,
|
||||
170.1
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"label": "c4-p4-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.0,
|
||||
"drain_seconds": 15.92,
|
||||
"throughput_ready_per_s": 13.171,
|
||||
"time_to_ready_s": {
|
||||
"p50": 5.745,
|
||||
"p95": 9.89,
|
||||
"p99": 10.199,
|
||||
"max": 10.259,
|
||||
"mean": 6.057
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.029,
|
||||
"p95": 0.051
|
||||
},
|
||||
"generator_max_late_s": 0.004,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 146,
|
||||
"max_broker_messages": 86
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 113.9,
|
||||
"idle_first": 92.1
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
11
|
||||
],
|
||||
[
|
||||
5.37,
|
||||
146
|
||||
],
|
||||
[
|
||||
6.15,
|
||||
133
|
||||
],
|
||||
[
|
||||
8.53,
|
||||
92
|
||||
],
|
||||
[
|
||||
9.34,
|
||||
79
|
||||
],
|
||||
[
|
||||
13.83,
|
||||
19
|
||||
],
|
||||
[
|
||||
14.63,
|
||||
6
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
86
|
||||
],
|
||||
[
|
||||
9.34,
|
||||
47
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
92.1
|
||||
],
|
||||
[
|
||||
6.15,
|
||||
107.5
|
||||
],
|
||||
[
|
||||
9.34,
|
||||
113.9
|
||||
],
|
||||
[
|
||||
14.63,
|
||||
107.3
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"label": "c4-p4-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 200,
|
||||
"rate_per_s": 40.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 200
|
||||
},
|
||||
"accepted": 200,
|
||||
"ready": 200,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.01,
|
||||
"drain_seconds": 16.28,
|
||||
"throughput_ready_per_s": 12.521,
|
||||
"time_to_ready_s": {
|
||||
"p50": 5.741,
|
||||
"p95": 10.568,
|
||||
"p99": 10.924,
|
||||
"max": 11.02,
|
||||
"mean": 5.689
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.035,
|
||||
"p95": 0.089
|
||||
},
|
||||
"generator_max_late_s": 0.007,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 121,
|
||||
"max_broker_messages": 137
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 169.5,
|
||||
"idle_first": 160.1
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
9
|
||||
],
|
||||
[
|
||||
6.28,
|
||||
121
|
||||
],
|
||||
[
|
||||
7.13,
|
||||
109
|
||||
],
|
||||
[
|
||||
9.61,
|
||||
78
|
||||
],
|
||||
[
|
||||
10.48,
|
||||
67
|
||||
],
|
||||
[
|
||||
15.43,
|
||||
6
|
||||
],
|
||||
[
|
||||
16.24,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
137
|
||||
],
|
||||
[
|
||||
10.48,
|
||||
72
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
160.1
|
||||
],
|
||||
[
|
||||
7.13,
|
||||
169.5
|
||||
],
|
||||
[
|
||||
10.48,
|
||||
160.1
|
||||
],
|
||||
[
|
||||
16.24,
|
||||
159.9
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,149 @@
|
||||
{
|
||||
"label": "backpressure-large",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 80,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "20",
|
||||
"upload_status": {
|
||||
"202": 26,
|
||||
"503": 54
|
||||
},
|
||||
"accepted": 26,
|
||||
"ready": 26,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 7.91,
|
||||
"drain_seconds": 26.08,
|
||||
"throughput_ready_per_s": 1.014,
|
||||
"time_to_ready_s": {
|
||||
"p50": 12.892,
|
||||
"p95": 19.227,
|
||||
"p99": 19.468,
|
||||
"max": 19.468,
|
||||
"mean": 12.189
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.006,
|
||||
"p95": 0.112
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 20,
|
||||
"max_broker_messages": 20
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 175.0,
|
||||
"idle_first": 118.4
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
2
|
||||
],
|
||||
[
|
||||
3.77,
|
||||
20
|
||||
],
|
||||
[
|
||||
4.49,
|
||||
20
|
||||
],
|
||||
[
|
||||
6.77,
|
||||
19
|
||||
],
|
||||
[
|
||||
7.51,
|
||||
20
|
||||
],
|
||||
[
|
||||
11.32,
|
||||
15
|
||||
],
|
||||
[
|
||||
12.07,
|
||||
15
|
||||
],
|
||||
[
|
||||
14.39,
|
||||
12
|
||||
],
|
||||
[
|
||||
15.16,
|
||||
11
|
||||
],
|
||||
[
|
||||
19.24,
|
||||
7
|
||||
],
|
||||
[
|
||||
20.01,
|
||||
7
|
||||
],
|
||||
[
|
||||
22.35,
|
||||
4
|
||||
],
|
||||
[
|
||||
23.17,
|
||||
4
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
12
|
||||
],
|
||||
[
|
||||
7.51,
|
||||
20
|
||||
],
|
||||
[
|
||||
15.16,
|
||||
11
|
||||
],
|
||||
[
|
||||
23.17,
|
||||
0
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
118.4
|
||||
],
|
||||
[
|
||||
4.49,
|
||||
110.8
|
||||
],
|
||||
[
|
||||
7.51,
|
||||
175.0
|
||||
],
|
||||
[
|
||||
12.07,
|
||||
129.1
|
||||
],
|
||||
[
|
||||
15.16,
|
||||
99.9
|
||||
],
|
||||
[
|
||||
20.01,
|
||||
118.2
|
||||
],
|
||||
[
|
||||
23.17,
|
||||
137.0
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"label": "c1-p1-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.85,
|
||||
"drain_seconds": 56.66,
|
||||
"throughput_ready_per_s": 0.714,
|
||||
"time_to_ready_s": {
|
||||
"p50": 24.858,
|
||||
"p95": 44.192,
|
||||
"p99": 46.347,
|
||||
"max": 46.347,
|
||||
"mean": 24.526
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.093,
|
||||
"p95": 0.154
|
||||
},
|
||||
"generator_max_late_s": 0.0,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 32,
|
||||
"max_broker_messages": 32
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 167.4,
|
||||
"idle_first": 161.7
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
5.27,
|
||||
19
|
||||
],
|
||||
[
|
||||
6.05,
|
||||
21
|
||||
],
|
||||
[
|
||||
8.4,
|
||||
29
|
||||
],
|
||||
[
|
||||
9.13,
|
||||
32
|
||||
],
|
||||
[
|
||||
13.05,
|
||||
31
|
||||
],
|
||||
[
|
||||
13.79,
|
||||
30
|
||||
],
|
||||
[
|
||||
16.22,
|
||||
29
|
||||
],
|
||||
[
|
||||
17.16,
|
||||
28
|
||||
],
|
||||
[
|
||||
21.77,
|
||||
26
|
||||
],
|
||||
[
|
||||
22.58,
|
||||
25
|
||||
],
|
||||
[
|
||||
24.93,
|
||||
24
|
||||
],
|
||||
[
|
||||
25.78,
|
||||
23
|
||||
],
|
||||
[
|
||||
29.66,
|
||||
20
|
||||
],
|
||||
[
|
||||
30.52,
|
||||
20
|
||||
],
|
||||
[
|
||||
32.91,
|
||||
18
|
||||
],
|
||||
[
|
||||
33.67,
|
||||
17
|
||||
],
|
||||
[
|
||||
38.45,
|
||||
14
|
||||
],
|
||||
[
|
||||
39.21,
|
||||
13
|
||||
],
|
||||
[
|
||||
41.55,
|
||||
11
|
||||
],
|
||||
[
|
||||
42.31,
|
||||
11
|
||||
],
|
||||
[
|
||||
46.91,
|
||||
7
|
||||
],
|
||||
[
|
||||
47.71,
|
||||
7
|
||||
],
|
||||
[
|
||||
50.03,
|
||||
5
|
||||
],
|
||||
[
|
||||
50.79,
|
||||
4
|
||||
],
|
||||
[
|
||||
54.84,
|
||||
1
|
||||
],
|
||||
[
|
||||
55.59,
|
||||
1
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
0
|
||||
],
|
||||
[
|
||||
9.13,
|
||||
32
|
||||
],
|
||||
[
|
||||
17.16,
|
||||
27
|
||||
],
|
||||
[
|
||||
25.78,
|
||||
24
|
||||
],
|
||||
[
|
||||
33.67,
|
||||
17
|
||||
],
|
||||
[
|
||||
42.31,
|
||||
9
|
||||
],
|
||||
[
|
||||
50.79,
|
||||
5
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
161.7
|
||||
],
|
||||
[
|
||||
6.05,
|
||||
105.0
|
||||
],
|
||||
[
|
||||
9.13,
|
||||
119.2
|
||||
],
|
||||
[
|
||||
13.79,
|
||||
90.5
|
||||
],
|
||||
[
|
||||
17.16,
|
||||
122.9
|
||||
],
|
||||
[
|
||||
22.58,
|
||||
167.4
|
||||
],
|
||||
[
|
||||
25.78,
|
||||
82.7
|
||||
],
|
||||
[
|
||||
30.52,
|
||||
105.7
|
||||
],
|
||||
[
|
||||
33.67,
|
||||
167.3
|
||||
],
|
||||
[
|
||||
39.21,
|
||||
81.4
|
||||
],
|
||||
[
|
||||
42.31,
|
||||
117.2
|
||||
],
|
||||
[
|
||||
47.71,
|
||||
117.4
|
||||
],
|
||||
[
|
||||
50.79,
|
||||
81.4
|
||||
],
|
||||
[
|
||||
55.59,
|
||||
81.4
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,232 @@
|
||||
{
|
||||
"label": "c1-p1-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.84,
|
||||
"drain_seconds": 52.43,
|
||||
"throughput_ready_per_s": 0.772,
|
||||
"time_to_ready_s": {
|
||||
"p50": 21.413,
|
||||
"p95": 39.883,
|
||||
"p99": 42.128,
|
||||
"max": 42.128,
|
||||
"mean": 21.96
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.09,
|
||||
"p95": 0.148
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 31,
|
||||
"max_broker_messages": 28
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 167.5,
|
||||
"idle_first": 81.4
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
3.81,
|
||||
13
|
||||
],
|
||||
[
|
||||
4.57,
|
||||
16
|
||||
],
|
||||
[
|
||||
6.92,
|
||||
24
|
||||
],
|
||||
[
|
||||
7.68,
|
||||
27
|
||||
],
|
||||
[
|
||||
12.19,
|
||||
31
|
||||
],
|
||||
[
|
||||
12.99,
|
||||
31
|
||||
],
|
||||
[
|
||||
15.28,
|
||||
29
|
||||
],
|
||||
[
|
||||
16.07,
|
||||
28
|
||||
],
|
||||
[
|
||||
21.01,
|
||||
25
|
||||
],
|
||||
[
|
||||
21.72,
|
||||
24
|
||||
],
|
||||
[
|
||||
24.01,
|
||||
22
|
||||
],
|
||||
[
|
||||
24.78,
|
||||
21
|
||||
],
|
||||
[
|
||||
28.59,
|
||||
18
|
||||
],
|
||||
[
|
||||
29.36,
|
||||
18
|
||||
],
|
||||
[
|
||||
31.66,
|
||||
17
|
||||
],
|
||||
[
|
||||
32.46,
|
||||
16
|
||||
],
|
||||
[
|
||||
36.31,
|
||||
13
|
||||
],
|
||||
[
|
||||
37.08,
|
||||
12
|
||||
],
|
||||
[
|
||||
39.39,
|
||||
10
|
||||
],
|
||||
[
|
||||
40.17,
|
||||
10
|
||||
],
|
||||
[
|
||||
43.83,
|
||||
7
|
||||
],
|
||||
[
|
||||
44.56,
|
||||
6
|
||||
],
|
||||
[
|
||||
46.98,
|
||||
4
|
||||
],
|
||||
[
|
||||
47.75,
|
||||
4
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
0
|
||||
],
|
||||
[
|
||||
7.68,
|
||||
28
|
||||
],
|
||||
[
|
||||
16.07,
|
||||
27
|
||||
],
|
||||
[
|
||||
24.78,
|
||||
23
|
||||
],
|
||||
[
|
||||
32.46,
|
||||
15
|
||||
],
|
||||
[
|
||||
40.17,
|
||||
11
|
||||
],
|
||||
[
|
||||
47.75,
|
||||
3
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
81.4
|
||||
],
|
||||
[
|
||||
4.57,
|
||||
105.0
|
||||
],
|
||||
[
|
||||
7.68,
|
||||
167.5
|
||||
],
|
||||
[
|
||||
12.99,
|
||||
119.3
|
||||
],
|
||||
[
|
||||
16.07,
|
||||
97.4
|
||||
],
|
||||
[
|
||||
21.72,
|
||||
107.4
|
||||
],
|
||||
[
|
||||
24.78,
|
||||
167.5
|
||||
],
|
||||
[
|
||||
29.36,
|
||||
92.6
|
||||
],
|
||||
[
|
||||
32.46,
|
||||
106.2
|
||||
],
|
||||
[
|
||||
37.08,
|
||||
85.2
|
||||
],
|
||||
[
|
||||
40.17,
|
||||
122.4
|
||||
],
|
||||
[
|
||||
44.56,
|
||||
100.4
|
||||
],
|
||||
[
|
||||
47.75,
|
||||
163.3
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
{
|
||||
"label": "c1-p1-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.98,
|
||||
"drain_seconds": 111.41,
|
||||
"throughput_ready_per_s": 0.542,
|
||||
"time_to_ready_s": {
|
||||
"p50": 64.273,
|
||||
"p95": 102.861,
|
||||
"p99": 104.752,
|
||||
"max": 104.752,
|
||||
"mean": 62.41
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.044,
|
||||
"p95": 0.139
|
||||
},
|
||||
"generator_max_late_s": 0.027,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 57,
|
||||
"max_broker_messages": 58
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 112.6,
|
||||
"idle_first": 108.9
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
9.74,
|
||||
57
|
||||
],
|
||||
[
|
||||
11.52,
|
||||
57
|
||||
],
|
||||
[
|
||||
15.64,
|
||||
56
|
||||
],
|
||||
[
|
||||
17.81,
|
||||
55
|
||||
],
|
||||
[
|
||||
25.92,
|
||||
53
|
||||
],
|
||||
[
|
||||
27.24,
|
||||
52
|
||||
],
|
||||
[
|
||||
30.32,
|
||||
51
|
||||
],
|
||||
[
|
||||
31.63,
|
||||
51
|
||||
],
|
||||
[
|
||||
39.09,
|
||||
47
|
||||
],
|
||||
[
|
||||
40.06,
|
||||
46
|
||||
],
|
||||
[
|
||||
42.47,
|
||||
45
|
||||
],
|
||||
[
|
||||
43.53,
|
||||
44
|
||||
],
|
||||
[
|
||||
48.48,
|
||||
41
|
||||
],
|
||||
[
|
||||
49.36,
|
||||
41
|
||||
],
|
||||
[
|
||||
51.81,
|
||||
39
|
||||
],
|
||||
[
|
||||
52.71,
|
||||
38
|
||||
],
|
||||
[
|
||||
57.49,
|
||||
35
|
||||
],
|
||||
[
|
||||
58.6,
|
||||
35
|
||||
],
|
||||
[
|
||||
61.16,
|
||||
33
|
||||
],
|
||||
[
|
||||
62.18,
|
||||
32
|
||||
],
|
||||
[
|
||||
70.26,
|
||||
28
|
||||
],
|
||||
[
|
||||
71.11,
|
||||
28
|
||||
],
|
||||
[
|
||||
73.63,
|
||||
26
|
||||
],
|
||||
[
|
||||
74.46,
|
||||
25
|
||||
],
|
||||
[
|
||||
79.85,
|
||||
22
|
||||
],
|
||||
[
|
||||
80.94,
|
||||
22
|
||||
],
|
||||
[
|
||||
83.78,
|
||||
20
|
||||
],
|
||||
[
|
||||
84.58,
|
||||
19
|
||||
],
|
||||
[
|
||||
89.99,
|
||||
15
|
||||
],
|
||||
[
|
||||
90.9,
|
||||
15
|
||||
],
|
||||
[
|
||||
93.86,
|
||||
13
|
||||
],
|
||||
[
|
||||
94.93,
|
||||
13
|
||||
],
|
||||
[
|
||||
101.49,
|
||||
9
|
||||
],
|
||||
[
|
||||
102.25,
|
||||
8
|
||||
],
|
||||
[
|
||||
104.62,
|
||||
5
|
||||
],
|
||||
[
|
||||
105.42,
|
||||
5
|
||||
],
|
||||
[
|
||||
109.78,
|
||||
1
|
||||
],
|
||||
[
|
||||
110.53,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
58
|
||||
],
|
||||
[
|
||||
17.81,
|
||||
55
|
||||
],
|
||||
[
|
||||
31.63,
|
||||
49
|
||||
],
|
||||
[
|
||||
43.53,
|
||||
43
|
||||
],
|
||||
[
|
||||
52.71,
|
||||
36
|
||||
],
|
||||
[
|
||||
62.18,
|
||||
31
|
||||
],
|
||||
[
|
||||
74.46,
|
||||
24
|
||||
],
|
||||
[
|
||||
84.58,
|
||||
18
|
||||
],
|
||||
[
|
||||
94.93,
|
||||
12
|
||||
],
|
||||
[
|
||||
105.42,
|
||||
4
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
108.9
|
||||
],
|
||||
[
|
||||
11.52,
|
||||
88.5
|
||||
],
|
||||
[
|
||||
17.81,
|
||||
97.3
|
||||
],
|
||||
[
|
||||
27.24,
|
||||
109.7
|
||||
],
|
||||
[
|
||||
31.63,
|
||||
83.6
|
||||
],
|
||||
[
|
||||
40.06,
|
||||
110.5
|
||||
],
|
||||
[
|
||||
43.53,
|
||||
109.8
|
||||
],
|
||||
[
|
||||
49.36,
|
||||
98.1
|
||||
],
|
||||
[
|
||||
52.71,
|
||||
81.9
|
||||
],
|
||||
[
|
||||
58.6,
|
||||
101.2
|
||||
],
|
||||
[
|
||||
62.18,
|
||||
107.2
|
||||
],
|
||||
[
|
||||
71.11,
|
||||
109.1
|
||||
],
|
||||
[
|
||||
74.46,
|
||||
89.0
|
||||
],
|
||||
[
|
||||
80.94,
|
||||
102.9
|
||||
],
|
||||
[
|
||||
84.58,
|
||||
76.5
|
||||
],
|
||||
[
|
||||
90.9,
|
||||
97.3
|
||||
],
|
||||
[
|
||||
94.93,
|
||||
98.0
|
||||
],
|
||||
[
|
||||
102.25,
|
||||
112.6
|
||||
],
|
||||
[
|
||||
105.42,
|
||||
108.8
|
||||
],
|
||||
[
|
||||
110.53,
|
||||
72.9
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
{
|
||||
"label": "c1-p1-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "1",
|
||||
"WORKER_PREFETCH": "1",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.92,
|
||||
"drain_seconds": 59.44,
|
||||
"throughput_ready_per_s": 1.019,
|
||||
"time_to_ready_s": {
|
||||
"p50": 28.487,
|
||||
"p95": 51.179,
|
||||
"p99": 53.022,
|
||||
"max": 53.022,
|
||||
"mean": 27.832
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.023,
|
||||
"p95": 0.047
|
||||
},
|
||||
"generator_max_late_s": 0.0,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 54,
|
||||
"max_broker_messages": 53
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 118.3,
|
||||
"idle_first": 103.1
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
5.4,
|
||||
52
|
||||
],
|
||||
[
|
||||
6.14,
|
||||
54
|
||||
],
|
||||
[
|
||||
8.52,
|
||||
52
|
||||
],
|
||||
[
|
||||
9.36,
|
||||
51
|
||||
],
|
||||
[
|
||||
13.5,
|
||||
47
|
||||
],
|
||||
[
|
||||
14.26,
|
||||
47
|
||||
],
|
||||
[
|
||||
16.58,
|
||||
44
|
||||
],
|
||||
[
|
||||
17.39,
|
||||
43
|
||||
],
|
||||
[
|
||||
21.55,
|
||||
39
|
||||
],
|
||||
[
|
||||
22.33,
|
||||
39
|
||||
],
|
||||
[
|
||||
24.72,
|
||||
37
|
||||
],
|
||||
[
|
||||
25.55,
|
||||
36
|
||||
],
|
||||
[
|
||||
29.35,
|
||||
32
|
||||
],
|
||||
[
|
||||
30.13,
|
||||
31
|
||||
],
|
||||
[
|
||||
32.6,
|
||||
29
|
||||
],
|
||||
[
|
||||
33.31,
|
||||
28
|
||||
],
|
||||
[
|
||||
38.15,
|
||||
23
|
||||
],
|
||||
[
|
||||
38.9,
|
||||
22
|
||||
],
|
||||
[
|
||||
41.18,
|
||||
19
|
||||
],
|
||||
[
|
||||
41.93,
|
||||
18
|
||||
],
|
||||
[
|
||||
46.26,
|
||||
14
|
||||
],
|
||||
[
|
||||
47.02,
|
||||
13
|
||||
],
|
||||
[
|
||||
49.32,
|
||||
10
|
||||
],
|
||||
[
|
||||
50.08,
|
||||
10
|
||||
],
|
||||
[
|
||||
53.97,
|
||||
6
|
||||
],
|
||||
[
|
||||
54.7,
|
||||
5
|
||||
],
|
||||
[
|
||||
56.99,
|
||||
2
|
||||
],
|
||||
[
|
||||
57.8,
|
||||
2
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
30
|
||||
],
|
||||
[
|
||||
9.36,
|
||||
53
|
||||
],
|
||||
[
|
||||
17.39,
|
||||
43
|
||||
],
|
||||
[
|
||||
25.55,
|
||||
34
|
||||
],
|
||||
[
|
||||
33.31,
|
||||
28
|
||||
],
|
||||
[
|
||||
41.93,
|
||||
17
|
||||
],
|
||||
[
|
||||
50.08,
|
||||
7
|
||||
],
|
||||
[
|
||||
57.8,
|
||||
1
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
103.1
|
||||
],
|
||||
[
|
||||
6.14,
|
||||
90.3
|
||||
],
|
||||
[
|
||||
9.36,
|
||||
98.5
|
||||
],
|
||||
[
|
||||
14.26,
|
||||
92.4
|
||||
],
|
||||
[
|
||||
17.39,
|
||||
98.6
|
||||
],
|
||||
[
|
||||
22.33,
|
||||
106.2
|
||||
],
|
||||
[
|
||||
25.55,
|
||||
103.1
|
||||
],
|
||||
[
|
||||
30.13,
|
||||
97.4
|
||||
],
|
||||
[
|
||||
33.31,
|
||||
118.3
|
||||
],
|
||||
[
|
||||
38.9,
|
||||
118.3
|
||||
],
|
||||
[
|
||||
41.93,
|
||||
97.2
|
||||
],
|
||||
[
|
||||
47.02,
|
||||
108.7
|
||||
],
|
||||
[
|
||||
50.08,
|
||||
86.6
|
||||
],
|
||||
[
|
||||
54.7,
|
||||
81.7
|
||||
],
|
||||
[
|
||||
57.8,
|
||||
81.4
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"label": "c2-p16-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.87,
|
||||
"drain_seconds": 43.42,
|
||||
"throughput_ready_per_s": 0.938,
|
||||
"time_to_ready_s": {
|
||||
"p50": 19.011,
|
||||
"p95": 32.083,
|
||||
"p99": 32.918,
|
||||
"max": 32.918,
|
||||
"mean": 18.912
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.126,
|
||||
"p95": 0.504
|
||||
},
|
||||
"generator_max_late_s": 0.04,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 34,
|
||||
"max_broker_messages": 33
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 187.0,
|
||||
"idle_first": 166.5
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
5.74,
|
||||
22
|
||||
],
|
||||
[
|
||||
6.53,
|
||||
24
|
||||
],
|
||||
[
|
||||
8.85,
|
||||
31
|
||||
],
|
||||
[
|
||||
9.71,
|
||||
34
|
||||
],
|
||||
[
|
||||
13.71,
|
||||
30
|
||||
],
|
||||
[
|
||||
14.54,
|
||||
29
|
||||
],
|
||||
[
|
||||
16.85,
|
||||
27
|
||||
],
|
||||
[
|
||||
17.7,
|
||||
26
|
||||
],
|
||||
[
|
||||
21.8,
|
||||
22
|
||||
],
|
||||
[
|
||||
22.59,
|
||||
21
|
||||
],
|
||||
[
|
||||
24.98,
|
||||
19
|
||||
],
|
||||
[
|
||||
25.69,
|
||||
18
|
||||
],
|
||||
[
|
||||
30.21,
|
||||
14
|
||||
],
|
||||
[
|
||||
30.97,
|
||||
13
|
||||
],
|
||||
[
|
||||
33.31,
|
||||
11
|
||||
],
|
||||
[
|
||||
34.05,
|
||||
10
|
||||
],
|
||||
[
|
||||
39.01,
|
||||
5
|
||||
],
|
||||
[
|
||||
39.75,
|
||||
4
|
||||
],
|
||||
[
|
||||
42.11,
|
||||
2
|
||||
],
|
||||
[
|
||||
42.93,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
9.71,
|
||||
33
|
||||
],
|
||||
[
|
||||
17.7,
|
||||
23
|
||||
],
|
||||
[
|
||||
25.69,
|
||||
18
|
||||
],
|
||||
[
|
||||
34.05,
|
||||
8
|
||||
],
|
||||
[
|
||||
42.93,
|
||||
0
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
166.5
|
||||
],
|
||||
[
|
||||
6.53,
|
||||
116.5
|
||||
],
|
||||
[
|
||||
9.71,
|
||||
132.6
|
||||
],
|
||||
[
|
||||
14.54,
|
||||
120.2
|
||||
],
|
||||
[
|
||||
17.7,
|
||||
123.7
|
||||
],
|
||||
[
|
||||
22.59,
|
||||
187.0
|
||||
],
|
||||
[
|
||||
25.69,
|
||||
119.1
|
||||
],
|
||||
[
|
||||
30.97,
|
||||
123.9
|
||||
],
|
||||
[
|
||||
34.05,
|
||||
141.0
|
||||
],
|
||||
[
|
||||
39.75,
|
||||
124.6
|
||||
],
|
||||
[
|
||||
42.93,
|
||||
99.8
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"label": "c2-p16-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.86,
|
||||
"drain_seconds": 42.32,
|
||||
"throughput_ready_per_s": 0.969,
|
||||
"time_to_ready_s": {
|
||||
"p50": 17.383,
|
||||
"p95": 30.681,
|
||||
"p99": 31.583,
|
||||
"max": 31.583,
|
||||
"mean": 17.421
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.111,
|
||||
"p95": 0.168
|
||||
},
|
||||
"generator_max_late_s": 0.0,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 30,
|
||||
"max_broker_messages": 25
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 204.7,
|
||||
"idle_first": 204.7
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
3.97,
|
||||
14
|
||||
],
|
||||
[
|
||||
4.73,
|
||||
17
|
||||
],
|
||||
[
|
||||
7.12,
|
||||
25
|
||||
],
|
||||
[
|
||||
7.97,
|
||||
27
|
||||
],
|
||||
[
|
||||
12.0,
|
||||
30
|
||||
],
|
||||
[
|
||||
12.81,
|
||||
29
|
||||
],
|
||||
[
|
||||
15.16,
|
||||
27
|
||||
],
|
||||
[
|
||||
15.92,
|
||||
26
|
||||
],
|
||||
[
|
||||
20.26,
|
||||
22
|
||||
],
|
||||
[
|
||||
21.14,
|
||||
21
|
||||
],
|
||||
[
|
||||
23.54,
|
||||
19
|
||||
],
|
||||
[
|
||||
24.27,
|
||||
18
|
||||
],
|
||||
[
|
||||
29.65,
|
||||
13
|
||||
],
|
||||
[
|
||||
30.41,
|
||||
12
|
||||
],
|
||||
[
|
||||
32.73,
|
||||
10
|
||||
],
|
||||
[
|
||||
33.46,
|
||||
9
|
||||
],
|
||||
[
|
||||
37.3,
|
||||
5
|
||||
],
|
||||
[
|
||||
38.1,
|
||||
4
|
||||
],
|
||||
[
|
||||
40.43,
|
||||
2
|
||||
],
|
||||
[
|
||||
41.21,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
9
|
||||
],
|
||||
[
|
||||
7.97,
|
||||
25
|
||||
],
|
||||
[
|
||||
15.92,
|
||||
25
|
||||
],
|
||||
[
|
||||
24.27,
|
||||
15
|
||||
],
|
||||
[
|
||||
33.46,
|
||||
10
|
||||
],
|
||||
[
|
||||
41.21,
|
||||
0
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
204.7
|
||||
],
|
||||
[
|
||||
4.73,
|
||||
147.2
|
||||
],
|
||||
[
|
||||
7.97,
|
||||
136.4
|
||||
],
|
||||
[
|
||||
12.81,
|
||||
175.8
|
||||
],
|
||||
[
|
||||
15.92,
|
||||
130.8
|
||||
],
|
||||
[
|
||||
21.14,
|
||||
129.4
|
||||
],
|
||||
[
|
||||
24.27,
|
||||
147.5
|
||||
],
|
||||
[
|
||||
30.41,
|
||||
141.5
|
||||
],
|
||||
[
|
||||
33.46,
|
||||
138.8
|
||||
],
|
||||
[
|
||||
38.1,
|
||||
131.9
|
||||
],
|
||||
[
|
||||
41.21,
|
||||
114.2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
{
|
||||
"label": "c2-p16-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.92,
|
||||
"drain_seconds": 52.56,
|
||||
"throughput_ready_per_s": 1.155,
|
||||
"time_to_ready_s": {
|
||||
"p50": 23.311,
|
||||
"p95": 45.166,
|
||||
"p99": 46.079,
|
||||
"max": 46.079,
|
||||
"mean": 23.808
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.023,
|
||||
"p95": 0.038
|
||||
},
|
||||
"generator_max_late_s": 0.0,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 52,
|
||||
"max_broker_messages": 53
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 112.5,
|
||||
"idle_first": 84.1
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
3.66,
|
||||
36
|
||||
],
|
||||
[
|
||||
4.45,
|
||||
43
|
||||
],
|
||||
[
|
||||
6.77,
|
||||
52
|
||||
],
|
||||
[
|
||||
7.49,
|
||||
52
|
||||
],
|
||||
[
|
||||
11.93,
|
||||
47
|
||||
],
|
||||
[
|
||||
12.67,
|
||||
46
|
||||
],
|
||||
[
|
||||
14.95,
|
||||
43
|
||||
],
|
||||
[
|
||||
15.68,
|
||||
42
|
||||
],
|
||||
[
|
||||
20.49,
|
||||
36
|
||||
],
|
||||
[
|
||||
21.24,
|
||||
36
|
||||
],
|
||||
[
|
||||
23.56,
|
||||
33
|
||||
],
|
||||
[
|
||||
24.33,
|
||||
32
|
||||
],
|
||||
[
|
||||
28.17,
|
||||
28
|
||||
],
|
||||
[
|
||||
28.9,
|
||||
27
|
||||
],
|
||||
[
|
||||
31.2,
|
||||
25
|
||||
],
|
||||
[
|
||||
32.0,
|
||||
24
|
||||
],
|
||||
[
|
||||
35.77,
|
||||
19
|
||||
],
|
||||
[
|
||||
36.52,
|
||||
19
|
||||
],
|
||||
[
|
||||
38.87,
|
||||
16
|
||||
],
|
||||
[
|
||||
39.62,
|
||||
15
|
||||
],
|
||||
[
|
||||
43.47,
|
||||
11
|
||||
],
|
||||
[
|
||||
44.27,
|
||||
10
|
||||
],
|
||||
[
|
||||
46.61,
|
||||
7
|
||||
],
|
||||
[
|
||||
47.34,
|
||||
6
|
||||
],
|
||||
[
|
||||
52.37,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
20
|
||||
],
|
||||
[
|
||||
7.49,
|
||||
53
|
||||
],
|
||||
[
|
||||
15.68,
|
||||
41
|
||||
],
|
||||
[
|
||||
24.33,
|
||||
30
|
||||
],
|
||||
[
|
||||
32.0,
|
||||
24
|
||||
],
|
||||
[
|
||||
39.62,
|
||||
13
|
||||
],
|
||||
[
|
||||
47.34,
|
||||
7
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
84.1
|
||||
],
|
||||
[
|
||||
4.45,
|
||||
98.0
|
||||
],
|
||||
[
|
||||
7.49,
|
||||
96.7
|
||||
],
|
||||
[
|
||||
12.67,
|
||||
111.4
|
||||
],
|
||||
[
|
||||
15.68,
|
||||
111.0
|
||||
],
|
||||
[
|
||||
21.24,
|
||||
109.8
|
||||
],
|
||||
[
|
||||
24.33,
|
||||
92.0
|
||||
],
|
||||
[
|
||||
28.9,
|
||||
95.9
|
||||
],
|
||||
[
|
||||
32.0,
|
||||
111.0
|
||||
],
|
||||
[
|
||||
36.52,
|
||||
112.5
|
||||
],
|
||||
[
|
||||
39.62,
|
||||
108.6
|
||||
],
|
||||
[
|
||||
44.27,
|
||||
101.7
|
||||
],
|
||||
[
|
||||
47.34,
|
||||
99.0
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"label": "c2-p16-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "16",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.93,
|
||||
"drain_seconds": 54.36,
|
||||
"throughput_ready_per_s": 1.129,
|
||||
"time_to_ready_s": {
|
||||
"p50": 24.153,
|
||||
"p95": 46.414,
|
||||
"p99": 47.296,
|
||||
"max": 47.296,
|
||||
"mean": 24.656
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.023,
|
||||
"p95": 0.028
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 53,
|
||||
"max_broker_messages": 54
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 142.1,
|
||||
"idle_first": 124.7
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
3.7,
|
||||
35
|
||||
],
|
||||
[
|
||||
4.43,
|
||||
43
|
||||
],
|
||||
[
|
||||
6.74,
|
||||
53
|
||||
],
|
||||
[
|
||||
7.55,
|
||||
52
|
||||
],
|
||||
[
|
||||
11.29,
|
||||
48
|
||||
],
|
||||
[
|
||||
12.05,
|
||||
47
|
||||
],
|
||||
[
|
||||
14.43,
|
||||
45
|
||||
],
|
||||
[
|
||||
15.24,
|
||||
44
|
||||
],
|
||||
[
|
||||
19.09,
|
||||
39
|
||||
],
|
||||
[
|
||||
19.93,
|
||||
39
|
||||
],
|
||||
[
|
||||
22.28,
|
||||
36
|
||||
],
|
||||
[
|
||||
23.07,
|
||||
35
|
||||
],
|
||||
[
|
||||
28.32,
|
||||
29
|
||||
],
|
||||
[
|
||||
29.09,
|
||||
28
|
||||
],
|
||||
[
|
||||
31.41,
|
||||
26
|
||||
],
|
||||
[
|
||||
32.2,
|
||||
25
|
||||
],
|
||||
[
|
||||
36.02,
|
||||
21
|
||||
],
|
||||
[
|
||||
36.75,
|
||||
20
|
||||
],
|
||||
[
|
||||
39.06,
|
||||
17
|
||||
],
|
||||
[
|
||||
39.84,
|
||||
16
|
||||
],
|
||||
[
|
||||
43.64,
|
||||
12
|
||||
],
|
||||
[
|
||||
44.41,
|
||||
11
|
||||
],
|
||||
[
|
||||
46.7,
|
||||
8
|
||||
],
|
||||
[
|
||||
47.5,
|
||||
8
|
||||
],
|
||||
[
|
||||
51.3,
|
||||
3
|
||||
],
|
||||
[
|
||||
52.08,
|
||||
2
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
12
|
||||
],
|
||||
[
|
||||
7.55,
|
||||
54
|
||||
],
|
||||
[
|
||||
15.24,
|
||||
43
|
||||
],
|
||||
[
|
||||
23.07,
|
||||
32
|
||||
],
|
||||
[
|
||||
32.2,
|
||||
26
|
||||
],
|
||||
[
|
||||
39.84,
|
||||
15
|
||||
],
|
||||
[
|
||||
47.5,
|
||||
9
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
124.7
|
||||
],
|
||||
[
|
||||
4.43,
|
||||
110.4
|
||||
],
|
||||
[
|
||||
7.55,
|
||||
118.9
|
||||
],
|
||||
[
|
||||
12.05,
|
||||
124.8
|
||||
],
|
||||
[
|
||||
15.24,
|
||||
139.3
|
||||
],
|
||||
[
|
||||
19.93,
|
||||
112.3
|
||||
],
|
||||
[
|
||||
23.07,
|
||||
140.7
|
||||
],
|
||||
[
|
||||
29.09,
|
||||
123.4
|
||||
],
|
||||
[
|
||||
32.2,
|
||||
139.4
|
||||
],
|
||||
[
|
||||
36.75,
|
||||
142.1
|
||||
],
|
||||
[
|
||||
39.84,
|
||||
126.4
|
||||
],
|
||||
[
|
||||
44.41,
|
||||
132.7
|
||||
],
|
||||
[
|
||||
47.5,
|
||||
119.8
|
||||
],
|
||||
[
|
||||
52.08,
|
||||
102.8
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
{
|
||||
"label": "c2-p2-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.83,
|
||||
"drain_seconds": 42.01,
|
||||
"throughput_ready_per_s": 0.974,
|
||||
"time_to_ready_s": {
|
||||
"p50": 16.79,
|
||||
"p95": 30.743,
|
||||
"p99": 31.45,
|
||||
"max": 31.45,
|
||||
"mean": 17.089
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.108,
|
||||
"p95": 0.181
|
||||
},
|
||||
"generator_max_late_s": 0.0,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 29,
|
||||
"max_broker_messages": 31
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 191.5,
|
||||
"idle_first": 103.5
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
4.46,
|
||||
16
|
||||
],
|
||||
[
|
||||
5.28,
|
||||
18
|
||||
],
|
||||
[
|
||||
7.66,
|
||||
26
|
||||
],
|
||||
[
|
||||
8.46,
|
||||
28
|
||||
],
|
||||
[
|
||||
12.59,
|
||||
29
|
||||
],
|
||||
[
|
||||
13.37,
|
||||
28
|
||||
],
|
||||
[
|
||||
15.81,
|
||||
26
|
||||
],
|
||||
[
|
||||
16.55,
|
||||
25
|
||||
],
|
||||
[
|
||||
20.47,
|
||||
21
|
||||
],
|
||||
[
|
||||
21.27,
|
||||
21
|
||||
],
|
||||
[
|
||||
23.75,
|
||||
18
|
||||
],
|
||||
[
|
||||
24.49,
|
||||
17
|
||||
],
|
||||
[
|
||||
29.86,
|
||||
12
|
||||
],
|
||||
[
|
||||
30.71,
|
||||
11
|
||||
],
|
||||
[
|
||||
33.03,
|
||||
9
|
||||
],
|
||||
[
|
||||
33.79,
|
||||
8
|
||||
],
|
||||
[
|
||||
38.33,
|
||||
4
|
||||
],
|
||||
[
|
||||
39.09,
|
||||
3
|
||||
],
|
||||
[
|
||||
41.46,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
2
|
||||
],
|
||||
[
|
||||
8.46,
|
||||
31
|
||||
],
|
||||
[
|
||||
16.55,
|
||||
27
|
||||
],
|
||||
[
|
||||
24.49,
|
||||
16
|
||||
],
|
||||
[
|
||||
33.79,
|
||||
7
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
103.5
|
||||
],
|
||||
[
|
||||
5.28,
|
||||
187.9
|
||||
],
|
||||
[
|
||||
8.46,
|
||||
104.7
|
||||
],
|
||||
[
|
||||
13.37,
|
||||
187.5
|
||||
],
|
||||
[
|
||||
16.55,
|
||||
108.2
|
||||
],
|
||||
[
|
||||
21.27,
|
||||
191.5
|
||||
],
|
||||
[
|
||||
24.49,
|
||||
114.2
|
||||
],
|
||||
[
|
||||
30.71,
|
||||
188.2
|
||||
],
|
||||
[
|
||||
33.79,
|
||||
110.3
|
||||
],
|
||||
[
|
||||
39.09,
|
||||
122.7
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
{
|
||||
"label": "c2-p2-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.89,
|
||||
"drain_seconds": 42.23,
|
||||
"throughput_ready_per_s": 0.964,
|
||||
"time_to_ready_s": {
|
||||
"p50": 17.196,
|
||||
"p95": 31.073,
|
||||
"p99": 31.762,
|
||||
"max": 31.762,
|
||||
"mean": 17.437
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.105,
|
||||
"p95": 0.157
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 32,
|
||||
"max_broker_messages": 30
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 214.4,
|
||||
"idle_first": 125.8
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
5.67,
|
||||
20
|
||||
],
|
||||
[
|
||||
6.44,
|
||||
22
|
||||
],
|
||||
[
|
||||
8.78,
|
||||
29
|
||||
],
|
||||
[
|
||||
9.6,
|
||||
32
|
||||
],
|
||||
[
|
||||
13.66,
|
||||
28
|
||||
],
|
||||
[
|
||||
14.52,
|
||||
28
|
||||
],
|
||||
[
|
||||
16.8,
|
||||
25
|
||||
],
|
||||
[
|
||||
17.64,
|
||||
25
|
||||
],
|
||||
[
|
||||
21.57,
|
||||
21
|
||||
],
|
||||
[
|
||||
22.37,
|
||||
20
|
||||
],
|
||||
[
|
||||
24.81,
|
||||
18
|
||||
],
|
||||
[
|
||||
25.55,
|
||||
17
|
||||
],
|
||||
[
|
||||
29.48,
|
||||
13
|
||||
],
|
||||
[
|
||||
30.27,
|
||||
12
|
||||
],
|
||||
[
|
||||
32.68,
|
||||
10
|
||||
],
|
||||
[
|
||||
33.44,
|
||||
9
|
||||
],
|
||||
[
|
||||
38.27,
|
||||
4
|
||||
],
|
||||
[
|
||||
39.09,
|
||||
4
|
||||
],
|
||||
[
|
||||
41.44,
|
||||
0
|
||||
],
|
||||
[
|
||||
42.17,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
6
|
||||
],
|
||||
[
|
||||
9.6,
|
||||
30
|
||||
],
|
||||
[
|
||||
17.64,
|
||||
26
|
||||
],
|
||||
[
|
||||
25.55,
|
||||
16
|
||||
],
|
||||
[
|
||||
33.44,
|
||||
6
|
||||
],
|
||||
[
|
||||
42.17,
|
||||
0
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
125.8
|
||||
],
|
||||
[
|
||||
6.44,
|
||||
131.5
|
||||
],
|
||||
[
|
||||
9.6,
|
||||
129.1
|
||||
],
|
||||
[
|
||||
14.52,
|
||||
187.2
|
||||
],
|
||||
[
|
||||
17.64,
|
||||
214.4
|
||||
],
|
||||
[
|
||||
22.37,
|
||||
141.2
|
||||
],
|
||||
[
|
||||
25.55,
|
||||
142.2
|
||||
],
|
||||
[
|
||||
30.27,
|
||||
213.7
|
||||
],
|
||||
[
|
||||
33.44,
|
||||
141.6
|
||||
],
|
||||
[
|
||||
39.09,
|
||||
181.2
|
||||
],
|
||||
[
|
||||
42.17,
|
||||
116.8
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"label": "c2-p2-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.92,
|
||||
"drain_seconds": 54.15,
|
||||
"throughput_ready_per_s": 1.118,
|
||||
"time_to_ready_s": {
|
||||
"p50": 24.31,
|
||||
"p95": 46.982,
|
||||
"p99": 47.842,
|
||||
"max": 47.842,
|
||||
"mean": 24.75
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.023,
|
||||
"p95": 0.036
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 53,
|
||||
"max_broker_messages": 50
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 113.3,
|
||||
"idle_first": 84.0
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
2
|
||||
],
|
||||
[
|
||||
4.36,
|
||||
43
|
||||
],
|
||||
[
|
||||
5.07,
|
||||
49
|
||||
],
|
||||
[
|
||||
7.37,
|
||||
53
|
||||
],
|
||||
[
|
||||
8.1,
|
||||
52
|
||||
],
|
||||
[
|
||||
11.8,
|
||||
47
|
||||
],
|
||||
[
|
||||
12.53,
|
||||
46
|
||||
],
|
||||
[
|
||||
14.8,
|
||||
44
|
||||
],
|
||||
[
|
||||
15.57,
|
||||
43
|
||||
],
|
||||
[
|
||||
19.52,
|
||||
39
|
||||
],
|
||||
[
|
||||
20.3,
|
||||
38
|
||||
],
|
||||
[
|
||||
22.61,
|
||||
35
|
||||
],
|
||||
[
|
||||
23.37,
|
||||
34
|
||||
],
|
||||
[
|
||||
27.32,
|
||||
30
|
||||
],
|
||||
[
|
||||
28.14,
|
||||
29
|
||||
],
|
||||
[
|
||||
30.49,
|
||||
27
|
||||
],
|
||||
[
|
||||
31.27,
|
||||
26
|
||||
],
|
||||
[
|
||||
36.62,
|
||||
19
|
||||
],
|
||||
[
|
||||
37.41,
|
||||
19
|
||||
],
|
||||
[
|
||||
39.8,
|
||||
15
|
||||
],
|
||||
[
|
||||
40.56,
|
||||
15
|
||||
],
|
||||
[
|
||||
44.72,
|
||||
11
|
||||
],
|
||||
[
|
||||
45.46,
|
||||
10
|
||||
],
|
||||
[
|
||||
47.84,
|
||||
7
|
||||
],
|
||||
[
|
||||
48.61,
|
||||
7
|
||||
],
|
||||
[
|
||||
52.52,
|
||||
2
|
||||
],
|
||||
[
|
||||
53.31,
|
||||
2
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
0
|
||||
],
|
||||
[
|
||||
8.1,
|
||||
50
|
||||
],
|
||||
[
|
||||
15.57,
|
||||
44
|
||||
],
|
||||
[
|
||||
23.37,
|
||||
33
|
||||
],
|
||||
[
|
||||
31.27,
|
||||
23
|
||||
],
|
||||
[
|
||||
40.56,
|
||||
17
|
||||
],
|
||||
[
|
||||
48.61,
|
||||
6
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
84.0
|
||||
],
|
||||
[
|
||||
5.07,
|
||||
89.6
|
||||
],
|
||||
[
|
||||
8.1,
|
||||
75.4
|
||||
],
|
||||
[
|
||||
12.53,
|
||||
97.0
|
||||
],
|
||||
[
|
||||
15.57,
|
||||
111.9
|
||||
],
|
||||
[
|
||||
20.3,
|
||||
104.7
|
||||
],
|
||||
[
|
||||
23.37,
|
||||
92.0
|
||||
],
|
||||
[
|
||||
28.14,
|
||||
91.6
|
||||
],
|
||||
[
|
||||
31.27,
|
||||
109.9
|
||||
],
|
||||
[
|
||||
37.41,
|
||||
98.9
|
||||
],
|
||||
[
|
||||
40.56,
|
||||
92.2
|
||||
],
|
||||
[
|
||||
45.46,
|
||||
98.8
|
||||
],
|
||||
[
|
||||
48.61,
|
||||
113.3
|
||||
],
|
||||
[
|
||||
53.31,
|
||||
74.2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"label": "c2-p2-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.92,
|
||||
"drain_seconds": 57.86,
|
||||
"throughput_ready_per_s": 1.044,
|
||||
"time_to_ready_s": {
|
||||
"p50": 28.017,
|
||||
"p95": 50.808,
|
||||
"p99": 51.638,
|
||||
"max": 51.638,
|
||||
"mean": 27.604
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.024,
|
||||
"p95": 0.041
|
||||
},
|
||||
"generator_max_late_s": 0.01,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 54,
|
||||
"max_broker_messages": 48
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 139.0,
|
||||
"idle_first": 124.9
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
5.4,
|
||||
52
|
||||
],
|
||||
[
|
||||
6.22,
|
||||
54
|
||||
],
|
||||
[
|
||||
8.53,
|
||||
51
|
||||
],
|
||||
[
|
||||
9.31,
|
||||
50
|
||||
],
|
||||
[
|
||||
15.24,
|
||||
47
|
||||
],
|
||||
[
|
||||
16.01,
|
||||
46
|
||||
],
|
||||
[
|
||||
18.34,
|
||||
43
|
||||
],
|
||||
[
|
||||
19.17,
|
||||
43
|
||||
],
|
||||
[
|
||||
23.02,
|
||||
38
|
||||
],
|
||||
[
|
||||
23.77,
|
||||
37
|
||||
],
|
||||
[
|
||||
26.11,
|
||||
36
|
||||
],
|
||||
[
|
||||
26.97,
|
||||
35
|
||||
],
|
||||
[
|
||||
30.73,
|
||||
30
|
||||
],
|
||||
[
|
||||
31.5,
|
||||
30
|
||||
],
|
||||
[
|
||||
33.83,
|
||||
27
|
||||
],
|
||||
[
|
||||
34.56,
|
||||
26
|
||||
],
|
||||
[
|
||||
39.24,
|
||||
21
|
||||
],
|
||||
[
|
||||
40.03,
|
||||
20
|
||||
],
|
||||
[
|
||||
42.36,
|
||||
17
|
||||
],
|
||||
[
|
||||
43.1,
|
||||
17
|
||||
],
|
||||
[
|
||||
47.84,
|
||||
11
|
||||
],
|
||||
[
|
||||
48.58,
|
||||
10
|
||||
],
|
||||
[
|
||||
50.89,
|
||||
8
|
||||
],
|
||||
[
|
||||
51.61,
|
||||
7
|
||||
],
|
||||
[
|
||||
56.09,
|
||||
3
|
||||
],
|
||||
[
|
||||
56.81,
|
||||
2
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
42
|
||||
],
|
||||
[
|
||||
9.31,
|
||||
48
|
||||
],
|
||||
[
|
||||
19.17,
|
||||
43
|
||||
],
|
||||
[
|
||||
26.97,
|
||||
32
|
||||
],
|
||||
[
|
||||
34.56,
|
||||
26
|
||||
],
|
||||
[
|
||||
43.1,
|
||||
15
|
||||
],
|
||||
[
|
||||
51.61,
|
||||
5
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
124.9
|
||||
],
|
||||
[
|
||||
6.22,
|
||||
123.6
|
||||
],
|
||||
[
|
||||
9.31,
|
||||
137.0
|
||||
],
|
||||
[
|
||||
16.01,
|
||||
135.8
|
||||
],
|
||||
[
|
||||
19.17,
|
||||
115.2
|
||||
],
|
||||
[
|
||||
23.77,
|
||||
136.0
|
||||
],
|
||||
[
|
||||
26.97,
|
||||
139.0
|
||||
],
|
||||
[
|
||||
31.5,
|
||||
138.9
|
||||
],
|
||||
[
|
||||
34.56,
|
||||
125.9
|
||||
],
|
||||
[
|
||||
40.03,
|
||||
118.6
|
||||
],
|
||||
[
|
||||
43.1,
|
||||
109.3
|
||||
],
|
||||
[
|
||||
48.58,
|
||||
118.1
|
||||
],
|
||||
[
|
||||
51.61,
|
||||
136.5
|
||||
],
|
||||
[
|
||||
56.81,
|
||||
100.4
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
{
|
||||
"label": "c4-p4-large-r1",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.85,
|
||||
"drain_seconds": 41.36,
|
||||
"throughput_ready_per_s": 0.999,
|
||||
"time_to_ready_s": {
|
||||
"p50": 17.132,
|
||||
"p95": 30.34,
|
||||
"p99": 30.651,
|
||||
"max": 30.651,
|
||||
"mean": 17.34
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.114,
|
||||
"p95": 0.171
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 28,
|
||||
"max_broker_messages": 31
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 223.9,
|
||||
"idle_first": 190.5
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
3.99,
|
||||
14
|
||||
],
|
||||
[
|
||||
4.91,
|
||||
18
|
||||
],
|
||||
[
|
||||
7.44,
|
||||
25
|
||||
],
|
||||
[
|
||||
8.2,
|
||||
28
|
||||
],
|
||||
[
|
||||
13.92,
|
||||
28
|
||||
],
|
||||
[
|
||||
14.74,
|
||||
28
|
||||
],
|
||||
[
|
||||
17.04,
|
||||
25
|
||||
],
|
||||
[
|
||||
17.81,
|
||||
25
|
||||
],
|
||||
[
|
||||
22.01,
|
||||
19
|
||||
],
|
||||
[
|
||||
22.82,
|
||||
19
|
||||
],
|
||||
[
|
||||
25.18,
|
||||
16
|
||||
],
|
||||
[
|
||||
25.98,
|
||||
16
|
||||
],
|
||||
[
|
||||
29.78,
|
||||
13
|
||||
],
|
||||
[
|
||||
30.5,
|
||||
10
|
||||
],
|
||||
[
|
||||
32.91,
|
||||
10
|
||||
],
|
||||
[
|
||||
33.74,
|
||||
7
|
||||
],
|
||||
[
|
||||
37.66,
|
||||
4
|
||||
],
|
||||
[
|
||||
38.45,
|
||||
4
|
||||
],
|
||||
[
|
||||
40.94,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
12
|
||||
],
|
||||
[
|
||||
8.2,
|
||||
31
|
||||
],
|
||||
[
|
||||
17.81,
|
||||
25
|
||||
],
|
||||
[
|
||||
25.98,
|
||||
15
|
||||
],
|
||||
[
|
||||
33.74,
|
||||
10
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
190.5
|
||||
],
|
||||
[
|
||||
4.91,
|
||||
138.5
|
||||
],
|
||||
[
|
||||
8.2,
|
||||
124.8
|
||||
],
|
||||
[
|
||||
14.74,
|
||||
179.0
|
||||
],
|
||||
[
|
||||
17.81,
|
||||
128.9
|
||||
],
|
||||
[
|
||||
22.82,
|
||||
139.0
|
||||
],
|
||||
[
|
||||
25.98,
|
||||
150.6
|
||||
],
|
||||
[
|
||||
30.5,
|
||||
223.9
|
||||
],
|
||||
[
|
||||
33.74,
|
||||
214.0
|
||||
],
|
||||
[
|
||||
38.45,
|
||||
205.9
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
{
|
||||
"label": "c4-p4-large-r2",
|
||||
"size": "large",
|
||||
"bytes": 3015368,
|
||||
"count": 40,
|
||||
"rate_per_s": 4.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 40
|
||||
},
|
||||
"accepted": 40,
|
||||
"ready": 40,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 9.86,
|
||||
"drain_seconds": 39.66,
|
||||
"throughput_ready_per_s": 1.018,
|
||||
"time_to_ready_s": {
|
||||
"p50": 17.391,
|
||||
"p95": 29.583,
|
||||
"p99": 29.973,
|
||||
"max": 29.973,
|
||||
"mean": 17.584
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.122,
|
||||
"p95": 0.17
|
||||
},
|
||||
"generator_max_late_s": 0.003,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 31,
|
||||
"max_broker_messages": 29
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 226.4,
|
||||
"idle_first": 226.4
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
4.99,
|
||||
19
|
||||
],
|
||||
[
|
||||
5.9,
|
||||
22
|
||||
],
|
||||
[
|
||||
8.22,
|
||||
28
|
||||
],
|
||||
[
|
||||
9.0,
|
||||
31
|
||||
],
|
||||
[
|
||||
13.04,
|
||||
31
|
||||
],
|
||||
[
|
||||
13.81,
|
||||
28
|
||||
],
|
||||
[
|
||||
16.2,
|
||||
28
|
||||
],
|
||||
[
|
||||
16.96,
|
||||
25
|
||||
],
|
||||
[
|
||||
20.83,
|
||||
22
|
||||
],
|
||||
[
|
||||
21.57,
|
||||
22
|
||||
],
|
||||
[
|
||||
23.96,
|
||||
19
|
||||
],
|
||||
[
|
||||
24.73,
|
||||
19
|
||||
],
|
||||
[
|
||||
29.1,
|
||||
13
|
||||
],
|
||||
[
|
||||
29.9,
|
||||
13
|
||||
],
|
||||
[
|
||||
32.23,
|
||||
10
|
||||
],
|
||||
[
|
||||
33.01,
|
||||
10
|
||||
],
|
||||
[
|
||||
37.93,
|
||||
4
|
||||
],
|
||||
[
|
||||
38.69,
|
||||
4
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
15
|
||||
],
|
||||
[
|
||||
9.0,
|
||||
29
|
||||
],
|
||||
[
|
||||
16.96,
|
||||
25
|
||||
],
|
||||
[
|
||||
24.73,
|
||||
19
|
||||
],
|
||||
[
|
||||
33.01,
|
||||
8
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
226.4
|
||||
],
|
||||
[
|
||||
5.9,
|
||||
153.0
|
||||
],
|
||||
[
|
||||
9.0,
|
||||
154.4
|
||||
],
|
||||
[
|
||||
13.81,
|
||||
155.9
|
||||
],
|
||||
[
|
||||
16.96,
|
||||
168.5
|
||||
],
|
||||
[
|
||||
21.57,
|
||||
157.0
|
||||
],
|
||||
[
|
||||
24.73,
|
||||
186.0
|
||||
],
|
||||
[
|
||||
29.9,
|
||||
155.6
|
||||
],
|
||||
[
|
||||
33.01,
|
||||
170.0
|
||||
],
|
||||
[
|
||||
38.69,
|
||||
132.5
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
{
|
||||
"label": "c4-p4-small-r1",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.92,
|
||||
"drain_seconds": 51.81,
|
||||
"throughput_ready_per_s": 1.171,
|
||||
"time_to_ready_s": {
|
||||
"p50": 25.537,
|
||||
"p95": 44.701,
|
||||
"p99": 45.387,
|
||||
"max": 45.387,
|
||||
"mean": 24.63
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.022,
|
||||
"p95": 0.035
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 53,
|
||||
"max_broker_messages": 53
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 123.1,
|
||||
"idle_first": 112.2
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
3.71,
|
||||
37
|
||||
],
|
||||
[
|
||||
4.46,
|
||||
43
|
||||
],
|
||||
[
|
||||
6.76,
|
||||
53
|
||||
],
|
||||
[
|
||||
7.53,
|
||||
52
|
||||
],
|
||||
[
|
||||
11.18,
|
||||
49
|
||||
],
|
||||
[
|
||||
11.91,
|
||||
49
|
||||
],
|
||||
[
|
||||
14.23,
|
||||
45
|
||||
],
|
||||
[
|
||||
14.97,
|
||||
45
|
||||
],
|
||||
[
|
||||
18.93,
|
||||
41
|
||||
],
|
||||
[
|
||||
19.67,
|
||||
38
|
||||
],
|
||||
[
|
||||
22.09,
|
||||
36
|
||||
],
|
||||
[
|
||||
22.81,
|
||||
35
|
||||
],
|
||||
[
|
||||
27.13,
|
||||
31
|
||||
],
|
||||
[
|
||||
28.11,
|
||||
31
|
||||
],
|
||||
[
|
||||
30.44,
|
||||
28
|
||||
],
|
||||
[
|
||||
31.18,
|
||||
26
|
||||
],
|
||||
[
|
||||
36.25,
|
||||
21
|
||||
],
|
||||
[
|
||||
37.01,
|
||||
17
|
||||
],
|
||||
[
|
||||
39.33,
|
||||
17
|
||||
],
|
||||
[
|
||||
40.08,
|
||||
14
|
||||
],
|
||||
[
|
||||
43.94,
|
||||
10
|
||||
],
|
||||
[
|
||||
44.69,
|
||||
9
|
||||
],
|
||||
[
|
||||
47.02,
|
||||
5
|
||||
],
|
||||
[
|
||||
47.8,
|
||||
5
|
||||
],
|
||||
[
|
||||
51.71,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
0
|
||||
],
|
||||
[
|
||||
7.53,
|
||||
53
|
||||
],
|
||||
[
|
||||
14.97,
|
||||
46
|
||||
],
|
||||
[
|
||||
22.81,
|
||||
35
|
||||
],
|
||||
[
|
||||
31.18,
|
||||
24
|
||||
],
|
||||
[
|
||||
40.08,
|
||||
17
|
||||
],
|
||||
[
|
||||
47.8,
|
||||
5
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
112.2
|
||||
],
|
||||
[
|
||||
4.46,
|
||||
104.2
|
||||
],
|
||||
[
|
||||
7.53,
|
||||
100.2
|
||||
],
|
||||
[
|
||||
11.91,
|
||||
116.0
|
||||
],
|
||||
[
|
||||
14.97,
|
||||
95.9
|
||||
],
|
||||
[
|
||||
19.67,
|
||||
110.3
|
||||
],
|
||||
[
|
||||
22.81,
|
||||
105.2
|
||||
],
|
||||
[
|
||||
28.11,
|
||||
123.1
|
||||
],
|
||||
[
|
||||
31.18,
|
||||
117.9
|
||||
],
|
||||
[
|
||||
37.01,
|
||||
116.7
|
||||
],
|
||||
[
|
||||
40.08,
|
||||
105.9
|
||||
],
|
||||
[
|
||||
44.69,
|
||||
118.3
|
||||
],
|
||||
[
|
||||
47.8,
|
||||
108.2
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
{
|
||||
"label": "c4-p4-small-r2",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "4",
|
||||
"WORKER_PREFETCH": "4",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "200",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.92,
|
||||
"drain_seconds": 54.95,
|
||||
"throughput_ready_per_s": 1.096,
|
||||
"time_to_ready_s": {
|
||||
"p50": 24.86,
|
||||
"p95": 48.903,
|
||||
"p99": 49.058,
|
||||
"max": 49.058,
|
||||
"mean": 25.564
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.023,
|
||||
"p95": 0.029
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 53,
|
||||
"max_broker_messages": 53
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 157.3,
|
||||
"idle_first": 143.6
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
3.83,
|
||||
37
|
||||
],
|
||||
[
|
||||
4.6,
|
||||
44
|
||||
],
|
||||
[
|
||||
6.89,
|
||||
53
|
||||
],
|
||||
[
|
||||
7.77,
|
||||
52
|
||||
],
|
||||
[
|
||||
11.78,
|
||||
48
|
||||
],
|
||||
[
|
||||
12.53,
|
||||
48
|
||||
],
|
||||
[
|
||||
14.91,
|
||||
45
|
||||
],
|
||||
[
|
||||
15.69,
|
||||
44
|
||||
],
|
||||
[
|
||||
19.5,
|
||||
40
|
||||
],
|
||||
[
|
||||
20.32,
|
||||
39
|
||||
],
|
||||
[
|
||||
22.69,
|
||||
36
|
||||
],
|
||||
[
|
||||
23.42,
|
||||
36
|
||||
],
|
||||
[
|
||||
28.49,
|
||||
29
|
||||
],
|
||||
[
|
||||
29.27,
|
||||
29
|
||||
],
|
||||
[
|
||||
31.56,
|
||||
25
|
||||
],
|
||||
[
|
||||
32.3,
|
||||
24
|
||||
],
|
||||
[
|
||||
36.72,
|
||||
20
|
||||
],
|
||||
[
|
||||
37.46,
|
||||
20
|
||||
],
|
||||
[
|
||||
39.81,
|
||||
16
|
||||
],
|
||||
[
|
||||
40.58,
|
||||
16
|
||||
],
|
||||
[
|
||||
44.6,
|
||||
12
|
||||
],
|
||||
[
|
||||
45.42,
|
||||
12
|
||||
],
|
||||
[
|
||||
47.85,
|
||||
8
|
||||
],
|
||||
[
|
||||
48.77,
|
||||
8
|
||||
],
|
||||
[
|
||||
53.12,
|
||||
5
|
||||
],
|
||||
[
|
||||
53.85,
|
||||
4
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
32
|
||||
],
|
||||
[
|
||||
7.77,
|
||||
53
|
||||
],
|
||||
[
|
||||
15.69,
|
||||
43
|
||||
],
|
||||
[
|
||||
23.42,
|
||||
37
|
||||
],
|
||||
[
|
||||
32.3,
|
||||
24
|
||||
],
|
||||
[
|
||||
40.58,
|
||||
13
|
||||
],
|
||||
[
|
||||
48.77,
|
||||
10
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
143.6
|
||||
],
|
||||
[
|
||||
4.6,
|
||||
156.8
|
||||
],
|
||||
[
|
||||
7.77,
|
||||
141.3
|
||||
],
|
||||
[
|
||||
12.53,
|
||||
147.3
|
||||
],
|
||||
[
|
||||
15.69,
|
||||
140.9
|
||||
],
|
||||
[
|
||||
20.32,
|
||||
149.8
|
||||
],
|
||||
[
|
||||
23.42,
|
||||
151.1
|
||||
],
|
||||
[
|
||||
29.27,
|
||||
136.1
|
||||
],
|
||||
[
|
||||
32.3,
|
||||
130.9
|
||||
],
|
||||
[
|
||||
37.46,
|
||||
157.3
|
||||
],
|
||||
[
|
||||
40.58,
|
||||
129.1
|
||||
],
|
||||
[
|
||||
45.42,
|
||||
156.3
|
||||
],
|
||||
[
|
||||
48.77,
|
||||
145.1
|
||||
],
|
||||
[
|
||||
53.85,
|
||||
120.4
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
{
|
||||
"label": "recovery-small",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 10.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "20",
|
||||
"upload_status": {
|
||||
"202": 26,
|
||||
"503": 34
|
||||
},
|
||||
"accepted": 26,
|
||||
"ready": 26,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 5.91,
|
||||
"drain_seconds": 23.91,
|
||||
"throughput_ready_per_s": 1.133,
|
||||
"time_to_ready_s": {
|
||||
"p50": 11.215,
|
||||
"p95": 17.979,
|
||||
"p99": 18.519,
|
||||
"max": 18.519,
|
||||
"mean": 10.686
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.006,
|
||||
"p95": 0.027
|
||||
},
|
||||
"generator_max_late_s": 0.0,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 20,
|
||||
"max_broker_messages": 21
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 126.6,
|
||||
"idle_first": 118.3
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
3
|
||||
],
|
||||
[
|
||||
4.6,
|
||||
20
|
||||
],
|
||||
[
|
||||
5.32,
|
||||
20
|
||||
],
|
||||
[
|
||||
7.6,
|
||||
18
|
||||
],
|
||||
[
|
||||
8.36,
|
||||
17
|
||||
],
|
||||
[
|
||||
12.21,
|
||||
13
|
||||
],
|
||||
[
|
||||
12.99,
|
||||
12
|
||||
],
|
||||
[
|
||||
15.32,
|
||||
9
|
||||
],
|
||||
[
|
||||
16.13,
|
||||
9
|
||||
],
|
||||
[
|
||||
20.2,
|
||||
4
|
||||
],
|
||||
[
|
||||
20.96,
|
||||
3
|
||||
],
|
||||
[
|
||||
23.29,
|
||||
0
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
21
|
||||
],
|
||||
[
|
||||
8.36,
|
||||
17
|
||||
],
|
||||
[
|
||||
16.13,
|
||||
6
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
118.3
|
||||
],
|
||||
[
|
||||
5.32,
|
||||
104.1
|
||||
],
|
||||
[
|
||||
8.36,
|
||||
125.6
|
||||
],
|
||||
[
|
||||
12.99,
|
||||
126.6
|
||||
],
|
||||
[
|
||||
16.13,
|
||||
105.7
|
||||
],
|
||||
[
|
||||
20.96,
|
||||
107.4
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
## Матрица concurrency/prefetch
|
||||
|
||||
| Размер | concurrency / prefetch | Повторов | Принято / ready / failed / потеряно | Throughput, ready/s | p50 до ready, с | p95 до ready, с | p99 до ready, с | Макс. незавершённых | Пик памяти worker, MiB | Разгрузка, с |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| small | 1 / 1 | 2 | 120 / 120 / 0 / 0 | 0.78 [0.54..1.02] | 46.38 [28.49..64.27] | 77.02 [51.18..102.86] | 78.89 [53.02..104.75] | 56 [54..57] | 115 [113..118] | 85.4 [59.4..111.4] |
|
||||
| small | 2 / 2 | 2 | 120 / 120 / 0 / 0 | 1.08 [1.04..1.12] | 26.16 [24.31..28.02] | 48.89 [46.98..50.81] | 49.74 [47.84..51.64] | 54 [53..54] | 126 [113..139] | 56.0 [54.1..57.9] |
|
||||
| small | 2 / 16 | 2 | 120 / 120 / 0 / 0 | 1.14 [1.13..1.16] | 23.73 [23.31..24.15] | 45.79 [45.17..46.41] | 46.69 [46.08..47.30] | 52 [52..53] | 127 [112..142] | 53.5 [52.6..54.4] |
|
||||
| small | 4 / 4 | 2 | 120 / 120 / 0 / 0 | 1.13 [1.10..1.17] | 25.20 [24.86..25.54] | 46.80 [44.70..48.90] | 47.22 [45.39..49.06] | 53 | 140 [123..157] | 53.4 [51.8..55.0] |
|
||||
| large | 1 / 1 | 2 | 80 / 80 / 0 / 0 | 0.74 [0.71..0.77] | 23.14 [21.41..24.86] | 42.04 [39.88..44.19] | 44.24 [42.13..46.35] | 32 [31..32] | 167 [167..168] | 54.5 [52.4..56.7] |
|
||||
| large | 2 / 2 | 2 | 80 / 80 / 0 / 0 | 0.97 [0.96..0.97] | 16.99 [16.79..17.20] | 30.91 [30.74..31.07] | 31.61 [31.45..31.76] | 30 [29..32] | 203 [192..214] | 42.1 [42.0..42.2] |
|
||||
| large | 2 / 16 | 2 | 80 / 80 / 0 / 0 | 0.95 [0.94..0.97] | 18.20 [17.38..19.01] | 31.38 [30.68..32.08] | 32.25 [31.58..32.92] | 32 [30..34] | 196 [187..205] | 42.9 [42.3..43.4] |
|
||||
| large | 4 / 4 | 2 | 80 / 80 / 0 / 0 | 1.01 [1.00..1.02] | 17.26 [17.13..17.39] | 29.96 [29.58..30.34] | 30.31 [29.97..30.65] | 30 [28..31] | 225 [224..226] | 40.5 [39.7..41.4] |
|
||||
|
||||
## Backpressure и восстановление
|
||||
|
||||
| Прогон | MAX_PENDING_JOBS | Поток | Ответы upload | Ready / failed / потеряно | Макс. незавершённых | p95 до ready, с | Throughput, ready/s | Пик памяти, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| backpressure-large | 20 | 80 x large @ 10.0/s | 202: 26, 503: 54 | 26 / 0 / 0 | 20 | 19.227 | 1.014 | 175.0 |
|
||||
| recovery-small | 20 | 60 x small @ 10.0/s | 202: 26, 503: 34 | 26 / 0 / 0 | 20 | 17.979 | 1.133 | 126.6 |
|
||||
@@ -0,0 +1,19 @@
|
||||
{"label": "c1-p1-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.542, "time_to_ready_s": {"p50": 64.273, "p95": 102.861, "p99": 104.752, "max": 104.752, "mean": 62.41}, "queue_lag": {"max_pending_jobs": 57, "max_broker_messages": 58}, "worker_memory_mib": {"peak": 112.6, "idle_first": 108.9}, "drain_seconds": 111.41}
|
||||
{"label": "c1-p1-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.714, "time_to_ready_s": {"p50": 24.858, "p95": 44.192, "p99": 46.347, "max": 46.347, "mean": 24.526}, "queue_lag": {"max_pending_jobs": 32, "max_broker_messages": 32}, "worker_memory_mib": {"peak": 167.4, "idle_first": 161.7}, "drain_seconds": 56.66}
|
||||
{"label": "c1-p1-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.019, "time_to_ready_s": {"p50": 28.487, "p95": 51.179, "p99": 53.022, "max": 53.022, "mean": 27.832}, "queue_lag": {"max_pending_jobs": 54, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 118.3, "idle_first": 103.1}, "drain_seconds": 59.44}
|
||||
{"label": "c1-p1-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.772, "time_to_ready_s": {"p50": 21.413, "p95": 39.883, "p99": 42.128, "max": 42.128, "mean": 21.96}, "queue_lag": {"max_pending_jobs": 31, "max_broker_messages": 28}, "worker_memory_mib": {"peak": 167.5, "idle_first": 81.4}, "drain_seconds": 52.43}
|
||||
{"label": "c2-p2-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.118, "time_to_ready_s": {"p50": 24.31, "p95": 46.982, "p99": 47.842, "max": 47.842, "mean": 24.75}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 50}, "worker_memory_mib": {"peak": 113.3, "idle_first": 84.0}, "drain_seconds": 54.15}
|
||||
{"label": "c2-p2-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.974, "time_to_ready_s": {"p50": 16.79, "p95": 30.743, "p99": 31.45, "max": 31.45, "mean": 17.089}, "queue_lag": {"max_pending_jobs": 29, "max_broker_messages": 31}, "worker_memory_mib": {"peak": 191.5, "idle_first": 103.5}, "drain_seconds": 42.01}
|
||||
{"label": "c2-p2-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.044, "time_to_ready_s": {"p50": 28.017, "p95": 50.808, "p99": 51.638, "max": 51.638, "mean": 27.604}, "queue_lag": {"max_pending_jobs": 54, "max_broker_messages": 48}, "worker_memory_mib": {"peak": 139.0, "idle_first": 124.9}, "drain_seconds": 57.86}
|
||||
{"label": "c2-p2-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.964, "time_to_ready_s": {"p50": 17.196, "p95": 31.073, "p99": 31.762, "max": 31.762, "mean": 17.437}, "queue_lag": {"max_pending_jobs": 32, "max_broker_messages": 30}, "worker_memory_mib": {"peak": 214.4, "idle_first": 125.8}, "drain_seconds": 42.23}
|
||||
{"label": "c4-p4-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.171, "time_to_ready_s": {"p50": 25.537, "p95": 44.701, "p99": 45.387, "max": 45.387, "mean": 24.63}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 123.1, "idle_first": 112.2}, "drain_seconds": 51.81}
|
||||
{"label": "c4-p4-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.999, "time_to_ready_s": {"p50": 17.132, "p95": 30.34, "p99": 30.651, "max": 30.651, "mean": 17.34}, "queue_lag": {"max_pending_jobs": 28, "max_broker_messages": 31}, "worker_memory_mib": {"peak": 223.9, "idle_first": 190.5}, "drain_seconds": 41.36}
|
||||
{"label": "c4-p4-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.096, "time_to_ready_s": {"p50": 24.86, "p95": 48.903, "p99": 49.058, "max": 49.058, "mean": 25.564}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 157.3, "idle_first": 143.6}, "drain_seconds": 54.95}
|
||||
{"label": "c4-p4-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.018, "time_to_ready_s": {"p50": 17.391, "p95": 29.583, "p99": 29.973, "max": 29.973, "mean": 17.584}, "queue_lag": {"max_pending_jobs": 31, "max_broker_messages": 29}, "worker_memory_mib": {"peak": 226.4, "idle_first": 226.4}, "drain_seconds": 39.66}
|
||||
{"label": "c2-p16-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.155, "time_to_ready_s": {"p50": 23.311, "p95": 45.166, "p99": 46.079, "max": 46.079, "mean": 23.808}, "queue_lag": {"max_pending_jobs": 52, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 112.5, "idle_first": 84.1}, "drain_seconds": 52.56}
|
||||
{"label": "c2-p16-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.938, "time_to_ready_s": {"p50": 19.011, "p95": 32.083, "p99": 32.918, "max": 32.918, "mean": 18.912}, "queue_lag": {"max_pending_jobs": 34, "max_broker_messages": 33}, "worker_memory_mib": {"peak": 187.0, "idle_first": 166.5}, "drain_seconds": 43.42}
|
||||
{"label": "c2-p16-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.129, "time_to_ready_s": {"p50": 24.153, "p95": 46.414, "p99": 47.296, "max": 47.296, "mean": 24.656}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 54}, "worker_memory_mib": {"peak": 142.1, "idle_first": 124.7}, "drain_seconds": 54.36}
|
||||
{"label": "c2-p16-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.969, "time_to_ready_s": {"p50": 17.383, "p95": 30.681, "p99": 31.583, "max": 31.583, "mean": 17.421}, "queue_lag": {"max_pending_jobs": 30, "max_broker_messages": 25}, "worker_memory_mib": {"peak": 204.7, "idle_first": 204.7}, "drain_seconds": 42.32}
|
||||
{"label": "backpressure-large", "upload_status": {"202": 26, "503": 54}, "ready": 26, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.014, "time_to_ready_s": {"p50": 12.892, "p95": 19.227, "p99": 19.468, "max": 19.468, "mean": 12.189}, "queue_lag": {"max_pending_jobs": 20, "max_broker_messages": 20}, "worker_memory_mib": {"peak": 175.0, "idle_first": 118.4}, "drain_seconds": 26.08}
|
||||
{"label": "recovery-small", "upload_status": {"202": 26, "503": 34}, "ready": 26, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.133, "time_to_ready_s": {"p50": 11.215, "p95": 17.979, "p99": 18.519, "max": 18.519, "mean": 10.686}, "queue_lag": {"max_pending_jobs": 20, "max_broker_messages": 21}, "worker_memory_mib": {"peak": 126.6, "idle_first": 118.3}, "drain_seconds": 23.91}
|
||||
track1 done
|
||||
@@ -0,0 +1,104 @@
|
||||
{
|
||||
"label": "recovery-small",
|
||||
"size": "small",
|
||||
"bytes": 111817,
|
||||
"count": 60,
|
||||
"rate_per_s": 5.0,
|
||||
"worker": {
|
||||
"WORKER_CONCURRENCY": "2",
|
||||
"WORKER_PREFETCH": "2",
|
||||
"WORKER_TASK_MEMORY_MB": "640"
|
||||
},
|
||||
"worker_replicas": 1,
|
||||
"max_pending_jobs": "20",
|
||||
"upload_status": {
|
||||
"202": 60
|
||||
},
|
||||
"accepted": 60,
|
||||
"ready": 60,
|
||||
"failed": 0,
|
||||
"lost_accepted": 0,
|
||||
"submit_seconds": 11.82,
|
||||
"drain_seconds": 12.1,
|
||||
"throughput_ready_per_s": 5.031,
|
||||
"time_to_ready_s": {
|
||||
"p50": 0.113,
|
||||
"p95": 0.145,
|
||||
"p99": 0.15,
|
||||
"max": 0.15,
|
||||
"mean": 0.116
|
||||
},
|
||||
"upload_latency_s": {
|
||||
"p50": 0.022,
|
||||
"p95": 0.029
|
||||
},
|
||||
"generator_max_late_s": 0.001,
|
||||
"queue_lag": {
|
||||
"max_pending_jobs": 1,
|
||||
"max_broker_messages": 0
|
||||
},
|
||||
"worker_memory_mib": {
|
||||
"peak": 125.6,
|
||||
"idle_first": 122.6
|
||||
},
|
||||
"timeline": {
|
||||
"pending": [
|
||||
[
|
||||
0.0,
|
||||
1
|
||||
],
|
||||
[
|
||||
3.6,
|
||||
0
|
||||
],
|
||||
[
|
||||
4.33,
|
||||
1
|
||||
],
|
||||
[
|
||||
6.61,
|
||||
1
|
||||
],
|
||||
[
|
||||
7.37,
|
||||
0
|
||||
],
|
||||
[
|
||||
10.94,
|
||||
0
|
||||
],
|
||||
[
|
||||
11.67,
|
||||
1
|
||||
]
|
||||
],
|
||||
"broker": [
|
||||
[
|
||||
0.0,
|
||||
0
|
||||
],
|
||||
[
|
||||
7.37,
|
||||
0
|
||||
]
|
||||
],
|
||||
"memory": [
|
||||
[
|
||||
0.0,
|
||||
122.6
|
||||
],
|
||||
[
|
||||
4.33,
|
||||
122.2
|
||||
],
|
||||
[
|
||||
7.37,
|
||||
125.6
|
||||
],
|
||||
[
|
||||
11.67,
|
||||
122.3
|
||||
]
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
## Матрица concurrency/prefetch
|
||||
|
||||
| Размер | concurrency / prefetch | Повторов | Принято / ready / failed / потеряно | Throughput, ready/s | p50 до ready, с | p95 до ready, с | p99 до ready, с | Макс. незавершённых | Пик памяти worker, MiB | Разгрузка, с |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| small | 1 / 1 | 2 | 400 / 400 / 0 / 0 | 4.50 [3.75..5.26] | 19.71 [14.65..24.77] | 38.18 [30.79..45.56] | 40.16 [32.73..47.59] | 179 [177..181] | 120 [110..130] | 46.4 [39.0..53.7] |
|
||||
| small | 2 / 2 | 2 | 400 / 400 / 0 / 0 | 5.69 [5.29..6.08] | 18.03 [16.32..19.74] | 29.41 [27.00..31.83] | 30.27 [27.79..32.75] | 170 | 133 [113..153] | 36.2 [34.1..38.4] |
|
||||
| small | 2 / 16 | 2 | 400 / 400 / 0 / 0 | 9.81 [9.60..10.02] | 7.84 [7.19..8.48] | 14.72 [14.26..15.17] | 15.28 [14.84..15.71] | 156 [148..165] | 133 [117..149] | 21.0 [20.9..21.0] |
|
||||
| small | 4 / 4 | 2 | 400 / 400 / 0 / 0 | 12.85 [12.52..13.17] | 5.74 [5.74..5.75] | 10.23 [9.89..10.57] | 10.56 [10.20..10.92] | 134 [121..146] | 142 [114..170] | 16.1 [15.9..16.3] |
|
||||
| large | 1 / 1 | 2 | 120 / 120 / 0 / 0 | 1.26 [1.13..1.39] | 19.35 [18.49..20.21] | 39.68 [34.84..44.53] | 40.86 [36.04..45.67] | 50 | 179 [173..186] | 49.4 [44.3..54.5] |
|
||||
| large | 2 / 2 | 2 | 120 / 120 / 0 / 0 | 2.43 [2.25..2.62] | 10.13 [8.96..11.31] | 17.30 [15.30..19.30] | 17.62 [15.67..19.57] | 46 [41..51] | 246 [239..254] | 25.6 [23.6..27.5] |
|
||||
| large | 2 / 16 | 2 | 120 / 120 / 0 / 0 | 3.23 [2.98..3.48] | 7.10 [5.55..8.64] | 11.09 [9.66..12.52] | 11.35 [9.91..12.79] | 40 [36..43] | 222 [205..239] | 19.7 [18.2..21.1] |
|
||||
| large | 4 / 4 | 2 | 120 / 120 / 0 / 0 | 3.81 [3.73..3.88] | 5.02 [4.50..5.54] | 8.44 [8.07..8.81] | 8.53 [8.15..8.90] | 33 [30..36] | 294 [248..339] | 16.4 [15.9..16.8] |
|
||||
|
||||
## Backpressure и восстановление
|
||||
|
||||
| Прогон | MAX_PENDING_JOBS | Поток | Ответы upload | Ready / failed / потеряно | Макс. незавершённых | p95 до ready, с | Throughput, ready/s | Пик памяти, MiB |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| backpressure-large | 20 | 120 x large @ 15.0/s | 202: 48, 503: 72 | 48 / 0 / 0 | 22 | 5.646 | 3.691 | 236.9 |
|
||||
| baseline-small | 200 | 60 x small @ 5.0/s | 202: 60 | 60 / 0 / 0 | 1 | 1.171 | 5.031 | 109.7 |
|
||||
| recovery-small | 20 | 60 x small @ 5.0/s | 202: 60 | 60 / 0 / 0 | 1 | 0.145 | 5.031 | 125.6 |
|
||||
@@ -0,0 +1,20 @@
|
||||
{"label": "c1-p1-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.256, "time_to_ready_s": {"p50": 14.65, "p95": 30.791, "p99": 32.731, "max": 33.144, "mean": 15.373}, "queue_lag": {"max_pending_jobs": 177, "max_broker_messages": 141}, "worker_memory_mib": {"peak": 109.7, "idle_first": 91.2}, "drain_seconds": 39.01}
|
||||
{"label": "c1-p1-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.131, "time_to_ready_s": {"p50": 20.214, "p95": 44.534, "p99": 45.674, "max": 45.674, "mean": 21.052}, "queue_lag": {"max_pending_jobs": 50, "max_broker_messages": 43}, "worker_memory_mib": {"peak": 172.9, "idle_first": 105.8}, "drain_seconds": 54.51}
|
||||
{"label": "c1-p1-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.745, "time_to_ready_s": {"p50": 24.775, "p95": 45.564, "p99": 47.589, "max": 48.149, "mean": 24.871}, "queue_lag": {"max_pending_jobs": 181, "max_broker_messages": 156}, "worker_memory_mib": {"peak": 130.1, "idle_first": 123.8}, "drain_seconds": 53.7}
|
||||
{"label": "c1-p1-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.385, "time_to_ready_s": {"p50": 18.493, "p95": 34.835, "p99": 36.043, "max": 36.043, "mean": 18.892}, "queue_lag": {"max_pending_jobs": 50, "max_broker_messages": 47}, "worker_memory_mib": {"peak": 185.6, "idle_first": 176.1}, "drain_seconds": 44.28}
|
||||
{"label": "c2-p2-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 6.085, "time_to_ready_s": {"p50": 16.317, "p95": 26.997, "p99": 27.79, "max": 27.979, "mean": 16.313}, "queue_lag": {"max_pending_jobs": 170, "max_broker_messages": 180}, "worker_memory_mib": {"peak": 113.1, "idle_first": 78.4}, "drain_seconds": 34.07}
|
||||
{"label": "c2-p2-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 2.615, "time_to_ready_s": {"p50": 8.955, "p95": 15.298, "p99": 15.667, "max": 15.667, "mean": 8.775}, "queue_lag": {"max_pending_jobs": 41, "max_broker_messages": 30}, "worker_memory_mib": {"peak": 239.2, "idle_first": 138.2}, "drain_seconds": 23.59}
|
||||
{"label": "c2-p2-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.29, "time_to_ready_s": {"p50": 19.744, "p95": 31.828, "p99": 32.749, "max": 32.946, "mean": 18.353}, "queue_lag": {"max_pending_jobs": 170, "max_broker_messages": 131}, "worker_memory_mib": {"peak": 153.1, "idle_first": 139.4}, "drain_seconds": 38.38}
|
||||
{"label": "c2-p2-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 2.25, "time_to_ready_s": {"p50": 11.312, "p95": 19.302, "p99": 19.573, "max": 19.573, "mean": 11.959}, "queue_lag": {"max_pending_jobs": 51, "max_broker_messages": 50}, "worker_memory_mib": {"peak": 253.8, "idle_first": 253.8}, "drain_seconds": 27.53}
|
||||
{"label": "c4-p4-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 13.171, "time_to_ready_s": {"p50": 5.745, "p95": 9.89, "p99": 10.199, "max": 10.259, "mean": 6.057}, "queue_lag": {"max_pending_jobs": 146, "max_broker_messages": 86}, "worker_memory_mib": {"peak": 113.9, "idle_first": 92.1}, "drain_seconds": 15.92}
|
||||
{"label": "c4-p4-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.729, "time_to_ready_s": {"p50": 5.538, "p95": 8.81, "p99": 8.9, "max": 8.9, "mean": 5.31}, "queue_lag": {"max_pending_jobs": 36, "max_broker_messages": 19}, "worker_memory_mib": {"peak": 248.5, "idle_first": 174.6}, "drain_seconds": 16.84}
|
||||
{"label": "c4-p4-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 12.521, "time_to_ready_s": {"p50": 5.741, "p95": 10.568, "p99": 10.924, "max": 11.02, "mean": 5.689}, "queue_lag": {"max_pending_jobs": 121, "max_broker_messages": 137}, "worker_memory_mib": {"peak": 169.5, "idle_first": 160.1}, "drain_seconds": 16.28}
|
||||
{"label": "c4-p4-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.883, "time_to_ready_s": {"p50": 4.504, "p95": 8.075, "p99": 8.152, "max": 8.152, "mean": 4.407}, "queue_lag": {"max_pending_jobs": 30, "max_broker_messages": 17}, "worker_memory_mib": {"peak": 338.9, "idle_first": 286.2}, "drain_seconds": 15.93}
|
||||
{"label": "c2-p16-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 9.604, "time_to_ready_s": {"p50": 8.479, "p95": 15.174, "p99": 15.711, "max": 15.851, "mean": 8.697}, "queue_lag": {"max_pending_jobs": 165, "max_broker_messages": 107}, "worker_memory_mib": {"peak": 117.2, "idle_first": 100.4}, "drain_seconds": 20.95}
|
||||
{"label": "c2-p16-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 2.98, "time_to_ready_s": {"p50": 8.645, "p95": 12.515, "p99": 12.789, "max": 12.789, "mean": 7.964}, "queue_lag": {"max_pending_jobs": 43, "max_broker_messages": 40}, "worker_memory_mib": {"peak": 238.7, "idle_first": 117.7}, "drain_seconds": 21.1}
|
||||
{"label": "c2-p16-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 10.022, "time_to_ready_s": {"p50": 7.194, "p95": 14.257, "p99": 14.841, "max": 14.998, "mean": 7.499}, "queue_lag": {"max_pending_jobs": 148, "max_broker_messages": 120}, "worker_memory_mib": {"peak": 149.3, "idle_first": 144.1}, "drain_seconds": 20.97}
|
||||
{"label": "c2-p16-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.475, "time_to_ready_s": {"p50": 5.555, "p95": 9.662, "p99": 9.912, "max": 9.912, "mean": 5.505}, "queue_lag": {"max_pending_jobs": 36, "max_broker_messages": 34}, "worker_memory_mib": {"peak": 204.8, "idle_first": 138.9}, "drain_seconds": 18.25}
|
||||
{"label": "baseline-small", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.031, "time_to_ready_s": {"p50": 0.115, "p95": 1.171, "p99": 1.377, "max": 1.377, "mean": 0.248}, "queue_lag": {"max_pending_jobs": 1, "max_broker_messages": 4}, "worker_memory_mib": {"peak": 109.7, "idle_first": 101.1}, "drain_seconds": 12.1}
|
||||
{"label": "backpressure-large", "upload_status": {"202": 48, "503": 72}, "ready": 48, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.691, "time_to_ready_s": {"p50": 4.937, "p95": 5.646, "p99": 5.682, "max": 5.682, "mean": 4.161}, "queue_lag": {"max_pending_jobs": 22, "max_broker_messages": 20}, "worker_memory_mib": {"peak": 236.9, "idle_first": 110.5}, "drain_seconds": 13.33}
|
||||
{"label": "recovery-small", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.031, "time_to_ready_s": {"p50": 0.113, "p95": 0.145, "p99": 0.15, "max": 0.15, "mean": 0.116}, "queue_lag": {"max_pending_jobs": 1, "max_broker_messages": 0}, "worker_memory_mib": {"peak": 125.6, "idle_first": 122.6}, "drain_seconds": 12.1}
|
||||
track1 done
|
||||
@@ -0,0 +1,89 @@
|
||||
# HAProxy — единственная публичная точка входа lab3.
|
||||
# Реплики API обнаруживаются через DNS Docker (имя сервиса "api" -> все IP его контейнеров),
|
||||
# поэтому масштабирование `--scale api=N` (N <= 8) не требует правки конфигурации.
|
||||
# Sticky session нет намеренно: любая реплика обслуживает любой запрос, состояние — в PostgreSQL/Valkey.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 4096
|
||||
hard-stop-after 30s
|
||||
|
||||
# Встроенный DNS Docker. Интервал повторного разрешения 1 с: остановленный контейнер исчезает
|
||||
# из ответа DNS, и его слот server-template уходит в MAINT ("No IP for server") за 1–2 с.
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 2
|
||||
timeout resolve 1s
|
||||
timeout retry 500ms
|
||||
hold valid 1s
|
||||
hold obsolete 1s
|
||||
hold nx 1s
|
||||
hold other 1s
|
||||
hold refused 1s
|
||||
hold timeout 1s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 1s # до живой реплики в той же сети — миллисекунды; 1 с = признак отказа
|
||||
timeout client 30s
|
||||
timeout server 15s # дольше statement_timeout БД (5 с) и Argon2id под нагрузкой
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 10s # меньше keep-alive uvicorn (30 с): idle-соединение закрывает HAProxy первым
|
||||
timeout queue 5s
|
||||
timeout check 1s
|
||||
# Повтор на ДРУГОЙ реплике (redispatch) при неудачном соединении; L7-повторы ограничены ниже.
|
||||
retries 2
|
||||
option redispatch 1
|
||||
# Ответы, созданные самим LB, — в формате ошибок контракта (без X-Instance-Id: их создал не API).
|
||||
http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Bad gateway\",\"request_id\":\"%[req.hdr(x-request-id)]\"}}" hdr X-Request-Id "%[req.hdr(x-request-id)]"
|
||||
http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API replica available\",\"request_id\":\"%[req.hdr(x-request-id)]\"}}" hdr X-Request-Id "%[req.hdr(x-request-id)]"
|
||||
http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Gateway timeout\",\"request_id\":\"%[req.hdr(x-request-id)]\"}}" hdr X-Request-Id "%[req.hdr(x-request-id)]"
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
# Сквозной request id: API принимает корректный входящий X-Request-Id, LB подставляет свой, если его нет.
|
||||
unique-id-format %[uuid()]
|
||||
http-request set-header X-Request-Id %[unique-id] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ }
|
||||
# Если /health/ready не проходит НИ У ОДНОЙ реплики (например, недоступна БД), не отвечаем
|
||||
# безликим 503 от LB, а отдаём запрос живым процессам: они вернут честный 503 контракта,
|
||||
# а /health/live продолжит работать.
|
||||
use_backend api_live if { nbsrv(api) eq 0 }
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready
|
||||
http-check expect status 200
|
||||
# Активная проверка раз в 1 с, вывод из ротации после 2 неудач подряд, возврат после 2 успехов.
|
||||
# Пассивно: 2 ошибки соединения подряд на реальном трафике — сразу DOWN, не дожидаясь проверки.
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 observe layer4 error-limit 2 on-error mark-down
|
||||
# L7-повтор на другой реплике безопасен только для GET и POST сообщения (защищён Idempotency-Key).
|
||||
# Для POST /users, /chats, /auth/sessions и DELETE остаются лишь повторы до отправки запроса (conn-failure).
|
||||
retry-on conn-failure empty-response response-timeout
|
||||
acl safe_method method GET HEAD
|
||||
acl is_post method POST
|
||||
acl messages_path path_reg ^/api/v1/chats/[^/]+/messages$
|
||||
http-request disable-l7-retry unless safe_method || is_post messages_path
|
||||
server-template api- 8 api:8080 resolvers docker init-addr none resolve-prefer ipv4
|
||||
|
||||
backend api_live
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/live
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fall 2 rise 2
|
||||
server-template live- 8 api:8080 resolvers docker init-addr none resolve-prefer ipv4
|
||||
|
||||
# Наблюдаемость LB: состояние backend-ов и метрики. Публикуется только на 127.0.0.1 хоста.
|
||||
frontend stats
|
||||
bind :8404
|
||||
http-request use-service prometheus-exporter if { path /metrics }
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 2s
|
||||
stats show-legends
|
||||
@@ -0,0 +1,59 @@
|
||||
-- 0001: базовая схема мессенджера lab2.
|
||||
-- Применяется раннером app/migrate.py внутри одной транзакции; версия пишется в schema_migrations.
|
||||
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
username text NOT NULL,
|
||||
display_name text NOT NULL,
|
||||
password_hash text NOT NULL, -- строка Argon2id ($argon2id$v=19$m=...,t=...,p=...$salt$hash)
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
CONSTRAINT users_username_key UNIQUE (username), -- защита от гонки регистраций: решает БД, а не приложение
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
-- Серверные сессии. Источник истины; Valkey хранит только копию с TTL.
|
||||
-- В БД лежит SHA-256 от токена, а не сам токен: утечка дампа не даёт войти.
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32),
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
revoked_at timestamptz,
|
||||
CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at)
|
||||
);
|
||||
CREATE INDEX sessions_user_id_idx ON sessions (user_id);
|
||||
CREATE INDEX sessions_expires_at_idx ON sessions (expires_at); -- для периодической очистки истёкших
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_by uuid NOT NULL REFERENCES users (id),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
-- Список чатов сортируется по (created_at, id).
|
||||
CREATE INDEX chats_created_at_id_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id),
|
||||
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (chat_id, user_id) -- один пользователь входит в чат один раз
|
||||
);
|
||||
-- "Чаты пользователя": поиск по user_id (PK начинается с chat_id и тут не помогает).
|
||||
CREATE INDEX chat_members_user_id_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
|
||||
chat_id uuid NOT NULL,
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
-- Отправитель обязан быть участником этого чата: гарантирует сама БД.
|
||||
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
|
||||
REFERENCES chat_members (chat_id, user_id)
|
||||
);
|
||||
-- Индекс под keyset-пагинацию истории:
|
||||
-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
|
||||
-- Равенство по chat_id + диапазон по (created_at, id) читаются одним проходом индекса без сортировки.
|
||||
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
|
||||
@@ -0,0 +1,21 @@
|
||||
-- 0002: общая идемпотентность POST /chats/{chat_id}/messages (lab3).
|
||||
-- Только добавляет таблицу (expand): версия lab2 продолжает работать на этой схеме, пока идёт замена реплик.
|
||||
|
||||
-- Область уникальности контракта: пользователь + метод + путь чата + ключ.
|
||||
-- Метод один (POST), путь однозначно задаётся chat_id, поэтому PK = (user_id, chat_id, key).
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash bytea NOT NULL CHECK (octet_length(request_hash) = 32), -- SHA-256 канонического JSON тела
|
||||
message_id uuid NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (user_id, chat_id, key),
|
||||
CONSTRAINT idempotency_sender_is_member FOREIGN KEY (chat_id, user_id)
|
||||
REFERENCES chat_members (chat_id, user_id),
|
||||
-- Ключ вставляется ДО сообщения (он и есть "замок" против гонки), поэтому FK проверяется при COMMIT.
|
||||
CONSTRAINT idempotency_message_fk FOREIGN KEY (message_id)
|
||||
REFERENCES messages (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED
|
||||
);
|
||||
-- Периодическая очистка ключей старше IDEMPOTENCY_TTL_SECONDS.
|
||||
CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,10 @@
|
||||
-- 0003: общий для всех реплик признак "кэшу сессий не доверять до ...".
|
||||
-- В lab2 окно недоверия после сбоя Valkey жило в памяти единственного процесса. С несколькими репликами
|
||||
-- это локальное состояние, от которого зависит корректность: реплика, не видевшая сбоя, отдала бы
|
||||
-- копию отозванной сессии. Теперь реплика, получившая ошибку Valkey, публикует окно здесь.
|
||||
-- Только добавление (expand): версии без этой таблицы её просто не используют.
|
||||
CREATE TABLE cache_state (
|
||||
id smallint PRIMARY KEY CHECK (id = 1), -- одна строка на весь стенд
|
||||
distrust_until timestamptz NOT NULL
|
||||
);
|
||||
INSERT INTO cache_state (id, distrust_until) VALUES (1, '-infinity');
|
||||
@@ -0,0 +1,50 @@
|
||||
-- 0004: вложения (lab4). Метаданные — здесь, в PostgreSQL; байты — в приватном S3-bucket.
|
||||
-- Только expand: версия lab3 продолжает работать на этой схеме (новые таблицы она не читает).
|
||||
|
||||
-- Одно вложение = один оригинал в S3. Файл заранее привязан к чату (chat_id), а не к сообщению:
|
||||
-- сначала upload, потом POST сообщения с attachment_ids.
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id),
|
||||
owner_id uuid NOT NULL,
|
||||
filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), -- только для отображения
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
width integer NOT NULL CHECK (width > 0),
|
||||
height integer NOT NULL CHECK (height > 0),
|
||||
-- Ключ объекта назначает сервер (attachments/<uuid>), имя файла клиента в нём не участвует.
|
||||
-- В публичный API ключ не попадает.
|
||||
object_key text NOT NULL UNIQUE,
|
||||
status text NOT NULL DEFAULT 'ready' CHECK (status IN ('ready')),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
-- Загрузивший обязан быть участником чата.
|
||||
CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id)
|
||||
REFERENCES chat_members (chat_id, user_id),
|
||||
-- Цель для составного FK из message_attachments: вложение и сообщение — из одного чата.
|
||||
CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id)
|
||||
);
|
||||
CREATE INDEX attachments_chat_id_idx ON attachments (chat_id);
|
||||
|
||||
-- Связь сообщение -> вложения (0..10, порядок как в запросе).
|
||||
-- chat_id входит в оба FK: БД сама не даст прикрепить вложение другого чата.
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position),
|
||||
CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id)
|
||||
REFERENCES attachments (id, chat_id)
|
||||
);
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
|
||||
-- Сообщение может состоять только из вложений: text="" допустим (непустота text ИЛИ вложений
|
||||
-- проверяется в API, т.к. вложения вставляются отдельной строкой в той же транзакции).
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_check;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000);
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id);
|
||||
ALTER TABLE message_attachments ADD CONSTRAINT message_attachments_message_fk
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id) ON DELETE CASCADE;
|
||||
@@ -0,0 +1,7 @@
|
||||
-- 0005_nul_safe_text: точное хранение строк с U+0000 (замечание к lab3, PR #99: "abc\u0000def" давал 500).
|
||||
-- PostgreSQL text не хранит 0x00. Если строка содержит U+0000, её точные UTF-8 байты пишутся в *_raw,
|
||||
-- а в основной столбец — та же строка с U+FFFD вместо U+0000 (см. app/pgtext.py).
|
||||
-- Только expand: столбцы nullable, версия без них продолжает работать на этой схеме.
|
||||
ALTER TABLE users ADD COLUMN display_name_raw bytea CHECK (octet_length(display_name_raw) <= 400);
|
||||
ALTER TABLE chats ADD COLUMN title_raw bytea CHECK (octet_length(title_raw) <= 400);
|
||||
ALTER TABLE messages ADD COLUMN text_raw bytea CHECK (octet_length(text_raw) <= 16000);
|
||||
@@ -0,0 +1,81 @@
|
||||
-- 0006: асинхронная обработка вложений (lab5): состояния, задания, варианты, transactional outbox, audit.
|
||||
-- Только expand: новые таблицы и столбцы; прежняя (lab4) версия видит ready-вложения как раньше.
|
||||
|
||||
-- Вложение: queued -> processing -> ready | failed (contracts/attachment-states.md).
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status_check;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status_check
|
||||
CHECK (status IN ('queued', 'processing', 'ready', 'failed'));
|
||||
ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued';
|
||||
-- Безопасный машинный код для failed (invalid_image, retry_exhausted, ...); у ready всегда NULL.
|
||||
ALTER TABLE attachments ADD COLUMN error_code text CHECK (error_code ~ '^[a-z_]{1,64}$');
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_error_only_failed
|
||||
CHECK ((status = 'failed') = (error_code IS NOT NULL));
|
||||
-- Устойчивый staging: приватный prefix staging/<id> того же bucket. object_key — конечный ключ оригинала
|
||||
-- (originals/<id>); он появляется в S3, когда worker публикует результат.
|
||||
ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE;
|
||||
ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1);
|
||||
|
||||
-- Варианты. Ключ эффекта контракта: (attachment_id, pipeline_version, variant) — повтор не создаёт второй набор.
|
||||
CREATE TABLE attachment_variants (
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL,
|
||||
name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')),
|
||||
size_bytes bigint NOT NULL CHECK (size_bytes > 0),
|
||||
width integer NOT NULL CHECK (width > 0),
|
||||
height integer NOT NULL CHECK (height > 0),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL UNIQUE, -- variants/<id>/v<pipeline>/<name>: детерминирован
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (attachment_id, pipeline_version, name)
|
||||
);
|
||||
|
||||
-- Одна логическая работа на (вложение, версия pipeline). job_id не меняется при redelivery и retry.
|
||||
CREATE TABLE attachment_jobs (
|
||||
job_id uuid PRIMARY KEY,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id),
|
||||
pipeline_version integer NOT NULL DEFAULT 1,
|
||||
status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'processing', 'done', 'failed')),
|
||||
attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0), -- число начатых попыток
|
||||
max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 50),
|
||||
next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
lease_until timestamptz, -- аренда processing; продлевается heartbeat-ом worker
|
||||
worker_id text, -- владелец аренды (fencing при фиксации результата)
|
||||
error_code text,
|
||||
last_error text, -- краткая причина без секретов и stack trace
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
updated_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
started_at timestamptz,
|
||||
finished_at timestamptz,
|
||||
UNIQUE (attachment_id, pipeline_version),
|
||||
CONSTRAINT jobs_lease_when_processing CHECK ((status = 'processing') = (lease_until IS NOT NULL))
|
||||
);
|
||||
-- Backpressure считает незавершённую работу; reaper ищет просроченные аренды.
|
||||
CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (status) WHERE status IN ('queued', 'processing');
|
||||
CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing';
|
||||
|
||||
-- Transactional outbox: намерение опубликовать job фиксируется в той же транзакции, что и вложение.
|
||||
-- Relay публикует с publisher confirm и только потом ставит published_at. available_at > now() —
|
||||
-- отложенная публикация (retry с backoff) без плагинов брокера.
|
||||
CREATE TABLE outbox (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id),
|
||||
payload jsonb NOT NULL, -- contracts/jobs.schema.json: только ID и служебные поля
|
||||
available_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
published_at timestamptz,
|
||||
publish_attempts integer NOT NULL DEFAULT 0,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE INDEX outbox_unpublished_idx ON outbox (available_at, id) WHERE published_at IS NULL;
|
||||
|
||||
-- Audit операторских действий (явный replay failed-задания).
|
||||
CREATE TABLE job_audit (
|
||||
id bigserial PRIMARY KEY,
|
||||
job_id uuid NOT NULL REFERENCES attachment_jobs (job_id),
|
||||
action text NOT NULL CHECK (action IN ('replay')),
|
||||
previous_status text NOT NULL,
|
||||
previous_error text,
|
||||
actor text NOT NULL,
|
||||
reason text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
@@ -0,0 +1,38 @@
|
||||
# Полный lock (pip freeze в python:3.12.12-slim-bookworm); устанавливается с --no-deps.
|
||||
# Прямые зависимости: fastapi, uvicorn, asyncpg, argon2-cffi, redis (клиент совместим с Valkey),
|
||||
# lab4: minio (S3-клиент, работает с RustFS), pillow (проверка и обработка PNG/JPEG),
|
||||
# lab5: aio-pika (RabbitMQ: publisher confirms, manual ack), jsonschema (проверка job по contracts/jobs.schema.json).
|
||||
aio-pika==9.5.7
|
||||
aiormq==6.9.4
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==23.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
asyncpg==0.30.0
|
||||
attrs==26.1.0
|
||||
certifi==2026.10.23
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
fastapi==0.115.12
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
jsonschema==4.25.1
|
||||
jsonschema-specifications==2025.9.1
|
||||
minio==7.2.18
|
||||
multidict==7.2.1
|
||||
pamqp==3.3.0
|
||||
pillow==11.3.0
|
||||
propcache==0.5.4
|
||||
pycparser==3.12
|
||||
pycryptodome==4.4.0
|
||||
pydantic==2.14.1
|
||||
pydantic_core==2.50.1
|
||||
redis==5.2.1
|
||||
referencing==0.37.0
|
||||
rpds-py==2026.9.1
|
||||
starlette==0.46.2
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.2
|
||||
yarl==1.25.1
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/bin/sh
|
||||
# Повторяемый сбор доказательств уровней 3–4 lab5 в evidence/ (запуск на хосте из корня, стенд уже поднят).
|
||||
# Токены/пароли/ключи S3 в вывод не попадают: тесты их не печатают, hook-вывод подавлен тестом.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
export MSYS_NO_PATHCONV=1 PYTHONUTF8=1
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
tester() { compose --profile tools run --rm tester sh -c "$1" 2>&1 | grep -v '^ Container'; }
|
||||
stamp() { echo "# $(date -u +%Y-%m-%dT%H:%M:%SZ), commit $(git rev-parse --short HEAD 2>/dev/null || echo uncommitted)"; }
|
||||
mkdir -p evidence
|
||||
: > evidence/stop-one.log
|
||||
|
||||
echo '== stand and S3 bootstrap'
|
||||
{ stamp
|
||||
compose ps -a --format 'table {{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}'
|
||||
echo; echo '# s3-init (idempotent bucket bootstrap + anonymous access check):'
|
||||
compose logs --no-log-prefix s3-init
|
||||
echo; echo '# repeated bootstrap run (must not touch existing bucket):'
|
||||
compose run --rm -T s3-init 2>&1 | grep -v '^ Container'
|
||||
echo; echo '# volumes:'
|
||||
docker volume ls --filter label=com.docker.compose.project=mtusi-lab5 --format '{{.Name}}'
|
||||
echo; echo '# RabbitMQ: очереди (quorum), consumers и prefetch:'
|
||||
compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable messages consumers
|
||||
compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required
|
||||
} | tee evidence/stand.txt
|
||||
|
||||
echo '== public checks'
|
||||
{ stamp
|
||||
tester 'pip install -q --disable-pip-version-check --root-user-action=ignore -r requirements-dev.txt && make check && make validate PYTHON=python3 && make test'
|
||||
} | tee evidence/public-tests.txt
|
||||
|
||||
echo '== failover: stop-one for each replica (restore between runs)'
|
||||
{ stamp
|
||||
for i in 1 2 3 4; do
|
||||
tester 'make test-failover ACTION_SCRIPT=scripts/stop-one.sh'
|
||||
tester 'sh scripts/restore.sh && curl -s -o /dev/null http://lb:8080/health/live'
|
||||
[ "$(cut -d' ' -f4 evidence/stop-one.log | sort -u | wc -l)" -ge 2 ] && break
|
||||
done
|
||||
echo; echo '# evidence/stop-one.log (what the hook really stopped):'; cat evidence/stop-one.log
|
||||
} | tee evidence/failover-hook.txt
|
||||
|
||||
echo '== persistence: restart of lb, api, worker, postgres, s3, rabbitmq, valkey with volumes'
|
||||
{ stamp
|
||||
tester 'make test-persistence ACTION_SCRIPT=scripts/restart.sh'
|
||||
echo; echo '# hook scope (StartedAt before/after of every process):'
|
||||
sh scripts/restart.sh
|
||||
tester 'until curl -fs http://lb:8080/health/ready >/dev/null; do sleep 0.5; done; echo ready'
|
||||
} | tee evidence/persistence.txt
|
||||
|
||||
echo '== student tests (lab5 + lab4/lab3/lab2 regression)'
|
||||
{ stamp; sh scripts/test-lab5.sh 2>&1 | grep -v '^ Container'; } | tee evidence/student-tests.txt
|
||||
Executable
+49
@@ -0,0 +1,49 @@
|
||||
#!/bin/sh
|
||||
# Создаёт локальные секреты в ./secrets (каталог в .gitignore). Повторный запуск ничего не перезаписывает.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Каталог закрыт для других пользователей хоста; сами файлы 0644, т.к. compose монтирует их
|
||||
# в контейнеры с правами хоста, а процессы там работают под непривилегированными uid.
|
||||
umask 022
|
||||
mkdir -p secrets
|
||||
chmod 700 secrets
|
||||
|
||||
random_hex() { od -An -tx1 -N24 /dev/urandom | tr -d ' \n'; }
|
||||
|
||||
if [ ! -f secrets/postgres_password ]; then
|
||||
pg=$(random_hex)
|
||||
printf '%s' "$pg" > secrets/postgres_password
|
||||
printf 'postgresql://app:%s@postgres:5432/messenger' "$pg" > secrets/database_url
|
||||
echo 'created secrets/postgres_password, secrets/database_url'
|
||||
fi
|
||||
if [ ! -f secrets/valkey.conf ]; then
|
||||
vk=$(random_hex)
|
||||
# Valkey — только кэш сессий: persistence выключена намеренно, источник истины — PostgreSQL.
|
||||
cat > secrets/valkey.conf <<EOF
|
||||
requirepass $vk
|
||||
save ""
|
||||
appendonly no
|
||||
maxmemory 64mb
|
||||
maxmemory-policy volatile-lru
|
||||
EOF
|
||||
printf 'redis://:%s@valkey:6379/0' "$vk" > secrets/cache_url
|
||||
echo 'created secrets/valkey.conf, secrets/cache_url'
|
||||
fi
|
||||
if [ ! -f secrets/s3_secret_access_key ]; then
|
||||
# Учётные данные S3 (RustFS): access key — не секретный идентификатор, но тоже вне Git.
|
||||
printf 'msg%s' "$(od -An -tx1 -N8 /dev/urandom | tr -d ' \n')" > secrets/s3_access_key_id
|
||||
random_hex > secrets/s3_secret_access_key
|
||||
echo 'created secrets/s3_access_key_id, secrets/s3_secret_access_key'
|
||||
fi
|
||||
if [ ! -f secrets/rabbitmq.conf ]; then
|
||||
# RabbitMQ: отдельный пользователь приложения вместо guest (guest разрешён только с localhost).
|
||||
rp=$(random_hex)
|
||||
printf 'default_user = messenger\ndefault_pass = %s\ndefault_vhost = /\nloopback_users = none\n' "$rp" > secrets/rabbitmq.conf
|
||||
printf 'amqp://messenger:%s@rabbitmq:5672/%%2F' "$rp" > secrets/broker_url
|
||||
echo 'created secrets/rabbitmq.conf, secrets/broker_url'
|
||||
fi
|
||||
if [ ! -f .env ]; then
|
||||
cp .env.example .env
|
||||
echo 'created .env from .env.example'
|
||||
fi
|
||||
echo 'secrets ready (values are not printed)'
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Перезапуск ВСЕГО стенда: lb, API-реплики, worker, PostgreSQL, S3 (RustFS), RabbitMQ, Valkey — с сохранением volumes.
|
||||
# Не выполняет down -v, FLUSHALL и повторный seed. Готовность API ожидает сам тест.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
|
||||
started_at() { docker inspect -f '{{.Name}} {{.State.StartedAt}}' $(compose ps -q lb api worker postgres s3 rabbitmq valkey); }
|
||||
|
||||
echo 'before:'; started_at
|
||||
compose stop lb api worker # сначала входная точка и клиенты хранилищ
|
||||
compose restart postgres s3 rabbitmq valkey # обычный restart: volumes pgdata, s3data, rabbitdata остаются
|
||||
compose start api worker lb
|
||||
echo 'after:'; started_at
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/sh
|
||||
# Возвращает остановленные API-реплики (после stop-one.sh) и ждёт их healthcheck.
|
||||
set -eu
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi-lab5}
|
||||
filter="--filter label=com.docker.compose.project=$project --filter label=com.docker.compose.service=api"
|
||||
stopped=$(docker ps -aq $filter --filter status=exited)
|
||||
[ -n "$stopped" ] && docker start $stopped >/dev/null
|
||||
for _ in $(seq 1 60); do
|
||||
unhealthy=$(docker ps -q $filter | xargs docker inspect -f '{{.State.Health.Status}}' | grep -vc healthy || true)
|
||||
[ "$unhealthy" = 0 ] && break
|
||||
sleep 1
|
||||
done
|
||||
docker ps $filter --format '{{.Names}} {{.Status}}'
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/bin/sh
|
||||
# Поочерёдная замена API-реплик без остановки сервиса (Compose не умеет rolling update сам).
|
||||
# 1. собрать новый образ (BUILD_ID попадает в label образа);
|
||||
# 2. применить миграции ОДИН раз (только expand-миграции: старая версия работает на новой схеме);
|
||||
# 3. поднять столько же новых реплик рядом со старыми (surge x2) и дождаться их healthcheck;
|
||||
# 4. по одной останавливать старые: docker stop -> SIGTERM -> drain -> graceful shutdown, затем удалить.
|
||||
# HAProxy сам находит новые реплики через DNS и исключает старые по /health/ready.
|
||||
# Запускать на хосте из корня репозитория (нужны bind-пути compose.yaml). NO_BUILD=1 — без пересборки.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
project=mtusi-lab5
|
||||
filter="--filter label=com.docker.compose.project=$project --filter label=com.docker.compose.service=api"
|
||||
|
||||
old=$(docker ps -q $filter)
|
||||
n=$(echo "$old" | wc -w)
|
||||
[ "$n" -ge 1 ] || { echo 'no running api replicas' >&2; exit 2; }
|
||||
build_id=${BUILD_ID:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date +%s)}
|
||||
|
||||
echo "$(date -u +%H:%M:%S) old replicas ($n): $(docker inspect -f '{{.Name}}={{index .Config.Labels "org.opencontainers.image.revision"}}' $old | tr -d / | tr '\n' ' ')"
|
||||
if [ "${NO_BUILD:-0}" != 1 ]; then
|
||||
compose build -q --build-arg BUILD_ID="$build_id" api
|
||||
fi
|
||||
compose run --rm -T migrate # однократно, под advisory lock
|
||||
compose up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api 2>&1 | grep -v '^ Container' || true
|
||||
new=$(docker ps -q $filter | grep -vxF "$old" || true)
|
||||
[ "$(echo "$new" | wc -w)" -eq "$n" ] || { echo 'new replicas did not start' >&2; exit 3; }
|
||||
echo "$(date -u +%H:%M:%S) new replicas healthy: $(docker inspect -f '{{.Name}}={{index .Config.Labels "org.opencontainers.image.revision"}}' $new | tr -d / | tr '\n' ' ')"
|
||||
sleep "${SETTLE_SECONDS:-3}" # LB: rise 2 x inter 1 s + DNS 1 с
|
||||
for c in $old; do
|
||||
name=$(docker inspect -f '{{.Name}}' "$c" | tr -d /)
|
||||
docker stop -t 20 "$c" >/dev/null
|
||||
docker rm "$c" >/dev/null
|
||||
echo "$(date -u +%H:%M:%S) replaced $name"
|
||||
done
|
||||
docker ps $filter --format '{{.Names}} {{.Status}} {{.Label "org.opencontainers.image.revision"}}'
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
# Останавливает ОДНУ API-реплику — ту, что обслужила тест, — и оставляет её остановленной.
|
||||
# TARGET_INSTANCE_ID = "<hostname контейнера>-<случайный суффикс>", hostname = короткий ID контейнера.
|
||||
# STOP_MODE=stop (по умолчанию): docker stop -> SIGTERM -> drain -> graceful shutdown.
|
||||
# STOP_MODE=kill: SIGKILL, внезапная смерть процесса без drain.
|
||||
# БД, LB и volumes не трогает. Восстановление — отдельно: sh scripts/restore.sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
project=${COMPOSE_PROJECT_NAME:-mtusi-lab5}
|
||||
cid=${TARGET_INSTANCE_ID%-*}
|
||||
case "$cid" in ''|*[!0-9a-f]*) echo 'TARGET_INSTANCE_ID has unexpected format' >&2; exit 2 ;; esac
|
||||
|
||||
# Защита от остановки чужого контейнера: только сервис api этого проекта.
|
||||
owner=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$cid")
|
||||
[ "$owner" = "$project/api" ] || { echo "container $cid is not $project/api" >&2; exit 3; }
|
||||
name=$(docker inspect -f '{{.Name}}' "$cid" | tr -d /)
|
||||
|
||||
started=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
if [ "${STOP_MODE:-stop}" = kill ]; then
|
||||
docker kill "$cid" >/dev/null
|
||||
else
|
||||
docker stop -t 20 "$cid" >/dev/null
|
||||
fi
|
||||
state=$(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}' "$cid")
|
||||
mkdir -p evidence
|
||||
printf '%s mode=%s instance=%s container=%s state=%s\n' \
|
||||
"$started" "${STOP_MODE:-stop}" "$TARGET_INSTANCE_ID" "$name" "$state" >> evidence/stop-one.log
|
||||
echo "stopped $name: $state"
|
||||
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/sh
|
||||
# Собственные проверки lab5 + регрессия lab4, lab3 и уровня 4 lab2. Запускать на хосте (Linux/macOS/Git Bash) из корня.
|
||||
# Поднимает api-ttl (SESSION_TTL_SECONDS=2), запускает student-tests в контейнере tester, гасит api-ttl.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
compose --profile test up -d --wait api-ttl
|
||||
status=0
|
||||
MSYS_NO_PATHCONV=1 compose --profile tools run --rm tester sh -c \
|
||||
'python -u student-tests/test_lab5.py; a=$?; python -u student-tests/test_lab4.py; b=$?; python -u student-tests/test_lab3.py; c=$?; python -u student-tests/test_lab2_regression.py; d=$?; exit $((a | b | c | d))' || status=$?
|
||||
compose --profile test stop api-ttl
|
||||
exit $status
|
||||
Executable
+40
@@ -0,0 +1,40 @@
|
||||
#!/bin/sh
|
||||
# Трек 1 «Очередь под перегрузкой»: матрица concurrency/prefetch x размер изображения x повтор,
|
||||
# затем backpressure (MAX_PENDING_JOBS) и восстановление. Запускать на хосте из корня, стенд поднят.
|
||||
# Результаты: evidence/track1/*.json + evidence/track1/summary.txt. Конфигурация стенда в конце возвращается.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
export MSYS_NO_PATHCONV=1
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
load() { compose --profile tools run --rm tester python -u student-tests/overload.py "$@" 2>&1 | grep -v '^ Container'; }
|
||||
REPEATS=${REPEATS:-2}
|
||||
# Поток выше пропускной способности worker при любой из конфигураций -> очередь растёт, видно lag/backpressure.
|
||||
SMALL="--size small --count 200 --rate 40"
|
||||
LARGE="--size large --count 60 --rate 8"
|
||||
mkdir -p evidence/track1
|
||||
: > evidence/track1/summary.txt
|
||||
|
||||
worker() { # пересоздать worker с заданными concurrency/prefetch (env shell важнее .env)
|
||||
WORKER_CONCURRENCY=$1 WORKER_PREFETCH=$2 compose up -d --no-deps --wait worker 2>&1 | grep -v '^ Container' || true
|
||||
}
|
||||
|
||||
for cfg in "1 1" "2 2" "4 4" "2 16"; do
|
||||
set -- $cfg
|
||||
worker "$1" "$2"
|
||||
for r in $(seq 1 "$REPEATS"); do
|
||||
load --label "c$1-p$2-small-r$r" $SMALL | tee -a evidence/track1/summary.txt
|
||||
load --label "c$1-p$2-large-r$r" $LARGE | tee -a evidence/track1/summary.txt
|
||||
done
|
||||
done
|
||||
|
||||
# Backpressure: принятая работа ограничена 20 незавершёнными заданиями; поток 15/с больших изображений
|
||||
# заведомо выше пропускной способности worker при c=2 -> часть upload получает 503 + Retry-After.
|
||||
worker 2 2
|
||||
# База для сравнения с восстановлением: та же нагрузка ниже пропускной способности, до перегрузки.
|
||||
load --label baseline-small --size small --count 60 --rate 5 | tee -a evidence/track1/summary.txt
|
||||
MAX_PENDING_JOBS=20 compose up -d --no-deps --wait api 2>&1 | grep -v '^ Container' || true
|
||||
load --label backpressure-large --size large --count 120 --rate 15 | tee -a evidence/track1/summary.txt
|
||||
# Восстановление: сразу после разгрузки — та же лёгкая нагрузка, что в базовом прогоне.
|
||||
load --label recovery-small --size small --count 60 --rate 5 | tee -a evidence/track1/summary.txt
|
||||
compose up -d --no-deps --wait api 2>&1 | grep -v '^ Container' || true
|
||||
echo 'track1 done' | tee -a evidence/track1/summary.txt
|
||||
Executable
+312
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Серии отказов под нагрузкой (уровень 4): RPS, ошибки, время обнаружения и восстановления.
|
||||
|
||||
Запускается на ХОСТЕ из корня репозитория (нужны docker CLI и, для rolling, compose с bind-путями):
|
||||
python student-tests/failover_series.py --scenario stop # graceful stop каждой реплики по очереди
|
||||
python student-tests/failover_series.py --scenario kill # SIGKILL каждой реплики по очереди
|
||||
python student-tests/failover_series.py --scenario rolling # scripts/rolling-update.sh под нагрузкой
|
||||
|
||||
Нагрузка closed-loop: W писателей (POST сообщения; неопределённый результат повторяется с ТЕМ ЖЕ ключом,
|
||||
как в tests/resilience.py) и R читателей (GET истории). Параллельно раз в 200 мс опрашивается
|
||||
CSV-статистика HAProxy: когда слот целевой реплики перестал быть UP — это время обнаружения отказа LB.
|
||||
После серии проверяется, что все подтверждённые сообщения на месте и нет дублей.
|
||||
Токены остаются в памяти процесса; в отчёт попадают только агрегаты.
|
||||
"""
|
||||
import argparse
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
PROJECT = 'mtusi-lab5'
|
||||
EDGE_NETWORK = PROJECT + '_edge'
|
||||
API_FILTER = ['--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', 'label=com.docker.compose.service=api']
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
return subprocess.run(['docker', *args], check=True, text=True, capture_output=True).stdout.strip()
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
names = docker('ps', *API_FILTER, '--format', '{{.Names}}').split()
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def container_info(name: str) -> dict:
|
||||
fmt = ('{{.Config.Hostname}}|{{index .Config.Labels "org.opencontainers.image.revision"}}|'
|
||||
'{{(index .NetworkSettings.Networks "%s").IPAddress}}') % EDGE_NETWORK
|
||||
hostname, revision, ip = docker('inspect', '-f', fmt, name).split('|')
|
||||
return {'name': name, 'hostname': hostname, 'revision': revision, 'ip': ip}
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float):
|
||||
values = sorted(values)
|
||||
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) if values else None
|
||||
|
||||
|
||||
class LbWatcher(threading.Thread):
|
||||
"""Опрос /stats;csv: состояние слотов backend `api` и счётчики retry/redispatch."""
|
||||
|
||||
def __init__(self, stats_url: str, t0: float):
|
||||
super().__init__(daemon=True)
|
||||
self.url = stats_url.rstrip('/') + ';csv'
|
||||
self.t0 = t0
|
||||
self.stop_flag = threading.Event()
|
||||
self.samples = [] # (t, {addr: status})
|
||||
self.first = self.last = None
|
||||
|
||||
def snapshot(self):
|
||||
with urllib.request.urlopen(self.url, timeout=1) as r:
|
||||
rows = list(csv.DictReader(io.StringIO(r.read().decode().lstrip('# '))))
|
||||
servers = {row['addr']: row['status'] for row in rows
|
||||
if row['pxname'] == 'api' and row['svname'] not in ('FRONTEND', 'BACKEND') and row.get('addr')}
|
||||
backend = next(row for row in rows if row['pxname'] == 'api' and row['svname'] == 'BACKEND')
|
||||
return servers, {'retries': int(backend['wretr'] or 0), 'redispatches': int(backend['wredis'] or 0)}
|
||||
|
||||
def run(self):
|
||||
while not self.stop_flag.is_set():
|
||||
try:
|
||||
servers, counters = self.snapshot()
|
||||
self.samples.append((time.monotonic() - self.t0, servers))
|
||||
self.first = self.first or counters
|
||||
self.last = counters
|
||||
except Exception: # LB-статистика — вспомогательная, не влияет на результат серии
|
||||
pass
|
||||
time.sleep(0.2)
|
||||
|
||||
def first_not_up(self, ip: str, after: float):
|
||||
"""Первый момент после `after`, когда слот с этим IP не UP (DOWN/MAINT/исчез из DNS)."""
|
||||
addr = ip + ':8080'
|
||||
for t, servers in self.samples:
|
||||
if t >= after and not servers.get(addr, 'MISSING').startswith('UP'):
|
||||
return round(t - after, 3), servers.get(addr, 'removed from DNS')
|
||||
return None, None
|
||||
|
||||
|
||||
def run_probe(base_url: str, stats_url: str, action, targets: list[dict], duration: float,
|
||||
action_at: float, writers: int, readers: int) -> dict:
|
||||
c = Client(base_url)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
events = [] # (t, kind, ok, instance, latency)
|
||||
confirmed = [] # подтверждённые сообщения
|
||||
lock = threading.Lock()
|
||||
retries_with_same_key = [0]
|
||||
t0 = time.monotonic()
|
||||
deadline = t0 + duration
|
||||
watcher = LbWatcher(stats_url, t0)
|
||||
watcher.start()
|
||||
|
||||
def record(kind, ok, instance, latency):
|
||||
with lock:
|
||||
events.append((time.monotonic() - t0, kind, ok, instance, latency))
|
||||
|
||||
def writer(i):
|
||||
local = Client(base_url, timeout=3)
|
||||
key, n = uuid.uuid4().hex, 0
|
||||
text = f'probe-{i}-{n}'
|
||||
while time.monotonic() < deadline:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = local.message(alice, chat, text, key=key)
|
||||
record('write', True, r['headers'].get('X-Instance-Id', ''), time.monotonic() - start)
|
||||
with lock:
|
||||
confirmed.append(r['json'])
|
||||
n += 1
|
||||
key, text = uuid.uuid4().hex, f'probe-{i}-{n}'
|
||||
except Failure:
|
||||
record('write', False, '', time.monotonic() - start)
|
||||
with lock:
|
||||
retries_with_same_key[0] += 1
|
||||
time.sleep(0.05) # тот же key и тот же body при следующей попытке
|
||||
|
||||
def reader(_i):
|
||||
local = Client(base_url, timeout=3)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=20"
|
||||
while time.monotonic() < deadline:
|
||||
start = time.monotonic()
|
||||
try:
|
||||
r = local.call('GET', path, actor=bob, schema='MessagePage')
|
||||
record('read', True, r['headers'].get('X-Instance-Id', ''), time.monotonic() - start)
|
||||
except Failure:
|
||||
record('read', False, '', time.monotonic() - start)
|
||||
time.sleep(0.05)
|
||||
|
||||
threads = [threading.Thread(target=writer, args=(i,)) for i in range(writers)]
|
||||
threads += [threading.Thread(target=reader, args=(i,)) for i in range(readers)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
time.sleep(max(0.0, t0 + action_at - time.monotonic()))
|
||||
a_start = time.monotonic() - t0
|
||||
action_output = action()
|
||||
a_end = time.monotonic() - t0
|
||||
for t in threads:
|
||||
t.join()
|
||||
watcher.stop_flag.set()
|
||||
watcher.join()
|
||||
|
||||
# Сохранность: тот же токен, полная история через LB, без дублей.
|
||||
history, cursor = [], None
|
||||
while True:
|
||||
path = f"/api/v1/chats/{chat['id']}/messages?limit=100" + (f'&cursor={urllib.parse.quote(cursor, safe="")}' if cursor else '')
|
||||
page = c.call('GET', path, actor=bob, schema='MessagePage')['json']
|
||||
history += page['items']
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
break
|
||||
ids = {m['id'] for m in history}
|
||||
texts = [m['text'] for m in history]
|
||||
lost = sum(1 for m in confirmed if m['id'] not in ids)
|
||||
check(len(ids) == len(history), 'duplicated message id in history')
|
||||
check(len(texts) == len(set(texts)), 'retry with same key produced a duplicate message')
|
||||
check(lost == 0, f'{lost} confirmed messages lost')
|
||||
|
||||
def window(lo, hi):
|
||||
sel = [e for e in events if lo <= e[0] < hi]
|
||||
ok = [e for e in sel if e[2]]
|
||||
span = max(hi - lo, 1e-9)
|
||||
return {'seconds': round(span, 2), 'ok_rps': round(len(ok) / span, 1),
|
||||
'write_rps': round(sum(1 for e in ok if e[1] == 'write') / span, 1),
|
||||
'read_rps': round(sum(1 for e in ok if e[1] == 'read') / span, 1),
|
||||
'errors': len(sel) - len(ok),
|
||||
'p50_ms': percentile([e[4] for e in ok], .50), 'p95_ms': percentile([e[4] for e in ok], .95),
|
||||
'p99_ms': percentile([e[4] for e in ok], .99), 'max_ms': percentile([e[4] for e in ok], 1.0)}
|
||||
|
||||
errors = [e for e in events if not e[2]]
|
||||
target_prefixes = tuple(t['hostname'] + '-' for t in targets)
|
||||
served_by_target = [e[0] for e in events if e[2] and e[3].startswith(target_prefixes)]
|
||||
per_second = []
|
||||
for s in range(int(duration)):
|
||||
sel = [e for e in events if s <= e[0] < s + 1]
|
||||
per_second.append({'t': s, 'ok': sum(1 for e in sel if e[2]), 'err': sum(1 for e in sel if not e[2]),
|
||||
'instances': len({e[3] for e in sel if e[2]})})
|
||||
detection = {}
|
||||
for t in targets:
|
||||
seconds, state = watcher.first_not_up(t['ip'], a_start)
|
||||
detection[t['name']] = {'lb_marked_not_up_after_s': seconds, 'lb_state': state}
|
||||
lb_counters = None
|
||||
if watcher.first and watcher.last:
|
||||
lb_counters = {k: watcher.last[k] - watcher.first[k] for k in watcher.last}
|
||||
return {
|
||||
'targets': [{k: t[k] for k in ('name', 'hostname', 'revision')} for t in targets],
|
||||
'action_started_s': round(a_start, 3), 'action_duration_s': round(a_end - a_start, 3),
|
||||
'action_output': action_output,
|
||||
'before': window(1, a_start), 'during_action_plus_5s': window(a_start, a_end + 5),
|
||||
'after': window(a_end + 5, duration),
|
||||
'errors_total': len(errors),
|
||||
'write_attempts_retried_same_key': retries_with_same_key[0],
|
||||
'error_window_s': ([round(errors[0][0] - a_start, 3), round(errors[-1][0] - a_start, 3)] if errors else None),
|
||||
'recovery_s': round(errors[-1][0] - a_start, 3) if errors else 0.0,
|
||||
'target_last_served_after_action_s': (round(max(served_by_target) - a_start, 3)
|
||||
if served_by_target and max(served_by_target) >= a_start else None),
|
||||
'lb_detection': detection,
|
||||
'lb_retries_redispatches': lb_counters,
|
||||
'instances_after': sorted({e[3] for e in events if e[2] and e[0] > a_end + 5}),
|
||||
'confirmed_writes': len(confirmed), 'history_messages': len(history),
|
||||
'duplicates': 0, 'lost_confirmed': lost,
|
||||
'timeline_per_second': per_second,
|
||||
}
|
||||
|
||||
|
||||
def wait_lb_up(stats_url: str, ip: str, seconds: float = 60) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
url = stats_url.rstrip('/') + ';csv'
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=1) as r:
|
||||
rows = csv.DictReader(io.StringIO(r.read().decode().lstrip('# ')))
|
||||
if any(row['pxname'] == 'api' and row.get('addr') == ip + ':8080' and row['status'].startswith('UP')
|
||||
for row in rows):
|
||||
return
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise Failure('replica did not return to LB rotation')
|
||||
|
||||
|
||||
def restore(stats_url: str) -> None:
|
||||
subprocess.run(['sh', str(ROOT / 'scripts/restore.sh')], check=True, capture_output=True)
|
||||
for name in api_containers():
|
||||
wait_lb_up(stats_url, container_info(name)['ip'])
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__.split('\n')[0])
|
||||
p.add_argument('--scenario', choices=['stop', 'kill', 'rolling'], required=True)
|
||||
p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080'))
|
||||
p.add_argument('--stats-url', default=os.environ.get('LB_STATS_URL', 'http://localhost:8404/stats'))
|
||||
p.add_argument('--duration', type=float, default=None)
|
||||
p.add_argument('--action-at', type=float, default=8)
|
||||
p.add_argument('--writers', type=int, default=4)
|
||||
p.add_argument('--readers', type=int, default=2)
|
||||
p.add_argument('--output', default=None)
|
||||
args = p.parse_args()
|
||||
duration = args.duration or (45 if args.scenario == 'rolling' else 25)
|
||||
|
||||
runs = []
|
||||
if args.scenario == 'rolling':
|
||||
targets = [container_info(n) for n in api_containers()]
|
||||
|
||||
def action():
|
||||
r = subprocess.run(['sh', str(ROOT / 'scripts/rolling-update.sh')], text=True, capture_output=True)
|
||||
check(r.returncode == 0, 'rolling-update.sh failed: ' + r.stderr[-500:])
|
||||
return r.stdout.strip().splitlines()
|
||||
result = run_probe(args.base_url, args.stats_url, action, targets, duration, args.action_at,
|
||||
args.writers, args.readers)
|
||||
result['replicas_after'] = [container_info(n) for n in api_containers()]
|
||||
runs.append(result)
|
||||
else:
|
||||
for name in api_containers(): # по очереди каждую реплику, восстанавливая предыдущую
|
||||
restore(args.stats_url)
|
||||
target = container_info(name)
|
||||
|
||||
def action(name=name):
|
||||
started = time.monotonic()
|
||||
if args.scenario == 'kill':
|
||||
docker('kill', name)
|
||||
else:
|
||||
docker('stop', '-t', '20', name)
|
||||
state = docker('inspect', '-f', '{{.State.Status}} exit={{.State.ExitCode}}', name)
|
||||
return [f'{args.scenario} {name}: {state} in {time.monotonic() - started:.2f}s']
|
||||
result = run_probe(args.base_url, args.stats_url, action, [target], duration, args.action_at,
|
||||
args.writers, args.readers)
|
||||
runs.append(result)
|
||||
print(f"{args.scenario} {name}: errors={result['errors_total']} recovery={result['recovery_s']}s "
|
||||
f"lb_detection={result['lb_detection'][name]['lb_marked_not_up_after_s']}s "
|
||||
f"rps before/after={result['before']['ok_rps']}/{result['after']['ok_rps']}", flush=True)
|
||||
restore(args.stats_url)
|
||||
|
||||
report = {
|
||||
'scenario': args.scenario, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()),
|
||||
'profile': {'duration_s': duration, 'action_at_s': args.action_at, 'writers': args.writers,
|
||||
'readers': args.readers, 'client_timeout_s': 3,
|
||||
'workload': 'closed-loop: POST message (retry same key on failure) + GET history limit=20'},
|
||||
'runs': runs,
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
Path(args.output).parent.mkdir(parents=True, exist_ok=True)
|
||||
Path(args.output).write_text(text + '\n', encoding='utf-8')
|
||||
summary = [{k: r[k] for k in ('targets', 'errors_total', 'recovery_s', 'lb_detection', 'action_duration_s',
|
||||
'target_last_served_after_action_s', 'lb_retries_redispatches')}
|
||||
| {'rps_before': r['before']['ok_rps'], 'rps_after': r['after']['ok_rps'],
|
||||
'p99_during_ms': r['during_action_plus_5s']['p99_ms']} for r in runs]
|
||||
print(json.dumps(summary, ensure_ascii=False, indent=1))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
Executable
+207
@@ -0,0 +1,207 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Трек 1: нагрузка на очередь обработки изображений (запуск внутри сервиса tester; драйвер — scripts/track1.sh).
|
||||
|
||||
Методика:
|
||||
- OPEN-LOOP поступление: upload отправляются по расписанию (rate/s) независимо от скорости ответа, поэтому
|
||||
нет coordinated omission генератора; задержки отправки (если пул потоков не успел) фиксируются отдельно;
|
||||
- время до ready считается по серверным меткам PostgreSQL: attachments.created_at (момент COMMIT приёма) ->
|
||||
updated_at перехода в ready; это то, что видит клиент, опрашивая метаданные (без шума опроса);
|
||||
- queue lag: каждые 0.5 с — число незавершённых заданий (queued + processing) из attachment_jobs и
|
||||
глубина очереди RabbitMQ (messages_ready + unacked) — раз в 2 с;
|
||||
- память worker: docker stats контейнера worker (процесс + дочерние процессы обработки), раз в ~1 с;
|
||||
- throughput: число ready / (последний ready - первый принятый upload).
|
||||
Результат — JSON в evidence/track1/<label>.json (без токенов и ID пользователей).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
BASE = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
PROJECT = 'mtusi-lab5'
|
||||
IMAGES = {
|
||||
'small': (ROOT / 'tests/fixtures/sample.png', 'image/png'), # 320x200, 112 KB
|
||||
'large': (ROOT / 'student-tests/.generated/large.jpg', 'image/jpeg'), # 4000x3000, ~3 MB
|
||||
}
|
||||
|
||||
|
||||
def sh(*args: str) -> str:
|
||||
return subprocess.run(args, check=True, text=True, capture_output=True).stdout.strip()
|
||||
|
||||
|
||||
def sql(query: str) -> str:
|
||||
return sh('docker', 'compose', '--env-file', str(ROOT / '.env'), 'exec', '-T', 'postgres',
|
||||
'psql', '-U', 'app', '-d', 'messenger', '-tAc', query)
|
||||
|
||||
|
||||
def worker_ids() -> list[str]:
|
||||
return sh('docker', 'ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', 'label=com.docker.compose.service=worker').split()
|
||||
|
||||
|
||||
def mem_mib(value: str) -> float:
|
||||
number = float(''.join(ch for ch in value if ch.isdigit() or ch == '.'))
|
||||
unit = value.lstrip('0123456789.').strip().lower()
|
||||
return number * {'b': 1 / 2**20, 'kib': 1 / 1024, 'mib': 1, 'gib': 1024, 'kb': 1 / 1024, 'mb': 1, 'gb': 1024}.get(unit, 1)
|
||||
|
||||
|
||||
def pct(values: list[float], p: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
k = max(0, min(len(ordered) - 1, round(p / 100 * len(ordered) + 0.5) - 1))
|
||||
return round(ordered[k], 3)
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
def __init__(self, since: str):
|
||||
super().__init__(daemon=True)
|
||||
self.since = since
|
||||
self.stop = threading.Event()
|
||||
self.pending: list[tuple[float, int]] = []
|
||||
self.broker: list[tuple[float, int]] = []
|
||||
self.memory: list[tuple[float, float]] = []
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def run(self):
|
||||
tick = 0
|
||||
while not self.stop.is_set():
|
||||
t = round(time.monotonic() - self.t0, 2)
|
||||
try:
|
||||
n = int(sql("SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing') "
|
||||
f"AND created_at >= '{self.since}'"))
|
||||
self.pending.append((t, n))
|
||||
if tick % 2 == 0:
|
||||
stats = sh('docker', 'stats', '--no-stream', '--format', '{{.MemUsage}}', *worker_ids())
|
||||
self.memory.append((t, round(sum(mem_mib(line.split('/')[0]) for line in stats.splitlines()), 1)))
|
||||
if tick % 4 == 0:
|
||||
out = sh('docker', 'compose', '--env-file', str(ROOT / '.env'), 'exec', '-T', 'rabbitmq',
|
||||
'rabbitmqctl', '-q', 'list_queues', 'name', 'messages')
|
||||
for line in out.splitlines():
|
||||
parts = line.split()
|
||||
if parts and parts[0] == 'attachment.process.v1':
|
||||
self.broker.append((t, int(parts[1])))
|
||||
except (subprocess.CalledProcessError, ValueError):
|
||||
pass
|
||||
tick += 1
|
||||
self.stop.wait(0.5)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument('--label', required=True)
|
||||
p.add_argument('--size', choices=IMAGES, required=True)
|
||||
p.add_argument('--count', type=int, required=True)
|
||||
p.add_argument('--rate', type=float, required=True, help='uploads per second (open loop)')
|
||||
p.add_argument('--timeout', type=float, default=600)
|
||||
args = p.parse_args()
|
||||
|
||||
path, ctype = IMAGES[args.size]
|
||||
data = path.read_bytes()
|
||||
c = Client(BASE, timeout=60)
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
url = f"{BASE}/api/v1/chats/{chat['id']}/attachments?filename=load.{'png' if ctype == 'image/png' else 'jpg'}"
|
||||
config = {k: sh('docker', 'inspect', '-f', '{{range .Config.Env}}{{println .}}{{end}}', worker_ids()[0])
|
||||
for k in ['env']}['env']
|
||||
env = dict(line.split('=', 1) for line in config.splitlines() if '=' in line)
|
||||
api = sh('docker', 'ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', 'label=com.docker.compose.service=api').split()[0]
|
||||
api_env = dict(line.split('=', 1) for line in sh('docker', 'inspect', '-f',
|
||||
'{{range .Config.Env}}{{println .}}{{end}}', api).splitlines() if '=' in line)
|
||||
since = sql('SELECT clock_timestamp()')
|
||||
|
||||
results, lock = [], threading.Lock()
|
||||
|
||||
def send(i: int, planned: float) -> None:
|
||||
late = time.monotonic() - planned
|
||||
req = urllib.request.Request(url, data=data, method='POST', headers={
|
||||
'Content-Type': ctype, 'Authorization': 'Bearer ' + alice['token']})
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
status, body = r.status, json.loads(r.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
status, body = e.code, None
|
||||
e.read()
|
||||
except OSError:
|
||||
status, body = 'error', None
|
||||
with lock:
|
||||
results.append({'status': status, 'id': body['id'] if body else None,
|
||||
'upload_s': time.monotonic() - started, 'late_s': late})
|
||||
|
||||
sampler = Sampler(since)
|
||||
sampler.start()
|
||||
t0 = time.monotonic()
|
||||
with ThreadPoolExecutor(max_workers=48) as pool:
|
||||
for i in range(args.count):
|
||||
planned = t0 + i / args.rate
|
||||
delay = planned - time.monotonic()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
pool.submit(send, i, planned)
|
||||
submit_done = time.monotonic() - t0
|
||||
accepted = [r['id'] for r in results if r['status'] == 202]
|
||||
deadline = time.monotonic() + args.timeout
|
||||
while time.monotonic() < deadline:
|
||||
left = int(sql("SELECT count(*) FROM attachment_jobs WHERE status IN ('queued','processing') "
|
||||
f"AND created_at >= '{since}'"))
|
||||
if left == 0:
|
||||
break
|
||||
time.sleep(1)
|
||||
drained_at = time.monotonic() - t0
|
||||
sampler.stop.set()
|
||||
sampler.join(5)
|
||||
|
||||
rows = sql("SELECT a.status, extract(epoch FROM a.updated_at - a.created_at), "
|
||||
"extract(epoch FROM a.created_at), extract(epoch FROM a.updated_at) FROM attachments a "
|
||||
f"WHERE a.chat_id = '{chat['id']}'").splitlines()
|
||||
ready = [list(map(float, r.split('|')[1:])) for r in rows if r.startswith('ready|')]
|
||||
lat = [x[0] for x in ready]
|
||||
statuses = {}
|
||||
for r in results:
|
||||
statuses[str(r['status'])] = statuses.get(str(r['status']), 0) + 1
|
||||
span = (max(x[2] for x in ready) - min(x[1] for x in ready)) if ready else 0
|
||||
summary = {
|
||||
'label': args.label, 'size': args.size, 'bytes': len(data), 'count': args.count, 'rate_per_s': args.rate,
|
||||
'worker': {k: env.get(k) for k in ('WORKER_CONCURRENCY', 'WORKER_PREFETCH', 'WORKER_TASK_MEMORY_MB')},
|
||||
'worker_replicas': len(worker_ids()), 'max_pending_jobs': api_env.get('MAX_PENDING_JOBS'),
|
||||
'upload_status': statuses, 'accepted': len(accepted),
|
||||
'ready': len(ready), 'failed': sum(1 for r in rows if r.startswith('failed|')),
|
||||
'lost_accepted': len(accepted) - len(rows),
|
||||
'submit_seconds': round(submit_done, 2), 'drain_seconds': round(drained_at, 2),
|
||||
'throughput_ready_per_s': round(len(ready) / span, 3) if span else None,
|
||||
'time_to_ready_s': {'p50': pct(lat, 50), 'p95': pct(lat, 95), 'p99': pct(lat, 99),
|
||||
'max': round(max(lat), 3) if lat else None,
|
||||
'mean': round(statistics.mean(lat), 3) if lat else None},
|
||||
'upload_latency_s': {'p50': pct([r['upload_s'] for r in results], 50),
|
||||
'p95': pct([r['upload_s'] for r in results], 95)},
|
||||
'generator_max_late_s': round(max(r['late_s'] for r in results), 3),
|
||||
'queue_lag': {'max_pending_jobs': max((n for _, n in sampler.pending), default=0),
|
||||
'max_broker_messages': max((n for _, n in sampler.broker), default=0)},
|
||||
'worker_memory_mib': {'peak': max((m for _, m in sampler.memory), default=None),
|
||||
'idle_first': sampler.memory[0][1] if sampler.memory else None},
|
||||
'timeline': {'pending': sampler.pending, 'broker': sampler.broker, 'memory': sampler.memory},
|
||||
}
|
||||
out = ROOT / 'evidence/track1' / f'{args.label}.json'
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(summary, ensure_ascii=False, indent=1), encoding='utf-8')
|
||||
short = {k: summary[k] for k in ('label', 'upload_status', 'ready', 'failed', 'lost_accepted',
|
||||
'throughput_ready_per_s', 'time_to_ready_s', 'queue_lag', 'worker_memory_mib',
|
||||
'drain_seconds')}
|
||||
print(json.dumps(short, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Один вход (одна сессия) -> 10 запросов через LB: какие реплики обслужили. Токен не печатается."""
|
||||
import os
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
c = Client(os.environ.get('BASE_URL', 'http://lb:8080'))
|
||||
alice = c.user('instances')
|
||||
served = [c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id'] for _ in range(10)]
|
||||
print('X-Instance-Id sequence:', ' '.join(served))
|
||||
print('distinct replicas for one session without re-login:', dict(Counter(served)))
|
||||
Executable
+256
@@ -0,0 +1,256 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Регрессия: собственные проверки уровня 4 из lab2, перенесённые на стенд lab3 (через LB).
|
||||
Запуск: sh scripts/test-lab3.sh (внутри сервиса tester).
|
||||
|
||||
Требуют запущенный стенд и доступ к docker (останавливают/ставят на паузу valkey и postgres,
|
||||
затем возвращают их). Используют только синтетические данные.
|
||||
"""
|
||||
import base64
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
TTL_BASE_URL = os.environ.get('TTL_BASE_URL', 'http://api-ttl:8080')
|
||||
VALKEY_CLI = 'valkey-cli --no-auth-warning -a "$(sed -n "s/^requirepass //p" /run/secrets/valkey_conf)" '
|
||||
|
||||
|
||||
def compose(*args: str, capture: bool = False) -> str:
|
||||
result = subprocess.run(['docker', 'compose', '--env-file', str(ROOT / '.env'), *args],
|
||||
cwd=ROOT, check=True, text=True,
|
||||
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
return (result.stdout or '').strip()
|
||||
|
||||
|
||||
def sql(query: str) -> str:
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'app', '-d', 'messenger', '-tAc', query, capture=True)
|
||||
|
||||
|
||||
def valkey(command: str) -> str:
|
||||
return compose('exec', '-T', 'valkey', 'sh', '-c', VALKEY_CLI + command, capture=True)
|
||||
|
||||
|
||||
def basic(username: str, password: str) -> dict:
|
||||
return {'Authorization': 'Basic ' + base64.b64encode(f'{username}:{password}'.encode()).decode()}
|
||||
|
||||
|
||||
def wait_ready(c: Client, seconds: float = 60) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'API did not become ready')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def wait_healthy(service: str, seconds: float = 60) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
container = compose('ps', '-q', service, capture=True)
|
||||
while subprocess.run(['docker', 'inspect', '-f', '{{.State.Health.Status}}', container],
|
||||
text=True, capture_output=True).stdout.strip() != 'healthy':
|
||||
check(time.monotonic() < deadline, service + ' did not become healthy')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def cache_key(actor: dict) -> str:
|
||||
return 'session:' + hashlib.sha256(actor['token'].encode()).hexdigest()
|
||||
|
||||
|
||||
def wait_cache_used(c: Client, actor: dict, seconds: float = 75) -> None:
|
||||
"""После недавнего сбоя Valkey API законно не доверяет кэшу SESSION_CACHE_TTL_SECONDS — ждём конца окна."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
c.call('GET', '/api/v1/users/me', actor=actor, schema='User')
|
||||
if valkey('EXISTS ' + cache_key(actor)) == '1':
|
||||
return
|
||||
check(time.monotonic() < deadline, 'session was never cached in Valkey')
|
||||
time.sleep(2)
|
||||
|
||||
|
||||
def bearer(token: str) -> dict:
|
||||
return {'Authorization': 'Bearer ' + token}
|
||||
|
||||
# ---------------------------------------------------------------- тесты
|
||||
|
||||
|
||||
def test_wrong_password(c: Client):
|
||||
alice = c.user('wrongpw')
|
||||
username = alice['registration']['username']
|
||||
wrong = c.call('POST', '/api/v1/auth/sessions', 401, headers=basic(username, 'definitely-wrong-password'))
|
||||
unknown = c.call('POST', '/api/v1/auth/sessions', 401, headers=basic('no_such_' + uuid.uuid4().hex[:8], 'whatever-password-123'))
|
||||
# Одинаковый ответ: по нему нельзя узнать, существует ли username.
|
||||
check(wrong['json']['error']['message'] == unknown['json']['error']['message'], 'different 401 for unknown user')
|
||||
check(wrong['headers']['WWW-Authenticate'] == 'Basic realm="messenger", charset="UTF-8"', 'wrong Basic challenge')
|
||||
c.call('POST', '/api/v1/auth/sessions', 401, headers={'Authorization': 'Basic !!!not-base64'})
|
||||
c.call('POST', '/api/v1/auth/sessions', 401, headers=bearer(alice['token'])) # Bearer на входе не принимается
|
||||
|
||||
|
||||
def test_logout(c: Client):
|
||||
alice = c.user('logout')
|
||||
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
r = c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
||||
check(r['headers']['WWW-Authenticate'] == 'Bearer realm="messenger"', 'wrong Bearer challenge')
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=alice)
|
||||
revoked = sql("SELECT count(*) FROM sessions s JOIN users u ON u.id = s.user_id "
|
||||
f"WHERE u.username = '{alice['registration']['username']}' AND s.revoked_at IS NOT NULL")
|
||||
check(revoked == '1', 'revocation not recorded in PostgreSQL')
|
||||
|
||||
|
||||
def test_short_ttl(_c: Client):
|
||||
# Отдельная реплика api-ttl с SESSION_TTL_SECONDS=2, общая БД и кэш.
|
||||
c = Client(TTL_BASE_URL)
|
||||
wait_ready(c)
|
||||
alice = c.user('ttl')
|
||||
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||||
time.sleep(3)
|
||||
c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
||||
|
||||
|
||||
def test_username_race(c: Client):
|
||||
username = 'race_' + uuid.uuid4().hex[:12]
|
||||
body = {'username': username, 'display_name': 'Гонка', 'password': 'Race-password-' + uuid.uuid4().hex}
|
||||
statuses, barrier = [], threading.Barrier(16)
|
||||
|
||||
def register():
|
||||
barrier.wait()
|
||||
statuses.append(Client(BASE_URL, timeout=30).call('POST', '/api/v1/users', [201, 409], body=body)['status'])
|
||||
|
||||
threads = [threading.Thread(target=register) for _ in range(16)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
check(sorted(statuses) == [201] + [409] * 15, f'race result {sorted(statuses)}')
|
||||
check(sql(f"SELECT count(*) FROM users WHERE username = '{username}'") == '1', 'duplicate username row')
|
||||
|
||||
|
||||
def test_chat_rollback(c: Client):
|
||||
alice = c.user('rollback')
|
||||
before = sql('SELECT (SELECT count(*) FROM chats) || \' \' || (SELECT count(*) FROM chat_members)')
|
||||
# Второй участник не существует: INSERT chats уже выполнен, INSERT chat_members вставит 1 из 2 -> ROLLBACK.
|
||||
c.call('POST', '/api/v1/chats', 400, actor=alice,
|
||||
body={'title': 'Откат', 'member_ids': [alice['id'], str(uuid.uuid4())]})
|
||||
after = sql('SELECT (SELECT count(*) FROM chats) || \' \' || (SELECT count(*) FROM chat_members)')
|
||||
check(before == after, f'partial chat left after failure: {before} -> {after}')
|
||||
listed = c.call('GET', '/api/v1/chats', actor=alice, schema='ChatPage')['json']
|
||||
check(listed['items'] == [], 'failed chat is visible')
|
||||
|
||||
|
||||
def test_cursor_rules(c: Client):
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Другой', 'member_ids': [alice['id']]},
|
||||
schema='Chat')['json']
|
||||
for i in range(3):
|
||||
c.message(alice, chat, f'm{i}')
|
||||
c.message(alice, other, f'o{i}')
|
||||
page = c.call('GET', f'/api/v1/chats/{chat["id"]}/messages?limit=1', actor=alice, schema='MessagePage')['json']
|
||||
from urllib.parse import quote
|
||||
cursor = quote(page['next_cursor'], safe='')
|
||||
c.call('GET', f'/api/v1/chats/{other["id"]}/messages?cursor={cursor}', 400, actor=alice) # курсор чужого чата
|
||||
c.call('GET', f'/api/v1/chats/{chat["id"]}/messages?cursor=garbage', 400, actor=alice)
|
||||
c.call('GET', f'/api/v1/chats/{chat["id"]}/messages?limit=101', 400, actor=alice)
|
||||
c.call('POST', '/api/v1/chats', 400, actor=alice, body={'title': 'x', 'member_ids': [alice['id']], 'extra': 1})
|
||||
c.call('POST', f'/api/v1/chats/{chat["id"]}/messages', 400, actor=bob, body={'text': 'x', 'sender_id': alice['id']})
|
||||
|
||||
|
||||
def test_valkey_cache_role(c: Client):
|
||||
alice = c.user('cache')
|
||||
wait_cache_used(c, alice)
|
||||
ttl = int(valkey('TTL ' + cache_key(alice)))
|
||||
check(0 < ttl <= 60, f'unexpected cache TTL {ttl}')
|
||||
valkey('FLUSHALL') # потеря всего кэша
|
||||
c.call('GET', '/api/v1/users/me', actor=alice, schema='User') # источник истины — PostgreSQL
|
||||
check(valkey('EXISTS ' + cache_key(alice)) == '1', 'cache was not refilled on miss')
|
||||
|
||||
|
||||
def test_valkey_down(c: Client):
|
||||
alice = c.user('vkdown')
|
||||
try:
|
||||
compose('stop', 'valkey')
|
||||
c.call('GET', '/api/v1/users/me', actor=alice, schema='User') # обход кэша через PostgreSQL
|
||||
c.call('GET', '/api/v1/users/me', 401, headers=bearer('A' * 43)) # неизвестный токен — не доверяем
|
||||
c.call('GET', '/api/v1/users/me', 401) # аноним — 401
|
||||
bob = c.user('vkdown_login') # вход работает без кэша
|
||||
c.call('GET', '/health/ready', schema='Health') # Valkey не критичен
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=bob)
|
||||
c.call('GET', '/api/v1/users/me', 401, actor=bob)
|
||||
finally:
|
||||
compose('start', 'valkey')
|
||||
wait_healthy('valkey')
|
||||
|
||||
|
||||
def test_revoke_during_cache_partition(c: Client):
|
||||
# Худший случай инвалидации: logout во время разрыва API <-> Valkey, DEL потерян, а копия отозванной
|
||||
# сессии в Valkey пережила разрыв. Окно недоверия к кэшу (SESSION_CACHE_TTL_SECONDS после ошибки)
|
||||
# не даёт её использовать. TCP может доставить DEL повторной передачей после восстановления сети,
|
||||
# поэтому потерю DEL моделируем явно: возвращаем сохранённую до разрыва копию записи.
|
||||
alice = c.user('partition')
|
||||
wait_cache_used(c, alice)
|
||||
key = cache_key(alice)
|
||||
stale_value = valkey('GET ' + key)
|
||||
container = compose('ps', '-q', 'valkey', capture=True)
|
||||
network = 'mtusi-lab5_data' # Valkey в lab3 только во внутренней сети data
|
||||
subprocess.run(['docker', 'network', 'disconnect', network, container], check=True)
|
||||
try:
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) # COMMIT в PostgreSQL прошёл
|
||||
finally:
|
||||
subprocess.run(['docker', 'network', 'connect', '--alias', 'valkey', network, container], check=True)
|
||||
valkey(f"SET {key} '{stale_value}' EX 60")
|
||||
c.call('GET', '/api/v1/users/me', 401, actor=alice)
|
||||
|
||||
|
||||
def test_postgres_down(c: Client):
|
||||
alice = c.user('pgdown')
|
||||
c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||||
try:
|
||||
compose('stop', 'postgres')
|
||||
c.call('GET', '/health/live', schema='Health')
|
||||
c.call('GET', '/health/ready', 503)
|
||||
r = c.call('GET', '/api/v1/users/me', 503, actor=alice) # ограниченная ошибка, не 500/200
|
||||
check(r['json']['error']['code'] == 'unavailable', 'expected unavailable')
|
||||
c.call('GET', '/api/v1/users/me', 503, headers=bearer('B' * 43)) # неизвестный токен не проходит
|
||||
c.call('GET', '/api/v1/chats', 401) # аноним не пропускается
|
||||
c.call('POST', '/api/v1/users', 503, body={'username': 'pg_' + uuid.uuid4().hex[:8],
|
||||
'display_name': 'x', 'password': 'Password-123456'})
|
||||
finally:
|
||||
compose('start', 'postgres')
|
||||
wait_ready(c)
|
||||
c.call('GET', '/api/v1/users/me', actor=alice, schema='User') # та же сессия жива после рестарта БД
|
||||
|
||||
|
||||
TESTS = [test_wrong_password, test_logout, test_short_ttl, test_username_race, test_chat_rollback,
|
||||
test_cursor_rules, test_valkey_cache_role, test_revoke_during_cache_partition, test_valkey_down,
|
||||
test_postgres_down]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
c = Client(BASE_URL)
|
||||
wait_ready(c)
|
||||
failed = 0
|
||||
for test in TESTS:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
test(c)
|
||||
print(f'PASS {test.__name__} ({time.monotonic() - started:.1f}s)')
|
||||
except (Failure, subprocess.CalledProcessError, OSError) as exc:
|
||||
failed += 1
|
||||
print(f'FAIL {test.__name__}: {exc}')
|
||||
print(f'{len(TESTS) - failed}/{len(TESTS)} passed')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+308
@@ -0,0 +1,308 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Собственные проверки lab3 (уровни 3–4), в lab4 — регрессия. Запуск: sh scripts/test-lab4.sh (внутри сервиса tester).
|
||||
|
||||
Обращаются к репликам и НАПРЯМУЮ (по IP из DNS Docker внутри сети edge), чтобы гарантировать, что
|
||||
конкурентные повторы действительно приходят в разные процессы, а не полагаться на round-robin LB.
|
||||
Требуют доступ к docker (останавливают postgres, шлют SIGTERM реплике и возвращают всё обратно).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
PROJECT = 'mtusi-lab5'
|
||||
API_FILTER = ['--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', 'label=com.docker.compose.service=api']
|
||||
|
||||
|
||||
def compose(*args: str, capture: bool = False) -> str:
|
||||
result = subprocess.run(['docker', 'compose', '--env-file', str(ROOT / '.env'), *args],
|
||||
cwd=ROOT, check=True, text=True,
|
||||
stdout=subprocess.PIPE if capture else subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL)
|
||||
return (result.stdout or '').strip()
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
return subprocess.run(['docker', *args], check=True, text=True, capture_output=True).stdout.strip()
|
||||
|
||||
|
||||
def sql(query: str) -> str:
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'app', '-d', 'messenger', '-tAc', query, capture=True)
|
||||
|
||||
|
||||
def replica_urls() -> list[str]:
|
||||
"""Все IP сервиса api из DNS Docker -> прямые адреса реплик (порты наружу не опубликованы)."""
|
||||
ips = sorted({info[4][0] for info in socket.getaddrinfo('api', 8080, socket.AF_INET, socket.SOCK_STREAM)})
|
||||
check(len(ips) >= 2, f'expected >= 2 api replicas in DNS, got {len(ips)}')
|
||||
return [f'http://{ip}:8080' for ip in ips]
|
||||
|
||||
|
||||
def wait_ready(c: Client, seconds: float = 60) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'API did not become ready')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def raw_post(base: str, path: str, actor: dict, raw: bytes, key: str | None):
|
||||
"""POST с заданными байтами тела (для проверки семантического равенства JSON)."""
|
||||
headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer ' + actor['token']}
|
||||
if key is not None:
|
||||
headers['Idempotency-Key'] = key
|
||||
return Client(base).call('POST', path, [201, 400, 409], raw=raw, headers=headers, schema='Message')
|
||||
|
||||
|
||||
def concurrent(calls: list) -> list:
|
||||
"""Запускает функции одновременно (barrier) и возвращает результаты в исходном порядке."""
|
||||
results = [None] * len(calls)
|
||||
barrier = threading.Barrier(len(calls))
|
||||
|
||||
def run(i):
|
||||
barrier.wait()
|
||||
try:
|
||||
results[i] = calls[i]()
|
||||
except Exception as exc: # результат проверяет вызывающий тест
|
||||
results[i] = exc
|
||||
|
||||
threads = [threading.Thread(target=run, args=(i,)) for i in range(len(calls))]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
return results
|
||||
|
||||
|
||||
def count_messages(chat: dict) -> int:
|
||||
return int(sql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'"))
|
||||
|
||||
# ---------------------------------------------------------------- тесты
|
||||
|
||||
|
||||
def test_session_across_replicas(c: Client):
|
||||
alice = c.user('xrepl') # вход через LB (обслужила какая-то одна реплика)
|
||||
seen = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||||
for _ in range(20)}
|
||||
check(len(seen) >= 2, f'one session must be served by >= 2 replicas, saw {seen}')
|
||||
urls = replica_urls()
|
||||
for url in urls: # каждая реплика напрямую принимает тот же токен
|
||||
me = Client(url).call('GET', '/api/v1/users/me', actor=alice, schema='User')['json']
|
||||
check(me['id'] == alice['id'], 'replica resolved another user')
|
||||
Client(urls[0]).call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
for url in urls: # logout на одной реплике виден всем (общий store)
|
||||
Client(url).call('GET', '/api/v1/users/me', 401, actor=alice)
|
||||
|
||||
|
||||
def test_same_key_concurrent_across_replicas(c: Client):
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
urls, key = replica_urls(), uuid.uuid4().hex
|
||||
calls = [lambda u=urls[i % len(urls)]: Client(u, timeout=15).message(alice, chat, 'один раз', key=key)
|
||||
for i in range(24)]
|
||||
results = concurrent(calls)
|
||||
failures = [r for r in results if isinstance(r, Exception)]
|
||||
check(not failures, f'concurrent retry failed: {failures[:1]}')
|
||||
bodies = {json.dumps(r['json'], sort_keys=True) for r in results}
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(bodies) == 1, f'{len(bodies)} different messages returned for one key')
|
||||
check(len(instances) >= 2, 'retries did not hit different replicas')
|
||||
check(count_messages(chat) == 1, f'{count_messages(chat)} messages stored for one key')
|
||||
via_lb = c.message(alice, chat, 'один раз', key=key)['json'] # и позже через LB — тот же ответ
|
||||
check(json.dumps(via_lb, sort_keys=True) in bodies, 'later replay differs')
|
||||
|
||||
|
||||
def test_same_key_concurrent_different_bodies(c: Client):
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
urls, key = replica_urls(), uuid.uuid4().hex
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
def send(url, text):
|
||||
return Client(url, timeout=15).call('POST', path, [201, 409], actor=alice, body={'text': text},
|
||||
headers={'Idempotency-Key': key}, schema='Message')
|
||||
calls = [lambda u=urls[i % len(urls)], t=('A' if i % 2 else 'B'): send(u, t) for i in range(20)]
|
||||
results = concurrent(calls)
|
||||
check(not any(isinstance(r, Exception) for r in results), 'unexpected status in conflicting retries')
|
||||
winners = {r['json']['text'] for r in results if r['status'] == 201}
|
||||
check(len(winners) == 1, f'both bodies accepted for one key: {winners}')
|
||||
loser_statuses = {r['status'] for r in results if r['status'] != 201}
|
||||
check(loser_statuses <= {409}, 'different body must be 409')
|
||||
check(sum(r['status'] == 409 for r in results) == 10, 'every other-body request must get 409')
|
||||
check(count_messages(chat) == 1, 'conflicting retries stored more than one message')
|
||||
|
||||
|
||||
def test_key_scope_and_format(c: Client):
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
other = c.call('POST', '/api/v1/chats', 201, actor=alice, body={'title': 'Другой', 'member_ids': [alice['id']]},
|
||||
schema='Chat')['json']
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
key = 'scope.' + uuid.uuid4().hex + ':1-x_y' # все допустимые символы
|
||||
first = c.message(alice, chat, 'область', key=key)['json']
|
||||
check(c.message(bob, chat, 'область', key=key)['json']['id'] != first['id'], 'key shared between users')
|
||||
check(c.message(alice, other, 'область', key=key)['json']['id'] != first['id'], 'key shared between chats')
|
||||
# Семантически тот же JSON другими байтами (пробелы, \u-экранирование) — тот же ответ.
|
||||
same = raw_post(BASE_URL, path, alice, b'{ "text" : "\\u043e\\u0431\\u043b\\u0430\\u0441\\u0442\\u044c" }', key)
|
||||
check(same['status'] == 201 and same['json'] == first, 'semantically equal JSON was not a replay')
|
||||
conflict = raw_post(BASE_URL, path, alice, json.dumps({'text': 'иначе'}).encode(), key)
|
||||
check(conflict['status'] == 409 and conflict['json']['error']['code'] == 'conflict', 'expected 409 conflict')
|
||||
check(c.message(alice, chat, 'область', key=key)['json'] == first, '409 must not replace stored result')
|
||||
for bad in (None, '', 'has space', 'café', 'x' * 129, 'a/b'):
|
||||
r = raw_post(BASE_URL, path, alice, b'{"text":"bad key"}', bad)
|
||||
check(r['status'] == 400, f'invalid key {bad!r} accepted')
|
||||
check(len(c.message(alice, chat, 'max', key='k' * 128)['json']['id']) == 36, '128-char key rejected')
|
||||
check(count_messages(chat) == 3, 'unexpected number of messages in scope test') # alice x2, bob x1
|
||||
|
||||
|
||||
def test_replay_rechecks_session(c: Client):
|
||||
alice, _bob, eve, chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
c.message(alice, chat, 'до выхода', key=key)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
# Посторонний с тем же ключом не получает чужой результат.
|
||||
c.call('POST', path, 404, actor=eve, body={'text': 'до выхода'}, headers={'Idempotency-Key': key})
|
||||
c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice)
|
||||
c.call('POST', path, 401, actor=alice, body={'text': 'до выхода'}, headers={'Idempotency-Key': key})
|
||||
|
||||
|
||||
def test_transient_failure_not_recorded(c: Client):
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
try:
|
||||
compose('stop', 'postgres')
|
||||
r = c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 503, actor=alice,
|
||||
body={'text': 'после сбоя'}, headers={'Idempotency-Key': key})
|
||||
check(r['json']['error']['code'] == 'unavailable', 'expected unavailable')
|
||||
check(bool(r['headers'].get('X-Instance-Id')), '503 must come from API (fallback backend), not LB')
|
||||
c.call('GET', '/health/live', schema='Health')
|
||||
finally:
|
||||
compose('start', 'postgres')
|
||||
wait_ready(c)
|
||||
first = c.message(alice, chat, 'после сбоя', key=key)['json'] # повтор с тем же ключом создаёт запись
|
||||
check(c.message(alice, chat, 'после сбоя', key=key)['json'] == first, 'replay after recovery differs')
|
||||
check(count_messages(chat) == 1, 'transient failure produced extra message')
|
||||
|
||||
|
||||
def test_sigterm_drain(c: Client):
|
||||
"""SIGTERM: реплика сначала отвечает ready=503, но ещё обслуживает запросы, затем завершается с кодом 0."""
|
||||
alice = c.user('drain')
|
||||
names = docker('ps', *API_FILTER, '--format', '{{.Names}}').split()
|
||||
name = sorted(names)[0]
|
||||
ip = docker('inspect', '-f', '{{(index .NetworkSettings.Networks "%s_edge").IPAddress}}' % PROJECT, name)
|
||||
direct = Client(f'http://{ip}:8080', timeout=2)
|
||||
docker('kill', '-s', 'TERM', name)
|
||||
signalled = time.monotonic()
|
||||
try:
|
||||
time.sleep(0.3)
|
||||
direct.call('GET', '/health/ready', 503)
|
||||
direct.call('GET', '/health/live', schema='Health')
|
||||
r = direct.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||||
check(r['headers'].get('Connection', '').lower() == 'close', 'draining replica must close keep-alive')
|
||||
time.sleep(max(0.0, signalled + 2.0 - time.monotonic())) # окно обнаружения LB: fall 2 x 0.5-1 с
|
||||
served = {c.call('GET', '/api/v1/users/me', actor=alice, schema='User')['headers']['X-Instance-Id']
|
||||
for _ in range(10)} # LB уже не отправляет на неё запросы
|
||||
deadline = time.monotonic() + 20
|
||||
while docker('inspect', '-f', '{{.State.Status}}', name) != 'exited':
|
||||
check(time.monotonic() < deadline, 'replica did not exit after drain')
|
||||
time.sleep(0.2)
|
||||
code = docker('inspect', '-f', '{{.State.ExitCode}}', name)
|
||||
check(code in ('0', '143'), f'unclean exit code {code}')
|
||||
hostname = docker('inspect', '-f', '{{.Config.Hostname}}', name)
|
||||
check(not any(i.startswith(hostname) for i in served), 'LB still routes to the draining replica')
|
||||
finally:
|
||||
subprocess.run(['sh', str(ROOT / 'scripts/restore.sh')], check=True, capture_output=True)
|
||||
deadline = time.monotonic() + 30
|
||||
while len({c.call('GET', '/health/live', schema='Health')['headers']['X-Instance-Id'] for _ in range(10)}) < 2:
|
||||
check(time.monotonic() < deadline, 'restored replica did not return to LB')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def test_infrastructure_rules(_c: Client):
|
||||
containers = docker('ps', '--filter', f'label=com.docker.compose.project={PROJECT}', '--format', '{{.Names}}').split()
|
||||
for name in containers:
|
||||
cfg = json.loads(docker('inspect', name))[0]
|
||||
service = cfg['Config']['Labels']['com.docker.compose.service']
|
||||
if service == 'tester':
|
||||
continue
|
||||
host = cfg['HostConfig']
|
||||
check(host['NanoCpus'] > 0 and host['Memory'] > 0, f'{name}: no CPU/RAM limit')
|
||||
check(name.startswith(f'{PROJECT}-{service}-'), f'{name}: fixed container_name prevents scaling')
|
||||
bindings = host.get('PortBindings') or {}
|
||||
if service == 'lb':
|
||||
check(all(b['HostIp'] == '127.0.0.1' for v in bindings.values() for b in v), 'LB published beyond loopback')
|
||||
else:
|
||||
check(not bindings, f'{name} publishes ports in bypass of LB')
|
||||
if service == 'api':
|
||||
check(cfg['HostConfig']['ReadonlyRootfs'], 'api root fs must be read-only (no local state)')
|
||||
mounts = {m['Destination'] for m in cfg['Mounts']}
|
||||
check(mounts <= {'/run/secrets/database_url', '/run/secrets/cache_url', '/run/secrets/s3_access_key_id',
|
||||
'/run/secrets/s3_secret_access_key', '/run/secrets/broker_url'},
|
||||
f'api has unexpected mounts {mounts}') # lab4: + S3, lab5: + broker
|
||||
check(sql('SELECT count(*) FROM schema_migrations') == '6', 'unexpected migration count') # lab4: + 0004_attachments, 0005_nul_safe_text; lab5: + 0006_jobs
|
||||
with urllib.request.urlopen(os.environ.get('LB_STATS_URL', 'http://lb:8404/stats') + ';csv', timeout=2) as r:
|
||||
text = r.read().decode()
|
||||
check(',UP,' in text or ',UP ' in text, 'LB stats show no UP backend')
|
||||
|
||||
|
||||
def test_nul_in_text_roundtrip(c: Client):
|
||||
"""Замечание к PR #99: {"text":"abc\\u0000def"} давал 500. Строка с U+0000 сохраняется точно."""
|
||||
alice, bob, _eve, _chat = c.bootstrap()
|
||||
text = 'abc\u0000def'
|
||||
chat = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'чат\u0000NUL', 'member_ids': [alice['id'], bob['id']]}, schema='Chat')['json']
|
||||
check(chat['title'] == 'чат\u0000NUL', 'title with NUL changed in response')
|
||||
got = c.call('GET', '/api/v1/chats/' + chat['id'], actor=bob, schema='Chat')['json']
|
||||
check(got['title'] == 'чат\u0000NUL', 'title with NUL changed on read')
|
||||
key = uuid.uuid4().hex
|
||||
first = c.message(alice, chat, text, key=key)['json']
|
||||
check(first['text'] == text, 'response text differs')
|
||||
replay = c.message(alice, chat, text, key=key)['json']
|
||||
check(replay == first, 'idempotent replay differs')
|
||||
c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice,
|
||||
body={'text': 'abc\ufffddef'}, headers={'Idempotency-Key': key}) # U+FFFD — другой текст
|
||||
c.message(alice, chat, '\u0000') # только NUL — не пробел, допустим
|
||||
history = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob,
|
||||
schema='MessagePage')['json']['items']
|
||||
check([m['text'] for m in history] == [text, '\u0000'], 'history lost NUL')
|
||||
name = 'Имя\u0000Фамилия'
|
||||
user = c.call('POST', '/api/v1/users', 201, body={'username': 'nul_' + uuid.uuid4().hex[:12],
|
||||
'display_name': name, 'password': 'T3st-' + uuid.uuid4().hex}, schema='User')['json']
|
||||
check(user['display_name'] == name, 'display_name with NUL changed')
|
||||
|
||||
|
||||
TESTS = [test_session_across_replicas, test_same_key_concurrent_across_replicas,
|
||||
test_same_key_concurrent_different_bodies, test_key_scope_and_format, test_replay_rechecks_session,
|
||||
test_transient_failure_not_recorded, test_sigterm_drain, test_infrastructure_rules,
|
||||
test_nul_in_text_roundtrip]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
c = Client(BASE_URL)
|
||||
wait_ready(c)
|
||||
failed = 0
|
||||
for test in TESTS:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
test(c)
|
||||
print(f'PASS {test.__name__} ({time.monotonic() - started:.1f}s)', flush=True)
|
||||
except (Failure, subprocess.CalledProcessError, OSError) as exc:
|
||||
failed += 1
|
||||
print(f'FAIL {test.__name__}: {exc}', flush=True)
|
||||
print(f'{len(TESTS) - failed}/{len(TESTS)} passed')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+284
@@ -0,0 +1,284 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Собственные проверки lab4 (уровень 3), в lab5 — регрессия с переходом 201 -> 202 (после upload ждём ready).
|
||||
Запуск: sh scripts/test-lab5.sh (внутри сервиса tester).
|
||||
|
||||
Покрывают критерии README «3»: PNG/JPEG upload -> 201 ready, побайтный download с SHA-256, доступ только
|
||||
участникам, сообщение с attachment_ids, лимиты (10 MiB, 40 Мпикс до декодирования), MIME по содержимому,
|
||||
опасные filename, failover загрузившей реплики, недоступность S3, приватность bucket вне API и отсутствие
|
||||
локальной копии файла на API. Требуют доступ к docker: останавливают реплику и S3, затем возвращают.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
PROJECT = 'mtusi-lab5'
|
||||
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
JPEG = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes()
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
return subprocess.run(['docker', *args], check=True, text=True, capture_output=True).stdout.strip()
|
||||
|
||||
|
||||
def service_containers(service: str, running_only: bool = True) -> list[str]:
|
||||
args = ['ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', f'label=com.docker.compose.service={service}']
|
||||
if not running_only:
|
||||
args.insert(1, '-a')
|
||||
return docker(*args).split()
|
||||
|
||||
|
||||
def wait_ready(c: Client, seconds: float = 60) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'API did not become ready')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def upload(c: Client, actor: dict, chat: dict, data: bytes, ctype: str, filename: str = 'photo.png',
|
||||
expected=202, wait=True) -> dict:
|
||||
path = f"/api/v1/chats/{chat['id']}/attachments?filename={quote(filename, safe='')}"
|
||||
response = c.call('POST', path, expected, actor=actor, raw=data, headers={'Content-Type': ctype},
|
||||
schema='Attachment' if expected == 202 else None)
|
||||
if expected == 202:
|
||||
check(response['json']['status'] == 'queued', 'lab5 upload must return queued snapshot')
|
||||
if wait: # lab5: готовность — через опрос метаданных (заголовки ответа upload сохраняем)
|
||||
response['json'] = c.wait_ready(actor, response['json'])
|
||||
return response
|
||||
|
||||
|
||||
def download(c: Client, actor: dict, att_id: str, expected=200) -> dict:
|
||||
return c.call('GET', f'/api/v1/attachments/{att_id}/content', expected, actor=actor)
|
||||
|
||||
|
||||
def png_chunk(kind: bytes, data: bytes) -> bytes:
|
||||
return struct.pack('>I', len(data)) + kind + data + struct.pack('>I', zlib.crc32(kind + data) & 0xffffffff)
|
||||
|
||||
|
||||
def png_header_only(width: int, height: int) -> bytes:
|
||||
"""Валидная сигнатура и IHDR (с верным CRC) огромного изображения, без пиксельных данных."""
|
||||
ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0)
|
||||
idat = zlib.compress(b'\x00' * 64) # заведомо меньше, чем нужно: декодировать сервер не должен
|
||||
return b'\x89PNG\r\n\x1a\n' + png_chunk(b'IHDR', ihdr) + png_chunk(b'IDAT', idat) + png_chunk(b'IEND', b'')
|
||||
|
||||
|
||||
# ---------- тесты ----------
|
||||
|
||||
def test_png_and_jpeg_roundtrip(c: Client):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
for data, ctype, name in ((PNG, 'image/png', 'Фото 1.png'), (JPEG, 'image/jpeg', 'кадр.jpg')):
|
||||
att = upload(c, alice, chat, data, ctype, name)['json']
|
||||
check(att['status'] == 'ready' and len(att['variants']) == 3 and att['error_code'] is None, 'not ready')
|
||||
check(att['filename'] == name and att['content_type'] == ctype, 'metadata mismatch')
|
||||
check(att['sha256'] == hashlib.sha256(data).hexdigest() and att['size_bytes'] == len(data), 'checksum')
|
||||
meta = c.call('GET', f"/api/v1/attachments/{att['id']}", actor=bob, schema='Attachment')['json']
|
||||
check(meta == att, 'metadata via GET differs from upload response')
|
||||
got = download(c, bob, att['id'])
|
||||
check(got['bytes'] == data, 'downloaded bytes differ')
|
||||
check(got['headers'].get_content_type() == ctype, 'wrong Content-Type on download')
|
||||
check(hashlib.sha256(got['bytes']).hexdigest() == att['sha256'], 'sha256 mismatch')
|
||||
for suffix in ('', '/content'):
|
||||
c.call('GET', f"/api/v1/attachments/{att['id']}{suffix}", 404, actor=eve)
|
||||
c.call('GET', f"/api/v1/attachments/{att['id']}/content", 401)
|
||||
|
||||
|
||||
def test_message_with_attachments(c: Client):
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
a1 = upload(c, alice, chat, PNG, 'image/png')['json']
|
||||
a2 = upload(c, bob, chat, JPEG, 'image/jpeg', 'b.jpg')['json']
|
||||
key = uuid.uuid4().hex
|
||||
msg = c.message(bob, chat, '', key=key, attachments=[a2['id'], a1['id']])['json'] # любой участник
|
||||
check(msg['attachment_ids'] == [a2['id'], a1['id']] and msg['text'] == '', 'attachment order lost')
|
||||
replay = c.message(bob, chat, '', key=key, attachments=[a2['id'], a1['id']])['json']
|
||||
check(replay == msg, 'idempotent replay with attachments differs')
|
||||
plain = c.message(alice, chat, 'только текст')['json']
|
||||
check(plain['attachment_ids'] == [], 'text message must have attachment_ids=[]')
|
||||
history = c.call('GET', f"/api/v1/chats/{chat['id']}/messages?limit=100", actor=alice,
|
||||
schema='MessagePage')['json']['items']
|
||||
check([m['attachment_ids'] for m in history] == [[a2['id'], a1['id']], []], 'history lost attachments')
|
||||
h = {'Idempotency-Key': uuid.uuid4().hex}
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice, body={'text': ' '}, headers=h)
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 400, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [a1['id'], a1['id']]}, headers=h)
|
||||
|
||||
|
||||
def test_foreign_attachment_rejected(c: Client):
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
other = c.call('POST', '/api/v1/chats', 201, actor=eve, body={'title': 'чужой', 'member_ids': [eve['id']]},
|
||||
schema='Chat')['json']
|
||||
foreign = upload(c, eve, other, PNG, 'image/png')['json']
|
||||
h = {'Idempotency-Key': uuid.uuid4().hex}
|
||||
# Вложение другого чата и несуществующее — 404, сообщение не создаётся.
|
||||
for ids in ([foreign['id']], [str(uuid.uuid4())]):
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 404, actor=alice,
|
||||
body={'text': 'x', 'attachment_ids': ids}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
items = c.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=alice, schema='MessagePage')['json']['items']
|
||||
check(items == [], 'message with foreign attachment was created')
|
||||
# Посторонний не может загрузить файл в чужой чат.
|
||||
upload(c, eve, chat, PNG, 'image/png', expected=404)
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 404, actor=eve,
|
||||
body={'text': 'x'}, headers=h)
|
||||
|
||||
|
||||
def test_limits_and_mime(c: Client):
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
# 10 MiB + 1 байт: 413 (тело дальше лимита сервер не читает).
|
||||
upload(c, alice, chat, PNG[:8] + b'\0' * (10 * 1024 * 1024 - 7), 'image/png', expected=413)
|
||||
# Ровно 10 MiB проходит проверку размера (а затем отклоняется как повреждённый PNG — 415, не 413).
|
||||
upload(c, alice, chat, PNG[:8] + b'\0' * (10 * 1024 * 1024 - 8), 'image/png', expected=415)
|
||||
# 10000 x 5000 = 50 Мпикс по заголовку -> 413 до декодирования (файл всего ~100 байт).
|
||||
upload(c, alice, chat, png_header_only(10000, 5000), 'image/png', expected=413)
|
||||
# 6000 x 6000 = 36 Мпикс укладывается в лимит, но пикселей нет -> повреждённый -> 415.
|
||||
upload(c, alice, chat, png_header_only(6000, 6000), 'image/png', expected=415)
|
||||
# MIME по содержимому: PNG с Content-Type image/jpeg и наоборот -> 415.
|
||||
upload(c, alice, chat, PNG, 'image/jpeg', expected=415)
|
||||
upload(c, alice, chat, JPEG, 'image/png', expected=415)
|
||||
# Не изображение, обрезанные файлы, пустое тело, чужие Content-Type.
|
||||
upload(c, alice, chat, b'GIF89a' + b'\0' * 100, 'image/png', expected=415)
|
||||
upload(c, alice, chat, PNG[:len(PNG) // 2], 'image/png', expected=415)
|
||||
upload(c, alice, chat, JPEG[:len(JPEG) // 2], 'image/jpeg', expected=415)
|
||||
upload(c, alice, chat, b'', 'image/png', expected=415)
|
||||
upload(c, alice, chat, PNG, 'text/plain', expected=415)
|
||||
upload(c, alice, chat, PNG, 'multipart/form-data; boundary=x', expected=415)
|
||||
|
||||
|
||||
def test_dangerous_filenames(c: Client):
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
for name in ('../etc/passwd', 'a/b.png', 'a\\b.png', 'nul\x00.png', 'tab\t.png', 'new\nline.png',
|
||||
'\x7f.png', '', ' ', '..', 'x' * 256):
|
||||
upload(c, alice, chat, PNG, 'image/png', name, expected=400)
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/attachments", 400, actor=alice, raw=PNG,
|
||||
headers={'Content-Type': 'image/png'}) # без filename
|
||||
ok = upload(c, alice, chat, PNG, 'image/png', 'отчёт (финал) ✓.png')['json']
|
||||
check(ok['filename'] == 'отчёт (финал) ✓.png', 'unicode filename changed')
|
||||
upload(c, alice, chat, PNG, 'image/png', 'x' * 255)
|
||||
|
||||
|
||||
def test_failover_of_uploading_replica(c: Client):
|
||||
"""Upload -> остановить реплику, которая его обслужила -> download через другую = тот же файл."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
response = upload(c, alice, chat, PNG, 'image/png')
|
||||
att = response['json']
|
||||
served_by = response['headers']['X-Instance-Id']
|
||||
cid = served_by.rsplit('-', 1)[0]
|
||||
check(cid in [x[:12] for x in service_containers('api')], 'uploader replica not found among containers')
|
||||
docker('stop', '-t', '20', cid)
|
||||
try:
|
||||
time.sleep(2)
|
||||
got = download(c, bob, att['id'])
|
||||
check(got['headers']['X-Instance-Id'] != served_by, 'download served by stopped replica')
|
||||
check(got['bytes'] == PNG and hashlib.sha256(got['bytes']).hexdigest() == att['sha256'],
|
||||
'file differs after uploader replica stopped')
|
||||
print(f' uploaded via {served_by}, downloaded via {got["headers"]["X-Instance-Id"]}')
|
||||
finally:
|
||||
docker('start', cid)
|
||||
wait_ready(c)
|
||||
|
||||
|
||||
def test_s3_unavailable(c: Client):
|
||||
"""S3 остановлен: upload/download -> 503 без 500, текстовые сообщения работают, после старта — всё снова."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat, PNG, 'image/png')['json']
|
||||
s3 = service_containers('s3')
|
||||
check(len(s3) == 1, 'expected one s3 container')
|
||||
docker('stop', s3[0])
|
||||
try:
|
||||
started = time.monotonic()
|
||||
upload(c, alice, chat, PNG, 'image/png', expected=503)
|
||||
download(c, bob, att['id'], expected=503)
|
||||
check(time.monotonic() - started < 15, 'S3 failure is not bounded by timeouts')
|
||||
c.message(alice, chat, 'S3 недоступен, а текст отправляется')
|
||||
c.call('GET', f"/api/v1/attachments/{att['id']}", actor=bob, schema='Attachment') # метаданные в БД
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
finally:
|
||||
docker('start', s3[0])
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
check(download(c, bob, att['id'])['bytes'] == PNG, 'file changed after S3 restart')
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'S3 did not come back')
|
||||
time.sleep(1)
|
||||
upload(c, alice, chat, JPEG, 'image/jpeg', 'after.jpg')
|
||||
|
||||
|
||||
def test_bucket_private_outside_api(c: Client):
|
||||
"""Прямой анонимный GET объекта в S3 (в обход API) отклоняется; ключ в API не раскрывается."""
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat, PNG, 'image/png')['json']
|
||||
check('object_key' not in att and 'originals/' not in str(att) and 'staging/' not in str(att), 'object key leaked to API')
|
||||
api = service_containers('api')[0]
|
||||
code = docker('exec', api, 'python', '-c',
|
||||
'import urllib.request,urllib.error,sys\n'
|
||||
'try:\n'
|
||||
f' urllib.request.urlopen("http://s3:9000/messenger-attachments/originals/{att["id"]}", timeout=5)\n'
|
||||
' print(200)\n'
|
||||
'except urllib.error.HTTPError as e:\n'
|
||||
' print(e.code)\n')
|
||||
check(code in ('401', '403'), f'anonymous S3 GET returned {code}')
|
||||
# Листинг bucket анонимно тоже запрещён.
|
||||
code = docker('exec', api, 'python', '-c',
|
||||
'import urllib.request,urllib.error\n'
|
||||
'try:\n'
|
||||
' urllib.request.urlopen("http://s3:9000/messenger-attachments", timeout=5); print(200)\n'
|
||||
'except urllib.error.HTTPError as e:\n'
|
||||
' print(e.code)\n')
|
||||
check(code in ('401', '403'), f'anonymous S3 list returned {code}')
|
||||
print(f' anonymous GET object/list -> {code}')
|
||||
|
||||
|
||||
def test_no_local_copy_on_api(c: Client):
|
||||
"""У API read-only корневая ФС и пустой tmpfs: оригинал не хранится локально."""
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
upload(c, alice, chat, PNG, 'image/png')
|
||||
for cid in service_containers('api'):
|
||||
check(docker('inspect', '-f', '{{.HostConfig.ReadonlyRootfs}}', cid) == 'true', 'api rootfs is writable')
|
||||
# Допустимы только read-only секреты compose (/run/secrets/*); volume и записываемых bind нет.
|
||||
mounts = docker('inspect', '-f', '{{range .Mounts}}{{.Type}}:{{.Destination}}:{{.RW}} {{end}}', cid).split()
|
||||
writable = [m for m in mounts if m.startswith('volume:') or
|
||||
(m.startswith('bind:') and not (m.startswith('bind:/run/secrets/') and m.endswith(':false')))]
|
||||
check(not writable, f'api has persistent/writable mounts: {writable}')
|
||||
files = docker('exec', cid, 'find', '/tmp', '-type', 'f')
|
||||
check(files == '', f'unexpected files in api /tmp: {files}')
|
||||
|
||||
|
||||
TESTS = [test_png_and_jpeg_roundtrip, test_message_with_attachments, test_foreign_attachment_rejected,
|
||||
test_limits_and_mime, test_dangerous_filenames, test_bucket_private_outside_api,
|
||||
test_no_local_copy_on_api, test_failover_of_uploading_replica, test_s3_unavailable]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
c = Client(BASE_URL, timeout=30)
|
||||
wait_ready(c)
|
||||
failed = 0
|
||||
for test in TESTS:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
test(c)
|
||||
print(f'PASS {test.__name__} ({time.monotonic() - started:.1f}s)', flush=True)
|
||||
except (Failure, subprocess.CalledProcessError, OSError) as exc:
|
||||
failed += 1
|
||||
print(f'FAIL {test.__name__}: {exc}', flush=True)
|
||||
print(f'{len(TESTS) - failed}/{len(TESTS)} passed')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+515
@@ -0,0 +1,515 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Собственные проверки lab5 (уровни 3–4). Запуск: sh scripts/test-lab5.sh (внутри сервиса tester).
|
||||
|
||||
Управляют стендом через docker (останавливают worker/S3/RabbitMQ, внедряют отказы файлами /tmp/fault-*),
|
||||
читают состояние из PostgreSQL и RabbitMQ, чтобы проверять не только HTTP-ответы, но и инварианты:
|
||||
нет второго набора вариантов, попытки ограничены, принятая работа не теряется.
|
||||
"""
|
||||
import hashlib
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
from image_probe import jpeg_size # noqa: E402
|
||||
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://lb:8080')
|
||||
PROJECT = 'mtusi-lab5'
|
||||
PNG = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
JPEG = (ROOT / 'student-tests/fixtures/sample.jpg').read_bytes()
|
||||
|
||||
|
||||
# ---------- инфраструктура ----------
|
||||
|
||||
def docker(*args: str, stdin: str | None = None) -> str:
|
||||
return subprocess.run(['docker', *args], check=True, text=True, capture_output=True, input=stdin).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args: str) -> str:
|
||||
return subprocess.run(['docker', 'compose', '--env-file', str(ROOT / '.env'), *args], cwd=ROOT, check=True,
|
||||
text=True, capture_output=True).stdout.strip()
|
||||
|
||||
|
||||
def containers(service: str) -> list[str]:
|
||||
return docker('ps', '-q', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', f'label=com.docker.compose.service={service}').split()
|
||||
|
||||
|
||||
def sql(query: str) -> str:
|
||||
return compose('exec', '-T', 'postgres', 'psql', '-U', 'app', '-d', 'messenger', '-tAc', query)
|
||||
|
||||
|
||||
def in_container(service: str, script: str) -> str:
|
||||
"""Python-скрипт внутри контейнера сервиса (там есть minio/aio-pika/Pillow и секреты).
|
||||
|
||||
Скрипт пишется в /tmp (tmpfs) и запускается файлом: multiprocessing forkserver не умеет стартовать
|
||||
из stdin. Код с побочными эффектами в скриптах — под if __name__ == '__main__'.
|
||||
"""
|
||||
return docker('exec', '-i', containers(service)[0], 'sh', '-c', 'cat > /tmp/_t.py && PYTHONPATH=/srv python /tmp/_t.py; rc=$?; rm -f /tmp/_t.py; exit $rc',
|
||||
stdin=script)
|
||||
|
||||
|
||||
def logs(container: str, since: str | None = None) -> str:
|
||||
"""docker logs: приложение пишет logging в stderr, поэтому склеиваем оба потока."""
|
||||
args = ['docker', 'logs'] + (['--since', since] if since else []) + [container]
|
||||
return subprocess.run(args, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT).stdout
|
||||
|
||||
|
||||
def start_worker() -> None:
|
||||
"""docker start, а не compose start: compose поднял бы и намеренно остановленные зависимости (S3)."""
|
||||
ids = docker('ps', '-aq', '--filter', f'label=com.docker.compose.project={PROJECT}',
|
||||
'--filter', 'label=com.docker.compose.service=worker').split()
|
||||
docker('start', *ids)
|
||||
|
||||
|
||||
S3_PRELUDE = '''
|
||||
from minio import Minio
|
||||
import io
|
||||
k=open('/run/secrets/s3_access_key_id').read().strip(); s=open('/run/secrets/s3_secret_access_key').read().strip()
|
||||
m=Minio('s3:9000', access_key=k, secret_key=s, secure=False, region='us-east-1')
|
||||
B='messenger-attachments'
|
||||
def exists(key):
|
||||
try:
|
||||
m.stat_object(B, key); return True
|
||||
except Exception:
|
||||
return False
|
||||
'''
|
||||
|
||||
|
||||
def queue_counts() -> dict:
|
||||
out = compose('exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_queues', 'name', 'messages')
|
||||
counts = {}
|
||||
for line in out.splitlines():
|
||||
parts = line.split()
|
||||
if len(parts) == 2 and parts[1].isdigit():
|
||||
counts[parts[0]] = int(parts[1])
|
||||
return counts
|
||||
|
||||
|
||||
def wait_queue(name: str, minimum: int, seconds: float = 20) -> int:
|
||||
"""Счётчики list_queues у quorum queue обновляются периодически (метрики) — опрашиваем."""
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
n = queue_counts().get(name, 0)
|
||||
if n >= minimum or time.monotonic() > deadline:
|
||||
return n
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def wait_healthy(service: str, seconds: float = 90) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
ids = containers(service)
|
||||
if ids and all(docker('inspect', '-f', '{{.State.Health.Status}}', i) == 'healthy' for i in ids):
|
||||
return
|
||||
time.sleep(1)
|
||||
raise Failure(f'{service} did not become healthy')
|
||||
|
||||
|
||||
def wait_ready_api(c: Client, seconds: float = 90) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
try:
|
||||
c.call('GET', '/health/ready', schema='Health')
|
||||
return
|
||||
except Failure:
|
||||
check(time.monotonic() < deadline, 'API did not become ready')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def touch_fault(service: str, name: str) -> list[str]:
|
||||
ids = containers(service)
|
||||
for i in ids:
|
||||
docker('exec', '-u', '10001', i, 'touch', f'/tmp/fault-{name}')
|
||||
return ids
|
||||
|
||||
|
||||
def clear_fault(service: str, name: str) -> None:
|
||||
for i in containers(service):
|
||||
docker('exec', '-u', '10001', i, 'rm', '-f', f'/tmp/fault-{name}')
|
||||
|
||||
|
||||
# ---------- API ----------
|
||||
|
||||
def upload(c: Client, actor: dict, chat: dict, data: bytes = PNG, ctype: str = 'image/png') -> dict:
|
||||
att = c.call('POST', f"/api/v1/chats/{chat['id']}/attachments?filename=photo.png", 202, actor=actor, raw=data,
|
||||
headers={'Content-Type': ctype}, schema='Attachment')['json']
|
||||
check(att['status'] == 'queued' and att['variants'] == [] and att['error_code'] is None, 'bad queued snapshot')
|
||||
return att
|
||||
|
||||
|
||||
def meta(c: Client, actor: dict, att_id: str) -> dict:
|
||||
return c.call('GET', f'/api/v1/attachments/{att_id}', actor=actor, schema='Attachment')['json']
|
||||
|
||||
|
||||
def wait_status(c: Client, actor: dict, att_id: str, wanted: set, seconds: float = 90) -> dict:
|
||||
deadline = time.monotonic() + seconds
|
||||
while True:
|
||||
m = meta(c, actor, att_id)
|
||||
if m['status'] in wanted:
|
||||
return m
|
||||
check(time.monotonic() < deadline, f'attachment stayed {m["status"]}, wanted {wanted}')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def content(c: Client, actor: dict, att_id: str, variant: str = 'original', expected=200) -> dict:
|
||||
return c.call('GET', f'/api/v1/attachments/{att_id}/content?variant={variant}', expected, actor=actor)
|
||||
|
||||
|
||||
def job(att_id: str) -> dict:
|
||||
row = sql(f"SELECT status, attempt, coalesce(error_code,''), "
|
||||
f"(SELECT count(*) FROM attachment_variants v WHERE v.attachment_id = j.attachment_id) "
|
||||
f"FROM attachment_jobs j WHERE attachment_id = '{att_id}'").split('|')
|
||||
return {'status': row[0], 'attempt': int(row[1]), 'error_code': row[2], 'variants': int(row[3])}
|
||||
|
||||
|
||||
def wait_published(att_id: str, seconds: float = 30) -> None:
|
||||
deadline = time.monotonic() + seconds
|
||||
while sql(f"SELECT count(*) FROM outbox o JOIN attachment_jobs j USING (job_id) "
|
||||
f"WHERE j.attachment_id = '{att_id}' AND o.published_at IS NULL") != '0':
|
||||
check(time.monotonic() < deadline, 'outbox row not published')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def webp_size(b: bytes) -> tuple[int, int]:
|
||||
check(b[:4] == b'RIFF' and b[8:12] == b'WEBP', 'not a WebP file')
|
||||
kind = b[12:16]
|
||||
if kind == b'VP8 ':
|
||||
return int.from_bytes(b[26:28], 'little') & 0x3fff, int.from_bytes(b[28:30], 'little') & 0x3fff
|
||||
if kind == b'VP8L':
|
||||
bits = int.from_bytes(b[21:25], 'little')
|
||||
return (bits & 0x3fff) + 1, ((bits >> 14) & 0x3fff) + 1
|
||||
return int.from_bytes(b[24:27], 'little') + 1, int.from_bytes(b[27:30], 'little') + 1
|
||||
|
||||
|
||||
# ---------- уровень 3 ----------
|
||||
|
||||
def test_variants_contract(c: Client):
|
||||
"""Оригинал неизменен; thumbnail 256x160 JPEG, crop 128x128 JPEG, optimized WebP; checksums и реальные размеры."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)
|
||||
ready = wait_status(c, bob, att['id'], {'ready'})
|
||||
check(ready['error_code'] is None, 'ready with error_code')
|
||||
variants = {v['name']: v for v in ready['variants']}
|
||||
check(sorted(variants) == ['crop', 'optimized', 'thumbnail'], f'variants: {sorted(variants)}')
|
||||
expect = {'thumbnail': ('image/jpeg', (256, 160)), 'crop': ('image/jpeg', (128, 128)),
|
||||
'optimized': ('image/webp', (320, 200))}
|
||||
for name, (ctype, size) in expect.items():
|
||||
v = variants[name]
|
||||
got = content(c, bob, att['id'], name)
|
||||
check(got['headers'].get_content_type() == ctype, f'{name}: content type')
|
||||
check(len(got['bytes']) == v['size_bytes'] and hashlib.sha256(got['bytes']).hexdigest() == v['sha256'],
|
||||
f'{name}: checksum')
|
||||
real = jpeg_size(got['bytes']) if ctype == 'image/jpeg' else webp_size(got['bytes'])
|
||||
check(real == size == (v['width'], v['height']), f'{name}: real size {real}, metadata {v["width"]}x{v["height"]}')
|
||||
original = content(c, bob, att['id'])
|
||||
check(original['bytes'] == PNG and original['headers'].get_content_type() == 'image/png', 'original changed')
|
||||
jpg = upload(c, alice, chat, JPEG, 'image/jpeg')
|
||||
r = wait_status(c, alice, jpg['id'], {'ready'})
|
||||
thumb = {v['name']: v for v in r['variants']}['thumbnail']
|
||||
check((thumb['width'], thumb['height']) == (160, 100), 'small image must not be upscaled')
|
||||
|
||||
|
||||
def test_worker_down_demo(c: Client):
|
||||
"""Сценарий защиты: worker остановлен -> 202 -> 409 -> сообщение создано -> рестарт API и RabbitMQ -> worker -> ready."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
att = upload(c, alice, chat)
|
||||
print(f' worker stopped; upload 202 attachment={att["id"]}')
|
||||
check(meta(c, bob, att['id'])['status'] == 'queued', 'must stay queued without worker')
|
||||
for variant in ('original', 'thumbnail'):
|
||||
content(c, bob, att['id'], variant, 409)
|
||||
msg = c.message(alice, chat, 'фото без worker', attachments=[att['id']])['json']
|
||||
check(msg['attachment_ids'] == [att['id']], 'message lost attachment')
|
||||
c.message(bob, chat, 'другие сообщения работают')
|
||||
wait_published(att['id'])
|
||||
before = wait_queue('attachment.process.v1', 1)
|
||||
check(before >= 1, 'job is not in the broker queue')
|
||||
compose('restart', 'api')
|
||||
wait_ready_api(c)
|
||||
compose('restart', 'rabbitmq')
|
||||
wait_healthy('rabbitmq')
|
||||
after = wait_queue('attachment.process.v1', 1)
|
||||
check(after >= 1, 'job lost on broker restart')
|
||||
print(f' api and rabbitmq restarted; queue depth before/after broker restart: {before}/{after}')
|
||||
check(meta(c, bob, att['id'])['status'] == 'queued', 'state changed without worker')
|
||||
finally:
|
||||
start_worker()
|
||||
started = time.monotonic()
|
||||
ready = wait_status(c, bob, att['id'], {'ready'}, 60)
|
||||
thumb = content(c, bob, att['id'], 'thumbnail')
|
||||
check(jpeg_size(thumb['bytes']) == (256, 160), 'thumbnail after restart is wrong')
|
||||
check(len(ready['variants']) == 3, 'variants missing')
|
||||
print(f' worker started -> ready in {time.monotonic() - started:.1f}s, thumbnail OK')
|
||||
|
||||
|
||||
def test_failed_state_and_409(c: Client):
|
||||
"""Постоянная ошибка (staging подменён в S3, sha256 не совпал) -> failed с безопасным кодом; 409 на content и attach."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
att = upload(c, alice, chat)
|
||||
in_container('api', S3_PRELUDE + f"m.put_object(B, 'staging/{att['id']}', io.BytesIO(b'garbage'), 7)\n")
|
||||
finally:
|
||||
start_worker()
|
||||
failed = wait_status(c, bob, att['id'], {'failed', 'ready'}, 60)
|
||||
check(failed['status'] == 'failed' and failed['error_code'] == 'source_corrupted', f'got {failed}')
|
||||
check(failed['variants'] == [], 'failed must have no variants')
|
||||
content(c, bob, att['id'], 'original', 409)
|
||||
c.call('POST', f"/api/v1/chats/{chat['id']}/messages", 409, actor=alice,
|
||||
body={'text': '', 'attachment_ids': [att['id']]}, headers={'Idempotency-Key': uuid.uuid4().hex})
|
||||
j = job(att['id'])
|
||||
check(j['status'] == 'failed' and j['attempt'] == 1, f'permanent error must not be retried: {j}')
|
||||
|
||||
|
||||
def test_retry_bounded_then_replay(c: Client):
|
||||
"""Временная ошибка (S3 недоступен) -> ограниченные повторы с backoff -> failed retry_exhausted -> replay -> ready."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
compose('stop', 'worker')
|
||||
s3 = containers('s3')[0]
|
||||
try:
|
||||
att = upload(c, alice, chat)
|
||||
docker('stop', s3)
|
||||
start_worker()
|
||||
started = time.monotonic()
|
||||
failed = wait_status(c, bob, att['id'], {'failed', 'ready'}, 150)
|
||||
check(failed['status'] == 'failed' and failed['error_code'] == 'retry_exhausted', f'got {failed}')
|
||||
max_attempts = int(sql("SELECT max_attempts FROM attachment_jobs WHERE attachment_id = '%s'" % att['id']))
|
||||
j = job(att['id'])
|
||||
check(j['attempt'] == max_attempts, f'attempts {j["attempt"]} != JOB_MAX_ATTEMPTS {max_attempts}')
|
||||
print(f' S3 down: failed retry_exhausted after {j["attempt"]} attempts, {time.monotonic() - started:.0f}s')
|
||||
worker_logs = logs(containers('worker')[0])
|
||||
job_id = att_job_id(att['id']) # один запрос, не на каждую строку лога
|
||||
delays = [line.split('retry in ')[1].split('s')[0] for line in worker_logs.splitlines()
|
||||
if job_id in line and 'retry in' in line]
|
||||
print(f' backoff delays (s): {", ".join(delays)}')
|
||||
check(len(delays) == max_attempts - 1, 'expected a retry delay per non-final attempt')
|
||||
finally:
|
||||
docker('start', s3)
|
||||
wait_healthy('s3')
|
||||
out = compose('exec', '-T', 'worker', 'python', '-m', 'app.replay', 'replay', '--attachment', att['id'],
|
||||
'--reason', 'S3 restored', '--dry-run')
|
||||
check('would replay' in out and meta(c, bob, att['id'])['status'] == 'failed', 'dry-run changed state')
|
||||
compose('exec', '-T', '-e', 'REPLAY_ACTOR=test', 'worker', 'python', '-m', 'app.replay', 'replay',
|
||||
'--attachment', att['id'], '--reason', 'S3 restored')
|
||||
ready = wait_status(c, bob, att['id'], {'ready'}, 60)
|
||||
check(len(ready['variants']) == 3, 'replay did not produce variants')
|
||||
out = compose('exec', '-T', 'worker', 'python', '-m', 'app.replay', 'replay', '--attachment', att['id'],
|
||||
'--reason', 'again')
|
||||
check('replay only for failed' in out, 'replay of a done job must be refused')
|
||||
audit = sql(f"SELECT count(*) FROM job_audit a JOIN attachment_jobs j USING (job_id) WHERE j.attachment_id = '{att['id']}'")
|
||||
check(audit == '1' and job(att['id'])['variants'] == 3, 'audit/variants after replay')
|
||||
|
||||
|
||||
def att_job_id(att_id: str) -> str:
|
||||
return sql(f"SELECT job_id FROM attachment_jobs WHERE attachment_id = '{att_id}'")
|
||||
|
||||
|
||||
def test_duplicate_delivery_is_noop(c: Client):
|
||||
"""Повторная доставка завершённой job: no-op + ack, без второго набора вариантов и без изменения updated_at."""
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)
|
||||
ready = wait_status(c, alice, att['id'], {'ready'})
|
||||
job_id = att_job_id(att['id'])
|
||||
for _ in range(3):
|
||||
sql(f"INSERT INTO outbox (job_id, payload) SELECT job_id, payload FROM outbox WHERE job_id = '{job_id}' LIMIT 1")
|
||||
wait_published(att['id'])
|
||||
time.sleep(3)
|
||||
check(meta(c, alice, att['id'])['updated_at'] == ready['updated_at'], 'duplicate changed ready attachment')
|
||||
j = job(att['id'])
|
||||
check(j['attempt'] == 1 and j['variants'] == 3, f'duplicate created extra work: {j}')
|
||||
check(f'job {job_id}: no-op, already done' in logs(containers('worker')[0], '60s'), 'worker did not report no-op')
|
||||
|
||||
|
||||
# ---------- уровень 4 ----------
|
||||
|
||||
def test_crash_between_commit_and_publish(c: Client):
|
||||
"""API падает сразу после COMMIT (до 202 и publish): outbox восстанавливает намерение, файл доходит до ready."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
touch_fault('api', 'api_crash_after_commit')
|
||||
try:
|
||||
req = urllib.request.Request(f"{BASE_URL}/api/v1/chats/{chat['id']}/attachments?filename=crash.png", data=PNG,
|
||||
method='POST', headers={'Content-Type': 'image/png',
|
||||
'Authorization': 'Bearer ' + alice['token']})
|
||||
try:
|
||||
status = urllib.request.urlopen(req, timeout=30).status
|
||||
except urllib.error.HTTPError as exc:
|
||||
status = exc.code
|
||||
except (urllib.error.URLError, OSError):
|
||||
status = 'connection error'
|
||||
check(status != 202, f'API should have crashed before 202, got {status}')
|
||||
print(f' client saw: {status} (API process exited after COMMIT)')
|
||||
finally:
|
||||
clear_fault('api', 'api_crash_after_commit')
|
||||
att_id = sql(f"SELECT id FROM attachments WHERE chat_id = '{chat['id']}'")
|
||||
check(len(att_id) == 36, 'attachment row must be committed')
|
||||
wait_ready_api(c)
|
||||
ready = wait_status(c, bob, att_id, {'ready'}, 60)
|
||||
check(len(ready['variants']) == 3, 'variants missing')
|
||||
print(f' attachment {att_id} reached ready via outbox relay')
|
||||
|
||||
|
||||
def test_worker_crash_after_s3_before_commit(c: Client):
|
||||
"""Worker умирает после записи объектов в S3, до COMMIT: аренда истекает, повтор даёт тот же результат."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
worker = touch_fault('worker', 'worker_crash_after_s3')[0]
|
||||
restarts = int(docker('inspect', '-f', '{{.RestartCount}}', worker))
|
||||
att = upload(c, alice, chat)
|
||||
started = time.monotonic()
|
||||
ready = wait_status(c, bob, att['id'], {'ready'}, 90)
|
||||
check(int(docker('inspect', '-f', '{{.RestartCount}}', worker)) == restarts + 1, 'worker did not crash')
|
||||
j = job(att['id'])
|
||||
check(j['attempt'] == 2 and j['variants'] == 3, f'expected second attempt with one variant set: {j}')
|
||||
keys = in_container('api', S3_PRELUDE + f"print(len(list(m.list_objects(B, prefix='variants/{att['id']}/', recursive=True))))\n")
|
||||
check(keys == '3', f'S3 has {keys} variant objects, expected 3 (same keys overwritten)')
|
||||
check(len(ready['variants']) == 3, 'variants missing')
|
||||
print(f' crash after S3 writes -> ready on attempt 2 in {time.monotonic() - started:.1f}s, 3 objects in S3')
|
||||
wait_healthy('worker')
|
||||
|
||||
|
||||
def test_worker_crash_after_commit_before_ack(c: Client):
|
||||
"""Worker умирает после COMMIT результата, до ack: redelivery видит done и подтверждает без повторной работы."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
worker = touch_fault('worker', 'worker_crash_after_commit')[0]
|
||||
att = upload(c, alice, chat)
|
||||
wait_status(c, bob, att['id'], {'ready'}, 60)
|
||||
wait_healthy('worker')
|
||||
job_id = att_job_id(att['id'])
|
||||
deadline = time.monotonic() + 30
|
||||
while f'job {job_id}: no-op, already done' not in logs(containers('worker')[0]):
|
||||
check(time.monotonic() < deadline, 'redelivered message was not acknowledged as no-op')
|
||||
time.sleep(1)
|
||||
j = job(att['id'])
|
||||
check(j['attempt'] == 1 and j['variants'] == 3, f'work was repeated: {j}')
|
||||
deadline = time.monotonic() + 20
|
||||
while queue_counts().get('attachment.process.v1', 0) != 0:
|
||||
check(time.monotonic() < deadline, 'message left in queue')
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def test_expired_lease_takeover(c: Client):
|
||||
"""Зависшая processing-задача (аренда истекла, владельца нет) перехватывается новой попыткой."""
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
compose('stop', 'worker')
|
||||
try:
|
||||
att = upload(c, alice, chat)
|
||||
wait_published(att['id'])
|
||||
job_id = att_job_id(att['id'])
|
||||
sql(f"UPDATE attachment_jobs SET status='processing', attempt=1, worker_id='ghost-dead', "
|
||||
f"lease_until=clock_timestamp() - interval '1 second' WHERE job_id='{job_id}'")
|
||||
sql(f"UPDATE attachments SET status='processing' WHERE id='{att['id']}'")
|
||||
check(meta(c, bob, att['id'])['status'] == 'processing', 'processing state not visible in API')
|
||||
finally:
|
||||
start_worker()
|
||||
wait_status(c, bob, att['id'], {'ready'}, 60)
|
||||
j = job(att['id'])
|
||||
check(j['attempt'] == 2 and j['variants'] == 3, f'takeover result: {j}')
|
||||
|
||||
|
||||
def test_staging_cleanup_after_commit(c: Client):
|
||||
alice, _bob, _eve, chat = c.bootstrap()
|
||||
att = upload(c, alice, chat)
|
||||
wait_status(c, alice, att['id'], {'ready'})
|
||||
out = in_container('api', S3_PRELUDE + f"print(exists('staging/{att['id']}'), exists('originals/{att['id']}'))\n")
|
||||
check(out == 'False True', f'staging/originals after ready: {out}')
|
||||
|
||||
|
||||
def test_poison_message_to_dlq(c: Client):
|
||||
before = queue_counts().get('attachment.process.v1.dlq', 0)
|
||||
in_container('api', '''
|
||||
import asyncio, aio_pika
|
||||
async def main():
|
||||
url = open('/run/secrets/broker_url').read().strip()
|
||||
conn = await aio_pika.connect(url)
|
||||
ch = await conn.channel(publisher_confirms=True)
|
||||
ex = await ch.get_exchange('attachments')
|
||||
await ex.publish(aio_pika.Message(b'{"not":"a job"}', delivery_mode=2), routing_key='attachment.process.v1')
|
||||
await conn.close()
|
||||
asyncio.run(main())
|
||||
''')
|
||||
deadline = time.monotonic() + 15
|
||||
while queue_counts().get('attachment.process.v1.dlq', 0) != before + 1:
|
||||
check(time.monotonic() < deadline, 'invalid message did not reach DLQ')
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def test_prefetch_limited(c: Client):
|
||||
out = compose('exec', '-T', 'rabbitmq', 'rabbitmqctl', '-q', 'list_consumers', 'queue_name', 'prefetch_count')
|
||||
rows = [l.split() for l in out.splitlines() if l.startswith('attachment.process.v1')]
|
||||
expected = os.environ.get('WORKER_PREFETCH_EXPECTED', '2')
|
||||
check(rows and all(r[1] == expected for r in rows), f'consumers/prefetch: {rows}')
|
||||
|
||||
|
||||
def test_worker_resource_limits(c: Client):
|
||||
"""Лимиты worker: бомба памяти -> resource_limit (permanent), долгая обработка -> timeout (transient)."""
|
||||
out = in_container('worker', '''
|
||||
import asyncio, io, struct, zlib
|
||||
from PIL import Image
|
||||
from app import imaging
|
||||
def ch(k, d): return struct.pack('>I', len(d)) + k + d + struct.pack('>I', zlib.crc32(k + d) & 0xffffffff)
|
||||
async def main():
|
||||
w = h = 6000 # 36 Мпикс: проходит лимит пикселей, но RGB-декодирование ~108 MiB
|
||||
raw = zlib.compress(b''.join(b'\\0' + b'\\0' * (3 * w) for _ in range(h)), 1)
|
||||
big = b'\\x89PNG\\r\\n\\x1a\\n' + ch(b'IHDR', struct.pack('>IIBBBBB', w, h, 8, 2, 0, 0, 0)) + ch(b'IDAT', raw) + ch(b'IEND', b'')
|
||||
try:
|
||||
await imaging.run_isolated(big, 40_000_000, 96, 60); print('mem: no error')
|
||||
except imaging.PermanentImageError as e:
|
||||
print('mem:', e.code)
|
||||
try:
|
||||
await imaging.run_isolated(big, 40_000_000, 1024, 0.2); print('time: no error')
|
||||
except imaging.TransientImageError as e:
|
||||
print('time:', e.code)
|
||||
rgba = Image.new('RGBA', (300, 200), (255, 0, 0, 0)); b = io.BytesIO(); rgba.save(b, 'PNG')
|
||||
v = await imaging.run_isolated(b.getvalue(), 40_000_000, 512, 30)
|
||||
print('alpha:', Image.open(io.BytesIO(v[0]['data'])).convert('RGB').getpixel((10, 10)))
|
||||
im = Image.new('RGB', (300, 100)); ex = im.getexif(); ex[0x0112] = 6; b = io.BytesIO(); im.save(b, 'JPEG', exif=ex)
|
||||
v = await imaging.run_isolated(b.getvalue(), 40_000_000, 512, 30)
|
||||
t = Image.open(io.BytesIO(v[0]['data']))
|
||||
print('exif:', t.size, t.getexif().get(0x0112))
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
''')
|
||||
print(' ' + out.replace('\n', '; '))
|
||||
check('mem: resource_limit' in out, 'memory limit not enforced')
|
||||
check('time: processing_timeout' in out, 'timeout not enforced')
|
||||
alpha = out.split('alpha: (')[1].split(')')[0].split(', ')
|
||||
check(all(int(x) >= 250 for x in alpha), 'alpha not composited on white')
|
||||
check('exif: (85, 256) None' in out, 'EXIF orientation not applied or metadata not stripped')
|
||||
|
||||
|
||||
TESTS = [test_variants_contract, test_worker_down_demo, test_failed_state_and_409, test_duplicate_delivery_is_noop,
|
||||
test_retry_bounded_then_replay, test_crash_between_commit_and_publish,
|
||||
test_worker_crash_after_s3_before_commit, test_worker_crash_after_commit_before_ack,
|
||||
test_expired_lease_takeover, test_staging_cleanup_after_commit, test_poison_message_to_dlq,
|
||||
test_prefetch_limited, test_worker_resource_limits]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
c = Client(BASE_URL, timeout=30)
|
||||
wait_ready_api(c)
|
||||
only = set(sys.argv[1:])
|
||||
failed = 0
|
||||
selected = [t for t in TESTS if not only or t.__name__ in only]
|
||||
for test in selected:
|
||||
started = time.monotonic()
|
||||
try:
|
||||
test(c)
|
||||
print(f'PASS {test.__name__} ({time.monotonic() - started:.1f}s)', flush=True)
|
||||
except (Failure, subprocess.CalledProcessError, OSError) as exc:
|
||||
failed += 1
|
||||
detail = getattr(exc, 'stderr', '') or ''
|
||||
print(f'FAIL {test.__name__}: {exc} {detail.strip()[-300:]}', flush=True)
|
||||
print(f'{len(selected) - failed}/{len(selected)} passed')
|
||||
return 1 if failed else 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Executable
+64
@@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Сводка трека 1: evidence/track1/*.json -> markdown-таблицы (stdout и evidence/track1/report.md).
|
||||
|
||||
Запуск (Python 3.9+, без зависимостей): python student-tests/track1_report.py [каталог, по умолчанию evidence/track1]
|
||||
Повторы одной конфигурации показываются как среднее и диапазон [min..max], чтобы был виден разброс.
|
||||
"""
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DIR = Path(sys.argv[1]) if len(sys.argv) > 1 else ROOT / 'evidence/track1'
|
||||
|
||||
|
||||
def rng(values, fmt='{:.2f}'):
|
||||
values = [v for v in values if v is not None]
|
||||
if not values:
|
||||
return '—'
|
||||
mean = sum(values) / len(values)
|
||||
if len(values) == 1 or max(values) == min(values):
|
||||
return fmt.format(mean)
|
||||
return f"{fmt.format(mean)} [{fmt.format(min(values))}..{fmt.format(max(values))}]"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
runs = [json.loads(p.read_text(encoding='utf-8')) for p in sorted(DIR.glob('*.json'))]
|
||||
matrix = defaultdict(list)
|
||||
for r in runs:
|
||||
m = re.fullmatch(r'c(\d+)-p(\d+)-(small|large)-r\d+', r['label'])
|
||||
if m:
|
||||
matrix[(int(m[1]), int(m[2]), m[3])].append(r)
|
||||
lines = ['## Матрица concurrency/prefetch', '',
|
||||
'| Размер | concurrency / prefetch | Повторов | Принято / ready / failed / потеряно | Throughput, ready/s | '
|
||||
'p50 до ready, с | p95 до ready, с | p99 до ready, с | Макс. незавершённых | Пик памяти worker, MiB | Разгрузка, с |',
|
||||
'| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |']
|
||||
for (c, p, size), rs in sorted(matrix.items(), key=lambda kv: (kv[0][2] != 'small', kv[0][0], kv[0][1])):
|
||||
lines.append(
|
||||
f"| {size} | {c} / {p} | {len(rs)} | "
|
||||
f"{sum(r['accepted'] for r in rs)} / {sum(r['ready'] for r in rs)} / {sum(r['failed'] for r in rs)} / "
|
||||
f"{sum(r['lost_accepted'] for r in rs)} | "
|
||||
f"{rng([r['throughput_ready_per_s'] for r in rs])} | {rng([r['time_to_ready_s']['p50'] for r in rs])} | "
|
||||
f"{rng([r['time_to_ready_s']['p95'] for r in rs])} | {rng([r['time_to_ready_s']['p99'] for r in rs])} | "
|
||||
f"{rng([r['queue_lag']['max_pending_jobs'] for r in rs], '{:.0f}')} | "
|
||||
f"{rng([r['worker_memory_mib']['peak'] for r in rs], '{:.0f}')} | {rng([r['drain_seconds'] for r in rs], '{:.1f}')} |")
|
||||
lines += ['', '## Backpressure и восстановление', '',
|
||||
'| Прогон | MAX_PENDING_JOBS | Поток | Ответы upload | Ready / failed / потеряно | Макс. незавершённых | '
|
||||
'p95 до ready, с | Throughput, ready/s | Пик памяти, MiB |',
|
||||
'| --- | --- | --- | --- | --- | --- | --- | --- | --- |']
|
||||
for r in runs:
|
||||
if r['label'].startswith(('baseline', 'backpressure', 'recovery')):
|
||||
lines.append(
|
||||
f"| {r['label']} | {r['max_pending_jobs']} | {r['count']} x {r['size']} @ {r['rate_per_s']}/s | "
|
||||
f"{', '.join(f'{k}: {v}' for k, v in sorted(r['upload_status'].items()))} | "
|
||||
f"{r['ready']} / {r['failed']} / {r['lost_accepted']} | {r['queue_lag']['max_pending_jobs']} | "
|
||||
f"{r['time_to_ready_s']['p95']} | {r['throughput_ready_per_s']} | {r['worker_memory_mib']['peak']} |")
|
||||
text = '\n'.join(lines) + '\n'
|
||||
(DIR / 'report.md').write_text(text, encoding='utf-8')
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
# Служебный образ для запуска make check / make test / hook-скриптов из Linux на любой ОС хоста.
|
||||
FROM python:3.12.12-alpine3.22
|
||||
RUN apk add --no-cache make curl docker-cli docker-cli-compose
|
||||
WORKDIR /work
|
||||
Reference in New Issue
Block a user