From 0a221b0675b237ba842369f6b26362ff5b123d9c Mon Sep 17 00:00:00 2001 From: ZapcMihail Date: Tue, 17 Nov 2026 18:33:52 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=9B=D0=A05=20=D0=91=D0=92=D0=A22304=20?= =?UTF-8?q?=D0=97=D0=B0=D0=BF=D0=BE=D1=80=D0=BE=D0=B6=D0=B5=D1=86=20=D0=BD?= =?UTF-8?q?=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RabbitMQ (quorum queue, publisher confirms, manual ack, DLX) + отдельный worker: staging в S3 + вложение/job/outbox одной транзакцией -> 202 queued; relay outbox, lease с heartbeat и fencing (worker_id, attempt), retry с backoff+jitter, failed-реестр, безопасный replay с audit; thumbnail/crop/WebP в изолированном процессе с лимитами. Тесты окон отказа (fault injection), трек 1 «Очередь под перегрузкой» с backpressure. Перенесено исправление замечания lab3 PR #99 (U+0000 в тексте). Co-Authored-By: Claude Opus 5.5 --- .env.example | 45 +- .gitattributes | 6 + .gitignore | 1 + Dockerfile | 31 + REPORT.md | 186 +++++- app/__init__.py | 0 app/__main__.py | 48 ++ app/broker.py | 39 ++ app/config.py | 116 ++++ app/db.py | 65 ++ app/errors.py | 48 ++ app/faults.py | 28 + app/images.py | 156 +++++ app/imaging.py | 184 ++++++ app/jobs.py | 95 +++ app/lifecycle.py | 16 + app/main.py | 588 ++++++++++++++++++ app/migrate.py | 60 ++ app/outbox.py | 149 +++++ app/pgtext.py | 22 + app/replay.py | 98 +++ app/s3_bootstrap.py | 72 +++ app/security.py | 47 ++ app/sessions.py | 187 ++++++ app/storage.py | 120 ++++ app/validation.py | 138 ++++ app/worker.py | 307 +++++++++ compose.yaml | 334 ++++++++++ evidence/failover-hook.txt | 17 + evidence/persistence.txt | 25 + evidence/public-tests.txt | 7 + evidence/stand.txt | 35 ++ evidence/student-tests.txt | 57 ++ evidence/track1/backpressure-large.json | 97 +++ evidence/track1/baseline-small.json | 104 ++++ evidence/track1/c1-p1-large-r1.json | 188 ++++++ evidence/track1/c1-p1-large-r2.json | 180 ++++++ evidence/track1/c1-p1-small-r1.json | 180 ++++++ evidence/track1/c1-p1-small-r2.json | 204 ++++++ evidence/track1/c2-p16-large-r1.json | 120 ++++ evidence/track1/c2-p16-large-r2.json | 120 ++++ evidence/track1/c2-p16-small-r1.json | 120 ++++ evidence/track1/c2-p16-small-r2.json | 120 ++++ evidence/track1/c2-p2-large-r1.json | 120 ++++ evidence/track1/c2-p2-large-r2.json | 124 ++++ evidence/track1/c2-p2-small-r1.json | 148 +++++ evidence/track1/c2-p2-small-r2.json | 152 +++++ evidence/track1/c4-p4-large-r1.json | 104 ++++ evidence/track1/c4-p4-large-r2.json | 104 ++++ evidence/track1/c4-p4-small-r1.json | 104 ++++ evidence/track1/c4-p4-small-r2.json | 104 ++++ .../iter1-before-fix/backpressure-large.json | 149 +++++ .../iter1-before-fix/c1-p1-large-r1.json | 244 ++++++++ .../iter1-before-fix/c1-p1-large-r2.json | 232 +++++++ .../iter1-before-fix/c1-p1-small-r1.json | 328 ++++++++++ .../iter1-before-fix/c1-p1-small-r2.json | 260 ++++++++ .../iter1-before-fix/c2-p16-large-r1.json | 204 ++++++ .../iter1-before-fix/c2-p16-large-r2.json | 204 ++++++ .../iter1-before-fix/c2-p16-small-r1.json | 236 +++++++ .../iter1-before-fix/c2-p16-small-r2.json | 244 ++++++++ .../iter1-before-fix/c2-p2-large-r1.json | 192 ++++++ .../iter1-before-fix/c2-p2-large-r2.json | 204 ++++++ .../iter1-before-fix/c2-p2-small-r1.json | 244 ++++++++ .../iter1-before-fix/c2-p2-small-r2.json | 244 ++++++++ .../iter1-before-fix/c4-p4-large-r1.json | 192 ++++++ .../iter1-before-fix/c4-p4-large-r2.json | 188 ++++++ .../iter1-before-fix/c4-p4-small-r1.json | 236 +++++++ .../iter1-before-fix/c4-p4-small-r2.json | 244 ++++++++ .../iter1-before-fix/recovery-small.json | 137 ++++ evidence/track1/iter1-before-fix/report.md | 19 + evidence/track1/iter1-before-fix/summary.txt | 19 + evidence/track1/recovery-small.json | 104 ++++ evidence/track1/report.md | 20 + evidence/track1/summary.txt | 20 + infra/haproxy/haproxy.cfg | 89 +++ migrations/0001_init.sql | 59 ++ migrations/0002_idempotency.sql | 21 + migrations/0003_cache_state.sql | 10 + migrations/0004_attachments.sql | 50 ++ migrations/0005_nul_safe_text.sql | 7 + migrations/0006_jobs.sql | 81 +++ requirements.txt | 38 ++ scripts/collect-evidence.sh | 51 ++ scripts/init-secrets.sh | 49 ++ scripts/restart.sh | 14 + scripts/restore.sh | 13 + scripts/rolling-update.sh | 36 ++ scripts/stop-one.sh | 29 + scripts/test-lab5.sh | 12 + scripts/track1.sh | 40 ++ student-tests/failover_series.py | 312 ++++++++++ student-tests/fixtures/sample.jpg | Bin 0 -> 1992 bytes student-tests/overload.py | 207 ++++++ student-tests/show_instances.py | 15 + student-tests/test_lab2_regression.py | 256 ++++++++ student-tests/test_lab3.py | 308 +++++++++ student-tests/test_lab4.py | 284 +++++++++ student-tests/test_lab5.py | 515 +++++++++++++++ student-tests/track1_report.py | 64 ++ tools/tester.Dockerfile | 4 + 100 files changed, 12431 insertions(+), 17 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/__main__.py create mode 100644 app/broker.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/errors.py create mode 100644 app/faults.py create mode 100644 app/images.py create mode 100644 app/imaging.py create mode 100644 app/jobs.py create mode 100644 app/lifecycle.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/outbox.py create mode 100644 app/pgtext.py create mode 100644 app/replay.py create mode 100644 app/s3_bootstrap.py create mode 100644 app/security.py create mode 100644 app/sessions.py create mode 100644 app/storage.py create mode 100644 app/validation.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 evidence/failover-hook.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/public-tests.txt create mode 100644 evidence/stand.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/track1/backpressure-large.json create mode 100644 evidence/track1/baseline-small.json create mode 100644 evidence/track1/c1-p1-large-r1.json create mode 100644 evidence/track1/c1-p1-large-r2.json create mode 100644 evidence/track1/c1-p1-small-r1.json create mode 100644 evidence/track1/c1-p1-small-r2.json create mode 100644 evidence/track1/c2-p16-large-r1.json create mode 100644 evidence/track1/c2-p16-large-r2.json create mode 100644 evidence/track1/c2-p16-small-r1.json create mode 100644 evidence/track1/c2-p16-small-r2.json create mode 100644 evidence/track1/c2-p2-large-r1.json create mode 100644 evidence/track1/c2-p2-large-r2.json create mode 100644 evidence/track1/c2-p2-small-r1.json create mode 100644 evidence/track1/c2-p2-small-r2.json create mode 100644 evidence/track1/c4-p4-large-r1.json create mode 100644 evidence/track1/c4-p4-large-r2.json create mode 100644 evidence/track1/c4-p4-small-r1.json create mode 100644 evidence/track1/c4-p4-small-r2.json create mode 100644 evidence/track1/iter1-before-fix/backpressure-large.json create mode 100644 evidence/track1/iter1-before-fix/c1-p1-large-r1.json create mode 100644 evidence/track1/iter1-before-fix/c1-p1-large-r2.json create mode 100644 evidence/track1/iter1-before-fix/c1-p1-small-r1.json create mode 100644 evidence/track1/iter1-before-fix/c1-p1-small-r2.json create mode 100644 evidence/track1/iter1-before-fix/c2-p16-large-r1.json create mode 100644 evidence/track1/iter1-before-fix/c2-p16-large-r2.json create mode 100644 evidence/track1/iter1-before-fix/c2-p16-small-r1.json create mode 100644 evidence/track1/iter1-before-fix/c2-p16-small-r2.json create mode 100644 evidence/track1/iter1-before-fix/c2-p2-large-r1.json create mode 100644 evidence/track1/iter1-before-fix/c2-p2-large-r2.json create mode 100644 evidence/track1/iter1-before-fix/c2-p2-small-r1.json create mode 100644 evidence/track1/iter1-before-fix/c2-p2-small-r2.json create mode 100644 evidence/track1/iter1-before-fix/c4-p4-large-r1.json create mode 100644 evidence/track1/iter1-before-fix/c4-p4-large-r2.json create mode 100644 evidence/track1/iter1-before-fix/c4-p4-small-r1.json create mode 100644 evidence/track1/iter1-before-fix/c4-p4-small-r2.json create mode 100644 evidence/track1/iter1-before-fix/recovery-small.json create mode 100644 evidence/track1/iter1-before-fix/report.md create mode 100644 evidence/track1/iter1-before-fix/summary.txt create mode 100644 evidence/track1/recovery-small.json create mode 100644 evidence/track1/report.md create mode 100644 evidence/track1/summary.txt create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_cache_state.sql create mode 100644 migrations/0004_attachments.sql create mode 100644 migrations/0005_nul_safe_text.sql create mode 100644 migrations/0006_jobs.sql create mode 100644 requirements.txt create mode 100755 scripts/collect-evidence.sh create mode 100755 scripts/init-secrets.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/restore.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/test-lab5.sh create mode 100755 scripts/track1.sh create mode 100755 student-tests/failover_series.py create mode 100644 student-tests/fixtures/sample.jpg create mode 100755 student-tests/overload.py create mode 100755 student-tests/show_instances.py create mode 100755 student-tests/test_lab2_regression.py create mode 100755 student-tests/test_lab3.py create mode 100755 student-tests/test_lab4.py create mode 100755 student-tests/test_lab5.py create mode 100755 student-tests/track1_report.py create mode 100644 tools/tester.Dockerfile diff --git a/.env.example b/.env.example index 119384b5..767dbc46 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,9 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# postgres - sessions only in PostgreSQL +# postgres+valkey - PostgreSQL is the source of truth, Valkey is a shared read-through session cache with TTL +SESSION_STORE=postgres+valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 @@ -23,3 +25,44 @@ WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 STAGING_RETENTION_SECONDS=86400 +# --- solution-specific (lab3) --- +# Max lifetime of a cached session copy (also the cache-distrust window after a Valkey error) +SESSION_CACHE_TTL_SECONDS=60 +# On SIGTERM a replica keeps serving but reports not-ready this long, so the LB drains it +SHUTDOWN_DRAIN_MS=3000 +# Bounded pool and timeouts, per replica +DB_POOL_MAX=10 +DB_STATEMENT_TIMEOUT_MS=5000 +DB_ACQUIRE_TIMEOUT_MS=2000 +CACHE_TIMEOUT_MS=300 +# Parallel Argon2id computations per replica (each uses 64 MiB) +PASSWORD_HASH_WORKERS=2 +# --- solution-specific (lab4) --- +# Deadline of one S3 operation (connect 2 s, no retries); after it the API answers 503 +S3_TIMEOUT_MS=5000 +# Parallel image validations per replica (CPU/RAM bound) +IMAGE_CHECK_WORKERS=2 +# --- solution-specific (lab5) --- +# Unacked messages RabbitMQ hands to one worker (defaults to WORKER_CONCURRENCY) +WORKER_PREFETCH=2 +# Processing lease, extended by worker heartbeat every lease/3; expired lease -> job re-queued by reaper +JOB_LEASE_SECONDS=15 +# Retry backoff: min(RETRY_MAX_MS, RETRY_BASE_MS * 2^(attempt-1)), equal jitter +RETRY_BASE_MS=1000 +RETRY_MAX_MS=60000 +# Backpressure: uploads get 503 + Retry-After when queued+processing jobs reach this number +MAX_PENDING_JOBS=200 +# Outbox relay poll interval (relay is also woken right after each upload COMMIT) +OUTBOX_POLL_MS=500 +# Address-space limit of one image-processing child process +WORKER_TASK_MEMORY_MB=640 +WORKER_REPLICAS=1 +WORKER_CPUS=2.0 +WORKER_MEMORY=1536M +# Per-replica limits +API_CPUS=1.0 +API_MEMORY=384M +PG_CPUS=2.0 +# Host ports bound to 127.0.0.1: LB entrypoint and LB stats page +LB_PUBLISHED_PORT=8080 +LB_STATS_PORT=8404 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..7558e95a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Shell-скрипты выполняются в Linux-контейнерах: всегда LF, даже при checkout на Windows. +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +Dockerfile text eol=lf +*.Dockerfile text eol=lf diff --git a/.gitignore b/.gitignore index b5a71896..708ae4c3 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +student-tests/.generated/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..27bf1ddf --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +# API и worker мессенджера lab5: один образ, разные команды (python -m app | python -m app.worker). +FROM python:3.12.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /srv + +# Сначала зависимости (кэшируемый слой), потом код. +COPY requirements.txt . +RUN pip install --no-deps -r requirements.txt + +COPY app ./app +COPY migrations ./migrations +COPY contracts/jobs.schema.json ./contracts/jobs.schema.json + +RUN useradd --system --uid 10001 --no-create-home appuser +USER appuser + +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + +# Метка сборки (последний слой, кэш не ломает): rolling update показывает, что реплики заменены новым образом. +ARG BUILD_ID=dev +LABEL org.opencontainers.image.revision=$BUILD_ID + +# exec-форма: python — PID 1 и сам получает SIGTERM от docker stop (drain -> graceful shutdown). +CMD ["python", "-m", "app"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..7d312524 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,191 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Запорожец Михаил, БВТ2304 +- Целевая оценка: **5**, трек **1 «Очередь под перегрузкой»** +- Commit/PR: TODO после публикации (ветка `solution/bvt2304-zaporozhets`) +- Использованные источники, библиотеки и помощь ИИ: RabbitMQ docs (Reliability Guide, Consumer Acknowledgements and Publisher Confirms, Quorum Queues, Dead Lettering, Consumer Prefetch), паттерн Transactional Outbox, AWS Architecture Blog «Exponential Backoff And Jitter», Python `multiprocessing` (forkserver, preload), Pillow (`draft`, `ImageOps.exif_transpose`, WebP), PostgreSQL (`FOR UPDATE SKIP LOCKED`, advisory locks). Библиотеки: FastAPI 0.115, uvicorn 0.34, asyncpg 0.30, argon2-cffi 23.1, redis-py 5.2, minio-py 7.2.18, Pillow 11.3.0, **aio-pika 9.5.7**, **jsonschema 4.25.1**. Код, конфигурации и тесты написаны с помощью ИИ-ассистента (Claude); решения разобраны и проверены автором. + +## Исправления по замечаниям преподавателя (перенесены из lab3/lab4) + +| Замечание | Исправление | Проверка | +| --- | --- | --- | +| PR lab3 #99: `{"text":"abc\u0000def"}` возвращает 500 | PostgreSQL `text` не хранит 0x00. Миграция `0005_nul_safe_text.sql` (expand) добавляет `*_raw bytea` для text, display_name, title: при наличии U+0000 там хранятся точные UTF-8 байты, в основном столбце — строка с U+FFFD (длина та же, CHECK работают); API отдаёт исходную строку (`app/pgtext.py`). Исправлено и запушено также в PR lab3 #99 и lab4 #115 | `test_nul_in_text_roundtrip` (регрессия lab3): 201, точный текст в ответе, истории и идемпотентном повторе, 409 на другой текст с тем же ключом (`evidence/student-tests.txt`) | ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Нужно:** Docker Engine/Desktop с Compose v2.20+ (проверено: Docker 29.8.2, Compose 5.6.0, Windows 11 + Docker Desktop/WSL2, VM 8 vCPU / 4 GiB, Intel i5-1135G7) и POSIX-shell (Git Bash). Python/make на хосте не нужны: проверки идут в сервисе `tester`. + +| Роль | Сервис | Образ | Health/readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| Балансировщик | `lb` | `haproxy:3.2.8-alpine` | `/stats`; API по `/health/ready` | — | edge; **единственная** публикация `127.0.0.1:8080` (+8404 stats) | — | +| API × N | `api` (`API_REPLICAS=2`) | сборка `Dockerfile` | `/health/ready` (критична только PostgreSQL) | нет; rootfs read-only | edge + data | secrets `database_url`, `cache_url`, `s3_*`, `broker_url` | +| **Worker** | `worker` (`WORKER_REPLICAS=1`) | тот же образ, `python -m app.worker` | пульс `/tmp/worker-alive` моложе 20 с | нет; rootfs read-only | только data | `database_url`, `s3_*`, `broker_url` | +| **Брокер** | `rabbitmq` | `rabbitmq:4.3.6-alpine@sha256:6931816a…f5` | `rabbitmq-diagnostics check_port_connectivity` | **`rabbitdata`** | data (internal); порты не публикуются, management UI не включён | `secrets/rabbitmq.conf` → `/etc/rabbitmq/conf.d/90-credentials.conf` (отдельный пользователь, не guest) | +| S3 | `s3` | `rustfs/rustfs:1.0.4@sha256:ed9210d6…dcc2` | `/health` | `s3data` | data | `RUSTFS_*_FILE` | +| PostgreSQL | `postgres` | `postgres:17.7-bookworm` | `pg_isready` | `pgdata` | data | `POSTGRES_PASSWORD_FILE` | +| Кэш сессий | `valkey` | `valkey/valkey:8.1.4` | ping | — | data | `valkey_conf` | +| Bootstrap | `migrate`, `s3-init` (one-shot) | образ API | exit 0 | — | data | как у API | +| Тесты | `tester` (profile tools), `api-ttl` (profile test) | python:3.12.12-alpine3.22 / образ API | — | bind `.` | edge | docker.sock | + +```sh +git clone && cd +sh scripts/init-secrets.sh # ./secrets/* (БД, Valkey, S3, RabbitMQ) и .env; значения не печатает +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +# postgres, s3, rabbitmq, valkey -> migrate (схема, 0006_jobs) -> s3-init (bucket) -> api x2, worker -> lb + +docker compose --env-file .env --profile tools run --rm tester make check +docker compose --env-file .env --profile tools run --rm tester make test +docker compose --env-file .env --profile tools run --rm tester make test-failover ACTION_SCRIPT=scripts/stop-one.sh +docker compose --env-file .env --profile tools run --rm tester sh scripts/restore.sh +docker compose --env-file .env --profile tools run --rm tester make test-persistence ACTION_SCRIPT=scripts/restart.sh + +sh scripts/test-lab5.sh # собственные тесты lab5 (уровни 3–4) + регрессия lab4/lab3/lab2 +sh scripts/collect-evidence.sh # доказательства evidence/*.txt +sh scripts/track1.sh # трек 1 (~25 мин): evidence/track1/*.json +python student-tests/track1_report.py # сводные таблицы -> evidence/track1/report.md + +# Оператор: реестр failed и безопасный replay +docker compose --env-file .env exec worker python -m app.replay list +docker compose --env-file .env exec worker python -m app.replay replay --attachment --reason "..." [--dry-run] +docker compose --env-file .env exec worker python -m app.replay replay --all-failed --reason "..." --dry-run +``` + +- **Bootstrap:** `migrate` применяет `migrations/*.sql` (новая — `0006_jobs.sql`, только expand). Топологию RabbitMQ (exchange, quorum queue, DLX/DLQ) идемпотентно объявляют и API (relay), и worker при подключении. Bucket создаёт `s3-init`, как в lab4. +- **Действий оператора**, кроме `init-secrets.sh`, нет. Если на 8080 работает стенд lab3/lab4 — остановить (`docker compose -p mtusi-lab4 stop`). +- **Безопасная остановка:** `docker compose --env-file .env stop` (или `down`) — volumes сохраняются, принятые задания остаются в PostgreSQL/RabbitMQ. +- **Удаление учебных данных:** `docker compose --env-file .env down -v` (volumes `mtusi-lab5_pgdata`, `_s3data`, `_rabbitdata`); секреты — `secrets/`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент ─HTTP─> lb ─> api x2 ──(1) PUT staging/ ─────────────────────────────────> S3 (RustFS) + │ (2) BEGIN; INSERT attachment(queued) + job + outbox; COMMIT ▲ + │ (3) 202 queued │ + ▼ │ + PostgreSQL ◄── relay (в каждой API): outbox FOR UPDATE SKIP LOCKED │ + attachments, ── publish persistent + publisher confirm ──> RabbitMQ │ + attachment_jobs, quorum queue attachment.process.v1 │ + attachment_variants, (DLX -> attachment.process.v1.dlq) │ + outbox, job_audit │ prefetch=N, manual ack │ + ▲ ▼ │ + └──── claim / lease+heartbeat / COMMIT ── worker ── forkserver-процессы ──┘ + (Pillow, RLIMIT_AS, timeout) + originals/, variants//v1/* +``` + +**Перенесено из lab4 без изменений:** HAProxy, реплики API, Bearer-сессии, идемпотентность сообщений, приватный bucket, синхронная валидация upload (10 MiB / 40 Мпикс до декодирования, MIME по содержимому, filename), авторизованный download без redirect. **Изменено по объявленному переходу:** upload → `202 queued`, готовность — через `GET /attachments/{id}`; `variant` = original/thumbnail/crop/optimized; content для queued/processing/failed — 409; failed-вложение в сообщение — 409, queued/processing — можно. + +**Что именно сохранено в момент 202.** (1) Все байты проверенного оригинала — в устойчивом staging (приватный prefix `staging/` S3, а не память/диск API). (2) Одной транзакцией PostgreSQL — метаданные вложения (`queued`, size, sha256), логическое задание `attachment_jobs` и **намерение публикации** (строка `outbox`). Публикация в брокер к этому моменту могла ещё не произойти — это допустимо: после COMMIT работу восстановит relay. Граница честная: staging уже в S3, worker переносит оригинал server-side copy `staging/ → originals/`. + +**Transactional outbox.** Разрыв «COMMIT — publish» закрыт тем, что публикация — отдельный повторяемый шаг над строкой outbox. Relay (фоновая задача каждой реплики API) берёт строки `published_at IS NULL AND available_at <= now()` через `FOR UPDATE SKIP LOCKED`, публикует persistent-сообщение с `mandatory`, ждёт **publisher confirm** и только потом ставит `published_at`. Падение между confirm и UPDATE даёт повторную публикацию — допустимый дубль (at-least-once). `available_at` в будущем используется для retry с backoff без плагинов брокера. Сразу после COMMIT upload будит relay (`kick`), поэтому задержка до брокера — миллисекунды, а не период опроса. + +**Publisher confirm и consumer ack — разные границы.** Confirm — брокер принял и записал сообщение (producer→broker; после него можно отметить outbox). Ack — consumer устойчиво обработал сообщение (broker→worker); worker шлёт ack только после COMMIT результата или после того, как убедился, что работа уже сделана/не его. Без ack сообщение вернётся при обрыве канала. + +**Брокер.** RabbitMQ 4.3, quorum queue (Raft-журнал на диске, volume `rabbitdata`), сообщения persistent. `x-delivery-limit=20` + DLX: poison message, который валит worker до ack, или сообщение, не прошедшее `contracts/jobs.schema.json`, уходит в `attachment.process.v1.dlq`. В сообщении только `schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at` — без bytes, URL, ключей S3 и credentials; worker берёт доверенные метаданные (staging_key, size, sha256) из БД. + +**Ключ идемпотентности job** — `job_id` (одна логическая работа на `(attachment_id, pipeline_version)`, UNIQUE); не меняется при redelivery, retry и replay. **Ключ эффекта** — `(attachment_id, pipeline_version, variant)`: PK `attachment_variants` и детерминированные S3-ключи `variants//v1/.`. Повтор после сбоя перезаписывает те же объекты тем же содержимым и вставляет варианты `ON CONFLICT DO NOTHING` — второго набора нет. Повторная доставка уже `done` job — no-op + ack. `ready` никогда не откатывается: все переходы назад ограничены `WHERE status IN ('queued','processing')`. **Почему повторная доставка нормальна:** at-least-once неизбежно даёт дубли (обрыв до ack, повтор publish после confirm), поэтому корректность обеспечивает идемпотентный эффект, а не «exactly-once доставка». + +**Захват, аренда, fencing — как отличить зависший worker от долгого и не запустить два неконтролируемых дубля.** Claim — транзакция `SELECT … FOR UPDATE` строки job: queued (или processing с **истёкшей** арендой) → processing, `attempt+1`, `worker_id`, `lease_until = now + JOB_LEASE_SECONDS (15)`. Пока worker жив, **heartbeat** каждые lease/3 продлевает аренду: долгая обработка аренду не теряет, а мёртвый или зависший процесс перестаёт её продлевать. Истёкшую аренду находит reaper в relay (раз в ~1 с, `pg_try_advisory_xact_lock` — одна реплика) и возвращает задание в queued с новой строкой outbox (или failed, если попытки исчерпаны). Дубль доставки при живой аренде — no-op. Чтобы «воскресший» старый исполнитель не перезаписал результат, фиксация результата/ошибки фенсится токеном `(worker_id, attempt)`: `UPDATE … WHERE status='processing' AND worker_id=$me AND attempt=$n` — при перехваченной аренде COMMIT не проходит. Отдельно — жёсткий лимит одной попытки `JOB_TIMEOUT_SECONDS=60` (дочерний процесс убивается). + +**Retry.** **Постоянные** ошибки (`invalid_image`, `image_too_large`, `resource_limit`, `source_missing`, `source_corrupted`, `processing_failed`) — сразу `failed`, без повторов. **Временные** (S3/БД недоступны, `processing_timeout`) — повтор с задержкой `min(RETRY_MAX_MS, RETRY_BASE_MS·2^(attempt−1))` с equal jitter (половина фиксирована, половина случайна), не более `JOB_MAX_ATTEMPTS=5`, затем `failed` / `retry_exhausted`. Реестр failed — `attachment_jobs` (`status='failed'`, `error_code`, `last_error`, время) + DLQ для непарсящихся сообщений. **Replay** (`python -m app.replay`) — только для `failed` (строка под `FOR UPDATE`), есть `--dry-run`, запись в `job_audit` (кто, почему, прежняя ошибка), тот же `job_id`; для `done` — отказ: «повторить» не исполняет дубль успешной работы. + +**Варианты** (`app/imaging.py`). thumbnail — JPEG q85, вписан в 256×256 без увеличения, размеры целочисленно вниз, min 1 px (320×200 → 256×160); crop — JPEG q85, центральный квадрат → 128×128 (маленькие увеличиваются); optimized — WebP q80 method 4, вписан в 1280×1280 без увеличения (маленький файл может не уменьшиться — это не обещается). EXIF orientation применяется (`exif_transpose`), alpha компонуется на белый, EXIF/ICC/XMP не записываются. **Ограничения worker:** bytes (size и sha256 сверяются с БД), пиксели (по заголовку до декодирования, ≤ 40 Мпикс), JPEG декодируется в draft-масштабе ≥ 1280, обработка — в отдельном процессе (`forkserver`) с `RLIMIT_AS = WORKER_TASK_MEMORY_MB` (640 MiB) и таймаутом `JOB_TIMEOUT_SECONDS` (kill), конкуренция `WORKER_CONCURRENCY` (семафор) и `prefetch_count = WORKER_PREFETCH`. + +**Staging cleanup.** Staging удаляется worker-ом сразу после COMMIT результата (до ack). Если удаление не удалось — retention sweeper в worker (раз в 5 мин, advisory lock) удаляет `staging/*` старше `STAGING_RETENTION_SECONDS` (24 ч), **по которым нет queued/processing задания**. Staging failed-вложений хранится до retention, чтобы был возможен replay. + +**Backpressure.** Перед чтением тела upload API считает незавершённые задания (`queued + processing`, частичный индекс). При `>= MAX_PENDING_JOBS` — `503 unavailable` + `Retry-After: 5`, байты не принимаются, в S3/БД ничего не пишется. Счётчик без блокировки: при одновременных upload порог можно превысить на число параллельных запросов (измерено: 22 при пороге 20). + +**Готовность:** `/health/ready` — только PostgreSQL. Брокер и worker не критичны для приёма: при недоступном RabbitMQ API продолжает отвечать 202 (намерение в outbox), при остановленном worker — тоже. Resize синхронно в API не выполняется никогда. + +**Соответствие `.env.example`:** `BROKER_URL_FILE` → secret `broker_url`; `WORKER_CONCURRENCY` → семафор worker; `JOB_MAX_ATTEMPTS` → `attachment_jobs.max_attempts`; `JOB_TIMEOUT_SECONDS` → таймаут дочернего процесса; `STAGING_RETENTION_SECONDS` → sweeper. Добавлены решением: `WORKER_PREFETCH`, `JOB_LEASE_SECONDS`, `RETRY_BASE_MS`, `RETRY_MAX_MS`, `MAX_PENDING_JOBS`, `OUTBOX_POLL_MS`, `WORKER_TASK_MEMORY_MB`, `WORKER_REPLICAS/CPUS/MEMORY`. ## Доказательства критериев +Текстовые доказательства собраны `sh scripts/collect-evidence.sh` 17.11.2026; трек 1 — `sh scripts/track1.sh`. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `make check`, `make validate`, `make test` | `PASS: lab5 public API scenario` (202, ready, оригинал, настоящий JPEG 256×160) | `evidence/public-tests.txt` | +| Минимум lab4 сохранён (201→202) | регрессия `test_lab4.py`, `make test-failover` ×2, `make test-persistence` | 9/9; failover без потерь; после рестарта всего стенда оригинал скачивается | `evidence/student-tests.txt`, `failover-hook.txt`, `persistence.txt` | +| 202 только после устойчивого приёма; остановка API не теряет работу | `test_worker_down_demo`, `test_crash_between_commit_and_publish` | API перезапущен после 202 → ready; API убит сразу после COMMIT (клиент видит обрыв) → вложение всё равно ready через outbox | `evidence/student-tests.txt` | +| Отдельные broker и worker, durable, at-least-once, ack после эффекта | `evidence/stand.txt` (quorum, durable, consumer `ack_required`), `test_worker_crash_after_commit_before_ack` | очередь `quorum true`, consumer с prefetch 2; падение после COMMIT до ack → redelivery → `no-op, already done` | `evidence/stand.txt`, `student-tests.txt` | +| Worker недоступен — API не делает resize | `test_worker_down_demo` | upload 202 и запись `queued` при остановленном worker; content 409 | `evidence/student-tests.txt` | +| Состояния queued/processing/ready/failed в API | `test_worker_down_demo`, `test_expired_lease_takeover`, `test_failed_state_and_409` | все 4 состояния видны в `GET /attachments/{id}` | там же | +| Оригинал неизменен, thumbnail по контракту | `test_variants_contract` | оригинал побайтно; thumbnail JPEG 256×160 (реальные размеры), 160×100 без увеличения для маленького | там же | +| Ограниченный retry, безопасный error_code, нет второго набора | `test_retry_bounded_then_replay`, `test_duplicate_delivery_is_noop` | S3 недоступен → 5 попыток, задержки 0.6 / 1.5 / 2.9 / 7.7 с (экспонента с jitter) → `failed retry_exhausted`; 3 дубля сообщения → `attempt=1`, 3 варианта | там же | +| Демо: worker стоп → 202 → message → рестарт API → рестарт broker → worker → ready | `test_worker_down_demo` | глубина очереди до/после рестарта RabbitMQ: 1/1; после старта worker ready и thumbnail | там же | +| **4:** crop 128×128, optimized WebP | `test_variants_contract` | crop JPEG 128×128, optimized WebP 320×200 (реальные размеры из файла) | там же | +| **4:** лимиты worker (bytes/пиксели/время/память) | `test_worker_resource_limits` | 6000×6000 при RLIMIT 96 MiB → `resource_limit`; таймаут 0.2 с → `processing_timeout`; alpha → белый; EXIF 6 → 85×256, EXIF удалён | там же | +| **4:** outbox, crash между COMMIT и publish | `test_crash_between_commit_and_publish` | `faults: api_crash_after_commit` → процесс API завершён до publish → ready без ручного создания задания | там же | +| **4:** backoff + jitter, постоянные/временные, DLQ/реестр, безопасный replay | `test_retry_bounded_then_replay`, `test_failed_state_and_409`, `test_poison_message_to_dlq` | sha-mismatch → `failed source_corrupted` с 1 попыткой; replay `--dry-run` ничего не меняет, replay → ready, audit 1 строка, повторный replay `done` отклонён; невалидное сообщение → DLQ | там же | +| **4:** остановка worker после записи объекта, до ack | `test_worker_crash_after_s3_before_commit` | `worker_crash_after_s3` → аренда истекла → попытка 2 → ready, в S3 ровно 3 объекта варианта | там же | +| **4:** lease зависшего processing, prefetch, очистка staging | `test_expired_lease_takeover`, `test_prefetch_limited`, `test_staging_cleanup_after_commit` | processing с истёкшей арендой перехвачен (attempt 2); `prefetch_count=2`; после ready `staging/` нет, `originals/` есть | там же | +| **5, трек 1** | `scripts/track1.sh`, `student-tests/track1_report.py` | см. раздел ниже | `evidence/track1/` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Отказы (все — автоматические тесты, `evidence/student-tests.txt`) -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| worker остановлен, upload, рестарт API и RabbitMQ | задание не теряется | queue depth 1 → 1 после рестарта брокера | ready через ~3 с после старта worker; потерь нет | +| API убит после COMMIT, до publish | outbox допубликует | клиент получил обрыв соединения; запись в БД есть | ready через ~1–3 с (relay другой реплики); потерь нет | +| worker убит после записи S3, до COMMIT | повтор тем же результатом | attempt 2, 3 варианта, 3 объекта в S3 | ~18 с (аренда 15 с + повтор); дублей нет | +| worker убит после COMMIT, до ack | redelivery — no-op | `no-op, already done`, attempt 1 | сразу после рестарта; дублей нет | +| S3 недоступен для worker | ограниченные повторы | 5 попыток за ~14 с → `retry_exhausted` | replay после возврата S3 → ready | +| staging подменён (sha256 не совпал) | постоянная ошибка без повторов | `failed source_corrupted`, attempt 1 | content/attach → 409 | +| poison message | не блокирует очередь | уходит в DLQ | — | + +### Трек 1 «Очередь под перегрузкой» + +**Гипотеза.** Пропускная способность worker растёт с `WORKER_CONCURRENCY` почти линейно, пока не упрётся в CPU-лимит контейнера (2 vCPU), а большой prefetch при той же concurrency не ускоряет обработку, но ухудшает справедливость и увеличивает объём «невидимой» для других consumers работы. Без ограничения принятой работы время до ready при перегрузке растёт неограниченно (вместе с очередью); с `MAX_PENDING_JOBS` оно ограничено ценой 503 для части клиентов. + +**Методика** (`student-tests/overload.py`, драйвер `scripts/track1.sh`): +- Стенд: Docker Desktop/WSL2, VM 8 vCPU / 4 GiB, Intel i5-1135G7; worker — 1 реплика, лимит 2 CPU / 1536 MiB; API ×2 по 1 CPU. +- Изображения: **small** — `tests/fixtures/sample.png` (320×200, 112 KB); **large** — синтетический JPEG 4000×3000 (12 Мпикс, 2.9 MB, генерируется скриптом, в Git не хранится). +- **Open-loop** поступление (расписание отправки не зависит от ответов — нет coordinated omission генератора; опоздание отправки фиксируется, максимум в прогонах < 0.2 с). Поток заведомо выше пропускной способности: small — 200 файлов по 40/с, large — 60 файлов по 8/с. +- Конфигурации worker `concurrency/prefetch`: 1/1, 2/2, 4/4, 2/16; каждая — 2 повтора на каждом размере (16 прогонов). +- Метрики: **время до ready** — по серверным меткам PostgreSQL (`created_at` COMMIT приёма → `updated_at` перехода в ready), p50/p95/p99; **throughput** — ready / (последний ready − первый приём); **queue lag** — число незавершённых заданий (queued+processing, раз в 0.5 с) и глубина очереди RabbitMQ; **память** — `docker stats` контейнера worker (процесс + дочерние процессы обработки); время полной разгрузки. +- Backpressure: `MAX_PENDING_JOBS=20`, 120 больших по 15/с; затем **восстановление** — 60 маленьких по 5/с (ниже пропускной способности), сравнение с таким же прогоном **до** перегрузки. + +**Итерация 1 — эксперимент нашёл узкое место** (`evidence/track1/iter1-before-fix/report.md`). Throughput был ≈ 1 задание/с **при любой concurrency и для обоих размеров**: small 0.78–1.14/s, large 0.74–1.01/s, хотя 320×200 обрабатывается в ~100 раз быстрее 12 Мпикс. Значит, время уходит не на изображение. Поэтапные замеры worker (`fetch/build/store/commit/cleanup` в логе) показали `build` 0.7–2.3 с при 0.07 с полезной работы: каждый дочерний процесс `forkserver` заново импортировал главный модуль `app.worker` со всеми зависимостями (при запуске `python -m` multiprocessing восстанавливает `__main__` по имени, а preload `'__main__'` при `-m` не срабатывает). **Исправление:** `set_forkserver_preload(['app.worker', 'app.imaging'])` — зависимости импортируются один раз в forkserver. Задание small: **1.8 с → 0.15 с**. + +**Итерация 2 — после исправления** (`evidence/track1/report.md`, среднее по 2 повторам, в скобках min..max): + +| Размер | conc / prefetch | Принято / ready / потеряно | Throughput, ready/s | p50 до ready, с | p95 до ready, с | Макс. незавершённых | Пик памяти worker, MiB | Разгрузка, с | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| small | 1 / 1 | 400 / 400 / 0 | 4.50 [3.75..5.26] | 19.7 | 38.2 [30.8..45.6] | 179 | 120 | 46.4 | +| small | 2 / 2 | 400 / 400 / 0 | 5.69 [5.29..6.08] | 18.0 | 29.4 [27.0..31.8] | 170 | 133 | 36.2 | +| small | 2 / 16 | 400 / 400 / 0 | 9.81 [9.60..10.02] | 7.8 | 14.7 [14.3..15.2] | 156 | 133 | 21.0 | +| small | 4 / 4 | 400 / 400 / 0 | 12.85 [12.52..13.17] | 5.7 | 10.2 [9.9..10.6] | 134 | 142 | 16.1 | +| large | 1 / 1 | 120 / 120 / 0 | 1.26 [1.13..1.39] | 19.4 | 39.7 [34.8..44.5] | 50 | 179 | 49.4 | +| large | 2 / 2 | 120 / 120 / 0 | 2.43 [2.25..2.62] | 10.1 | 17.3 [15.3..19.3] | 46 | 246 | 25.6 | +| large | 2 / 16 | 120 / 120 / 0 | 3.23 [2.98..3.48] | 7.1 | 11.1 [9.7..12.5] | 40 | 222 | 19.7 | +| large | 4 / 4 | 120 / 120 / 0 | 3.81 [3.73..3.88] | 5.0 | 8.4 [8.1..8.8] | 33 | 294 | 16.4 | + +Для сравнения итерация 1 (до исправления): large 4/4 — 1.01 ready/s, p95 30.0 с; small 4/4 — 1.13 ready/s, p95 46.8 с. + +| Прогон | MAX_PENDING_JOBS | Поток | Ответы upload | Ready / failed / потеряно | Макс. незавершённых | p50 / p95 до ready, с | +| --- | --- | --- | --- | --- | --- | --- | +| baseline-small (до перегрузки) | 200 | 60 small @ 5/s | 202: 60 | 60 / 0 / 0 | 1 | 0.115 / 1.171 | +| backpressure-large | 20 | 120 large @ 15/s | **202: 48, 503: 72** | 48 / 0 / 0 | **22** | 4.94 / 5.65 | +| recovery-small (сразу после) | 20 | 60 small @ 5/s | 202: 60 | 60 / 0 / 0 | 1 | 0.113 / 0.145 | + +**Результаты и выводы.** +1. **Concurrency.** С 1 до 4 throughput large вырос в 3.0 раза (1.26 → 3.81/s), p95 до ready упал с 39.7 до 8.4 с; small — в 2.9 раза. Рост сублинейный: с 2 до 4 large прибавил лишь 57 % — это насыщение лимита 2 CPU worker (обработка 12 Мпикс CPU-bound; I/O S3/БД перекрывается, поэтому 4 > 2 при 2 CPU). Цена — память: пик worker 179 → 294 MiB (каждый параллельный дочерний процесс держит декодированное изображение); при 4 и `WORKER_TASK_MEMORY_MB=640` наихудший случай (40 Мпикс PNG) упирается в лимит контейнера 1536 MiB — поэтому по умолчанию оставлено 2. +2. **Prefetch — гипотеза опровергнута частично.** При той же concurrency 2 prefetch 16 дал +33 % (large) и +72 % (small) throughput относительно prefetch 2. Объяснение (подтверждено характером эффекта, не профилированием): при prefetch = concurrency слот после ack простаивает на время доставки следующего сообщения и его проверки, а при коротких заданиях (small ~0.15 с) эта пауза — заметная доля цикла; запас сообщений внутри worker её скрывает. Цена большого prefetch: до 16 сообщений «заперты» в одном worker (вторая реплика их не получит — хуже справедливость и масштабирование), при падении worker все они возвращаются в очередь разом, а очередь брокера перестаёт отражать реальный lag (сообщения unacked). Компромисс для учебного стенда: prefetch ≈ 2–4 × concurrency; оставлено по умолчанию = concurrency, параметр вынесен в `WORKER_PREFETCH`. +3. **Queue lag и latency.** Без ограничения принятой работы при потоке выше пропускной способности число незавершённых заданий росло до 134–179 (small) и 33–50 (large), а p95 до ready — пропорционально очереди (до 40 с). Время ожидания = очередь / throughput — классический компромисс latency/ресурсы: больше concurrency — быстрее разгрузка, но больше RAM. +4. **Backpressure.** С `MAX_PENDING_JOBS=20` система приняла 48 из 120 больших файлов, остальным ответила `503` + `Retry-After` **до чтения тела** (upload p50 7 мс); незавершённых заданий — не больше 22 (перебор на 2 из-за неблокирующего счётчика, см. ограничения). p95 до ready принятых — 5.6 с против 17.3 с без ограничения при меньшем потоке (8/с, конфигурация 2/2): принятая работа выполняется с ограниченной задержкой, лишняя отклоняется явно, а не копится. +5. **Восстановление после перегрузки.** Сразу после backpressure-прогона лёгкая нагрузка обслужена так же, как до перегрузки: p50 0.113 с против 0.115 с, очередь ≤ 1, все 60 приняты. (p95 базового прогона 1.17 с — холодный старт forkserver только что пересозданного worker в первых заданиях; в recovery forkserver уже «тёплый».) Во всех 19 прогонах итерации 2 — 0 потерянных и 0 failed среди принятых. + +**Ограничения измерения.** Один хост: генератор, API, worker, S3 и БД делят 8 vCPU VM, поэтому абсолютные значения не переносимы на другую машину — сравнимы только конфигурации между собой. 2 повтора дают видимый разброс (до ±20 % по p95 у c=1), выводы опираются на различия больше разброса. Память — `docker stats` контейнера (RSS + page cache), снимок раз в ~1 с, короткие пики дочерних процессов могли не попасть в выборку. Throughput считается по серверным меткам и не включает время опроса клиента. Large-изображение одно (синтетическое, шум + градиент); реальные фото дают другое соотношение decode/encode. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Backpressure-счётчик неблокирующий** — при N одновременных upload порог превышается до N (измерено 22/20). Строгое ограничение — семафор в БД/Valkey или `SELECT … FOR UPDATE` счётчика; не сделано, чтобы не сериализовать все upload. +- **Prefetch по умолчанию = concurrency** — по результатам трека больший prefetch быстрее на коротких заданиях; выбор оставлен консервативным (справедливость между репликами worker, честный lag в брокере). +- **Takeover только после истечения аренды** (до 15 с задержки после смерти worker в середине задания) — цена простого и безопасного fencing без отдельного реестра живых worker. +- **Один узел RabbitMQ и RustFS** — quorum queue на одном узле защищает от рестарта, но не от потери диска; HA брокера/S3 вне рамок работы. +- **Staging failed-вложений** удаляется через 24 ч — replay после этого вернёт `source_missing`. +- Один worker-контейнер; горизонтальное масштабирование (`WORKER_REPLICAS`) поддержано конфигурацией, но в треке не измерялось. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..1f057ca2 --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,48 @@ +"""Запуск: python -m app (HTTP-процесс — PID 1 в контейнере). + +SIGTERM (docker stop) обрабатывается в два этапа: +1. drain: SHUTDOWN_DRAIN_MS процесс продолжает обслуживать запросы, но /health/ready отвечает 503 — + балансировщик за это время исключает реплику по активной проверке и перестаёт слать новые запросы; +2. обычный graceful shutdown uvicorn: новые соединения не принимаются, начатые запросы + дообрабатываются до timeout_graceful_shutdown. +""" +import logging +import os +import threading + +import uvicorn + +from . import lifecycle + +log = logging.getLogger('messenger') + + +class DrainingServer(uvicorn.Server): + drain_seconds = 0.0 + + def handle_exit(self, sig, frame): + if lifecycle.draining or self.drain_seconds <= 0: + # Повторный сигнал (или drain выключен) — сразу штатное завершение uvicorn. + return super().handle_exit(sig, frame) + lifecycle.start_draining() + log.info('signal %s: draining for %.1fs before shutdown, instance=%s', + sig, self.drain_seconds, lifecycle.INSTANCE_ID) + timer = threading.Timer(self.drain_seconds, super().handle_exit, (sig, frame)) + timer.daemon = True + timer.start() + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s %(message)s') + config = uvicorn.Config('app.main:app', host=os.environ.get('APP_HOST', '0.0.0.0'), + port=int(os.environ.get('APP_PORT', '8080')), proxy_headers=False, + server_header=False, timeout_graceful_shutdown=10, + timeout_keep_alive=30) # дольше, чем держит idle-соединение HAProxy + server = DrainingServer(config) + server.drain_seconds = int(os.environ.get('SHUTDOWN_DRAIN_MS', '3000')) / 1000 + log.info('starting instance %s', lifecycle.INSTANCE_ID) + server.run() + + +if __name__ == '__main__': + main() diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 00000000..da287311 --- /dev/null +++ b/app/broker.py @@ -0,0 +1,39 @@ +"""RabbitMQ: топология очередей заданий. Объявление идемпотентно — его выполняют и API (relay), и worker. + +attachments (direct, durable) --attachment.process.v1--> quorum queue attachment.process.v1 + отклонённые/непарсящиеся и превысившие x-delivery-limit сообщения + --> attachments.dlx --> quorum queue attachment.process.v1.dlq (просмотр оператором) + +Quorum queue: сообщения реплицируются в Raft-журнал на диске (volume rabbitdata) и переживают рестарт +брокера; persistent delivery_mode + publisher confirm у publisher; manual ack у consumer. +""" +import aio_pika +from aio_pika import ExchangeType + +EXCHANGE = 'attachments' +DLX = 'attachments.dlx' +QUEUE = 'attachment.process.v1' +DLQ = 'attachment.process.v1.dlq' +ROUTING_KEY = 'attachment.process.v1' +# Защита от poison message, который валит worker до ack: после N доставок — в DLQ. +# Состояние в БД при этом не теряется: reaper/sweeper вернёт работу по lease (см. app/outbox.py). +DELIVERY_LIMIT = 20 + + +async def connect(url: str) -> aio_pika.abc.AbstractRobustConnection: + return await aio_pika.connect_robust(url, timeout=5, client_properties={'connection_name': 'messenger'}) + + +async def declare(channel: aio_pika.abc.AbstractChannel): + dlx = await channel.declare_exchange(DLX, ExchangeType.DIRECT, durable=True) + dlq = await channel.declare_queue(DLQ, durable=True, arguments={'x-queue-type': 'quorum'}) + await dlq.bind(dlx, ROUTING_KEY) + exchange = await channel.declare_exchange(EXCHANGE, ExchangeType.DIRECT, durable=True) + queue = await channel.declare_queue(QUEUE, durable=True, arguments={ + 'x-queue-type': 'quorum', + 'x-dead-letter-exchange': DLX, + 'x-dead-letter-routing-key': ROUTING_KEY, + 'x-delivery-limit': DELIVERY_LIMIT, + }) + await queue.bind(exchange, ROUTING_KEY) + return exchange, queue diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..1b0ad4b4 --- /dev/null +++ b/app/config.py @@ -0,0 +1,116 @@ +"""Настройки из переменных окружения. Секреты читаются только из файлов (*_FILE).""" +import os +from dataclasses import dataclass +from pathlib import Path + + +def _int(name: str, default: int, low: int, high: int) -> int: + value = int(os.environ.get(name, default)) + if not low <= value <= high: + raise ValueError(f'{name} must be in [{low}, {high}]') + return value + + +def _secret(name: str) -> str | None: + """Значение из файла, путь к которому лежит в переменной NAME_FILE.""" + path = os.environ.get(name + '_FILE') + if not path: + return None + return Path(path).read_text(encoding='utf-8').strip() + + +def _required_secret(name: str) -> str: + value = _secret(name) + if not value: + raise RuntimeError(f'{name}_FILE is not configured') + return value + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + session_ttl: int + session_store: str + session_cache_ttl: int + database_url: str + cache_url: str | None + db_pool_max: int + db_statement_timeout_ms: int + db_acquire_timeout: float + cache_timeout: float + hash_workers: int + idempotency_ttl: int + drain_seconds: float + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key: str + s3_secret_key: str + s3_timeout: float + max_upload_bytes: int + max_image_pixels: int + image_workers: int + broker_url: str + worker_concurrency: int + worker_prefetch: int + job_max_attempts: int + job_timeout: int + job_lease: int + retry_base_ms: int + retry_max_ms: int + staging_retention: int + max_pending_jobs: int + outbox_poll_ms: int + worker_memory_mb: int + worker_max_pixels: int + + +def load() -> Settings: + database_url = _secret('DATABASE_URL') or os.environ.get('DATABASE_URL') + if not database_url: + raise RuntimeError('DATABASE_URL_FILE is not configured') + store = os.environ.get('SESSION_STORE', 'postgres') + if store not in ('postgres', 'postgres+valkey'): + raise ValueError('SESSION_STORE must be postgres or postgres+valkey') + cache_url = _secret('CACHE_URL') if store == 'postgres+valkey' else None + return Settings( + host=os.environ.get('APP_HOST', '0.0.0.0'), + port=_int('APP_PORT', 8080, 1, 65535), + session_ttl=_int('SESSION_TTL_SECONDS', 3600, 1, 30 * 24 * 3600), + session_store=store, + session_cache_ttl=_int('SESSION_CACHE_TTL_SECONDS', 60, 1, 3600), + database_url=database_url, + cache_url=cache_url, + db_pool_max=_int('DB_POOL_MAX', 10, 1, 100), + db_statement_timeout_ms=_int('DB_STATEMENT_TIMEOUT_MS', 5000, 100, 600000), + db_acquire_timeout=_int('DB_ACQUIRE_TIMEOUT_MS', 2000, 100, 60000) / 1000, + cache_timeout=_int('CACHE_TIMEOUT_MS', 300, 10, 10000) / 1000, + hash_workers=_int('PASSWORD_HASH_WORKERS', 4, 1, 32), + idempotency_ttl=_int('IDEMPOTENCY_TTL_SECONDS', 86400, 86400, 30 * 24 * 3600), # контракт: не менее 24 ч + drain_seconds=_int('SHUTDOWN_DRAIN_MS', 3000, 0, 60000) / 1000, + s3_endpoint=os.environ.get('S3_ENDPOINT', 'http://s3:9000'), + s3_region=os.environ.get('S3_REGION', 'us-east-1'), + s3_bucket=os.environ.get('S3_BUCKET', 'messenger-attachments'), + s3_access_key=_required_secret('S3_ACCESS_KEY_ID'), + s3_secret_key=_required_secret('S3_SECRET_ACCESS_KEY'), + s3_timeout=_int('S3_TIMEOUT_MS', 5000, 100, 12000) / 1000, # < timeout server HAProxy (15 с) + # Контракт: не более 10 MiB и 40 млн пикселей; значения можно только уменьшить. + max_upload_bytes=_int('MAX_UPLOAD_BYTES', 10485760, 1, 10485760), + max_image_pixels=_int('MAX_IMAGE_PIXELS', 40_000_000, 1, 40_000_000), + image_workers=_int('IMAGE_CHECK_WORKERS', 2, 1, 16), + # lab5: брокер, worker, повторы, backpressure + broker_url=_required_secret('BROKER_URL'), + worker_concurrency=_int('WORKER_CONCURRENCY', 2, 1, 32), + worker_prefetch=_int('WORKER_PREFETCH', int(os.environ.get('WORKER_CONCURRENCY', 2)), 1, 256), + job_max_attempts=_int('JOB_MAX_ATTEMPTS', 5, 1, 50), + job_timeout=_int('JOB_TIMEOUT_SECONDS', 60, 5, 3600), # жёсткий лимит одной попытки + job_lease=_int('JOB_LEASE_SECONDS', 15, 3, 600), # аренда, продлевается heartbeat + retry_base_ms=_int('RETRY_BASE_MS', 1000, 10, 600000), + retry_max_ms=_int('RETRY_MAX_MS', 60000, 10, 3600000), + staging_retention=_int('STAGING_RETENTION_SECONDS', 86400, 60, 30 * 86400), + max_pending_jobs=_int('MAX_PENDING_JOBS', 200, 1, 1000000), + outbox_poll_ms=_int('OUTBOX_POLL_MS', 500, 50, 60000), + worker_memory_mb=_int('WORKER_TASK_MEMORY_MB', 512, 64, 8192), + worker_max_pixels=_int('MAX_IMAGE_PIXELS', 40_000_000, 1, 40_000_000), + ) diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..dbcb5fa9 --- /dev/null +++ b/app/db.py @@ -0,0 +1,65 @@ +"""Ограниченный пул соединений asyncpg и перевод сбоев БД в 503.""" +import asyncio +from contextlib import asynccontextmanager + +import asyncpg + +from .config import Settings +from .errors import unavailable + +# Ошибки "БД недоступна/перегружена" -> 503, а не 500 и тем более не пропуск запроса. +DB_UNAVAILABLE = ( + OSError, + asyncio.TimeoutError, + asyncpg.PostgresConnectionError, + asyncpg.InterfaceError, + asyncpg.CannotConnectNowError, + asyncpg.TooManyConnectionsError, + asyncpg.QueryCanceledError, # statement_timeout + asyncpg.AdminShutdownError, +) + + +class Database: + def __init__(self, settings: Settings): + self.settings = settings + self.pool: asyncpg.Pool | None = None + + async def start(self) -> None: + s = self.settings + # min_size соединений создаётся лениво (connect при первом acquire), поэтому API стартует + # даже при недоступной БД и честно отвечает 503 в /health/ready. + self.pool = asyncpg.create_pool( + s.database_url, + min_size=0, + max_size=s.db_pool_max, + timeout=3, # таймаут установления соединения + command_timeout=s.db_statement_timeout_ms / 1000 + 1, + max_inactive_connection_lifetime=300, + server_settings={ + 'application_name': 'messenger-api', + 'statement_timeout': str(s.db_statement_timeout_ms), + 'idle_in_transaction_session_timeout': '10000', + 'timezone': 'UTC', + }, + ) + await self.pool + + async def close(self) -> None: + if self.pool is not None: + await self.pool.close() + + @asynccontextmanager + async def connection(self): + try: + async with self.pool.acquire(timeout=self.settings.db_acquire_timeout) as conn: + yield conn + except DB_UNAVAILABLE: + raise unavailable() from None + + async def ping(self) -> bool: + try: + async with self.pool.acquire(timeout=1) as conn: + return await conn.fetchval('SELECT 1', timeout=1) == 1 + except DB_UNAVAILABLE: + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..82a88acc --- /dev/null +++ b/app/errors.py @@ -0,0 +1,48 @@ +"""Единый формат ошибок {"error":{"code","message","request_id"}}.""" +from contextvars import ContextVar + +from fastapi.responses import JSONResponse + +request_id_var: ContextVar[str] = ContextVar('request_id', default='-') + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def invalid(message: str = 'Некорректный запрос') -> ApiError: + return ApiError(400, 'invalid_request', message) + + +def not_found() -> ApiError: + # Одинаковый ответ для "нет такого" и "чужое": существование чужого чата не раскрывается. + return ApiError(404, 'not_found', 'Объект не найден') + + +def unauthorized_bearer() -> ApiError: + return ApiError(401, 'unauthorized', 'Требуется действительная сессия', {'WWW-Authenticate': BEARER_CHALLENGE}) + + +def unauthorized_basic() -> ApiError: + return ApiError(401, 'unauthorized', 'Неверные учётные данные', {'WWW-Authenticate': BASIC_CHALLENGE}) + + +def unavailable() -> ApiError: + return ApiError(503, 'unavailable', 'Сервис временно недоступен') + + +def error_response(status: int, code: str, message: str, headers: dict | None = None) -> JSONResponse: + rid = request_id_var.get() + return JSONResponse( + status_code=status, + content={'error': {'code': code, 'message': message, 'request_id': rid}}, + headers=headers, + ) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 00000000..11b5ff41 --- /dev/null +++ b/app/faults.py @@ -0,0 +1,28 @@ +"""Точки внедрения отказов для тестов окон отказа (contracts/attachment-states.md). + +Точка активируется файлом /tmp/fault- внутри контейнера (tmpfs; создаётся тестом через +`docker exec -u 10001 touch /tmp/fault-`). Срабатывает один раз: файл удаляется, процесс +завершается os._exit — без finally, ack, закрытия соединений, как при kill -9. Docker перезапускает +контейнер (restart: unless-stopped), tmpfs при этом пустой. В обычной работе файлов нет — это no-op. + +Точки: + api_crash_after_commit API: после COMMIT вложения+job+outbox, до ответа 202 и до publish; + worker_crash_after_s3 worker: варианты и оригинал записаны в S3, COMMIT результата ещё нет; + worker_crash_after_commit worker: результат зафиксирован (ready), ack брокеру ещё не отправлен. +""" +import logging +import os + +log = logging.getLogger('messenger.faults') + + +def hit(name: str) -> None: + path = f'/tmp/fault-{name}' + if not os.path.exists(path): + return + try: + os.remove(path) + except OSError: + pass + log.error('FAULT INJECTION %s: hard exit', name) + os._exit(86) diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..5fb8bb51 --- /dev/null +++ b/app/images.py @@ -0,0 +1,156 @@ +"""Приём изображения: ограниченное чтение тела и проверка PNG/JPEG по содержимому. + +Порядок выбран так, чтобы ни один лимит не проверялся ПОСЛЕ дорогой операции: +1. Content-Length больше лимита -> 413 сразу, тело не читается; +2. чтение потоком с обрывом на MAX_UPLOAD_BYTES + 1 байте (chunked без длины тоже ограничен); +3. сигнатура (magic bytes) -> тип; несовпадение с Content-Type или чужой формат -> 415; +4. размеры из заголовка изображения (Pillow читает только header) -> > MAX_IMAGE_PIXELS -> 413 + ДО распаковки пикселей (защита от decompression bomb); +5. только потом проверка целостности: PNG verify() (CRC чанков), JPEG — декодирование в draft-режиме 1/8. +""" +import asyncio +import io +import warnings + +from fastapi import Request +from PIL import Image + +from .errors import ApiError + +# Лимит пикселей проверяем сами (п.4) с нужным кодом ответа; встроенный в Pillow оставляем как страховку. +Image.MAX_IMAGE_PIXELS = 40_000_000 +Image.LOAD_TRUNCATED_IMAGES = False + +SIGNATURES = { + 'image/png': b'\x89PNG\r\n\x1a\n', + 'image/jpeg': b'\xff\xd8\xff', +} +PIL_FORMATS = {'image/png': 'PNG', 'image/jpeg': 'JPEG'} + + +def too_large(message: str = 'Файл превышает лимит') -> ApiError: + return ApiError(413, 'payload_too_large', message) + + +def unsupported(message: str = 'Ожидается корректное изображение PNG или JPEG') -> ApiError: + return ApiError(415, 'unsupported_media_type', message) + + +def declared_type(request: Request) -> str: + media_type = request.headers.get('content-type', '').split(';')[0].strip().lower() + if media_type not in SIGNATURES: + raise unsupported('Content-Type должен быть image/png или image/jpeg') + return media_type + + +async def read_limited(request: Request, limit: int) -> bytes: + length = request.headers.get('content-length') + if length is not None and length.isdigit() and int(length) > limit: + raise too_large() + buf = bytearray() + async for chunk in request.stream(): + buf += chunk + if len(buf) > limit: + raise too_large() + return bytes(buf) + + +class ImageChecker: + def __init__(self, max_pixels: int, workers: int): + self.max_pixels = max_pixels + # Не больше N одновременных проверок на реплику: декодирование занимает CPU и память. + self.slots = asyncio.Semaphore(workers) + + async def inspect(self, data: bytes, declared: str) -> tuple[str, int, int]: + detected = next((t for t, sig in SIGNATURES.items() if data.startswith(sig)), None) + if detected is None or detected != declared: + raise unsupported() + async with self.slots: + return await asyncio.to_thread(self._inspect, data, detected) + + def _inspect(self, data: bytes, content_type: str) -> tuple[str, int, int]: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + try: + img = Image.open(io.BytesIO(data)) # ленивый: читает только заголовок + width, height = img.size + if img.format != PIL_FORMATS[content_type]: + raise unsupported() + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise too_large('Изображение превышает лимит пикселей') from None + except ApiError: + raise + except Exception: + raise unsupported() from None + if width <= 0 or height <= 0: + raise unsupported() + if width * height > self.max_pixels: + raise too_large('Изображение превышает лимит пикселей') + try: + if content_type == 'image/png': + img.verify() # CRC и структура чанков без хранения пикселей + check_png_pixel_stream(data) # объём пиксельных данных совпадает с IHDR + else: + img.draft('RGB', (max(1, width // 8), max(1, height // 8))) + img.load() # обрезанный/битый JPEG -> исключение + except Exception: + raise unsupported() from None + finally: + img.close() + return content_type, width, height + + +# ---------- PNG: потоковая проверка пиксельных данных ---------- +# verify() Pillow проверяет только CRC чанков. Файл с честным IHDR 6000x6000 и почти пустым IDAT он +# пропустит. Полное декодирование 40 Мпикс заняло бы 160 MiB на запрос, поэтому IDAT распаковывается +# потоково кусками по 64 KiB: считаются только байты, пиксели не хранятся. Распакованный объём должен +# точно совпасть с вычисленным по IHDR, иначе файл повреждён (или это zip-бомба сверх заявленного). + +CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) + + +def _row_bytes(width: int, bits_per_pixel: int) -> int: + return (width * bits_per_pixel + 7) // 8 + + +def expected_png_raw_size(width: int, height: int, bit_depth: int, color_type: int, interlace: int) -> int: + bpp = CHANNELS[color_type] * bit_depth + if not interlace: + return height * (1 + _row_bytes(width, bpp)) + total = 0 + for xs, ys, dx, dy in ADAM7: + pw = (width - xs + dx - 1) // dx if width > xs else 0 + ph = (height - ys + dy - 1) // dy if height > ys else 0 + if pw and ph: + total += ph * (1 + _row_bytes(pw, bpp)) + return total + + +def check_png_pixel_stream(data: bytes) -> None: + import struct + import zlib + pos, expected, produced = 8, None, 0 + inflater = zlib.decompressobj() + while pos + 8 <= len(data): + length, kind = struct.unpack('>I4s', data[pos:pos + 8]) + body = data[pos + 8:pos + 8 + length] + pos += 12 + length + if kind == b'IHDR': + width, height, bit_depth, color_type, _c, _f, interlace = struct.unpack('>IIBBBBB', body) + expected = expected_png_raw_size(width, height, bit_depth, color_type, interlace) + elif kind == b'IDAT': + if expected is None: + raise ValueError('IDAT before IHDR') + chunk = body + while chunk: + out = inflater.decompress(chunk, 64 * 1024) + produced += len(out) + if produced > expected: + raise ValueError('pixel data larger than IHDR') + chunk = inflater.unconsumed_tail + elif kind == b'IEND': + break + produced += len(inflater.flush()) + if expected is None or produced != expected or not inflater.eof: + raise ValueError('pixel data does not match IHDR') diff --git a/app/imaging.py b/app/imaging.py new file mode 100644 index 00000000..b0f680ec --- /dev/null +++ b/app/imaging.py @@ -0,0 +1,184 @@ +"""Построение вариантов изображения (выполняется worker-ом в отдельном дочернем процессе). + +Контракт (contracts/README.md, «Фоновая обработка»): +- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, EXIF orientation применяется, + alpha компонуется на белый, размеры округляются вниз, минимум 1 px (320x200 -> 256x160); +- crop: JPEG, центрированный квадрат 128x128 (маленький исходник увеличивается); +- optimized: WebP, вписан в 1280x1280 без увеличения. + +Параметры: JPEG quality=85 (thumbnail, crop), optimize=True, progressive=False; WebP quality=80, method=4. +Метаданные удаляются: EXIF/ICC/XMP не передаются в save() (Pillow не копирует их без явного exif=/icc_profile=), +ориентация EXIF уже применена к пикселям. Все варианты — 8-bit sRGB без alpha. + +Ограничения ресурсов: лимит пикселей проверяется по заголовку ДО декодирования; JPEG декодируется в draft- +режиме с уменьшением до ближайшего масштаба >= 1280 (меньше памяти и CPU); процесс запускается с RLIMIT_AS +(память) и убивается по таймауту (время) — см. run_isolated(). +""" +import asyncio +import hashlib +import io +import multiprocessing +import resource +import warnings + +from PIL import Image, ImageOps + +THUMB = 256 +CROP = 128 +OPTIMIZED = 1280 + + +class PermanentImageError(Exception): + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +class TransientImageError(Exception): + def __init__(self, code: str): + super().__init__(code) + self.code = code + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + """Вписать в box x box без увеличения; целочисленно, округление вниз, минимум 1 px.""" + m = max(width, height) + if m <= box: + return width, height + return max(1, width * box // m), max(1, height * box // m) + + +def flatten(img: Image.Image) -> Image.Image: + """Alpha (RGBA/LA/P с прозрачностью) компонуется на белый фон; результат RGB.""" + if img.mode in ('RGBA', 'LA') or (img.mode == 'P' and 'transparency' in img.info): + rgba = img.convert('RGBA') + background = Image.new('RGB', rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel('A')) + return background + if img.mode in ('I;16', 'I;16B', 'I;16L', 'I'): + img = img.point(lambda v: v / 256).convert('L') # 16-bit серый -> 8-bit + return img.convert('RGB') + + +def encode(name: str, img: Image.Image, fmt: str, content_type: str, **params) -> dict: + buf = io.BytesIO() + img.save(buf, fmt, **params) + data = buf.getvalue() + return {'name': name, 'content_type': content_type, 'width': img.width, 'height': img.height, + 'size_bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'data': data} + + +def build_variants(data: bytes, max_pixels: int) -> list[dict]: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + Image.MAX_IMAGE_PIXELS = max_pixels + try: + img = Image.open(io.BytesIO(data)) + width, height = img.size + except (Image.DecompressionBombError, Image.DecompressionBombWarning): + raise PermanentImageError('image_too_large') from None + except Exception: + raise PermanentImageError('invalid_image') from None + if img.format not in ('PNG', 'JPEG'): + raise PermanentImageError('invalid_image') + if width * height > max_pixels: + raise PermanentImageError('image_too_large') + try: + if img.format == 'JPEG': + img.draft('RGB', (OPTIMIZED, OPTIMIZED)) # декодер сам уменьшает в 2/4/8 раз, если можно + img.load() + img = ImageOps.exif_transpose(img) + rgb = flatten(img) + except MemoryError: + raise + except Exception: + raise PermanentImageError('invalid_image') from None + + # Размеры считаются от ориентированного ОРИГИНАЛА (не от draft-копии), иначе draft менял бы результат. + ow, oh = _oriented_size(data, width, height) + variants = [] + tw, th = fit(ow, oh, THUMB) + variants.append(encode('thumbnail', rgb.resize((tw, th), Image.LANCZOS) if (tw, th) != rgb.size else rgb, + 'JPEG', 'image/jpeg', quality=85, optimize=True)) + side = min(rgb.size) + left, top = (rgb.width - side) // 2, (rgb.height - side) // 2 + square = rgb.crop((left, top, left + side, top + side)) + variants.append(encode('crop', square.resize((CROP, CROP), Image.LANCZOS), 'JPEG', 'image/jpeg', + quality=85, optimize=True)) + pw, ph = fit(ow, oh, OPTIMIZED) + variants.append(encode('optimized', rgb.resize((pw, ph), Image.LANCZOS) if (pw, ph) != rgb.size else rgb, + 'WEBP', 'image/webp', quality=80, method=4)) + return variants + + +def _orientation(data: bytes) -> int: + try: + return Image.open(io.BytesIO(data)).getexif().get(0x0112, 1) + except Exception: + return 1 + + +def _rotated(data: bytes) -> bool: + return _orientation(data) in (5, 6, 7, 8) + + +def _oriented_size(data: bytes, width: int, height: int) -> tuple[int, int]: + """Размер оригинала после применения EXIF orientation (5–8 меняют местами ширину и высоту).""" + return (height, width) if _rotated(data) else (width, height) + + +# ---------- изоляция: отдельный процесс с лимитом памяти и времени ---------- + +def _child(conn, data: bytes, max_pixels: int, memory_mb: int) -> None: + limit = memory_mb * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (limit, limit)) + try: + conn.send(('ok', build_variants(data, max_pixels))) + except PermanentImageError as exc: + conn.send(('permanent', exc.code)) + except MemoryError: + conn.send(('permanent', 'resource_limit')) + except Exception: + conn.send(('permanent', 'processing_failed')) + finally: + conn.close() + + +_ctx = multiprocessing.get_context('forkserver') +# Preload app.worker: forkserver один раз импортирует главный модуль worker со всеми зависимостями +# (aio-pika, jsonschema, minio, asyncpg, Pillow), дочерние процессы получают их готовыми через fork. +# Без этого каждый запуск обработки заново импортировал app.worker (при `python -m` multiprocessing +# восстанавливает __main__ по имени модуля): 0.7–2.3 с накладных расходов на задание при 0.07 с полезной +# работы для 320x200 — найдено в эксперименте трека 1 (evidence/track1/, раздел «Итерация 1» в REPORT.md). +# '__main__' по пути здесь не работает: при запуске через -m пути к главному модулю нет. +_ctx.set_forkserver_preload(['app.worker', 'app.imaging']) + + +async def run_isolated(data: bytes, max_pixels: int, memory_mb: int, timeout: float) -> list[dict]: + """Варианты строятся в дочернем процессе: зависание/бомба памяти не роняют и не блокируют worker. + + Таймаут -> процесс убивается, TransientImageError('processing_timeout') (мог помешать перегруз CPU — + повтор с backoff; при исчерпании попыток — failed retry_exhausted). + RLIMIT_AS/смерть процесса -> PermanentImageError('resource_limit'). + """ + parent, child = _ctx.Pipe(duplex=False) + proc = _ctx.Process(target=_child, args=(child, data, max_pixels, memory_mb), daemon=True) + proc.start() + child.close() + loop = asyncio.get_running_loop() + try: + ready = await loop.run_in_executor(None, parent.poll, timeout) + if not ready: + raise TransientImageError('processing_timeout') + try: + status, value = await loop.run_in_executor(None, parent.recv) + except EOFError: + raise PermanentImageError('resource_limit') from None + finally: + if proc.is_alive(): + proc.kill() + await loop.run_in_executor(None, proc.join, 5) + parent.close() + if status == 'ok': + return value + raise PermanentImageError(value) diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 00000000..b27010b1 --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,95 @@ +"""Общие операции над заданиями обработки вложений (API, relay, worker, replay). + +Все переходы состояний выполняются SQL-транзакциями в PostgreSQL — это источник истины о работе. +Брокер только доставляет "напоминание" (job_id), поэтому потеря/дубль сообщения не теряет и не +дублирует эффект: worker всегда сверяется с attachment_jobs. +""" +import json +import random +import uuid +from datetime import datetime, timezone + +PIPELINE_VERSION = 1 +JOB_TYPE = 'attachment.process.v1' + + +def payload(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int) -> dict: + """Сообщение по contracts/jobs.schema.json: только ID и служебные поля — без bytes, URL и credentials.""" + return { + 'schema_version': 1, + 'job_id': str(job_id), + 'type': JOB_TYPE, + 'attachment_id': str(attachment_id), + 'pipeline_version': PIPELINE_VERSION, + 'attempt': attempt, + 'created_at': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ'), + } + + +def backoff_seconds(attempt: int, base_ms: int, max_ms: int) -> float: + """Экспоненциальный backoff с "equal jitter": половина задержки фиксирована, половина случайна. + + Фиксированная половина не даёт повтору прийти мгновенно (full jitter может дать ~0), + случайная — разводит повторы разных заданий после общего сбоя (нет "стада"). + """ + cap = min(max_ms, base_ms * 2 ** max(0, attempt - 1)) + return (cap / 2 + random.uniform(0, cap / 2)) / 1000 + + +async def enqueue(conn, job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int, delay_s: float = 0.0) -> None: + """Намерение опубликовать job — строка outbox в ТЕКУЩЕЙ транзакции вызывающего.""" + await conn.execute( + 'INSERT INTO outbox (job_id, payload, available_at) ' + 'VALUES ($1, $2::jsonb, clock_timestamp() + make_interval(secs => $3))', + job_id, json.dumps(payload(job_id, attachment_id, attempt)), delay_s) + + +async def create(conn, attachment_id: uuid.UUID, max_attempts: int) -> uuid.UUID: + job_id = uuid.uuid4() + await conn.execute( + 'INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, max_attempts) VALUES ($1, $2, $3, $4)', + job_id, attachment_id, PIPELINE_VERSION, max_attempts) + await enqueue(conn, job_id, attachment_id, attempt=1) + return job_id + + +async def requeue(conn, job_id, attachment_id, next_attempt: int, delay_s: float, reason: str, + worker_id: str | None = None, attempt: int | None = None) -> bool: + """processing -> queued: временная ошибка (worker) или истёкшая аренда (reaper). + + worker_id/attempt заданы — фиксация от имени владельца аренды (fencing token = worker_id + номер попытки): + чужую или уже перехваченную работу не трогаем. + """ + fence = 'AND worker_id = $4 AND attempt = $5' if worker_id else '' + args = [job_id, delay_s, reason[:200]] + ([worker_id, attempt] if worker_id else []) + updated = await conn.fetchval( + "UPDATE attachment_jobs SET status = 'queued', lease_until = NULL, worker_id = NULL, " + "next_attempt_at = clock_timestamp() + make_interval(secs => $2), last_error = $3, " + f"updated_at = clock_timestamp() WHERE job_id = $1 AND status = 'processing' {fence} RETURNING 1", + *args) + if not updated: + return False + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp() " + "WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id) + await enqueue(conn, job_id, attachment_id, next_attempt, delay_s) + return True + + +async def fail(conn, job_id, attachment_id, code: str, reason: str, worker_id: str | None = None, + attempt: int | None = None) -> bool: + """-> failed: постоянная ошибка или исчерпаны попытки. Код машинный и безопасный для клиента.""" + fence = "AND worker_id = $5 AND attempt = $6 AND status = 'processing'" if worker_id else '' + args = [job_id, code, reason[:200], attachment_id] + ([worker_id, attempt] if worker_id else []) + updated = await conn.fetchval( + "UPDATE attachment_jobs SET status = 'failed', lease_until = NULL, error_code = $2, last_error = $3, " + "finished_at = clock_timestamp(), updated_at = clock_timestamp() " + f"WHERE job_id = $1 AND attachment_id = $4 AND status IN ('queued', 'processing') {fence} RETURNING 1", + *args) + if not updated: + return False + # ready никогда не откатывается: условие по статусу. + await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = $2, updated_at = clock_timestamp() " + "WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id, code) + return True diff --git a/app/lifecycle.py b/app/lifecycle.py new file mode 100644 index 00000000..acb168e4 --- /dev/null +++ b/app/lifecycle.py @@ -0,0 +1,16 @@ +"""Состояние процесса реплики: идентификатор для X-Instance-Id и флаг drain при SIGTERM.""" +import os +import secrets +import socket + +# Стабилен в пределах жизни процесса, новый после перезапуска. Префикс — hostname контейнера +# (= короткий ID контейнера в Docker), по нему scripts/stop-one.sh находит именно эту реплику. +# В lab7 по контракту понадобится непрозрачный alias без hostname. +INSTANCE_ID = os.environ.get('INSTANCE_ID') or f'{socket.gethostname()}-{secrets.token_hex(3)}' + +draining = False + + +def start_draining() -> None: + global draining + draining = True diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..5e4210c8 --- /dev/null +++ b/app/main.py @@ -0,0 +1,588 @@ +"""HTTP API мессенджера lab4 по contracts/openapi.json (stateless-реплика за балансировщиком). + +lab4: вложения. Метаданные — PostgreSQL, байты — приватный S3-bucket (RustFS). Локальных файлов у API нет. +lab5: upload -> устойчивый staging в S3 + вложение/job/outbox одной транзакцией -> 202 queued; +обработку выполняет отдельный worker через RabbitMQ (app/worker.py), публикацию — relay outbox (app/outbox.py). +""" +import asyncio +import base64 +import binascii +import hashlib +import json +import logging +import re +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from urllib.parse import quote +from uuid import UUID + +import asyncpg +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import config, faults, images, jobs, lifecycle, pgtext, validation as v +from .db import DB_UNAVAILABLE, Database +from .errors import (ApiError, error_response, invalid, not_found, request_id_var, + unauthorized_basic, unauthorized_bearer, unavailable) +from .security import PasswordService, new_token +from .outbox import Relay +from .sessions import SessionStore +from .storage import Storage + +log = logging.getLogger('messenger') +settings = config.load() +db = Database(settings) +sessions = SessionStore(settings, db) +passwords = PasswordService(settings.hash_workers) +storage = Storage(settings) +image_checker = images.ImageChecker(settings.max_image_pixels, settings.image_workers) +relay = Relay(settings, db) + + +async def purge_loop() -> None: + while True: + await asyncio.sleep(3600) + try: + removed = await sessions.purge_expired() + keys = await purge_idempotency_keys() + log.info('purged %d expired sessions, %d idempotency keys', removed, keys) + except Exception: # очистка — фоновая гигиена, её сбой не должен ронять процесс + log.warning('purge failed', exc_info=True) + + +async def purge_idempotency_keys() -> int: + # Удаление по одному ключу индекса created_at; параллельный запуск на нескольких репликах безопасен. + async with db.connection() as conn: + status = await conn.execute( + 'DELETE FROM idempotency_keys WHERE created_at < clock_timestamp() - make_interval(secs => $1)', + settings.idempotency_ttl) + return int(status.split()[-1]) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + await db.start() + task = asyncio.create_task(purge_loop()) + relay_task = asyncio.create_task(relay.run()) + yield + task.cancel() + relay_task.cancel() + await relay.close() + await sessions.cache.close() + await db.close() + passwords.close() + + +app = FastAPI(lifespan=lifespan, openapi_url=None, docs_url=None, redoc_url=None) + +# ---------- request id, ошибки ---------- + +REQUEST_ID_RE = re.compile(r'[A-Za-z0-9._:-]{1,128}') + + +@app.middleware('http') +async def request_id_middleware(request: Request, call_next): + incoming = request.headers.get('x-request-id', '') + rid = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex + request_id_var.set(rid) + try: + response = await call_next(request) + except Exception: + log.exception('unhandled error, request_id=%s', rid) + response = error_response(500, 'internal_error', 'Внутренняя ошибка сервера') + response.headers['X-Request-Id'] = rid + response.headers['X-Instance-Id'] = lifecycle.INSTANCE_ID + if lifecycle.draining: + # Во время drain закрываем keep-alive: следующий запрос LB откроет соединение уже к живой реплике. + response.headers['Connection'] = 'close' + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(_request: Request, exc: ApiError): + return error_response(exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(StarletteHTTPException) +async def http_error_handler(_request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(404, 'not_found', 'Объект не найден') + if exc.status_code == 405: + return error_response(405, 'invalid_request', 'Метод не поддерживается', dict(exc.headers or {})) + return error_response(exc.status_code, 'invalid_request', 'Некорректный запрос') + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(_request: Request, _exc: RequestValidationError): + return error_response(400, 'invalid_request', 'Некорректный запрос') + + +async def db_unavailable_handler(_request: Request, _exc: Exception): + # Страховка: сбой связи с БД вне db.connection() тоже даёт ограниченный 503, а не 500. + return error_response(503, 'unavailable', 'Сервис временно недоступен') + + +for _exc_type in DB_UNAVAILABLE: + app.add_exception_handler(_exc_type, db_unavailable_handler) + +# ---------- представление ---------- + + +def ts(value: datetime) -> str: + # RFC 3339 UTC с микросекундами: точность совпадает с timestamptz и с ключом пагинации. + return value.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +def user_json(row) -> dict: + return {'id': str(row['id']), 'username': row['username'], + 'display_name': pgtext.join(row['display_name'], row['display_name_raw']), 'created_at': ts(row['created_at'])} + + +def chat_json(row) -> dict: + return {'id': str(row['id']), 'title': pgtext.join(row['title'], row['title_raw']), + 'member_ids': [str(m) for m in row['member_ids']], 'created_at': ts(row['created_at'])} + + +def message_json(row) -> dict: + return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'sender_id': str(row['sender_id']), + 'text': pgtext.join(row['text'], row['text_raw']), 'created_at': ts(row['created_at']), + 'attachment_ids': [str(a) for a in row['attachment_ids']]} + + +def attachment_json(row, variants=()) -> dict: + # object_key/staging_key и внутренние адреса S3 в ответ не попадают. + return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']), + 'filename': row['filename'], 'content_type': row['content_type'], + 'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'], + 'variants': [{'name': x['name'], 'content_type': x['content_type'], 'size_bytes': x['size_bytes'], + 'width': x['width'], 'height': x['height'], 'sha256': x['sha256']} for x in variants], + 'error_code': row['error_code'], + 'created_at': ts(row['created_at']), 'updated_at': ts(row['updated_at'])} + + +USER_COLUMNS = 'id, username, display_name, display_name_raw, created_at' +CHAT_COLUMNS = ('c.id, c.title, c.title_raw, c.created_at, ' + 'ARRAY(SELECT cm.user_id FROM chat_members cm WHERE cm.chat_id = c.id ORDER BY cm.user_id) AS member_ids') +MESSAGE_ATTACHMENTS = ('ARRAY(SELECT ma.attachment_id FROM message_attachments ma ' + 'WHERE ma.message_id = {m}.id ORDER BY ma.position) AS attachment_ids') +MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, text_raw, created_at, ' + MESSAGE_ATTACHMENTS.format(m='messages') +ATTACHMENT_COLUMNS = ('a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, ' + 'a.status, a.error_code, a.object_key, a.pipeline_version, a.created_at, a.updated_at') + +# ---------- аутентификация ---------- + +TOKEN_RE = re.compile(r'[A-Za-z0-9_-]{32,128}') + + +def bearer_token(request: Request) -> str: + """Только Authorization: Bearer. X-User-Id и Basic здесь игнорируются/отклоняются.""" + scheme, _, token = request.headers.get('authorization', '').partition(' ') + token = token.strip() + if scheme.lower() != 'bearer' or not TOKEN_RE.fullmatch(token): + raise unauthorized_bearer() + return token + + +async def current_user(request: Request) -> UUID: + session = await sessions.resolve(bearer_token(request)) + if session is None: + raise unauthorized_bearer() + return session.user_id + + +def basic_credentials(request: Request) -> tuple[str, str]: + scheme, _, encoded = request.headers.get('authorization', '').partition(' ') + if scheme.lower() != 'basic': + raise unauthorized_basic() + try: + decoded = base64.b64decode(encoded.strip(), validate=True) + username, sep, password = decoded.partition(b':') # двоеточие допустимо внутри пароля + if not sep: + raise ValueError + return username.decode('ascii'), password.decode('utf-8') + except (ValueError, binascii.Error, UnicodeDecodeError): + raise unauthorized_basic() from None + +# ---------- health ---------- + + +@app.get('/health/live') +async def live(): + return {'status': 'ok'} + + +@app.get('/health/ready') +async def ready(): + # Критична только PostgreSQL (источник истины). Valkey — кэш: без него API работает медленнее, но корректно. + # Во время drain (получен SIGTERM) реплика честно сообщает LB, что новых запросов ей давать не надо. + if lifecycle.draining or not await db.ping(): + raise unavailable() + return {'status': 'ok'} + +# ---------- пользователи и сессии ---------- + + +@app.post('/api/v1/users', status_code=201) +async def create_user(request: Request): + data = await v.read_json(request) + v.fields(data, {'username', 'display_name', 'password'}) + username = v.username(data) + display_name = v.text_field(data, 'display_name', 1, 100) + password = v.text_field(data, 'password', 12, 128) + password_hash = await passwords.hash(password) + try: + async with db.connection() as conn: + row = await conn.fetchrow( + 'INSERT INTO users (username, display_name, display_name_raw, password_hash) ' + f'VALUES ($1, $2, $3, $4) RETURNING {USER_COLUMNS}', + username, *pgtext.split(display_name), password_hash) + except asyncpg.UniqueViolationError: + # Гонка двух одинаковых регистраций решается UNIQUE-индексом: COMMIT пройдёт только у одной. + raise ApiError(409, 'conflict', 'username уже занят') from None + return JSONResponse(user_json(row), status_code=201) + + +@app.post('/api/v1/auth/sessions', status_code=201) +async def create_session(request: Request): + if await request.body(): + raise invalid('Тело запроса не допускается; используйте Authorization: Basic') + username, password = basic_credentials(request) + row = None + if v.USERNAME_RE.fullmatch(username) and 12 <= len(password) <= 128: + async with db.connection() as conn: + row = await conn.fetchrow( + f'SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = $1', username) + # verify выполняется всегда (и для несуществующего пользователя) — одинаковое время ответа. + if not await passwords.verify(row['password_hash'] if row else None, password): + raise unauthorized_basic() + token = new_token() + session = await sessions.create(row['id'], token) + return JSONResponse({'token': token, 'token_type': 'Bearer', 'expires_at': ts(session.expires_at), + 'user': user_json(row)}, status_code=201, headers={'Cache-Control': 'no-store'}) + + +@app.delete('/api/v1/auth/sessions/current', status_code=204) +async def delete_session(request: Request): + token = bearer_token(request) + if not await sessions.revoke(token): + raise unauthorized_bearer() + return Response(status_code=204) + + +@app.get('/api/v1/users/me') +async def me(request: Request): + user_id = await current_user(request) + async with db.connection() as conn: + row = await conn.fetchrow(f'SELECT {USER_COLUMNS} FROM users WHERE id = $1', user_id) + if row is None: + raise not_found() + return user_json(row) + +# ---------- чаты ---------- + + +@app.post('/api/v1/chats', status_code=201) +async def create_chat(request: Request): + user_id = await current_user(request) + data = await v.read_json(request) + v.fields(data, {'title', 'member_ids'}) + title = v.text_field(data, 'title', 1, 100) + raw_members = data['member_ids'] + if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100: + raise invalid('member_ids должен быть массивом из 1..100 UUID') + members = [v.uuid_value(m, 'member_id') for m in raw_members] + if len(set(members)) != len(members): + raise invalid('member_ids содержит дубликаты') + if user_id not in members: + raise invalid('Создатель должен входить в member_ids') + + async with db.connection() as conn: + # Одна транзакция: чат и все участники появляются вместе или не появляются вовсе. + async with conn.transaction(): + chat = await conn.fetchrow( + 'INSERT INTO chats (title, title_raw, created_by) VALUES ($1, $2, $3) RETURNING id, created_at', + *pgtext.split(title), user_id) + added = await conn.fetch( + 'INSERT INTO chat_members (chat_id, user_id) ' + 'SELECT $1, u.id FROM users u WHERE u.id = ANY($2::uuid[]) RETURNING user_id', + chat['id'], members) + if len(added) != len(members): + # Исключение внутри conn.transaction() -> ROLLBACK, строка chats тоже откатывается. + raise invalid('member_ids содержит неизвестных пользователей') + return JSONResponse({'id': str(chat['id']), 'title': title, + 'member_ids': sorted(str(m) for m in members), + 'created_at': ts(chat['created_at'])}, status_code=201) + + +@app.get('/api/v1/chats') +async def list_chats(request: Request): + user_id = await current_user(request) + limit = v.limit_param(request) + after = v.decode_cursor(request, 'chats') + query = (f'SELECT {CHAT_COLUMNS} FROM chats c ' + 'JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $1 ') + params = [user_id, limit + 1] + if after: + query += 'WHERE (c.created_at, c.id) > ($3, $4) ' + params += after + query += 'ORDER BY c.created_at, c.id LIMIT $2' + async with db.connection() as conn: + rows = await conn.fetch(query, *params) + return page(rows, limit, 'chats', chat_json) + + +def page(rows, limit: int, scope: str, render) -> dict: + # Читаем limit+1 строку: если лишняя есть — следующая страница существует. + items = rows[:limit] + cursor = (v.encode_cursor(scope, items[-1]['created_at'], items[-1]['id']) + if len(rows) > limit else None) + return {'items': [render(r) for r in items], 'next_cursor': cursor} + + +async def chat_for_member(conn, chat_id: UUID, user_id: UUID): + return await conn.fetchrow( + f'SELECT {CHAT_COLUMNS} FROM chats c ' + 'WHERE c.id = $1 AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = $2)', + chat_id, user_id) + + +@app.get('/api/v1/chats/{chat_id}') +async def get_chat(chat_id: str, request: Request): + user_id = await current_user(request) + cid = v.uuid_value(chat_id, 'chat_id') + async with db.connection() as conn: + row = await chat_for_member(conn, cid, user_id) + if row is None: + raise not_found() + return chat_json(row) + +# ---------- сообщения ---------- + + +IDEMPOTENCY_KEY_RE = re.compile(r'[A-Za-z0-9._:-]{1,128}') + + +def idempotency_key(request: Request) -> str: + key = request.headers.get('idempotency-key') + if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key): + raise invalid('Требуется заголовок Idempotency-Key: [A-Za-z0-9._:-]{1,128}') + return key + + +def request_fingerprint(text: str, attachments: list[UUID]) -> bytes: + # Хешируем разобранное тело, а не байты: порядок ключей и пробелы JSON не делают запрос "другим", + # отсутствующий attachment_ids равен []. Порядок вложений значим — он сохраняется в сообщении. + canonical = json.dumps({'text': text, 'attachment_ids': [str(a) for a in attachments]}, + ensure_ascii=False, sort_keys=True, separators=(',', ':')) + return hashlib.sha256(canonical.encode('utf-8')).digest() + + +@app.post('/api/v1/chats/{chat_id}/messages', status_code=201) +async def create_message(chat_id: str, request: Request): + # Повтор заново проверяет сессию и членство: сохранённый ответ не выдаётся без текущего доступа. + user_id = await current_user(request) + cid = v.uuid_value(chat_id, 'chat_id') + key = idempotency_key(request) + data = await v.read_json(request) + # sender_id в запросе не принимается — отправитель берётся из сессии + v.fields(data, {'text'}, {'attachment_ids'}) + text = v.text_field(data, 'text', 0, 4000, allow_blank=True) + attachments = v.attachment_ids(data) + if not attachments and not text.strip(): + raise invalid('Нужен непустой текст или хотя бы одно вложение') + fingerprint = request_fingerprint(text, attachments) + async with db.connection() as conn: + async with conn.transaction(): + member = await conn.fetchval( + 'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id) + if member is None: + raise not_found() + # Ключ — "замок": параллельный INSERT того же PK с другой реплики ждёт COMMIT/ROLLBACK этой + # транзакции. Просроченный (старше TTL) ключ атомарно переиспользуется как новый. + message_id = await conn.fetchval( + 'INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) ' + 'VALUES ($1, $2, $3, $4, gen_random_uuid()) ' + 'ON CONFLICT (user_id, chat_id, key) DO UPDATE ' + 'SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, ' + ' created_at = clock_timestamp() ' + 'WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => $5) ' + 'RETURNING message_id', + user_id, cid, key, fingerprint, settings.idempotency_ttl) + if message_id is not None: + # Ключ и сообщение фиксируются одним COMMIT: при сбое не остаётся ни того, ни другого, + # поэтому транзиентная 5xx не становится "окончательным" результатом ключа. + if attachments: + # Все вложения — из ЭТОГО чата (иначе 404, как для чужого объекта). FOR SHARE + # фиксирует строки до COMMIT; составной FK (attachment_id, chat_id) страхует на уровне БД. + # lab5: queued/processing прикреплять можно, failed — 409 (контракт). + statuses = await conn.fetch( + 'SELECT status FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[]) FOR SHARE', + cid, attachments) + if len(statuses) != len(attachments): + raise not_found() + if any(r['status'] == 'failed' for r in statuses): + raise ApiError(409, 'conflict', 'Вложение не обработано (failed)') + row = await conn.fetchrow( + 'INSERT INTO messages (id, chat_id, sender_id, text, text_raw) VALUES ($1, $2, $3, $4, $5) ' + 'RETURNING id, chat_id, sender_id, text, text_raw, created_at', + message_id, cid, user_id, *pgtext.split(text)) + if attachments: + await conn.execute( + 'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) ' + 'SELECT $1, a.id, $2, a.ord - 1 FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord)', + message_id, cid, attachments) + row = {**dict(row), 'attachment_ids': attachments} + else: + row = await conn.fetchrow( + 'SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.text_raw, m.created_at, ' + + MESSAGE_ATTACHMENTS.format(m='m') + ' ' + 'FROM idempotency_keys k JOIN messages m ON m.id = k.message_id ' + 'WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3', user_id, cid, key) + if row is None: + raise unavailable() # не должно случаться: ключ без сообщения не коммитится + if row['request_hash'] != fingerprint: + raise ApiError(409, 'conflict', 'Idempotency-Key уже использован с другим сообщением') + return JSONResponse(message_json(row), status_code=201) + + +@app.get('/api/v1/chats/{chat_id}/messages') +async def list_messages(chat_id: str, request: Request): + user_id = await current_user(request) + cid = v.uuid_value(chat_id, 'chat_id') + limit = v.limit_param(request) + after = v.decode_cursor(request, 'messages:' + str(cid)) + async with db.connection() as conn: + member = await conn.fetchval( + 'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id) + if member is None: + raise not_found() + if after: + # Keyset: продолжаем строго после последнего элемента прошлой страницы (без OFFSET). + rows = await conn.fetch( + f'SELECT {MESSAGE_COLUMNS} FROM messages WHERE chat_id = $1 AND (created_at, id) > ($3, $4) ' + 'ORDER BY created_at, id LIMIT $2', cid, limit + 1, *after) + else: + rows = await conn.fetch( + f'SELECT {MESSAGE_COLUMNS} FROM messages WHERE chat_id = $1 ' + 'ORDER BY created_at, id LIMIT $2', cid, limit + 1) + return page(rows, limit, 'messages:' + str(cid), message_json) + +# ---------- вложения (lab4, асинхронная обработка с lab5) ---------- + + +async def attachment_for_member(conn, attachment_id: UUID, user_id: UUID): + # Доступ = членство в чате вложения. Чужое и несуществующее неразличимы (404). + return await conn.fetchrow( + f'SELECT {ATTACHMENT_COLUMNS} FROM attachments a ' + 'JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 WHERE a.id = $1', + attachment_id, user_id) + + +async def variants_of(conn, row) -> list: + if row['status'] != 'ready': + return [] # варианты объявляются только вместе с ready + return await conn.fetch( + 'SELECT name, content_type, size_bytes, width, height, sha256, object_key FROM attachment_variants ' + 'WHERE attachment_id = $1 AND pipeline_version = $2 ORDER BY name', row['id'], row['pipeline_version']) + + +@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202) +async def upload_attachment(chat_id: str, request: Request): + user_id = await current_user(request) + cid = v.uuid_value(chat_id, 'chat_id') + async with db.connection() as conn: + member = await conn.fetchval( + 'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id) + if member is None: + raise not_found() # посторонний не загружает ни байта в чужой чат + # Backpressure: ограничение принятой, но не обработанной работы. При переполнении — 503 до + # чтения тела, клиент повторит позже (Retry-After). Счётчик — по частичному индексу. + pending = await conn.fetchval( + "SELECT count(*) FROM attachment_jobs WHERE status IN ('queued', 'processing')") + if pending >= settings.max_pending_jobs: + raise ApiError(503, 'unavailable', 'Очередь обработки переполнена, повторите позже', + {'Retry-After': '5'}) + filename = v.filename_param(request) + declared = images.declared_type(request) + data = await images.read_limited(request, settings.max_upload_bytes) + # Дешёвые синхронные проверки (сигнатура, лимит пикселей по заголовку, целостность) остаются в API: + # заведомо битый/огромный файл получает 413/415 сразу. Декодирование и resize — только в worker. + content_type, width, height = await image_checker.inspect(data, declared) + sha256 = hashlib.sha256(data).hexdigest() + + attachment_id = uuid.uuid4() + staging_key = f'staging/{attachment_id}' # устойчивый staging: приватный prefix S3 + object_key = f'originals/{attachment_id}' # конечный ключ, его публикует worker + # 1) все байты — в устойчивый staging. Сбой -> 503, в БД ничего нет. + await storage.put(staging_key, data, content_type) + # 2) вложение (queued) + job + outbox — ОДНОЙ транзакцией. После COMMIT работа не теряется: + # даже если процесс умрёт до publish, relay любой реплики опубликует job из outbox. + try: + async with db.connection() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + 'INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, ' + "width, height, object_key, staging_key, status) " + "VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, 'queued') " + 'RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, ' + 'error_code, created_at, updated_at', + attachment_id, cid, user_id, filename, content_type, len(data), sha256, width, height, + object_key, staging_key) + await jobs.create(conn, attachment_id, settings.job_max_attempts) + except BaseException: + await storage.delete_quietly(staging_key) # COMMIT не прошёл: staging без метаданных не нужен + raise + faults.hit('api_crash_after_commit') # окно отказа «после COMMIT до publish» (тесты) + relay.kick() + return JSONResponse(attachment_json(row), status_code=202) + + +@app.get('/api/v1/attachments/{attachment_id}') +async def get_attachment(attachment_id: str, request: Request): + user_id = await current_user(request) + aid = v.uuid_value(attachment_id, 'attachment_id') + async with db.connection() as conn: + row = await attachment_for_member(conn, aid, user_id) + if row is None: + raise not_found() + variants = await variants_of(conn, row) + return attachment_json(row, variants) + + +VARIANTS = {'original', 'thumbnail', 'crop', 'optimized'} + + +@app.get('/api/v1/attachments/{attachment_id}/content') +async def download_attachment(attachment_id: str, request: Request): + user_id = await current_user(request) + aid = v.uuid_value(attachment_id, 'attachment_id') + variant = request.query_params.get('variant', 'original') + if variant not in VARIANTS: + raise invalid('Неизвестный variant') + async with db.connection() as conn: + row = await attachment_for_member(conn, aid, user_id) + if row is None: + raise not_found() + if row['status'] != 'ready': + # queued/processing/failed: содержимое ещё (или уже) недоступно. + raise ApiError(409, 'conflict', 'Вложение ещё не готово' if row['status'] != 'failed' + else 'Вложение не обработано') + if variant == 'original': + key, content_type, digest, name = row['object_key'], row['content_type'], row['sha256'], row['filename'] + else: + found = [x for x in await variants_of(conn, row) if x['name'] == variant] + if not found: + raise not_found() # отсутствующий вариант ready-объекта + x = found[0] + key, content_type, digest, name = x['object_key'], x['content_type'], x['sha256'], variant + # Байты идут через API после проверки доступа, без redirect на S3. + data = await storage.get(key) + return Response(data, media_type=content_type, headers={ + 'Cache-Control': 'private, no-store', + 'X-Content-Type-Options': 'nosniff', + 'Content-Disposition': "inline; filename*=UTF-8''" + quote(name, safe=''), + 'ETag': '"' + digest + '"', + }) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..64ff1043 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,60 @@ +"""Версионированные миграции: migrations/NNNN_name.sql применяются по порядку, каждая в своей транзакции. + +Повторный запуск безопасен: применённые версии записаны в schema_migrations и пропускаются, +изменённый задним числом файл (другая контрольная сумма) останавливает запуск. +Advisory lock не даёт двум параллельным запускам применить одну миграцию дважды. +""" +import asyncio +import hashlib +from pathlib import Path + +import asyncpg + +from . import config + +MIGRATIONS = Path(__file__).resolve().parent.parent / 'migrations' +LOCK_ID = 2026_0002 + + +async def connect(url: str, attempts: int = 30) -> asyncpg.Connection: + for attempt in range(attempts): + try: + return await asyncpg.connect(url, timeout=3) + except (OSError, asyncpg.CannotConnectNowError, asyncpg.PostgresConnectionError): + if attempt == attempts - 1: + raise + await asyncio.sleep(1) + + +async def migrate() -> None: + # Только DATABASE_URL: сервису migrate не нужны (и не выдаются) S3-credentials API. + database_url = config._secret('DATABASE_URL') + if not database_url: + raise SystemExit('DATABASE_URL_FILE is not configured') + conn = await connect(database_url) + try: + await conn.execute('SELECT pg_advisory_lock($1)', LOCK_ID) + await conn.execute( + 'CREATE TABLE IF NOT EXISTS schema_migrations (' + ' version text PRIMARY KEY, checksum text NOT NULL,' + ' applied_at timestamptz NOT NULL DEFAULT clock_timestamp())') + applied = {r['version']: r['checksum'] for r in await conn.fetch('SELECT version, checksum FROM schema_migrations')} + for path in sorted(MIGRATIONS.glob('[0-9][0-9][0-9][0-9]_*.sql')): + version = path.stem + sql = path.read_text(encoding='utf-8') + checksum = hashlib.sha256(sql.encode()).hexdigest() + if version in applied: + if applied[version] != checksum: + raise SystemExit(f'migration {version} was modified after being applied') + print(f'skip {version}') + continue + async with conn.transaction(): + await conn.execute(sql) + await conn.execute('INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)', version, checksum) + print(f'apply {version}') + finally: + await conn.close() + + +if __name__ == '__main__': + asyncio.run(migrate()) diff --git a/app/outbox.py b/app/outbox.py new file mode 100644 index 00000000..d08931a2 --- /dev/null +++ b/app/outbox.py @@ -0,0 +1,149 @@ +"""Transactional outbox relay + восстановление работы. Фоновая задача в каждой реплике API. + +1. publish: строки outbox (published_at IS NULL, available_at <= now) берутся FOR UPDATE SKIP LOCKED — + две реплики не публикуют одну строку одновременно. Каждое сообщение публикуется persistent с + publisher confirm; published_at ставится только после confirm. Падение между confirm и UPDATE даёт + повторную публикацию — это at-least-once, worker идемпотентен. +2. reap: processing с истёкшей арендой (worker умер, heartbeat прекратился) -> queued + новая строка outbox + (или failed, если попытки исчерпаны). +3. sweep: queued-задания без неопубликованной строки outbox, "застрявшие" дольше STALE_QUEUED_SECONDS + (сообщение потеряно в DLQ по delivery-limit и т.п.), получают новую строку outbox. + +Брокер недоступен -> API продолжает отвечать 202: намерение уже в PostgreSQL, relay допубликует позже. +""" +import asyncio +import json +import logging + +import aio_pika + +from . import broker, jobs +from .config import Settings +from .db import Database + +log = logging.getLogger('messenger.outbox') +BATCH = 100 +STALE_QUEUED_SECONDS = 120 + + +class Relay: + def __init__(self, settings: Settings, db: Database): + self.settings = settings + self.db = db + self.wake = asyncio.Event() + self.connection = None + self.channel = None + self.exchange = None + self.broker_ok = False + self._last_maintenance = 0.0 + + def kick(self) -> None: + """Сразу после COMMIT upload: не ждать следующего опроса (низкая задержка до worker).""" + self.wake.set() + + async def _ensure_channel(self) -> None: + if self.channel is not None and not self.channel.is_closed: + return + if self.connection is None: + self.connection = await broker.connect(self.settings.broker_url) + self.channel = await self.connection.channel(publisher_confirms=True) + self.exchange, _queue = await broker.declare(self.channel) + + async def publish_batch(self) -> int: + await self._ensure_channel() + async with self.db.connection() as conn: + async with conn.transaction(): + rows = await conn.fetch( + 'SELECT id, job_id, payload FROM outbox ' + 'WHERE published_at IS NULL AND available_at <= clock_timestamp() ' + 'ORDER BY available_at, id LIMIT $1 FOR UPDATE SKIP LOCKED', BATCH) + published, error = [], None + for row in rows: + body = row['payload'] if isinstance(row['payload'], str) else json.dumps(row['payload']) + try: + # await = ожидание publisher confirm; mandatory=True — неприроутированное сообщение + # вернётся ошибкой, а не исчезнет молча. + await self.exchange.publish( + aio_pika.Message(body.encode(), content_type='application/json', + delivery_mode=aio_pika.DeliveryMode.PERSISTENT, + message_id=str(row['job_id']), type=jobs.JOB_TYPE), + routing_key=broker.ROUTING_KEY, mandatory=True, timeout=5) + except Exception as exc: + error = exc + break + published.append(row['id']) + # Подтверждённые брокером отмечаем даже при ошибке на середине пачки (COMMIT ниже), + # неподтверждённые остаются для следующего прохода. + if published: + await conn.execute('UPDATE outbox SET published_at = clock_timestamp(), ' + 'publish_attempts = publish_attempts + 1 WHERE id = ANY($1::bigint[])', + published) + if error is not None: + raise error + return len(rows) + + async def maintenance(self) -> None: + """reap + sweep раз в ~1 с; pg_try_advisory_xact_lock — одна реплика за раз.""" + loop = asyncio.get_running_loop() + if loop.time() - self._last_maintenance < 1.0: + return + self._last_maintenance = loop.time() + async with self.db.connection() as conn: + async with conn.transaction(): + if not await conn.fetchval('SELECT pg_try_advisory_xact_lock(20260005)'): + return + expired = await conn.fetch( + "SELECT job_id, attachment_id, attempt, max_attempts FROM attachment_jobs " + "WHERE status = 'processing' AND lease_until < clock_timestamp() " + "ORDER BY lease_until LIMIT $1 FOR UPDATE SKIP LOCKED", BATCH) + for j in expired: + if j['attempt'] >= j['max_attempts']: + await jobs.fail(conn, j['job_id'], j['attachment_id'], 'retry_exhausted', + 'lease expired on last attempt') + else: + await jobs.requeue(conn, j['job_id'], j['attachment_id'], j['attempt'] + 1, 0.0, + 'lease expired (worker died or hung)') + log.warning('lease expired: job %s attempt %s', j['job_id'], j['attempt']) + stale = await conn.fetch( + "SELECT j.job_id, j.attachment_id, j.attempt FROM attachment_jobs j " + "WHERE j.status = 'queued' AND j.updated_at < clock_timestamp() - make_interval(secs => $1) " + "AND j.next_attempt_at < clock_timestamp() - make_interval(secs => $1) " + "AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND o.published_at IS NULL) " + "LIMIT $2 FOR UPDATE OF j SKIP LOCKED", STALE_QUEUED_SECONDS, BATCH) + for j in stale: + await jobs.enqueue(conn, j['job_id'], j['attachment_id'], j['attempt'] + 1) + await conn.execute('UPDATE attachment_jobs SET updated_at = clock_timestamp() WHERE job_id = $1', + j['job_id']) + log.warning('stale queued job %s re-enqueued', j['job_id']) + + async def run(self) -> None: + delay = self.settings.outbox_poll_ms / 1000 + while True: + try: + n = await self.publish_batch() + self.broker_ok = True + await self.maintenance() + if n == BATCH: + continue # есть ещё — без паузы + except asyncio.CancelledError: + raise + except Exception as exc: + if self.broker_ok: + log.warning('outbox relay error (%s); will retry', type(exc).__name__) + self.broker_ok = False + self.channel = None + try: + await self.maintenance() # reap/sweep не зависят от брокера + except Exception: + pass + await asyncio.sleep(1.0) + continue + self.wake.clear() + try: + await asyncio.wait_for(self.wake.wait(), delay) + except asyncio.TimeoutError: + pass + + async def close(self) -> None: + if self.connection is not None: + await self.connection.close() diff --git a/app/pgtext.py b/app/pgtext.py new file mode 100644 index 00000000..fdc8baea --- /dev/null +++ b/app/pgtext.py @@ -0,0 +1,22 @@ +"""Точное хранение строк с U+0000 в PostgreSQL. + +Тип text в PostgreSQL не может содержать байт 0x00, а JSON-контракт допускает "\\u0000" в тексте +сообщения, display_name и title. Раньше такой INSERT падал с 500 (замечание к lab3, PR #99). + +Схема (только expand, совместима с предыдущей версией при rolling update): +- столбец text/display_name/title получает ту же строку, где U+0000 заменён на U+FFFD: длина в code points + не меняется, поэтому CHECK char_length и прежние реплики продолжают работать; +- столбец *_raw bytea получает точные UTF-8 байты исходной строки, но ТОЛЬКО если в ней был U+0000 + (иначе NULL, лишнего места не занимает); +- при чтении API возвращает декодированный *_raw, если он есть, иначе обычный столбец. +""" + + +def split(value: str) -> tuple[str, bytes | None]: + if '\x00' in value: + return value.replace('\x00', '\ufffd'), value.encode('utf-8') + return value, None + + +def join(text: str, raw: bytes | None) -> str: + return bytes(raw).decode('utf-8') if raw is not None else text diff --git a/app/replay.py b/app/replay.py new file mode 100644 index 00000000..41a25ae3 --- /dev/null +++ b/app/replay.py @@ -0,0 +1,98 @@ +"""Явный операторский replay failed-заданий (failed -> queued) и просмотр реестра failed/DLQ. + + docker compose --env-file .env run --rm --no-deps worker python -m app.replay list + docker compose --env-file .env run --rm --no-deps worker python -m app.replay replay --attachment --reason "..." [--dry-run] + docker compose --env-file .env run --rm --no-deps worker python -m app.replay replay --all-failed --reason "..." [--dry-run] + +Безопасность: переводится только задание в статусе failed (строка блокируется FOR UPDATE). Для ready/done +replay — отказ: "повторить" не означает исполнить дубль успешной работы. Каждое действие пишется в +job_audit (прежний статус и ошибка, actor, причина), счётчик попыток начинается заново, job_id тот же — +логический эффект (attachment_id, pipeline_version, variant) защищён от дубля так же, как при redelivery. +""" +import argparse +import asyncio +import getpass +import os +import sys +from uuid import UUID + +from . import config, jobs +from .db import Database + + +async def list_failed(db: Database) -> None: + async with db.connection() as conn: + rows = await conn.fetch( + "SELECT j.job_id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.finished_at " + "FROM attachment_jobs j WHERE j.status = 'failed' ORDER BY j.finished_at") + print(f'failed jobs: {len(rows)}') + for r in rows: + print(f" attachment={r['attachment_id']} job={r['job_id']} attempts={r['attempt']}/{r['max_attempts']} " + f"code={r['error_code']} reason={r['last_error']!r} at={r['finished_at']:%Y-%m-%dT%H:%M:%SZ}") + + +async def replay(db: Database, attachment: UUID | None, all_failed: bool, reason: str, dry_run: bool) -> int: + actor = os.environ.get('REPLAY_ACTOR') or getpass.getuser() + done = 0 + async with db.connection() as conn: + async with conn.transaction(): + if all_failed: + targets = await conn.fetch("SELECT job_id, attachment_id, status, error_code FROM attachment_jobs " + "WHERE status = 'failed' ORDER BY finished_at FOR UPDATE") + else: + targets = await conn.fetch('SELECT job_id, attachment_id, status, error_code FROM attachment_jobs ' + 'WHERE attachment_id = $1 FOR UPDATE', attachment) + if not targets: + print('no such job') + return 1 + for t in targets: + if t['status'] != 'failed': + print(f"skip attachment={t['attachment_id']}: status={t['status']} — replay only for failed") + continue + print(f"{'would replay' if dry_run else 'replay'} attachment={t['attachment_id']} " + f"job={t['job_id']} previous={t['error_code']}") + if dry_run: + continue + await conn.execute( + 'INSERT INTO job_audit (job_id, action, previous_status, previous_error, actor, reason) ' + "VALUES ($1, 'replay', $2, $3, $4, $5)", t['job_id'], t['status'], t['error_code'], actor, reason) + await conn.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt = 0, error_code = NULL, last_error = NULL, " + "next_attempt_at = clock_timestamp(), finished_at = NULL, worker_id = NULL, " + "updated_at = clock_timestamp() WHERE job_id = $1 AND status = 'failed'", t['job_id']) + await conn.execute( + "UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp() " + "WHERE id = $1 AND status = 'failed'", t['attachment_id']) + await jobs.enqueue(conn, t['job_id'], t['attachment_id'], attempt=1) + done += 1 + print(f'replayed: {done}' + (' (dry run, nothing changed)' if dry_run else '')) + return 0 + + +async def main_async(args) -> int: + db = Database(config.load()) + await db.start() + try: + if args.command == 'list': + await list_failed(db) + return 0 + return await replay(db, args.attachment, args.all_failed, args.reason, args.dry_run) + finally: + await db.close() + + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + sub = p.add_subparsers(dest='command', required=True) + sub.add_parser('list') + r = sub.add_parser('replay') + target = r.add_mutually_exclusive_group(required=True) + target.add_argument('--attachment', type=UUID) + target.add_argument('--all-failed', action='store_true') + r.add_argument('--reason', required=True) + r.add_argument('--dry-run', action='store_true') + sys.exit(asyncio.run(main_async(p.parse_args()))) + + +if __name__ == '__main__': + main() diff --git a/app/s3_bootstrap.py b/app/s3_bootstrap.py new file mode 100644 index 00000000..0087d3fe --- /dev/null +++ b/app/s3_bootstrap.py @@ -0,0 +1,72 @@ +"""Bootstrap S3: создаёт приватный bucket для вложений. Запуск: python -m app.s3_bootstrap (сервис s3-init). + +Идемпотентно: повторный запуск не трогает существующий bucket и данные в нём. +"Приватный" = у bucket нет политики анонимного доступа. Если политика есть (например, кто-то включил +public read руками), она удаляется. В конце выполняется проверка снаружи API: анонимный GET объекта +должен получить отказ (403), иначе bootstrap завершается ошибкой и API не стартует. +""" +import io +import os +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from urllib.parse import urlparse + +from minio import Minio +from minio.error import S3Error + + +def secret(name: str) -> str: + return Path(os.environ[name + '_FILE']).read_text(encoding='utf-8').strip() + + +def main() -> None: + endpoint = os.environ.get('S3_ENDPOINT', 'http://s3:9000') + bucket = os.environ.get('S3_BUCKET', 'messenger-attachments') + region = os.environ.get('S3_REGION', 'us-east-1') + url = urlparse(endpoint) + client = Minio(url.netloc, access_key=secret('S3_ACCESS_KEY_ID'), secret_key=secret('S3_SECRET_ACCESS_KEY'), + secure=url.scheme == 'https', region=region) + + for attempt in range(30): + try: + exists = client.bucket_exists(bucket) + break + except Exception as exc: # S3 ещё стартует + if attempt == 29: + raise SystemExit(f's3 not reachable: {type(exc).__name__}') + time.sleep(1) + if exists: + print(f'bucket {bucket}: exists') + else: + client.make_bucket(bucket, location=region) + print(f'bucket {bucket}: created') + + try: + client.get_bucket_policy(bucket) + client.delete_bucket_policy(bucket) + print(f'bucket {bucket}: anonymous policy removed') + except S3Error as exc: + if exc.code not in ('NoSuchBucketPolicy', 'NoSuchPolicy'): + raise + print(f'bucket {bucket}: no bucket policy (private)') + + # Проверка приватности без credentials: служебный объект, анонимный GET, ожидаем 403. + probe = '_bootstrap/private-check' + client.put_object(bucket, probe, io.BytesIO(b'private'), 7, content_type='text/plain') + try: + urllib.request.urlopen(f'{endpoint}/{bucket}/{probe}', timeout=5) + print('ERROR: anonymous GET returned object, bucket is public', file=sys.stderr) + sys.exit(1) + except urllib.error.HTTPError as exc: + print(f'anonymous GET {bucket}/{probe}: HTTP {exc.code} (denied)') + if exc.code not in (401, 403): + sys.exit(1) + finally: + client.remove_object(bucket, probe) + + +if __name__ == '__main__': + main() diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..327032f3 --- /dev/null +++ b/app/security.py @@ -0,0 +1,47 @@ +"""Пароли (Argon2id) и непрозрачные токены сессий.""" +import asyncio +import hashlib +import secrets +from concurrent.futures import ThreadPoolExecutor + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError + +# Параметры по умолчанию argon2-cffi = профиль RFC 9106 "low memory": Argon2id, m=64 MiB, t=3, p=4. +_hasher = PasswordHasher() +# Хеш для несуществующего пользователя: вход выполняет ту же работу и не выдаёт по времени, +# что такого username нет. +_DUMMY_HASH = _hasher.hash('dummy-password-for-timing') + + +class PasswordService: + """KDF дорогой по CPU и памяти: считаем в отдельном ограниченном пуле потоков, + чтобы не блокировать event loop и не съесть всю память параллельными входами.""" + + def __init__(self, workers: int): + self._pool = ThreadPoolExecutor(max_workers=workers, thread_name_prefix='argon2') + + async def hash(self, password: str) -> str: + return await asyncio.get_running_loop().run_in_executor(self._pool, _hasher.hash, password) + + async def verify(self, password_hash: str | None, password: str) -> bool: + def check() -> bool: + try: + return _hasher.verify(password_hash or _DUMMY_HASH, password) and password_hash is not None + except (VerificationError, InvalidHashError): + return False + return await asyncio.get_running_loop().run_in_executor(self._pool, check) + + def close(self) -> None: + self._pool.shutdown(wait=False) + + +def new_token() -> str: + # 32 случайных байта = 256 бит энтропии из CSPRNG ОС; base64url -> 43 символа. + return secrets.token_urlsafe(32) + + +def token_hash(token: str) -> bytes: + # Для токена с 256 битами энтропии быстрый SHA-256 достаточен: перебор невозможен. + # Для паролей (низкая энтропия) так нельзя — там нужен медленный KDF. + return hashlib.sha256(token.encode('ascii')).digest() diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..205bc2b7 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,187 @@ +"""Сессии: PostgreSQL — источник истины, Valkey — read-through кэш с TTL. + +Политика: +- создание: INSERT в PostgreSQL (COMMIT), затем SET в кэш с TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS); +- проверка: кэш -> при промахе PostgreSQL -> заполнить кэш; +- logout: UPDATE revoked_at в PostgreSQL (COMMIT), затем DEL из кэша; +- любой сбой Valkey: работаем только с PostgreSQL и SESSION_CACHE_TTL_SECONDS не доверяем кэшу + (все записи, сделанные до сбоя, за это время гарантированно истекут — например, не удалённая + при logout копия отозванной сессии). Flush/рестарт Valkey = пустой кэш, сессии не теряются. +- lab3, несколько реплик: окно недоверия ОБЩЕЕ. Реплика, получившая ошибку Valkey, записывает его + в PostgreSQL (cache_state), а попадание в кэш принимается, только если общий признак проверен + не раньше SHARED_STATE_MAX_AGE назад. Иначе реплика, которая сама сбоя не видела, могла бы + отдать копию сессии, отозванной на другой реплике во время разрыва с Valkey. +""" +import asyncio +import json +import logging +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from uuid import UUID + +import redis.asyncio as redis +from redis.exceptions import RedisError + +from .config import Settings +from .db import Database +from .security import token_hash + +log = logging.getLogger('messenger.sessions') + + +@dataclass(frozen=True) +class Session: + user_id: UUID + expires_at: datetime + + +# Насколько устаревшим может быть знание реплики об общем окне недоверия (не чаще 2 запросов/с к БД). +SHARED_STATE_MAX_AGE = 0.5 + + +class SessionCache: + def __init__(self, url: str | None, ttl: int, timeout: float, db: Database): + self.ttl = ttl + self.db = db + self.client = redis.Redis.from_url( + url, socket_timeout=timeout, socket_connect_timeout=timeout, + retry_on_timeout=False, health_check_interval=30, + ) if url else None + self._distrust_until = 0.0 # локально (monotonic): своя ошибка действует сразу + self._shared_trusted = False # результат последней проверки общего признака + self._shared_checked = -1e9 + self._lock = asyncio.Lock() + + @staticmethod + def key(digest: bytes) -> str: + return 'session:' + digest.hex() + + def usable(self) -> bool: + return self.client is not None and time.monotonic() >= self._distrust_until + + async def trusted(self) -> bool: + """Можно ли принять попадание в кэш: нет ни своего, ни общего (с других реплик) окна недоверия.""" + if not self.usable(): + return False + if time.monotonic() - self._shared_checked > SHARED_STATE_MAX_AGE: + async with self._lock: + if time.monotonic() - self._shared_checked > SHARED_STATE_MAX_AGE: + try: + async with self.db.pool.acquire(timeout=1) as conn: + self._shared_trusted = await conn.fetchval( + 'SELECT distrust_until <= clock_timestamp() FROM cache_state WHERE id = 1', + timeout=1) is True + except Exception: # без БД сессию всё равно не проверить; кэшу не доверяем + self._shared_trusted = False + self._shared_checked = time.monotonic() + return self._shared_trusted + + async def _failed(self, op: str) -> None: + # Не знаем, что успело/не успело записаться: перестаём читать кэш на один максимальный TTL — + # сами сразу и все остальные реплики через общий признак в PostgreSQL. + self._distrust_until = time.monotonic() + self.ttl + self._shared_checked = -1e9 + log.warning('valkey %s failed; bypassing cache for %ss on all replicas', op, self.ttl) + try: + async with self.db.pool.acquire(timeout=1) as conn: + await conn.execute( + 'UPDATE cache_state SET distrust_until = greatest(distrust_until, ' + 'clock_timestamp() + make_interval(secs => $1)) WHERE id = 1', self.ttl, timeout=1) + except Exception: # БД недоступна — и кэш, и сессии всё равно проверить нельзя + log.warning('could not publish cache distrust window', exc_info=True) + + async def get(self, digest: bytes) -> Session | None: + if not await self.trusted(): + return None + try: + raw = await self.client.get(self.key(digest)) + except (RedisError, OSError): + await self._failed('get') + return None + if raw is None: + return None + data = json.loads(raw) + return Session(UUID(data['u']), datetime.fromtimestamp(data['e'], timezone.utc)) + + async def put(self, digest: bytes, session: Session) -> None: + if not self.usable(): + return + remaining = int((session.expires_at - datetime.now(timezone.utc)).total_seconds()) + ttl = min(self.ttl, remaining) + if ttl <= 0: + return + value = json.dumps({'u': str(session.user_id), 'e': session.expires_at.timestamp()}) + try: + await self.client.set(self.key(digest), value, ex=ttl) + except (RedisError, OSError): + await self._failed('set') + + async def delete(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.delete(self.key(digest)) + except (RedisError, OSError): + await self._failed('delete') + + async def close(self) -> None: + if self.client is not None: + await self.client.aclose() + + +class SessionStore: + def __init__(self, settings: Settings, db: Database): + self.ttl = settings.session_ttl + self.db = db + self.cache = SessionCache(settings.cache_url, settings.session_cache_ttl, settings.cache_timeout, db) + + async def create(self, user_id: UUID, token: str) -> Session: + digest = token_hash(token) + async with self.db.connection() as conn: + # Время берём из БД, чтобы срок не зависел от часов конкретной реплики API. + expires_at = await conn.fetchval( + 'INSERT INTO sessions (token_hash, user_id, expires_at) ' + 'VALUES ($1, $2, clock_timestamp() + make_interval(secs => $3)) RETURNING expires_at', + digest, user_id, self.ttl, + ) + session = Session(user_id, expires_at) + await self.cache.put(digest, session) + return session + + async def resolve(self, token: str) -> Session | None: + digest = token_hash(token) + now = datetime.now(timezone.utc) + cached = await self.cache.get(digest) + if cached is not None and cached.expires_at > now: + return cached + async with self.db.connection() as conn: + row = await conn.fetchrow( + 'SELECT user_id, expires_at FROM sessions ' + 'WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > clock_timestamp()', + digest, + ) + if row is None: + return None + session = Session(row['user_id'], row['expires_at']) + await self.cache.put(digest, session) + return session + + async def revoke(self, token: str) -> bool: + digest = token_hash(token) + async with self.db.connection() as conn: + status = await conn.execute( + 'UPDATE sessions SET revoked_at = clock_timestamp() ' + 'WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > clock_timestamp()', + digest, + ) + # Инвалидация кэша строго после COMMIT в источнике истины. + await self.cache.delete(digest) + return status == 'UPDATE 1' + + async def purge_expired(self) -> int: + async with self.db.connection() as conn: + status = await conn.execute( + "DELETE FROM sessions WHERE expires_at < clock_timestamp() - interval '1 day'") + return int(status.split()[-1]) + diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..0f966a91 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,120 @@ +"""Приватное S3-хранилище оригиналов вложений (MinIO, S3 API). + +Клиент minio-py синхронный, поэтому вызовы уходят в пул потоков (asyncio.to_thread) и не блокируют +event loop. Таймауты и число повторов ограничены: недоступный S3 даёт 503 за секунды, а не зависание. +Ключи объектов назначает сервер; bucket приватный, байты отдаёт только API после проверки доступа. +""" +import asyncio +import io +import logging +from urllib.parse import urlparse + +import urllib3 +from minio import Minio +from minio.error import MinioException, S3Error + +from .config import Settings +from .errors import ApiError, unavailable + +log = logging.getLogger('messenger.storage') + + +class ObjectMissing(Exception): + """Метаданные в БД есть, а объекта в bucket нет (рассогласование хранилищ).""" + + +class Storage: + def __init__(self, settings: Settings): + endpoint = urlparse(settings.s3_endpoint) + self.bucket = settings.s3_bucket + # Без повторов на уровне HTTP-клиента: повтор удваивал бы время отказа. Общий дедлайн операции — + # в _call: getaddrinfo (DNS остановленного контейнера ~8 с) таймаутами urllib3 не ограничен. + self.deadline = settings.s3_timeout + http = urllib3.PoolManager( + timeout=urllib3.Timeout(connect=2, read=settings.s3_timeout), + retries=urllib3.Retry(total=0, redirect=0), + maxsize=16, + ) + self.client = Minio(endpoint.netloc, access_key=settings.s3_access_key, + secret_key=settings.s3_secret_key, secure=endpoint.scheme == 'https', + region=settings.s3_region, http_client=http) + + async def _call(self, fn, *args, **kwargs): + try: + return await asyncio.wait_for(asyncio.to_thread(fn, *args, **kwargs), self.deadline) + except asyncio.TimeoutError: + # Поток дорабатывает в фоне (его нельзя прервать), но клиент получает 503 в пределах дедлайна. + log.warning('S3 deadline %.1fs exceeded on %s', self.deadline, fn.__name__) + raise unavailable() from None + except S3Error as exc: + if exc.code in ('NoSuchKey', 'NoSuchObject'): + raise ObjectMissing() from None + log.warning('S3 error %s on %s', exc.code, fn.__name__) + raise unavailable() from None + except (MinioException, urllib3.exceptions.HTTPError, OSError, ValueError): + # Нет связи, таймаут, 5xx после повтора — для клиента это временная недоступность. + log.warning('S3 unavailable on %s', fn.__name__, exc_info=True) + raise unavailable() from None + + async def put(self, key: str, data: bytes, content_type: str) -> None: + await self._call(self.client.put_object, self.bucket, key, io.BytesIO(data), len(data), + content_type=content_type) + + async def get(self, key: str) -> bytes: + def read() -> bytes: + response = self.client.get_object(self.bucket, key) + try: + return response.read() + finally: + response.close() + response.release_conn() + try: + return await self._call(read) + except ObjectMissing: + log.error('object %s is missing in bucket %s', key, self.bucket) + raise ApiError(500, 'internal_error', 'Внутренняя ошибка сервера') from None + + async def delete(self, key: str) -> None: + await self._call(self.client.remove_object, self.bucket, key) + + async def delete_quietly(self, key: str) -> None: + """Компенсация после неудачной записи метаданных. Сбой здесь не меняет ответ клиенту.""" + try: + await asyncio.shield(self.delete(key)) + except Exception: + log.error('compensation failed: orphan object %s left in bucket', key) + + # ---------- lab5: операции worker ---------- + + async def get_raw(self, key: str) -> bytes: + """Как get(), но отсутствие объекта — ObjectMissing (worker решает сам: это постоянная ошибка).""" + def read() -> bytes: + response = self.client.get_object(self.bucket, key) + try: + return response.read() + finally: + response.close() + response.release_conn() + return await self._call(read) + + async def copy(self, source_key: str, target_key: str) -> None: + """Server-side copy внутри bucket: байты оригинала не гоняются через worker повторно.""" + from minio.commonconfig import CopySource + await self._call(self.client.copy_object, self.bucket, target_key, CopySource(self.bucket, source_key)) + + async def exists(self, key: str) -> bool: + try: + await self._call(self.client.stat_object, self.bucket, key) + return True + except ObjectMissing: + return False + + async def list_older(self, prefix: str, older_than_seconds: int) -> list[str]: + """Ключи под prefix старше заданного возраста (для retention staging).""" + import datetime as dt + + def scan() -> list[str]: + border = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=older_than_seconds) + return [o.object_name for o in self.client.list_objects(self.bucket, prefix=prefix, recursive=True) + if o.last_modified is not None and o.last_modified < border] + return await self._call(scan) diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..d8b4613c --- /dev/null +++ b/app/validation.py @@ -0,0 +1,138 @@ +"""Строгий разбор JSON и параметров: лишние поля, неверные типы, malformed JSON -> 400.""" +import base64 +import json +import re +from datetime import datetime +from uuid import UUID + +from fastapi import Request + +from .errors import ApiError, invalid + +MAX_BODY_BYTES = 64 * 1024 +USERNAME_RE = re.compile(r'[a-z0-9_]{3,32}') + + +async def read_json(request: Request) -> dict: + media_type = request.headers.get('content-type', '').split(';')[0].strip().lower() + if media_type != 'application/json': + raise ApiError(415, 'unsupported_media_type', 'Ожидается Content-Type: application/json') + body = await request.body() + if len(body) > MAX_BODY_BYTES: + raise ApiError(413, 'payload_too_large', 'Слишком большое тело запроса') + try: + data = json.loads(body.decode('utf-8')) + except (UnicodeDecodeError, ValueError): + raise invalid('Некорректный JSON') from None + if not isinstance(data, dict): + raise invalid('Ожидается JSON-объект') + return data + + +def fields(data: dict, required: set[str], optional: set[str] = frozenset()) -> None: + extra = set(data) - required - optional + if extra: + raise invalid('Неизвестные поля: ' + ', '.join(sorted(extra))) + missing = required - set(data) + if missing: + raise invalid('Отсутствуют поля: ' + ', '.join(sorted(missing))) + + +def text_field(data: dict, name: str, min_len: int, max_len: int, *, allow_blank: bool = False) -> str: + value = data[name] + if not isinstance(value, str): + raise invalid(f'Поле {name} должно быть строкой') + try: + value.encode('utf-8') # отсекает одиночные суррогаты из \uD800-экранирования + except UnicodeEncodeError: + raise invalid(f'Поле {name} содержит некорректный Unicode') from None + if not min_len <= len(value) <= max_len: # len(str) в Python = число code points + raise invalid(f'Длина поля {name} должна быть {min_len}..{max_len} символов') + if not allow_blank and not value.strip(): + raise invalid(f'Поле {name} не может состоять из пробелов') + return value + + +def username(data: dict) -> str: + value = data['username'] + if not isinstance(value, str) or not USERNAME_RE.fullmatch(value): + raise invalid('username должен соответствовать [a-z0-9_]{3,32}') + return value + + +def uuid_value(value, what: str = 'идентификатор') -> UUID: + if not isinstance(value, str): + raise invalid(f'Некорректный {what}') + try: + parsed = UUID(value) + except ValueError: + raise invalid(f'Некорректный {what}') from None + if str(parsed) != value.lower(): # только каноническая форма 8-4-4-4-12 + raise invalid(f'Некорректный {what}') + return parsed + + +def limit_param(request: Request) -> int: + raw = request.query_params.get('limit') + if raw is None: + return 50 + if not re.fullmatch(r'[0-9]{1,3}', raw) or not 1 <= int(raw) <= 100: + raise invalid('limit должен быть целым 1..100') + return int(raw) + + +# Курсор: base64url(JSON {"s": область, "t": created_at, "i": id}). Для клиента это непрозрачная строка. +# Подделка курсора ничего не даёт: доступ к чату проверяется отдельно, а область привязывает +# курсор к конкретному чату/списку — курсор другого чата отклоняется с 400. + +def encode_cursor(scope: str, created_at: datetime, item_id: UUID) -> str: + raw = json.dumps({'s': scope, 't': created_at.isoformat(), 'i': str(item_id)}, separators=(',', ':')) + return base64.urlsafe_b64encode(raw.encode()).decode().rstrip('=') + + +def decode_cursor(request: Request, scope: str) -> tuple[datetime, UUID] | None: + raw = request.query_params.get('cursor') + if raw is None: + return None + try: + if len(raw) > 512: + raise ValueError + data = json.loads(base64.urlsafe_b64decode(raw + '=' * (-len(raw) % 4))) + if not isinstance(data, dict) or set(data) != {'s', 't', 'i'} or data['s'] != scope: + raise ValueError + created_at = datetime.fromisoformat(data['t']) + if created_at.tzinfo is None: + raise ValueError + return created_at, UUID(data['i']) + except (ValueError, TypeError, UnicodeDecodeError): + raise invalid('Некорректный cursor') from None + + +# Имя файла — только для отображения: в ключ S3 и пути оно не попадает. Разделители путей, +# NUL и управляющие символы (C0, DEL, C1) отклоняются, чтобы имя нельзя было выдать за путь. +FILENAME_FORBIDDEN_RE = re.compile(r'[/\\\x00-\x1f\x7f-\x9f]') + + +def filename_param(request: Request) -> str: + values = request.query_params.getlist('filename') + if len(values) != 1: + raise invalid('Требуется ровно один параметр filename') + value = values[0] + try: + value.encode('utf-8') + except UnicodeEncodeError: + raise invalid('filename содержит некорректный Unicode') from None + if not 1 <= len(value) <= 255 or not value.strip() or value in ('.', '..') \ + or FILENAME_FORBIDDEN_RE.search(value): + raise invalid('Недопустимое имя файла') + return value + + +def attachment_ids(data: dict) -> list[UUID]: + raw = data.get('attachment_ids', []) + if not isinstance(raw, list) or len(raw) > 10: + raise invalid('attachment_ids должен быть массивом из 0..10 UUID') + ids = [uuid_value(x, 'attachment_id') for x in raw] + if len(set(ids)) != len(ids): + raise invalid('attachment_ids содержит дубликаты') + return ids diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..e998a3bb --- /dev/null +++ b/app/worker.py @@ -0,0 +1,307 @@ +"""Worker обработки вложений: python -m app.worker (отдельный сервис, не API). + +Одна доставка сообщения: + 1. проверить сообщение по contracts/jobs.schema.json (иначе reject без requeue -> DLQ); + 2. claim в PostgreSQL: queued (или processing с истёкшей арендой) -> processing, attempt+1, lease; + done/failed/чужая живая аренда/ещё не время retry -> no-op + ack; + 3. heartbeat продлевает lease, пока идёт работа (отличает долгую обработку от зависшей/мёртвой); + 4. оригинал из staging, сверка size/sha256 с доверенными метаданными из БД; + 5. варианты в дочернем процессе (лимиты памяти/времени), запись в S3 по ДЕТЕРМИНИРОВАННЫМ ключам + variants//v/ + server-side copy staging -> originals/; + 6. COMMIT результата с fencing по (worker_id, attempt): variants (ON CONFLICT DO NOTHING) + ready; + 7. удалить staging (после безопасной фиксации), затем ack. +Сбой на любом шаге до ack -> redelivery/reaper -> повтор шагов 4–6 перезаписывает те же ключи тем же +содержимым и не создаёт второй набор вариантов (ключ эффекта (attachment_id, pipeline_version, variant)). +""" +import asyncio +import hashlib +import json +import logging +import os +import secrets +import signal +import socket +import time +from pathlib import Path + +import jsonschema + +from . import broker, config, faults, imaging, jobs +from .db import Database +from .errors import ApiError +from .storage import ObjectMissing, Storage + +log = logging.getLogger('messenger.worker') +SCHEMA = json.loads((Path(__file__).resolve().parent.parent / 'contracts/jobs.schema.json').read_text(encoding='utf-8')) +VALIDATOR = jsonschema.Draft202012Validator(SCHEMA, format_checker=jsonschema.FormatChecker()) +ALIVE_FILE = Path('/tmp/worker-alive') + + +class LostLease(Exception): + """Аренду перехватили (истекла): результат фиксирует новый владелец, этот — только ack.""" + + +class Worker: + def __init__(self): + self.settings = config.load() + self.db = Database(self.settings) + self.storage = Storage(self.settings) + self.worker_id = f'{socket.gethostname()}-{secrets.token_hex(3)}' + self.slots = asyncio.Semaphore(self.settings.worker_concurrency) + self.inflight: set[asyncio.Task] = set() + self.stopping = asyncio.Event() + self.connection = None + + # ---------- приём сообщения ---------- + + async def on_message(self, message) -> None: + task = asyncio.current_task() + self.inflight.add(task) + try: + try: + payload = json.loads(message.body) + VALIDATOR.validate(payload) + except (ValueError, jsonschema.ValidationError): + log.error('invalid job message %s -> DLQ', message.message_id) + await message.reject(requeue=False) + return + async with self.slots: + try: + await self.process(payload) + except Exception: + # Инфраструктура (БД/S3) недоступна настолько, что даже состояние не записать: + # вернуть сообщение в очередь с паузой; delivery-limit страхует от бесконечного цикла. + log.exception('job %s: infrastructure error, requeue', payload['job_id']) + await asyncio.sleep(2) + await message.nack(requeue=True) + return + await message.ack() # только после устойчивого эффекта (или no-op) + finally: + self.inflight.discard(task) + + # ---------- обработка ---------- + + async def claim(self, job_id: str, attachment_id: str): + async with self.db.connection() as conn: + async with conn.transaction(): + j = await conn.fetchrow( + 'SELECT j.status, j.attempt, j.max_attempts, ' + ' coalesce(j.lease_until > clock_timestamp(), false) AS leased, ' + ' j.next_attempt_at > clock_timestamp() AS not_yet, ' + ' a.staging_key, a.object_key, a.size_bytes, a.sha256, a.pipeline_version ' + 'FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id ' + 'WHERE j.job_id = $1 AND j.attachment_id = $2 FOR UPDATE OF j', job_id, attachment_id) + if j is None: + return None, 'unknown job' + if j['status'] in ('done', 'failed'): + return None, f"already {j['status']} (duplicate delivery)" + if j['status'] == 'processing' and j['leased']: + return None, 'leased by another live worker (duplicate delivery)' + if j['status'] == 'queued' and j['not_yet']: + return None, 'retry is scheduled later (stale message)' + if j['attempt'] >= j['max_attempts']: + await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', 'no attempts left') + return None, 'attempts exhausted' + attempt = await conn.fetchval( + "UPDATE attachment_jobs SET status = 'processing', attempt = attempt + 1, worker_id = $2, " + "lease_until = clock_timestamp() + make_interval(secs => $3), started_at = clock_timestamp(), " + "updated_at = clock_timestamp() WHERE job_id = $1 RETURNING attempt", + job_id, self.worker_id, self.settings.job_lease) + await conn.execute( + "UPDATE attachments SET status = 'processing', updated_at = clock_timestamp() " + "WHERE id = $1 AND status IN ('queued', 'processing')", attachment_id) + return {**dict(j), 'attempt': attempt, 'max_attempts': j['max_attempts']}, None + + async def heartbeat(self, job_id: str, attempt: int) -> None: + interval = max(1.0, self.settings.job_lease / 3) + while True: + await asyncio.sleep(interval) + try: + async with self.db.connection() as conn: + ok = await conn.fetchval( + "UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => $3) " + "WHERE job_id = $1 AND worker_id = $2 AND attempt = $4 AND status = 'processing' RETURNING 1", + job_id, self.worker_id, self.settings.job_lease, attempt) + if not ok: + log.warning('job %s: lease lost', job_id) + return + except Exception: + log.warning('job %s: heartbeat failed', job_id) + + async def process(self, payload: dict) -> None: + job_id, attachment_id = payload['job_id'], payload['attachment_id'] + job, reason = await self.claim(job_id, attachment_id) + if job is None: + log.info('job %s: no-op, %s', job_id, reason) + return + started = time.monotonic() + beat = asyncio.create_task(self.heartbeat(job_id, job['attempt'])) + try: + stages = await self.execute(job_id, attachment_id, job) + log.info('job %s: ready in %.2fs (attempt %d) %s', job_id, time.monotonic() - started, job['attempt'], + ' '.join(f'{k}={v:.3f}' for k, v in stages.items())) + except imaging.PermanentImageError as exc: + await self.finish_failed(job_id, attachment_id, job['attempt'], exc.code, exc.code) + except LostLease: + log.warning('job %s: result not committed, lease taken over', job_id) + except (imaging.TransientImageError, ApiError) as exc: + code = getattr(exc, 'code', 'unavailable') + await self.retry_or_fail(job_id, attachment_id, job, f'transient: {code}') + finally: + beat.cancel() + + async def execute(self, job_id: str, attachment_id: str, job) -> None: + version = job['pipeline_version'] + stages, mark = {}, time.monotonic() + + def lap(name): + nonlocal mark + now = time.monotonic() + stages[name] = now - mark + mark = now + try: + data = await self.storage.get_raw(job['staging_key']) + except ObjectMissing: + raise imaging.PermanentImageError('source_missing') from None + # Доверенные метаданные — из БД, а не из сообщения брокера. + if len(data) != job['size_bytes'] or hashlib.sha256(data).hexdigest() != job['sha256']: + raise imaging.PermanentImageError('source_corrupted') + lap('fetch') + variants = await imaging.run_isolated(data, self.settings.worker_max_pixels, + self.settings.worker_memory_mb, self.settings.job_timeout) + lap('build') + ext = {'image/jpeg': 'jpg', 'image/webp': 'webp', 'image/png': 'png'} + for v in variants: + v['object_key'] = f"variants/{attachment_id}/v{version}/{v['name']}.{ext[v['content_type']]}" + await self.storage.put(v['object_key'], v.pop('data'), v['content_type']) + await self.storage.copy(job['staging_key'], job['object_key']) + lap('store') + faults.hit('worker_crash_after_s3') + async with self.db.connection() as conn: + async with conn.transaction(): + owned = await conn.fetchval( + "UPDATE attachment_jobs SET status = 'done', lease_until = NULL, error_code = NULL, " + "last_error = NULL, finished_at = clock_timestamp(), updated_at = clock_timestamp() " + "WHERE job_id = $1 AND status = 'processing' AND worker_id = $2 AND attempt = $3 RETURNING 1", + job_id, self.worker_id, job['attempt']) + if not owned: + raise LostLease() + for v in variants: + await conn.execute( + 'INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, ' + 'size_bytes, width, height, sha256, object_key) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9) ' + 'ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING', + attachment_id, version, v['name'], v['content_type'], v['size_bytes'], + v['width'], v['height'], v['sha256'], v['object_key']) + await conn.execute( + "UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() " + "WHERE id = $1 AND status = 'processing'", attachment_id) + lap('commit') + faults.hit('worker_crash_after_commit') + try: + await self.storage.delete(job['staging_key']) # staging больше не нужен: результат зафиксирован + except ApiError: + log.warning('job %s: staging cleanup deferred to retention sweeper', job_id) + lap('cleanup') + return stages + + async def finish_failed(self, job_id, attachment_id, attempt: int, code: str, reason: str) -> None: + async with self.db.connection() as conn: + async with conn.transaction(): + await jobs.fail(conn, job_id, attachment_id, code, reason, worker_id=self.worker_id, attempt=attempt) + log.warning('job %s: failed (%s)', job_id, code) + + async def retry_or_fail(self, job_id, attachment_id, job, reason: str) -> None: + async with self.db.connection() as conn: + async with conn.transaction(): + if job['attempt'] >= job['max_attempts']: + await jobs.fail(conn, job_id, attachment_id, 'retry_exhausted', reason, + worker_id=self.worker_id, attempt=job['attempt']) + log.warning('job %s: failed, retry_exhausted after %d attempts', job_id, job['attempt']) + return + delay = jobs.backoff_seconds(job['attempt'], self.settings.retry_base_ms, self.settings.retry_max_ms) + await jobs.requeue(conn, job_id, attachment_id, job['attempt'] + 1, delay, reason, + worker_id=self.worker_id, attempt=job['attempt']) + log.warning('job %s: %s, retry in %.1fs (attempt %d/%d)', job_id, reason, delay, + job['attempt'], job['max_attempts']) + + # ---------- обслуживание ---------- + + async def maintenance(self) -> None: + """Пульс для healthcheck + retention staging (одна реплика за раз через advisory lock).""" + last_sweep = 0.0 + while not self.stopping.is_set(): + if self.connection is not None and not self.connection.is_closed: + ALIVE_FILE.touch() + if time.monotonic() - last_sweep > 300: + last_sweep = time.monotonic() + try: + await self.sweep_staging() + except Exception: + log.warning('staging sweep failed', exc_info=True) + try: + await asyncio.wait_for(self.stopping.wait(), 5) + except asyncio.TimeoutError: + pass + + async def sweep_staging(self) -> None: + old = await self.storage.list_older('staging/', self.settings.staging_retention) + if not old: + return + async with self.db.connection() as conn: + async with conn.transaction(): + if not await conn.fetchval('SELECT pg_try_advisory_xact_lock(20260006)'): + return + # Удаляем только staging, по которому НЕТ текущей работы (queued/processing). + active = {r['staging_key'] for r in await conn.fetch( + "SELECT staging_key FROM attachments WHERE staging_key = ANY($1::text[]) " + "AND status IN ('queued', 'processing')", old)} + removed = 0 + for key in old: + if key not in active: + await self.storage.delete(key) + removed += 1 + log.info('staging retention: removed %d objects older than %ds', removed, self.settings.staging_retention) + + # ---------- жизненный цикл ---------- + + async def run(self) -> None: + await self.db.start() + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, self.stopping.set) + housekeeping = asyncio.create_task(self.maintenance()) + while True: + try: + self.connection = await broker.connect(self.settings.broker_url) + break + except Exception as exc: + if self.stopping.is_set(): + return + log.warning('broker not reachable (%s), retrying', type(exc).__name__) + await asyncio.sleep(2) + channel = await self.connection.channel() + # prefetch = сколько неподтверждённых сообщений брокер отдаёт этому worker (ограниченная конкуренция). + await channel.set_qos(prefetch_count=self.settings.worker_prefetch) + _exchange, queue = await broker.declare(channel) + tag = await queue.consume(self.on_message, no_ack=False) + log.info('worker %s consuming %s, concurrency=%d prefetch=%d', self.worker_id, broker.QUEUE, + self.settings.worker_concurrency, self.settings.worker_prefetch) + await self.stopping.wait() + # Graceful: перестать брать новые, дождаться текущих (неподтверждённые prefetch вернутся в очередь). + await queue.cancel(tag) + if self.inflight: + log.info('draining %d in-flight jobs', len(self.inflight)) + await asyncio.wait(self.inflight, timeout=self.settings.job_timeout) + housekeeping.cancel() + await self.connection.close() + await self.db.close() + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s %(name)s %(message)s') + asyncio.run(Worker().run()) + + +if __name__ == '__main__': + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..054c2d87 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,334 @@ +# Стенд lab5: lb (HAProxy) -> api x N -> postgres (метаданные, jobs, outbox) + s3 (RustFS: staging/, originals/, +# variants/) + rabbitmq (очередь заданий) <- worker (обработка изображений) ; valkey — кэш сессий. +# migrate и s3-init — одноразовый bootstrap схемы БД и приватного bucket. +# Подготовка: sh scripts/init-secrets.sh +# Запуск: docker compose --env-file .env up -d --build --wait +name: mtusi-lab5 + +x-api-env: &api-env + APP_HOST: ${APP_HOST:-0.0.0.0} + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: ${SESSION_STORE:-postgres+valkey} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + SHUTDOWN_DRAIN_MS: ${SHUTDOWN_DRAIN_MS:-3000} + DATABASE_URL_FILE: ${DATABASE_URL_FILE:-/run/secrets/database_url} + CACHE_URL_FILE: ${CACHE_URL_FILE:-/run/secrets/cache_url} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + DB_ACQUIRE_TIMEOUT_MS: ${DB_ACQUIRE_TIMEOUT_MS:-2000} + CACHE_TIMEOUT_MS: ${CACHE_TIMEOUT_MS:-300} + PASSWORD_HASH_WORKERS: ${PASSWORD_HASH_WORKERS:-2} + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: ${S3_ACCESS_KEY_ID_FILE:-/run/secrets/s3_access_key_id} + S3_SECRET_ACCESS_KEY_FILE: ${S3_SECRET_ACCESS_KEY_FILE:-/run/secrets/s3_secret_access_key} + S3_TIMEOUT_MS: ${S3_TIMEOUT_MS:-5000} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + IMAGE_CHECK_WORKERS: ${IMAGE_CHECK_WORKERS:-2} + BROKER_URL_FILE: ${BROKER_URL_FILE:-/run/secrets/broker_url} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15} + RETRY_BASE_MS: ${RETRY_BASE_MS:-1000} + RETRY_MAX_MS: ${RETRY_MAX_MS:-60000} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200} + OUTBOX_POLL_MS: ${OUTBOX_POLL_MS:-500} + WORKER_TASK_MEMORY_MB: ${WORKER_TASK_MEMORY_MB:-640} + +x-api-depends: &api-depends + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + # bucket создан и проверен на приватность до старта API + condition: service_completed_successfully + rabbitmq: + # Брокер не критичен для приёма: API пишет намерение в outbox и допубликует позже. + condition: service_started + required: false + valkey: + # Valkey — кэш, не источник истины: API должен стартовать и без него. + condition: service_started + required: false + +x-api-limits: &api-limits + limits: + cpus: "${API_CPUS:-1.0}" + memory: ${API_MEMORY:-384M} # 2 Argon2id x 64 MiB + 2 загрузки x (10 MiB тело + проверка) + интерпретатор + +services: + lb: + image: haproxy:3.2.8-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + # Единственная публичная точка входа API (только loopback хоста) + страница состояния LB. + - "127.0.0.1:${LB_PUBLISHED_PORT:-8080}:8080" + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" + networks: [edge] + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/stats"] + interval: 5s + timeout: 2s + retries: 5 + depends_on: + api: + condition: service_healthy + deploy: + resources: + limits: {cpus: "1.0", memory: 128M} + restart: unless-stopped + + api: + image: mtusi-lab5-api:local + build: . + # Нет container_name и ports: реплики взаимозаменяемы, доступны только через lb. + deploy: + replicas: ${API_REPLICAS:-2} + resources: *api-limits + environment: *api-env + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url] + networks: [edge, data] + # Корневая ФС только для чтения, tmpfs /tmp — у API нет места для локальной копии оригинала. + read_only: true + tmpfs: [/tmp] + stop_grace_period: 20s # drain 3 с + graceful shutdown uvicorn до 10 с, с запасом + depends_on: *api-depends + restart: unless-stopped + + migrate: + # Один запуск на весь стенд (а не в каждой реплике): pg_advisory_lock + schema_migrations. + build: . + image: mtusi-lab5-api:local + command: ["python", "-m", "app.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + SESSION_STORE: postgres + secrets: [database_url] + networks: [data] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: "no" + + # S3-совместимое хранилище оригиналов. Порты не публикуются на хост: доступ только из сети data + # (API и s3-init). Веб-консоль RustFS выключена (по умолчанию, RUSTFS_CONSOLE_ENABLE не задан). + s3: + image: rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 + environment: + RUSTFS_ACCESS_KEY_FILE: /run/secrets/s3_access_key_id + RUSTFS_SECRET_KEY_FILE: /run/secrets/s3_secret_access_key + RUSTFS_REGION: ${S3_REGION:-us-east-1} + RUSTFS_VOLUMES: /data + RUSTFS_OBS_LOG_DIRECTORY: "" # логи в stdout контейнера + RUSTFS_OBS_LOGGER_LEVEL: warn + secrets: [s3_access_key_id, s3_secret_access_key] + volumes: + - s3data:/data + networks: [data] + healthcheck: + test: ["CMD", "curl", "-fs", "-o", "/dev/null", "http://127.0.0.1:9000/health"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + stop_grace_period: 20s + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + restart: unless-stopped + + s3-init: + # Идемпотентный bootstrap: bucket создаётся, если его нет; анонимная политика удаляется; + # анонимный GET проверяется на отказ. Повторный запуск данные не трогает. + image: mtusi-lab5-api:local + command: ["python", "-m", "app.s3_bootstrap"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3: + condition: service_healthy + migrate: + condition: service_completed_successfully # образ api уже собран сервисом migrate + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + restart: "no" + + # Брокер заданий. Quorum queue + persistent messages на volume rabbitdata переживают рестарт. + # hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume. + # Порты не публикуются; management UI не включён (только встроенный prometheus на 15692 в сети data). + rabbitmq: + image: rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5 + hostname: rabbitmq + secrets: + - source: rabbitmq_conf + target: /etc/rabbitmq/conf.d/90-credentials.conf + volumes: + - rabbitdata:/var/lib/rabbitmq + networks: [data] + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 5s + timeout: 10s + retries: 30 + start_period: 10s + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + restart: unless-stopped + + # Worker: отдельный процесс обработки (тот же образ, другая команда). Не в сети edge, портов нет. + worker: + image: mtusi-lab5-api:local + command: ["python", "-m", "app.worker"] + environment: + <<: *api-env + SESSION_STORE: postgres # worker не работает с сессиями + secrets: [database_url, s3_access_key_id, s3_secret_access_key, broker_url] + networks: [data] + read_only: true + tmpfs: [/tmp] + healthcheck: + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-alive')<20 else 1)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 20s + stop_grace_period: 70s # дождаться текущих заданий (JOB_TIMEOUT_SECONDS) + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: + cpus: "${WORKER_CPUS:-2.0}" + memory: ${WORKER_MEMORY:-1536M} # WORKER_CONCURRENCY x WORKER_TASK_MEMORY_MB + процесс worker + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + rabbitmq: + condition: service_healthy + restart: unless-stopped + + postgres: + image: postgres:17.7-bookworm + environment: + POSTGRES_USER: app + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql/data + networks: [data] + healthcheck: + test: ["CMD", "pg_isready", "-U", "app", "-d", "messenger"] + interval: 2s + timeout: 3s + retries: 30 + shm_size: 128m + stop_grace_period: 30s # даём PostgreSQL завершиться с checkpoint, без recovery при старте + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2.0}", memory: 768M} + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1.4 + command: ["valkey-server", "/run/secrets/valkey_conf"] + secrets: [valkey_conf] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "valkey-cli --no-auth-warning -a \"$$(sed -n 's/^requirepass //p' /run/secrets/valkey_conf)\" ping | grep -q PONG"] + interval: 2s + timeout: 3s + retries: 30 + deploy: + resources: + limits: {cpus: "0.5", memory: 96M} + restart: unless-stopped + + # Отдельная реплика с коротким TTL только для автотеста истечения сессии (lab2, уровень 4). + # Порт не публикуется: tester обращается к ней по внутренней сети. + api-ttl: + profiles: [test] + image: mtusi-lab5-api:local + environment: + <<: *api-env + SESSION_TTL_SECONDS: "2" + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key, broker_url] + networks: [edge, data] + read_only: true + tmpfs: [/tmp] + deploy: + resources: *api-limits + depends_on: *api-depends + + # Linux-окружение для make/тестов/hook-скриптов (нужно на Windows без make и WSL). + tester: + profiles: [tools] + build: + context: . + dockerfile: tools/tester.Dockerfile + image: mtusi-lab5-tester:local + working_dir: /work + environment: + BASE_URL: http://lb:8080 + TTL_BASE_URL: http://api-ttl:8080 + LB_STATS_URL: http://lb:8404/stats + PYTHONUTF8: "1" + PYTHONDONTWRITEBYTECODE: "1" + networks: [edge] + volumes: + - .:/work + - /var/run/docker.sock:/var/run/docker.sock + +networks: + # edge: lb <-> api (+ tester). data: api/migrate <-> хранилища; lb в неё не входит. + edge: + data: + internal: true + +volumes: + pgdata: + s3data: + rabbitdata: + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + valkey_conf: + file: ./secrets/valkey.conf + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + broker_url: + file: ./secrets/broker_url + rabbitmq_conf: + file: ./secrets/rabbitmq.conf diff --git a/evidence/failover-hook.txt b/evidence/failover-hook.txt new file mode 100644 index 00000000..caa84ef0 --- /dev/null +++ b/evidence/failover-hook.txt @@ -0,0 +1,17 @@ +# 2026-11-17T15:27:35Z, commit 276dbda +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 137, "failed_attempts": 0, "first_recovery_seconds": 4.458} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab5-api-2 Up 5 seconds (healthy) +mtusi-lab5-api-1 Up About a minute (healthy) +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://lb:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 137, "failed_attempts": 0, "first_recovery_seconds": 4.163} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab5-api-2 Up 31 seconds (healthy) +mtusi-lab5-api-1 Up 5 seconds (healthy) + +# evidence/stop-one.log (what the hook really stopped): +2026-11-17T15:27:38Z mode=stop instance=cdb3725c641e-b9c808 container=mtusi-lab5-api-2 state=exited exit=0 finished=2026-11-17T15:27:42.125871236Z +2026-11-17T15:28:04Z mode=stop instance=824c92061861-f52bc3 container=mtusi-lab5-api-1 state=exited exit=0 finished=2026-11-17T15:28:07.582211619Z diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..c3989f5b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,25 @@ +# 2026-11-17T15:28:26Z, commit 276dbda +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://lb:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +# hook scope (StartedAt before/after of every process): +before: +/mtusi-lab5-api-1 2026-11-17T15:28:44.882592618Z +/mtusi-lab5-api-2 2026-11-17T15:28:44.5554688Z +/mtusi-lab5-lb-1 2026-11-17T15:28:50.680908369Z +/mtusi-lab5-postgres-1 2026-11-17T15:28:36.844686149Z +/mtusi-lab5-rabbitmq-1 2026-11-17T15:28:37.794780107Z +/mtusi-lab5-s3-1 2026-11-17T15:28:36.68421067Z +/mtusi-lab5-valkey-1 2026-11-17T15:28:36.995474977Z +/mtusi-lab5-worker-1 2026-11-17T15:28:45.270340876Z +after: +/mtusi-lab5-api-1 2026-11-17T15:29:08.027213384Z +/mtusi-lab5-api-2 2026-11-17T15:29:07.790819877Z +/mtusi-lab5-lb-1 2026-11-17T15:29:13.806020318Z +/mtusi-lab5-postgres-1 2026-11-17T15:28:59.585810192Z +/mtusi-lab5-rabbitmq-1 2026-11-17T15:29:00.317641583Z +/mtusi-lab5-s3-1 2026-11-17T15:28:59.897534526Z +/mtusi-lab5-valkey-1 2026-11-17T15:28:59.750297747Z +/mtusi-lab5-worker-1 2026-11-17T15:29:07.569183681Z +ready diff --git a/evidence/public-tests.txt b/evidence/public-tests.txt new file mode 100644 index 00000000..56b1ec9d --- /dev/null +++ b/evidence/public-tests.txt @@ -0,0 +1,7 @@ +# 2026-11-17T15:27:18Z, commit 276dbda +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. +python3 tests/smoke.py --base-url "http://lb:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stand.txt b/evidence/stand.txt new file mode 100644 index 00000000..ecddfca2 --- /dev/null +++ b/evidence/stand.txt @@ -0,0 +1,35 @@ +# 2026-11-17T15:27:09Z, commit 276dbda +NAME IMAGE STATUS PORTS +mtusi-lab5-api-1 mtusi-lab5-api:local Up 37 seconds (healthy) 8080/tcp +mtusi-lab5-api-2 mtusi-lab5-api:local Up 38 seconds (healthy) 8080/tcp +mtusi-lab5-api-ttl-1 sha256:d872bf2ffa4ed28b4db76ab08df13745276eff2c8b4e56b54531370f3eb6f2fb Exited (137) 37 minutes ago +mtusi-lab5-lb-1 haproxy:3.2.8-alpine Up 6 hours (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +mtusi-lab5-migrate-1 mtusi-lab5-api:local Exited (0) 12 minutes ago +mtusi-lab5-postgres-1 postgres:17.7-bookworm Up 37 minutes (healthy) 5432/tcp +mtusi-lab5-rabbitmq-1 rabbitmq:4.3.6-alpine@sha256:6931816af1a943f471c038b278835d36692d55dec769232739296bb91b7de0f5 Up 36 minutes (healthy) 4369/tcp, 5671-5672/tcp, 15691-15692/tcp, 25672/tcp +mtusi-lab5-s3-1 rustfs/rustfs:1.0.4@sha256:ed9210d6411d77ffceb70d94b964a3cb7d225983368bc25bebe01e995293dcc2 Up 35 minutes (healthy) 9000-9001/tcp +mtusi-lab5-s3-init-1 mtusi-lab5-api:local Exited (0) 12 minutes ago +mtusi-lab5-valkey-1 valkey/valkey:8.1.4 Up 37 minutes (healthy) 6379/tcp +mtusi-lab5-worker-1 mtusi-lab5-api:local Up About a minute (healthy) 8080/tcp + +# s3-init (idempotent bucket bootstrap + anonymous access check): +bucket messenger-attachments: exists +bucket messenger-attachments: no bucket policy (private) +anonymous GET messenger-attachments/_bootstrap/private-check: HTTP 403 (denied) + +# repeated bootstrap run (must not touch existing bucket): +bucket messenger-attachments: exists +bucket messenger-attachments: no bucket policy (private) +anonymous GET messenger-attachments/_bootstrap/private-check: HTTP 403 (denied) + +# volumes: +mtusi-lab5_pgdata +mtusi-lab5_rabbitdata +mtusi-lab5_s3data + +# RabbitMQ: очереди (quorum), consumers и prefetch: +name type durable messages consumers +attachment.process.v1.dlq quorum true 2 0 +attachment.process.v1 quorum true 0 1 +queue_name ack_required prefetch_count +attachment.process.v1 true 2 diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..dcca9ef9 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,57 @@ +# 2026-11-17T15:29:15Z, commit 276dbda +PASS test_variants_contract (3.2s) + worker stopped; upload 202 attachment=89ff98d0-59f5-4296-a4ac-455e23e234a0 + api and rabbitmq restarted; queue depth before/after broker restart: 1/1 + worker started -> ready in 2.6s, thumbnail OK +PASS test_worker_down_demo (24.4s) +PASS test_failed_state_and_409 (4.7s) +PASS test_duplicate_delivery_is_noop (7.0s) + S3 down: failed retry_exhausted after 5 attempts, 15s + backoff delays (s): 0.6, 1.5, 2.9, 7.7 +PASS test_retry_bounded_then_replay (28.8s) + client saw: connection error (API process exited after COMMIT) + attachment e6e30a4d-1e9e-47b2-8ef4-8976af20b767 reached ready via outbox relay +PASS test_crash_between_commit_and_publish (2.3s) + crash after S3 writes -> ready on attempt 2 in 17.8s, 3 objects in S3 +PASS test_worker_crash_after_s3_before_commit (20.5s) +PASS test_worker_crash_after_commit_before_ack (14.6s) +PASS test_expired_lease_takeover (5.5s) +PASS test_staging_cleanup_after_commit (2.2s) +PASS test_poison_message_to_dlq (3.8s) +PASS test_prefetch_limited (1.4s) + mem: resource_limit; time: processing_timeout; alpha: (255, 255, 255); exif: (85, 256) None +PASS test_worker_resource_limits (2.3s) +13/13 passed +PASS test_png_and_jpeg_roundtrip (1.9s) +PASS test_message_with_attachments (1.9s) +PASS test_foreign_attachment_rejected (1.1s) +PASS test_limits_and_mime (1.2s) +PASS test_dangerous_filenames (1.9s) + anonymous GET object/list -> 403 +PASS test_bucket_private_outside_api (2.2s) +PASS test_no_local_copy_on_api (1.9s) + uploaded via cdb3725c641e-42f61c, downloaded via 824c92061861-21c48d +PASS test_failover_of_uploading_replica (8.0s) +PASS test_s3_unavailable (14.8s) +9/9 passed +PASS test_session_across_replicas (0.6s) +PASS test_same_key_concurrent_across_replicas (2.6s) +PASS test_same_key_concurrent_different_bodies (1.7s) +PASS test_key_scope_and_format (1.5s) +PASS test_replay_rechecks_session (0.8s) +PASS test_transient_failure_not_recorded (6.0s) +PASS test_sigterm_drain (10.3s) +PASS test_infrastructure_rules (0.5s) +PASS test_nul_in_text_roundtrip (1.3s) +9/9 passed +PASS test_wrong_password (0.6s) +PASS test_logout (0.5s) +PASS test_short_ttl (3.4s) +PASS test_username_race (3.8s) +PASS test_chat_rollback (0.9s) +PASS test_cursor_rules (1.3s) +PASS test_valkey_cache_role (1.0s) +PASS test_revoke_during_cache_partition (1.6s) +PASS test_valkey_down (4.5s) +PASS test_postgres_down (10.8s) +10/10 passed diff --git a/evidence/track1/backpressure-large.json b/evidence/track1/backpressure-large.json new file mode 100644 index 00000000..65a60973 --- /dev/null +++ b/evidence/track1/backpressure-large.json @@ -0,0 +1,97 @@ +{ + "label": "backpressure-large", + "size": "large", + "bytes": 3015368, + "count": 120, + "rate_per_s": 15.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "20", + "upload_status": { + "202": 48, + "503": 72 + }, + "accepted": 48, + "ready": 48, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.94, + "drain_seconds": 13.33, + "throughput_ready_per_s": 3.691, + "time_to_ready_s": { + "p50": 4.937, + "p95": 5.646, + "p99": 5.682, + "max": 5.682, + "mean": 4.161 + }, + "upload_latency_s": { + "p50": 0.007, + "p95": 0.142 + }, + "generator_max_late_s": 0.004, + "queue_lag": { + "max_pending_jobs": 22, + "max_broker_messages": 20 + }, + "worker_memory_mib": { + "peak": 236.9, + "idle_first": 110.5 + }, + "timeline": { + "pending": [ + [ + 0.0, + 2 + ], + [ + 4.58, + 19 + ], + [ + 5.34, + 21 + ], + [ + 7.73, + 22 + ], + [ + 8.5, + 18 + ], + [ + 13.21, + 0 + ] + ], + "broker": [ + [ + 0.0, + 20 + ], + [ + 8.5, + 19 + ] + ], + "memory": [ + [ + 0.0, + 110.5 + ], + [ + 5.34, + 236.9 + ], + [ + 8.5, + 236.0 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/baseline-small.json b/evidence/track1/baseline-small.json new file mode 100644 index 00000000..5cc33745 --- /dev/null +++ b/evidence/track1/baseline-small.json @@ -0,0 +1,104 @@ +{ + "label": "baseline-small", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 5.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 11.82, + "drain_seconds": 12.1, + "throughput_ready_per_s": 5.031, + "time_to_ready_s": { + "p50": 0.115, + "p95": 1.171, + "p99": 1.377, + "max": 1.377, + "mean": 0.248 + }, + "upload_latency_s": { + "p50": 0.024, + "p95": 0.035 + }, + "generator_max_late_s": 0.003, + "queue_lag": { + "max_pending_jobs": 1, + "max_broker_messages": 4 + }, + "worker_memory_mib": { + "peak": 109.7, + "idle_first": 101.1 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 3.67, + 1 + ], + [ + 4.4, + 0 + ], + [ + 6.68, + 1 + ], + [ + 7.43, + 1 + ], + [ + 11.08, + 1 + ], + [ + 11.81, + 0 + ] + ], + "broker": [ + [ + 0.0, + 4 + ], + [ + 7.43, + 1 + ] + ], + "memory": [ + [ + 0.0, + 101.1 + ], + [ + 4.4, + 109.7 + ], + [ + 7.43, + 98.9 + ], + [ + 11.81, + 98.9 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c1-p1-large-r1.json b/evidence/track1/c1-p1-large-r1.json new file mode 100644 index 00000000..248ec253 --- /dev/null +++ b/evidence/track1/c1-p1-large-r1.json @@ -0,0 +1,188 @@ +{ + "label": "c1-p1-large-r1", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.57, + "drain_seconds": 54.51, + "throughput_ready_per_s": 1.131, + "time_to_ready_s": { + "p50": 20.214, + "p95": 44.534, + "p99": 45.674, + "max": 45.674, + "mean": 21.052 + }, + "upload_latency_s": { + "p50": 0.143, + "p95": 0.209 + }, + "generator_max_late_s": 0.002, + "queue_lag": { + "max_pending_jobs": 50, + "max_broker_messages": 43 + }, + "worker_memory_mib": { + "peak": 172.9, + "idle_first": 105.8 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 6.22, + 44 + ], + [ + 7.27, + 50 + ], + [ + 9.82, + 46 + ], + [ + 10.57, + 44 + ], + [ + 14.83, + 37 + ], + [ + 15.67, + 36 + ], + [ + 18.82, + 34 + ], + [ + 20.43, + 33 + ], + [ + 27.3, + 27 + ], + [ + 28.37, + 26 + ], + [ + 31.19, + 23 + ], + [ + 32.43, + 22 + ], + [ + 39.52, + 13 + ], + [ + 40.41, + 12 + ], + [ + 42.94, + 9 + ], + [ + 43.8, + 7 + ], + [ + 51.79, + 2 + ], + [ + 52.68, + 1 + ] + ], + "broker": [ + [ + 0.0, + 11 + ], + [ + 10.57, + 43 + ], + [ + 20.43, + 32 + ], + [ + 32.43, + 18 + ], + [ + 43.8, + 6 + ] + ], + "memory": [ + [ + 0.0, + 105.8 + ], + [ + 7.27, + 115.9 + ], + [ + 10.57, + 120.7 + ], + [ + 15.67, + 146.7 + ], + [ + 20.43, + 172.7 + ], + [ + 28.37, + 172.9 + ], + [ + 32.43, + 116.0 + ], + [ + 40.41, + 130.2 + ], + [ + 43.8, + 116.2 + ], + [ + 52.68, + 119.1 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c1-p1-large-r2.json b/evidence/track1/c1-p1-large-r2.json new file mode 100644 index 00000000..56963d60 --- /dev/null +++ b/evidence/track1/c1-p1-large-r2.json @@ -0,0 +1,180 @@ +{ + "label": "c1-p1-large-r2", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.53, + "drain_seconds": 44.28, + "throughput_ready_per_s": 1.385, + "time_to_ready_s": { + "p50": 18.493, + "p95": 34.835, + "p99": 36.043, + "max": 36.043, + "mean": 18.892 + }, + "upload_latency_s": { + "p50": 0.166, + "p95": 0.473 + }, + "generator_max_late_s": 0.004, + "queue_lag": { + "max_pending_jobs": 50, + "max_broker_messages": 47 + }, + "worker_memory_mib": { + "peak": 185.6, + "idle_first": 176.1 + }, + "timeline": { + "pending": [ + [ + 0.0, + 2 + ], + [ + 5.13, + 39 + ], + [ + 6.05, + 45 + ], + [ + 8.57, + 50 + ], + [ + 9.46, + 49 + ], + [ + 13.98, + 43 + ], + [ + 14.86, + 41 + ], + [ + 17.29, + 37 + ], + [ + 18.14, + 36 + ], + [ + 23.74, + 28 + ], + [ + 24.69, + 27 + ], + [ + 27.22, + 23 + ], + [ + 28.12, + 22 + ], + [ + 34.56, + 13 + ], + [ + 35.4, + 11 + ], + [ + 37.92, + 8 + ], + [ + 38.79, + 6 + ], + [ + 43.93, + 0 + ] + ], + "broker": [ + [ + 0.0, + 10 + ], + [ + 9.46, + 47 + ], + [ + 18.14, + 32 + ], + [ + 28.12, + 18 + ], + [ + 38.79, + 3 + ] + ], + "memory": [ + [ + 0.0, + 176.1 + ], + [ + 6.05, + 141.2 + ], + [ + 9.46, + 128.0 + ], + [ + 14.86, + 128.0 + ], + [ + 18.14, + 130.1 + ], + [ + 24.69, + 132.4 + ], + [ + 28.12, + 170.3 + ], + [ + 35.4, + 185.6 + ], + [ + 38.79, + 185.2 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c1-p1-small-r1.json b/evidence/track1/c1-p1-small-r1.json new file mode 100644 index 00000000..dcd60324 --- /dev/null +++ b/evidence/track1/c1-p1-small-r1.json @@ -0,0 +1,180 @@ +{ + "label": "c1-p1-small-r1", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.0, + "drain_seconds": 39.01, + "throughput_ready_per_s": 5.256, + "time_to_ready_s": { + "p50": 14.65, + "p95": 30.791, + "p99": 32.731, + "max": 33.144, + "mean": 15.373 + }, + "upload_latency_s": { + "p50": 0.023, + "p95": 0.037 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 177, + "max_broker_messages": 141 + }, + "worker_memory_mib": { + "peak": 109.7, + "idle_first": 91.2 + }, + "timeline": { + "pending": [ + [ + 0.0, + 10 + ], + [ + 3.85, + 151 + ], + [ + 4.61, + 177 + ], + [ + 6.95, + 167 + ], + [ + 7.67, + 162 + ], + [ + 12.33, + 133 + ], + [ + 13.1, + 128 + ], + [ + 15.38, + 111 + ], + [ + 16.12, + 106 + ], + [ + 20.66, + 78 + ], + [ + 21.39, + 73 + ], + [ + 23.68, + 57 + ], + [ + 24.43, + 52 + ], + [ + 28.12, + 28 + ], + [ + 29.15, + 25 + ], + [ + 31.87, + 21 + ], + [ + 33.11, + 18 + ], + [ + 38.16, + 0 + ] + ], + "broker": [ + [ + 0.0, + 41 + ], + [ + 7.67, + 141 + ], + [ + 16.12, + 108 + ], + [ + 24.43, + 43 + ], + [ + 33.11, + 8 + ] + ], + "memory": [ + [ + 0.0, + 91.2 + ], + [ + 4.61, + 98.0 + ], + [ + 7.67, + 97.9 + ], + [ + 13.1, + 103.0 + ], + [ + 16.12, + 98.1 + ], + [ + 21.39, + 98.5 + ], + [ + 24.43, + 98.6 + ], + [ + 29.15, + 98.1 + ], + [ + 33.11, + 109.7 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c1-p1-small-r2.json b/evidence/track1/c1-p1-small-r2.json new file mode 100644 index 00000000..6fa685e1 --- /dev/null +++ b/evidence/track1/c1-p1-small-r2.json @@ -0,0 +1,204 @@ +{ + "label": "c1-p1-small-r2", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.35, + "drain_seconds": 53.7, + "throughput_ready_per_s": 3.745, + "time_to_ready_s": { + "p50": 24.775, + "p95": 45.564, + "p99": 47.589, + "max": 48.149, + "mean": 24.871 + }, + "upload_latency_s": { + "p50": 0.04, + "p95": 0.62 + }, + "generator_max_late_s": 0.11, + "queue_lag": { + "max_pending_jobs": 181, + "max_broker_messages": 156 + }, + "worker_memory_mib": { + "peak": 130.1, + "idle_first": 123.8 + }, + "timeline": { + "pending": [ + [ + 0.0, + 9 + ], + [ + 4.82, + 161 + ], + [ + 5.66, + 181 + ], + [ + 8.29, + 171 + ], + [ + 9.15, + 168 + ], + [ + 13.92, + 149 + ], + [ + 14.76, + 145 + ], + [ + 17.23, + 135 + ], + [ + 18.13, + 132 + ], + [ + 24.55, + 110 + ], + [ + 25.46, + 107 + ], + [ + 27.93, + 97 + ], + [ + 28.88, + 94 + ], + [ + 35.24, + 72 + ], + [ + 36.13, + 68 + ], + [ + 38.69, + 58 + ], + [ + 39.64, + 54 + ], + [ + 44.54, + 34 + ], + [ + 45.38, + 31 + ], + [ + 47.97, + 21 + ], + [ + 48.86, + 17 + ] + ], + "broker": [ + [ + 0.0, + 86 + ], + [ + 9.15, + 156 + ], + [ + 18.13, + 118 + ], + [ + 28.88, + 85 + ], + [ + 39.64, + 46 + ], + [ + 48.86, + 5 + ] + ], + "memory": [ + [ + 0.0, + 123.8 + ], + [ + 5.66, + 119.3 + ], + [ + 9.15, + 119.2 + ], + [ + 14.76, + 119.4 + ], + [ + 18.13, + 119.7 + ], + [ + 25.46, + 126.6 + ], + [ + 28.88, + 129.2 + ], + [ + 36.13, + 130.0 + ], + [ + 39.64, + 119.5 + ], + [ + 45.38, + 130.1 + ], + [ + 48.86, + 119.6 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p16-large-r1.json b/evidence/track1/c2-p16-large-r1.json new file mode 100644 index 00000000..eec8203c --- /dev/null +++ b/evidence/track1/c2-p16-large-r1.json @@ -0,0 +1,120 @@ +{ + "label": "c2-p16-large-r1", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.51, + "drain_seconds": 21.1, + "throughput_ready_per_s": 2.98, + "time_to_ready_s": { + "p50": 8.645, + "p95": 12.515, + "p99": 12.789, + "max": 12.789, + "mean": 7.964 + }, + "upload_latency_s": { + "p50": 0.167, + "p95": 1.092 + }, + "generator_max_late_s": 0.01, + "queue_lag": { + "max_pending_jobs": 43, + "max_broker_messages": 40 + }, + "worker_memory_mib": { + "peak": 238.7, + "idle_first": 117.7 + }, + "timeline": { + "pending": [ + [ + 0.0, + 2 + ], + [ + 5.13, + 29 + ], + [ + 6.27, + 43 + ], + [ + 9.23, + 42 + ], + [ + 10.21, + 38 + ], + [ + 14.18, + 23 + ], + [ + 14.98, + 21 + ], + [ + 17.39, + 11 + ], + [ + 18.21, + 8 + ] + ], + "broker": [ + [ + 0.0, + 1 + ], + [ + 10.21, + 40 + ], + [ + 18.21, + 1 + ] + ], + "memory": [ + [ + 0.0, + 117.7 + ], + [ + 6.27, + 136.3 + ], + [ + 10.21, + 238.7 + ], + [ + 14.98, + 194.3 + ], + [ + 18.21, + 195.0 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p16-large-r2.json b/evidence/track1/c2-p16-large-r2.json new file mode 100644 index 00000000..eee704f4 --- /dev/null +++ b/evidence/track1/c2-p16-large-r2.json @@ -0,0 +1,120 @@ +{ + "label": "c2-p16-large-r2", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.47, + "drain_seconds": 18.25, + "throughput_ready_per_s": 3.475, + "time_to_ready_s": { + "p50": 5.555, + "p95": 9.662, + "p99": 9.912, + "max": 9.912, + "mean": 5.505 + }, + "upload_latency_s": { + "p50": 0.11, + "p95": 0.168 + }, + "generator_max_late_s": 0.006, + "queue_lag": { + "max_pending_jobs": 36, + "max_broker_messages": 34 + }, + "worker_memory_mib": { + "peak": 204.8, + "idle_first": 138.9 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 4.2, + 24 + ], + [ + 5.04, + 27 + ], + [ + 7.44, + 36 + ], + [ + 8.25, + 34 + ], + [ + 13.41, + 15 + ], + [ + 14.19, + 12 + ], + [ + 16.58, + 3 + ], + [ + 17.36, + 0 + ] + ], + "broker": [ + [ + 0.0, + 16 + ], + [ + 8.25, + 34 + ], + [ + 17.36, + 0 + ] + ], + "memory": [ + [ + 0.0, + 138.9 + ], + [ + 5.04, + 204.8 + ], + [ + 8.25, + 190.6 + ], + [ + 14.19, + 181.0 + ], + [ + 17.36, + 147.6 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p16-small-r1.json b/evidence/track1/c2-p16-small-r1.json new file mode 100644 index 00000000..7fad4947 --- /dev/null +++ b/evidence/track1/c2-p16-small-r1.json @@ -0,0 +1,120 @@ +{ + "label": "c2-p16-small-r1", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.01, + "drain_seconds": 20.95, + "throughput_ready_per_s": 9.604, + "time_to_ready_s": { + "p50": 8.479, + "p95": 15.174, + "p99": 15.711, + "max": 15.851, + "mean": 8.697 + }, + "upload_latency_s": { + "p50": 0.025, + "p95": 0.035 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 165, + "max_broker_messages": 107 + }, + "worker_memory_mib": { + "peak": 117.2, + "idle_first": 100.4 + }, + "timeline": { + "pending": [ + [ + 0.0, + 9 + ], + [ + 3.99, + 143 + ], + [ + 4.77, + 165 + ], + [ + 7.27, + 138 + ], + [ + 8.0, + 129 + ], + [ + 13.91, + 70 + ], + [ + 14.78, + 62 + ], + [ + 17.12, + 36 + ], + [ + 17.9, + 28 + ] + ], + "broker": [ + [ + 0.0, + 15 + ], + [ + 8.0, + 107 + ], + [ + 17.9, + 6 + ] + ], + "memory": [ + [ + 0.0, + 100.4 + ], + [ + 4.77, + 112.8 + ], + [ + 8.0, + 110.4 + ], + [ + 14.78, + 117.0 + ], + [ + 17.9, + 117.2 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p16-small-r2.json b/evidence/track1/c2-p16-small-r2.json new file mode 100644 index 00000000..766b93c0 --- /dev/null +++ b/evidence/track1/c2-p16-small-r2.json @@ -0,0 +1,120 @@ +{ + "label": "c2-p16-small-r2", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.0, + "drain_seconds": 20.97, + "throughput_ready_per_s": 10.022, + "time_to_ready_s": { + "p50": 7.194, + "p95": 14.257, + "p99": 14.841, + "max": 14.998, + "mean": 7.499 + }, + "upload_latency_s": { + "p50": 0.025, + "p95": 0.035 + }, + "generator_max_late_s": 0.002, + "queue_lag": { + "max_pending_jobs": 148, + "max_broker_messages": 120 + }, + "worker_memory_mib": { + "peak": 149.3, + "idle_first": 144.1 + }, + "timeline": { + "pending": [ + [ + 0.0, + 7 + ], + [ + 5.09, + 148 + ], + [ + 5.89, + 139 + ], + [ + 8.23, + 113 + ], + [ + 8.98, + 104 + ], + [ + 13.3, + 63 + ], + [ + 14.15, + 56 + ], + [ + 16.59, + 31 + ], + [ + 17.46, + 25 + ] + ], + "broker": [ + [ + 0.0, + 88 + ], + [ + 8.98, + 120 + ], + [ + 17.46, + 24 + ] + ], + "memory": [ + [ + 0.0, + 144.1 + ], + [ + 5.89, + 149.3 + ], + [ + 8.98, + 148.1 + ], + [ + 14.15, + 139.2 + ], + [ + 17.46, + 143.6 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p2-large-r1.json b/evidence/track1/c2-p2-large-r1.json new file mode 100644 index 00000000..fc9a941c --- /dev/null +++ b/evidence/track1/c2-p2-large-r1.json @@ -0,0 +1,120 @@ +{ + "label": "c2-p2-large-r1", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.53, + "drain_seconds": 23.59, + "throughput_ready_per_s": 2.615, + "time_to_ready_s": { + "p50": 8.955, + "p95": 15.298, + "p99": 15.667, + "max": 15.667, + "mean": 8.775 + }, + "upload_latency_s": { + "p50": 0.167, + "p95": 0.332 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 41, + "max_broker_messages": 30 + }, + "worker_memory_mib": { + "peak": 239.2, + "idle_first": 138.2 + }, + "timeline": { + "pending": [ + [ + 0.0, + 2 + ], + [ + 4.95, + 32 + ], + [ + 5.97, + 37 + ], + [ + 8.59, + 41 + ], + [ + 9.57, + 38 + ], + [ + 14.5, + 25 + ], + [ + 15.36, + 22 + ], + [ + 17.83, + 15 + ], + [ + 18.69, + 12 + ] + ], + "broker": [ + [ + 0.0, + 18 + ], + [ + 9.57, + 30 + ], + [ + 18.69, + 2 + ] + ], + "memory": [ + [ + 0.0, + 138.2 + ], + [ + 5.97, + 187.7 + ], + [ + 9.57, + 222.7 + ], + [ + 15.36, + 229.2 + ], + [ + 18.69, + 239.2 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p2-large-r2.json b/evidence/track1/c2-p2-large-r2.json new file mode 100644 index 00000000..990e9ced --- /dev/null +++ b/evidence/track1/c2-p2-large-r2.json @@ -0,0 +1,124 @@ +{ + "label": "c2-p2-large-r2", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.51, + "drain_seconds": 27.53, + "throughput_ready_per_s": 2.25, + "time_to_ready_s": { + "p50": 11.312, + "p95": 19.302, + "p99": 19.573, + "max": 19.573, + "mean": 11.959 + }, + "upload_latency_s": { + "p50": 0.871, + "p95": 3.157 + }, + "generator_max_late_s": 0.019, + "queue_lag": { + "max_pending_jobs": 51, + "max_broker_messages": 50 + }, + "worker_memory_mib": { + "peak": 253.8, + "idle_first": 253.8 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 9.14, + 51 + ], + [ + 10.03, + 48 + ], + [ + 12.53, + 42 + ], + [ + 13.41, + 39 + ], + [ + 18.27, + 25 + ], + [ + 19.16, + 22 + ], + [ + 21.63, + 16 + ], + [ + 22.56, + 13 + ], + [ + 27.16, + 0 + ] + ], + "broker": [ + [ + 0.0, + 50 + ], + [ + 13.41, + 30 + ], + [ + 22.56, + 16 + ] + ], + "memory": [ + [ + 0.0, + 253.8 + ], + [ + 10.03, + 200.2 + ], + [ + 13.41, + 192.3 + ], + [ + 19.16, + 200.9 + ], + [ + 22.56, + 196.6 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p2-small-r1.json b/evidence/track1/c2-p2-small-r1.json new file mode 100644 index 00000000..bacf2ad5 --- /dev/null +++ b/evidence/track1/c2-p2-small-r1.json @@ -0,0 +1,148 @@ +{ + "label": "c2-p2-small-r1", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.03, + "drain_seconds": 34.07, + "throughput_ready_per_s": 6.085, + "time_to_ready_s": { + "p50": 16.317, + "p95": 26.997, + "p99": 27.79, + "max": 27.979, + "mean": 16.313 + }, + "upload_latency_s": { + "p50": 0.049, + "p95": 0.112 + }, + "generator_max_late_s": 0.009, + "queue_lag": { + "max_pending_jobs": 170, + "max_broker_messages": 180 + }, + "worker_memory_mib": { + "peak": 113.1, + "idle_first": 78.4 + }, + "timeline": { + "pending": [ + [ + 0.0, + 19 + ], + [ + 8.28, + 170 + ], + [ + 9.17, + 164 + ], + [ + 11.74, + 146 + ], + [ + 12.62, + 140 + ], + [ + 18.17, + 103 + ], + [ + 19.05, + 96 + ], + [ + 21.64, + 79 + ], + [ + 22.57, + 73 + ], + [ + 27.7, + 37 + ], + [ + 28.69, + 31 + ], + [ + 31.14, + 13 + ], + [ + 32.11, + 5 + ] + ], + "broker": [ + [ + 0.0, + 180 + ], + [ + 12.62, + 115 + ], + [ + 22.57, + 45 + ], + [ + 32.11, + 0 + ] + ], + "memory": [ + [ + 0.0, + 78.4 + ], + [ + 9.17, + 110.6 + ], + [ + 12.62, + 106.4 + ], + [ + 19.05, + 108.2 + ], + [ + 22.57, + 113.1 + ], + [ + 28.69, + 113.0 + ], + [ + 32.11, + 104.6 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c2-p2-small-r2.json b/evidence/track1/c2-p2-small-r2.json new file mode 100644 index 00000000..d2790731 --- /dev/null +++ b/evidence/track1/c2-p2-small-r2.json @@ -0,0 +1,152 @@ +{ + "label": "c2-p2-small-r2", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.01, + "drain_seconds": 38.38, + "throughput_ready_per_s": 5.29, + "time_to_ready_s": { + "p50": 19.744, + "p95": 31.828, + "p99": 32.749, + "max": 32.946, + "mean": 18.353 + }, + "upload_latency_s": { + "p50": 0.053, + "p95": 0.154 + }, + "generator_max_late_s": 0.014, + "queue_lag": { + "max_pending_jobs": 170, + "max_broker_messages": 131 + }, + "worker_memory_mib": { + "peak": 153.1, + "idle_first": 139.4 + }, + "timeline": { + "pending": [ + [ + 0.0, + 18 + ], + [ + 5.62, + 170 + ], + [ + 6.77, + 165 + ], + [ + 9.59, + 156 + ], + [ + 11.0, + 151 + ], + [ + 17.53, + 121 + ], + [ + 18.48, + 117 + ], + [ + 21.02, + 105 + ], + [ + 21.94, + 100 + ], + [ + 28.43, + 62 + ], + [ + 29.36, + 56 + ], + [ + 31.88, + 38 + ], + [ + 32.85, + 32 + ], + [ + 38.04, + 0 + ] + ], + "broker": [ + [ + 0.0, + 38 + ], + [ + 11.0, + 131 + ], + [ + 21.94, + 78 + ], + [ + 32.85, + 14 + ] + ], + "memory": [ + [ + 0.0, + 139.4 + ], + [ + 6.77, + 148.2 + ], + [ + 11.0, + 153.1 + ], + [ + 18.48, + 148.5 + ], + [ + 21.94, + 139.8 + ], + [ + 29.36, + 150.5 + ], + [ + 32.85, + 149.4 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c4-p4-large-r1.json b/evidence/track1/c4-p4-large-r1.json new file mode 100644 index 00000000..0baffb9e --- /dev/null +++ b/evidence/track1/c4-p4-large-r1.json @@ -0,0 +1,104 @@ +{ + "label": "c4-p4-large-r1", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.47, + "drain_seconds": 16.84, + "throughput_ready_per_s": 3.729, + "time_to_ready_s": { + "p50": 5.538, + "p95": 8.81, + "p99": 8.9, + "max": 8.9, + "mean": 5.31 + }, + "upload_latency_s": { + "p50": 0.131, + "p95": 0.237 + }, + "generator_max_late_s": 0.008, + "queue_lag": { + "max_pending_jobs": 36, + "max_broker_messages": 19 + }, + "worker_memory_mib": { + "peak": 248.5, + "idle_first": 174.6 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 6.89, + 36 + ], + [ + 7.83, + 35 + ], + [ + 10.29, + 25 + ], + [ + 11.07, + 21 + ], + [ + 15.37, + 4 + ], + [ + 16.26, + 0 + ] + ], + "broker": [ + [ + 0.0, + 11 + ], + [ + 11.07, + 19 + ] + ], + "memory": [ + [ + 0.0, + 174.6 + ], + [ + 7.83, + 196.0 + ], + [ + 11.07, + 248.5 + ], + [ + 16.26, + 159.9 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c4-p4-large-r2.json b/evidence/track1/c4-p4-large-r2.json new file mode 100644 index 00000000..5329afed --- /dev/null +++ b/evidence/track1/c4-p4-large-r2.json @@ -0,0 +1,104 @@ +{ + "label": "c4-p4-large-r2", + "size": "large", + "bytes": 3015368, + "count": 60, + "rate_per_s": 8.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.49, + "drain_seconds": 15.93, + "throughput_ready_per_s": 3.883, + "time_to_ready_s": { + "p50": 4.504, + "p95": 8.075, + "p99": 8.152, + "max": 8.152, + "mean": 4.407 + }, + "upload_latency_s": { + "p50": 0.114, + "p95": 0.225 + }, + "generator_max_late_s": 0.006, + "queue_lag": { + "max_pending_jobs": 30, + "max_broker_messages": 17 + }, + "worker_memory_mib": { + "peak": 338.9, + "idle_first": 286.2 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 5.59, + 27 + ], + [ + 6.5, + 30 + ], + [ + 8.92, + 27 + ], + [ + 9.77, + 23 + ], + [ + 14.76, + 3 + ], + [ + 15.6, + 0 + ] + ], + "broker": [ + [ + 0.0, + 17 + ], + [ + 9.77, + 9 + ] + ], + "memory": [ + [ + 0.0, + 286.2 + ], + [ + 6.5, + 338.9 + ], + [ + 9.77, + 324.9 + ], + [ + 15.6, + 170.1 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c4-p4-small-r1.json b/evidence/track1/c4-p4-small-r1.json new file mode 100644 index 00000000..d5080c56 --- /dev/null +++ b/evidence/track1/c4-p4-small-r1.json @@ -0,0 +1,104 @@ +{ + "label": "c4-p4-small-r1", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.0, + "drain_seconds": 15.92, + "throughput_ready_per_s": 13.171, + "time_to_ready_s": { + "p50": 5.745, + "p95": 9.89, + "p99": 10.199, + "max": 10.259, + "mean": 6.057 + }, + "upload_latency_s": { + "p50": 0.029, + "p95": 0.051 + }, + "generator_max_late_s": 0.004, + "queue_lag": { + "max_pending_jobs": 146, + "max_broker_messages": 86 + }, + "worker_memory_mib": { + "peak": 113.9, + "idle_first": 92.1 + }, + "timeline": { + "pending": [ + [ + 0.0, + 11 + ], + [ + 5.37, + 146 + ], + [ + 6.15, + 133 + ], + [ + 8.53, + 92 + ], + [ + 9.34, + 79 + ], + [ + 13.83, + 19 + ], + [ + 14.63, + 6 + ] + ], + "broker": [ + [ + 0.0, + 86 + ], + [ + 9.34, + 47 + ] + ], + "memory": [ + [ + 0.0, + 92.1 + ], + [ + 6.15, + 107.5 + ], + [ + 9.34, + 113.9 + ], + [ + 14.63, + 107.3 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/c4-p4-small-r2.json b/evidence/track1/c4-p4-small-r2.json new file mode 100644 index 00000000..3558fb96 --- /dev/null +++ b/evidence/track1/c4-p4-small-r2.json @@ -0,0 +1,104 @@ +{ + "label": "c4-p4-small-r2", + "size": "small", + "bytes": 111817, + "count": 200, + "rate_per_s": 40.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 200 + }, + "accepted": 200, + "ready": 200, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.01, + "drain_seconds": 16.28, + "throughput_ready_per_s": 12.521, + "time_to_ready_s": { + "p50": 5.741, + "p95": 10.568, + "p99": 10.924, + "max": 11.02, + "mean": 5.689 + }, + "upload_latency_s": { + "p50": 0.035, + "p95": 0.089 + }, + "generator_max_late_s": 0.007, + "queue_lag": { + "max_pending_jobs": 121, + "max_broker_messages": 137 + }, + "worker_memory_mib": { + "peak": 169.5, + "idle_first": 160.1 + }, + "timeline": { + "pending": [ + [ + 0.0, + 9 + ], + [ + 6.28, + 121 + ], + [ + 7.13, + 109 + ], + [ + 9.61, + 78 + ], + [ + 10.48, + 67 + ], + [ + 15.43, + 6 + ], + [ + 16.24, + 0 + ] + ], + "broker": [ + [ + 0.0, + 137 + ], + [ + 10.48, + 72 + ] + ], + "memory": [ + [ + 0.0, + 160.1 + ], + [ + 7.13, + 169.5 + ], + [ + 10.48, + 160.1 + ], + [ + 16.24, + 159.9 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/backpressure-large.json b/evidence/track1/iter1-before-fix/backpressure-large.json new file mode 100644 index 00000000..9d984e14 --- /dev/null +++ b/evidence/track1/iter1-before-fix/backpressure-large.json @@ -0,0 +1,149 @@ +{ + "label": "backpressure-large", + "size": "large", + "bytes": 3015368, + "count": 80, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "20", + "upload_status": { + "202": 26, + "503": 54 + }, + "accepted": 26, + "ready": 26, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 7.91, + "drain_seconds": 26.08, + "throughput_ready_per_s": 1.014, + "time_to_ready_s": { + "p50": 12.892, + "p95": 19.227, + "p99": 19.468, + "max": 19.468, + "mean": 12.189 + }, + "upload_latency_s": { + "p50": 0.006, + "p95": 0.112 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 20, + "max_broker_messages": 20 + }, + "worker_memory_mib": { + "peak": 175.0, + "idle_first": 118.4 + }, + "timeline": { + "pending": [ + [ + 0.0, + 2 + ], + [ + 3.77, + 20 + ], + [ + 4.49, + 20 + ], + [ + 6.77, + 19 + ], + [ + 7.51, + 20 + ], + [ + 11.32, + 15 + ], + [ + 12.07, + 15 + ], + [ + 14.39, + 12 + ], + [ + 15.16, + 11 + ], + [ + 19.24, + 7 + ], + [ + 20.01, + 7 + ], + [ + 22.35, + 4 + ], + [ + 23.17, + 4 + ] + ], + "broker": [ + [ + 0.0, + 12 + ], + [ + 7.51, + 20 + ], + [ + 15.16, + 11 + ], + [ + 23.17, + 0 + ] + ], + "memory": [ + [ + 0.0, + 118.4 + ], + [ + 4.49, + 110.8 + ], + [ + 7.51, + 175.0 + ], + [ + 12.07, + 129.1 + ], + [ + 15.16, + 99.9 + ], + [ + 20.01, + 118.2 + ], + [ + 23.17, + 137.0 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c1-p1-large-r1.json b/evidence/track1/iter1-before-fix/c1-p1-large-r1.json new file mode 100644 index 00000000..c7df6bd1 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c1-p1-large-r1.json @@ -0,0 +1,244 @@ +{ + "label": "c1-p1-large-r1", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.85, + "drain_seconds": 56.66, + "throughput_ready_per_s": 0.714, + "time_to_ready_s": { + "p50": 24.858, + "p95": 44.192, + "p99": 46.347, + "max": 46.347, + "mean": 24.526 + }, + "upload_latency_s": { + "p50": 0.093, + "p95": 0.154 + }, + "generator_max_late_s": 0.0, + "queue_lag": { + "max_pending_jobs": 32, + "max_broker_messages": 32 + }, + "worker_memory_mib": { + "peak": 167.4, + "idle_first": 161.7 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 5.27, + 19 + ], + [ + 6.05, + 21 + ], + [ + 8.4, + 29 + ], + [ + 9.13, + 32 + ], + [ + 13.05, + 31 + ], + [ + 13.79, + 30 + ], + [ + 16.22, + 29 + ], + [ + 17.16, + 28 + ], + [ + 21.77, + 26 + ], + [ + 22.58, + 25 + ], + [ + 24.93, + 24 + ], + [ + 25.78, + 23 + ], + [ + 29.66, + 20 + ], + [ + 30.52, + 20 + ], + [ + 32.91, + 18 + ], + [ + 33.67, + 17 + ], + [ + 38.45, + 14 + ], + [ + 39.21, + 13 + ], + [ + 41.55, + 11 + ], + [ + 42.31, + 11 + ], + [ + 46.91, + 7 + ], + [ + 47.71, + 7 + ], + [ + 50.03, + 5 + ], + [ + 50.79, + 4 + ], + [ + 54.84, + 1 + ], + [ + 55.59, + 1 + ] + ], + "broker": [ + [ + 0.0, + 0 + ], + [ + 9.13, + 32 + ], + [ + 17.16, + 27 + ], + [ + 25.78, + 24 + ], + [ + 33.67, + 17 + ], + [ + 42.31, + 9 + ], + [ + 50.79, + 5 + ] + ], + "memory": [ + [ + 0.0, + 161.7 + ], + [ + 6.05, + 105.0 + ], + [ + 9.13, + 119.2 + ], + [ + 13.79, + 90.5 + ], + [ + 17.16, + 122.9 + ], + [ + 22.58, + 167.4 + ], + [ + 25.78, + 82.7 + ], + [ + 30.52, + 105.7 + ], + [ + 33.67, + 167.3 + ], + [ + 39.21, + 81.4 + ], + [ + 42.31, + 117.2 + ], + [ + 47.71, + 117.4 + ], + [ + 50.79, + 81.4 + ], + [ + 55.59, + 81.4 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c1-p1-large-r2.json b/evidence/track1/iter1-before-fix/c1-p1-large-r2.json new file mode 100644 index 00000000..a84a5a7d --- /dev/null +++ b/evidence/track1/iter1-before-fix/c1-p1-large-r2.json @@ -0,0 +1,232 @@ +{ + "label": "c1-p1-large-r2", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.84, + "drain_seconds": 52.43, + "throughput_ready_per_s": 0.772, + "time_to_ready_s": { + "p50": 21.413, + "p95": 39.883, + "p99": 42.128, + "max": 42.128, + "mean": 21.96 + }, + "upload_latency_s": { + "p50": 0.09, + "p95": 0.148 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 31, + "max_broker_messages": 28 + }, + "worker_memory_mib": { + "peak": 167.5, + "idle_first": 81.4 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 3.81, + 13 + ], + [ + 4.57, + 16 + ], + [ + 6.92, + 24 + ], + [ + 7.68, + 27 + ], + [ + 12.19, + 31 + ], + [ + 12.99, + 31 + ], + [ + 15.28, + 29 + ], + [ + 16.07, + 28 + ], + [ + 21.01, + 25 + ], + [ + 21.72, + 24 + ], + [ + 24.01, + 22 + ], + [ + 24.78, + 21 + ], + [ + 28.59, + 18 + ], + [ + 29.36, + 18 + ], + [ + 31.66, + 17 + ], + [ + 32.46, + 16 + ], + [ + 36.31, + 13 + ], + [ + 37.08, + 12 + ], + [ + 39.39, + 10 + ], + [ + 40.17, + 10 + ], + [ + 43.83, + 7 + ], + [ + 44.56, + 6 + ], + [ + 46.98, + 4 + ], + [ + 47.75, + 4 + ] + ], + "broker": [ + [ + 0.0, + 0 + ], + [ + 7.68, + 28 + ], + [ + 16.07, + 27 + ], + [ + 24.78, + 23 + ], + [ + 32.46, + 15 + ], + [ + 40.17, + 11 + ], + [ + 47.75, + 3 + ] + ], + "memory": [ + [ + 0.0, + 81.4 + ], + [ + 4.57, + 105.0 + ], + [ + 7.68, + 167.5 + ], + [ + 12.99, + 119.3 + ], + [ + 16.07, + 97.4 + ], + [ + 21.72, + 107.4 + ], + [ + 24.78, + 167.5 + ], + [ + 29.36, + 92.6 + ], + [ + 32.46, + 106.2 + ], + [ + 37.08, + 85.2 + ], + [ + 40.17, + 122.4 + ], + [ + 44.56, + 100.4 + ], + [ + 47.75, + 163.3 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c1-p1-small-r1.json b/evidence/track1/iter1-before-fix/c1-p1-small-r1.json new file mode 100644 index 00000000..2c5f07df --- /dev/null +++ b/evidence/track1/iter1-before-fix/c1-p1-small-r1.json @@ -0,0 +1,328 @@ +{ + "label": "c1-p1-small-r1", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.98, + "drain_seconds": 111.41, + "throughput_ready_per_s": 0.542, + "time_to_ready_s": { + "p50": 64.273, + "p95": 102.861, + "p99": 104.752, + "max": 104.752, + "mean": 62.41 + }, + "upload_latency_s": { + "p50": 0.044, + "p95": 0.139 + }, + "generator_max_late_s": 0.027, + "queue_lag": { + "max_pending_jobs": 57, + "max_broker_messages": 58 + }, + "worker_memory_mib": { + "peak": 112.6, + "idle_first": 108.9 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 9.74, + 57 + ], + [ + 11.52, + 57 + ], + [ + 15.64, + 56 + ], + [ + 17.81, + 55 + ], + [ + 25.92, + 53 + ], + [ + 27.24, + 52 + ], + [ + 30.32, + 51 + ], + [ + 31.63, + 51 + ], + [ + 39.09, + 47 + ], + [ + 40.06, + 46 + ], + [ + 42.47, + 45 + ], + [ + 43.53, + 44 + ], + [ + 48.48, + 41 + ], + [ + 49.36, + 41 + ], + [ + 51.81, + 39 + ], + [ + 52.71, + 38 + ], + [ + 57.49, + 35 + ], + [ + 58.6, + 35 + ], + [ + 61.16, + 33 + ], + [ + 62.18, + 32 + ], + [ + 70.26, + 28 + ], + [ + 71.11, + 28 + ], + [ + 73.63, + 26 + ], + [ + 74.46, + 25 + ], + [ + 79.85, + 22 + ], + [ + 80.94, + 22 + ], + [ + 83.78, + 20 + ], + [ + 84.58, + 19 + ], + [ + 89.99, + 15 + ], + [ + 90.9, + 15 + ], + [ + 93.86, + 13 + ], + [ + 94.93, + 13 + ], + [ + 101.49, + 9 + ], + [ + 102.25, + 8 + ], + [ + 104.62, + 5 + ], + [ + 105.42, + 5 + ], + [ + 109.78, + 1 + ], + [ + 110.53, + 0 + ] + ], + "broker": [ + [ + 0.0, + 58 + ], + [ + 17.81, + 55 + ], + [ + 31.63, + 49 + ], + [ + 43.53, + 43 + ], + [ + 52.71, + 36 + ], + [ + 62.18, + 31 + ], + [ + 74.46, + 24 + ], + [ + 84.58, + 18 + ], + [ + 94.93, + 12 + ], + [ + 105.42, + 4 + ] + ], + "memory": [ + [ + 0.0, + 108.9 + ], + [ + 11.52, + 88.5 + ], + [ + 17.81, + 97.3 + ], + [ + 27.24, + 109.7 + ], + [ + 31.63, + 83.6 + ], + [ + 40.06, + 110.5 + ], + [ + 43.53, + 109.8 + ], + [ + 49.36, + 98.1 + ], + [ + 52.71, + 81.9 + ], + [ + 58.6, + 101.2 + ], + [ + 62.18, + 107.2 + ], + [ + 71.11, + 109.1 + ], + [ + 74.46, + 89.0 + ], + [ + 80.94, + 102.9 + ], + [ + 84.58, + 76.5 + ], + [ + 90.9, + 97.3 + ], + [ + 94.93, + 98.0 + ], + [ + 102.25, + 112.6 + ], + [ + 105.42, + 108.8 + ], + [ + 110.53, + 72.9 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c1-p1-small-r2.json b/evidence/track1/iter1-before-fix/c1-p1-small-r2.json new file mode 100644 index 00000000..9b2337e0 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c1-p1-small-r2.json @@ -0,0 +1,260 @@ +{ + "label": "c1-p1-small-r2", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "1", + "WORKER_PREFETCH": "1", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.92, + "drain_seconds": 59.44, + "throughput_ready_per_s": 1.019, + "time_to_ready_s": { + "p50": 28.487, + "p95": 51.179, + "p99": 53.022, + "max": 53.022, + "mean": 27.832 + }, + "upload_latency_s": { + "p50": 0.023, + "p95": 0.047 + }, + "generator_max_late_s": 0.0, + "queue_lag": { + "max_pending_jobs": 54, + "max_broker_messages": 53 + }, + "worker_memory_mib": { + "peak": 118.3, + "idle_first": 103.1 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 5.4, + 52 + ], + [ + 6.14, + 54 + ], + [ + 8.52, + 52 + ], + [ + 9.36, + 51 + ], + [ + 13.5, + 47 + ], + [ + 14.26, + 47 + ], + [ + 16.58, + 44 + ], + [ + 17.39, + 43 + ], + [ + 21.55, + 39 + ], + [ + 22.33, + 39 + ], + [ + 24.72, + 37 + ], + [ + 25.55, + 36 + ], + [ + 29.35, + 32 + ], + [ + 30.13, + 31 + ], + [ + 32.6, + 29 + ], + [ + 33.31, + 28 + ], + [ + 38.15, + 23 + ], + [ + 38.9, + 22 + ], + [ + 41.18, + 19 + ], + [ + 41.93, + 18 + ], + [ + 46.26, + 14 + ], + [ + 47.02, + 13 + ], + [ + 49.32, + 10 + ], + [ + 50.08, + 10 + ], + [ + 53.97, + 6 + ], + [ + 54.7, + 5 + ], + [ + 56.99, + 2 + ], + [ + 57.8, + 2 + ] + ], + "broker": [ + [ + 0.0, + 30 + ], + [ + 9.36, + 53 + ], + [ + 17.39, + 43 + ], + [ + 25.55, + 34 + ], + [ + 33.31, + 28 + ], + [ + 41.93, + 17 + ], + [ + 50.08, + 7 + ], + [ + 57.8, + 1 + ] + ], + "memory": [ + [ + 0.0, + 103.1 + ], + [ + 6.14, + 90.3 + ], + [ + 9.36, + 98.5 + ], + [ + 14.26, + 92.4 + ], + [ + 17.39, + 98.6 + ], + [ + 22.33, + 106.2 + ], + [ + 25.55, + 103.1 + ], + [ + 30.13, + 97.4 + ], + [ + 33.31, + 118.3 + ], + [ + 38.9, + 118.3 + ], + [ + 41.93, + 97.2 + ], + [ + 47.02, + 108.7 + ], + [ + 50.08, + 86.6 + ], + [ + 54.7, + 81.7 + ], + [ + 57.8, + 81.4 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p16-large-r1.json b/evidence/track1/iter1-before-fix/c2-p16-large-r1.json new file mode 100644 index 00000000..7fc90b68 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p16-large-r1.json @@ -0,0 +1,204 @@ +{ + "label": "c2-p16-large-r1", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.87, + "drain_seconds": 43.42, + "throughput_ready_per_s": 0.938, + "time_to_ready_s": { + "p50": 19.011, + "p95": 32.083, + "p99": 32.918, + "max": 32.918, + "mean": 18.912 + }, + "upload_latency_s": { + "p50": 0.126, + "p95": 0.504 + }, + "generator_max_late_s": 0.04, + "queue_lag": { + "max_pending_jobs": 34, + "max_broker_messages": 33 + }, + "worker_memory_mib": { + "peak": 187.0, + "idle_first": 166.5 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 5.74, + 22 + ], + [ + 6.53, + 24 + ], + [ + 8.85, + 31 + ], + [ + 9.71, + 34 + ], + [ + 13.71, + 30 + ], + [ + 14.54, + 29 + ], + [ + 16.85, + 27 + ], + [ + 17.7, + 26 + ], + [ + 21.8, + 22 + ], + [ + 22.59, + 21 + ], + [ + 24.98, + 19 + ], + [ + 25.69, + 18 + ], + [ + 30.21, + 14 + ], + [ + 30.97, + 13 + ], + [ + 33.31, + 11 + ], + [ + 34.05, + 10 + ], + [ + 39.01, + 5 + ], + [ + 39.75, + 4 + ], + [ + 42.11, + 2 + ], + [ + 42.93, + 0 + ] + ], + "broker": [ + [ + 0.0, + 3 + ], + [ + 9.71, + 33 + ], + [ + 17.7, + 23 + ], + [ + 25.69, + 18 + ], + [ + 34.05, + 8 + ], + [ + 42.93, + 0 + ] + ], + "memory": [ + [ + 0.0, + 166.5 + ], + [ + 6.53, + 116.5 + ], + [ + 9.71, + 132.6 + ], + [ + 14.54, + 120.2 + ], + [ + 17.7, + 123.7 + ], + [ + 22.59, + 187.0 + ], + [ + 25.69, + 119.1 + ], + [ + 30.97, + 123.9 + ], + [ + 34.05, + 141.0 + ], + [ + 39.75, + 124.6 + ], + [ + 42.93, + 99.8 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p16-large-r2.json b/evidence/track1/iter1-before-fix/c2-p16-large-r2.json new file mode 100644 index 00000000..77c7aeac --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p16-large-r2.json @@ -0,0 +1,204 @@ +{ + "label": "c2-p16-large-r2", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.86, + "drain_seconds": 42.32, + "throughput_ready_per_s": 0.969, + "time_to_ready_s": { + "p50": 17.383, + "p95": 30.681, + "p99": 31.583, + "max": 31.583, + "mean": 17.421 + }, + "upload_latency_s": { + "p50": 0.111, + "p95": 0.168 + }, + "generator_max_late_s": 0.0, + "queue_lag": { + "max_pending_jobs": 30, + "max_broker_messages": 25 + }, + "worker_memory_mib": { + "peak": 204.7, + "idle_first": 204.7 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 3.97, + 14 + ], + [ + 4.73, + 17 + ], + [ + 7.12, + 25 + ], + [ + 7.97, + 27 + ], + [ + 12.0, + 30 + ], + [ + 12.81, + 29 + ], + [ + 15.16, + 27 + ], + [ + 15.92, + 26 + ], + [ + 20.26, + 22 + ], + [ + 21.14, + 21 + ], + [ + 23.54, + 19 + ], + [ + 24.27, + 18 + ], + [ + 29.65, + 13 + ], + [ + 30.41, + 12 + ], + [ + 32.73, + 10 + ], + [ + 33.46, + 9 + ], + [ + 37.3, + 5 + ], + [ + 38.1, + 4 + ], + [ + 40.43, + 2 + ], + [ + 41.21, + 0 + ] + ], + "broker": [ + [ + 0.0, + 9 + ], + [ + 7.97, + 25 + ], + [ + 15.92, + 25 + ], + [ + 24.27, + 15 + ], + [ + 33.46, + 10 + ], + [ + 41.21, + 0 + ] + ], + "memory": [ + [ + 0.0, + 204.7 + ], + [ + 4.73, + 147.2 + ], + [ + 7.97, + 136.4 + ], + [ + 12.81, + 175.8 + ], + [ + 15.92, + 130.8 + ], + [ + 21.14, + 129.4 + ], + [ + 24.27, + 147.5 + ], + [ + 30.41, + 141.5 + ], + [ + 33.46, + 138.8 + ], + [ + 38.1, + 131.9 + ], + [ + 41.21, + 114.2 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p16-small-r1.json b/evidence/track1/iter1-before-fix/c2-p16-small-r1.json new file mode 100644 index 00000000..5ab3b5b9 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p16-small-r1.json @@ -0,0 +1,236 @@ +{ + "label": "c2-p16-small-r1", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.92, + "drain_seconds": 52.56, + "throughput_ready_per_s": 1.155, + "time_to_ready_s": { + "p50": 23.311, + "p95": 45.166, + "p99": 46.079, + "max": 46.079, + "mean": 23.808 + }, + "upload_latency_s": { + "p50": 0.023, + "p95": 0.038 + }, + "generator_max_late_s": 0.0, + "queue_lag": { + "max_pending_jobs": 52, + "max_broker_messages": 53 + }, + "worker_memory_mib": { + "peak": 112.5, + "idle_first": 84.1 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 3.66, + 36 + ], + [ + 4.45, + 43 + ], + [ + 6.77, + 52 + ], + [ + 7.49, + 52 + ], + [ + 11.93, + 47 + ], + [ + 12.67, + 46 + ], + [ + 14.95, + 43 + ], + [ + 15.68, + 42 + ], + [ + 20.49, + 36 + ], + [ + 21.24, + 36 + ], + [ + 23.56, + 33 + ], + [ + 24.33, + 32 + ], + [ + 28.17, + 28 + ], + [ + 28.9, + 27 + ], + [ + 31.2, + 25 + ], + [ + 32.0, + 24 + ], + [ + 35.77, + 19 + ], + [ + 36.52, + 19 + ], + [ + 38.87, + 16 + ], + [ + 39.62, + 15 + ], + [ + 43.47, + 11 + ], + [ + 44.27, + 10 + ], + [ + 46.61, + 7 + ], + [ + 47.34, + 6 + ], + [ + 52.37, + 0 + ] + ], + "broker": [ + [ + 0.0, + 20 + ], + [ + 7.49, + 53 + ], + [ + 15.68, + 41 + ], + [ + 24.33, + 30 + ], + [ + 32.0, + 24 + ], + [ + 39.62, + 13 + ], + [ + 47.34, + 7 + ] + ], + "memory": [ + [ + 0.0, + 84.1 + ], + [ + 4.45, + 98.0 + ], + [ + 7.49, + 96.7 + ], + [ + 12.67, + 111.4 + ], + [ + 15.68, + 111.0 + ], + [ + 21.24, + 109.8 + ], + [ + 24.33, + 92.0 + ], + [ + 28.9, + 95.9 + ], + [ + 32.0, + 111.0 + ], + [ + 36.52, + 112.5 + ], + [ + 39.62, + 108.6 + ], + [ + 44.27, + 101.7 + ], + [ + 47.34, + 99.0 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p16-small-r2.json b/evidence/track1/iter1-before-fix/c2-p16-small-r2.json new file mode 100644 index 00000000..8c4e5540 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p16-small-r2.json @@ -0,0 +1,244 @@ +{ + "label": "c2-p16-small-r2", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "16", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.93, + "drain_seconds": 54.36, + "throughput_ready_per_s": 1.129, + "time_to_ready_s": { + "p50": 24.153, + "p95": 46.414, + "p99": 47.296, + "max": 47.296, + "mean": 24.656 + }, + "upload_latency_s": { + "p50": 0.023, + "p95": 0.028 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 53, + "max_broker_messages": 54 + }, + "worker_memory_mib": { + "peak": 142.1, + "idle_first": 124.7 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 3.7, + 35 + ], + [ + 4.43, + 43 + ], + [ + 6.74, + 53 + ], + [ + 7.55, + 52 + ], + [ + 11.29, + 48 + ], + [ + 12.05, + 47 + ], + [ + 14.43, + 45 + ], + [ + 15.24, + 44 + ], + [ + 19.09, + 39 + ], + [ + 19.93, + 39 + ], + [ + 22.28, + 36 + ], + [ + 23.07, + 35 + ], + [ + 28.32, + 29 + ], + [ + 29.09, + 28 + ], + [ + 31.41, + 26 + ], + [ + 32.2, + 25 + ], + [ + 36.02, + 21 + ], + [ + 36.75, + 20 + ], + [ + 39.06, + 17 + ], + [ + 39.84, + 16 + ], + [ + 43.64, + 12 + ], + [ + 44.41, + 11 + ], + [ + 46.7, + 8 + ], + [ + 47.5, + 8 + ], + [ + 51.3, + 3 + ], + [ + 52.08, + 2 + ] + ], + "broker": [ + [ + 0.0, + 12 + ], + [ + 7.55, + 54 + ], + [ + 15.24, + 43 + ], + [ + 23.07, + 32 + ], + [ + 32.2, + 26 + ], + [ + 39.84, + 15 + ], + [ + 47.5, + 9 + ] + ], + "memory": [ + [ + 0.0, + 124.7 + ], + [ + 4.43, + 110.4 + ], + [ + 7.55, + 118.9 + ], + [ + 12.05, + 124.8 + ], + [ + 15.24, + 139.3 + ], + [ + 19.93, + 112.3 + ], + [ + 23.07, + 140.7 + ], + [ + 29.09, + 123.4 + ], + [ + 32.2, + 139.4 + ], + [ + 36.75, + 142.1 + ], + [ + 39.84, + 126.4 + ], + [ + 44.41, + 132.7 + ], + [ + 47.5, + 119.8 + ], + [ + 52.08, + 102.8 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p2-large-r1.json b/evidence/track1/iter1-before-fix/c2-p2-large-r1.json new file mode 100644 index 00000000..a1d96924 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p2-large-r1.json @@ -0,0 +1,192 @@ +{ + "label": "c2-p2-large-r1", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.83, + "drain_seconds": 42.01, + "throughput_ready_per_s": 0.974, + "time_to_ready_s": { + "p50": 16.79, + "p95": 30.743, + "p99": 31.45, + "max": 31.45, + "mean": 17.089 + }, + "upload_latency_s": { + "p50": 0.108, + "p95": 0.181 + }, + "generator_max_late_s": 0.0, + "queue_lag": { + "max_pending_jobs": 29, + "max_broker_messages": 31 + }, + "worker_memory_mib": { + "peak": 191.5, + "idle_first": 103.5 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 4.46, + 16 + ], + [ + 5.28, + 18 + ], + [ + 7.66, + 26 + ], + [ + 8.46, + 28 + ], + [ + 12.59, + 29 + ], + [ + 13.37, + 28 + ], + [ + 15.81, + 26 + ], + [ + 16.55, + 25 + ], + [ + 20.47, + 21 + ], + [ + 21.27, + 21 + ], + [ + 23.75, + 18 + ], + [ + 24.49, + 17 + ], + [ + 29.86, + 12 + ], + [ + 30.71, + 11 + ], + [ + 33.03, + 9 + ], + [ + 33.79, + 8 + ], + [ + 38.33, + 4 + ], + [ + 39.09, + 3 + ], + [ + 41.46, + 0 + ] + ], + "broker": [ + [ + 0.0, + 2 + ], + [ + 8.46, + 31 + ], + [ + 16.55, + 27 + ], + [ + 24.49, + 16 + ], + [ + 33.79, + 7 + ] + ], + "memory": [ + [ + 0.0, + 103.5 + ], + [ + 5.28, + 187.9 + ], + [ + 8.46, + 104.7 + ], + [ + 13.37, + 187.5 + ], + [ + 16.55, + 108.2 + ], + [ + 21.27, + 191.5 + ], + [ + 24.49, + 114.2 + ], + [ + 30.71, + 188.2 + ], + [ + 33.79, + 110.3 + ], + [ + 39.09, + 122.7 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p2-large-r2.json b/evidence/track1/iter1-before-fix/c2-p2-large-r2.json new file mode 100644 index 00000000..6e8b82c1 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p2-large-r2.json @@ -0,0 +1,204 @@ +{ + "label": "c2-p2-large-r2", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.89, + "drain_seconds": 42.23, + "throughput_ready_per_s": 0.964, + "time_to_ready_s": { + "p50": 17.196, + "p95": 31.073, + "p99": 31.762, + "max": 31.762, + "mean": 17.437 + }, + "upload_latency_s": { + "p50": 0.105, + "p95": 0.157 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 32, + "max_broker_messages": 30 + }, + "worker_memory_mib": { + "peak": 214.4, + "idle_first": 125.8 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 5.67, + 20 + ], + [ + 6.44, + 22 + ], + [ + 8.78, + 29 + ], + [ + 9.6, + 32 + ], + [ + 13.66, + 28 + ], + [ + 14.52, + 28 + ], + [ + 16.8, + 25 + ], + [ + 17.64, + 25 + ], + [ + 21.57, + 21 + ], + [ + 22.37, + 20 + ], + [ + 24.81, + 18 + ], + [ + 25.55, + 17 + ], + [ + 29.48, + 13 + ], + [ + 30.27, + 12 + ], + [ + 32.68, + 10 + ], + [ + 33.44, + 9 + ], + [ + 38.27, + 4 + ], + [ + 39.09, + 4 + ], + [ + 41.44, + 0 + ], + [ + 42.17, + 0 + ] + ], + "broker": [ + [ + 0.0, + 6 + ], + [ + 9.6, + 30 + ], + [ + 17.64, + 26 + ], + [ + 25.55, + 16 + ], + [ + 33.44, + 6 + ], + [ + 42.17, + 0 + ] + ], + "memory": [ + [ + 0.0, + 125.8 + ], + [ + 6.44, + 131.5 + ], + [ + 9.6, + 129.1 + ], + [ + 14.52, + 187.2 + ], + [ + 17.64, + 214.4 + ], + [ + 22.37, + 141.2 + ], + [ + 25.55, + 142.2 + ], + [ + 30.27, + 213.7 + ], + [ + 33.44, + 141.6 + ], + [ + 39.09, + 181.2 + ], + [ + 42.17, + 116.8 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p2-small-r1.json b/evidence/track1/iter1-before-fix/c2-p2-small-r1.json new file mode 100644 index 00000000..6aec325b --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p2-small-r1.json @@ -0,0 +1,244 @@ +{ + "label": "c2-p2-small-r1", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.92, + "drain_seconds": 54.15, + "throughput_ready_per_s": 1.118, + "time_to_ready_s": { + "p50": 24.31, + "p95": 46.982, + "p99": 47.842, + "max": 47.842, + "mean": 24.75 + }, + "upload_latency_s": { + "p50": 0.023, + "p95": 0.036 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 53, + "max_broker_messages": 50 + }, + "worker_memory_mib": { + "peak": 113.3, + "idle_first": 84.0 + }, + "timeline": { + "pending": [ + [ + 0.0, + 2 + ], + [ + 4.36, + 43 + ], + [ + 5.07, + 49 + ], + [ + 7.37, + 53 + ], + [ + 8.1, + 52 + ], + [ + 11.8, + 47 + ], + [ + 12.53, + 46 + ], + [ + 14.8, + 44 + ], + [ + 15.57, + 43 + ], + [ + 19.52, + 39 + ], + [ + 20.3, + 38 + ], + [ + 22.61, + 35 + ], + [ + 23.37, + 34 + ], + [ + 27.32, + 30 + ], + [ + 28.14, + 29 + ], + [ + 30.49, + 27 + ], + [ + 31.27, + 26 + ], + [ + 36.62, + 19 + ], + [ + 37.41, + 19 + ], + [ + 39.8, + 15 + ], + [ + 40.56, + 15 + ], + [ + 44.72, + 11 + ], + [ + 45.46, + 10 + ], + [ + 47.84, + 7 + ], + [ + 48.61, + 7 + ], + [ + 52.52, + 2 + ], + [ + 53.31, + 2 + ] + ], + "broker": [ + [ + 0.0, + 0 + ], + [ + 8.1, + 50 + ], + [ + 15.57, + 44 + ], + [ + 23.37, + 33 + ], + [ + 31.27, + 23 + ], + [ + 40.56, + 17 + ], + [ + 48.61, + 6 + ] + ], + "memory": [ + [ + 0.0, + 84.0 + ], + [ + 5.07, + 89.6 + ], + [ + 8.1, + 75.4 + ], + [ + 12.53, + 97.0 + ], + [ + 15.57, + 111.9 + ], + [ + 20.3, + 104.7 + ], + [ + 23.37, + 92.0 + ], + [ + 28.14, + 91.6 + ], + [ + 31.27, + 109.9 + ], + [ + 37.41, + 98.9 + ], + [ + 40.56, + 92.2 + ], + [ + 45.46, + 98.8 + ], + [ + 48.61, + 113.3 + ], + [ + 53.31, + 74.2 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c2-p2-small-r2.json b/evidence/track1/iter1-before-fix/c2-p2-small-r2.json new file mode 100644 index 00000000..7f69c8cb --- /dev/null +++ b/evidence/track1/iter1-before-fix/c2-p2-small-r2.json @@ -0,0 +1,244 @@ +{ + "label": "c2-p2-small-r2", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.92, + "drain_seconds": 57.86, + "throughput_ready_per_s": 1.044, + "time_to_ready_s": { + "p50": 28.017, + "p95": 50.808, + "p99": 51.638, + "max": 51.638, + "mean": 27.604 + }, + "upload_latency_s": { + "p50": 0.024, + "p95": 0.041 + }, + "generator_max_late_s": 0.01, + "queue_lag": { + "max_pending_jobs": 54, + "max_broker_messages": 48 + }, + "worker_memory_mib": { + "peak": 139.0, + "idle_first": 124.9 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 5.4, + 52 + ], + [ + 6.22, + 54 + ], + [ + 8.53, + 51 + ], + [ + 9.31, + 50 + ], + [ + 15.24, + 47 + ], + [ + 16.01, + 46 + ], + [ + 18.34, + 43 + ], + [ + 19.17, + 43 + ], + [ + 23.02, + 38 + ], + [ + 23.77, + 37 + ], + [ + 26.11, + 36 + ], + [ + 26.97, + 35 + ], + [ + 30.73, + 30 + ], + [ + 31.5, + 30 + ], + [ + 33.83, + 27 + ], + [ + 34.56, + 26 + ], + [ + 39.24, + 21 + ], + [ + 40.03, + 20 + ], + [ + 42.36, + 17 + ], + [ + 43.1, + 17 + ], + [ + 47.84, + 11 + ], + [ + 48.58, + 10 + ], + [ + 50.89, + 8 + ], + [ + 51.61, + 7 + ], + [ + 56.09, + 3 + ], + [ + 56.81, + 2 + ] + ], + "broker": [ + [ + 0.0, + 42 + ], + [ + 9.31, + 48 + ], + [ + 19.17, + 43 + ], + [ + 26.97, + 32 + ], + [ + 34.56, + 26 + ], + [ + 43.1, + 15 + ], + [ + 51.61, + 5 + ] + ], + "memory": [ + [ + 0.0, + 124.9 + ], + [ + 6.22, + 123.6 + ], + [ + 9.31, + 137.0 + ], + [ + 16.01, + 135.8 + ], + [ + 19.17, + 115.2 + ], + [ + 23.77, + 136.0 + ], + [ + 26.97, + 139.0 + ], + [ + 31.5, + 138.9 + ], + [ + 34.56, + 125.9 + ], + [ + 40.03, + 118.6 + ], + [ + 43.1, + 109.3 + ], + [ + 48.58, + 118.1 + ], + [ + 51.61, + 136.5 + ], + [ + 56.81, + 100.4 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c4-p4-large-r1.json b/evidence/track1/iter1-before-fix/c4-p4-large-r1.json new file mode 100644 index 00000000..58baeb4b --- /dev/null +++ b/evidence/track1/iter1-before-fix/c4-p4-large-r1.json @@ -0,0 +1,192 @@ +{ + "label": "c4-p4-large-r1", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.85, + "drain_seconds": 41.36, + "throughput_ready_per_s": 0.999, + "time_to_ready_s": { + "p50": 17.132, + "p95": 30.34, + "p99": 30.651, + "max": 30.651, + "mean": 17.34 + }, + "upload_latency_s": { + "p50": 0.114, + "p95": 0.171 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 28, + "max_broker_messages": 31 + }, + "worker_memory_mib": { + "peak": 223.9, + "idle_first": 190.5 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 3.99, + 14 + ], + [ + 4.91, + 18 + ], + [ + 7.44, + 25 + ], + [ + 8.2, + 28 + ], + [ + 13.92, + 28 + ], + [ + 14.74, + 28 + ], + [ + 17.04, + 25 + ], + [ + 17.81, + 25 + ], + [ + 22.01, + 19 + ], + [ + 22.82, + 19 + ], + [ + 25.18, + 16 + ], + [ + 25.98, + 16 + ], + [ + 29.78, + 13 + ], + [ + 30.5, + 10 + ], + [ + 32.91, + 10 + ], + [ + 33.74, + 7 + ], + [ + 37.66, + 4 + ], + [ + 38.45, + 4 + ], + [ + 40.94, + 0 + ] + ], + "broker": [ + [ + 0.0, + 12 + ], + [ + 8.2, + 31 + ], + [ + 17.81, + 25 + ], + [ + 25.98, + 15 + ], + [ + 33.74, + 10 + ] + ], + "memory": [ + [ + 0.0, + 190.5 + ], + [ + 4.91, + 138.5 + ], + [ + 8.2, + 124.8 + ], + [ + 14.74, + 179.0 + ], + [ + 17.81, + 128.9 + ], + [ + 22.82, + 139.0 + ], + [ + 25.98, + 150.6 + ], + [ + 30.5, + 223.9 + ], + [ + 33.74, + 214.0 + ], + [ + 38.45, + 205.9 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c4-p4-large-r2.json b/evidence/track1/iter1-before-fix/c4-p4-large-r2.json new file mode 100644 index 00000000..dc13876a --- /dev/null +++ b/evidence/track1/iter1-before-fix/c4-p4-large-r2.json @@ -0,0 +1,188 @@ +{ + "label": "c4-p4-large-r2", + "size": "large", + "bytes": 3015368, + "count": 40, + "rate_per_s": 4.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 40 + }, + "accepted": 40, + "ready": 40, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 9.86, + "drain_seconds": 39.66, + "throughput_ready_per_s": 1.018, + "time_to_ready_s": { + "p50": 17.391, + "p95": 29.583, + "p99": 29.973, + "max": 29.973, + "mean": 17.584 + }, + "upload_latency_s": { + "p50": 0.122, + "p95": 0.17 + }, + "generator_max_late_s": 0.003, + "queue_lag": { + "max_pending_jobs": 31, + "max_broker_messages": 29 + }, + "worker_memory_mib": { + "peak": 226.4, + "idle_first": 226.4 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 4.99, + 19 + ], + [ + 5.9, + 22 + ], + [ + 8.22, + 28 + ], + [ + 9.0, + 31 + ], + [ + 13.04, + 31 + ], + [ + 13.81, + 28 + ], + [ + 16.2, + 28 + ], + [ + 16.96, + 25 + ], + [ + 20.83, + 22 + ], + [ + 21.57, + 22 + ], + [ + 23.96, + 19 + ], + [ + 24.73, + 19 + ], + [ + 29.1, + 13 + ], + [ + 29.9, + 13 + ], + [ + 32.23, + 10 + ], + [ + 33.01, + 10 + ], + [ + 37.93, + 4 + ], + [ + 38.69, + 4 + ] + ], + "broker": [ + [ + 0.0, + 15 + ], + [ + 9.0, + 29 + ], + [ + 16.96, + 25 + ], + [ + 24.73, + 19 + ], + [ + 33.01, + 8 + ] + ], + "memory": [ + [ + 0.0, + 226.4 + ], + [ + 5.9, + 153.0 + ], + [ + 9.0, + 154.4 + ], + [ + 13.81, + 155.9 + ], + [ + 16.96, + 168.5 + ], + [ + 21.57, + 157.0 + ], + [ + 24.73, + 186.0 + ], + [ + 29.9, + 155.6 + ], + [ + 33.01, + 170.0 + ], + [ + 38.69, + 132.5 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c4-p4-small-r1.json b/evidence/track1/iter1-before-fix/c4-p4-small-r1.json new file mode 100644 index 00000000..21779703 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c4-p4-small-r1.json @@ -0,0 +1,236 @@ +{ + "label": "c4-p4-small-r1", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.92, + "drain_seconds": 51.81, + "throughput_ready_per_s": 1.171, + "time_to_ready_s": { + "p50": 25.537, + "p95": 44.701, + "p99": 45.387, + "max": 45.387, + "mean": 24.63 + }, + "upload_latency_s": { + "p50": 0.022, + "p95": 0.035 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 53, + "max_broker_messages": 53 + }, + "worker_memory_mib": { + "peak": 123.1, + "idle_first": 112.2 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 3.71, + 37 + ], + [ + 4.46, + 43 + ], + [ + 6.76, + 53 + ], + [ + 7.53, + 52 + ], + [ + 11.18, + 49 + ], + [ + 11.91, + 49 + ], + [ + 14.23, + 45 + ], + [ + 14.97, + 45 + ], + [ + 18.93, + 41 + ], + [ + 19.67, + 38 + ], + [ + 22.09, + 36 + ], + [ + 22.81, + 35 + ], + [ + 27.13, + 31 + ], + [ + 28.11, + 31 + ], + [ + 30.44, + 28 + ], + [ + 31.18, + 26 + ], + [ + 36.25, + 21 + ], + [ + 37.01, + 17 + ], + [ + 39.33, + 17 + ], + [ + 40.08, + 14 + ], + [ + 43.94, + 10 + ], + [ + 44.69, + 9 + ], + [ + 47.02, + 5 + ], + [ + 47.8, + 5 + ], + [ + 51.71, + 0 + ] + ], + "broker": [ + [ + 0.0, + 0 + ], + [ + 7.53, + 53 + ], + [ + 14.97, + 46 + ], + [ + 22.81, + 35 + ], + [ + 31.18, + 24 + ], + [ + 40.08, + 17 + ], + [ + 47.8, + 5 + ] + ], + "memory": [ + [ + 0.0, + 112.2 + ], + [ + 4.46, + 104.2 + ], + [ + 7.53, + 100.2 + ], + [ + 11.91, + 116.0 + ], + [ + 14.97, + 95.9 + ], + [ + 19.67, + 110.3 + ], + [ + 22.81, + 105.2 + ], + [ + 28.11, + 123.1 + ], + [ + 31.18, + 117.9 + ], + [ + 37.01, + 116.7 + ], + [ + 40.08, + 105.9 + ], + [ + 44.69, + 118.3 + ], + [ + 47.8, + 108.2 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/c4-p4-small-r2.json b/evidence/track1/iter1-before-fix/c4-p4-small-r2.json new file mode 100644 index 00000000..f5887397 --- /dev/null +++ b/evidence/track1/iter1-before-fix/c4-p4-small-r2.json @@ -0,0 +1,244 @@ +{ + "label": "c4-p4-small-r2", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "4", + "WORKER_PREFETCH": "4", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "200", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.92, + "drain_seconds": 54.95, + "throughput_ready_per_s": 1.096, + "time_to_ready_s": { + "p50": 24.86, + "p95": 48.903, + "p99": 49.058, + "max": 49.058, + "mean": 25.564 + }, + "upload_latency_s": { + "p50": 0.023, + "p95": 0.029 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 53, + "max_broker_messages": 53 + }, + "worker_memory_mib": { + "peak": 157.3, + "idle_first": 143.6 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 3.83, + 37 + ], + [ + 4.6, + 44 + ], + [ + 6.89, + 53 + ], + [ + 7.77, + 52 + ], + [ + 11.78, + 48 + ], + [ + 12.53, + 48 + ], + [ + 14.91, + 45 + ], + [ + 15.69, + 44 + ], + [ + 19.5, + 40 + ], + [ + 20.32, + 39 + ], + [ + 22.69, + 36 + ], + [ + 23.42, + 36 + ], + [ + 28.49, + 29 + ], + [ + 29.27, + 29 + ], + [ + 31.56, + 25 + ], + [ + 32.3, + 24 + ], + [ + 36.72, + 20 + ], + [ + 37.46, + 20 + ], + [ + 39.81, + 16 + ], + [ + 40.58, + 16 + ], + [ + 44.6, + 12 + ], + [ + 45.42, + 12 + ], + [ + 47.85, + 8 + ], + [ + 48.77, + 8 + ], + [ + 53.12, + 5 + ], + [ + 53.85, + 4 + ] + ], + "broker": [ + [ + 0.0, + 32 + ], + [ + 7.77, + 53 + ], + [ + 15.69, + 43 + ], + [ + 23.42, + 37 + ], + [ + 32.3, + 24 + ], + [ + 40.58, + 13 + ], + [ + 48.77, + 10 + ] + ], + "memory": [ + [ + 0.0, + 143.6 + ], + [ + 4.6, + 156.8 + ], + [ + 7.77, + 141.3 + ], + [ + 12.53, + 147.3 + ], + [ + 15.69, + 140.9 + ], + [ + 20.32, + 149.8 + ], + [ + 23.42, + 151.1 + ], + [ + 29.27, + 136.1 + ], + [ + 32.3, + 130.9 + ], + [ + 37.46, + 157.3 + ], + [ + 40.58, + 129.1 + ], + [ + 45.42, + 156.3 + ], + [ + 48.77, + 145.1 + ], + [ + 53.85, + 120.4 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/recovery-small.json b/evidence/track1/iter1-before-fix/recovery-small.json new file mode 100644 index 00000000..d990588a --- /dev/null +++ b/evidence/track1/iter1-before-fix/recovery-small.json @@ -0,0 +1,137 @@ +{ + "label": "recovery-small", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 10.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "20", + "upload_status": { + "202": 26, + "503": 34 + }, + "accepted": 26, + "ready": 26, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 5.91, + "drain_seconds": 23.91, + "throughput_ready_per_s": 1.133, + "time_to_ready_s": { + "p50": 11.215, + "p95": 17.979, + "p99": 18.519, + "max": 18.519, + "mean": 10.686 + }, + "upload_latency_s": { + "p50": 0.006, + "p95": 0.027 + }, + "generator_max_late_s": 0.0, + "queue_lag": { + "max_pending_jobs": 20, + "max_broker_messages": 21 + }, + "worker_memory_mib": { + "peak": 126.6, + "idle_first": 118.3 + }, + "timeline": { + "pending": [ + [ + 0.0, + 3 + ], + [ + 4.6, + 20 + ], + [ + 5.32, + 20 + ], + [ + 7.6, + 18 + ], + [ + 8.36, + 17 + ], + [ + 12.21, + 13 + ], + [ + 12.99, + 12 + ], + [ + 15.32, + 9 + ], + [ + 16.13, + 9 + ], + [ + 20.2, + 4 + ], + [ + 20.96, + 3 + ], + [ + 23.29, + 0 + ] + ], + "broker": [ + [ + 0.0, + 21 + ], + [ + 8.36, + 17 + ], + [ + 16.13, + 6 + ] + ], + "memory": [ + [ + 0.0, + 118.3 + ], + [ + 5.32, + 104.1 + ], + [ + 8.36, + 125.6 + ], + [ + 12.99, + 126.6 + ], + [ + 16.13, + 105.7 + ], + [ + 20.96, + 107.4 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/iter1-before-fix/report.md b/evidence/track1/iter1-before-fix/report.md new file mode 100644 index 00000000..d37d3f2a --- /dev/null +++ b/evidence/track1/iter1-before-fix/report.md @@ -0,0 +1,19 @@ +## Матрица concurrency/prefetch + +| Размер | concurrency / prefetch | Повторов | Принято / ready / failed / потеряно | Throughput, ready/s | p50 до ready, с | p95 до ready, с | p99 до ready, с | Макс. незавершённых | Пик памяти worker, MiB | Разгрузка, с | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| small | 1 / 1 | 2 | 120 / 120 / 0 / 0 | 0.78 [0.54..1.02] | 46.38 [28.49..64.27] | 77.02 [51.18..102.86] | 78.89 [53.02..104.75] | 56 [54..57] | 115 [113..118] | 85.4 [59.4..111.4] | +| small | 2 / 2 | 2 | 120 / 120 / 0 / 0 | 1.08 [1.04..1.12] | 26.16 [24.31..28.02] | 48.89 [46.98..50.81] | 49.74 [47.84..51.64] | 54 [53..54] | 126 [113..139] | 56.0 [54.1..57.9] | +| small | 2 / 16 | 2 | 120 / 120 / 0 / 0 | 1.14 [1.13..1.16] | 23.73 [23.31..24.15] | 45.79 [45.17..46.41] | 46.69 [46.08..47.30] | 52 [52..53] | 127 [112..142] | 53.5 [52.6..54.4] | +| small | 4 / 4 | 2 | 120 / 120 / 0 / 0 | 1.13 [1.10..1.17] | 25.20 [24.86..25.54] | 46.80 [44.70..48.90] | 47.22 [45.39..49.06] | 53 | 140 [123..157] | 53.4 [51.8..55.0] | +| large | 1 / 1 | 2 | 80 / 80 / 0 / 0 | 0.74 [0.71..0.77] | 23.14 [21.41..24.86] | 42.04 [39.88..44.19] | 44.24 [42.13..46.35] | 32 [31..32] | 167 [167..168] | 54.5 [52.4..56.7] | +| large | 2 / 2 | 2 | 80 / 80 / 0 / 0 | 0.97 [0.96..0.97] | 16.99 [16.79..17.20] | 30.91 [30.74..31.07] | 31.61 [31.45..31.76] | 30 [29..32] | 203 [192..214] | 42.1 [42.0..42.2] | +| large | 2 / 16 | 2 | 80 / 80 / 0 / 0 | 0.95 [0.94..0.97] | 18.20 [17.38..19.01] | 31.38 [30.68..32.08] | 32.25 [31.58..32.92] | 32 [30..34] | 196 [187..205] | 42.9 [42.3..43.4] | +| large | 4 / 4 | 2 | 80 / 80 / 0 / 0 | 1.01 [1.00..1.02] | 17.26 [17.13..17.39] | 29.96 [29.58..30.34] | 30.31 [29.97..30.65] | 30 [28..31] | 225 [224..226] | 40.5 [39.7..41.4] | + +## Backpressure и восстановление + +| Прогон | MAX_PENDING_JOBS | Поток | Ответы upload | Ready / failed / потеряно | Макс. незавершённых | p95 до ready, с | Throughput, ready/s | Пик памяти, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| backpressure-large | 20 | 80 x large @ 10.0/s | 202: 26, 503: 54 | 26 / 0 / 0 | 20 | 19.227 | 1.014 | 175.0 | +| recovery-small | 20 | 60 x small @ 10.0/s | 202: 26, 503: 34 | 26 / 0 / 0 | 20 | 17.979 | 1.133 | 126.6 | diff --git a/evidence/track1/iter1-before-fix/summary.txt b/evidence/track1/iter1-before-fix/summary.txt new file mode 100644 index 00000000..54851ae0 --- /dev/null +++ b/evidence/track1/iter1-before-fix/summary.txt @@ -0,0 +1,19 @@ +{"label": "c1-p1-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.542, "time_to_ready_s": {"p50": 64.273, "p95": 102.861, "p99": 104.752, "max": 104.752, "mean": 62.41}, "queue_lag": {"max_pending_jobs": 57, "max_broker_messages": 58}, "worker_memory_mib": {"peak": 112.6, "idle_first": 108.9}, "drain_seconds": 111.41} +{"label": "c1-p1-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.714, "time_to_ready_s": {"p50": 24.858, "p95": 44.192, "p99": 46.347, "max": 46.347, "mean": 24.526}, "queue_lag": {"max_pending_jobs": 32, "max_broker_messages": 32}, "worker_memory_mib": {"peak": 167.4, "idle_first": 161.7}, "drain_seconds": 56.66} +{"label": "c1-p1-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.019, "time_to_ready_s": {"p50": 28.487, "p95": 51.179, "p99": 53.022, "max": 53.022, "mean": 27.832}, "queue_lag": {"max_pending_jobs": 54, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 118.3, "idle_first": 103.1}, "drain_seconds": 59.44} +{"label": "c1-p1-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.772, "time_to_ready_s": {"p50": 21.413, "p95": 39.883, "p99": 42.128, "max": 42.128, "mean": 21.96}, "queue_lag": {"max_pending_jobs": 31, "max_broker_messages": 28}, "worker_memory_mib": {"peak": 167.5, "idle_first": 81.4}, "drain_seconds": 52.43} +{"label": "c2-p2-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.118, "time_to_ready_s": {"p50": 24.31, "p95": 46.982, "p99": 47.842, "max": 47.842, "mean": 24.75}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 50}, "worker_memory_mib": {"peak": 113.3, "idle_first": 84.0}, "drain_seconds": 54.15} +{"label": "c2-p2-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.974, "time_to_ready_s": {"p50": 16.79, "p95": 30.743, "p99": 31.45, "max": 31.45, "mean": 17.089}, "queue_lag": {"max_pending_jobs": 29, "max_broker_messages": 31}, "worker_memory_mib": {"peak": 191.5, "idle_first": 103.5}, "drain_seconds": 42.01} +{"label": "c2-p2-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.044, "time_to_ready_s": {"p50": 28.017, "p95": 50.808, "p99": 51.638, "max": 51.638, "mean": 27.604}, "queue_lag": {"max_pending_jobs": 54, "max_broker_messages": 48}, "worker_memory_mib": {"peak": 139.0, "idle_first": 124.9}, "drain_seconds": 57.86} +{"label": "c2-p2-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.964, "time_to_ready_s": {"p50": 17.196, "p95": 31.073, "p99": 31.762, "max": 31.762, "mean": 17.437}, "queue_lag": {"max_pending_jobs": 32, "max_broker_messages": 30}, "worker_memory_mib": {"peak": 214.4, "idle_first": 125.8}, "drain_seconds": 42.23} +{"label": "c4-p4-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.171, "time_to_ready_s": {"p50": 25.537, "p95": 44.701, "p99": 45.387, "max": 45.387, "mean": 24.63}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 123.1, "idle_first": 112.2}, "drain_seconds": 51.81} +{"label": "c4-p4-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.999, "time_to_ready_s": {"p50": 17.132, "p95": 30.34, "p99": 30.651, "max": 30.651, "mean": 17.34}, "queue_lag": {"max_pending_jobs": 28, "max_broker_messages": 31}, "worker_memory_mib": {"peak": 223.9, "idle_first": 190.5}, "drain_seconds": 41.36} +{"label": "c4-p4-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.096, "time_to_ready_s": {"p50": 24.86, "p95": 48.903, "p99": 49.058, "max": 49.058, "mean": 25.564}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 157.3, "idle_first": 143.6}, "drain_seconds": 54.95} +{"label": "c4-p4-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.018, "time_to_ready_s": {"p50": 17.391, "p95": 29.583, "p99": 29.973, "max": 29.973, "mean": 17.584}, "queue_lag": {"max_pending_jobs": 31, "max_broker_messages": 29}, "worker_memory_mib": {"peak": 226.4, "idle_first": 226.4}, "drain_seconds": 39.66} +{"label": "c2-p16-small-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.155, "time_to_ready_s": {"p50": 23.311, "p95": 45.166, "p99": 46.079, "max": 46.079, "mean": 23.808}, "queue_lag": {"max_pending_jobs": 52, "max_broker_messages": 53}, "worker_memory_mib": {"peak": 112.5, "idle_first": 84.1}, "drain_seconds": 52.56} +{"label": "c2-p16-large-r1", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.938, "time_to_ready_s": {"p50": 19.011, "p95": 32.083, "p99": 32.918, "max": 32.918, "mean": 18.912}, "queue_lag": {"max_pending_jobs": 34, "max_broker_messages": 33}, "worker_memory_mib": {"peak": 187.0, "idle_first": 166.5}, "drain_seconds": 43.42} +{"label": "c2-p16-small-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.129, "time_to_ready_s": {"p50": 24.153, "p95": 46.414, "p99": 47.296, "max": 47.296, "mean": 24.656}, "queue_lag": {"max_pending_jobs": 53, "max_broker_messages": 54}, "worker_memory_mib": {"peak": 142.1, "idle_first": 124.7}, "drain_seconds": 54.36} +{"label": "c2-p16-large-r2", "upload_status": {"202": 40}, "ready": 40, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 0.969, "time_to_ready_s": {"p50": 17.383, "p95": 30.681, "p99": 31.583, "max": 31.583, "mean": 17.421}, "queue_lag": {"max_pending_jobs": 30, "max_broker_messages": 25}, "worker_memory_mib": {"peak": 204.7, "idle_first": 204.7}, "drain_seconds": 42.32} +{"label": "backpressure-large", "upload_status": {"202": 26, "503": 54}, "ready": 26, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.014, "time_to_ready_s": {"p50": 12.892, "p95": 19.227, "p99": 19.468, "max": 19.468, "mean": 12.189}, "queue_lag": {"max_pending_jobs": 20, "max_broker_messages": 20}, "worker_memory_mib": {"peak": 175.0, "idle_first": 118.4}, "drain_seconds": 26.08} +{"label": "recovery-small", "upload_status": {"202": 26, "503": 34}, "ready": 26, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.133, "time_to_ready_s": {"p50": 11.215, "p95": 17.979, "p99": 18.519, "max": 18.519, "mean": 10.686}, "queue_lag": {"max_pending_jobs": 20, "max_broker_messages": 21}, "worker_memory_mib": {"peak": 126.6, "idle_first": 118.3}, "drain_seconds": 23.91} +track1 done diff --git a/evidence/track1/recovery-small.json b/evidence/track1/recovery-small.json new file mode 100644 index 00000000..699d1f4f --- /dev/null +++ b/evidence/track1/recovery-small.json @@ -0,0 +1,104 @@ +{ + "label": "recovery-small", + "size": "small", + "bytes": 111817, + "count": 60, + "rate_per_s": 5.0, + "worker": { + "WORKER_CONCURRENCY": "2", + "WORKER_PREFETCH": "2", + "WORKER_TASK_MEMORY_MB": "640" + }, + "worker_replicas": 1, + "max_pending_jobs": "20", + "upload_status": { + "202": 60 + }, + "accepted": 60, + "ready": 60, + "failed": 0, + "lost_accepted": 0, + "submit_seconds": 11.82, + "drain_seconds": 12.1, + "throughput_ready_per_s": 5.031, + "time_to_ready_s": { + "p50": 0.113, + "p95": 0.145, + "p99": 0.15, + "max": 0.15, + "mean": 0.116 + }, + "upload_latency_s": { + "p50": 0.022, + "p95": 0.029 + }, + "generator_max_late_s": 0.001, + "queue_lag": { + "max_pending_jobs": 1, + "max_broker_messages": 0 + }, + "worker_memory_mib": { + "peak": 125.6, + "idle_first": 122.6 + }, + "timeline": { + "pending": [ + [ + 0.0, + 1 + ], + [ + 3.6, + 0 + ], + [ + 4.33, + 1 + ], + [ + 6.61, + 1 + ], + [ + 7.37, + 0 + ], + [ + 10.94, + 0 + ], + [ + 11.67, + 1 + ] + ], + "broker": [ + [ + 0.0, + 0 + ], + [ + 7.37, + 0 + ] + ], + "memory": [ + [ + 0.0, + 122.6 + ], + [ + 4.33, + 122.2 + ], + [ + 7.37, + 125.6 + ], + [ + 11.67, + 122.3 + ] + ] + } +} \ No newline at end of file diff --git a/evidence/track1/report.md b/evidence/track1/report.md new file mode 100644 index 00000000..a9acb015 --- /dev/null +++ b/evidence/track1/report.md @@ -0,0 +1,20 @@ +## Матрица concurrency/prefetch + +| Размер | concurrency / prefetch | Повторов | Принято / ready / failed / потеряно | Throughput, ready/s | p50 до ready, с | p95 до ready, с | p99 до ready, с | Макс. незавершённых | Пик памяти worker, MiB | Разгрузка, с | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| small | 1 / 1 | 2 | 400 / 400 / 0 / 0 | 4.50 [3.75..5.26] | 19.71 [14.65..24.77] | 38.18 [30.79..45.56] | 40.16 [32.73..47.59] | 179 [177..181] | 120 [110..130] | 46.4 [39.0..53.7] | +| small | 2 / 2 | 2 | 400 / 400 / 0 / 0 | 5.69 [5.29..6.08] | 18.03 [16.32..19.74] | 29.41 [27.00..31.83] | 30.27 [27.79..32.75] | 170 | 133 [113..153] | 36.2 [34.1..38.4] | +| small | 2 / 16 | 2 | 400 / 400 / 0 / 0 | 9.81 [9.60..10.02] | 7.84 [7.19..8.48] | 14.72 [14.26..15.17] | 15.28 [14.84..15.71] | 156 [148..165] | 133 [117..149] | 21.0 [20.9..21.0] | +| small | 4 / 4 | 2 | 400 / 400 / 0 / 0 | 12.85 [12.52..13.17] | 5.74 [5.74..5.75] | 10.23 [9.89..10.57] | 10.56 [10.20..10.92] | 134 [121..146] | 142 [114..170] | 16.1 [15.9..16.3] | +| large | 1 / 1 | 2 | 120 / 120 / 0 / 0 | 1.26 [1.13..1.39] | 19.35 [18.49..20.21] | 39.68 [34.84..44.53] | 40.86 [36.04..45.67] | 50 | 179 [173..186] | 49.4 [44.3..54.5] | +| large | 2 / 2 | 2 | 120 / 120 / 0 / 0 | 2.43 [2.25..2.62] | 10.13 [8.96..11.31] | 17.30 [15.30..19.30] | 17.62 [15.67..19.57] | 46 [41..51] | 246 [239..254] | 25.6 [23.6..27.5] | +| large | 2 / 16 | 2 | 120 / 120 / 0 / 0 | 3.23 [2.98..3.48] | 7.10 [5.55..8.64] | 11.09 [9.66..12.52] | 11.35 [9.91..12.79] | 40 [36..43] | 222 [205..239] | 19.7 [18.2..21.1] | +| large | 4 / 4 | 2 | 120 / 120 / 0 / 0 | 3.81 [3.73..3.88] | 5.02 [4.50..5.54] | 8.44 [8.07..8.81] | 8.53 [8.15..8.90] | 33 [30..36] | 294 [248..339] | 16.4 [15.9..16.8] | + +## Backpressure и восстановление + +| Прогон | MAX_PENDING_JOBS | Поток | Ответы upload | Ready / failed / потеряно | Макс. незавершённых | p95 до ready, с | Throughput, ready/s | Пик памяти, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| backpressure-large | 20 | 120 x large @ 15.0/s | 202: 48, 503: 72 | 48 / 0 / 0 | 22 | 5.646 | 3.691 | 236.9 | +| baseline-small | 200 | 60 x small @ 5.0/s | 202: 60 | 60 / 0 / 0 | 1 | 1.171 | 5.031 | 109.7 | +| recovery-small | 20 | 60 x small @ 5.0/s | 202: 60 | 60 / 0 / 0 | 1 | 0.145 | 5.031 | 125.6 | diff --git a/evidence/track1/summary.txt b/evidence/track1/summary.txt new file mode 100644 index 00000000..fb7787dc --- /dev/null +++ b/evidence/track1/summary.txt @@ -0,0 +1,20 @@ +{"label": "c1-p1-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.256, "time_to_ready_s": {"p50": 14.65, "p95": 30.791, "p99": 32.731, "max": 33.144, "mean": 15.373}, "queue_lag": {"max_pending_jobs": 177, "max_broker_messages": 141}, "worker_memory_mib": {"peak": 109.7, "idle_first": 91.2}, "drain_seconds": 39.01} +{"label": "c1-p1-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.131, "time_to_ready_s": {"p50": 20.214, "p95": 44.534, "p99": 45.674, "max": 45.674, "mean": 21.052}, "queue_lag": {"max_pending_jobs": 50, "max_broker_messages": 43}, "worker_memory_mib": {"peak": 172.9, "idle_first": 105.8}, "drain_seconds": 54.51} +{"label": "c1-p1-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.745, "time_to_ready_s": {"p50": 24.775, "p95": 45.564, "p99": 47.589, "max": 48.149, "mean": 24.871}, "queue_lag": {"max_pending_jobs": 181, "max_broker_messages": 156}, "worker_memory_mib": {"peak": 130.1, "idle_first": 123.8}, "drain_seconds": 53.7} +{"label": "c1-p1-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 1.385, "time_to_ready_s": {"p50": 18.493, "p95": 34.835, "p99": 36.043, "max": 36.043, "mean": 18.892}, "queue_lag": {"max_pending_jobs": 50, "max_broker_messages": 47}, "worker_memory_mib": {"peak": 185.6, "idle_first": 176.1}, "drain_seconds": 44.28} +{"label": "c2-p2-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 6.085, "time_to_ready_s": {"p50": 16.317, "p95": 26.997, "p99": 27.79, "max": 27.979, "mean": 16.313}, "queue_lag": {"max_pending_jobs": 170, "max_broker_messages": 180}, "worker_memory_mib": {"peak": 113.1, "idle_first": 78.4}, "drain_seconds": 34.07} +{"label": "c2-p2-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 2.615, "time_to_ready_s": {"p50": 8.955, "p95": 15.298, "p99": 15.667, "max": 15.667, "mean": 8.775}, "queue_lag": {"max_pending_jobs": 41, "max_broker_messages": 30}, "worker_memory_mib": {"peak": 239.2, "idle_first": 138.2}, "drain_seconds": 23.59} +{"label": "c2-p2-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.29, "time_to_ready_s": {"p50": 19.744, "p95": 31.828, "p99": 32.749, "max": 32.946, "mean": 18.353}, "queue_lag": {"max_pending_jobs": 170, "max_broker_messages": 131}, "worker_memory_mib": {"peak": 153.1, "idle_first": 139.4}, "drain_seconds": 38.38} +{"label": "c2-p2-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 2.25, "time_to_ready_s": {"p50": 11.312, "p95": 19.302, "p99": 19.573, "max": 19.573, "mean": 11.959}, "queue_lag": {"max_pending_jobs": 51, "max_broker_messages": 50}, "worker_memory_mib": {"peak": 253.8, "idle_first": 253.8}, "drain_seconds": 27.53} +{"label": "c4-p4-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 13.171, "time_to_ready_s": {"p50": 5.745, "p95": 9.89, "p99": 10.199, "max": 10.259, "mean": 6.057}, "queue_lag": {"max_pending_jobs": 146, "max_broker_messages": 86}, "worker_memory_mib": {"peak": 113.9, "idle_first": 92.1}, "drain_seconds": 15.92} +{"label": "c4-p4-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.729, "time_to_ready_s": {"p50": 5.538, "p95": 8.81, "p99": 8.9, "max": 8.9, "mean": 5.31}, "queue_lag": {"max_pending_jobs": 36, "max_broker_messages": 19}, "worker_memory_mib": {"peak": 248.5, "idle_first": 174.6}, "drain_seconds": 16.84} +{"label": "c4-p4-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 12.521, "time_to_ready_s": {"p50": 5.741, "p95": 10.568, "p99": 10.924, "max": 11.02, "mean": 5.689}, "queue_lag": {"max_pending_jobs": 121, "max_broker_messages": 137}, "worker_memory_mib": {"peak": 169.5, "idle_first": 160.1}, "drain_seconds": 16.28} +{"label": "c4-p4-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.883, "time_to_ready_s": {"p50": 4.504, "p95": 8.075, "p99": 8.152, "max": 8.152, "mean": 4.407}, "queue_lag": {"max_pending_jobs": 30, "max_broker_messages": 17}, "worker_memory_mib": {"peak": 338.9, "idle_first": 286.2}, "drain_seconds": 15.93} +{"label": "c2-p16-small-r1", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 9.604, "time_to_ready_s": {"p50": 8.479, "p95": 15.174, "p99": 15.711, "max": 15.851, "mean": 8.697}, "queue_lag": {"max_pending_jobs": 165, "max_broker_messages": 107}, "worker_memory_mib": {"peak": 117.2, "idle_first": 100.4}, "drain_seconds": 20.95} +{"label": "c2-p16-large-r1", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 2.98, "time_to_ready_s": {"p50": 8.645, "p95": 12.515, "p99": 12.789, "max": 12.789, "mean": 7.964}, "queue_lag": {"max_pending_jobs": 43, "max_broker_messages": 40}, "worker_memory_mib": {"peak": 238.7, "idle_first": 117.7}, "drain_seconds": 21.1} +{"label": "c2-p16-small-r2", "upload_status": {"202": 200}, "ready": 200, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 10.022, "time_to_ready_s": {"p50": 7.194, "p95": 14.257, "p99": 14.841, "max": 14.998, "mean": 7.499}, "queue_lag": {"max_pending_jobs": 148, "max_broker_messages": 120}, "worker_memory_mib": {"peak": 149.3, "idle_first": 144.1}, "drain_seconds": 20.97} +{"label": "c2-p16-large-r2", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.475, "time_to_ready_s": {"p50": 5.555, "p95": 9.662, "p99": 9.912, "max": 9.912, "mean": 5.505}, "queue_lag": {"max_pending_jobs": 36, "max_broker_messages": 34}, "worker_memory_mib": {"peak": 204.8, "idle_first": 138.9}, "drain_seconds": 18.25} +{"label": "baseline-small", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.031, "time_to_ready_s": {"p50": 0.115, "p95": 1.171, "p99": 1.377, "max": 1.377, "mean": 0.248}, "queue_lag": {"max_pending_jobs": 1, "max_broker_messages": 4}, "worker_memory_mib": {"peak": 109.7, "idle_first": 101.1}, "drain_seconds": 12.1} +{"label": "backpressure-large", "upload_status": {"202": 48, "503": 72}, "ready": 48, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 3.691, "time_to_ready_s": {"p50": 4.937, "p95": 5.646, "p99": 5.682, "max": 5.682, "mean": 4.161}, "queue_lag": {"max_pending_jobs": 22, "max_broker_messages": 20}, "worker_memory_mib": {"peak": 236.9, "idle_first": 110.5}, "drain_seconds": 13.33} +{"label": "recovery-small", "upload_status": {"202": 60}, "ready": 60, "failed": 0, "lost_accepted": 0, "throughput_ready_per_s": 5.031, "time_to_ready_s": {"p50": 0.113, "p95": 0.145, "p99": 0.15, "max": 0.15, "mean": 0.116}, "queue_lag": {"max_pending_jobs": 1, "max_broker_messages": 0}, "worker_memory_mib": {"peak": 125.6, "idle_first": 122.6}, "drain_seconds": 12.1} +track1 done diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..3ab74694 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,89 @@ +# HAProxy — единственная публичная точка входа lab3. +# Реплики API обнаруживаются через DNS Docker (имя сервиса "api" -> все IP его контейнеров), +# поэтому масштабирование `--scale api=N` (N <= 8) не требует правки конфигурации. +# Sticky session нет намеренно: любая реплика обслуживает любой запрос, состояние — в PostgreSQL/Valkey. + +global + log stdout format raw local0 info + maxconn 4096 + hard-stop-after 30s + +# Встроенный DNS Docker. Интервал повторного разрешения 1 с: остановленный контейнер исчезает +# из ответа DNS, и его слот server-template уходит в MAINT ("No IP for server") за 1–2 с. +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 2 + timeout resolve 1s + timeout retry 500ms + hold valid 1s + hold obsolete 1s + hold nx 1s + hold other 1s + hold refused 1s + hold timeout 1s + +defaults + mode http + log global + option httplog + option dontlognull + timeout connect 1s # до живой реплики в той же сети — миллисекунды; 1 с = признак отказа + timeout client 30s + timeout server 15s # дольше statement_timeout БД (5 с) и Argon2id под нагрузкой + timeout http-request 10s + timeout http-keep-alive 10s # меньше keep-alive uvicorn (30 с): idle-соединение закрывает HAProxy первым + timeout queue 5s + timeout check 1s + # Повтор на ДРУГОЙ реплике (redispatch) при неудачном соединении; L7-повторы ограничены ниже. + retries 2 + option redispatch 1 + # Ответы, созданные самим LB, — в формате ошибок контракта (без X-Instance-Id: их создал не API). + http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Bad gateway\",\"request_id\":\"%[req.hdr(x-request-id)]\"}}" hdr X-Request-Id "%[req.hdr(x-request-id)]" + http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API replica available\",\"request_id\":\"%[req.hdr(x-request-id)]\"}}" hdr X-Request-Id "%[req.hdr(x-request-id)]" + http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Gateway timeout\",\"request_id\":\"%[req.hdr(x-request-id)]\"}}" hdr X-Request-Id "%[req.hdr(x-request-id)]" + +frontend public + bind :8080 + # Сквозной request id: API принимает корректный входящий X-Request-Id, LB подставляет свой, если его нет. + unique-id-format %[uuid()] + http-request set-header X-Request-Id %[unique-id] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ } + # Если /health/ready не проходит НИ У ОДНОЙ реплики (например, недоступна БД), не отвечаем + # безликим 503 от LB, а отдаём запрос живым процессам: они вернут честный 503 контракта, + # а /health/live продолжит работать. + use_backend api_live if { nbsrv(api) eq 0 } + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + # Активная проверка раз в 1 с, вывод из ротации после 2 неудач подряд, возврат после 2 успехов. + # Пассивно: 2 ошибки соединения подряд на реальном трафике — сразу DOWN, не дожидаясь проверки. + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 observe layer4 error-limit 2 on-error mark-down + # L7-повтор на другой реплике безопасен только для GET и POST сообщения (защищён Idempotency-Key). + # Для POST /users, /chats, /auth/sessions и DELETE остаются лишь повторы до отправки запроса (conn-failure). + retry-on conn-failure empty-response response-timeout + acl safe_method method GET HEAD + acl is_post method POST + acl messages_path path_reg ^/api/v1/chats/[^/]+/messages$ + http-request disable-l7-retry unless safe_method || is_post messages_path + server-template api- 8 api:8080 resolvers docker init-addr none resolve-prefer ipv4 + +backend api_live + balance roundrobin + option httpchk + http-check send meth GET uri /health/live + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 + server-template live- 8 api:8080 resolvers docker init-addr none resolve-prefer ipv4 + +# Наблюдаемость LB: состояние backend-ов и метрики. Публикуется только на 127.0.0.1 хоста. +frontend stats + bind :8404 + http-request use-service prometheus-exporter if { path /metrics } + stats enable + stats uri /stats + stats refresh 2s + stats show-legends diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..34533405 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,59 @@ +-- 0001: базовая схема мессенджера lab2. +-- Применяется раннером app/migrate.py внутри одной транзакции; версия пишется в schema_migrations. + +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, -- строка Argon2id ($argon2id$v=19$m=...,t=...,p=...$salt$hash) + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), -- защита от гонки регистраций: решает БД, а не приложение + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +-- Серверные сессии. Источник истины; Valkey хранит только копию с TTL. +-- В БД лежит SHA-256 от токена, а не сам токен: утечка дампа не даёт войти. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY CHECK (octet_length(token_hash) = 32), + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_expiry_after_create CHECK (expires_at > created_at) +); +CREATE INDEX sessions_user_id_idx ON sessions (user_id); +CREATE INDEX sessions_expires_at_idx ON sessions (expires_at); -- для периодической очистки истёкших + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +-- Список чатов сортируется по (created_at, id). +CREATE INDEX chats_created_at_id_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + joined_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) -- один пользователь входит в чат один раз +); +-- "Чаты пользователя": поиск по user_id (PK начинается с chat_id и тут не помогает). +CREATE INDEX chat_members_user_id_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + -- Отправитель обязан быть участником этого чата: гарантирует сама БД. + CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id) +); +-- Индекс под keyset-пагинацию истории: +-- WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n +-- Равенство по chat_id + диапазон по (created_at, id) читаются одним проходом индекса без сортировки. +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..d96c5bf1 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,21 @@ +-- 0002: общая идемпотентность POST /chats/{chat_id}/messages (lab3). +-- Только добавляет таблицу (expand): версия lab2 продолжает работать на этой схеме, пока идёт замена реплик. + +-- Область уникальности контракта: пользователь + метод + путь чата + ключ. +-- Метод один (POST), путь однозначно задаётся chat_id, поэтому PK = (user_id, chat_id, key). +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL, + chat_id uuid NOT NULL, + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash bytea NOT NULL CHECK (octet_length(request_hash) = 32), -- SHA-256 канонического JSON тела + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT idempotency_sender_is_member FOREIGN KEY (chat_id, user_id) + REFERENCES chat_members (chat_id, user_id), + -- Ключ вставляется ДО сообщения (он и есть "замок" против гонки), поэтому FK проверяется при COMMIT. + CONSTRAINT idempotency_message_fk FOREIGN KEY (message_id) + REFERENCES messages (id) ON DELETE CASCADE DEFERRABLE INITIALLY DEFERRED +); +-- Периодическая очистка ключей старше IDEMPOTENCY_TTL_SECONDS. +CREATE INDEX idempotency_keys_created_at_idx ON idempotency_keys (created_at); diff --git a/migrations/0003_cache_state.sql b/migrations/0003_cache_state.sql new file mode 100644 index 00000000..1d2a8821 --- /dev/null +++ b/migrations/0003_cache_state.sql @@ -0,0 +1,10 @@ +-- 0003: общий для всех реплик признак "кэшу сессий не доверять до ...". +-- В lab2 окно недоверия после сбоя Valkey жило в памяти единственного процесса. С несколькими репликами +-- это локальное состояние, от которого зависит корректность: реплика, не видевшая сбоя, отдала бы +-- копию отозванной сессии. Теперь реплика, получившая ошибку Valkey, публикует окно здесь. +-- Только добавление (expand): версии без этой таблицы её просто не используют. +CREATE TABLE cache_state ( + id smallint PRIMARY KEY CHECK (id = 1), -- одна строка на весь стенд + distrust_until timestamptz NOT NULL +); +INSERT INTO cache_state (id, distrust_until) VALUES (1, '-infinity'); diff --git a/migrations/0004_attachments.sql b/migrations/0004_attachments.sql new file mode 100644 index 00000000..5d976fa4 --- /dev/null +++ b/migrations/0004_attachments.sql @@ -0,0 +1,50 @@ +-- 0004: вложения (lab4). Метаданные — здесь, в PostgreSQL; байты — в приватном S3-bucket. +-- Только expand: версия lab3 продолжает работать на этой схеме (новые таблицы она не читает). + +-- Одно вложение = один оригинал в S3. Файл заранее привязан к чату (chat_id), а не к сообщению: +-- сначала upload, потом POST сообщения с attachment_ids. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL, + filename text NOT NULL CHECK (char_length(filename) BETWEEN 1 AND 255), -- только для отображения + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + -- Ключ объекта назначает сервер (attachments/), имя файла клиента в нём не участвует. + -- В публичный API ключ не попадает. + object_key text NOT NULL UNIQUE, + status text NOT NULL DEFAULT 'ready' CHECK (status IN ('ready')), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + -- Загрузивший обязан быть участником чата. + CONSTRAINT attachments_owner_is_member FOREIGN KEY (chat_id, owner_id) + REFERENCES chat_members (chat_id, user_id), + -- Цель для составного FK из message_attachments: вложение и сообщение — из одного чата. + CONSTRAINT attachments_id_chat_key UNIQUE (id, chat_id) +); +CREATE INDEX attachments_chat_id_idx ON attachments (chat_id); + +-- Связь сообщение -> вложения (0..10, порядок как в запросе). +-- chat_id входит в оба FK: БД сама не даст прикрепить вложение другого чата. +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + CONSTRAINT message_attachments_attachment_fk FOREIGN KEY (attachment_id, chat_id) + REFERENCES attachments (id, chat_id) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +-- Сообщение может состоять только из вложений: text="" допустим (непустота text ИЛИ вложений +-- проверяется в API, т.к. вложения вставляются отдельной строкой в той же транзакции). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (char_length(text) BETWEEN 0 AND 4000); +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_key UNIQUE (id, chat_id); +ALTER TABLE message_attachments ADD CONSTRAINT message_attachments_message_fk + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id) ON DELETE CASCADE; diff --git a/migrations/0005_nul_safe_text.sql b/migrations/0005_nul_safe_text.sql new file mode 100644 index 00000000..ec860181 --- /dev/null +++ b/migrations/0005_nul_safe_text.sql @@ -0,0 +1,7 @@ +-- 0005_nul_safe_text: точное хранение строк с U+0000 (замечание к lab3, PR #99: "abc\u0000def" давал 500). +-- PostgreSQL text не хранит 0x00. Если строка содержит U+0000, её точные UTF-8 байты пишутся в *_raw, +-- а в основной столбец — та же строка с U+FFFD вместо U+0000 (см. app/pgtext.py). +-- Только expand: столбцы nullable, версия без них продолжает работать на этой схеме. +ALTER TABLE users ADD COLUMN display_name_raw bytea CHECK (octet_length(display_name_raw) <= 400); +ALTER TABLE chats ADD COLUMN title_raw bytea CHECK (octet_length(title_raw) <= 400); +ALTER TABLE messages ADD COLUMN text_raw bytea CHECK (octet_length(text_raw) <= 16000); diff --git a/migrations/0006_jobs.sql b/migrations/0006_jobs.sql new file mode 100644 index 00000000..26bb43e6 --- /dev/null +++ b/migrations/0006_jobs.sql @@ -0,0 +1,81 @@ +-- 0006: асинхронная обработка вложений (lab5): состояния, задания, варианты, transactional outbox, audit. +-- Только expand: новые таблицы и столбцы; прежняя (lab4) версия видит ready-вложения как раньше. + +-- Вложение: queued -> processing -> ready | failed (contracts/attachment-states.md). +ALTER TABLE attachments DROP CONSTRAINT attachments_status_check; +ALTER TABLE attachments ADD CONSTRAINT attachments_status_check + CHECK (status IN ('queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ALTER COLUMN status SET DEFAULT 'queued'; +-- Безопасный машинный код для failed (invalid_image, retry_exhausted, ...); у ready всегда NULL. +ALTER TABLE attachments ADD COLUMN error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_error_only_failed + CHECK ((status = 'failed') = (error_code IS NOT NULL)); +-- Устойчивый staging: приватный prefix staging/ того же bucket. object_key — конечный ключ оригинала +-- (originals/); он появляется в S3, когда worker публикует результат. +ALTER TABLE attachments ADD COLUMN staging_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1 CHECK (pipeline_version >= 1); + +-- Варианты. Ключ эффекта контракта: (attachment_id, pipeline_version, variant) — повтор не создаёт второй набор. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL, + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes bigint NOT NULL CHECK (size_bytes > 0), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE, -- variants//v/: детерминирован + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- Одна логическая работа на (вложение, версия pipeline). job_id не меняется при redelivery и retry. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL DEFAULT 1, + status text NOT NULL DEFAULT 'queued' CHECK (status IN ('queued', 'processing', 'done', 'failed')), + attempt integer NOT NULL DEFAULT 0 CHECK (attempt >= 0), -- число начатых попыток + max_attempts integer NOT NULL CHECK (max_attempts BETWEEN 1 AND 50), + next_attempt_at timestamptz NOT NULL DEFAULT clock_timestamp(), + lease_until timestamptz, -- аренда processing; продлевается heartbeat-ом worker + worker_id text, -- владелец аренды (fencing при фиксации результата) + error_code text, + last_error text, -- краткая причина без секретов и stack trace + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + started_at timestamptz, + finished_at timestamptz, + UNIQUE (attachment_id, pipeline_version), + CONSTRAINT jobs_lease_when_processing CHECK ((status = 'processing') = (lease_until IS NOT NULL)) +); +-- Backpressure считает незавершённую работу; reaper ищет просроченные аренды. +CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (status) WHERE status IN ('queued', 'processing'); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing'; + +-- Transactional outbox: намерение опубликовать job фиксируется в той же транзакции, что и вложение. +-- Relay публикует с publisher confirm и только потом ставит published_at. available_at > now() — +-- отложенная публикация (retry с backoff) без плагинов брокера. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + payload jsonb NOT NULL, -- contracts/jobs.schema.json: только ID и служебные поля + available_at timestamptz NOT NULL DEFAULT clock_timestamp(), + published_at timestamptz, + publish_attempts integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX outbox_unpublished_idx ON outbox (available_at, id) WHERE published_at IS NULL; + +-- Audit операторских действий (явный replay failed-задания). +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + action text NOT NULL CHECK (action IN ('replay')), + previous_status text NOT NULL, + previous_error text, + actor text NOT NULL, + reason text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6ccd1ca0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,38 @@ +# Полный lock (pip freeze в python:3.12.12-slim-bookworm); устанавливается с --no-deps. +# Прямые зависимости: fastapi, uvicorn, asyncpg, argon2-cffi, redis (клиент совместим с Valkey), +# lab4: minio (S3-клиент, работает с RustFS), pillow (проверка и обработка PNG/JPEG), +# lab5: aio-pika (RabbitMQ: publisher confirms, manual ack), jsonschema (проверка job по contracts/jobs.schema.json). +aio-pika==9.5.7 +aiormq==6.9.4 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==23.1.0 +argon2-cffi-bindings==26.1.0 +asyncpg==0.30.0 +attrs==26.1.0 +certifi==2026.10.23 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +idna==3.20 +jsonschema==4.25.1 +jsonschema-specifications==2025.9.1 +minio==7.2.18 +multidict==7.2.1 +pamqp==3.3.0 +pillow==11.3.0 +propcache==0.5.4 +pycparser==3.12 +pycryptodome==4.4.0 +pydantic==2.14.1 +pydantic_core==2.50.1 +redis==5.2.1 +referencing==0.37.0 +rpds-py==2026.9.1 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.2 +yarl==1.25.1 diff --git a/scripts/collect-evidence.sh b/scripts/collect-evidence.sh new file mode 100755 index 00000000..defba587 --- /dev/null +++ b/scripts/collect-evidence.sh @@ -0,0 +1,51 @@ +#!/bin/sh +# Повторяемый сбор доказательств уровней 3–4 lab5 в evidence/ (запуск на хосте из корня, стенд уже поднят). +# Токены/пароли/ключи S3 в вывод не попадают: тесты их не печатают, hook-вывод подавлен тестом. +set -eu +cd "$(dirname "$0")/.." +export MSYS_NO_PATHCONV=1 PYTHONUTF8=1 +compose() { docker compose --env-file .env "$@"; } +tester() { compose --profile tools run --rm tester sh -c "$1" 2>&1 | grep -v '^ Container'; } +stamp() { echo "# $(date -u +%Y-%m-%dT%H:%M:%SZ), commit $(git rev-parse --short HEAD 2>/dev/null || echo uncommitted)"; } +mkdir -p evidence +: > evidence/stop-one.log + +echo '== stand and S3 bootstrap' +{ stamp + compose ps -a --format 'table {{.Name}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' + echo; echo '# s3-init (idempotent bucket bootstrap + anonymous access check):' + compose logs --no-log-prefix s3-init + echo; echo '# repeated bootstrap run (must not touch existing bucket):' + compose run --rm -T s3-init 2>&1 | grep -v '^ Container' + echo; echo '# volumes:' + docker volume ls --filter label=com.docker.compose.project=mtusi-lab5 --format '{{.Name}}' + echo; echo '# RabbitMQ: очереди (quorum), consumers и prefetch:' + compose exec -T rabbitmq rabbitmqctl -q list_queues name type durable messages consumers + compose exec -T rabbitmq rabbitmqctl -q list_consumers queue_name prefetch_count ack_required +} | tee evidence/stand.txt + +echo '== public checks' +{ stamp + tester 'pip install -q --disable-pip-version-check --root-user-action=ignore -r requirements-dev.txt && make check && make validate PYTHON=python3 && make test' +} | tee evidence/public-tests.txt + +echo '== failover: stop-one for each replica (restore between runs)' +{ stamp + for i in 1 2 3 4; do + tester 'make test-failover ACTION_SCRIPT=scripts/stop-one.sh' + tester 'sh scripts/restore.sh && curl -s -o /dev/null http://lb:8080/health/live' + [ "$(cut -d' ' -f4 evidence/stop-one.log | sort -u | wc -l)" -ge 2 ] && break + done + echo; echo '# evidence/stop-one.log (what the hook really stopped):'; cat evidence/stop-one.log +} | tee evidence/failover-hook.txt + +echo '== persistence: restart of lb, api, worker, postgres, s3, rabbitmq, valkey with volumes' +{ stamp + tester 'make test-persistence ACTION_SCRIPT=scripts/restart.sh' + echo; echo '# hook scope (StartedAt before/after of every process):' + sh scripts/restart.sh + tester 'until curl -fs http://lb:8080/health/ready >/dev/null; do sleep 0.5; done; echo ready' +} | tee evidence/persistence.txt + +echo '== student tests (lab5 + lab4/lab3/lab2 regression)' +{ stamp; sh scripts/test-lab5.sh 2>&1 | grep -v '^ Container'; } | tee evidence/student-tests.txt diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..9d5ab642 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# Создаёт локальные секреты в ./secrets (каталог в .gitignore). Повторный запуск ничего не перезаписывает. +set -eu +cd "$(dirname "$0")/.." +# Каталог закрыт для других пользователей хоста; сами файлы 0644, т.к. compose монтирует их +# в контейнеры с правами хоста, а процессы там работают под непривилегированными uid. +umask 022 +mkdir -p secrets +chmod 700 secrets + +random_hex() { od -An -tx1 -N24 /dev/urandom | tr -d ' \n'; } + +if [ ! -f secrets/postgres_password ]; then + pg=$(random_hex) + printf '%s' "$pg" > secrets/postgres_password + printf 'postgresql://app:%s@postgres:5432/messenger' "$pg" > secrets/database_url + echo 'created secrets/postgres_password, secrets/database_url' +fi +if [ ! -f secrets/valkey.conf ]; then + vk=$(random_hex) + # Valkey — только кэш сессий: persistence выключена намеренно, источник истины — PostgreSQL. + cat > secrets/valkey.conf < secrets/cache_url + echo 'created secrets/valkey.conf, secrets/cache_url' +fi +if [ ! -f secrets/s3_secret_access_key ]; then + # Учётные данные S3 (RustFS): access key — не секретный идентификатор, но тоже вне Git. + printf 'msg%s' "$(od -An -tx1 -N8 /dev/urandom | tr -d ' \n')" > secrets/s3_access_key_id + random_hex > secrets/s3_secret_access_key + echo 'created secrets/s3_access_key_id, secrets/s3_secret_access_key' +fi +if [ ! -f secrets/rabbitmq.conf ]; then + # RabbitMQ: отдельный пользователь приложения вместо guest (guest разрешён только с localhost). + rp=$(random_hex) + printf 'default_user = messenger\ndefault_pass = %s\ndefault_vhost = /\nloopback_users = none\n' "$rp" > secrets/rabbitmq.conf + printf 'amqp://messenger:%s@rabbitmq:5672/%%2F' "$rp" > secrets/broker_url + echo 'created secrets/rabbitmq.conf, secrets/broker_url' +fi +if [ ! -f .env ]; then + cp .env.example .env + echo 'created .env from .env.example' +fi +echo 'secrets ready (values are not printed)' diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..35fa465a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Перезапуск ВСЕГО стенда: lb, API-реплики, worker, PostgreSQL, S3 (RustFS), RabbitMQ, Valkey — с сохранением volumes. +# Не выполняет down -v, FLUSHALL и повторный seed. Готовность API ожидает сам тест. +set -eu +cd "$(dirname "$0")/.." +compose() { docker compose --env-file .env "$@"; } + +started_at() { docker inspect -f '{{.Name}} {{.State.StartedAt}}' $(compose ps -q lb api worker postgres s3 rabbitmq valkey); } + +echo 'before:'; started_at +compose stop lb api worker # сначала входная точка и клиенты хранилищ +compose restart postgres s3 rabbitmq valkey # обычный restart: volumes pgdata, s3data, rabbitdata остаются +compose start api worker lb +echo 'after:'; started_at diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..dea5cbfd --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Возвращает остановленные API-реплики (после stop-one.sh) и ждёт их healthcheck. +set -eu +project=${COMPOSE_PROJECT_NAME:-mtusi-lab5} +filter="--filter label=com.docker.compose.project=$project --filter label=com.docker.compose.service=api" +stopped=$(docker ps -aq $filter --filter status=exited) +[ -n "$stopped" ] && docker start $stopped >/dev/null +for _ in $(seq 1 60); do + unhealthy=$(docker ps -q $filter | xargs docker inspect -f '{{.State.Health.Status}}' | grep -vc healthy || true) + [ "$unhealthy" = 0 ] && break + sleep 1 +done +docker ps $filter --format '{{.Names}} {{.Status}}' diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..f52ac562 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,36 @@ +#!/bin/sh +# Поочерёдная замена API-реплик без остановки сервиса (Compose не умеет rolling update сам). +# 1. собрать новый образ (BUILD_ID попадает в label образа); +# 2. применить миграции ОДИН раз (только expand-миграции: старая версия работает на новой схеме); +# 3. поднять столько же новых реплик рядом со старыми (surge x2) и дождаться их healthcheck; +# 4. по одной останавливать старые: docker stop -> SIGTERM -> drain -> graceful shutdown, затем удалить. +# HAProxy сам находит новые реплики через DNS и исключает старые по /health/ready. +# Запускать на хосте из корня репозитория (нужны bind-пути compose.yaml). NO_BUILD=1 — без пересборки. +set -eu +cd "$(dirname "$0")/.." +compose() { docker compose --env-file .env "$@"; } +project=mtusi-lab5 +filter="--filter label=com.docker.compose.project=$project --filter label=com.docker.compose.service=api" + +old=$(docker ps -q $filter) +n=$(echo "$old" | wc -w) +[ "$n" -ge 1 ] || { echo 'no running api replicas' >&2; exit 2; } +build_id=${BUILD_ID:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date +%s)} + +echo "$(date -u +%H:%M:%S) old replicas ($n): $(docker inspect -f '{{.Name}}={{index .Config.Labels "org.opencontainers.image.revision"}}' $old | tr -d / | tr '\n' ' ')" +if [ "${NO_BUILD:-0}" != 1 ]; then + compose build -q --build-arg BUILD_ID="$build_id" api +fi +compose run --rm -T migrate # однократно, под advisory lock +compose up -d --wait --no-deps --no-recreate --scale api=$((n * 2)) api 2>&1 | grep -v '^ Container' || true +new=$(docker ps -q $filter | grep -vxF "$old" || true) +[ "$(echo "$new" | wc -w)" -eq "$n" ] || { echo 'new replicas did not start' >&2; exit 3; } +echo "$(date -u +%H:%M:%S) new replicas healthy: $(docker inspect -f '{{.Name}}={{index .Config.Labels "org.opencontainers.image.revision"}}' $new | tr -d / | tr '\n' ' ')" +sleep "${SETTLE_SECONDS:-3}" # LB: rise 2 x inter 1 s + DNS 1 с +for c in $old; do + name=$(docker inspect -f '{{.Name}}' "$c" | tr -d /) + docker stop -t 20 "$c" >/dev/null + docker rm "$c" >/dev/null + echo "$(date -u +%H:%M:%S) replaced $name" +done +docker ps $filter --format '{{.Names}} {{.Status}} {{.Label "org.opencontainers.image.revision"}}' diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..8fc49135 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Останавливает ОДНУ API-реплику — ту, что обслужила тест, — и оставляет её остановленной. +# TARGET_INSTANCE_ID = "-<случайный суффикс>", hostname = короткий ID контейнера. +# STOP_MODE=stop (по умолчанию): docker stop -> SIGTERM -> drain -> graceful shutdown. +# STOP_MODE=kill: SIGKILL, внезапная смерть процесса без drain. +# БД, LB и volumes не трогает. Восстановление — отдельно: sh scripts/restore.sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +project=${COMPOSE_PROJECT_NAME:-mtusi-lab5} +cid=${TARGET_INSTANCE_ID%-*} +case "$cid" in ''|*[!0-9a-f]*) echo 'TARGET_INSTANCE_ID has unexpected format' >&2; exit 2 ;; esac + +# Защита от остановки чужого контейнера: только сервис api этого проекта. +owner=$(docker inspect -f '{{index .Config.Labels "com.docker.compose.project"}}/{{index .Config.Labels "com.docker.compose.service"}}' "$cid") +[ "$owner" = "$project/api" ] || { echo "container $cid is not $project/api" >&2; exit 3; } +name=$(docker inspect -f '{{.Name}}' "$cid" | tr -d /) + +started=$(date -u +%Y-%m-%dT%H:%M:%SZ) +if [ "${STOP_MODE:-stop}" = kill ]; then + docker kill "$cid" >/dev/null +else + docker stop -t 20 "$cid" >/dev/null +fi +state=$(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}' "$cid") +mkdir -p evidence +printf '%s mode=%s instance=%s container=%s state=%s\n' \ + "$started" "${STOP_MODE:-stop}" "$TARGET_INSTANCE_ID" "$name" "$state" >> evidence/stop-one.log +echo "stopped $name: $state" diff --git a/scripts/test-lab5.sh b/scripts/test-lab5.sh new file mode 100755 index 00000000..391693d7 --- /dev/null +++ b/scripts/test-lab5.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Собственные проверки lab5 + регрессия lab4, lab3 и уровня 4 lab2. Запускать на хосте (Linux/macOS/Git Bash) из корня. +# Поднимает api-ttl (SESSION_TTL_SECONDS=2), запускает student-tests в контейнере tester, гасит api-ttl. +set -eu +cd "$(dirname "$0")/.." +compose() { docker compose --env-file .env "$@"; } +compose --profile test up -d --wait api-ttl +status=0 +MSYS_NO_PATHCONV=1 compose --profile tools run --rm tester sh -c \ + 'python -u student-tests/test_lab5.py; a=$?; python -u student-tests/test_lab4.py; b=$?; python -u student-tests/test_lab3.py; c=$?; python -u student-tests/test_lab2_regression.py; d=$?; exit $((a | b | c | d))' || status=$? +compose --profile test stop api-ttl +exit $status diff --git a/scripts/track1.sh b/scripts/track1.sh new file mode 100755 index 00000000..57be4bac --- /dev/null +++ b/scripts/track1.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Трек 1 «Очередь под перегрузкой»: матрица concurrency/prefetch x размер изображения x повтор, +# затем backpressure (MAX_PENDING_JOBS) и восстановление. Запускать на хосте из корня, стенд поднят. +# Результаты: evidence/track1/*.json + evidence/track1/summary.txt. Конфигурация стенда в конце возвращается. +set -eu +cd "$(dirname "$0")/.." +export MSYS_NO_PATHCONV=1 +compose() { docker compose --env-file .env "$@"; } +load() { compose --profile tools run --rm tester python -u student-tests/overload.py "$@" 2>&1 | grep -v '^ Container'; } +REPEATS=${REPEATS:-2} +# Поток выше пропускной способности worker при любой из конфигураций -> очередь растёт, видно lag/backpressure. +SMALL="--size small --count 200 --rate 40" +LARGE="--size large --count 60 --rate 8" +mkdir -p evidence/track1 +: > evidence/track1/summary.txt + +worker() { # пересоздать worker с заданными concurrency/prefetch (env shell важнее .env) + WORKER_CONCURRENCY=$1 WORKER_PREFETCH=$2 compose up -d --no-deps --wait worker 2>&1 | grep -v '^ Container' || true +} + +for cfg in "1 1" "2 2" "4 4" "2 16"; do + set -- $cfg + worker "$1" "$2" + for r in $(seq 1 "$REPEATS"); do + load --label "c$1-p$2-small-r$r" $SMALL | tee -a evidence/track1/summary.txt + load --label "c$1-p$2-large-r$r" $LARGE | tee -a evidence/track1/summary.txt + done +done + +# Backpressure: принятая работа ограничена 20 незавершёнными заданиями; поток 15/с больших изображений +# заведомо выше пропускной способности worker при c=2 -> часть upload получает 503 + Retry-After. +worker 2 2 +# База для сравнения с восстановлением: та же нагрузка ниже пропускной способности, до перегрузки. +load --label baseline-small --size small --count 60 --rate 5 | tee -a evidence/track1/summary.txt +MAX_PENDING_JOBS=20 compose up -d --no-deps --wait api 2>&1 | grep -v '^ Container' || true +load --label backpressure-large --size large --count 120 --rate 15 | tee -a evidence/track1/summary.txt +# Восстановление: сразу после разгрузки — та же лёгкая нагрузка, что в базовом прогоне. +load --label recovery-small --size small --count 60 --rate 5 | tee -a evidence/track1/summary.txt +compose up -d --no-deps --wait api 2>&1 | grep -v '^ Container' || true +echo 'track1 done' | tee -a evidence/track1/summary.txt diff --git a/student-tests/failover_series.py b/student-tests/failover_series.py new file mode 100755 index 00000000..3d350ebc --- /dev/null +++ b/student-tests/failover_series.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +"""Серии отказов под нагрузкой (уровень 4): RPS, ошибки, время обнаружения и восстановления. + +Запускается на ХОСТЕ из корня репозитория (нужны docker CLI и, для rolling, compose с bind-путями): + python student-tests/failover_series.py --scenario stop # graceful stop каждой реплики по очереди + python student-tests/failover_series.py --scenario kill # SIGKILL каждой реплики по очереди + python student-tests/failover_series.py --scenario rolling # scripts/rolling-update.sh под нагрузкой + +Нагрузка closed-loop: W писателей (POST сообщения; неопределённый результат повторяется с ТЕМ ЖЕ ключом, +как в tests/resilience.py) и R читателей (GET истории). Параллельно раз в 200 мс опрашивается +CSV-статистика HAProxy: когда слот целевой реплики перестал быть UP — это время обнаружения отказа LB. +После серии проверяется, что все подтверждённые сообщения на месте и нет дублей. +Токены остаются в памяти процесса; в отчёт попадают только агрегаты. +""" +import argparse +import csv +import io +import json +import math +import os +import subprocess +import sys +import threading +import time +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check # noqa: E402 + +PROJECT = 'mtusi-lab5' +EDGE_NETWORK = PROJECT + '_edge' +API_FILTER = ['--filter', f'label=com.docker.compose.project={PROJECT}', + '--filter', 'label=com.docker.compose.service=api'] + + +def docker(*args: str) -> str: + return subprocess.run(['docker', *args], check=True, text=True, capture_output=True).stdout.strip() + + +def api_containers() -> list[str]: + names = docker('ps', *API_FILTER, '--format', '{{.Names}}').split() + return sorted(names) + + +def container_info(name: str) -> dict: + fmt = ('{{.Config.Hostname}}|{{index .Config.Labels "org.opencontainers.image.revision"}}|' + '{{(index .NetworkSettings.Networks "%s").IPAddress}}') % EDGE_NETWORK + hostname, revision, ip = docker('inspect', '-f', fmt, name).split('|') + return {'name': name, 'hostname': hostname, 'revision': revision, 'ip': ip} + + +def percentile(values: list[float], p: float): + values = sorted(values) + return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 1) if values else None + + +class LbWatcher(threading.Thread): + """Опрос /stats;csv: состояние слотов backend `api` и счётчики retry/redispatch.""" + + def __init__(self, stats_url: str, t0: float): + super().__init__(daemon=True) + self.url = stats_url.rstrip('/') + ';csv' + self.t0 = t0 + self.stop_flag = threading.Event() + self.samples = [] # (t, {addr: status}) + self.first = self.last = None + + def snapshot(self): + with urllib.request.urlopen(self.url, timeout=1) as r: + rows = list(csv.DictReader(io.StringIO(r.read().decode().lstrip('# ')))) + servers = {row['addr']: row['status'] for row in rows + if row['pxname'] == 'api' and row['svname'] not in ('FRONTEND', 'BACKEND') and row.get('addr')} + backend = next(row for row in rows if row['pxname'] == 'api' and row['svname'] == 'BACKEND') + return servers, {'retries': int(backend['wretr'] or 0), 'redispatches': int(backend['wredis'] or 0)} + + def run(self): + while not self.stop_flag.is_set(): + try: + servers, counters = self.snapshot() + self.samples.append((time.monotonic() - self.t0, servers)) + self.first = self.first or counters + self.last = counters + except Exception: # LB-статистика — вспомогательная, не влияет на результат серии + pass + time.sleep(0.2) + + def first_not_up(self, ip: str, after: float): + """Первый момент после `after`, когда слот с этим IP не UP (DOWN/MAINT/исчез из DNS).""" + addr = ip + ':8080' + for t, servers in self.samples: + if t >= after and not servers.get(addr, 'MISSING').startswith('UP'): + return round(t - after, 3), servers.get(addr, 'removed from DNS') + return None, None + + +def run_probe(base_url: str, stats_url: str, action, targets: list[dict], duration: float, + action_at: float, writers: int, readers: int) -> dict: + c = Client(base_url) + alice, bob, _eve, chat = c.bootstrap() + events = [] # (t, kind, ok, instance, latency) + confirmed = [] # подтверждённые сообщения + lock = threading.Lock() + retries_with_same_key = [0] + t0 = time.monotonic() + deadline = t0 + duration + watcher = LbWatcher(stats_url, t0) + watcher.start() + + def record(kind, ok, instance, latency): + with lock: + events.append((time.monotonic() - t0, kind, ok, instance, latency)) + + def writer(i): + local = Client(base_url, timeout=3) + key, n = uuid.uuid4().hex, 0 + text = f'probe-{i}-{n}' + while time.monotonic() < deadline: + start = time.monotonic() + try: + r = local.message(alice, chat, text, key=key) + record('write', True, r['headers'].get('X-Instance-Id', ''), time.monotonic() - start) + with lock: + confirmed.append(r['json']) + n += 1 + key, text = uuid.uuid4().hex, f'probe-{i}-{n}' + except Failure: + record('write', False, '', time.monotonic() - start) + with lock: + retries_with_same_key[0] += 1 + time.sleep(0.05) # тот же key и тот же body при следующей попытке + + def reader(_i): + local = Client(base_url, timeout=3) + path = f"/api/v1/chats/{chat['id']}/messages?limit=20" + while time.monotonic() < deadline: + start = time.monotonic() + try: + r = local.call('GET', path, actor=bob, schema='MessagePage') + record('read', True, r['headers'].get('X-Instance-Id', ''), time.monotonic() - start) + except Failure: + record('read', False, '', time.monotonic() - start) + time.sleep(0.05) + + threads = [threading.Thread(target=writer, args=(i,)) for i in range(writers)] + threads += [threading.Thread(target=reader, args=(i,)) for i in range(readers)] + for t in threads: + t.start() + time.sleep(max(0.0, t0 + action_at - time.monotonic())) + a_start = time.monotonic() - t0 + action_output = action() + a_end = time.monotonic() - t0 + for t in threads: + t.join() + watcher.stop_flag.set() + watcher.join() + + # Сохранность: тот же токен, полная история через LB, без дублей. + history, cursor = [], None + while True: + path = f"/api/v1/chats/{chat['id']}/messages?limit=100" + (f'&cursor={urllib.parse.quote(cursor, safe="")}' if cursor else '') + page = c.call('GET', path, actor=bob, schema='MessagePage')['json'] + history += page['items'] + cursor = page['next_cursor'] + if cursor is None: + break + ids = {m['id'] for m in history} + texts = [m['text'] for m in history] + lost = sum(1 for m in confirmed if m['id'] not in ids) + check(len(ids) == len(history), 'duplicated message id in history') + check(len(texts) == len(set(texts)), 'retry with same key produced a duplicate message') + check(lost == 0, f'{lost} confirmed messages lost') + + def window(lo, hi): + sel = [e for e in events if lo <= e[0] < hi] + ok = [e for e in sel if e[2]] + span = max(hi - lo, 1e-9) + return {'seconds': round(span, 2), 'ok_rps': round(len(ok) / span, 1), + 'write_rps': round(sum(1 for e in ok if e[1] == 'write') / span, 1), + 'read_rps': round(sum(1 for e in ok if e[1] == 'read') / span, 1), + 'errors': len(sel) - len(ok), + 'p50_ms': percentile([e[4] for e in ok], .50), 'p95_ms': percentile([e[4] for e in ok], .95), + 'p99_ms': percentile([e[4] for e in ok], .99), 'max_ms': percentile([e[4] for e in ok], 1.0)} + + errors = [e for e in events if not e[2]] + target_prefixes = tuple(t['hostname'] + '-' for t in targets) + served_by_target = [e[0] for e in events if e[2] and e[3].startswith(target_prefixes)] + per_second = [] + for s in range(int(duration)): + sel = [e for e in events if s <= e[0] < s + 1] + per_second.append({'t': s, 'ok': sum(1 for e in sel if e[2]), 'err': sum(1 for e in sel if not e[2]), + 'instances': len({e[3] for e in sel if e[2]})}) + detection = {} + for t in targets: + seconds, state = watcher.first_not_up(t['ip'], a_start) + detection[t['name']] = {'lb_marked_not_up_after_s': seconds, 'lb_state': state} + lb_counters = None + if watcher.first and watcher.last: + lb_counters = {k: watcher.last[k] - watcher.first[k] for k in watcher.last} + return { + 'targets': [{k: t[k] for k in ('name', 'hostname', 'revision')} for t in targets], + 'action_started_s': round(a_start, 3), 'action_duration_s': round(a_end - a_start, 3), + 'action_output': action_output, + 'before': window(1, a_start), 'during_action_plus_5s': window(a_start, a_end + 5), + 'after': window(a_end + 5, duration), + 'errors_total': len(errors), + 'write_attempts_retried_same_key': retries_with_same_key[0], + 'error_window_s': ([round(errors[0][0] - a_start, 3), round(errors[-1][0] - a_start, 3)] if errors else None), + 'recovery_s': round(errors[-1][0] - a_start, 3) if errors else 0.0, + 'target_last_served_after_action_s': (round(max(served_by_target) - a_start, 3) + if served_by_target and max(served_by_target) >= a_start else None), + 'lb_detection': detection, + 'lb_retries_redispatches': lb_counters, + 'instances_after': sorted({e[3] for e in events if e[2] and e[0] > a_end + 5}), + 'confirmed_writes': len(confirmed), 'history_messages': len(history), + 'duplicates': 0, 'lost_confirmed': lost, + 'timeline_per_second': per_second, + } + + +def wait_lb_up(stats_url: str, ip: str, seconds: float = 60) -> None: + deadline = time.monotonic() + seconds + url = stats_url.rstrip('/') + ';csv' + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(url, timeout=1) as r: + rows = csv.DictReader(io.StringIO(r.read().decode().lstrip('# '))) + if any(row['pxname'] == 'api' and row.get('addr') == ip + ':8080' and row['status'].startswith('UP') + for row in rows): + return + except OSError: + pass + time.sleep(0.5) + raise Failure('replica did not return to LB rotation') + + +def restore(stats_url: str) -> None: + subprocess.run(['sh', str(ROOT / 'scripts/restore.sh')], check=True, capture_output=True) + for name in api_containers(): + wait_lb_up(stats_url, container_info(name)['ip']) + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + p.add_argument('--scenario', choices=['stop', 'kill', 'rolling'], required=True) + p.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + p.add_argument('--stats-url', default=os.environ.get('LB_STATS_URL', 'http://localhost:8404/stats')) + p.add_argument('--duration', type=float, default=None) + p.add_argument('--action-at', type=float, default=8) + p.add_argument('--writers', type=int, default=4) + p.add_argument('--readers', type=int, default=2) + p.add_argument('--output', default=None) + args = p.parse_args() + duration = args.duration or (45 if args.scenario == 'rolling' else 25) + + runs = [] + if args.scenario == 'rolling': + targets = [container_info(n) for n in api_containers()] + + def action(): + r = subprocess.run(['sh', str(ROOT / 'scripts/rolling-update.sh')], text=True, capture_output=True) + check(r.returncode == 0, 'rolling-update.sh failed: ' + r.stderr[-500:]) + return r.stdout.strip().splitlines() + result = run_probe(args.base_url, args.stats_url, action, targets, duration, args.action_at, + args.writers, args.readers) + result['replicas_after'] = [container_info(n) for n in api_containers()] + runs.append(result) + else: + for name in api_containers(): # по очереди каждую реплику, восстанавливая предыдущую + restore(args.stats_url) + target = container_info(name) + + def action(name=name): + started = time.monotonic() + if args.scenario == 'kill': + docker('kill', name) + else: + docker('stop', '-t', '20', name) + state = docker('inspect', '-f', '{{.State.Status}} exit={{.State.ExitCode}}', name) + return [f'{args.scenario} {name}: {state} in {time.monotonic() - started:.2f}s'] + result = run_probe(args.base_url, args.stats_url, action, [target], duration, args.action_at, + args.writers, args.readers) + runs.append(result) + print(f"{args.scenario} {name}: errors={result['errors_total']} recovery={result['recovery_s']}s " + f"lb_detection={result['lb_detection'][name]['lb_marked_not_up_after_s']}s " + f"rps before/after={result['before']['ok_rps']}/{result['after']['ok_rps']}", flush=True) + restore(args.stats_url) + + report = { + 'scenario': args.scenario, 'date_utc': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'profile': {'duration_s': duration, 'action_at_s': args.action_at, 'writers': args.writers, + 'readers': args.readers, 'client_timeout_s': 3, + 'workload': 'closed-loop: POST message (retry same key on failure) + GET history limit=20'}, + 'runs': runs, + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + Path(args.output).parent.mkdir(parents=True, exist_ok=True) + Path(args.output).write_text(text + '\n', encoding='utf-8') + summary = [{k: r[k] for k in ('targets', 'errors_total', 'recovery_s', 'lb_detection', 'action_duration_s', + 'target_last_served_after_action_s', 'lb_retries_redispatches')} + | {'rps_before': r['before']['ok_rps'], 'rps_after': r['after']['ok_rps'], + 'p99_during_ms': r['during_action_plus_5s']['p99_ms']} for r in runs] + print(json.dumps(summary, ensure_ascii=False, indent=1)) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c35b7ec510e35174fbb392137a479fbdfdb22c55 GIT binary patch literal 1992 zcmex=HEAm;@P_1sVSzVUP#9la&z+7@&ZWiJ1lHN)ApgZlHp#0t`%y z%*;$c_X9luR9y>{XJ8Rz6;d>GWD^cdWLGK_F>0K+kVDyN<3Z7&iyu^slZu)+xx~aJ zB&Af<)HO7Pb?HxGHT=yahkYr<3UbkKb$@|Xn~>=}LrDtucxi9c)CgzAu!>|5(LYaO5NE)e8lyDFvm*19Ef zSB)IsMtE&}6?jAhsA{Idx791zxB5(|4l`O)wWW~Nr1{n|pxRfCZ-Z6juC}a}TPow| ziDF`+*74Ou87Q=-YDyui3c^63_bjqS> z63}r#vq~Y(IuzO;)TginMdgO6$yXM4Xaaoz^lf+$*q>`uBfe%l;_?LhHuMVES*tn2 zY`q``0@Z>Hyu_acG%&1BVG+W>lIB}j4NN7*Kwuz4T=?+|A|!y8eHB_$FNP5bz%TTWaLO#5Ai3~SjC70h=IgK0!lIgCMHla0lSdBA7UjmXxFctn*3uEX_4SJ zp&Dl-K*9_d3B<*!26C(}_pk-UYLrD&7ct4?>f#QYN|f|n>$6(^C?w$m&7w_C0ES)< zJSPB?95k~6vleDf07k+^^t_5AR)Kl-@k^BS{Yo|BpALCB0T>C#z&T+VFed=Z2v8=( zl@owj3ndbOMF=z!KzSADTl{$y7zwbv3bYkiCV}#*#v^`dmw2z4g0&ip_8oHnxK_-k zW8YLGrC))riE33_B7v-s6WuReMO$XR3QPgAf*0v86$ulv_0j?A(MoasvZ(8j&)1Ai zK-S8M;+ML#IgPI@Iyq6?79w}deWmMUCHO&Re6o$wnG4hw0@8NO=c~r161X + updated_at перехода в ready; это то, что видит клиент, опрашивая метаданные (без шума опроса); +- queue lag: каждые 0.5 с — число незавершённых заданий (queued + processing) из attachment_jobs и + глубина очереди RabbitMQ (messages_ready + unacked) — раз в 2 с; +- память worker: docker stats контейнера worker (процесс + дочерние процессы обработки), раз в ~1 с; +- throughput: число ready / (последний ready - первый принятый upload). +Результат — JSON в evidence/track1/