From b4804ac5c1153d3cc6c0b4100437af3b0d8409c5 Mon Sep 17 00:00:00 2001 From: LOSYASH Date: Thu, 12 Nov 2026 15:14:48 +0100 Subject: [PATCH 1/2] =?UTF-8?q?=D0=9B=D0=A05:=20=D0=B1=D1=80=D0=BE=D0=BA?= =?UTF-8?q?=D0=B5=D1=80=20Redis=20Streams,=20worker,=20outbox,=20retry/DLQ?= =?UTF-8?q?/replay,=20=D0=B2=D0=B0=D1=80=D0=B8=D0=B0=D0=BD=D1=82=D1=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 39 +- .env.example | 61 ++- .gitattributes | 11 + Dockerfile | 27 ++ REPORT.md | 183 ++++++++- app/__init__.py | 0 app/__main__.py | 22 ++ app/broker.py | 140 +++++++ app/config.py | 129 ++++++ app/db.py | 50 +++ app/images.py | 235 +++++++++++ app/imaging.py | 153 +++++++ app/janitor.py | 180 +++++++++ app/main.py | 634 ++++++++++++++++++++++++++++++ app/migrate.py | 57 +++ app/replay.py | 109 +++++ app/repo.py | 380 ++++++++++++++++++ app/security.py | 46 +++ app/sessions.py | 168 ++++++++ app/storage.py | 284 +++++++++++++ app/tools.py | 313 +++++++++++++++ app/uploads.py | 106 +++++ app/worker.py | 584 +++++++++++++++++++++++++++ compose.yaml | 350 +++++++++++++++++ evidence/attachments.txt | 108 +++++ evidence/broker-config.txt | 22 ++ evidence/check.txt | 6 + evidence/failover.txt | 56 +++ evidence/jobs-broker-down.txt | 17 + evidence/jobs-crash-ack.txt | 15 + evidence/jobs-crash-s3.txt | 17 + evidence/jobs-lease.txt | 19 + evidence/jobs-outbox.txt | 16 + evidence/jobs-redelivery.txt | 15 + evidence/jobs-registry.txt | 21 + evidence/jobs-retry-replay.txt | 22 ++ evidence/jobs-worker-down.txt | 26 ++ evidence/local/attachments.txt | 105 +++++ evidence/local/failover.txt | 3 + evidence/local/jobs.txt | 93 +++++ evidence/local/persistence.txt | 2 + evidence/local/registry.txt | 5 + evidence/local/smoke.txt | 1 + evidence/local/student-tests.txt | 59 +++ evidence/local/worker-limits.txt | 7 + evidence/no-local-file.txt | 19 + evidence/orphans.txt | 44 +++ evidence/persistence.txt | 31 ++ evidence/queue-drain.txt | 7 + evidence/s3-bootstrap.txt | 30 ++ evidence/s3-outage.txt | 22 ++ evidence/s3-private.txt | 17 + evidence/smoke.txt | 6 + evidence/stand.txt | 30 ++ evidence/student-tests.txt | 67 ++++ evidence/upload-memory-10.json | 391 ++++++++++++++++++ evidence/upload-memory-20.json | 586 +++++++++++++++++++++++++++ evidence/upload-memory.txt | 156 ++++++++ evidence/worker-limits.txt | 12 + infra/haproxy/haproxy.cfg | 84 ++++ infra/minio/bootstrap.sh | 56 +++ migrations/0001_init.sql | 54 +++ migrations/0002_idempotency.sql | 17 + migrations/0003_attachments.sql | 56 +++ migrations/0004_jobs.sql | 88 +++++ requirements.txt | 12 + scripts/check-s3-private.cmd | 4 + scripts/check-s3-private.sh | 7 + scripts/restart.cmd | 4 + scripts/restart.sh | 6 + scripts/restore-api.cmd | 4 + scripts/restore-api.sh | 5 + scripts/start-worker.cmd | 4 + scripts/start-worker.sh | 5 + scripts/stop-one.cmd | 7 + scripts/stop-one.sh | 12 + scripts/stop-worker.cmd | 4 + scripts/stop-worker.sh | 5 + student-tests/attachments.py | 262 ++++++++++++ student-tests/collect_evidence.py | 180 +++++++++ student-tests/common.py | 215 ++++++++++ student-tests/expiry.py | 49 +++ student-tests/explain.sql | 60 +++ student-tests/failover_each.py | 115 ++++++ student-tests/fixtures/README.md | 7 + student-tests/fixtures/sample.jpg | Bin 0 -> 19177 bytes student-tests/idempotency.py | 91 +++++ student-tests/jobs.py | 415 +++++++++++++++++++ student-tests/loadgen.py | 110 ++++++ student-tests/logout_cache.py | 68 ++++ student-tests/orphans.py | 127 ++++++ student-tests/outage.py | 96 +++++ student-tests/race.py | 46 +++ student-tests/rollback_check.py | 52 +++ student-tests/rolling_update.py | 133 +++++++ student-tests/run.py | 125 ++++++ student-tests/s3_outage.py | 85 ++++ student-tests/upload_memory.py | 185 +++++++++ 98 files changed, 9131 insertions(+), 38 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/__main__.py create mode 100644 app/broker.py create mode 100644 app/config.py create mode 100644 app/db.py create mode 100644 app/images.py create mode 100644 app/imaging.py create mode 100644 app/janitor.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/replay.py create mode 100644 app/repo.py create mode 100644 app/security.py create mode 100644 app/sessions.py create mode 100644 app/storage.py create mode 100644 app/tools.py create mode 100644 app/uploads.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 evidence/attachments.txt create mode 100644 evidence/broker-config.txt create mode 100644 evidence/check.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/jobs-broker-down.txt create mode 100644 evidence/jobs-crash-ack.txt create mode 100644 evidence/jobs-crash-s3.txt create mode 100644 evidence/jobs-lease.txt create mode 100644 evidence/jobs-outbox.txt create mode 100644 evidence/jobs-redelivery.txt create mode 100644 evidence/jobs-registry.txt create mode 100644 evidence/jobs-retry-replay.txt create mode 100644 evidence/jobs-worker-down.txt create mode 100644 evidence/local/attachments.txt create mode 100644 evidence/local/failover.txt create mode 100644 evidence/local/jobs.txt create mode 100644 evidence/local/persistence.txt create mode 100644 evidence/local/registry.txt create mode 100644 evidence/local/smoke.txt create mode 100644 evidence/local/student-tests.txt create mode 100644 evidence/local/worker-limits.txt create mode 100644 evidence/no-local-file.txt create mode 100644 evidence/orphans.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/queue-drain.txt create mode 100644 evidence/s3-bootstrap.txt create mode 100644 evidence/s3-outage.txt create mode 100644 evidence/s3-private.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stand.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/upload-memory-10.json create mode 100644 evidence/upload-memory-20.json create mode 100644 evidence/upload-memory.txt create mode 100644 evidence/worker-limits.txt create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 infra/minio/bootstrap.sh create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_jobs.sql create mode 100644 requirements.txt create mode 100644 scripts/check-s3-private.cmd create mode 100644 scripts/check-s3-private.sh create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/restore-api.cmd create mode 100644 scripts/restore-api.sh create mode 100644 scripts/start-worker.cmd create mode 100644 scripts/start-worker.sh create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/stop-worker.cmd create mode 100644 scripts/stop-worker.sh create mode 100644 student-tests/attachments.py create mode 100644 student-tests/collect_evidence.py create mode 100644 student-tests/common.py create mode 100644 student-tests/expiry.py create mode 100644 student-tests/explain.sql create mode 100644 student-tests/failover_each.py create mode 100644 student-tests/fixtures/README.md create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/idempotency.py create mode 100644 student-tests/jobs.py create mode 100644 student-tests/loadgen.py create mode 100644 student-tests/logout_cache.py create mode 100644 student-tests/orphans.py create mode 100644 student-tests/outage.py create mode 100644 student-tests/race.py create mode 100644 student-tests/rollback_check.py create mode 100644 student-tests/rolling_update.py create mode 100644 student-tests/run.py create mode 100644 student-tests/s3_outage.py create mode 100644 student-tests/upload_memory.py diff --git a/.dockerignore b/.dockerignore index bc16e556..75492fa1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,39 @@ +# В образ попадает только то, что нужно рантайму. Всё лишнее исключаем: +# уменьшает контекст сборки, ускоряет build и не даёт утечь .env/секретам. .git -.github +.gitignore +.dockerignore +Dockerfile +Dockerfile.example .env .env.* !.env.example -secrets -certs -volumes -data -backups +.venv/ +__pycache__/ +*.py[cod] +tests/ +student-tests/ +scripts/ +contracts/ +evidence/ +*.md +.github/ +lab.json +requirements-dev.txt +compose.yaml +compose.example.yaml +examples/ +Makefile +MAINTAINERS.md +infra/ +secrets/ +certs/ +volumes/ +data/ +backups/ *.key *.pem *.p12 *.pfx *.jks -.venv -__pycache__ -node_modules -evidence .DS_Store diff --git a/.env.example b/.env.example index 119384b5..24270191 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,66 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Скопируйте в .env для ЛОКАЛЬНОГО учебного стенда; .env никогда не коммитится. +# Пароли и ключи ниже — синтетические примеры, задайте свои. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: + +# PostgreSQL — источник истины (пользователи, чаты, сообщения, сессии, ключи идемпотентности, +# метаданные вложений). Compose собирает из них DATABASE_URL. Вне compose: DATABASE_URL или DATABASE_URL_FILE. +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-local-only + +# Сессии в PostgreSQL, Redis — общий кэш сессий для всех реплик. SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +SESSION_TTL_SECONDS=3600 +SESSION_CACHE_TTL_SECONDS=60 +# CACHE_URL (или CACHE_URL_FILE) задаёт compose: redis://redis:6379/0. + +# lab3: число реплик API за балансировщиком и срок хранения Idempotency-Key. API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 + +# lab4: MinIO. ROOT — только для сервисов s3 и s3-init (bootstrap), в API не передаётся. +MINIO_ROOT_USER=minio-root +MINIO_ROOT_PASSWORD=change-me-root-local-only +# Отдельный пользователь API с правами только на свой bucket (создаёт s3-init). +# Вне compose можно передать файлами: S3_ACCESS_KEY_ID_FILE / S3_SECRET_ACCESS_KEY_FILE. S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=change-me-api-local-only +S3_CONNECT_TIMEOUT=2 +S3_READ_TIMEOUT=10 + +# Лимиты загрузки (контракт): 10 MiB и 40 млн пикселей. MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url +UPLOAD_BODY_TIMEOUT_SECONDS=60 +UPLOAD_SPOOL_MEMORY_BYTES=262144 +UPLOAD_MAX_CONCURRENT=16 +IMAGE_CHECK_CONCURRENCY=2 +# Очистка сирот: журнал загрузки старше grace period считается брошенным. +UPLOAD_GRACE_SECONDS=900 +JANITOR_INTERVAL_SECONDS=60 + +# Пул соединений и таймауты +DB_POOL_MAX=10 +DB_POOL_TIMEOUT=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_TIMEOUT=0.5 +SHUTDOWN_TIMEOUT=10 + +LOG_LEVEL=info + +# lab5: брокер и worker. BROKER_URL задаёт compose (redis://broker:6379/0 — отдельный Redis с AOF). +WORKER_REPLICAS=1 WORKER_CONCURRENCY=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=20 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=60 +CLAIM_IDLE_SECONDS=90 +IMAGE_MEMORY_MB=768 STAGING_RETENTION_SECONDS=86400 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..4723f5db --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# Скрипты, которые выполняются ВНУТРИ Linux-контейнеров, обязаны иметь LF, +# даже если репозиторий клонирован на Windows с core.autocrlf=true. +*.sh text eol=lf +*.py text eol=lf +*.sql text eol=lf +*.cfg text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf +*.cmd text eol=crlf +*.png binary +*.jpg binary diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..0237de68 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +# syntax=docker/dockerfile:1 + +# ---- Stage 1: builder: зависимости ставим отдельно, в рантайм несём только результат +FROM python:3.12.7-slim-bookworm AS builder +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 +WORKDIR /app +COPY requirements.txt . +RUN pip install --prefix=/install -r requirements.txt + +# ---- Stage 2: runtime +FROM python:3.12.7-slim-bookworm AS runtime +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 APP_PORT=8080 SHUTDOWN_TIMEOUT=10 +RUN useradd --create-home --uid 10001 appuser +WORKDIR /app +COPY --from=builder /install /usr/local +COPY app ./app +COPY migrations ./migrations +USER appuser +EXPOSE 8080 +# /health/ready проверяет PostgreSQL: без БД контейнер станет unhealthy. +HEALTHCHECK --interval=10s --timeout=3s --start-period=10s --retries=3 \ + CMD python -c "import os,urllib.request,sys; \ +u='http://127.0.0.1:'+os.getenv('APP_PORT','8080')+'/health/ready'; \ +sys.exit(0 if urllib.request.urlopen(u,timeout=2).status==200 else 1)" || exit 1 +# exec-форма: uvicorn — PID 1 и сам получает SIGTERM от docker stop. +CMD ["python", "-m", "app"] diff --git a/REPORT.md b/REPORT.md index e98f75e9..76d1ce4d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,184 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Санжаровский, БВТ2304 +- Целевая оценка: **4** (трек 5 не заявлен) +- Commit/PR: ветка `solution/bvt2304-sanzharovskiy` форка `Anatoly01/mtusi2026lab5`, PR в `mtusi2026/mtusi2026lab5` +- Источники и библиотеки: всё из lab4 (FastAPI, uvicorn, psycopg 3 + psycopg-pool, argon2-cffi, redis-py 8.1, boto3 1.38, Pillow 11.2, HAProxy 3.0, MinIO от Pigsty). Брокер — **Redis Streams** (тот же образ `redis:7.4.1-alpine`, отдельный экземпляр с AOF). Документация: Redis Streams (consumer groups, PEL, `XAUTOCLAIM`), Redis persistence (AOF, `appendfsync always`), RabbitMQ reliability/confirms (для сравнения границ confirm/ack), Pillow (`draft`, `ImageOps.exif_transpose`, WebP), Python `resource` (RLIMIT_AS/RLIMIT_CPU), `multiprocessing` forkserver; Transactional outbox (microservices.io). Помощь ИИ: Claude — код, тесты, отчёт; решения разобраны и проверены прогоном. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужно: Docker Desktop (Compose v2), Python 3.9+ для тестов (только stdlib). Образы закреплены: `python:3.12.7-slim-bookworm`, `postgres:16.4-alpine`, `redis:7.4.1-alpine` (кэш сессий **и** брокер), `haproxy:3.0.5-alpine`, `pgsty/minio:RELEASE.2026-08-04T00-00-00Z`, `pgsty/mc:RELEASE.2026-09-16T00-00-00Z`. Новых образов по сравнению с lab4 нет. + +```sh +cp .env.example .env # Windows: copy .env.example .env — задать свои пароли/ключи +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# порядок: postgres(healthy) → migrate (exit 0, миграции 0001–0004) +# s3(healthy) → s3-init (bootstrap, exit 0); broker(healthy) +# → api ×2 → worker → lb +curl -i http://localhost:8080/health/ready # 200 + X-Instance-Id +make check && make test # публичные проверки (Windows: set PYTHONUTF8=1) +python student-tests/collect_evidence.py # ВСЕ проверки → evidence/*.txt (+ сводка) +``` + +Демонстрация уровня 3 вручную: `scripts/stop-worker.(sh|cmd)` → upload (202 queued, content 409) → `docker compose restart api broker` → `scripts/start-worker.(sh|cmd)` → ready. Всё это автоматически делает `python student-tests/jobs.py --scenario worker-down`. + +Отдельные проверки: + +```sh +python student-tests/jobs.py --base-url http://localhost:8080 # все 8 сценариев отказов lab5 +python student-tests/jobs.py --scenario crash-s3 # один сценарий +docker compose --env-file .env exec -T api python -m app.tools limits-selftest # лимиты worker +docker compose --env-file .env exec -T api python -m app.tools queue # stream, pending, DLQ, outbox +docker compose --env-file .env exec -T api python -m app.tools job +docker compose --env-file .env exec -T api python -m app.replay --list # реестр failed +docker compose --env-file .env exec -T api python -m app.replay --attachment # явный replay +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh && scripts/restore-api.sh +``` + +Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных (БД, bucket **и** очередь): `docker compose --env-file .env down -v`. Имя проекта `mtusi-lab5`: volumes lab4 не затрагиваются. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент ─▶ lb (HAProxy, 127.0.0.1:8080) ─▶ api ×2 ──① PUT staging/ ──────────────▶ s3 (MinIO, private) + │ ② BEGIN; attachments(queued) + ▲ staging/ + │ attachment_jobs + job_outbox; COMMIT │ originals/ + │ ③ XADD → broker (ответ = «confirm») │ variants//v1/* + │ ④ 202 queued │ + ▼ │ + postgres ◀──────── ⑦ COMMIT ready ──── worker ───────┘ ⑥ copy + PUT + (истина: вложения, ⑤ XREADGROUP ◀── broker (Redis Streams, AOF always, + jobs, outbox, volume brokerdata) + variants, events) ⑧ XACK ─▶ broker ; ⑨ DELETE staging/ +``` + +| Роль | Service | Версия | Health/readiness | Volume | Порт | +| --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | `haproxy:3.0.5-alpine` | `/healthz`; backend `/health/ready` | — | `127.0.0.1:8080` (+8081 ttl, +8082 fault — тестовые профили) | +| api ×2 | `api` | `Dockerfile` | `/health/ready` (PostgreSQL) | нет | `expose 8080` | +| **worker** | `worker` | тот же образ, `python -m app.worker` | heartbeat-файл обновляется раз в 1 c | нет | нет | +| **broker** | `broker` | `redis:7.4.1-alpine` + `--appendonly yes --appendfsync always --maxmemory-policy noeviction` | `redis-cli ping` | `brokerdata` | нет | +| postgres, redis, s3, s3-init, migrate | как в lab4 | | | `pgdata`, `s3data` | нет | +| тестовые | `api-fault`, `worker-fault` (профиль `fault`), `api-ttl` (профиль `ttl`) | тот же образ | | нет | только через lb | + +**Что перенесено из lab4** без изменений: LB + 2 реплики API без состояния, сессии (PostgreSQL + Redis-кэш), идемпотентность сообщений, проверка изображения по содержимому в API (413/415 синхронно, как в lab4), приватный bucket и отдельный пользователь MinIO, журнал `upload_intents` + janitor, лимиты, хуки. **Переход** по контракту: upload `201 ready` → `202 queued`. + +**Что добавлено в lab5:** + +- **Брокер — Redis Streams на отдельном экземпляре Redis с persistence.** Обычный Pub/Sub не хранит сообщения; Stream хранит, а consumer group даёт at-least-once: выданное worker сообщение лежит в PEL (pending entries list), пока не будет `XACK`. `appendfsync always` — ответ на `XADD` отправляется только после fsync журнала AOF, это наш аналог publisher confirm. `noeviction` — сообщения никогда не вытесняются. Кэш сессий — другой экземпляр Redis без persistence (lab2). +- **Сообщение** (`app/broker.py`, строго по `contracts/jobs.schema.json`): `schema_version, job_id, type=attachment.process.v1, attachment_id, pipeline_version=1, attempt, created_at`. Ни байтов, ни URL, ни ключей bucket, ни credentials. Worker берёт ключи объектов и sha256 только из БД. Сообщение, не прошедшее схему или с неизвестным job, уходит в DLQ-stream `attachments:jobs:dlq`. +- **Модель** (`migrations/0004_jobs.sql`, только добавления — lab4 с новой схемой работает): `attachments.staging_key`; `attachment_jobs` (job_id, attempt, max_attempts, `lease_token/lease_owner/lease_until`, error_code, replays, `UNIQUE(attachment_id, pipeline_version)`); `job_outbox` (payload, `available_at`, `published_at`); `attachment_variants` с **ключом эффекта** `PRIMARY KEY (attachment_id, pipeline_version, variant)`; `job_events` — журнал (реестр для оператора и audit replay). +- **Варианты** (`app/imaging.py`): **thumbnail** — JPEG q85, вписан в 256×256, без увеличения, округление вниз, минимум 1 px (320×200 → 256×160); **crop** — JPEG q85, центрированный квадрат 128×128 (маленькие увеличиваются); **optimized** — WebP q80, `method=4`, вписан в 1280×1280 без увеличения, альфа сохраняется. EXIF orientation применяется до расчёта размеров; EXIF/XMP/ICC в результат не пишутся (проверено: у всех трёх вариантов `exif=False, icc=False`). Прозрачность в JPEG компонуется на белый. JPEG декодируется сразу в уменьшенном масштабе (`draft`), не меньше нужного самому большому варианту. +- **Ключи S3 детерминированы**: `originals/` (серверный `CopyObject` из `staging/`), `variants//v1/.`. Повтор job перезаписывает те же объекты теми же байтами; в БД — `ON CONFLICT DO NOTHING`. Второго набора вариантов не бывает. +- **Лимиты worker** (`app/worker.py` → дочерний процесс `forkserver`): байты — не больше `MAX_UPLOAD_BYTES` при скачивании из staging; пиксели — по заголовку до распаковки (`MAX_IMAGE_PIXELS`); память — `RLIMIT_AS = IMAGE_MEMORY_MB` (768 МиБ) у ребёнка, нехватка памяти даёт MemoryError в ребёнке, а не OOM всего worker; время — `JOB_TIMEOUT_SECONDS` (60 c) wall-clock, по истечении ребёнок убивается, плюс `RLIMIT_CPU`. Конкуренция — `WORKER_CONCURRENCY`=2 потока, каждый берёт по одному сообщению (`COUNT 1`, аналог prefetch=1). Контейнер worker: 1 CPU / 1 ГиБ. + +### Состояния и транзакционные границы + +| Переход | Где | Что устойчиво сохранено | +| --- | --- | --- | +| нет → queued | API: журнал → PUT `staging/` → **одна транзакция**: снять журнал, `attachments(queued)`, `attachment_jobs(queued, attempt 1)`, `job_outbox` → COMMIT → `XADD` → **202** | полный проверенный оригинал в staging, метаданные, job и сообщение в outbox | +| queued → processing | worker, `SELECT … FOR UPDATE`: новый `lease_token`, `lease_until = now()+20 c` | попытка, аренда, владелец | +| processing → queued | временная ошибка, попытки есть: `attempt+1`, строка outbox с `available_at = now()+backoff` | причина (`last_error`), время следующей попытки | +| processing → ready | worker: S3 (copy + 3 PUT) → **одна транзакция**: варианты, attachment и job → ready (только если `lease_token` всё ещё наш) → `XACK` → удалить staging | оригинал, три варианта, их sha256/размеры | +| processing → failed | постоянная ошибка или `attempt = max_attempts` | `error_code` (`invalid_image`, `processing_failed`, `retry_exhausted`), событие, копия в DLQ-stream | +| processing → queued (lease) | reaper: аренда истекла ещё на одну аренду назад | новая попытка через outbox | +| failed → queued | `python -m app.replay` (только failed, только если staging ещё есть) | событие `replay` (кто, когда), тот же job_id, `attempt+1`, бюджет ещё 5 попыток | + +Ready никогда не откатывается: повторная доставка ready-задачи — no-op с ack. + +### Окна отказа + +| Окно | Что происходит | Кто восстанавливает | Проверка | +| --- | --- | --- | --- | +| после staging, до записи job | объект в staging + строка `upload_intents` | API компенсирует сразу; иначе janitor после grace (как lab4) | `orphans.py` | +| после COMMIT, до publish | job и outbox в БД, сообщения в брокере нет | **outbox relay** (поток maintenance worker, раз в 1 c, строки старше 2 c, `FOR UPDATE SKIP LOCKED`) | `jobs.py outbox` | +| брокер недоступен при upload | `XADD` не прошёл за 2 c — 202 всё равно отдаётся | relay, когда брокер вернётся | `jobs.py broker-down` | +| после publish, до confirm | брокер мог принять сообщение, а API не узнал | relay опубликует ещё раз → дубль → no-op | `jobs.py redelivery` | +| после записи S3, до COMMIT результата | job в processing с арендой умершего worker, сообщение в PEL | перезапущенный worker перечитывает свой PEL; после истечения аренды — **takeover той же попытки**; объекты S3 те же | `jobs.py crash-s3` | +| после COMMIT результата, до ack | job ready, сообщение в PEL | redelivery → `already_ready` → ack | `jobs.py crash-ack` | +| worker завис (аренда не продлевается) | processing, lease истекает | reaper → attempt+1 → другой worker; проснувшийся не может зафиксировать (fencing по `lease_token`) | `jobs.py lease` | + +### Retry, DLQ, replay + +- **Постоянные** ошибки (без повторов): изображение не декодируется / превышает пиксели / не хватило памяти → `invalid_image`; sha256 staging не совпал с БД или исходника нет → `processing_failed`. +- **Временные**: S3 или БД недоступны, таймаут обработки, падение ребёнка. Повтор через outbox с **экспоненциальным backoff и jitter** («equal jitter»: `d = min(60, base·2^(n−1))`, задержка ∈ [d/2, d]). `JOB_MAX_ATTEMPTS`=5, затем `failed retry_exhausted`. Бесконечного немого retry нет. +- **Реестр failed** — таблица `attachment_jobs` (status=failed, error_code, last_error) + `job_events`; копия для оператора — DLQ-stream. `python -m app.replay --list` / `--dlq`. +- **Replay** (`app/replay.py`) безопасен: меняет только failed (`FOR UPDATE`), ready/queued/processing отклоняет — успешную задачу второй раз не исполнить. job_id тот же, attempt растёт, событие с actor. Если исходник уже удалён retention — отказ. + +### Аренда: долгая или зависшая задача + +Живой worker продлевает аренду каждые `lease/3` ≈ 6,7 c (отдельный поток heartbeat), пока ребёнок обрабатывает картинку. Поэтому **долгая** задача (в тесте 30 c при аренде 20 c) не перехватывается: аренда всё время свежая. Жёсткий предел — `JOB_TIMEOUT_SECONDS`. У **мёртвого или замёрзшего** worker аренда истекает. Тогда: + +1. сообщение из его PEL заберёт он сам после рестарта или другой worker (`XAUTOCLAIM`, 90 c); обработку получит тот, кто первым увидит истёкшую аренду; +2. если за ещё одну аренду никто не перехватил, reaper переводит job в queued с новой попыткой. + +Два неконтролируемых дубля невозможны: захват — `SELECT … FOR UPDATE`, фиксация результата сверяет `lease_token`. Старый worker после пробуждения получает `LeaseLost` и ничего не пишет в БД, а его объекты S3 совпадают байт в байт. + +### Очистка staging + +- ready: worker удаляет `staging/` **после** COMMIT ready и ack; если не успел, подчищает maintenance через 30 c; +- failed: исходник хранится `STAGING_RETENTION_SECONDS` (24 ч) для replay, потом удаляется; +- брошенные загрузки (нет job): janitor по `upload_intents`, обход bucket проверяет и `originals/`, и `staging/`. + +Текущую работу cleanup не трогает: удаляются только staging у ready и у failed старше retention. + +**Соответствие `.env.example`:** `WORKER_CONCURRENCY`, `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS`, `STAGING_RETENTION_SECONDS`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` используются как есть; добавлены `WORKER_REPLICAS`, `JOB_LEASE_SECONDS`, `JOB_BACKOFF_BASE/MAX_SECONDS`, `CLAIM_IDLE_SECONDS`, `IMAGE_MEMORY_MB`. `BROKER_URL` задаёт compose (`redis://broker:6379/0`), поддерживается и `BROKER_URL_FILE`. `SESSION_STORE` — сессии всегда в PostgreSQL. + +**Readiness:** `/health/ready` проверяет только PostgreSQL. Брокер не критичен для приёма — намерение лежит в outbox (проверено: брокер выключен → `/health/ready` 200, upload 202). S3 критичен только для вложений (как в lab4). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Основной прогон — **Docker на ПК** (Windows, Docker Desktop, стек из `compose.yaml` как есть): `python student-tests/collect_evidence.py`, все шаги PASS, выводы в `evidence/<шаг>.txt`. До этого код отлажен на локальном стенде без Docker (PostgreSQL 16, два процесса API за HAProxy, Redis-брокер с AOF, worker, S3-эмулятор moto) — `evidence/local/*.txt`, там те же результаты. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат (Docker) | Файл | +| --- | --- | --- | --- | +| Публичный smoke | `make test` | `PASS: lab5 public API scenario`: 202 → ready, thumbnail 256×160 JPEG, sha256 | `evidence/smoke.txt` | +| Брокер с persistence | `redis-cli CONFIG GET` | `appendonly yes`, `appendfsync always`, `maxmemory-policy noeviction` | `evidence/broker-config.txt` | +| **3:** 202 только после устойчивого приёма; брокер и worker отдельно; API не делает resize | `jobs.py worker-down` | worker остановлен: 202 queued за 0,06 c; через 3 c всё ещё queued; content и thumbnail → 409; сообщение с queued-вложением → 201; задание лежит в stream (len=1) | `evidence/jobs-worker-down.txt` | +| **3:** рестарт API и брокера с volumes → worker → ready | `jobs.py worker-down` | после `restart broker` + `restart api` сообщение на месте (len=1), тот же токен; worker запущен → ready за 2,5 c, оригинал побайтно, thumbnail 256×160, staging удалён | `evidence/jobs-worker-down.txt` | +| **3:** at-least-once, ack после эффекта, повтор не создаёт второй набор | `jobs.py redelivery`, `crash-ack` | 2 дубля ready-задачи → no-op: 3 строки вариантов, 3 объекта S3, `ready` записан один раз, pending 0; crash после COMMIT → redelivery → no-op | `evidence/jobs-redelivery.txt`, `jobs-crash-ack.txt` | +| **3:** ограниченный retry, финальный failed | `jobs.py retry-replay` | 5 попыток → `failed retry_exhausted` за 22 c; content → 409; прикрепить к сообщению → 409 | `evidence/jobs-retry-replay.txt` | +| **4:** crop 128×128 и WebP | smoke, `jobs.py` | у каждого ready-вложения варианты `thumbnail`, `crop`, `optimized` | `evidence/jobs-*.txt` | +| **4:** лимиты байт/пикселей/времени/памяти worker | `app.tools limits-selftest` | 5/5: 36 Мп при лимите 10 Мп → `invalid_image: pixel limit`; RLIMIT_AS 120 МиБ → `memory limit`; битый PNG → `invalid_image`; зависание 10 c убито через 2 c | `evidence/worker-limits.txt` | +| **4:** outbox: crash между COMMIT и publish | `jobs.py outbox` (api-fault) | клиент получил 502; вложение и job закоммичены; relay опубликовал строку outbox → ready без нового upload | `evidence/jobs-outbox.txt` | +| **4:** брокер недоступен при приёме | `jobs.py broker-down` | `/health/ready` 200; upload 202 за 2,1 c (таймаут publish 2 c); строка outbox не опубликована → брокер вернулся → relay → ready | `evidence/jobs-broker-down.txt` | +| **4:** backoff с jitter, постоянные/временные, DLQ/реестр, безопасный replay | `jobs.py retry-replay`, `app.replay` | задержки 0,7 / 1,1 / 3,6 / 6,3 c (база 1 c: d = 1, 2, 4, 8, каждая в [d/2, d]); задача в `replay --list`, копия в DLQ с причиной `retry_exhausted (injected_transient)`; replay → ready, тот же job_id, attempt 6, replays 1; повторный replay ready-задачи → `refused` | `evidence/jobs-retry-replay.txt`, `jobs-registry.txt` | +| **4:** worker остановлен после записи объекта, до ack | `jobs.py crash-s3` | `os._exit(1)` после записи S3; контейнер перезапустился, `lease_takeover` той же попытки (attempt 1) → ready за 26,9 c; по-прежнему ровно 3 варианта | `evidence/jobs-crash-s3.txt` | +| **4:** lease/timeout зависшего processing | `jobs.py lease` | долгая задача 32 c при аренде 20 c — без перехвата (heartbeat); замёрзший worker → reaper → attempt 2 → ready за 37,3 c; проснувшийся старый worker не смог зафиксировать (fencing), результат не изменился | `evidence/jobs-lease.txt` | +| **4:** ограниченная конкуренция, очередь под нагрузкой, очистка staging | `upload-memory` + `queue-drain`; `worker-down` | 90 загрузок по 9,6 МиБ → 90 × 202; worker (2 потока, 1 CPU) разобрал очередь за 103 c после окончания загрузок; staging удаляется после ready | `evidence/upload-memory.txt`, `queue-drain.txt` | +| Возможности lab2–4 | `run.py`, `idempotency.py`, `attachments.py`, `orphans.py`, `s3_outage.py`, `upload_memory.py`, persistence, failover ×2 | 30/30, 16/16, 91/91, orphans PASS, S3 outage 12/12, память процессов API +25–29 МиБ при 10–20 параллельных загрузках по 9,6 МиБ; persistence PASS; failover PASS дважды (0 неудачных попыток, восстановление 1,5 и 2,0 c) | `evidence/*.txt` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдали (Docker) | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| worker остановлен, upload | 202 queued, обработки нет | 202 за 0,06 c, 3 c queued | работа ждёт в stream; 0 потерь | +| рестарт брокера (AOF) с необработанным заданием | сообщение сохранено | len=1 до и после | ready через 2,5 c после старта worker | +| брокер остановлен при upload | 202 без брокера | 202 за 2,1 c, outbox не опубликован | relay опубликовал после старта брокера, сценарий 14 c | +| api убит между COMMIT и publish | задача не теряется | 502 клиенту, job в БД | ready без нового upload (сценарий 20 c) | +| worker убит после S3, до COMMIT | та же попытка доходит до ready | `lease_takeover`, attempt 1 | 26,9 c (рестарт + истечение аренды 20 c) | +| worker убит после COMMIT, до ack | дубль = no-op | redelivery → no-op, pending 0 | 0 c для пользователя | +| worker «завис» | после истечения аренды задачу берёт другой | reaper → attempt 2 → ready | 37,3 c (аренда 20 c + ещё одна) | +| 5 временных ошибок подряд | failed `retry_exhausted` | 22 c, задержки растут | replay → ready | +| 90 больших загрузок | приём не ждёт обработку | 90 × 202, p50 4,4–7,2 c (сама загрузка 9,6 МиБ) | очередь разобрана за 103 c | + +Время восстановления после смерти worker определяется арендой: `JOB_LEASE_SECONDS`=20 c. Короче — быстрее восстановление, но выше риск перехватить живую задачу при паузе GC/диска. Heartbeat каждые 6,7 c даёт запас в 3 продления. + +**Порядок шагов в `collect_evidence.py`.** Первый полный прогон уронил 4 унаследованных теста lab4: они скачивали файл сразу после ответа (теперь это 202, а не 201 ready), а persistence/failover стояли в очереди за 90 большими картинками от `upload-memory`. Тесты исправлены: ждут ready. Добавлен шаг `queue-drain`: ждёт пустой очереди перед persistence. В `upload_memory.py` критерий памяти теперь — память процессов (`anon`), без файлового кэша ядра. `usage` по-прежнему печатается: +68 МиБ, тоже меньше 100 МиБ. + +## Ответы на вопросы защиты + +- **Publisher confirm vs consumer ack.** Confirm — граница «брокер принял и сохранил»: у нас ответ `XADD` после fsync AOF. Ack (`XACK`) — граница «потребитель устойчиво выполнил работу»: у нас после COMMIT ready в PostgreSQL. Между ними сообщение лежит в PEL. +- **Что сохранено в момент 202.** Полный проверенный оригинал в приватном `staging/` (S3), строка `attachments` (queued), `attachment_jobs` и `job_outbox` — в одной транзакции. Сообщение в брокере — почти всегда, но не обязательно: если нет, его опубликует relay. +- **Почему повторная доставка нормальна и где ключ идемпотентности.** At-least-once: сообщение повторяется, если ack не дошёл. Ключ эффекта — `(attachment_id, pipeline_version, variant)`: PK `attachment_variants` + детерминированные ключи S3; логическая работа — `job_id` (UNIQUE по `attachment_id, pipeline_version`); устаревшие попытки отсекает `attempt` в сообщении. +- **Зависший или долгий.** Долгий продлевает аренду heartbeat-ом; у зависшего она истекает. Захват — `FOR UPDATE`, фиксация — только с актуальным `lease_token` (fencing), поэтому два неконтролируемых исполнителя невозможны. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Задержка обработки под нагрузкой ограничена одним worker (1 CPU, 2 потока): ~0,9 больших картинки в секунду. Масштабирование — `WORKER_REPLICAS`; в этой работе не измерялось. +- Брокер в одном экземпляре: отказ узла с брокером останавливает доставку (но не приём — outbox), реплики Redis/Sentinel не настроены. +- `appendfsync always` дороже `everysec`; для учебной нагрузки это не узкое место, при большой нагрузке нужна оценка. +- Relay и reaper работают в worker: если все worker выключены, отложенные retry ждут их запуска (новые задачи при этом публикует API). +- Exactly-once нет и не обещается: at-least-once + идемпотентный эффект. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..31667101 --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,22 @@ +""" +Точка входа контейнера: `python -m app`. Настройки — из окружения (app/config.py). +uvicorn ловит SIGTERM: перестаёт принимать запросы и дожидается текущих +(до SHUTDOWN_TIMEOUT), затем закрывает пул БД и соединение с Redis. +""" +import logging + +import uvicorn + +from .config import settings + +if __name__ == "__main__": + logging.basicConfig(level=settings.log_level.upper(), + format="%(asctime)s %(levelname)s %(name)s %(message)s") + uvicorn.run( + "app.main:app", + host=settings.host, + port=settings.port, + timeout_graceful_shutdown=settings.shutdown_timeout, + access_log=False, # свой лог в middleware: без query, токенов и тел + log_level=settings.log_level, + ) diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 00000000..7cd4630f --- /dev/null +++ b/app/broker.py @@ -0,0 +1,140 @@ +""" +Брокер заданий (lab5): Redis Streams на ОТДЕЛЬНОМ экземпляре Redis с AOF. + +Почему это durable-брокер, а не Pub/Sub: +- XADD записывает сообщение в stream; Redis запущен с `appendonly yes` и + `appendfsync always`: ответ на XADD отправляется только после fsync журнала AOF. + Ответ XADD (ID сообщения) — наш аналог publisher confirm: брокер принял и сохранил. +- Consumer group: XREADGROUP выдаёт сообщение одному worker и кладёт его в PEL + (pending entries list). Пока worker не сделал XACK, сообщение остаётся в PEL + и переживает рестарт брокера и смерть worker. XACK — consumer ack: worker + устойчиво зафиксировал результат. Это разные границы. +- Брошенные сообщения (worker умер до XACK) забирает другой worker через XAUTOCLAIM. +- `maxmemory-policy noeviction`: Redis никогда не вытеснит сообщения из памяти. + +В сообщении только ID и служебные поля (contracts/jobs.schema.json): +ни байтов, ни URL, ни ключей bucket, ни credentials. +""" +from __future__ import annotations + +import json +import uuid +from datetime import datetime, timezone + +import redis +import redis.asyncio as aioredis + +from .config import settings + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 +_REQUIRED = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", + "attempt", "created_at"} +_OPTIONAL = {"traceparent", "tracestate"} + + +class BrokerUnavailable(Exception): + pass + + +class BadMessage(ValueError): + """Сообщение не соответствует jobs.schema.json — в DLQ, без обработки.""" + + +def iso_z(ts: datetime) -> str: + return ts.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def make_payload(job_id, attachment_id, pipeline_version: int, attempt: int, + created_at: datetime) -> dict: + return {"schema_version": SCHEMA_VERSION, "job_id": str(job_id), "type": JOB_TYPE, + "attachment_id": str(attachment_id), "pipeline_version": int(pipeline_version), + "attempt": int(attempt), "created_at": iso_z(created_at)} + + +def parse(raw) -> dict: + """Строгая проверка по jobs.schema.json (additionalProperties: false).""" + try: + data = json.loads(raw) + except (TypeError, ValueError) as exc: + raise BadMessage("not json") from exc + if not isinstance(data, dict): + raise BadMessage("not an object") + keys = set(data) + if not _REQUIRED <= keys or keys - _REQUIRED - _OPTIONAL: + raise BadMessage("unexpected fields") + if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE: + raise BadMessage("unknown schema/type") + for k in ("job_id", "attachment_id"): + try: + uuid.UUID(str(data[k])) + except ValueError as exc: + raise BadMessage(f"bad {k}") from exc + for k in ("pipeline_version", "attempt"): + if not isinstance(data[k], int) or isinstance(data[k], bool) or data[k] < 1: + raise BadMessage(f"bad {k}") + if not isinstance(data["created_at"], str) or not data["created_at"].endswith("Z"): + raise BadMessage("bad created_at") + return data + + +def dumps(payload: dict) -> str: + return json.dumps(payload, separators=(",", ":"), sort_keys=True) + + +# ----------------------------------------------------------- sync (worker, CLI) +def sync_client(read_timeout: float | None = None) -> redis.Redis: + """read_timeout у worker больше BLOCK в XREADGROUP (2 с), иначе блокирующее + чтение пустой очереди выглядело бы как обрыв связи.""" + return redis.Redis.from_url(settings.broker_url, + socket_timeout=read_timeout or settings.broker_timeout, + socket_connect_timeout=settings.broker_timeout, + decode_responses=True) + + +def ensure_group(r: redis.Redis) -> None: + """Группа создаётся с ID 0: сообщения, опубликованные ДО первого запуска worker, + тоже будут прочитаны (с '$' они бы потерялись).""" + try: + r.xgroup_create(settings.jobs_stream, settings.jobs_group, id="0", mkstream=True) + except redis.ResponseError as exc: + if "BUSYGROUP" not in str(exc): + raise + + +def publish_sync(r: redis.Redis, payload: dict) -> str: + """Возвращает ID сообщения — подтверждение приёма брокером (после fsync AOF).""" + try: + return r.xadd(settings.jobs_stream, {"job": dumps(payload)}) + except redis.RedisError as exc: + raise BrokerUnavailable(type(exc).__name__) from exc + + +def to_dlq(r: redis.Redis, raw: str, reason: str, source_id: str = "") -> None: + r.xadd(settings.dlq_stream, {"job": raw if isinstance(raw, str) else "", "reason": reason[:200], + "source_id": source_id, "at": iso_z(datetime.now(timezone.utc))}) + + +# ----------------------------------------------------------- async (API) +_aclient: aioredis.Redis | None = None + + +def init_async() -> None: + global _aclient + _aclient = aioredis.Redis.from_url(settings.broker_url, socket_timeout=settings.broker_timeout, + socket_connect_timeout=settings.broker_timeout, + decode_responses=True) + + +async def close_async() -> None: + if _aclient is not None: + await _aclient.aclose() + + +async def publish(payload: dict) -> str: + if _aclient is None: + init_async() + try: + return await _aclient.xadd(settings.jobs_stream, {"job": dumps(payload)}) + except (redis.RedisError, OSError) as exc: + raise BrokerUnavailable(type(exc).__name__) from exc diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..7796b76e --- /dev/null +++ b/app/config.py @@ -0,0 +1,129 @@ +""" +Конфигурация только из переменных окружения (ничего не зашито в код и образ). +""" +from __future__ import annotations +import os +import socket +from dataclasses import dataclass + + +def _int(name: str, default: int) -> int: + return int(os.getenv(name, str(default))) + + +def _float(name: str, default: float) -> float: + return float(os.getenv(name, str(default))) + + +def _secret(name: str, default: str = "") -> str: + """Значение из NAME_FILE (Docker secret, файл) или из переменной NAME. + Файл приоритетнее: так credentials не видны в `docker inspect`.""" + path = os.getenv(name + "_FILE") + if path: + with open(path, encoding="utf-8") as f: + return f.read().strip() + return os.getenv(name, default) + + +@dataclass(frozen=True) +class Settings: + # HTTP + host: str = os.getenv("APP_HOST", "0.0.0.0") + port: int = _int("APP_PORT", 8080) + shutdown_timeout: int = _int("SHUTDOWN_TIMEOUT", 10) + log_level: str = os.getenv("LOG_LEVEL", "info") + # Имя реплики для X-Instance-Id: INSTANCE_ID или hostname контейнера + # (в Docker это короткий ID контейнера — по нему stop-one.sh находит реплику). + instance_id: str = os.getenv("INSTANCE_ID") or socket.gethostname() + + # PostgreSQL — источник истины + database_url: str = _secret( + "DATABASE_URL", "postgresql://messenger:devpass@127.0.0.1:5432/messenger") + db_pool_min: int = _int("DB_POOL_MIN", 1) + db_pool_max: int = _int("DB_POOL_MAX", 10) # ограниченный пул + db_pool_timeout: float = _float("DB_POOL_TIMEOUT", 3.0) # ждать свободное соединение + db_connect_timeout: int = _int("DB_CONNECT_TIMEOUT", 3) + db_statement_timeout_ms: int = _int("DB_STATEMENT_TIMEOUT_MS", 5000) + migrate_on_start: bool = os.getenv("MIGRATE_ON_START", "true").lower() == "true" + + # Одновременных вычислений Argon2id на реплику (каждое ≈ 64 МиБ RAM). + password_hash_concurrency: int = _int("PASSWORD_HASH_CONCURRENCY", 2) + + # Сессии + session_ttl_seconds: int = _int("SESSION_TTL_SECONDS", 3600) + + # Идемпотентность POST сообщений: сколько хранить ключ (контракт: 24 часа) + idempotency_ttl_seconds: int = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + + # Redis/Valkey — кэш сессий (не источник истины). Пусто = кэш выключен. + # Имя из skeleton — CACHE_URL(_FILE); REDIS_URL оставлено как синоним. + redis_url: str = _secret("CACHE_URL") or _secret("REDIS_URL") + redis_timeout: float = _float("REDIS_TIMEOUT", 0.5) + session_cache_ttl_seconds: int = _int("SESSION_CACHE_TTL_SECONDS", 60) + + # lab4: S3-совместимое хранилище (MinIO) — только байты вложений. + # API ходит отдельным пользователем с правами только на свой bucket + # (не root MinIO): S3_ACCESS_KEY_ID(_FILE) / S3_SECRET_ACCESS_KEY(_FILE). + s3_endpoint: str = os.getenv("S3_ENDPOINT", "http://s3:9000") + s3_region: str = os.getenv("S3_REGION", "us-east-1") + s3_bucket: str = os.getenv("S3_BUCKET", "messenger-attachments") + s3_access_key_id: str = _secret("S3_ACCESS_KEY_ID") + s3_secret_access_key: str = _secret("S3_SECRET_ACCESS_KEY") + s3_connect_timeout: float = _float("S3_CONNECT_TIMEOUT", 2.0) + s3_read_timeout: float = _float("S3_READ_TIMEOUT", 10.0) + s3_max_attempts: int = _int("S3_MAX_ATTEMPTS", 2) + # Сколько секунд после ошибки соединения с S3 сразу отвечать 503 (circuit breaker). + s3_breaker_seconds: float = _float("S3_BREAKER_SECONDS", 5.0) + + # Лимиты загрузки (контракт: 10 MiB и 40 млн пикселей). + max_upload_bytes: int = _int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) + max_image_pixels: int = _int("MAX_IMAGE_PIXELS", 40_000_000) + # Сколько ждать тело запроса целиком (защита от «медленной» загрузки). + upload_body_timeout_seconds: float = _float("UPLOAD_BODY_TIMEOUT_SECONDS", 60.0) + # Сколько байт тела держим в RAM; остальное — во временном файле, + # который удаляется сразу после ответа (не хранилище, только буфер). + upload_spool_memory_bytes: int = _int("UPLOAD_SPOOL_MEMORY_BYTES", 256 * 1024) + # Одновременных загрузок на реплику и одновременных проверок изображения. + upload_max_concurrent: int = _int("UPLOAD_MAX_CONCURRENT", 16) + image_check_concurrency: int = _int("IMAGE_CHECK_CONCURRENCY", 2) + + # Очистка «сирот» (объект в S3 без метаданных в БД). + # Журнал upload_intents старше grace period считается брошенным. + upload_grace_seconds: int = _int("UPLOAD_GRACE_SECONDS", 900) + janitor_interval_seconds: int = _int("JANITOR_INTERVAL_SECONDS", 60) + janitor_enabled: bool = os.getenv("JANITOR_ENABLED", "true").lower() == "true" + + # lab5: брокер — отдельный Redis со Streams и AOF (appendfsync always) на volume. + # Не путать с CACHE_URL: кэш сессий без persistence, брокер — с persistence. + broker_url: str = _secret("BROKER_URL", "redis://127.0.0.1:6380/0") + broker_timeout: float = _float("BROKER_TIMEOUT", 2.0) + jobs_stream: str = os.getenv("JOBS_STREAM", "attachments:jobs") + jobs_group: str = os.getenv("JOBS_GROUP", "workers") + dlq_stream: str = os.getenv("DLQ_STREAM", "attachments:jobs:dlq") + pipeline_version: int = _int("PIPELINE_VERSION", 1) + + # lab5: worker + worker_concurrency: int = _int("WORKER_CONCURRENCY", 2) # одновременных задач на процесс + job_max_attempts: int = _int("JOB_MAX_ATTEMPTS", 5) # управляемых попыток на job + job_timeout_seconds: float = _float("JOB_TIMEOUT_SECONDS", 60) # жёсткий лимит обработки + job_lease_seconds: float = _float("JOB_LEASE_SECONDS", 20) # аренда, продлевается heartbeat + job_backoff_base_seconds: float = _float("JOB_BACKOFF_BASE_SECONDS", 2) + job_backoff_max_seconds: float = _float("JOB_BACKOFF_MAX_SECONDS", 60) + # Через сколько чужое неподтверждённое сообщение считается брошенным (XAUTOCLAIM). + # Больше JOB_TIMEOUT: живой worker к этому времени точно закончил или убил обработку. + claim_idle_seconds: float = _float("CLAIM_IDLE_SECONDS", 90) + # Память дочернего процесса обработки (RLIMIT_AS), МиБ. + image_memory_mb: int = _int("IMAGE_MEMORY_MB", 768) + # Outbox relay: сколько ждать, прежде чем считать строку не опубликованной API. + outbox_relay_delay_seconds: float = _float("OUTBOX_RELAY_DELAY_SECONDS", 2) + maintenance_interval_seconds: float = _float("MAINTENANCE_INTERVAL_SECONDS", 1) + # Исходник в staging у failed хранится столько (для replay), потом удаляется. + staging_retention_seconds: int = _int("STAGING_RETENTION_SECONDS", 86400) + heartbeat_file: str = os.getenv("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat") + + # Внедрение отказов ТОЛЬКО для тестового профиля fault (api-fault). + # В обычных репликах выключено: заголовок X-Fault-Inject игнорируется. + fault_injection: bool = os.getenv("FAULT_INJECTION", "false").lower() == "true" + + +settings = Settings() diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..3a3368e1 --- /dev/null +++ b/app/db.py @@ -0,0 +1,50 @@ +""" +Пул соединений с PostgreSQL. + +- Пул ограничен (DB_POOL_MAX): при нагрузке запросы ждут свободное соединение + не дольше DB_POOL_TIMEOUT, а не открывают соединения без предела. +- connect_timeout и statement_timeout: зависший запрос или недоступная БД + не держат обработчик бесконечно — клиент получает 503. +- Все запросы параметризованы (%s), SQL-инъекция невозможна. +""" +from __future__ import annotations +from psycopg_pool import AsyncConnectionPool + +from .config import settings + +pool: AsyncConnectionPool | None = None + + +def make_pool() -> AsyncConnectionPool: + return AsyncConnectionPool( + conninfo=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_pool_timeout, + kwargs={ + "connect_timeout": settings.db_connect_timeout, + "options": f"-c statement_timeout={settings.db_statement_timeout_ms}", + "application_name": "messenger-api", + }, + # Перед выдачей соединение проверяется (SELECT 1 на уровне протокола): + # после рестарта PostgreSQL «мёртвые» соединения отбрасываются, а не дают 503. + check=AsyncConnectionPool.check_connection, + open=False, + ) + + +async def open_pool() -> None: + global pool + pool = make_pool() + # wait=False: API стартует, даже если БД ещё поднимается; /health/ready это покажет. + await pool.open(wait=False) + + +async def close_pool() -> None: + if pool is not None: + await pool.close() + + +def get_pool() -> AsyncConnectionPool: + assert pool is not None, "pool is not opened" + return pool diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..35db74ec --- /dev/null +++ b/app/images.py @@ -0,0 +1,235 @@ +""" +Проверка загруженного изображения ПО СОДЕРЖИМОМУ (lab4). + +Порядок важен: сначала дешёвые проверки, потом дорогие. + 1. Сигнатура (magic bytes) → реальный тип. Не PNG/JPEG или не совпадает + с заявленным Content-Type → 415 (подмена MIME). + 2. Размеры из заголовка (PNG IHDR / JPEG SOF) → width × height ≤ MAX_IMAGE_PIXELS, + иначе 413. Это делается ДО распаковки пикселей: «декомпрессионная бомба» + (маленький файл, огромная картинка) отсекается, не заняв память. + 3. Целостность → повреждённый файл даёт 415: + - PNG: потоково читаем чанки, сверяем CRC каждого, распаковываем IDAT + кусками по 64 КиБ и только СЧИТАЕМ байты (пиксели не храним). + Объём распакованных данных должен ровно совпасть с тем, что следует + из IHDR; лишнее сверх ожидаемого не распаковываем вообще. + - JPEG: Pillow в режиме draft (декодирование в масштабе 1/8) — + весь поток проверяется декодером, а памяти нужно в ~64 раза меньше. +Памяти на проверку: O(64 КиБ) для PNG и ~(пиксели/64)×3 байт для JPEG, +независимо от размера файла. +""" +from __future__ import annotations + +import struct +import warnings +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image, ImageFile + +PNG_SIG = b"\x89PNG\r\n\x1a\n" +JPEG_SIG = b"\xff\xd8\xff" +CHUNK = 64 * 1024 + +# Усечённый файл — ошибка, а не «догрузим серым». +ImageFile.LOAD_TRUNCATED_IMAGES = False +# Предупреждение Pillow о «бомбе» превращаем в исключение (→ 413), а не в строку лога. +warnings.simplefilter("error", Image.DecompressionBombWarning) + + +class ImageTooLarge(Exception): + """Слишком много пикселей → 413.""" + + +class BadImage(Exception): + """Не PNG/JPEG, тип не совпадает с Content-Type или файл повреждён → 415.""" + + +@dataclass(frozen=True) +class ImageInfo: + content_type: str + width: int + height: int + + +def sniff(head: bytes) -> str | None: + """Реальный тип по первым байтам. Расширение и заголовок клиента не учитываются.""" + if head.startswith(PNG_SIG): + return "image/png" + if head.startswith(JPEG_SIG): + return "image/jpeg" + return None + + +def inspect(f: BinaryIO, declared_type: str, max_pixels: int) -> ImageInfo: + """Проверить файл (seekable) и вернуть реальный тип и размеры. + Бросает ImageTooLarge или BadImage. Блокирующая: вызывать в пуле потоков.""" + f.seek(0) + actual = sniff(f.read(16)) + if actual is None: + raise BadImage("not a PNG or JPEG") + if actual != declared_type: + raise BadImage("content does not match Content-Type") + f.seek(0) + try: + if actual == "image/png": + w, h = _check_png(f, max_pixels) + else: + w, h = _check_jpeg(f, max_pixels) + except (ImageTooLarge, BadImage): + raise + except (zlib.error, struct.error, ValueError, OSError, EOFError, IndexError) as exc: + # Любая ошибка разбора содержимого клиента — это битый файл (415), а не 500. + raise BadImage("corrupt image data") from exc + return ImageInfo(actual, w, h) + + +# --------------------------------------------------------------------- PNG +# color type → число каналов; допустимые глубины цвета (спецификация PNG, 11.2.2) +_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +_PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +# Adam7: (x0, y0, dx, dy) семи проходов +_ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) + + +def _png_layout(w: int, h: int, bits_per_pixel: int, interlaced: bool) -> list[tuple[int, int]]: + """Список (число строк, длина строки с байтом фильтра) для всех проходов.""" + def stride(width: int) -> int: + return 1 + (width * bits_per_pixel + 7) // 8 + + if not interlaced: + return [(h, stride(w))] + out = [] + for x0, y0, dx, dy in _ADAM7: + pw = (w - x0 + dx - 1) // dx if w > x0 else 0 + ph = (h - y0 + dy - 1) // dy if h > y0 else 0 + if pw and ph: + out.append((ph, stride(pw))) + return out + + +def _read_exact(f: BinaryIO, n: int) -> bytes: + data = f.read(n) + if len(data) != n: + raise BadImage("truncated PNG") + return data + + +def _check_png(f: BinaryIO, max_pixels: int) -> tuple[int, int]: + if f.read(8) != PNG_SIG: + raise BadImage("bad PNG signature") + width = height = 0 + layout: list[tuple[int, int]] = [] + expected = produced = 0 + seg = row_left = 0 # позиция для проверки байтов фильтра строк + inflater = None + seen_ihdr = seen_plte = seen_idat = seen_iend = False + color_type = 0 + + def consume(raw: bytes) -> None: + """Учесть распакованные байты: длина и байт фильтра в начале каждой строки.""" + nonlocal produced, seg, row_left + produced += len(raw) + if produced > expected: + raise BadImage("PNG image data is longer than declared") + pos = 0 + while pos < len(raw): + if row_left == 0: # начало новой строки: байт фильтра 0..4 + if raw[pos] > 4: + raise BadImage("bad PNG filter type") + rows, stride = layout[seg] + layout[seg] = (rows - 1, stride) + if layout[seg][0] == 0: + seg += 1 + row_left = stride + take = min(row_left, len(raw) - pos) + row_left -= take + pos += take + + def feed(data: bytes) -> None: + """Распаковать порциями ≤64 КиБ: бомба внутри IDAT не займёт память. + Цикл продолжается, пока zlib отдаёт полный буфер (могут остаться данные).""" + while True: + out = inflater.decompress(data, CHUNK) + consume(out) + data = inflater.unconsumed_tail + if inflater.eof or (not data and len(out) < CHUNK): + return + + while not seen_iend: + length, ctype = struct.unpack(">I4s", _read_exact(f, 8)) + if length > 0x7FFFFFFF or not ctype.isalpha(): + raise BadImage("bad PNG chunk") + crc = zlib.crc32(ctype) + if not seen_ihdr and ctype != b"IHDR": + raise BadImage("PNG must start with IHDR") + + if ctype == b"IHDR": + if seen_ihdr or length != 13: + raise BadImage("bad IHDR") + data = _read_exact(f, 13) + crc = zlib.crc32(data, crc) + width, height, depth, color_type, comp, filt, interlace = struct.unpack(">IIBBBBB", data) + if width == 0 or height == 0 or width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise BadImage("bad PNG dimensions") + # Лимит пикселей — ДО распаковки данных. + if width * height > max_pixels: + raise ImageTooLarge(f"{width}x{height} exceeds pixel limit") + if color_type not in _PNG_CHANNELS or depth not in _PNG_DEPTHS[color_type] \ + or comp != 0 or filt != 0 or interlace not in (0, 1): + raise BadImage("unsupported PNG header") + layout = _png_layout(width, height, depth * _PNG_CHANNELS[color_type], interlace == 1) + expected = sum(rows * stride for rows, stride in layout) + inflater = zlib.decompressobj() + seen_ihdr = True + else: + left = length + while left: + part = _read_exact(f, min(CHUNK, left)) + left -= len(part) + crc = zlib.crc32(part, crc) + if ctype == b"IDAT": + if inflater is None or inflater.eof: + raise BadImage("unexpected PNG data") + feed(part) + if ctype == b"IDAT": + seen_idat = True + elif ctype == b"PLTE": + seen_plte = True + elif ctype == b"IEND": + seen_iend = True + stored_crc = struct.unpack(">I", _read_exact(f, 4))[0] + if stored_crc != crc & 0xFFFFFFFF: + raise BadImage("PNG CRC mismatch") + + if not seen_idat or inflater is None: + raise BadImage("PNG has no image data") + if color_type == 3 and not seen_plte: + raise BadImage("palette PNG without PLTE") + if not inflater.eof or produced != expected: + raise BadImage("PNG image data is incomplete") + return width, height + + +# -------------------------------------------------------------------- JPEG +def _check_jpeg(f: BinaryIO, max_pixels: int) -> tuple[int, int]: + # Страховка Pillow от бомб; основной лимит проверяем сами ниже. + Image.MAX_IMAGE_PIXELS = max_pixels + try: + with Image.open(f, formats=["JPEG"]) as img: # читает только заголовок + width, height = img.size + if width <= 0 or height <= 0: + raise BadImage("bad JPEG dimensions") + if width * height > max_pixels: + raise ImageTooLarge(f"{width}x{height} exceeds pixel limit") + # Декодируем весь поток в масштабе 1/8: битый/усечённый файл даст ошибку, + # а памяти нужно (W/8)×(H/8)×3 байт вместо W×H×3. + img.draft("RGB", (max(1, width // 8), max(1, height // 8))) + img.load() + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc: + raise ImageTooLarge("pixel limit") from exc + except (ImageTooLarge, BadImage): + raise + except (OSError, SyntaxError, ValueError, struct.error, EOFError, IndexError) as exc: + raise BadImage("corrupt JPEG") from exc + return width, height diff --git a/app/imaging.py b/app/imaging.py new file mode 100644 index 00000000..b51658e7 --- /dev/null +++ b/app/imaging.py @@ -0,0 +1,153 @@ +""" +Построение вариантов изображения (lab5). Выполняется в ДОЧЕРНЕМ процессе worker +(app/worker.py → run_isolated), чтобы ограничить память и время одной задачи: + +- RLIMIT_AS (IMAGE_MEMORY_MB): нехватка памяти → MemoryError в ребёнке, а не OOM всего worker; +- RLIMIT_CPU и wall-clock таймаут родителя (JOB_TIMEOUT_SECONDS): зависшее + декодирование убивается, worker живёт дальше; +- пиксели проверяются по заголовку ДО распаковки (MAX_IMAGE_PIXELS = 40 млн). + +Варианты (контракт, contracts/README.md «Фоновая обработка»): + thumbnail — JPEG, вписан в 256×256, пропорции сохраняются, маленькие НЕ увеличиваются, + размеры округляются вниз, минимум 1 px (320×200 → 256×160); + crop — JPEG, центрированный квадрат 128×128 (маленькие исходники увеличиваются); + optimized — WebP, вписан в 1280×1280 без увеличения, quality 80, method 4. +Во всех вариантах: EXIF orientation применяется; EXIF/XMP/ICC в результат НЕ пишутся +(Pillow пишет метаданные только если их явно передать в save()). Прозрачность в JPEG +компонуется на белый фон; в WebP альфа-канал сохраняется. +""" +from __future__ import annotations + +import json +import os +import resource +import traceback +import warnings + +from PIL import Image, ImageFile, ImageOps + +ImageFile.LOAD_TRUNCATED_IMAGES = False +warnings.simplefilter("error", Image.DecompressionBombWarning) + +THUMB_BOX = 256 +CROP_SIDE = 128 +OPTIMIZED_BOX = 1280 +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +# Порядок вариантов в ответе API и набор, который обещает уровень «4». +VARIANTS = ("thumbnail", "crop", "optimized") +VARIANT_TYPES = {"thumbnail": "image/jpeg", "crop": "image/jpeg", "optimized": "image/webp"} +VARIANT_EXT = {"thumbnail": "jpg", "crop": "jpg", "optimized": "webp"} + + +class PermanentImageError(Exception): + """Изображение не обработать никогда (битое, слишком большое) → failed invalid_image.""" + + +def fit_size(w: int, h: int, box: int) -> tuple[int, int]: + """Вписать в box×box без увеличения, округление вниз, минимум 1 px.""" + if w <= box and h <= box: + return w, h + if w >= h: + return box, max(1, h * box // w) + return max(1, w * box // h), box + + +def _oriented_size(img: Image.Image) -> tuple[int, int]: + w, h = img.size + try: + orientation = img.getexif().get(0x0112, 1) + except Exception: # noqa: BLE001 — битый EXIF не повод падать; ориентация по умолчанию + orientation = 1 + return (h, w) if orientation in (5, 6, 7, 8) else (w, h) + + +def _on_white(img: Image.Image) -> Image.Image: + """RGB для JPEG: прозрачность компонуется на белый.""" + if img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info): + rgba = img.convert("RGBA") + bg = Image.new("RGB", rgba.size, (255, 255, 255)) + bg.paste(rgba, mask=rgba.getchannel("A")) + return bg + return img.convert("RGB") + + +def _has_alpha(img: Image.Image) -> bool: + return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info) + + +def render(src_path: str, out_dir: str, content_type: str, max_pixels: int) -> dict: + fmt = {"image/png": "PNG", "image/jpeg": "JPEG"}[content_type] + Image.MAX_IMAGE_PIXELS = max_pixels + try: + with Image.open(src_path, formats=[fmt]) as img: + w0, h0 = img.size + if w0 <= 0 or h0 <= 0 or w0 * h0 > max_pixels: + raise PermanentImageError("pixel limit") + ow, oh = _oriented_size(img) # размеры после поворота по EXIF + thumb = fit_size(ow, oh, THUMB_BOX) + opt = fit_size(ow, oh, OPTIMIZED_BOX) + if fmt == "JPEG": + # Декодируем сразу в уменьшенном масштабе (1/2, 1/4, 1/8), но не меньше, + # чем нужно самому большому варианту (optimized) и crop: экономия памяти и времени. + rw, rh = (opt[1], opt[0]) if (ow, oh) != (w0, h0) else opt # в осях файла + img.draft("RGB", (max(rw, min(w0, CROP_SIDE)), max(rh, min(h0, CROP_SIDE)))) + img.load() + base = ImageOps.exif_transpose(img) # копия с применённой ориентацией + except PermanentImageError: + raise + except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc: + raise PermanentImageError("pixel limit") from exc + except (OSError, SyntaxError, ValueError, EOFError, IndexError, KeyError) as exc: + raise PermanentImageError(f"cannot decode: {type(exc).__name__}") from exc + + out = {} + rgb = _on_white(base) + + t = rgb.resize(thumb, Image.Resampling.LANCZOS, reducing_gap=3.0) + out["thumbnail"] = _save(t, out_dir, "thumbnail", "JPEG", quality=JPEG_QUALITY, optimize=True) + + bw, bh = rgb.size + side = min(bw, bh) + left, top = (bw - side) // 2, (bh - side) // 2 + c = rgb.crop((left, top, left + side, top + side)).resize( + (CROP_SIDE, CROP_SIDE), Image.Resampling.LANCZOS, reducing_gap=3.0) + out["crop"] = _save(c, out_dir, "crop", "JPEG", quality=JPEG_QUALITY, optimize=True) + + src = base.convert("RGBA") if _has_alpha(base) else rgb + o = src if src.size == opt else src.resize(opt, Image.Resampling.LANCZOS, reducing_gap=3.0) + out["optimized"] = _save(o, out_dir, "optimized", "WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD) + return out + + +def _save(img: Image.Image, out_dir: str, name: str, fmt: str, **opts) -> dict: + path = os.path.join(out_dir, f"{name}.{VARIANT_EXT[name]}") + img.save(path, fmt, **opts) # без exif=/icc_profile= → метаданных нет + return {"path": path, "content_type": VARIANT_TYPES[name], + "width": img.size[0], "height": img.size[1]} + + +def child_main(src_path: str, out_dir: str, content_type: str, max_pixels: int, + memory_mb: int, cpu_seconds: int, result_path: str, + slow_seconds: float = 0.0) -> None: + """Точка входа дочернего процесса. Результат — JSON-файл; код выхода 0. + slow_seconds > 0 — только для теста «долгой, но живой» задачи (fault-профиль).""" + resource.setrlimit(resource.RLIMIT_AS, (memory_mb * 1024 * 1024, memory_mb * 1024 * 1024)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu_seconds, cpu_seconds + 1)) + if slow_seconds > 0: + import time + time.sleep(slow_seconds) # sleep не тратит CPU: держит только wall-clock + try: + result = {"ok": True, "variants": render(src_path, out_dir, content_type, max_pixels)} + except PermanentImageError as exc: + result = {"ok": False, "permanent": True, "code": "invalid_image", "detail": str(exc)[:200]} + except MemoryError: + result = {"ok": False, "permanent": True, "code": "invalid_image", "detail": "memory limit"} + except Exception as exc: # noqa: BLE001 — неизвестное: временная ошибка, retry ограничен + result = {"ok": False, "permanent": False, "code": "processing_failed", + "detail": type(exc).__name__, + "trace": traceback.format_exc(limit=3)[-500:]} + with open(result_path, "w", encoding="utf-8") as f: + json.dump(result, f) diff --git a/app/janitor.py b/app/janitor.py new file mode 100644 index 00000000..d539f623 --- /dev/null +++ b/app/janitor.py @@ -0,0 +1,180 @@ +""" +Очистка «сирот» после сбоев между S3 и PostgreSQL (lab4, уровень 4). + +Протокол загрузки (app/main.py → upload_attachment): + 1. INSERT upload_intents(object_key) + COMMIT ← намерение записано + 2. PUT объекта в S3 + 3. BEGIN; DELETE upload_intents …; INSERT attachments …; COMMIT ← 201 только после этого +Если 2 или 3 не удались — API сразу компенсирует (удаляет объект и строку журнала). +Если компенсация не удалась или процесс упал (kill -9, OOM, рестарт контейнера) — +в журнале остаётся строка без метаданных. Её и обрабатывает janitor: + + строка upload_intents СТАРШЕ grace period → + BEGIN; DELETE строку (блокировка); DELETE объект в S3; COMMIT + - ошибка S3 → ROLLBACK, строка остаётся, повтор на следующем проходе; + - упали после DELETE в S3, до COMMIT → строка остаётся, повторное + удаление объекта безопасно (DELETE в S3 идемпотентен). + +Почему grace period безопасен: живая загрузка не может длиться дольше +UPLOAD_BODY_TIMEOUT_SECONDS (60 c) + таймауты S3/БД (~30 c). По умолчанию +UPLOAD_GRACE_SECONDS = 900 c (15 мин) — в 10 раз больше. Даже если +загрузка «проспала» grace period, шаг 3 удаляет строку журнала в той же +транзакции, что и INSERT метаданных: если janitor уже забрал строку, загрузка +получит 0 строк, откатится и ответит 503 — 201 на удалённый объект невозможен. +Готовые вложения (есть строка в attachments) janitor не трогает никогда. + +Страховка (--scan-bucket): обход bucket по prefix originals/ — объект старше +grace period, для которого нет ни attachments, ни upload_intents, — сирота. + +Запуск: + - в каждой реплике API фоновым циклом (JANITOR_INTERVAL_SECONDS), но + одновременно работает только одна: pg_try_advisory_lock; + - вручную: python -m app.janitor --once [--dry-run] [--scan-bucket] [--grace-seconds N] +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import random +import time +from datetime import datetime, timedelta, timezone + +import psycopg +from fastapi.concurrency import run_in_threadpool + +from . import db, storage +from .config import settings + +log = logging.getLogger("messenger.janitor") +LOCK_ID = 824_113_2027 +BATCH = 500 + + +async def _sweep_intents(conn, grace: int, dry_run: bool, stats: dict) -> None: + cur = await conn.execute( + "SELECT object_key FROM upload_intents " + "WHERE created_at < now() - make_interval(secs => %s) ORDER BY created_at LIMIT %s", + (grace, BATCH)) + keys = [r[0] for r in await cur.fetchall()] + for key in keys: + stats["intents_expired"] += 1 + try: + async with conn.transaction(): + cur = await conn.execute( + "DELETE FROM upload_intents WHERE object_key = %s " + "AND created_at < now() - make_interval(secs => %s) RETURNING 1", (key, grace)) + if await cur.fetchone() is None: + continue # загрузка успела завершиться — не наше + cur = await conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (key,)) + committed = await cur.fetchone() is not None + if dry_run: + stats["would_delete_objects"] += 0 if committed else 1 + raise psycopg.Rollback() + if not committed: + await storage.delete(key) # ошибка → ROLLBACK, строка останется + stats["objects_deleted"] += 1 + stats["intents_removed"] += 1 + except storage.StorageUnavailable: + stats["errors"] += 1 + log.warning("janitor: s3 unavailable, will retry later") + return + + +async def _scan_bucket(conn, grace: int, dry_run: bool, stats: dict) -> None: + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace) + + def old_keys() -> list[str]: + # lab5: оригиналы публикует worker в originals/, приём идёт в staging/ — обходим оба. + return [k for prefix in (storage.ORIGINALS_PREFIX, storage.STAGING_PREFIX) + for k, modified, _ in storage.list_sync(prefix) if modified < cutoff] + + try: + candidates = await run_in_threadpool(old_keys) + except storage.StorageUnavailable: + stats["errors"] += 1 + return + stats["objects_scanned"] = len(candidates) + for i in range(0, len(candidates), BATCH): + part = candidates[i:i + BATCH] + cur = await conn.execute( + "SELECT object_key FROM attachments WHERE object_key = ANY(%s) " + "UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s) " + "UNION SELECT object_key FROM upload_intents WHERE object_key = ANY(%s)", (part, part, part)) + known = {r[0] for r in await cur.fetchall()} + for key in part: + if key in known: + continue + stats["orphans_found"] += 1 + if dry_run: + continue + try: + await storage.delete(key) + stats["objects_deleted"] += 1 + except storage.StorageUnavailable: + stats["errors"] += 1 + return + + +async def run_once(grace: int | None = None, dry_run: bool = False, + scan_bucket: bool = False) -> dict: + grace = settings.upload_grace_seconds if grace is None else grace + stats = {"grace_seconds": grace, "dry_run": dry_run, "locked": False, + "intents_expired": 0, "intents_removed": 0, "objects_deleted": 0, + "would_delete_objects": 0, "orphans_found": 0, "errors": 0} + started = time.monotonic() + async with db.get_pool().connection() as conn: + await conn.set_autocommit(True) + cur = await conn.execute("SELECT pg_try_advisory_lock(%s)", (LOCK_ID,)) + if not (await cur.fetchone())[0]: + return stats # другая реплика уже чистит + stats["locked"] = True + try: + await _sweep_intents(conn, grace, dry_run, stats) + if scan_bucket: + await _scan_bucket(conn, grace, dry_run, stats) + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + await conn.set_autocommit(False) + stats["duration_ms"] = round((time.monotonic() - started) * 1000, 1) + return stats + + +async def loop_forever() -> None: + """Фоновый цикл в реплике API. Ошибки логируются, цикл не падает.""" + runs = 0 + while True: + # Случайный сдвиг: реплики не стучатся в advisory lock одновременно. + await asyncio.sleep(settings.janitor_interval_seconds * random.uniform(0.8, 1.2)) + runs += 1 + try: + stats = await run_once(scan_bucket=(runs % 60 == 0)) # обход bucket — примерно раз в час + if stats["intents_removed"] or stats["objects_deleted"] or stats["errors"]: + log.info("janitor: %s", stats) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 — фоновая задача не должна умирать + log.warning("janitor run failed: %s", type(exc).__name__) + + +async def _cli() -> None: + ap = argparse.ArgumentParser(description="cleanup of orphan S3 objects") + ap.add_argument("--once", action="store_true", help="один проход (по умолчанию)") + ap.add_argument("--dry-run", action="store_true", help="только показать, ничего не удалять") + ap.add_argument("--scan-bucket", action="store_true", help="дополнительно обойти bucket") + ap.add_argument("--grace-seconds", type=int, default=None) + args = ap.parse_args() + await db.open_pool() + storage.init() + try: + await db.get_pool().wait(timeout=10) + stats = await run_once(args.grace_seconds, args.dry_run, args.scan_bucket) + finally: + await db.close_pool() + print(json.dumps(stats, ensure_ascii=False)) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + asyncio.run(_cli()) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..81c139b1 --- /dev/null +++ b/app/main.py @@ -0,0 +1,634 @@ +""" +HTTP-слой мессенджера (lab4). + +Клиент → LB (HAProxy) → эта реплика API (их ≥2, взаимозаменяемы) + → PostgreSQL (источник истины: данные и метаданные вложений) + → Redis (общий кэш сессий) + → MinIO, приватный bucket (только байты вложений) + +lab2: пароль Argon2id, вход Basic → Bearer-сессия в PostgreSQL, X-User-Id игнорируется. +lab3: X-Instance-Id в каждом ответе; POST сообщения требует Idempotency-Key, + идемпотентность общая для всех реплик (таблица idempotency_keys); + у реплики нет локального состояния, от которого зависит корректность. +lab4: вложения PNG/JPEG: GET /attachments/{id} и /content (только участникам чата), + attachment_ids в сообщениях. +lab5: POST /chats/{id}/attachments → 202 queued. До ответа: оригинал целиком в приватном + staging (S3), в ОДНОЙ транзакции — вложение, job и строка outbox. Затем сообщение + публикуется в брокер (Redis Streams с AOF); если не вышло — его опубликует relay. + Тяжёлую обработку (варианты) делает отдельный процесс worker (app/worker.py). +""" +from __future__ import annotations +import asyncio +import base64 +import binascii +import contextlib +import logging +import os +import re +import time +import uuid +from contextlib import asynccontextmanager + +import psycopg +from fastapi import Depends, FastAPI, Header, Query, Request +from fastapi.concurrency import run_in_threadpool +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from psycopg_pool import PoolTimeout +from pydantic import BaseModel, Field, field_validator, model_validator +from starlette.exceptions import HTTPException as StarletteHTTPException + +from urllib.parse import quote + +from . import broker, db, images, janitor, repo, security, sessions, storage, uploads +from .config import settings +from .migrate import migrate + +logger = logging.getLogger("messenger") + +# Только fullmatch: в Python "$" совпадает и перед завершающим "\n", +# поэтому match("^…$") пропустил бы "review123\n" до CHECK в БД (500). +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +MAX_ATTACHMENTS_PER_MESSAGE = 10 +KNOWN_VARIANTS = ("thumbnail", "crop", "optimized") + + +@asynccontextmanager +async def lifespan(app: FastAPI): + if settings.migrate_on_start: + try: + applied = await migrate() + logger.info("migrations: %s", applied or "up to date") + except psycopg.OperationalError as exc: + # БД ещё не готова: API всё равно стартует, /health/ready покажет 503. + logger.warning("migrations skipped, database unavailable: %s", type(exc).__name__) + await db.open_pool() + sessions.init_redis() + storage.init() + broker.init_async() + cleaner = asyncio.create_task(janitor.loop_forever()) if settings.janitor_enabled else None + yield + if cleaner is not None: + cleaner.cancel() + with contextlib.suppress(asyncio.CancelledError): + await cleaner + await sessions.close_redis() + await broker.close_async() + await db.close_pool() + + +app = FastAPI(title="MTUSI Messenger — lab5", lifespan=lifespan) + + +# ============================================================================ +# 1. ЕДИНЫЙ ФОРМАТ ОШИБОК {"error":{"code","message","request_id"}} +# ============================================================================ +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: dict | None = None): + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def error_response(request: Request, status: int, code: str, message: str, + headers: dict | None = None) -> JSONResponse: + rid = getattr(request.state, "request_id", str(uuid.uuid4())) + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers={"X-Request-Id": rid, "X-Instance-Id": settings.instance_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError): + return error_response(request, exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def handle_validation(request: Request, exc: RequestValidationError): + return error_response(request, 400, "invalid_request", "invalid request body or parameters") + + +@app.exception_handler(StarletteHTTPException) +async def handle_http(request: Request, exc: StarletteHTTPException): + # Только коды из enum контракта (contracts/openapi.json → Error.code). + code = {401: "unauthorized", 404: "not_found", 409: "conflict", + 413: "payload_too_large", 415: "unsupported_media_type", + 429: "rate_limited", 503: "unavailable"}.get( + exc.status_code, "internal_error" if exc.status_code >= 500 else "invalid_request") + return error_response(request, exc.status_code, code, str(exc.detail)) + + +@app.exception_handler(psycopg.OperationalError) +@app.exception_handler(PoolTimeout) +async def handle_db_down(request: Request, exc: Exception): + # БД недоступна или не ответила за таймаут: ограниченная ошибка, а не зависание. + logger.warning("database unavailable: %s", type(exc).__name__) + return error_response(request, 503, "unavailable", "service temporarily unavailable") + + +@app.exception_handler(storage.StorageUnavailable) +async def handle_storage_down(request: Request, exc: Exception): + # S3 недоступен/не ответил за таймаут: 503, текстовая часть мессенджера продолжает работать. + return error_response(request, 503, "unavailable", "file storage temporarily unavailable") + + +@app.exception_handler(uploads.UploadError) +async def handle_upload_error(request: Request, exc: uploads.UploadError): + return error_response(request, exc.status, exc.code, exc.message) + + +@app.exception_handler(Exception) +async def handle_unexpected(request: Request, exc: Exception): + logger.exception("unexpected error") + return error_response(request, 500, "internal_error", "internal error") + + +# ============================================================================ +# 2. X-Request-Id, X-Instance-Id + лог без тела запроса, токенов и паролей +# ============================================================================ +@app.middleware("http") +async def add_request_id(request: Request, call_next): + rid = str(uuid.uuid4()) + request.state.request_id = rid + start = time.monotonic() + response = await call_next(request) + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = settings.instance_id # какая реплика ответила + logger.info("rid=%s %s %s -> %s %sms", rid, request.method, request.url.path, + response.status_code, round((time.monotonic() - start) * 1000, 1)) + return response + + +# ============================================================================ +# 3. АУТЕНТИФИКАЦИЯ +# Bearer — для всех бизнес-запросов. Basic — только на POST /auth/sessions. +# X-User-Id больше ничего не значит и ни на что не влияет. +# ============================================================================ +def unauthorized(message: str = "invalid or missing bearer token") -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": BEARER_CHALLENGE}) + + +def _bearer_token(request: Request) -> str: + auth = request.headers.get("authorization", "") + scheme, _, token = auth.partition(" ") + token = token.strip() + if scheme.lower() != "bearer" or not token or len(token) > 512: + raise unauthorized() + return token + + +async def current_user(request: Request) -> dict: + """Зависимость FastAPI: выполняется ДО разбора тела → сначала 401, потом 400.""" + token = _bearer_token(request) + user = await sessions.resolve(db.get_pool(), token, repo.get_user) + if user is None: + raise unauthorized() + request.state.token = token + return user + + +def _parse_basic(request: Request) -> tuple[str, str]: + bad = ApiError(401, "unauthorized", "invalid credentials", {"WWW-Authenticate": BASIC_CHALLENGE}) + scheme, _, value = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "basic" or not value: + raise bad + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + raise bad + username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо + if not sep or not username.isascii() or not (12 <= len(password) <= 128): + raise bad + return username, password + + +def require_json(request: Request) -> None: + """Тип тела проверяем ДО валидации тела: не JSON → 415, а не 400. + Подключается после current_user, поэтому без токена по-прежнему сначала 401. + Параметры вроде '; charset=utf-8' допустимы.""" + media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if media_type != "application/json": + raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json") + + +_kdf_slots: asyncio.Semaphore | None = None + + +async def kdf(fn, *args): + """Argon2id по умолчанию берёт 64 МиБ RAM на один хэш. Без ограничения 20 + одновременных регистраций/входов на реплике = >1 ГиБ и OOM-kill контейнера + с лимитом 256 МиБ. Поэтому одновременно считаем не больше + PASSWORD_HASH_CONCURRENCY хэшей, остальные ждут в очереди.""" + global _kdf_slots + if _kdf_slots is None: + _kdf_slots = asyncio.Semaphore(settings.password_hash_concurrency) + async with _kdf_slots: + return await run_in_threadpool(fn, *args) + + +# ============================================================================ +# 4. МОДЕЛИ ВВОДА +# ============================================================================ +class CreateUser(BaseModel): + model_config = {"extra": "forbid"} + username: str + display_name: str + password: str + + @field_validator("username") + @classmethod + def check_username(cls, v: str) -> str: + if not USERNAME_RE.fullmatch(v): + raise ValueError("username must match [a-z0-9_]{3,32}") + return v + + @field_validator("display_name") + @classmethod + def check_display_name(cls, v: str) -> str: + if not (1 <= len(v) <= 100) or not v.strip(): + raise ValueError("display_name must be 1..100 non-blank chars") + return v + + @field_validator("password") + @classmethod + def check_password(cls, v: str) -> str: + if not (12 <= len(v) <= 128) or not v.strip(): + raise ValueError("password must be 12..128 chars") + return v + + +class CreateChat(BaseModel): + model_config = {"extra": "forbid"} + title: str + member_ids: list[str] + + @field_validator("title") + @classmethod + def check_title(cls, v: str) -> str: + if not (1 <= len(v) <= 100) or not v.strip(): + raise ValueError("title must be 1..100 non-blank chars") + return v + + +class CreateMessage(BaseModel): + model_config = {"extra": "forbid"} + text: str + # lab4: 0–10 уникальных id вложений этого чата; отсутствие поля == []. + attachment_ids: list[str] = Field(default_factory=list) + + @field_validator("text") + @classmethod + def check_text(cls, v: str) -> str: + if len(v) > 4000: + raise ValueError("text must be at most 4000 chars") + return v + + @field_validator("attachment_ids") + @classmethod + def check_attachment_ids(cls, v: list[str]) -> list[str]: + if len(v) > MAX_ATTACHMENTS_PER_MESSAGE: + raise ValueError("at most 10 attachment_ids") + if len(set(v)) != len(v): + raise ValueError("attachment_ids must be unique") + return v + + @model_validator(mode="after") + def text_or_attachments(self): + # Пустой/пробельный текст допустим только вместе с вложениями. + if not self.attachment_ids and not self.text.strip(): + raise ValueError("text must be non-blank when there are no attachments") + return self + + +def parse_uuid(value: str, what: str) -> uuid.UUID: + try: + return uuid.UUID(value) + except ValueError: + raise ApiError(400, "invalid_request", f"{what} is not a valid uuid") + + +# ============================================================================ +# 5. ENDPOINTS +# ============================================================================ +@app.get("/health/live") +async def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def health_ready(request: Request): + # Критичная зависимость — только PostgreSQL. Redis — кэш: его отказ не делает API неготовым. + try: + async with db.get_pool().connection() as conn: + await conn.execute("SELECT 1 FROM schema_migrations LIMIT 1") + except (psycopg.Error, PoolTimeout): + return error_response(request, 503, "unavailable", "database is not ready") + return {"status": "ok"} + + +# -------- users и сессии -------- +@app.post("/api/v1/users", status_code=201) +async def create_user(body: CreateUser, _json: None = Depends(require_json)): + # Argon2 намеренно медленный — считаем в пуле потоков, чтобы не блокировать event loop. + password_hash = await kdf(security.hash_password, body.password) + async with db.get_pool().connection() as conn: + try: + return await repo.create_user(conn, body.username, body.display_name, password_hash) + except repo.UsernameTaken: + raise ApiError(409, "conflict", "username already taken") + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + username, password = _parse_basic(request) + async with db.get_pool().connection() as conn: + row = await repo.get_credentials(conn, username) + stored_hash = row[4] if row else None + # Проверяем хэш даже для несуществующего пользователя — одинаковое время ответа. + ok = await kdf(security.verify_password, stored_hash, password) + if not ok: + raise ApiError(401, "unauthorized", "invalid credentials", {"WWW-Authenticate": BASIC_CHALLENGE}) + user = repo.user_out(row) + async with db.get_pool().connection() as conn: + if security.needs_rehash(stored_hash): + await repo.update_password_hash(conn, row[0], await kdf(security.hash_password, password)) + token, expires_at = await sessions.create_session(conn, row[0]) + return {"token": token, "token_type": "Bearer", "expires_at": repo.iso(expires_at), "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def delete_session(request: Request, user: dict = Depends(current_user)): + try: + revoked = await sessions.revoke(db.get_pool(), request.state.token) + except sessions.CacheUnavailable: + # 204 только при гарантированном отзыве; иначе ограниченная ошибка, повторить позже. + raise ApiError(503, "unavailable", "session cache unavailable, logout not confirmed") + if not revoked: + raise unauthorized() + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def get_me(user: dict = Depends(current_user)): + return user + + +# -------- chats -------- +@app.post("/api/v1/chats", status_code=201) +async def create_chat(body: CreateChat, user: dict = Depends(current_user), + _json: None = Depends(require_json)): + members = body.member_ids + if not (1 <= len(members) <= 100): + raise ApiError(400, "invalid_request", "member_ids must have 1..100 items") + if len(set(members)) != len(members): + raise ApiError(400, "invalid_request", "duplicate member_ids") + ids = [parse_uuid(m, "member id") for m in members] + if uuid.UUID(user["id"]) not in ids: + raise ApiError(400, "invalid_request", "caller must be a member") + async with db.get_pool().connection() as conn: + # Одна транзакция: чат и все участники создаются вместе или не создаются вовсе. + async with conn.transaction(): + if await repo.count_existing_users(conn, ids) != len(ids): + raise ApiError(400, "invalid_request", "unknown member") + return await repo.insert_chat(conn, body.title, ids, user["id"]) + + +@app.get("/api/v1/chats") +async def list_chats(user: dict = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1)): + async with db.get_pool().connection() as conn: + try: + items, nxt = await repo.list_chats(conn, user["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "bad cursor") + return {"items": items, "next_cursor": nxt} + + +async def accessible_chat(conn, chat_id: str, user: dict) -> dict: + """Кривой UUID → 400. Чата нет ИЛИ ты не участник → одинаковый 404.""" + cid = parse_uuid(chat_id, "chat_id") + chat = await repo.get_chat_for_member(conn, cid, user["id"]) + if chat is None: + raise ApiError(404, "not_found", "chat not found") + return chat + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, user: dict = Depends(current_user)): + async with db.get_pool().connection() as conn: + return await accessible_chat(conn, chat_id, user) + + +# -------- messages -------- +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def send_message(chat_id: str, body: CreateMessage, user: dict = Depends(current_user), + _json: None = Depends(require_json), + idempotency_key: str | None = Header(default=None, alias="Idempotency-Key")): + # Ключ обязателен: без него клиент не может безопасно повторить запрос после таймаута. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise ApiError(400, "invalid_request", "Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}") + attachment_ids = [parse_uuid(a, "attachment_id") for a in body.attachment_ids] + if len(set(attachment_ids)) != len(attachment_ids): + raise ApiError(400, "invalid_request", "attachment_ids must be unique") + async with db.get_pool().connection() as conn: + # Повтор снова проверяет сессию (current_user) и доступ к чату. + chat = await accessible_chat(conn, chat_id, user) + try: + async with conn.transaction(): + # Отправитель — из сессии, а не из тела запроса. + return await repo.send_message_idempotent( + conn, chat["id"], user["id"], idempotency_key, body.text, + settings.idempotency_ttl_seconds, attachment_ids) + except repo.IdempotencyConflict: + raise ApiError(409, "conflict", "Idempotency-Key was already used with a different request") + except repo.AttachmentNotFound: + # Нет такого вложения ИЛИ оно из другого чата — одинаковый 404. + raise ApiError(404, "not_found", "attachment not found") + except repo.AttachmentNotReady: + raise ApiError(409, "conflict", "attachment is not usable") + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, user: dict = Depends(current_user), + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None, min_length=1)): + async with db.get_pool().connection() as conn: + chat = await accessible_chat(conn, chat_id, user) + try: + items, nxt = await repo.list_messages(conn, chat["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "bad cursor") + return {"items": items, "next_cursor": nxt} + + +# ============================================================================ +# 6. ВЛОЖЕНИЯ (lab4): байты — в приватном S3, метаданные — в PostgreSQL +# ============================================================================ +_upload_slots: asyncio.Semaphore | None = None +_image_checks: asyncio.Semaphore | None = None + + +def _slots() -> tuple[asyncio.Semaphore, asyncio.Semaphore]: + global _upload_slots, _image_checks + if _upload_slots is None: + _upload_slots = asyncio.Semaphore(settings.upload_max_concurrent) + _image_checks = asyncio.Semaphore(settings.image_check_concurrency) + return _upload_slots, _image_checks + + +async def _discard_upload(key: str) -> None: + """Компенсация: удалить объект и строку журнала. Если не вышло — + строка журнала останется, и объект удалит janitor после grace period.""" + try: + async with db.get_pool().connection() as conn: + async with conn.transaction(): + cur = await conn.execute( + "DELETE FROM upload_intents WHERE object_key = %s RETURNING 1", (key,)) + if await cur.fetchone() is not None: + await storage.delete(key) + except Exception as exc: # noqa: BLE001 — компенсация best effort + logger.warning("upload compensation failed, janitor will clean up: %s", type(exc).__name__) + + +def _fault(request: Request, point: str) -> None: + """Внедрение отказа для тестов уровня 4 — только при FAULT_INJECTION=true + (отдельная тестовая реплика api-fault). В обычных репликах — no-op.""" + if not settings.fault_injection or request.headers.get("x-fault-inject") != point: + return + logger.warning("FAULT INJECTION: %s", point) + if point in ("crash-after-s3-put", "crash-after-commit-before-publish"): + os._exit(1) # как kill -9: процесс умирает мгновенно, без finally + raise RuntimeError("injected fault: " + point) + + +async def _publish_now(outbox_id: int, payload: dict) -> None: + """Быстрый путь после COMMIT: сразу отдать сообщение брокеру и отметить строку outbox. + Любая ошибка здесь НЕ влияет на ответ 202: строка outbox останется неопубликованной, + и relay (app/worker.py, поток maintenance) опубликует её позже.""" + try: + async with asyncio.timeout(settings.broker_timeout): + stream_id = await broker.publish(payload) # ответ XADD = брокер принял (AOF fsync) + async with db.get_pool().connection() as conn: + await repo.mark_published(conn, outbox_id, stream_id) + except Exception as exc: # noqa: BLE001 — намерение уже в outbox + logger.warning("direct publish failed, outbox relay will retry: %s", type(exc).__name__) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(chat_id: str, request: Request, user: dict = Depends(current_user), + filename: str | None = Query(default=None)): + # 1) доступ к чату → 2) имя → 3) заявленный тип → 4) заявленный размер; всё — до чтения тела. + async with db.get_pool().connection() as conn: + chat = await accessible_chat(conn, chat_id, user) + name = uploads.validate_filename(filename) + ctype = uploads.declared_type(request) + uploads.check_declared_length(request, settings.max_upload_bytes) + + slots, checks = _slots() + async with slots: # не больше N загрузок на реплику + body = await uploads.receive(request, settings.max_upload_bytes, + settings.upload_body_timeout_seconds, + settings.upload_spool_memory_bytes) + try: + async with checks: # CPU-тяжёлая проверка — ограниченно + try: + info = await run_in_threadpool(images.inspect, body.file, ctype, + settings.max_image_pixels) + except images.ImageTooLarge: + raise ApiError(413, "payload_too_large", "image has too many pixels") + except images.BadImage: + raise ApiError(415, "unsupported_media_type", + "file is not a valid image of the declared type") + + storage.ensure_available() # S3 заведомо недоступен → 503 без записи в журнал + attachment_id = uuid.uuid4() + key = storage.staging_key(attachment_id) # ключ назначает сервер; сначала — staging + async with db.get_pool().connection() as conn: + await repo.add_upload_intent(conn, key, attachment_id) # журнал ДО PUT + try: + await storage.put(key, body.file, body.size, info.content_type, + body.md5_b64, body.sha256_hex) + except storage.StorageUnavailable: + await _discard_upload(key) + raise + try: + _fault(request, "crash-after-s3-put") + _fault(request, "db-error-after-s3-put") + async with db.get_pool().connection() as conn: + async with conn.transaction(): + att, outbox_id, payload = await repo.commit_attachment( + conn, attachment_id=attachment_id, chat_id=chat["id"], + owner_id=user["id"], filename=name, content_type=info.content_type, + size_bytes=body.size, sha256=body.sha256_hex, + width=info.width, height=info.height, + object_key=storage.object_key(attachment_id), staging_key=key, + pipeline_version=settings.pipeline_version, + max_attempts=settings.job_max_attempts) + except Exception as exc: + # PUT прошёл, COMMIT — нет: компенсируем сразу, 202 не отдаём. + logger.warning("attachment commit failed: %s", type(exc).__name__) + await _discard_upload(key) + raise ApiError(503, "unavailable", "attachment was not saved, retry later") + # COMMIT прошёл: оригинал в staging, job и outbox в БД — работа уже не потеряется. + _fault(request, "crash-after-commit-before-publish") + await _publish_now(outbox_id, payload) + return att + finally: + body.file.close() # буфер запроса удаляется всегда + + +async def accessible_attachment(conn, attachment_id: str, user: dict): + aid = parse_uuid(attachment_id, "attachment_id") + row = await repo.get_attachment_for_member(conn, aid, user["id"]) + if row is None: + # Нет вложения ИЛИ пользователь не участник его чата — одинаковый 404. + raise ApiError(404, "not_found", "attachment not found") + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, user: dict = Depends(current_user)): + async with db.get_pool().connection() as conn: + row = await accessible_attachment(conn, attachment_id, user) + variants = await repo.get_variants(conn, row[0], settings.pipeline_version) \ + if row[7] == "ready" else [] + return repo.attachment_out(row, variants) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, user: dict = Depends(current_user), + variant: str = Query(default="original")): + parse_uuid(attachment_id, "attachment_id") + if variant != "original" and variant not in KNOWN_VARIANTS: + raise ApiError(400, "invalid_request", "unknown variant") + async with db.get_pool().connection() as conn: + row = await accessible_attachment(conn, attachment_id, user) + att = repo.attachment_out(row) + if att["status"] != "ready": + # queued / processing / failed: байтов результата ещё (или уже) нет. + raise ApiError(409, "conflict", "attachment is not ready") + key, ctype, etag = row[11], att["content_type"], att["sha256"] + if variant != "original": + v = await repo.get_variant(conn, row[0], settings.pipeline_version, variant) + if v is None: + raise ApiError(404, "not_found", "variant not found") + key, ctype, etag = v[0], v[1], v[3] + try: + stream, length = await storage.open_object(key) + except storage.ObjectMissing: + logger.error("object for attachment %s is missing in storage", att["id"]) + raise ApiError(500, "internal_error", "internal error") + headers = { + "Content-Length": str(length), + "ETag": f'"{etag}"', + "Cache-Control": "private, no-cache", + "X-Content-Type-Options": "nosniff", + # filename только для отображения; RFC 5987 — любые символы безопасно закодированы. + "Content-Disposition": "inline; filename*=UTF-8''" + quote(att["filename"], safe=""), + } + # Байты идут из S3 клиенту кусками по 64 КиБ — файл целиком в RAM не загружается. + return StreamingResponse(storage.iter_body(stream), media_type=ctype, headers=headers) diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..e52cec81 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,57 @@ +""" +Версионируемые миграции: migrations/NNNN_name.sql. + +- Таблица schema_migrations хранит применённые версии → повторный запуск + ничего не дублирует (идемпотентно). +- Каждая миграция применяется в своей транзакции: либо целиком, либо никак. +- pg_advisory_lock: если стартуют несколько API одновременно, миграции + выполнит только один, остальные подождут. + +Запуск вручную: python -m app.migrate +""" +from __future__ import annotations +import asyncio +import logging +from pathlib import Path + +import psycopg + +from .config import settings + +log = logging.getLogger("messenger.migrate") +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" +LOCK_ID = 824_113_2026 # произвольная константа для advisory lock + + +async def migrate(conninfo: str | None = None) -> list[str]: + applied_now: list[str] = [] + async with await psycopg.AsyncConnection.connect( + conninfo or settings.database_url, + connect_timeout=settings.db_connect_timeout, autocommit=True) as conn: + await conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + try: + await conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY," + " applied_at timestamptz NOT NULL DEFAULT now())") + cur = await conn.execute("SELECT version FROM schema_migrations") + done = {row[0] for row in await cur.fetchall()} + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + version = path.stem + if version in done: + continue + async with conn.transaction(): + await conn.execute(path.read_text(encoding="utf-8")) + await conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", (version,)) + applied_now.append(version) + log.info("migration applied: %s", version) + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + return applied_now + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + result = asyncio.run(migrate()) + print("applied:", result or "nothing (up to date)") diff --git a/app/replay.py b/app/replay.py new file mode 100644 index 00000000..574ac755 --- /dev/null +++ b/app/replay.py @@ -0,0 +1,109 @@ +""" +Реестр failed-задач и безопасный явный replay (lab5, уровень 4). Запуск в контейнере worker: + + python -m app.replay --list # реестр failed (из БД) + python -m app.replay --dlq # последние записи DLQ-stream брокера + python -m app.replay --attachment [--dry-run] [--actor имя] + python -m app.replay --all-failed [--error-code retry_exhausted] [--dry-run] + +Почему «повторить» безопасно: +- replay меняет только job в статусе failed (FOR UPDATE): ready/queued/processing не трогает — + уже успешную задачу второй раз исполнить нельзя; +- job_id тот же, attempt растёт (attempt+1), бюджет попыток — ещё JOB_MAX_ATTEMPTS; +- эффект защищён ключом (attachment_id, pipeline_version, variant) и детерминированными + ключами S3: даже дубль сообщения не создаст второй набор вариантов; +- если исходник в staging уже удалён по retention — replay отказывает (повторять нечего); +- каждое действие пишется в job_events (audit: кто, когда, какая попытка). +""" +from __future__ import annotations + +import argparse +import getpass +import json + +from psycopg_pool import ConnectionPool + +from . import broker +from .config import settings + + +def replay_one(pool: ConnectionPool, attachment_id: str, actor: str, dry_run: bool) -> str: + with pool.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT j.job_id, j.status, j.attempt, j.error_code, a.staging_key, j.pipeline_version " + "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id " + "WHERE j.attachment_id = %s AND j.pipeline_version = %s FOR UPDATE OF j", + (attachment_id, settings.pipeline_version)).fetchone() + if row is None: + return "not found" + job_id, status, attempt, error_code, staging_key, pv = row + if status != "failed": + return f"refused: status is {status} (replay only for failed)" + if not staging_key: + return "refused: source already removed by staging retention" + if dry_run: + return f"would replay job {job_id} ({error_code}), next attempt {attempt + 1}" + nxt = attempt + 1 + conn.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt = %s, max_attempts = %s, " + "error_code = NULL, replays = replays + 1, finished_at = NULL, next_attempt_at = now(), " + "updated_at = now() WHERE job_id = %s", + (nxt, attempt + settings.job_max_attempts, job_id)) + conn.execute("UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = now() " + "WHERE id = %s", (attachment_id,)) + created = conn.execute("SELECT now()").fetchone()[0] + payload = broker.make_payload(job_id, attachment_id, pv, nxt, created) + conn.execute("INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) " + "VALUES (%s, %s, %s, now(), now())", (job_id, nxt, json.dumps(payload))) + conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) " + "VALUES (%s, 'replay', %s, %s, %s)", + (job_id, nxt, actor, f"was failed: {error_code}")) + return f"replayed job {job_id}: attempt {nxt} queued (outbox → broker)" + + +def main() -> None: + ap = argparse.ArgumentParser(description="failed jobs registry and explicit replay") + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--list", action="store_true") + g.add_argument("--dlq", action="store_true") + g.add_argument("--attachment") + g.add_argument("--all-failed", action="store_true") + ap.add_argument("--error-code") + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--actor", default=None) + args = ap.parse_args() + actor = args.actor or f"operator:{getpass.getuser()}" + + if args.dlq: + r = broker.sync_client() + for msg_id, fields in r.xrevrange(settings.dlq_stream, count=20): + print(msg_id, json.dumps(fields, ensure_ascii=False)) + return + + with ConnectionPool(settings.database_url, min_size=1, max_size=2, open=True) as pool: + if args.list: + with pool.connection() as conn: + rows = conn.execute( + "SELECT j.attachment_id, j.job_id, j.attempt, j.error_code, j.finished_at, " + "a.staging_key IS NOT NULL FROM attachment_jobs j JOIN attachments a " + "ON a.id = j.attachment_id WHERE j.status = 'failed' ORDER BY j.finished_at").fetchall() + for aid, jid, att, code, fin, has_src in rows: + print(f"{aid} job={jid} attempt={att} error={code} at={fin:%Y-%m-%d %H:%M:%S} " + f"replayable={'yes' if has_src else 'no'}") + print(f"failed jobs: {len(rows)}") + return + targets = [args.attachment] if args.attachment else [] + if args.all_failed: + with pool.connection() as conn: + q = "SELECT attachment_id FROM attachment_jobs WHERE status = 'failed'" + params: tuple = () + if args.error_code: + q += " AND error_code = %s" + params = (args.error_code,) + targets = [str(r[0]) for r in conn.execute(q, params).fetchall()] + for aid in targets: + print(aid, "→", replay_one(pool, aid, actor, args.dry_run)) + + +if __name__ == "__main__": + main() diff --git a/app/repo.py b/app/repo.py new file mode 100644 index 00000000..655ac18f --- /dev/null +++ b/app/repo.py @@ -0,0 +1,380 @@ +""" +Доступ к данным в PostgreSQL. Только параметризованные запросы (%s): +значения передаются отдельно от текста SQL, инъекция невозможна. +""" +from __future__ import annotations +import base64 +import hashlib +import json +import uuid +from datetime import datetime, timezone + +from psycopg.errors import UniqueViolation + + +def iso(dt: datetime) -> str: + """RFC 3339 UTC с Z — формат контракта.""" + return dt.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +def now() -> datetime: + return datetime.now(timezone.utc) + + +# ---------------------------------------------------------------- курсор +def encode_cursor(scope: str, created_at: datetime, id_) -> str: + raw = json.dumps({"s": scope, "c": iso(created_at), "i": str(id_)}).encode() + return base64.urlsafe_b64encode(raw).decode() + + +def decode_cursor(cursor: str, scope: str) -> tuple[datetime, uuid.UUID]: + """(created_at, id) последнего показанного элемента или ValueError → 400.""" + try: + data = json.loads(base64.urlsafe_b64decode(cursor.encode())) + if data.get("s") != scope: + raise ValueError("cursor scope mismatch") + return datetime.fromisoformat(data["c"].replace("Z", "+00:00")), uuid.UUID(data["i"]) + except (ValueError, KeyError, TypeError, AttributeError) as exc: + raise ValueError("bad cursor") from exc + + +def _page(rows: list, limit: int, scope: str, key): + """rows получены с LIMIT limit+1: лишняя строка = есть следующая страница.""" + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = encode_cursor(scope, *key(page[-1])) if has_more and page else None + return page, next_cursor + + +# ---------------------------------------------------------------- users +USER_COLS = "id, username, display_name, created_at" + + +def user_out(row) -> dict: + return {"id": str(row[0]), "username": row[1], "display_name": row[2], "created_at": iso(row[3])} + + +class UsernameTaken(Exception): + pass + + +async def create_user(conn, username: str, display_name: str, password_hash: str) -> dict: + try: + cur = await conn.execute( + f"INSERT INTO users (id, username, display_name, password_hash, created_at) " + f"VALUES (%s, %s, %s, %s, %s) RETURNING {USER_COLS}", + (uuid.uuid4(), username, display_name, password_hash, now())) + except UniqueViolation as exc: + # Уникальность username гарантирует БД, а не проверка «есть ли такой» + # перед вставкой: две одновременные регистрации не пройдут обе. + raise UsernameTaken() from exc + return user_out(await cur.fetchone()) + + +async def get_user(conn, user_id) -> dict | None: + cur = await conn.execute(f"SELECT {USER_COLS} FROM users WHERE id = %s", (user_id,)) + row = await cur.fetchone() + return user_out(row) if row else None + + +async def get_credentials(conn, username: str): + cur = await conn.execute( + f"SELECT {USER_COLS}, password_hash FROM users WHERE username = %s", (username,)) + return await cur.fetchone() + + +async def update_password_hash(conn, user_id, password_hash: str) -> None: + await conn.execute("UPDATE users SET password_hash = %s WHERE id = %s", (password_hash, user_id)) + + +# ---------------------------------------------------------------- chats +CHAT_SELECT = ( + "SELECT c.id, c.title, c.created_at, " + " (SELECT array_agg(m.user_id ORDER BY m.position) FROM chat_members m WHERE m.chat_id = c.id) " + "FROM chats c ") + + +def chat_out(row) -> dict: + return {"id": str(row[0]), "title": row[1], + "member_ids": [str(x) for x in row[3]], "created_at": iso(row[2])} + + +async def count_existing_users(conn, ids: list[uuid.UUID]) -> int: + cur = await conn.execute("SELECT count(*) FROM users WHERE id = ANY(%s)", (ids,)) + return (await cur.fetchone())[0] + + +async def insert_chat(conn, title: str, member_ids: list[uuid.UUID], creator) -> dict: + """Чат + все участники. Вызывать внутри транзакции: либо всё, либо ничего.""" + chat_id, created = uuid.uuid4(), now() + await conn.execute( + "INSERT INTO chats (id, title, created_by, created_at) VALUES (%s, %s, %s, %s)", + (chat_id, title, creator, created)) + async with conn.cursor() as cur: + await cur.executemany( + "INSERT INTO chat_members (chat_id, user_id, position) VALUES (%s, %s, %s)", + [(chat_id, uid, pos) for pos, uid in enumerate(member_ids)]) + return {"id": str(chat_id), "title": title, + "member_ids": [str(x) for x in member_ids], "created_at": iso(created)} + + +async def get_chat_for_member(conn, chat_id, user_id) -> dict | None: + """Чат, если он существует И пользователь в нём состоит; иначе None (→ 404).""" + cur = await conn.execute( + CHAT_SELECT + "WHERE c.id = %s AND EXISTS " + "(SELECT 1 FROM chat_members x WHERE x.chat_id = c.id AND x.user_id = %s)", + (chat_id, user_id)) + row = await cur.fetchone() + return chat_out(row) if row else None + + +async def list_chats(conn, user_id, limit: int, cursor: str | None): + scope = f"chats:{user_id}" + params: list = [user_id] + where = "" + if cursor: + c_at, c_id = decode_cursor(cursor, scope) + where = "WHERE (c.created_at, c.id) > (%s, %s) " + params += [c_at, c_id] + cur = await conn.execute( + CHAT_SELECT + "JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s " + + where + "ORDER BY c.created_at, c.id LIMIT %s", (*params, limit + 1)) + rows = await cur.fetchall() + page, nxt = _page(rows, limit, scope, key=lambda r: (r[2], r[0])) + return [chat_out(r) for r in page], nxt + + +# ---------------------------------------------------------------- messages +MSG_COLS = "id, chat_id, sender_id, text, created_at" +# lab4: id вложений сообщения в порядке из запроса (пустой массив, если их нет). +MSG_ATTACHMENTS = ( + "COALESCE((SELECT array_agg(ma.attachment_id ORDER BY ma.position) " + "FROM message_attachments ma WHERE ma.message_id = m.id), '{}'::uuid[])") +MSG_SELECT = f"SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, {MSG_ATTACHMENTS} FROM messages m " + + +def message_out(row) -> dict: + return {"id": str(row[0]), "chat_id": str(row[1]), "sender_id": str(row[2]), + "text": row[3], "created_at": iso(row[4]), + "attachment_ids": [str(x) for x in row[5]]} + + +# ---------------------------------------------------------------- идемпотентность +class IdempotencyConflict(Exception): + """Тот же Idempotency-Key, но другое тело запроса → 409.""" + + +class AttachmentNotFound(Exception): + """Вложения нет или оно из другого чата → 404.""" + + +class AttachmentNotReady(Exception): + """Вложение в состоянии failed → 409 (в lab4 статус всегда ready).""" + + +def request_hash(text: str, attachment_ids: list | None = None) -> bytes: + """Хэш канонического тела: порядок ключей JSON не влияет на результат. + Отсутствующий attachment_ids и [] — одно и то же (контракт), поэтому пустой + список в хэш не входит: ключи, сохранённые lab3 для текстовых сообщений, + после обновления дают тот же хэш.""" + body: dict = {"text": text} + if attachment_ids: + body["attachment_ids"] = [str(x) for x in attachment_ids] + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +async def check_attachments(conn, chat_id, attachment_ids: list[uuid.UUID]) -> None: + """Все вложения существуют и принадлежат ЭТОМУ чату; иначе 404 (не раскрываем, + что чужое вложение существует). FOR SHARE: строку нельзя удалить до COMMIT.""" + if not attachment_ids: + return + cur = await conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s) FOR SHARE", + (attachment_ids,)) + found = {row[0]: row for row in await cur.fetchall()} + for aid in attachment_ids: + row = found.get(aid) + if row is None or str(row[1]) != str(chat_id): + raise AttachmentNotFound() + if any(row[2] == "failed" for row in found.values()): + raise AttachmentNotReady() + + +async def get_message(conn, message_id) -> dict: + cur = await conn.execute(MSG_SELECT + "WHERE m.id = %s", (message_id,)) + return message_out(await cur.fetchone()) + + +async def send_message_idempotent(conn, chat_id, sender_id, key: str, text: str, + ttl_seconds: int, + attachment_ids: list[uuid.UUID] | None = None) -> dict: + """Записать сообщение ровно один раз для (пользователь, чат, ключ). + Вызывать внутри транзакции: ключ, сообщение и его связи с вложениями + фиксируются вместе или никак, поэтому ошибка (5xx) не оставляет «записанного результата». + + Конкурентные повторы через разные реплики: второй INSERT того же PK ждёт + COMMIT первого (блокировка уникального индекса в PostgreSQL), затем получает + конфликт и читает уже сохранённый результат.""" + attachment_ids = attachment_ids or [] + rhash = request_hash(text, attachment_ids) + msg_id, created = uuid.uuid4(), now() + cur = await conn.execute( + "INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id, created_at) " + "VALUES (%s, %s, %s, %s, %s, %s) " + "ON CONFLICT (user_id, chat_id, idem_key) DO NOTHING RETURNING 1", + (sender_id, chat_id, key, rhash, msg_id, created)) + if await cur.fetchone() is None: + cur = await conn.execute( + "SELECT request_hash, message_id, created_at < now() - make_interval(secs => %s) " + "FROM idempotency_keys WHERE user_id = %s AND chat_id = %s AND idem_key = %s FOR UPDATE", + (ttl_seconds, sender_id, chat_id, key)) + stored_hash, stored_msg, expired = await cur.fetchone() + if not expired: + if bytes(stored_hash) != rhash: + raise IdempotencyConflict() + return await get_message(conn, stored_msg) # тот же Message, что и в первый раз + # Ключ старше TTL: используем его заново как новый. + await conn.execute( + "UPDATE idempotency_keys SET request_hash = %s, message_id = %s, created_at = %s " + "WHERE user_id = %s AND chat_id = %s AND idem_key = %s", + (rhash, msg_id, created, sender_id, chat_id, key)) + # Вложения проверяются в той же транзакции, что и запись сообщения. + await check_attachments(conn, chat_id, attachment_ids) + await conn.execute( + f"INSERT INTO messages ({MSG_COLS}) VALUES (%s, %s, %s, %s, %s)", + (msg_id, chat_id, sender_id, text, created)) + if attachment_ids: + async with conn.cursor() as c: + await c.executemany( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + [(msg_id, aid, pos) for pos, aid in enumerate(attachment_ids)]) + return {"id": str(msg_id), "chat_id": str(chat_id), "sender_id": str(sender_id), + "text": text, "created_at": iso(created), + "attachment_ids": [str(x) for x in attachment_ids]} + + +async def list_messages(conn, chat_id, limit: int, cursor: str | None): + """Keyset-пагинация по индексу (chat_id, created_at, id): без OFFSET и без скана истории.""" + scope = f"messages:{chat_id}" + params: list = [chat_id] + where = "" + if cursor: + c_at, c_id = decode_cursor(cursor, scope) + where = "AND (m.created_at, m.id) > (%s, %s) " + params += [c_at, c_id] + cur = await conn.execute( + MSG_SELECT + "WHERE m.chat_id = %s " + where + + "ORDER BY m.created_at, m.id LIMIT %s", (*params, limit + 1)) + rows = await cur.fetchall() + page, nxt = _page(rows, limit, scope, key=lambda r: (r[4], r[0])) + return [message_out(r) for r in page], nxt + + +# ---------------------------------------------------------------- вложения (lab4–5) +ATT_COLS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, " + "error_code, created_at, updated_at, object_key") +VARIANT_ORDER = ("thumbnail", "crop", "optimized") + + +def attachment_out(row, variants: list | None = None) -> dict: + """Публичное представление: без object_key, staging, bucket и внутренних адресов. + size_bytes/sha256/content_type всегда описывают ОРИГИНАЛ; у вариантов — свои поля. + Варианты показываются только у ready (статус ставится после записи всех вариантов).""" + return {"id": str(row[0]), "chat_id": str(row[1]), "owner_id": str(row[2]), + "filename": row[3], "content_type": row[4], "size_bytes": row[5], + "sha256": row[6], "status": row[7], + "variants": (variants or []) if row[7] == "ready" else [], + "error_code": row[8] if row[7] == "failed" else None, + "created_at": iso(row[9]), "updated_at": iso(row[10])} + + +async def add_upload_intent(conn, object_key: str, attachment_id) -> None: + """Журнал намерения: запись ДО PUT в S3 (staging). Если процесс упадёт между PUT и + COMMIT метаданных, по этой строке janitor найдёт и удалит объект-сироту.""" + await conn.execute( + "INSERT INTO upload_intents (object_key, attachment_id, created_at) VALUES (%s, %s, now())", + (object_key, attachment_id)) + + +class IntentLost(Exception): + """Журнал загрузки уже забрал janitor (загрузка шла дольше grace period).""" + + +async def commit_attachment(conn, *, attachment_id, chat_id, owner_id, filename: str, + content_type: str, size_bytes: int, sha256: str, + width: int, height: int, object_key: str, staging_key: str, + pipeline_version: int, max_attempts: int) -> tuple[dict, int, dict]: + """lab5. Вызывать внутри ОДНОЙ транзакции. Атомарно фиксирует: + - снятие журнала загрузки (staging-объект теперь «принадлежит» вложению), + - вложение status=queued, + - job (queued, attempt=1), + - строку outbox с сообщением для брокера (transactional outbox). + После COMMIT работа не теряется: оригинал в staging, намерение — в БД. + Возвращает (Attachment, id строки outbox, payload сообщения).""" + from .broker import make_payload # локальный импорт: repo не зависит от redis + cur = await conn.execute( + "DELETE FROM upload_intents WHERE object_key = %s RETURNING 1", (staging_key,)) + if await cur.fetchone() is None: + raise IntentLost() + ts = now() + cur = await conn.execute( + f"INSERT INTO attachments ({ATT_COLS}, width, height, staging_key) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, 'queued', NULL, %s, %s, %s, %s, %s, %s) " + f"RETURNING {ATT_COLS}", + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + ts, ts, object_key, width, height, staging_key)) + att = attachment_out(await cur.fetchone()) + job_id = uuid.uuid4() + await conn.execute( + "INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, " + "max_attempts, created_at, updated_at) VALUES (%s, %s, %s, 'queued', 1, %s, %s, %s)", + (job_id, attachment_id, pipeline_version, max_attempts, ts, ts)) + payload = make_payload(job_id, attachment_id, pipeline_version, 1, ts) + cur = await conn.execute( + "INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) " + "VALUES (%s, 1, %s, %s, %s) RETURNING id", + (job_id, json.dumps(payload), ts, ts)) + outbox_id = (await cur.fetchone())[0] + await conn.execute( + "INSERT INTO job_events (job_id, event, attempt, actor) VALUES (%s, 'accepted', 1, 'api')", + (job_id,)) + return att, outbox_id, payload + + +async def mark_published(conn, outbox_id: int, stream_id: str) -> None: + await conn.execute( + "UPDATE job_outbox SET published_at = now(), stream_id = %s, " + "publish_attempts = publish_attempts + 1 WHERE id = %s AND published_at IS NULL", + (stream_id, outbox_id)) + + +async def get_variants(conn, attachment_id, pipeline_version: int) -> list[dict]: + cur = await conn.execute( + "SELECT variant, content_type, size_bytes, width, height, sha256 FROM attachment_variants " + "WHERE attachment_id = %s AND pipeline_version = %s", (attachment_id, pipeline_version)) + rows = {r[0]: r for r in await cur.fetchall()} + return [{"name": r[0], "content_type": r[1], "size_bytes": r[2], "width": r[3], + "height": r[4], "sha256": r[5]} for name in VARIANT_ORDER if (r := rows.get(name))] + + +async def get_variant(conn, attachment_id, pipeline_version: int, name: str): + """(object_key, content_type, size_bytes, sha256) или None.""" + cur = await conn.execute( + "SELECT object_key, content_type, size_bytes, sha256 FROM attachment_variants " + "WHERE attachment_id = %s AND pipeline_version = %s AND variant = %s", + (attachment_id, pipeline_version, name)) + return await cur.fetchone() + + +async def drop_upload_intent(conn, object_key: str) -> None: + await conn.execute("DELETE FROM upload_intents WHERE object_key = %s", (object_key,)) + + +async def get_attachment_for_member(conn, attachment_id, user_id): + """Строка вложения, если пользователь состоит в его чате; иначе None (→ 404).""" + cur = await conn.execute( + f"SELECT {ATT_COLS} FROM attachments a WHERE a.id = %s AND EXISTS " + "(SELECT 1 FROM chat_members x WHERE x.chat_id = a.chat_id AND x.user_id = %s)", + (attachment_id, user_id)) + return await cur.fetchone() diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..c900880b --- /dev/null +++ b/app/security.py @@ -0,0 +1,46 @@ +""" +Пароли и токены. + +Пароль: Argon2id (password KDF). Он специально медленный и требует память, +поэтому перебор украденных хэшей дорог. Соль своя у каждого хэша. + +Токен сессии: 32 случайных байта (256 бит) из secrets — непрозрачная строка, +не связанная с username/UUID. В БД кладём SHA-256 от токена: для случайного +256-битного значения быстрый хэш достаточен (перебирать нечего), а утечка +таблицы sessions не даёт рабочих токенов. +""" +from __future__ import annotations +import hashlib +import secrets + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +_hasher = PasswordHasher() # Argon2id, параметры по умолчанию библиотеки (RFC 9106 low-memory) + +# Хэш-заглушка: проверяем пароль даже для несуществующего пользователя, +# чтобы по времени ответа нельзя было узнать, существует ли username. +_DUMMY_HASH = _hasher.hash("dummy-password-for-timing") + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password_hash: str | None, password: str) -> bool: + try: + return _hasher.verify(password_hash or _DUMMY_HASH, password) and password_hash is not None + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def needs_rehash(password_hash: str) -> bool: + return _hasher.check_needs_rehash(password_hash) + + +def new_token() -> str: + return secrets.token_urlsafe(32) # 256 бит энтропии + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("utf-8")).digest() diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..5f9ced65 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,168 @@ +""" +Сессии: PostgreSQL — источник истины, Redis/Valkey — кэш поиска сессии. + +Чтение (проверка Bearer): + 1. Ищем в Redis ключ sess:. Нашли и не истёк → пользователь. + 2. Нет в кэше / Redis недоступен → читаем sessions в PostgreSQL + (не отозвана и expires_at > now()) и кладём результат в кэш. + +Политика expiry: срок жизни сессии задаёт PostgreSQL (expires_at). +TTL ключа в Redis = min(SESSION_CACHE_TTL_SECONDS, время до expires_at), +поэтому кэш никогда не продлевает сессию. + +Инвалидация (logout) — атомарно для БД и кэша: + в одной транзакции UPDATE sessions SET revoked_at (строка блокируется), + затем DEL ключа в Redis. Если Redis не ответил — транзакция откатывается и + клиент получает 503: logout НЕ подтверждён, 204 отдаётся только когда + копии отозванной сессии в кэше гарантированно нет. + Гонка «чтение кладёт старую сессию в кэш после DEL» закрыта блокировкой: + чтение из БД идёт с FOR SHARE и кладёт в кэш, не отпуская строку, поэтому + logout ждёт окончания такого чтения и удаляет ключ уже после него. + +Отказ Redis: кэш обходится, всё читается из PostgreSQL (медленнее, но верно). +Flush/рестарт Redis: кэш пустой → сессии восстанавливаются из PostgreSQL. +Отказ PostgreSQL: новые проверки идут в БД → 503, анонимный запрос не принимается. +""" +from __future__ import annotations +import json +import logging +from datetime import datetime, timedelta, timezone + +import redis.asyncio as aioredis +from redis.exceptions import RedisError + +from .config import settings +from .security import new_token, token_hash + +log = logging.getLogger("messenger.sessions") + +_redis: aioredis.Redis | None = None + + +def init_redis() -> None: + global _redis + if settings.redis_url: + _redis = aioredis.Redis.from_url( + settings.redis_url, + socket_timeout=settings.redis_timeout, + socket_connect_timeout=settings.redis_timeout, + decode_responses=True, + ) + + +async def close_redis() -> None: + if _redis is not None: + await _redis.aclose() + + +def redis_enabled() -> bool: + return _redis is not None + + +async def redis_ping() -> bool: + if _redis is None: + return False + try: + return bool(await _redis.ping()) + except (RedisError, OSError): + return False + + +def _key(th: bytes) -> str: + return "sess:" + th.hex() + + +async def _cache_get(th: bytes) -> dict | None: + if _redis is None: + return None + try: + raw = await _redis.get(_key(th)) + except (RedisError, OSError) as exc: + log.warning("redis unavailable on read, bypassing cache: %s", type(exc).__name__) + return None + return json.loads(raw) if raw else None + + +async def _cache_put(th: bytes, user: dict, expires_at: datetime) -> None: + if _redis is None: + return + left = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + ttl = min(settings.session_cache_ttl_seconds, left) + if ttl <= 0: + return + try: + await _redis.set(_key(th), json.dumps({"user": user, "exp": expires_at.isoformat()}), ex=ttl) + except (RedisError, OSError) as exc: + log.warning("redis unavailable on write, skipping cache: %s", type(exc).__name__) + + +class CacheUnavailable(Exception): + """Redis не подтвердил удаление ключа — отзыв сессии нельзя гарантировать.""" + + +async def _cache_delete_strict(th: bytes) -> None: + """Удалить копию сессии из кэша или поднять CacheUnavailable. + Молча проглатывать ошибку нельзя: если Redis «приостановлен», ключ + переживёт сбой и после восстановления отозванный токен снова примут.""" + if _redis is None: + return + try: + await _redis.delete(_key(th)) + except (RedisError, OSError) as exc: + log.warning("redis unavailable on invalidate: %s", type(exc).__name__) + raise CacheUnavailable() from exc + + +async def create_session(conn, user_id) -> tuple[str, datetime]: + token = new_token() + now = datetime.now(timezone.utc) + expires_at = now + timedelta(seconds=settings.session_ttl_seconds) + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at) " + "VALUES (%s, %s, %s, %s)", + (token_hash(token), user_id, now, expires_at)) + return token, expires_at + + +async def resolve(pool, token: str, load_user) -> dict | None: + """Вернёт пользователя (dict) для действующего токена или None.""" + th = token_hash(token) + cached = await _cache_get(th) + if cached is not None: + if datetime.fromisoformat(cached["exp"]) > datetime.now(timezone.utc): + return cached["user"] + return None + async with pool.connection() as conn: + # FOR SHARE: пока эта транзакция не закончилась (вместе с записью в кэш), + # logout не сможет отозвать строку — он дождётся и удалит ключ после нас. + cur = await conn.execute( + "SELECT user_id, expires_at FROM sessions " + "WHERE token_hash = %s AND revoked_at IS NULL AND expires_at > now() " + "FOR SHARE", + (th,)) + row = await cur.fetchone() + if row is None: + return None + user = await load_user(conn, row[0]) + if user is None: + return None + await _cache_put(th, user, row[1]) + return user + + +async def revoke(pool, token: str) -> bool: + """Отозвать сессию. True — была действующая и теперь отозвана (и в БД, и в кэше). + False — сессии нет/уже отозвана. CacheUnavailable — Redis не ответил: + транзакция откатана, сессия осталась действующей, logout не подтверждён.""" + th = token_hash(token) + async with pool.connection() as conn: + async with conn.transaction(): + cur = await conn.execute( + "UPDATE sessions SET revoked_at = now() " + "WHERE token_hash = %s AND revoked_at IS NULL AND expires_at > now() " + "RETURNING 1", (th,)) + if await cur.fetchone() is None: + return False + # Внутри транзакции: ошибка Redis → исключение → ROLLBACK отзыва. + await _cache_delete_strict(th) + return True diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..d1936797 --- /dev/null +++ b/app/storage.py @@ -0,0 +1,284 @@ +""" +Объектное хранилище (S3-совместимое, MinIO) — только БАЙТЫ вложений. + +- Метаданные (кто, какой чат, sha256, размер, тип) — в PostgreSQL, источник истины. +- Ключ объекта назначает сервер: originals/. Имя файла от клиента + в ключ не попадает (нет path traversal и подбора чужих ключей по имени). +- Bucket приватный, API ходит отдельными credentials с правами только на + этот bucket (см. infra/minio/). Root MinIO у API нет. +- boto3 синхронный → все вызовы идут в пуле потоков (не блокируют event loop). +- Таймауты и ограниченные повторы: недоступный S3 даёт 503, а не зависание. +""" +from __future__ import annotations + +import base64 +import hashlib +import io +import logging +import time +from datetime import datetime +from typing import BinaryIO, Iterator + +import boto3 +from botocore.config import Config +from botocore.exceptions import (BotoCoreError, ClientError, ConnectionClosedError, + ConnectTimeoutError, EndpointConnectionError) +from fastapi.concurrency import run_in_threadpool + +from .config import settings + +log = logging.getLogger("messenger.storage") + +ORIGINALS_PREFIX = "originals/" +STAGING_PREFIX = "staging/" +VARIANTS_PREFIX = "variants/" +_client = None + +# Предохранитель (circuit breaker) реплики: если S3 не отвечает на уровне соединения +# (контейнер остановлен, имя не резолвится), следующие S3_BREAKER_SECONDS секунд +# запросы к S3 сразу получают 503, не тратя время на DNS/connect и повторы. +_CONNECTION_ERRORS = (EndpointConnectionError, ConnectTimeoutError, ConnectionClosedError) +_down_until = 0.0 + + +class StorageUnavailable(Exception): + """S3 недоступен/не ответил/отказал → 503.""" + + +class ObjectMissing(Exception): + """Объекта нет (NoSuchKey).""" + + +def object_key(attachment_id) -> str: + """Конечный ключ оригинала. С lab5 объект там появляется, когда worker его опубликует.""" + return f"{ORIGINALS_PREFIX}{attachment_id}" + + +def staging_key(attachment_id) -> str: + """lab5: приватный prefix приёма. До 202 оригинал целиком лежит здесь.""" + return f"{STAGING_PREFIX}{attachment_id}" + + +def variant_key(attachment_id, pipeline_version: int, variant: str, ext: str) -> str: + """Детерминированный ключ: повтор job перезаписывает тот же объект теми же байтами, + второго набора вариантов не появляется.""" + return f"{VARIANTS_PREFIX}{attachment_id}/v{pipeline_version}/{variant}.{ext}" + + +def make_client(): + return boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + region_name=settings.s3_region, + aws_access_key_id=settings.s3_access_key_id or None, + aws_secret_access_key=settings.s3_secret_access_key or None, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, # MinIO: http://s3:9000// + connect_timeout=settings.s3_connect_timeout, + read_timeout=settings.s3_read_timeout, + retries={"max_attempts": settings.s3_max_attempts, "mode": "standard"}, + max_pool_connections=max(10, settings.upload_max_concurrent), + # Контрольную сумму передаём сами (Content-MD5); не включаем + # aws-chunked/CRC по умолчанию новых botocore — совместимость с MinIO. + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + +def init() -> None: + global _client + _client = make_client() + + +def client(): + if _client is None: + init() + return _client + + +def _why(exc: Exception) -> str: + """Тип и код ошибки S3 для лога (без ключей, подписей и адресов).""" + if isinstance(exc, ClientError): + return f"{type(exc).__name__}:{exc.response.get('Error', {}).get('Code', '?')}" + return type(exc).__name__ + + +def _guard() -> None: + if time.monotonic() < _down_until: + raise StorageUnavailable("circuit open") + + +def ensure_available() -> None: + """Быстрый отказ ДО записи журнала загрузки, пока предохранитель открыт.""" + _guard() + + +def _trip(exc: Exception) -> None: + global _down_until + if isinstance(exc, _CONNECTION_ERRORS): + _down_until = time.monotonic() + settings.s3_breaker_seconds + + +def _is_missing(exc: ClientError) -> bool: + code = exc.response.get("Error", {}).get("Code", "") + return code in ("NoSuchKey", "404", "NotFound") + + +# ----------------------------------------------------------- синхронные вызовы +def put_sync(key: str, body: BinaryIO, size: int, content_type: str, + md5_b64: str, sha256_hex: str) -> None: + _guard() + body.seek(0) + try: + client().put_object( + Bucket=settings.s3_bucket, Key=key, Body=body, ContentLength=size, + ContentType=content_type, + ContentMD5=md5_b64, # S3 сверит и отклонит повреждённую передачу + Metadata={"sha256": sha256_hex}, + ) + except (BotoCoreError, ClientError) as exc: + _trip(exc) + log.warning("s3 put failed: %s", _why(exc)) + raise StorageUnavailable() from exc + + +def open_sync(key: str): + """(поток байтов, длина). Поток читается кусками — файл не грузится в RAM целиком.""" + _guard() + try: + obj = client().get_object(Bucket=settings.s3_bucket, Key=key) + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing() from exc + log.warning("s3 get failed: %s", _why(exc)) + raise StorageUnavailable() from exc + except BotoCoreError as exc: + _trip(exc) + log.warning("s3 get failed: %s", _why(exc)) + raise StorageUnavailable() from exc + return obj["Body"], obj["ContentLength"] + + +def iter_body(body, chunk_size: int = 64 * 1024) -> Iterator[bytes]: + try: + yield from body.iter_chunks(chunk_size) + finally: + body.close() + + +def delete_sync(key: str) -> None: + """Идемпотентно: удаление отсутствующего объекта в S3 — тоже успех.""" + _guard() + try: + client().delete_object(Bucket=settings.s3_bucket, Key=key) + except (BotoCoreError, ClientError) as exc: + _trip(exc) + log.warning("s3 delete failed: %s", _why(exc)) + raise StorageUnavailable() from exc + + +def exists_sync(key: str) -> bool: + try: + client().head_object(Bucket=settings.s3_bucket, Key=key) + return True + except ClientError as exc: + if _is_missing(exc): + return False + raise StorageUnavailable() from exc + except BotoCoreError as exc: + raise StorageUnavailable() from exc + + +def list_sync(prefix: str = ORIGINALS_PREFIX) -> Iterator[tuple[str, datetime, int]]: + """(key, LastModified, Size) всех объектов под prefix — постранично, без загрузки всего списка.""" + try: + pages = client().get_paginator("list_objects_v2").paginate( + Bucket=settings.s3_bucket, Prefix=prefix, PaginationConfig={"PageSize": 1000}) + for page in pages: + for obj in page.get("Contents", []): + yield obj["Key"], obj["LastModified"], obj["Size"] + except (BotoCoreError, ClientError) as exc: + raise StorageUnavailable() from exc + + +def head_sync(key: str) -> dict | None: + """{size, sha256} объекта или None, если его нет.""" + _guard() + try: + h = client().head_object(Bucket=settings.s3_bucket, Key=key) + except ClientError as exc: + if _is_missing(exc): + return None + raise StorageUnavailable() from exc + except BotoCoreError as exc: + _trip(exc) + raise StorageUnavailable() from exc + return {"size": h["ContentLength"], "sha256": h.get("Metadata", {}).get("sha256")} + + +def download_sync(key: str, fileobj, limit: int) -> int: + """Скачать объект в файл кусками; больше limit байт — ошибка (защита worker).""" + body, length = open_sync(key) + if length > limit: + body.close() + raise ValueError("object larger than limit") + total = 0 + for chunk in iter_body(body, 256 * 1024): + total += len(chunk) + if total > limit: + raise ValueError("object larger than limit") + fileobj.write(chunk) + return total + + +def copy_sync(src_key: str, dst_key: str) -> None: + """Серверное копирование внутри bucket (staging → originals). Идемпотентно.""" + _guard() + try: + client().copy_object(Bucket=settings.s3_bucket, Key=dst_key, + CopySource={"Bucket": settings.s3_bucket, "Key": src_key}, + MetadataDirective="COPY") + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing() from exc + log.warning("s3 copy failed: %s", _why(exc)) + raise StorageUnavailable() from exc + except BotoCoreError as exc: + _trip(exc) + log.warning("s3 copy failed: %s", _why(exc)) + raise StorageUnavailable() from exc + + +def put_file_sync(key: str, path: str, content_type: str, sha256_hex: str) -> None: + """Загрузить небольшой файл варианта (≤ нескольких МиБ) с проверкой Content-MD5.""" + with open(path, "rb") as f: + data = f.read() + md5_b64 = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode() + put_sync(key, io.BytesIO(data), len(data), content_type, md5_b64, sha256_hex) + + +def ping_sync() -> bool: + try: + client().head_bucket(Bucket=settings.s3_bucket) + return True + except (BotoCoreError, ClientError): + return False + + +# ----------------------------------------------------------- async-обёртки +async def put(key, body, size, content_type, md5_b64, sha256_hex) -> None: + await run_in_threadpool(put_sync, key, body, size, content_type, md5_b64, sha256_hex) + + +async def open_object(key: str): + return await run_in_threadpool(open_sync, key) + + +async def delete(key: str) -> None: + await run_in_threadpool(delete_sync, key) + + +async def ping() -> bool: + return await run_in_threadpool(ping_sync) diff --git a/app/tools.py b/app/tools.py new file mode 100644 index 00000000..7953479d --- /dev/null +++ b/app/tools.py @@ -0,0 +1,313 @@ +""" +Служебные проверки хранилища для тестов и защиты (не HTTP API). +Запуск внутри контейнера api (он в сети storage): + + python -m app.tools private-check # bucket приватный, права API ограничены + python -m app.tools stats # число объектов / attachments / upload_intents + python -m app.tools state # объекты staging/originals, метаданные, журнал + python -m app.tools job # lab5: job, попытки, события, варианты, объекты S3 + python -m app.tools republish # lab5: отправить ДУБЛЬ сообщения job в брокер + python -m app.tools queue # lab5: длина stream, pending, DLQ, outbox + python -m app.tools chat-attachments # lab5: вложения чата и их статусы (для тестов) + python -m app.tools limits-selftest # lab5: лимиты памяти/времени/пикселей worker + python -m app.tools find-local # нет ли на диске реплики копии файла + +Секреты не печатаются. Ключи объектов печатаются только для служебного prefix. +""" +from __future__ import annotations + +import json +import sys +import urllib.error +import urllib.request +import uuid + +import psycopg +from botocore.exceptions import ClientError + +from . import storage +from .config import settings + + +def _anon(method: str, path: str, data: bytes | None = None) -> int: + """HTTP-запрос к MinIO БЕЗ подписи (как посторонний клиент).""" + req = urllib.request.Request(settings.s3_endpoint.rstrip("/") + path, data=data, method=method) + try: + with urllib.request.urlopen(req, timeout=5) as r: + return r.status + except urllib.error.HTTPError as e: + return e.code + + +def _denied(fn) -> str: + try: + fn() + return "ALLOWED" + except ClientError as exc: + return exc.response.get("Error", {}).get("Code", "error") + + +def private_check() -> int: + s3 = storage.client() + bucket = settings.s3_bucket + probe = f"{storage.ORIGINALS_PREFIX}privacy-probe-{uuid.uuid4()}" + results: list[tuple[str, str, bool]] = [] + + def check(name: str, observed, ok: bool) -> None: + results.append((name, str(observed), ok)) + + s3.put_object(Bucket=bucket, Key=probe, Body=b"probe", ContentType="text/plain") + try: + # Позитивный контроль: объект существует и API-пользователь его читает. + body = s3.get_object(Bucket=bucket, Key=probe)["Body"].read() + check("API credentials: GET own object", "200", body == b"probe") + # Аноним: ни чтения, ни листинга, ни записи. + st = _anon("GET", f"/{bucket}/{probe}") + check("anonymous GET object", st, st == 403) + st = _anon("GET", f"/{bucket}?list-type=2") + check("anonymous LIST bucket", st, st == 403) + st = _anon("PUT", f"/{bucket}/{storage.ORIGINALS_PREFIX}anon-{uuid.uuid4()}", b"x") + check("anonymous PUT object", st, st == 403) + # Ограниченные права API-пользователя: только объекты своего bucket. + # MinIO без s3:ListAllMyBuckets либо отказывает, либо показывает только + # бакеты, на которые есть права, — чужие бакеты API-пользователь не видит. + try: + names = sorted(b["Name"] for b in s3.list_buckets().get("Buckets", [])) + r = "only " + ",".join(names) if names else "empty" + ok = set(names) <= {bucket} + except ClientError as exc: + r = exc.response.get("Error", {}).get("Code", "error") + ok = r == "AccessDenied" + check("API credentials: ListBuckets (no foreign buckets)", r, ok) + r = _denied(lambda: s3.create_bucket(Bucket="lab4-forbidden-" + uuid.uuid4().hex[:8])) + check("API credentials: CreateBucket", r, r == "AccessDenied") + public = json.dumps({"Version": "2012-10-17", "Statement": [{ + "Effect": "Allow", "Principal": {"AWS": ["*"]}, "Action": ["s3:GetObject"], + "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) + r = _denied(lambda: s3.put_bucket_policy(Bucket=bucket, Policy=public)) + check("API credentials: PutBucketPolicy (make public)", r, r == "AccessDenied") + r = _denied(lambda: s3.delete_bucket(Bucket=bucket)) + check("API credentials: DeleteBucket", r, r == "AccessDenied") + finally: + s3.delete_object(Bucket=bucket, Key=probe) + + width = max(len(n) for n, _, _ in results) + for name, observed, ok in results: + print(f" {'ok ' if ok else 'FAIL'} {name.ljust(width)} -> {observed}") + failed = sum(1 for _, _, ok in results if not ok) + print(f"\n{len(results) - failed} passed, {failed} failed") + return 1 if failed else 0 + + +def _db(): + return psycopg.connect(settings.database_url, connect_timeout=3) + + +def stats() -> int: + # Оригиналы и staging (lab5): сирота может остаться в любом из двух prefix. + objects = sum(1 for _ in storage.list_sync(storage.ORIGINALS_PREFIX)) \ + + sum(1 for _ in storage.list_sync(storage.STAGING_PREFIX)) + with _db() as conn: + attachments = conn.execute("SELECT count(*) FROM attachments").fetchone()[0] + intents = conn.execute("SELECT count(*) FROM upload_intents").fetchone()[0] + print(json.dumps({"objects": objects, "attachments": attachments, "upload_intents": intents})) + return 0 + + +def state(attachment_id: str) -> int: + aid = uuid.UUID(attachment_id) + original, staging = storage.object_key(aid), storage.staging_key(aid) + with _db() as conn: + att = conn.execute("SELECT status FROM attachments WHERE id = %s", (aid,)).fetchone() + intent = conn.execute("SELECT 1 FROM upload_intents WHERE attachment_id = %s", (aid,)).fetchone() + print(json.dumps({"staging_object_exists": storage.exists_sync(staging), + "original_object_exists": storage.exists_sync(original), + "attachment_row": att is not None, + "attachment_status": att[0] if att else None, + "upload_intent": intent is not None})) + return 0 + + +def job(attachment_id: str) -> int: + """Полное состояние обработки одного вложения (без секретов).""" + aid = uuid.UUID(attachment_id) + with _db() as conn: + j = conn.execute( + "SELECT job_id, status, attempt, max_attempts, error_code, last_error, replays, " + "lease_owner, lease_until FROM attachment_jobs WHERE attachment_id = %s", (aid,)).fetchone() + a = conn.execute("SELECT status, error_code, staging_key IS NOT NULL FROM attachments " + "WHERE id = %s", (aid,)).fetchone() + variants = conn.execute( + "SELECT variant, width, height, size_bytes, object_key FROM attachment_variants " + "WHERE attachment_id = %s ORDER BY variant", (aid,)).fetchall() + events = conn.execute( + "SELECT to_char(at, 'HH24:MI:SS.MS'), event, attempt, actor, coalesce(detail, '') " + "FROM job_events e WHERE job_id = %s ORDER BY id", (j[0],)).fetchall() if j else [] + outbox = conn.execute("SELECT attempt, published_at IS NOT NULL FROM job_outbox WHERE job_id = %s " + "ORDER BY id", (j[0],)).fetchall() if j else [] + objs = sorted(k for k, _, _ in storage.list_sync(f"{storage.VARIANTS_PREFIX}{aid}/")) + print(json.dumps({ + "attachment": {"status": a[0], "error_code": a[1], "staging_present": a[2]} if a else None, + "job": {"job_id": str(j[0]), "status": j[1], "attempt": j[2], "max_attempts": j[3], + "error_code": j[4], "last_error": j[5], "replays": j[6], "lease_owner": j[7], + "lease_until": j[8].isoformat() if j[8] else None} if j else None, + "variant_rows": [{"name": v[0], "size": f"{v[1]}x{v[2]}", "bytes": v[3]} for v in variants], + "variant_objects_in_s3": len(objs), + "original_object_exists": storage.exists_sync(storage.object_key(aid)), + "staging_object_exists": storage.exists_sync(storage.staging_key(aid)), + "outbox": [{"attempt": o[0], "published": o[1]} for o in outbox], + "events": [" ".join(str(x) for x in e if x not in (None, "")) for e in events], + }, ensure_ascii=False, indent=2)) + return 0 + + +def republish(attachment_id: str) -> int: + """Отправить в брокер копию сообщения ТЕКУЩЕЙ попытки job (проверка повторной доставки).""" + from . import broker + aid = uuid.UUID(attachment_id) + with _db() as conn: + j = conn.execute("SELECT job_id, pipeline_version, attempt, created_at FROM attachment_jobs " + "WHERE attachment_id = %s", (aid,)).fetchone() + if j is None: + print("job not found") + return 1 + payload = broker.make_payload(j[0], aid, j[1], j[2], j[3]) + print("published duplicate:", broker.publish_sync(broker.sync_client(), payload), json.dumps(payload)) + return 0 + + +def chat_attachments(chat_id: str) -> int: + with _db() as conn: + rows = conn.execute("SELECT id, status FROM attachments WHERE chat_id = %s ORDER BY created_at", + (uuid.UUID(chat_id),)).fetchall() + print(json.dumps([{"id": str(r[0]), "status": r[1]} for r in rows])) + return 0 + + +def limits_selftest() -> int: + """Защита worker от «тяжёлых» изображений проверяется на настоящем дочернем процессе + (тот же код и те же лимиты, что в worker), без API — API такие файлы и так отсекает.""" + import multiprocessing + import tempfile + import time + from pathlib import Path + from PIL import Image + from . import imaging + ctx = multiprocessing.get_context("forkserver") + results = [] + + def run(name, src, ctype, max_pixels, memory_mb, timeout, slow=0.0, expect=""): + d = Path(tempfile.mkdtemp()) + res = d / "r.json" + (d / "out").mkdir() + t0 = time.monotonic() + p = ctx.Process(target=imaging.child_main, args=(str(src), str(d / "out"), ctype, max_pixels, + memory_mb, int(timeout) + 5, str(res), slow)) + p.start() + p.join(timeout) + killed = p.is_alive() + if killed: + p.kill() + p.join() + got = "timeout(killed)" if killed else (json.loads(res.read_text()) if res.exists() else f"exit {p.exitcode}") + outcome = got if isinstance(got, str) else ("ok" if got["ok"] else f'{got["code"]}: {got["detail"]}') + ok = expect in outcome + results.append(ok) + print(f" {'ok ' if ok else 'FAIL'} {name}: {outcome} ({time.monotonic() - t0:.1f}s)") + + with tempfile.TemporaryDirectory() as tmp: + big = Path(tmp) / "big.png" # 6000×6000 = 36 Мп: в пределах 40 Мп, но ~100+ МБ после распаковки + Image.new("RGB", (6000, 6000), (200, 30, 30)).save(big, optimize=False, compress_level=1) + small = Path(tmp) / "small.png" + Image.new("RGBA", (320, 200), (0, 0, 255, 128)).save(small) + corrupt = Path(tmp) / "corrupt.png" + corrupt.write_bytes(small.read_bytes()[:200]) + run("normal 320x200 with alpha", small, "image/png", 40_000_000, 768, 30, expect="ok") + run("pixel limit (36 Mp > limit 10 Mp)", big, "image/png", 10_000_000, 768, 30, expect="invalid_image: pixel limit") + run("memory limit (36 Mp, RLIMIT_AS 120 MiB)", big, "image/png", 40_000_000, 120, 30, expect="invalid_image") + run("corrupt/truncated PNG", corrupt, "image/png", 40_000_000, 768, 30, expect="invalid_image") + run("time limit (hangs 10 s, timeout 2 s)", small, "image/png", 40_000_000, 768, 2, slow=10, expect="timeout") + failed = results.count(False) + print(f"\n{len(results) - failed} passed, {failed} failed") + return 1 if failed else 0 + + +def queue() -> int: + from . import broker + r = broker.sync_client() + try: + groups = r.xinfo_groups(settings.jobs_stream) + except Exception: # noqa: BLE001 + groups = [] + with _db() as conn: + pending_outbox = conn.execute("SELECT count(*) FROM job_outbox WHERE published_at IS NULL").fetchone()[0] + by_status = dict(conn.execute("SELECT status, count(*) FROM attachment_jobs GROUP BY status").fetchall()) + print(json.dumps({ + "stream_length": r.xlen(settings.jobs_stream) if r.exists(settings.jobs_stream) else 0, + "groups": [{"name": g["name"], "consumers": g["consumers"], "pending": g["pending"], + "lag": g.get("lag")} for g in groups], + "dlq_length": r.xlen(settings.dlq_stream) if r.exists(settings.dlq_stream) else 0, + "outbox_unpublished": pending_outbox, + "jobs_by_status": by_status, + }, ensure_ascii=False)) + return 0 + + +def find_local(digest: str) -> int: + """Обойти файловую систему контейнера API и найти файлы с таким SHA-256. + Доказательство, что оригинал не хранится на реплике (только в S3).""" + import hashlib + import os + skip = ("/proc", "/sys", "/dev", "/usr", "/lib", "/bin", "/sbin", "/etc", "/var/lib/dpkg") + scanned, found = 0, [] + for root, dirs, files in os.walk("/"): + if root.startswith(skip): + dirs[:] = [] + continue + for name in files: + path = os.path.join(root, name) + try: + if not os.path.isfile(path) or os.path.getsize(path) > 32 * 1024 * 1024: + continue + h = hashlib.sha256() + with open(path, "rb") as f: + for block in iter(lambda: f.read(1 << 20), b""): + h.update(block) + scanned += 1 + if h.hexdigest() == digest: + found.append(path) + except OSError: + continue + print(json.dumps({"files_scanned": scanned, "copies_found": found})) + return 1 if found else 0 + + +def main(argv: list[str]) -> int: + if not argv: + print(__doc__) + return 2 + cmd, args = argv[0], argv[1:] + if cmd == "private-check": + return private_check() + if cmd == "stats": + return stats() + if cmd == "state" and len(args) == 1: + return state(args[0]) + if cmd == "job" and len(args) == 1: + return job(args[0]) + if cmd == "republish" and len(args) == 1: + return republish(args[0]) + if cmd == "queue": + return queue() + if cmd == "chat-attachments" and len(args) == 1: + return chat_attachments(args[0]) + if cmd == "limits-selftest": + return limits_selftest() + if cmd == "find-local" and len(args) == 1: + return find_local(args[0]) + print(__doc__) + return 2 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/app/uploads.py b/app/uploads.py new file mode 100644 index 00000000..e83f2669 --- /dev/null +++ b/app/uploads.py @@ -0,0 +1,106 @@ +""" +Приём тела загрузки (lab4): потоково, с лимитами, без накопления файла в RAM. + +- Content-Length больше лимита → 413 сразу, тело не читается. +- Тело читается кусками (как пришли из сокета) и одновременно: + считается SHA-256 и MD5, растёт счётчик байт. Как только байт больше лимита — + 413, чтение прекращается (работает и для chunked без Content-Length). +- Байты копятся в SpooledTemporaryFile: первые UPLOAD_SPOOL_MEMORY_BYTES в памяти, + дальше — во временном файле. Это буфер одного запроса, а не хранилище: + файл удаляется при закрытии (в finally), оригинал живёт только в S3. +- Общий таймаут на всё тело: «медленный» клиент не держит слот вечно. +""" +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import tempfile +import unicodedata +from dataclasses import dataclass +from typing import BinaryIO + +from starlette.requests import ClientDisconnect, Request + +ALLOWED_TYPES = ("image/png", "image/jpeg") +# Управляющие символы направления текста: «photo‮gnp.exe» выглядит как «photoexe.png». +_BIDI_CONTROLS = set("‎‏‪‫‬‭‮⁦⁧⁨⁩") + + +class UploadError(Exception): + def __init__(self, status: int, code: str, message: str): + self.status, self.code, self.message = status, code, message + + +def validate_filename(name: str | None) -> str: + """filename — только отображаемое имя. Отклоняем всё, что похоже на путь + или прячет настоящий вид имени. Никаких «исправлений»: либо как есть, либо 400.""" + if name is None or not (1 <= len(name) <= 255) or not name.strip(): + raise UploadError(400, "invalid_request", "filename must be 1..255 characters") + if name in (".", ".."): + raise UploadError(400, "invalid_request", "filename is not allowed") + for ch in name: + if ch in "/\\" or ch == "\x00" or ch in _BIDI_CONTROLS \ + or unicodedata.category(ch) in ("Cc", "Cs"): + raise UploadError(400, "invalid_request", "filename contains forbidden characters") + return name + + +def declared_type(request: Request) -> str: + media = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if media not in ALLOWED_TYPES: + raise UploadError(415, "unsupported_media_type", "Content-Type must be image/png or image/jpeg") + return media + + +def check_declared_length(request: Request, limit: int) -> int | None: + raw = request.headers.get("content-length") + if raw is None: + return None + if not raw.isdigit(): + raise UploadError(400, "invalid_request", "bad Content-Length") + length = int(raw) + if length > limit: + # Отказ до чтения тела: ни байта сверх заголовков не принимаем. + raise UploadError(413, "payload_too_large", f"file is larger than {limit} bytes") + return length + + +@dataclass +class Received: + file: BinaryIO + size: int + sha256_hex: str + md5_b64: str + + +async def receive(request: Request, limit: int, timeout: float, spool_memory: int) -> Received: + spool = tempfile.SpooledTemporaryFile(max_size=spool_memory, prefix="upload-") + sha, md5 = hashlib.sha256(), hashlib.md5(usedforsecurity=False) + size = 0 + try: + async with asyncio.timeout(timeout): + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > limit: + raise UploadError(413, "payload_too_large", f"file is larger than {limit} bytes") + sha.update(chunk) + md5.update(chunk) + spool.write(chunk) + except TimeoutError: + spool.close() + raise UploadError(400, "invalid_request", "request body was not received in time") from None + except ClientDisconnect: + # Прерванная загрузка: в S3 ничего не записано, буфер удаляется здесь. + spool.close() + raise UploadError(400, "invalid_request", "upload was interrupted") from None + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise UploadError(400, "invalid_request", "request body is empty") + spool.seek(0) + return Received(spool, size, sha.hexdigest(), base64.b64encode(md5.digest()).decode()) diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..0e49ee8e --- /dev/null +++ b/app/worker.py @@ -0,0 +1,584 @@ +""" +Worker фоновой обработки вложений (lab5). Отдельный процесс/контейнер: `python -m app.worker`. + +Поток одной задачи (сообщение брокера = только ID, см. app/broker.py): + + XREADGROUP ─► claim в БД (FOR UPDATE): queued → processing, lease_token, lease_until + ─► скачать оригинал из staging, сверить sha256 с БД (доверенные метаданные) + ─► дочерний процесс: thumbnail / crop / optimized (лимиты памяти и времени) + ─► S3: copy staging → originals/, PUT variants//v1/* (детерминированные ключи) + ─► COMMIT результата (только владелец аренды): variants ON CONFLICT DO NOTHING, + attachment и job → ready + ─► XACK (+XDEL) ─► удалить staging (после безопасной фиксации) + +Ошибки: + постоянные (битое изображение, sha256 не совпал, исходника нет) → failed + error_code; + временные (S3/БД недоступны, таймаут обработки) → queued, attempt+1, backoff с jitter + через outbox (available_at в будущем); попытки кончились → failed retry_exhausted. + +Отказы worker: + умер до XACK → сообщение остаётся в PEL брокера; после рестарта worker читает свой PEL, + чужой брошенный PEL забирает XAUTOCLAIM (CLAIM_IDLE_SECONDS); + завис/умер с арендой → аренду никто не продлевает, после истечения задачу + забирает другой (lease takeover) или reaper переводит её в queued с новой попыткой. + Устаревший worker после пробуждения ничего не перезапишет: COMMIT проверяет lease_token. + +Поток maintenance (в каждом worker, безопасно параллельно — FOR UPDATE SKIP LOCKED): + outbox relay, reaper просроченных аренд, очистка staging, heartbeat-файл для healthcheck. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import multiprocessing +import os +import random +import shutil +import signal +import tempfile +import threading +import time +import uuid +from dataclasses import dataclass +from pathlib import Path + +import psycopg +import redis +from psycopg_pool import ConnectionPool + +from . import broker, imaging, storage +from .config import settings + +log = logging.getLogger("messenger.worker") + + +class Transient(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail + + +class Permanent(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail + + +class LeaseLost(Exception): + """Аренду забрали (истекла и задачу взял другой) — результат фиксировать нельзя.""" + + +@dataclass +class Job: + job_id: str + attachment_id: str + pipeline_version: int + attempt: int + max_attempts: int + token: str + staging_key: str | None + object_key: str + sha256: str + size_bytes: int + content_type: str + + +def backoff_seconds(attempt: int) -> float: + """Экспоненциальный backoff с jitter («equal jitter»): половина задержки + фиксирована, половина случайна — повторы разных задач не идут синхронной волной.""" + d = min(settings.job_backoff_max_seconds, settings.job_backoff_base_seconds * 2 ** (attempt - 1)) + return d / 2 + random.uniform(0, d / 2) + + +class Worker: + def __init__(self) -> None: + self.stop = threading.Event() + self.name = settings.instance_id + self.pool = ConnectionPool( + conninfo=settings.database_url, min_size=1, + max_size=settings.worker_concurrency + 3, timeout=settings.db_pool_timeout, + kwargs={"connect_timeout": settings.db_connect_timeout, + "options": f"-c statement_timeout={settings.db_statement_timeout_ms}", + "application_name": "messenger-worker"}, + check=ConnectionPool.check_connection, open=False) + self.r = broker.sync_client(read_timeout=settings.broker_timeout + 5) + self.mp = multiprocessing.get_context("forkserver") + self.mp.set_forkserver_preload(["app.imaging"]) + self.work_root = Path(tempfile.gettempdir()) / "worker-jobs" + + # ================================================================ запуск + def run(self) -> None: + signal.signal(signal.SIGTERM, lambda *_: self.stop.set()) + signal.signal(signal.SIGINT, lambda *_: self.stop.set()) + self.pool.open(wait=False) + storage.init() + shutil.rmtree(self.work_root, ignore_errors=True) # мусор прошлой жизни контейнера + self.work_root.mkdir(parents=True, exist_ok=True) + threads = [threading.Thread(target=self.maintenance_loop, name="maintenance", daemon=True)] + for i in range(settings.worker_concurrency): + threads.append(threading.Thread(target=self.consume_loop, args=(f"{self.name}-{i}",), + name=f"consumer-{i}", daemon=True)) + for t in threads: + t.start() + log.info("worker %s started: concurrency=%d max_attempts=%d timeout=%ss lease=%ss", + self.name, settings.worker_concurrency, settings.job_max_attempts, + settings.job_timeout_seconds, settings.job_lease_seconds) + while not self.stop.is_set(): + self.stop.wait(1) + log.info("worker %s stopping: finishing current jobs", self.name) + for t in threads[1:]: + t.join(timeout=settings.job_timeout_seconds + 10) + self.pool.close() + + # ================================================================ потребитель + def consume_loop(self, consumer: str) -> None: + """Ограниченная конкуренция: каждый поток берёт по ОДНОМУ сообщению (COUNT 1), + потоков WORKER_CONCURRENCY → не больше N задач на процесс (аналог prefetch).""" + last_scan = 0.0 + while not self.stop.is_set(): + try: + self._ensure_group() + batch = [] + if time.monotonic() - last_scan > 10: + last_scan = time.monotonic() + # 1) свои неподтверждённые (worker перезапустился в том же контейнере); + resp = self.r.xreadgroup(settings.jobs_group, consumer, + {settings.jobs_stream: "0"}, count=10) + batch += [m for _, msgs in resp for m in msgs] + # 2) брошенные другими (их процесс умер до XACK). + claimed = self.r.xautoclaim(settings.jobs_stream, settings.jobs_group, consumer, + min_idle_time=int(settings.claim_idle_seconds * 1000), + start_id="0-0", count=5) + batch += [m for m in claimed[1] if m] + if not batch: + resp = self.r.xreadgroup(settings.jobs_group, consumer, + {settings.jobs_stream: ">"}, count=1, block=2000) + batch = [m for _, msgs in resp for m in msgs] + for msg_id, fields in batch: + if self.stop.is_set(): + break + if self.handle(consumer, msg_id, fields) == "ack": + self._ack(msg_id) + except redis.ResponseError as exc: + if "NOGROUP" in str(exc): # брокер пересоздан без данных — создаём группу снова + self._group_ok = False + log.warning("broker error (%s), retry in 1s", str(exc)[:80]) + self.stop.wait(1) + except redis.RedisError as exc: + log.warning("broker unavailable (%s), retry in 1s", type(exc).__name__) + self.stop.wait(1) + except Exception: # noqa: BLE001 — поток не должен умирать + log.exception("consumer loop error") + self.stop.wait(1) + + def _ensure_group(self) -> None: + if getattr(self, "_group_ok", False): + return + broker.ensure_group(self.r) + self._group_ok = True + + def _ack(self, msg_id: str) -> None: + # XACK = consumer ack: результат уже устойчиво зафиксирован в БД и S3. + self.r.xack(settings.jobs_stream, settings.jobs_group, msg_id) + self.r.xdel(settings.jobs_stream, msg_id) + + # ================================================================ одна задача + def handle(self, consumer: str, msg_id: str, fields: dict | None) -> str: + raw = (fields or {}).get("job") + if raw is None: # запись удалена из stream (XDEL) — подтверждаем + return "ack" + try: + payload = broker.parse(raw) + except broker.BadMessage as exc: + log.error("bad message %s → DLQ: %s", msg_id, exc) + broker.to_dlq(self.r, raw, f"bad_message: {exc}", msg_id) + return "ack" + try: + decision, job = self.claim(payload, consumer) + except Exception as exc: # noqa: BLE001 — БД недоступна: не ack, повтор позже + log.warning("claim failed for job %s: %s", payload["job_id"], type(exc).__name__) + self.stop.wait(1) + return "leave" + if decision != "process": + log.info("job=%s attempt=%s msg=%s → %s", payload["job_id"], payload["attempt"], + msg_id, decision) + if decision == "poison": + broker.to_dlq(self.r, raw, "unknown job", msg_id) + return "leave" if decision == "busy" else "ack" + return self.process(job, msg_id) + + def claim(self, p: dict, consumer: str) -> tuple[str, Job | None]: + """Таблица решений (сообщение несёт attempt): + job нет / не то вложение → poison (DLQ, ack) + ready → already_ready (no-op, ack): повторная доставка + failed → failed (ack); вернуть может только replay + attempt сообщения ≠ attempt job → stale (ack): устаревшая попытка + queued → берём в работу + processing, аренда истекла → takeover той же попытки + processing, аренда жива → busy: не ack, проверим позже (другой работает)""" + with self.pool.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT j.status, j.attempt, j.max_attempts, j.lease_until < now(), " + "j.pipeline_version, j.attachment_id, a.staging_key, a.object_key, a.sha256, " + "a.size_bytes, a.content_type " + "FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id " + "WHERE j.job_id = %s FOR UPDATE OF j", (p["job_id"],)).fetchone() + if row is None or str(row[5]) != p["attachment_id"] or row[4] != p["pipeline_version"]: + return "poison", None + status, attempt, max_attempts, expired = row[0], row[1], row[2], row[3] + if status == "ready": + return "already_ready", None + if status == "failed": + return "failed", None + if p["attempt"] != attempt: + return "stale", None + if status == "processing" and not expired: + return "busy", None + event = "lease_takeover" if status == "processing" else "claimed" + token = str(uuid.uuid4()) + conn.execute( + "UPDATE attachment_jobs SET status = 'processing', lease_token = %s, lease_owner = %s, " + "lease_until = now() + make_interval(secs => %s), updated_at = now() WHERE job_id = %s", + (token, consumer, settings.job_lease_seconds, p["job_id"])) + conn.execute("UPDATE attachments SET status = 'processing', updated_at = now() " + "WHERE id = %s AND status <> 'ready'", (p["attachment_id"],)) + conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) VALUES (%s, %s, %s, %s)", + (p["job_id"], event, attempt, consumer)) + return "process", Job(p["job_id"], p["attachment_id"], row[4], attempt, max_attempts, token, + row[6], row[7], row[8], row[9], row[10]) + + def process(self, job: Job, msg_id: str) -> str: + log.info("job=%s attachment=%s attempt=%d/%d msg=%s: processing", job.job_id, + job.attachment_id, job.attempt, job.max_attempts, msg_id) + lost = threading.Event() + frozen = threading.Event() + done = threading.Event() + hb = threading.Thread(target=self._heartbeat, args=(job, lost, frozen, done), daemon=True) + hb.start() + workdir = Path(tempfile.mkdtemp(prefix=f"{job.attachment_id}-", dir=self.work_root)) + try: + if self._fault("freeze-after-claim"): + # Имитация зависшего worker: аренду не продлеваем, «спим» дольше аренды. + frozen.set() + log.warning("FAULT INJECTION: freeze-after-claim, sleeping %ss", 3 * settings.job_lease_seconds) + time.sleep(3 * settings.job_lease_seconds) + self._check_lease(job) + variants = self._build(job, workdir, lost) + self._check_lease(job) # не пишем в S3, если задачу уже забрали + self._publish_objects(job, variants) + if self._fault("crash-after-s3-write"): + log.warning("FAULT INJECTION: crash-after-s3-write (before COMMIT and ack)") + os._exit(1) + self._finalize(job, variants) + if self._fault("crash-after-commit-before-ack"): + log.warning("FAULT INJECTION: crash-after-commit-before-ack") + os._exit(1) + log.info("job=%s attachment=%s attempt=%d: ready", job.job_id, job.attachment_id, job.attempt) + done.set() + self._ack(msg_id) + self._drop_staging(job) + return "acked" + except LeaseLost: + log.warning("job=%s attempt=%d: lease lost, result discarded (another worker owns it)", + job.job_id, job.attempt) + return "ack" + except (Transient, Permanent) as exc: + return self._failure(job, exc, msg_id) + except storage.StorageUnavailable as exc: + return self._failure(job, Transient("storage_unavailable", type(exc).__name__), msg_id) + except storage.ObjectMissing: + return self._failure(job, Permanent("processing_failed", "staging object is missing"), msg_id) + except psycopg.Error as exc: + return self._failure(job, Transient("database_unavailable", type(exc).__name__), msg_id) + finally: + done.set() + hb.join(timeout=2) + shutil.rmtree(workdir, ignore_errors=True) + + def _heartbeat(self, job: Job, lost: threading.Event, frozen: threading.Event, + done: threading.Event) -> None: + """Живой worker продлевает аренду каждые lease/3 с — так «долгая» задача + отличается от «зависшей»: у долгой аренда свежая, у мёртвой/замёрзшей — истекает.""" + interval = settings.job_lease_seconds / 3 + while not done.wait(interval): + if frozen.is_set(): + continue + try: + with self.pool.connection() as conn: + ok = conn.execute( + "UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), " + "updated_at = now() WHERE job_id = %s AND lease_token = %s " + "AND status = 'processing' RETURNING 1", + (settings.job_lease_seconds, job.job_id, job.token)).fetchone() + if ok is None: + lost.set() + return + except psycopg.Error as exc: + log.warning("lease renew failed: %s", type(exc).__name__) + + # ---------------------------------------------------------------- шаги + def _check_lease(self, job: Job) -> None: + with self.pool.connection() as conn: + row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s", + (job.job_id,)).fetchone() + if row is None or row[0] != "processing" or str(row[1]) != job.token: + raise LeaseLost() + + def _build(self, job: Job, workdir: Path, lost: threading.Event) -> dict: + if not job.staging_key: + raise Permanent("processing_failed", "staging source is gone") + if self._fault("transient-error", counter=True): + raise Transient("injected_transient", "fault injection") + src = workdir / "source" + with open(src, "wb") as f: + try: + storage.download_sync(job.staging_key, f, settings.max_upload_bytes) + except storage.ObjectMissing: + raise Permanent("processing_failed", "staging object is missing") + except ValueError: + raise Permanent("invalid_image", "source larger than limit") + h = hashlib.sha256() + with open(src, "rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + h.update(chunk) + if h.hexdigest() != job.sha256: + raise Permanent("processing_failed", "source checksum mismatch") + + out = workdir / "out" + out.mkdir() + result_path = workdir / "result.json" + slow = float(self.r.get("fault:slow-seconds") or 0) if settings.fault_injection else 0.0 + proc = self.mp.Process(target=imaging.child_main, args=( + str(src), str(out), job.content_type, settings.max_image_pixels, + settings.image_memory_mb, int(settings.job_timeout_seconds) + 5, str(result_path), slow)) + proc.start() + deadline = time.monotonic() + settings.job_timeout_seconds + while proc.is_alive() and time.monotonic() < deadline and not lost.is_set(): + proc.join(timeout=0.5) + if proc.is_alive(): + proc.kill() + proc.join() + if lost.is_set(): + raise LeaseLost() + raise Transient("processing_timeout", f"> {settings.job_timeout_seconds}s") + if lost.is_set(): + raise LeaseLost() + if not result_path.exists(): + raise Transient("processing_crashed", f"exit code {proc.exitcode}") + result = json.loads(result_path.read_text(encoding="utf-8")) + if not result["ok"]: + if result.get("trace"): + log.warning("job=%s child error: %s", job.job_id, result["trace"]) + exc = Permanent if result["permanent"] else Transient + raise exc(result["code"], result.get("detail", "")) + variants = {} + for name in imaging.VARIANTS: + v = result["variants"][name] + data = Path(v["path"]).read_bytes() + v["size_bytes"] = len(data) + v["sha256"] = hashlib.sha256(data).hexdigest() + v["object_key"] = storage.variant_key(job.attachment_id, job.pipeline_version, name, + imaging.VARIANT_EXT[name]) + variants[name] = v + return variants + + def _publish_objects(self, job: Job, variants: dict) -> None: + """S3-эффекты идемпотентны: те же ключи и те же байты при повторе. + Оригинал копируется на стороне S3 (staging → originals), без скачивания.""" + storage.copy_sync(job.staging_key, job.object_key) + for name, v in variants.items(): + storage.put_file_sync(v["object_key"], v["path"], v["content_type"], v["sha256"]) + + def _finalize(self, job: Job, variants: dict) -> None: + """Атомарная публикация результата. Только владелец аренды (lease_token); + ready не откатывается; ключ эффекта (attachment, pipeline_version, variant).""" + with self.pool.connection() as conn, conn.transaction(): + row = conn.execute("SELECT status, lease_token FROM attachment_jobs WHERE job_id = %s " + "FOR UPDATE", (job.job_id,)).fetchone() + if row is None or row[0] != "processing" or str(row[1]) != job.token: + raise LeaseLost() + for name, v in variants.items(): + conn.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, variant, " + "object_key, content_type, width, height, size_bytes, sha256, created_at) " + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, now()) " + "ON CONFLICT (attachment_id, pipeline_version, variant) DO NOTHING", + (job.attachment_id, job.pipeline_version, name, v["object_key"], + v["content_type"], v["width"], v["height"], v["size_bytes"], v["sha256"])) + conn.execute( + "UPDATE attachment_jobs SET status = 'ready', lease_token = NULL, lease_owner = NULL, " + "lease_until = NULL, error_code = NULL, finished_at = now(), updated_at = now() " + "WHERE job_id = %s", (job.job_id,)) + conn.execute("UPDATE attachments SET status = 'ready', error_code = NULL, updated_at = now() " + "WHERE id = %s", (job.attachment_id,)) + conn.execute("INSERT INTO job_events (job_id, event, attempt, actor) " + "VALUES (%s, 'ready', %s, %s)", (job.job_id, job.attempt, self.name)) + + def _drop_staging(self, job: Job) -> None: + """Staging удаляется только ПОСЛЕ COMMIT ready и ack. Не вышло — подчистит maintenance.""" + try: + storage.delete_sync(job.staging_key) + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s AND status = 'ready'", + (job.attachment_id,)) + except Exception as exc: # noqa: BLE001 + log.info("staging cleanup deferred: %s", type(exc).__name__) + + def _failure(self, job: Job, exc: Transient | Permanent, msg_id: str) -> str: + permanent = isinstance(exc, Permanent) + log.warning("job=%s attempt=%d/%d %s error: %s %s", job.job_id, job.attempt, job.max_attempts, + "permanent" if permanent else "transient", exc.code, exc.detail) + try: + outcome = schedule_retry_or_fail(self.pool, job.job_id, job.token, exc.code, exc.detail, + permanent, self.name) + except psycopg.Error as db_exc: + log.warning("cannot record failure (%s): leave message pending", type(db_exc).__name__) + return "leave" + if outcome == "failed": + try: + broker.to_dlq(self.r, broker.dumps({"job_id": job.job_id, "attachment_id": job.attachment_id}), + exc.code if permanent else f"retry_exhausted ({exc.code})", msg_id) + except redis.RedisError: + pass # реестр failed — в БД; DLQ-stream лишь копия для оператора + return "ack" + + # ---------------------------------------------------------------- fault injection + def _fault(self, point: str, counter: bool = False) -> bool: + """Только при FAULT_INJECTION=true (сервис worker-fault). Флаг ставит тест: + redis-cli SET fault: 1 (одноразовый), для counter — число срабатываний.""" + if not settings.fault_injection: + return False + key = f"fault:{point}" + try: + if counter: + left = self.r.decr(key) + if left < 0: + self.r.delete(key) + return False + return True + return self.r.getdel(key) is not None + except redis.RedisError: + return False + + # ================================================================ maintenance + def maintenance_loop(self) -> None: + while not self.stop.is_set(): + Path(settings.heartbeat_file).touch() + for step in (self.relay_outbox, self.reap_leases, self.sweep_staging): + try: + step() + except Exception as exc: # noqa: BLE001 + log.warning("maintenance %s failed: %s", step.__name__, type(exc).__name__) + self.stop.wait(settings.maintenance_interval_seconds) + + def relay_outbox(self) -> None: + """Transactional outbox → брокер. Публикует строки, которые API не отметил + (упал между COMMIT и publish, брокер был недоступен) и отложенные retry. + SKIP LOCKED: несколько worker не публикуют одну строку одновременно. + Упали после XADD, до COMMIT → строка опубликуется ещё раз: дубль безопасен.""" + with self.pool.connection() as conn, conn.transaction(): + rows = conn.execute( + "SELECT id, payload FROM job_outbox WHERE published_at IS NULL " + "AND available_at <= now() - make_interval(secs => %s) " + "ORDER BY available_at, id LIMIT 100 FOR UPDATE SKIP LOCKED", + (settings.outbox_relay_delay_seconds,)).fetchall() + for oid, payload in rows: + try: + stream_id = broker.publish_sync(self.r, payload) + except broker.BrokerUnavailable as exc: + conn.execute("UPDATE job_outbox SET publish_attempts = publish_attempts + 1, " + "last_error = %s WHERE id = %s", (str(exc)[:100], oid)) + break + conn.execute("UPDATE job_outbox SET published_at = now(), stream_id = %s, " + "publish_attempts = publish_attempts + 1 WHERE id = %s", (stream_id, oid)) + log.info("relay: published outbox=%s job=%s attempt=%s", oid, payload["job_id"], + payload["attempt"]) + + def reap_leases(self) -> None: + """Аренда истекла и за ещё одну аренду никто не перехватил (redelivery) → + worker мёртв: новая управляемая попытка через outbox (или failed, если попыток нет).""" + with self.pool.connection() as conn: + ids = [r[0] for r in conn.execute( + "SELECT job_id FROM attachment_jobs WHERE status = 'processing' " + "AND lease_until < now() - make_interval(secs => %s) LIMIT 20", + (settings.job_lease_seconds,)).fetchall()] + for job_id in ids: + outcome = schedule_retry_or_fail(self.pool, job_id, None, "lease_expired", + "worker stopped renewing lease", False, "reaper") + log.warning("reaper: job=%s lease expired → %s", job_id, outcome) + + def sweep_staging(self) -> None: + """Очистка staging: ready — сразу после фиксации (страховка, если worker не успел); + failed — после STAGING_RETENTION_SECONDS (до этого исходник нужен для replay).""" + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT id, staging_key FROM attachments WHERE staging_key IS NOT NULL AND (" + " (status = 'ready' AND updated_at < now() - interval '30 seconds') OR" + " (status = 'failed' AND updated_at < now() - make_interval(secs => %s))) LIMIT 50", + (settings.staging_retention_seconds,)).fetchall() + for aid, key in rows: + storage.delete_sync(key) + with self.pool.connection() as conn: + conn.execute("UPDATE attachments SET staging_key = NULL WHERE id = %s " + "AND status IN ('ready', 'failed')", (aid,)) + log.info("staging cleaned for attachment %s", aid) + + +def schedule_retry_or_fail(pool: ConnectionPool, job_id, token: str | None, code: str, detail: str, + permanent: bool, actor: str) -> str: + """processing → queued (новая попытка через outbox с backoff) или → failed. + token задан — проверяем владельца аренды; None — вызывает reaper (аренда уже истекла).""" + with pool.connection() as conn, conn.transaction(): + row = conn.execute( + "SELECT status, attempt, max_attempts, lease_token, attachment_id, pipeline_version, " + "lease_until < now() - make_interval(secs => %s) FROM attachment_jobs " + "WHERE job_id = %s FOR UPDATE SKIP LOCKED", (settings.job_lease_seconds, job_id)).fetchone() + if row is None or row[0] != "processing": + return "skipped" + status, attempt, max_attempts, lease_token, attachment_id, pv, long_expired = row + if token is not None and str(lease_token) != token: + return "skipped" # аренду уже забрали + if token is None and not long_expired: + return "skipped" + if permanent or attempt >= max_attempts: + final = code if permanent else "retry_exhausted" + conn.execute( + "UPDATE attachment_jobs SET status = 'failed', error_code = %s, last_error = %s, " + "lease_token = NULL, lease_owner = NULL, lease_until = NULL, finished_at = now(), " + "updated_at = now() WHERE job_id = %s", (final, f"{code}: {detail}"[:300], job_id)) + conn.execute("UPDATE attachments SET status = 'failed', error_code = %s, updated_at = now() " + "WHERE id = %s", (final, attachment_id)) + conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) " + "VALUES (%s, 'failed', %s, %s, %s)", (job_id, attempt, actor, f"{code}: {detail}"[:300])) + return "failed" + delay = backoff_seconds(attempt) + nxt = attempt + 1 + conn.execute( + "UPDATE attachment_jobs SET status = 'queued', attempt = %s, last_error = %s, " + "next_attempt_at = now() + make_interval(secs => %s), lease_token = NULL, " + "lease_owner = NULL, lease_until = NULL, updated_at = now() WHERE job_id = %s", + (nxt, f"{code}: {detail}"[:300], delay, job_id)) + conn.execute("UPDATE attachments SET status = 'queued', updated_at = now() WHERE id = %s", + (attachment_id,)) + created = conn.execute("SELECT now()").fetchone()[0] + payload = broker.make_payload(job_id, attachment_id, pv, nxt, created) + conn.execute( + "INSERT INTO job_outbox (job_id, attempt, payload, available_at, created_at) " + "VALUES (%s, %s, %s, now() + make_interval(secs => %s), now())", + (job_id, nxt, json.dumps(payload), delay)) + conn.execute("INSERT INTO job_events (job_id, event, attempt, actor, detail) " + "VALUES (%s, 'retry_scheduled', %s, %s, %s)", + (job_id, attempt, actor, f"{code}; next attempt {nxt} in {delay:.1f}s")) + return f"retry #{nxt} in {delay:.1f}s" + + +def main() -> None: + logging.basicConfig(level=settings.log_level.upper(), + format="%(asctime)s %(levelname)s %(name)s %(message)s") + logging.getLogger("botocore").setLevel(logging.WARNING) + Worker().run() + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..c83195cd --- /dev/null +++ b/compose.yaml @@ -0,0 +1,350 @@ +# Стенд lab5: клиент → lb (HAProxy) → api × 2 → PostgreSQL (метаданные, jobs, outbox) + Redis (кэш сессий) +# ├→ MinIO, приватный bucket: staging/ → originals/, variants/ +# └→ broker (Redis Streams + AOF) → worker → MinIO + PostgreSQL +# Запуск: docker compose --env-file .env up -d --build +# Публичный только lb (127.0.0.1:8080). Реплики API, worker, broker и MinIO наружу не публикуются. +# +# Сети: +# app — lb, api, worker, postgres, redis, broker; +# storage — internal: только s3, s3-init, api и worker. Ни lb, ни хост туда не попадают: +# API и админ-API MinIO (порт 9000) доступны лишь внутри этой сети, +# веб-консоль MinIO выключена (MINIO_BROWSER=off). +name: mtusi-lab5 + +x-api-env: &api-env + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT: ${DB_POOL_TIMEOUT:-3} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + REDIS_TIMEOUT: ${REDIS_TIMEOUT:-0.5} + SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10} + LOG_LEVEL: ${LOG_LEVEL:-info} + # Миграции выполняет отдельный сервис migrate, а не каждая реплика. + MIGRATE_ON_START: "false" + # lab4: S3. У API — ОТДЕЛЬНЫЙ пользователь MinIO с правами только на свой bucket. + # Root-credentials MinIO в API не передаются. + S3_ENDPOINT: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_CONNECT_TIMEOUT: ${S3_CONNECT_TIMEOUT:-2} + S3_READ_TIMEOUT: ${S3_READ_TIMEOUT:-10} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + UPLOAD_BODY_TIMEOUT_SECONDS: ${UPLOAD_BODY_TIMEOUT_SECONDS:-60} + UPLOAD_SPOOL_MEMORY_BYTES: ${UPLOAD_SPOOL_MEMORY_BYTES:-262144} + UPLOAD_MAX_CONCURRENT: ${UPLOAD_MAX_CONCURRENT:-16} + IMAGE_CHECK_CONCURRENCY: ${IMAGE_CHECK_CONCURRENCY:-2} + UPLOAD_GRACE_SECONDS: ${UPLOAD_GRACE_SECONDS:-900} + JANITOR_INTERVAL_SECONDS: ${JANITOR_INTERVAL_SECONDS:-60} + S3_BREAKER_SECONDS: ${S3_BREAKER_SECONDS:-5} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2} + # lab5: брокер заданий (ОТДЕЛЬНЫЙ Redis с AOF, не кэш сессий) и параметры обработки. + BROKER_URL: redis://broker:6379/0 + BROKER_TIMEOUT: ${BROKER_TIMEOUT:-2} + PIPELINE_VERSION: "1" + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-20} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} + JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60} + CLAIM_IDLE_SECONDS: ${CLAIM_IDLE_SECONDS:-90} + IMAGE_MEMORY_MB: ${IMAGE_MEMORY_MB:-768} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + +# Сеть storage — internal: если s3 остановлен, его имя не резолвится, и встроенный DNS +# Docker пытается спросить внешний DNS, которого из internal-сети нет → по 5+ c на запрос. +# Короткий таймаут резолвера: недоступный S3 даёт быстрый 503, а не «зависание». +x-dns-opt: &dns-opt ["timeout:1", "attempts:1"] + +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - pgdata:/var/lib/postgresql/data + networks: [app] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 5s + timeout: 3s + retries: 20 + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + restart: unless-stopped + + redis: + image: redis:7.4.1-alpine + # Только кэш сессий: источник истины — PostgreSQL, persistence не нужен. + command: ["redis-server", "--save", "", "--appendonly", "no", + "--maxmemory", "64mb", "--maxmemory-policy", "allkeys-lru"] + networks: [app] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + deploy: + resources: + limits: {cpus: "0.25", memory: 96M} + restart: unless-stopped + + # lab5: брокер заданий — Redis Streams с persistence (тот же образ, другой экземпляр). + # --appendonly yes --appendfsync always: ответ на XADD уходит после fsync AOF + # (это наш «publisher confirm»: принятое брокером сообщение переживёт рестарт); + # --save "": RDB-снимки не нужны, источник — AOF; + # --maxmemory-policy noeviction: сообщения никогда не вытесняются из памяти. + broker: + image: redis:7.4.1-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", "", + "--maxmemory", "200mb", "--maxmemory-policy", "noeviction"] + volumes: + - brokerdata:/data + networks: [app] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + restart: unless-stopped + + # lab4: S3-совместимое хранилище. Версия закреплена; данные — в volume s3data. + # Официальные образы MinIO больше не скачиваются анонимно (Docker Hub — удалены, + # quay.io — 401). Используем сборку MinIO от сообщества Pigsty (тот же код MinIO, + # тот же CLI/env), версия закреплена. + s3: + image: pgsty/minio:RELEASE.2026-08-04T00-00-00Z + command: ["server", "/data"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + MINIO_BROWSER: "off" # веб-консоль выключена совсем + volumes: + - s3data:/data + networks: [storage] # портов наружу нет + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 20 + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} + restart: unless-stopped + + # Однократный идемпотентный bootstrap: bucket (private), политика, пользователь API. + # Повторный запуск ничего не ломает (mb --ignore-existing, policy/user перезаписываются). + s3-init: + image: pgsty/mc:RELEASE.2026-09-16T00-00-00Z # клиент MinIO (mc) той же сборки + entrypoint: ["/bin/sh", "/infra/bootstrap.sh"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + volumes: + - ./infra/minio:/infra:ro + networks: [storage] + depends_on: + s3: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + restart: "no" + + # Однократные миграции до старта реплик (advisory lock защищает и от повтора). + migrate: + build: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.migrate"] + environment: *api-env + networks: [app] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + restart: "no" + + api: + build: . + image: mtusi-messenger:lab5 + environment: *api-env + dns_opt: *dns-opt + # Портов наружу нет: доступ только через lb. container_name не задан — сервис масштабируется. + expose: ["8080"] + networks: [app, storage] + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "0.5", memory: 256M} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=2).status==200 else 1)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s + # docker stop → SIGTERM: uvicorn перестаёт принимать соединения и дожидается текущих запросов. + stop_grace_period: 15s + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_started + broker: + condition: service_healthy + restart: unless-stopped + + # lab5: worker — отдельный процесс, тот же образ. Портов нет, в lb не входит. + # Ограниченная конкуренция: WORKER_CONCURRENCY задач на процесс, каждая — в дочернем + # процессе с лимитом памяти (IMAGE_MEMORY_MB) и времени (JOB_TIMEOUT_SECONDS). + # docker stop → SIGTERM: новых задач не берёт, текущую дорабатывает (до таймаута). + worker: + image: mtusi-messenger:lab5 + pull_policy: never # образ собирает сервис api + command: ["python", "-m", "app.worker"] + environment: *api-env + dns_opt: *dns-opt + networks: [app, storage] + deploy: + replicas: ${WORKER_REPLICAS:-1} + resources: + limits: {cpus: "1.0", memory: 1G} + healthcheck: + # heartbeat-файл обновляет поток maintenance раз в секунду + test: ["CMD", "python", "-c", "import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat')<30 else 1)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 10s + stop_grace_period: 75s + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + broker: + condition: service_healthy + api: + condition: service_started # образ mtusi-messenger:lab5 собирается сервисом api + restart: unless-stopped + + lb: + image: haproxy:3.0.5-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8081:8081" + - "127.0.0.1:8082:8082" + networks: [app] + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/healthz"] + interval: 5s + timeout: 3s + retries: 3 + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + depends_on: + api: + condition: service_started + restart: unless-stopped + + # API с коротким сроком сессии — только для теста истечения (профиль ttl). + # Доступен тоже только через lb: http://localhost:8081 + api-ttl: + image: mtusi-messenger:lab5 + pull_policy: never # образ собирается локально сервисом api + environment: + <<: *api-env + SESSION_TTL_SECONDS: "2" + JANITOR_ENABLED: "false" + dns_opt: *dns-opt + expose: ["8080"] + networks: [app, storage] + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + depends_on: + postgres: + condition: service_healthy + profiles: ["ttl"] + + # Реплика с внедрением отказов (профиль fault) — только для student-tests/orphans.py. + # Заголовок X-Fault-Inject работает ТОЛЬКО здесь (FAULT_INJECTION=true). + # Доступна только через lb: http://localhost:8082 + api-fault: + image: mtusi-messenger:lab5 + pull_policy: never + environment: + <<: *api-env + FAULT_INJECTION: "true" + JANITOR_ENABLED: "false" + dns_opt: *dns-opt + expose: ["8080"] + networks: [app, storage] + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + depends_on: + postgres: + condition: service_healthy + restart: unless-stopped # после crash-after-s3-put контейнер поднимется сам + profiles: ["fault"] + + # lab5: worker с внедрением отказов (профиль fault) — только для student-tests/jobs.py. + # Отказ включает одноразовый флаг в брокере: redis-cli SET fault:<точка> 1. + # restart: unless-stopped — после os._exit(1) контейнер поднимается с тем же hostname, + # поэтому worker находит своё неподтверждённое сообщение в PEL. + worker-fault: + image: mtusi-messenger:lab5 + pull_policy: never + command: ["python", "-m", "app.worker"] + environment: + <<: *api-env + FAULT_INJECTION: "true" + JOB_BACKOFF_BASE_SECONDS: "1" # короче паузы в тесте retry + dns_opt: *dns-opt + networks: [app, storage] + deploy: + resources: + limits: {cpus: "1.0", memory: 1G} + stop_grace_period: 75s + depends_on: + broker: + condition: service_healthy + restart: unless-stopped + profiles: ["fault"] + +networks: + app: {} + storage: + internal: true # без выхода наружу и без публикации портов + +volumes: + pgdata: + s3data: + brokerdata: diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 00000000..4d953d47 --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,108 @@ +# attachments +# date: 2026-11-11T23:52:28 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/attachments.py --base-url http://localhost:8080 +== позитивный сценарий == + ok - upload PNG -> 202 (got 202) + ok - 202 queued, variants=[], error_code=null + ok - sha256/size describe the original + ok - no S3 key/bucket in public JSON + ok - upload JPEG with Unicode filename -> 202 (got 202) + ok - filename returned as is (display only) + ok - both become ready (worker) + ok - member downloads PNG -> 200 (got 200) + ok - downloaded bytes are identical (sha256) + ok - true Content-Type image/png + ok - nosniff header + ok - member downloads JPEG (variant=original) -> 200 (got 200) + ok - JPEG bytes and type + ok - message with empty text + 2 attachments -> 201 (got 201) + ok - attachment_ids kept in request order + ok - history shows attachment_ids + ok - text-only POST (lab3) still valid, attachment_ids=[] + ok - member reads metadata -> 200 (got 200) + ok - thumbnail variant exists in lab5 -> 200 (got 200) + ok - unknown variant name -> 400 (got 400) +== доступ == + ok - outsider GET attachment metadata -> 404 (got 404) + ok - unknown attachment metadata -> 404 (got 404) + ok - outsider GET attachment/content -> 404 (got 404) + ok - unknown attachment/content -> 404 (got 404) + ok - anonymous download -> 401 (got 401) + ok - attachment id is not a UUID -> 400 (got 400) + ok - outsider uploads into foreign chat -> 404 (got 404) + ok - anonymous upload -> 401 (got 401) +== привязка к сообщению == + ok - eve uploads into her own chat -> 202 (got 202) + ok - alice uploads into her second chat -> 202 (got 202) + ok - foreign attachment_id (eve's chat) in message -> 404 (got 404) + ok - attachment of ANOTHER chat of the same user -> 404 (got 404) + ok - unknown attachment_id -> 404 (got 404) + ok - one valid + one foreign -> whole message rejected -> 404 (got 404) + ok - duplicate attachment_ids -> 400 (got 400) + ok - same UUID in different case is a duplicate -> 400 (got 400) + ok - more than 10 attachment_ids -> 400 (got 400) + ok - attachment_id is not a UUID -> 400 (got 400) + ok - attachment_ids: null -> 400 (got 400) + ok - blank text without attachments -> 400 (got 400) + ok - text field is required -> 400 (got 400) + ok - any member may reference a chat attachment again -> 201 (got 201) + ok - eve attaches alice's file to her own chat -> 404 (got 404) +== идемпотентность с вложениями == + ok - missing attachment_ids == [] (same Message) + ok - same key, different attachment_ids -> 409 (got 409) + ok - replay with attachments returns the same Message +== размер == + ok - Content-Length = 10 MiB + 1, body never sent (limit before reading) -> 413 (got 413) + ok - Content-Length = 500 MiB -> 413 (got 413) + ok - chunked body > 10 MiB without Content-Length (stream cut at limit) -> 413 (got 413) + ok - real PNG of 11424004 bytes -> 413 (got 413) + ok - empty body -> 400 (got 400) +== пиксели == + ok - decompression bomb 8000x8000 in 62290 bytes -> 413 (got 413) + ok - 40 040 001 pixels (just over the limit) -> 413 (got 413) +== повреждённые файлы == + ok - truncated PNG -> 415 (got 415) + ok - truncated PNG: error code + ok - PNG with flipped byte (CRC) -> 415 (got 415) + ok - PNG with flipped byte (CRC): error code + ok - PNG signature only -> 415 (got 415) + ok - PNG signature only: error code + ok - PNG header + garbage -> 415 (got 415) + ok - PNG header + garbage: error code + ok - truncated JPEG -> 415 (got 415) + ok - truncated JPEG: error code + ok - JPEG SOI + garbage -> 415 (got 415) + ok - JPEG SOI + garbage: error code +== fuzz: случайные порчи байтов (только 202 или 415, никогда 500) == + ok - 30 corrupted files -> only 202/415 (got [202, 415]) +== подмена MIME == + ok - PNG bytes sent as image/jpeg -> 415 (got 415) + ok - JPEG bytes sent as image/png -> 415 (got 415) + ok - text renamed to .png -> 415 (got 415) + ok - GIF sent as image/png -> 415 (got 415) + ok - HTML sent as image/jpeg -> 415 (got 415) + ok - PNG with Content-Type image/gif -> 415 (got 415) + ok - PNG with Content-Type application/octet-stream -> 415 (got 415) + ok - PNG with Content-Type text/html -> 415 (got 415) +== опасные filename == + ok - filename '../../etc/passwd' -> 400 (got 400) + ok - filename 'dir/a.png' -> 400 (got 400) + ok - filename 'C:\\Windows\\a.png' -> 400 (got 400) + ok - filename 'a\x00.png' -> 400 (got 400) + ok - filename 'a\nb.png' -> 400 (got 400) + ok - filename 'tab\t.png' -> 400 (got 400) + ok - filename '\x1b[31mred.png' -> 400 (got 400) + ok - filename 'photo\u202egnp.exe' -> 400 (got 400) + ok - filename '..' -> 400 (got 400) + ok - filename '.' -> 400 (got 400) + ok - filename ' ' -> 400 (got 400) + ok - filename 'xxxxxxxxxxxxxxxxxxxx' -> 400 (got 400) + ok - filename missing -> 400 (got 400) + ok - filename of exactly 255 chars -> 202 (got 202) + ok - filename with spaces, Unicode and punctuation -> 202 (got 202) + ok - media type parameters are ignored -> 202 (got 202) + ok - no 500 and no dropped connection among 99 checked responses + +91 passed, 0 failed +[exit 0] diff --git a/evidence/broker-config.txt b/evidence/broker-config.txt new file mode 100644 index 00000000..4892dd77 --- /dev/null +++ b/evidence/broker-config.txt @@ -0,0 +1,22 @@ +# broker-config +# date: 2026-11-11T23:54:10 commit: 276dbda + +$ docker compose --env-file .env exec -T broker redis-cli CONFIG GET append* +appendfilename +appendonly.aof +appendonly +yes +appendfsync +always +appenddirname +appendonlydir +[exit 0] + +$ docker compose --env-file .env exec -T broker redis-cli CONFIG GET maxmemory-policy +maxmemory-policy +noeviction +[exit 0] + +$ docker compose --env-file .env exec -T api python -m app.tools queue +{"stream_length": 11, "groups": [{"name": "workers", "consumers": 2, "pending": 2, "lag": 9}], "dlq_length": 0, "outbox_unpublished": 0, "jobs_by_status": {"ready": 8, "processing": 2, "queued": 9}} +[exit 0] diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..3ef5381d --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check +# date: 2026-11-11T23:51:41 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +[exit 0] diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..fc501e47 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,56 @@ +# failover +# date: 2026-11-12T15:08:35 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/resilience.py --mode failover --action-script C:\Users\79152\Desktop\mtusi2026lab5\scripts\stop-one.cmd --base-url http://localhost:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 116, "failed_attempts": 0, "first_recovery_seconds": 1.536} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +[exit 0] + +$ C:\Users\79152\Desktop\mtusi2026lab5\scripts\restore-api.cmd + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-migrate-1 Starting + Container mtusi-lab5-s3-init-1 Starting + Container mtusi-lab5-migrate-1 Started + Container mtusi-lab5-s3-init-1 Started + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-s3-init-1 Waiting + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-s3-init-1 Exited + Container mtusi-lab5-api-1 Starting + Container mtusi-lab5-api-1 Started +[exit 0] + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/resilience.py --mode failover --action-script C:\Users\79152\Desktop\mtusi2026lab5\scripts\stop-one.cmd --base-url http://localhost:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 0, "first_recovery_seconds": 2.007} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +[exit 0] + +$ C:\Users\79152\Desktop\mtusi2026lab5\scripts\restore-api.cmd + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-s3-init-1 Starting + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-migrate-1 Starting + Container mtusi-lab5-s3-init-1 Started + Container mtusi-lab5-migrate-1 Started + Container mtusi-lab5-migrate-1 Waiting + Container mtusi-lab5-s3-init-1 Waiting + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-broker-1 Waiting + Container mtusi-lab5-broker-1 Healthy + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-migrate-1 Exited + Container mtusi-lab5-s3-init-1 Exited + Container mtusi-lab5-api-1 Starting + Container mtusi-lab5-api-1 Started +[exit 0] diff --git a/evidence/jobs-broker-down.txt b/evidence/jobs-broker-down.txt new file mode 100644 index 00000000..e708866f --- /dev/null +++ b/evidence/jobs-broker-down.txt @@ -0,0 +1,17 @@ +# jobs-broker-down +# date: 2026-11-11T23:55:16 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario broker-down +== брокер недоступен: приём не зависит от брокера (transactional outbox) == + ok - /health/ready = 200: broker is not critical for API + ok - upload → 202 queued while broker is down (2.1s, publish timeout ≤ 2s) + ok - intent kept in outbox, not published yet + ok - after broker is back: status ready, error_code null + ok - after broker is back: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after broker is back: original byte-for-byte + ok - after broker is back: thumbnail 256x160, sha256 matches + ok - relay published the outbox row when broker came back + [broker-down: 14s] + +8 passed, 0 failed +[exit 0] diff --git a/evidence/jobs-crash-ack.txt b/evidence/jobs-crash-ack.txt new file mode 100644 index 00000000..50b8fc3b --- /dev/null +++ b/evidence/jobs-crash-ack.txt @@ -0,0 +1,15 @@ +# jobs-crash-ack +# date: 2026-11-11T23:56:55 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario crash-ack +== crash после COMMIT ready, до ack == + ok - commit before crash: status ready, error_code null + ok - commit before crash: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - commit before crash: original byte-for-byte + ok - commit before crash: thumbnail 256x160, sha256 matches + ok - unacked message redelivered after restart and acked as no-op + ok - ready recorded once, 3 variants + [crash-ack: 26s] + +6 passed, 0 failed +[exit 0] diff --git a/evidence/jobs-crash-s3.txt b/evidence/jobs-crash-s3.txt new file mode 100644 index 00000000..104487cb --- /dev/null +++ b/evidence/jobs-crash-s3.txt @@ -0,0 +1,17 @@ +# jobs-crash-s3 +# date: 2026-11-11T23:56:03 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario crash-s3 +== crash после записи в S3, до COMMIT и ack == + recovered in 26.9s (worker restart + lease expiry) + ok - after crash before ack: status ready, error_code null + ok - after crash before ack: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after crash before ack: original byte-for-byte + ok - after crash before ack: thumbnail 256x160, sha256 matches + ok - same attempt taken over after lease expiry (redelivery, not a new job) + ok - S3 objects from the crashed attempt reused: still exactly 3 variants + events: ['22:56:14.755 accepted 1 api', '22:56:14.773 claimed 1 e4d49d50907a-1', '22:56:39.238 lease_takeover 1 e4d49d50907a-1', '22:56:41.607 ready 1 e4d49d50907a'] + [crash-s3: 48s] + +6 passed, 0 failed +[exit 0] diff --git a/evidence/jobs-lease.txt b/evidence/jobs-lease.txt new file mode 100644 index 00000000..8d21cde1 --- /dev/null +++ b/evidence/jobs-lease.txt @@ -0,0 +1,19 @@ +# jobs-lease +# date: 2026-11-11T23:58:48 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario lease +== долгая, но живая задача: heartbeat продлевает аренду == + ok - job ran 32s > lease 20s, no takeover, attempt 1 +== замёрзший worker: аренда истекает, задачу забирают == + ok - claimed by worker-fault, then it froze (processing) + recovered in 37.3s + ok - after frozen worker: status ready, error_code null + ok - after frozen worker: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after frozen worker: original byte-for-byte + ok - after frozen worker: thumbnail 256x160, sha256 matches + ok - reaper: lease expired → attempt 2 → ready + ok - woken stale worker could not commit (lease_token fencing), result unchanged + [lease: 127s] + +8 passed, 0 failed +[exit 0] diff --git a/evidence/jobs-outbox.txt b/evidence/jobs-outbox.txt new file mode 100644 index 00000000..da3f3240 --- /dev/null +++ b/evidence/jobs-outbox.txt @@ -0,0 +1,16 @@ +# jobs-outbox +# date: 2026-11-11T23:57:25 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario outbox +== crash API между COMMIT и publish (transactional outbox) == + ok - client did NOT get 202: API died after COMMIT (got 502) + ok - attachment and job were committed before the crash (1 new) + ok - relay-published job: status ready, error_code null + ok - relay-published job: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - relay-published job: original byte-for-byte + ok - relay-published job: thumbnail 256x160, sha256 matches + ok - outbox row published by relay (no new job created) + [outbox: 20s] + +7 passed, 0 failed +[exit 0] diff --git a/evidence/jobs-redelivery.txt b/evidence/jobs-redelivery.txt new file mode 100644 index 00000000..1b0cadaa --- /dev/null +++ b/evidence/jobs-redelivery.txt @@ -0,0 +1,15 @@ +# jobs-redelivery +# date: 2026-11-11T23:55:34 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario redelivery +== повторная доставка уже ready-задачи == + published duplicate: 1794437748730-0 {"schema_version": 1, "job_id": "29c2c47f-13f7-4064-ab1e-56d9860223d8", "type": "at + published duplicate: 1794437752117-0 {"schema_version": 1, "job_id": "29c2c47f-13f7-4064-ab1e-56d9860223d8", "type": "at + ok - duplicates: status stays ready, variants unchanged (same sha256) + ok - no second set of variants: rows=3, S3 objects=3 + ok - job finished exactly once, attempt unchanged + ok - duplicates acked (no pending messages) + [redelivery: 25s] + +4 passed, 0 failed +[exit 0] diff --git a/evidence/jobs-registry.txt b/evidence/jobs-registry.txt new file mode 100644 index 00000000..c89235d0 --- /dev/null +++ b/evidence/jobs-registry.txt @@ -0,0 +1,21 @@ +# jobs-registry +# date: 2026-11-12T00:00:58 commit: 276dbda + +$ docker compose --env-file .env exec -T api python -m app.replay --list +failed jobs: 0 +[exit 0] + +$ docker compose --env-file .env exec -T api python -m app.replay --dlq +1794437902645-0 {"job": "{\"attachment_id\":\"3916ea43-cfd7-4506-838a-5b206345e6b4\",\"job_id\":\"0bb5d8ca-8fad-4a3c-974e-1781acfa27e4\"}", "reason": "retry_exhausted (injected_transient)", "source_id": "1794437902611-0", "at": "2026-11-11T22:58:22.645225Z"} +[exit 0] + +$ docker compose --env-file .env exec -T api python -m app.tools queue +{"stream_length": 0, "groups": [{"name": "workers", "consumers": 4, "pending": 0, "lag": 0}], "dlq_length": 1, "outbox_unpublished": 0, "jobs_by_status": {"ready": 28}} +[exit 0] + +$ docker compose --env-file .env --profile fault stop api-fault worker-fault + Container mtusi-lab5-worker-fault-1 Stopping + Container mtusi-lab5-api-fault-1 Stopping + Container mtusi-lab5-worker-fault-1 Stopped + Container mtusi-lab5-api-fault-1 Stopped +[exit 0] diff --git a/evidence/jobs-retry-replay.txt b/evidence/jobs-retry-replay.txt new file mode 100644 index 00000000..f8cf8cd2 --- /dev/null +++ b/evidence/jobs-retry-replay.txt @@ -0,0 +1,22 @@ +# jobs-retry-replay +# date: 2026-11-11T23:57:49 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario retry-replay +== временные ошибки → backoff+jitter → failed → replay == + ok - after 22s: failed, error_code=retry_exhausted + ok - bounded: 5 attempts, 4 scheduled retries, delays [0.7, 1.1, 3.6, 6.3] + ok - backoff grows 2^n with jitter in [d/2, d] (JOB_BACKOFF_BASE_SECONDS=1) + ok - content of failed → 409 (got 409) + ok - failed attachment cannot be attached → 409 (got 409) + ok - failed job is in the registry (python -m app.replay --list) + ok - explicit replay accepted + ok - after replay: status ready, error_code null + ok - after replay: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after replay: original byte-for-byte + ok - after replay: thumbnail 256x160, sha256 matches + ok - same job_id, attempt 6, replays=1 (audit) + ok - replay of a ready job is refused (no duplicate execution) + [retry-replay: 55s] + +13 passed, 0 failed +[exit 0] diff --git a/evidence/jobs-worker-down.txt b/evidence/jobs-worker-down.txt new file mode 100644 index 00000000..574ce43c --- /dev/null +++ b/evidence/jobs-worker-down.txt @@ -0,0 +1,26 @@ +# jobs-worker-down +# date: 2026-11-11T23:54:34 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/jobs.py --base-url http://localhost:8080 --scenario worker-down +== worker остановлен: приём не зависит от обработки == + ok - upload → 202 queued without worker (0.06s) + ok - still queued after 3 s: API did not resize synchronously + ok - content of queued attachment → 409 (got 409) + ok - thumbnail of queued attachment → 409 (got 409) + ok - message with queued attachment → 201 (got 201) + ok - other messages keep working + ok - job is in the broker stream (len=1), outbox published +== рестарт брокера и API с сохранением volumes == + ok - after broker restart the message is still there (len=1) + ok - same session, same attachment, still queued +== worker запущен == + ok - after restart: status ready, error_code null + ok - after restart: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after restart: original byte-for-byte + ok - after restart: thumbnail 256x160, sha256 matches + ok - staging removed after ready (original lives in originals/) + attachment 6297945b-c47d-4659-846b-8a2bba4cedb2, job 41928f68-bbf4-415a-b615-022121ec24f2, events: ['22:54:40.477 accepted 1 api', '22:55:06.052 claimed 1 7f4ae0224d1d-0', '22:55:08.563 ready 1 7f4ae0224d1d'] + [worker-down: 38s] + +14 passed, 0 failed +[exit 0] diff --git a/evidence/local/attachments.txt b/evidence/local/attachments.txt new file mode 100644 index 00000000..8e3359ec --- /dev/null +++ b/evidence/local/attachments.txt @@ -0,0 +1,105 @@ +== позитивный сценарий == + ok - upload PNG -> 202 (got 202) + ok - 202 queued, variants=[], error_code=null + ok - sha256/size describe the original + ok - no S3 key/bucket in public JSON + ok - upload JPEG with Unicode filename -> 202 (got 202) + ok - filename returned as is (display only) + ok - both become ready (worker) + ok - member downloads PNG -> 200 (got 200) + ok - downloaded bytes are identical (sha256) + ok - true Content-Type image/png + ok - nosniff header + ok - member downloads JPEG (variant=original) -> 200 (got 200) + ok - JPEG bytes and type + ok - message with empty text + 2 attachments -> 201 (got 201) + ok - attachment_ids kept in request order + ok - history shows attachment_ids + ok - text-only POST (lab3) still valid, attachment_ids=[] + ok - member reads metadata -> 200 (got 200) + ok - thumbnail variant exists in lab5 -> 200 (got 200) + ok - unknown variant name -> 400 (got 400) +== доступ == + ok - outsider GET attachment metadata -> 404 (got 404) + ok - unknown attachment metadata -> 404 (got 404) + ok - outsider GET attachment/content -> 404 (got 404) + ok - unknown attachment/content -> 404 (got 404) + ok - anonymous download -> 401 (got 401) + ok - attachment id is not a UUID -> 400 (got 400) + ok - outsider uploads into foreign chat -> 404 (got 404) + ok - anonymous upload -> 401 (got 401) +== привязка к сообщению == + ok - eve uploads into her own chat -> 202 (got 202) + ok - alice uploads into her second chat -> 202 (got 202) + ok - foreign attachment_id (eve's chat) in message -> 404 (got 404) + ok - attachment of ANOTHER chat of the same user -> 404 (got 404) + ok - unknown attachment_id -> 404 (got 404) + ok - one valid + one foreign -> whole message rejected -> 404 (got 404) + ok - duplicate attachment_ids -> 400 (got 400) + ok - same UUID in different case is a duplicate -> 400 (got 400) + ok - more than 10 attachment_ids -> 400 (got 400) + ok - attachment_id is not a UUID -> 400 (got 400) + ok - attachment_ids: null -> 400 (got 400) + ok - blank text without attachments -> 400 (got 400) + ok - text field is required -> 400 (got 400) + ok - any member may reference a chat attachment again -> 201 (got 201) + ok - eve attaches alice's file to her own chat -> 404 (got 404) +== идемпотентность с вложениями == + ok - missing attachment_ids == [] (same Message) + ok - same key, different attachment_ids -> 409 (got 409) + ok - replay with attachments returns the same Message +== размер == + ok - Content-Length = 10 MiB + 1, body never sent (limit before reading) -> 413 (got 413) + ok - Content-Length = 500 MiB -> 413 (got 413) + ok - chunked body > 10 MiB without Content-Length (stream cut at limit) -> 413 (got 413) + ok - real PNG of 10835273 bytes -> 413 (got 413) + ok - real PNG of 10439723 bytes (≤ 10 MiB) accepted -> 202 (got 202) + ok - big file sha256/size + ok - empty body -> 400 (got 400) +== пиксели == + ok - decompression bomb 8000x8000 in 62290 bytes -> 413 (got 413) + ok - 40 040 001 pixels (just over the limit) -> 413 (got 413) +== повреждённые файлы == + ok - truncated PNG -> 415 (got 415) + ok - truncated PNG: error code + ok - PNG with flipped byte (CRC) -> 415 (got 415) + ok - PNG with flipped byte (CRC): error code + ok - PNG signature only -> 415 (got 415) + ok - PNG signature only: error code + ok - PNG header + garbage -> 415 (got 415) + ok - PNG header + garbage: error code + ok - truncated JPEG -> 415 (got 415) + ok - truncated JPEG: error code + ok - JPEG SOI + garbage -> 415 (got 415) + ok - JPEG SOI + garbage: error code +== fuzz: случайные порчи байтов (только 202 или 415, никогда 500) == + ok - 30 corrupted files -> only 202/415 (got [202, 415]) +== подмена MIME == + ok - PNG bytes sent as image/jpeg -> 415 (got 415) + ok - JPEG bytes sent as image/png -> 415 (got 415) + ok - text renamed to .png -> 415 (got 415) + ok - GIF sent as image/png -> 415 (got 415) + ok - HTML sent as image/jpeg -> 415 (got 415) + ok - PNG with Content-Type image/gif -> 415 (got 415) + ok - PNG with Content-Type application/octet-stream -> 415 (got 415) + ok - PNG with Content-Type text/html -> 415 (got 415) +== опасные filename == + ok - filename '../../etc/passwd' -> 400 (got 400) + ok - filename 'dir/a.png' -> 400 (got 400) + ok - filename 'C:\\Windows\\a.png' -> 400 (got 400) + ok - filename 'a\x00.png' -> 400 (got 400) + ok - filename 'a\nb.png' -> 400 (got 400) + ok - filename 'tab\t.png' -> 400 (got 400) + ok - filename '\x1b[31mred.png' -> 400 (got 400) + ok - filename 'photo\u202egnp.exe' -> 400 (got 400) + ok - filename '..' -> 400 (got 400) + ok - filename '.' -> 400 (got 400) + ok - filename ' ' -> 400 (got 400) + ok - filename 'xxxxxxxxxxxxxxxxxxxx' -> 400 (got 400) + ok - filename missing -> 400 (got 400) + ok - filename of exactly 255 chars -> 202 (got 202) + ok - filename with spaces, Unicode and punctuation -> 202 (got 202) + ok - media type parameters are ignored -> 202 (got 202) + ok - no 500 and no dropped connection among 100 checked responses + +93 passed, 0 failed diff --git a/evidence/local/failover.txt b/evidence/local/failover.txt new file mode 100644 index 00000000..854c1e9b --- /dev/null +++ b/evidence/local/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 143, "failed_attempts": 0, "first_recovery_seconds": 0.114} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/local/jobs.txt b/evidence/local/jobs.txt new file mode 100644 index 00000000..d245a2af --- /dev/null +++ b/evidence/local/jobs.txt @@ -0,0 +1,93 @@ +== worker остановлен: приём не зависит от обработки == + ok - upload → 202 queued without worker (0.02s) + ok - still queued after 3 s: API did not resize synchronously + ok - content of queued attachment → 409 (got 409) + ok - thumbnail of queued attachment → 409 (got 409) + ok - message with queued attachment → 201 (got 201) + ok - other messages keep working + ok - job is in the broker stream (len=1), outbox published +== рестарт брокера и API с сохранением volumes == + ok - after broker restart the message is still there (len=1) + ok - same session, same attachment, still queued +== worker запущен == + ok - after restart: status ready, error_code null + ok - after restart: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after restart: original byte-for-byte + ok - after restart: thumbnail 256x160, sha256 matches + ok - staging removed after ready (original lives in originals/) + attachment b442df77-ff58-4ab7-89e9-979b507b38f8, job e2871dfb-6dd2-46c7-8dd1-ce29c7f22558, events: ['01:30:28.325 accepted 1 api', '01:30:37.814 claimed 1 worker-1-0', '01:30:38.282 ready 1 worker-1'] + [worker-down: 14s] +== брокер недоступен: приём не зависит от брокера (transactional outbox) == + ok - /health/ready = 200: broker is not critical for API + ok - upload → 202 queued while broker is down (0.0s, publish timeout ≤ 2s) + ok - intent kept in outbox, not published yet + ok - after broker is back: status ready, error_code null + ok - after broker is back: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after broker is back: original byte-for-byte + ok - after broker is back: thumbnail 256x160, sha256 matches + ok - relay published the outbox row when broker came back + [broker-down: 4s] +== повторная доставка уже ready-задачи == + published duplicate: 1794436247461-0 {"schema_version": 1, "job_id": "5c7dd513-a5f2-4663-b2ad-c498ea0163e5", "type": "at + published duplicate: 1794436247942-0 {"schema_version": 1, "job_id": "5c7dd513-a5f2-4663-b2ad-c498ea0163e5", "type": "at + ok - duplicates: status stays ready, variants unchanged (same sha256) + ok - no second set of variants: rows=3, S3 objects=3 + ok - job finished exactly once, attempt unchanged + ok - duplicates acked (no pending messages) + [redelivery: 7s] +== crash после записи в S3, до COMMIT и ack == + recovered in 22.7s (worker restart + lease expiry) + ok - after crash before ack: status ready, error_code null + ok - after crash before ack: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after crash before ack: original byte-for-byte + ok - after crash before ack: thumbnail 256x160, sha256 matches + ok - same attempt taken over after lease expiry (redelivery, not a new job) + ok - S3 objects from the crashed attempt reused: still exactly 3 variants + events: ['01:30:56.132 accepted 1 api', '01:30:56.135 claimed 1 worker-fault-1', '01:31:18.241 lease_takeover 1 worker-fault-1', '01:31:18.745 ready 1 worker-fault'] + [crash-s3: 27s] +== crash после COMMIT ready, до ack == + ok - commit before crash: status ready, error_code null + ok - commit before crash: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - commit before crash: original byte-for-byte + ok - commit before crash: thumbnail 256x160, sha256 matches + ok - unacked message redelivered after restart and acked as no-op + ok - ready recorded once, 3 variants + [crash-ack: 8s] +== crash API между COMMIT и publish (transactional outbox) == + ok - client did NOT get 202: API died after COMMIT (got 502) + ok - attachment and job were committed before the crash (1 new) + ok - relay-published job: status ready, error_code null + ok - relay-published job: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - relay-published job: original byte-for-byte + ok - relay-published job: thumbnail 256x160, sha256 matches + ok - outbox row published by relay (no new job created) + [outbox: 4s] +== временные ошибки → backoff+jitter → failed → replay == + ok - after 23s: failed, error_code=retry_exhausted + ok - bounded: 5 attempts, 4 scheduled retries, delays [0.6, 1.8, 3.8, 6.4] + ok - backoff grows 2^n with jitter in [d/2, d] (JOB_BACKOFF_BASE_SECONDS=1) + ok - content of failed → 409 (got 409) + ok - failed attachment cannot be attached → 409 (got 409) + ok - failed job is in the registry (python -m app.replay --list) + ok - explicit replay accepted + ok - after replay: status ready, error_code null + ok - after replay: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after replay: original byte-for-byte + ok - after replay: thumbnail 256x160, sha256 matches + ok - same job_id, attempt 6, replays=1 (audit) + ok - replay of a ready job is refused (no duplicate execution) + [retry-replay: 31s] +== долгая, но живая задача: heartbeat продлевает аренду == + ok - job ran 31s > lease 20s, no takeover, attempt 1 +== замёрзший worker: аренда истекает, задачу забирают == + ok - claimed by worker-fault, then it froze (processing) + recovered in 40.8s + ok - after frozen worker: status ready, error_code null + ok - after frozen worker: variants thumbnail+crop+optimized (['thumbnail', 'crop', 'optimized']) + ok - after frozen worker: original byte-for-byte + ok - after frozen worker: thumbnail 256x160, sha256 matches + ok - reaper: lease expired → attempt 2 → ready + ok - woken stale worker could not commit (lease_token fencing), result unchanged + [lease: 104s] + +66 passed, 0 failed diff --git a/evidence/local/persistence.txt b/evidence/local/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/local/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/local/registry.txt b/evidence/local/registry.txt new file mode 100644 index 00000000..009c99e3 --- /dev/null +++ b/evidence/local/registry.txt @@ -0,0 +1,5 @@ +failed jobs: 0 +1794436317733-0 {"job": "{\"attachment_id\":\"e8a81276-b5df-45cc-8362-f38abe00a557\",\"job_id\":\"2db55464-1734-40c4-9ce1-05b1db9fb834\"}", "reason": "injected_transient", "source_id": "1794436317728-0", "at": "2026-11-11T22:31:57.733543Z"} +1794436083241-0 {"job": "{\"attachment_id\":\"abc0488f-8870-4a6b-96de-299a6571d281\",\"job_id\":\"900a5962-c5a8-4680-8cfd-d3033df2e16d\"}", "reason": "injected_transient", "source_id": "1794436083233-0", "at": "2026-11-11T22:28:03.241832Z"} +1794435590706-0 {"job": "{\"attachment_id\":\"92896e61-6806-4f6e-b4ac-31c8c7aa1c86\",\"job_id\":\"fae19d4e-898f-4d18-bc22-5dd0dbe3c53a\"}", "reason": "injected_transient", "source_id": "1794435590697-0", "at": "2026-11-11T22:19:50.706703Z"} +{"stream_length": 0, "groups": [{"name": "workers", "consumers": 4, "pending": 0, "lag": 0}], "dlq_length": 3, "outbox_unpublished": 0, "jobs_by_status": {"ready": 70}} diff --git a/evidence/local/smoke.txt b/evidence/local/smoke.txt new file mode 100644 index 00000000..5c90482b --- /dev/null +++ b/evidence/local/smoke.txt @@ -0,0 +1 @@ +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/local/student-tests.txt b/evidence/local/student-tests.txt new file mode 100644 index 00000000..401f1ed6 --- /dev/null +++ b/evidence/local/student-tests.txt @@ -0,0 +1,59 @@ +== аутентификация == + ok - wrong password -> 401 + Basic challenge + ok - unknown user -> 401 (same as wrong password) + ok - no token -> 401 + Bearer challenge + ok - X-User-Id alone gives no access + ok - unknown/forged token -> 401 + ok - Basic on business endpoint -> 401 + ok - X-User-Id cannot override bearer identity + ok - password shorter than 12 -> 400 +== logout == + ok - logout -> 204 + ok - token after logout -> 401 + ok - second logout -> 401 +== гонка регистрации одного username == + ok - 20 parallel registrations -> one 201, nineteen 409 (got [201, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409]) +== ошибки и доступ (из lab1) == + ok - malformed JSON -> 400 + ok - duplicate username -> 409 + ok - invalid username 'review123\n' -> 400, not 500 (got 400) + ok - invalid username 'review123\r\n' -> 400, not 500 (got 400) + ok - invalid username ' review123' -> 400, not 500 (got 400) + ok - invalid username 'Review123' -> 400, not 500 (got 400) + ok - invalid username 'ab' -> 400, not 500 (got 400) + ok - blank message -> 400 + ok - sender_id in body is rejected + ok - outsider POST to foreign chat -> 404 + ok - outsider GET foreign history -> 404 + ok - invalid chat uuid -> 400 +== пагинация == + ok - 5 messages / limit 2 -> 3 pages, no duplicates + ok - cursor from another chat -> 400 +== Content-Type == + ok - valid JSON with text/plain -> 415 (got 415) + ok - message with text/plain -> 415 (got 415) + ok - text/plain without token -> 401 first (got 401) + ok - application/json; charset=utf-8 accepted (got 201) + +30 passed, 0 failed +== 20 параллельных запросов с одним ключом == + ok - all parallel repeats -> 201 (got [201]) + ok - all responses carry the same message id (1 distinct) + ok - requests were served by >=2 replicas (2) + ok - exactly one message in history + ok - every replay returns the identical Message +== семантика ключа == + ok - same key, different body -> 409 (got 409) + ok - other JSON formatting = same request (got 201) + ok - same key in another chat is independent + ok - same key from another user is independent + ok - missing Idempotency-Key -> 400 (got 400) + ok - invalid key 'has space'... -> 400 (got 400) + ok - invalid key 'xxxxxxxxxxxx'... -> 400 (got 400) + ok - invalid key 'slash/key'... -> 400 (got 400) + ok - replay without session -> 401, access re-checked (got 401) +== неопределённый результат: клиент повторяет с тем же ключом == + ok - retry after uncertain result -> 201 + ok - still exactly one message + +16 passed, 0 failed diff --git a/evidence/local/worker-limits.txt b/evidence/local/worker-limits.txt new file mode 100644 index 00000000..e8d069c3 --- /dev/null +++ b/evidence/local/worker-limits.txt @@ -0,0 +1,7 @@ + ok normal 320x200 with alpha: ok (0.4s) + ok pixel limit (36 Mp > limit 10 Mp): invalid_image: pixel limit (0.4s) + ok memory limit (36 Mp, RLIMIT_AS 120 MiB): invalid_image: memory limit (0.3s) + ok corrupt/truncated PNG: invalid_image: cannot decode: OSError (0.3s) + ok time limit (hangs 10 s, timeout 2 s): timeout(killed) (2.0s) + +5 passed, 0 failed diff --git a/evidence/no-local-file.txt b/evidence/no-local-file.txt new file mode 100644 index 00000000..d3914d39 --- /dev/null +++ b/evidence/no-local-file.txt @@ -0,0 +1,19 @@ +# no-local-file +# date: 2026-11-12T00:01:21 commit: 276dbda + +$ docker compose --env-file .env exec -T --index 1 api python -m app.tools find-local 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 +{"files_scanned": 64, "copies_found": []} +[exit 0] + +$ docker compose --env-file .env exec -T --index 2 api python -m app.tools find-local 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 +{"files_scanned": 57, "copies_found": []} +[exit 0] + +$ docker compose --env-file .env exec -T worker python -m app.tools find-local 99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75 +{"files_scanned": 58, "copies_found": []} +[exit 0] + +$ docker inspect -f {{.Name}} mounts={{json .Mounts}} 858c692aadd78f68ceaadb4d8ee90ab48f93af82b843580a957dfb564d076383 96a8e9076e16f4080345c85bcd5f2267e5abe6a8b0e8b47e56d200fbe29ec603 +/mtusi-lab5-api-1 mounts=[] +/mtusi-lab5-api-2 mounts=[] +[exit 0] diff --git a/evidence/orphans.txt b/evidence/orphans.txt new file mode 100644 index 00000000..0245bf39 --- /dev/null +++ b/evidence/orphans.txt @@ -0,0 +1,44 @@ +# orphans +# date: 2026-11-12T00:01:29 commit: 276dbda + +$ docker compose --env-file .env --profile fault up -d api-fault + Container mtusi-lab5-postgres-1 Running + Container mtusi-lab5-postgres-1 Waiting + Container mtusi-lab5-postgres-1 Healthy + Container mtusi-lab5-api-fault-1 Starting + Container mtusi-lab5-api-fault-1 Started +[exit 0] + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/orphans.py --base-url http://localhost:8080 --fault-url http://localhost:8082 + ok - fault replica (profile fault) is ready on --fault-url + ok - committed attachment that must survive every cleanup + ok - it is processed to ready (lab5) + baseline: {'objects': 29, 'attachments': 29, 'upload_intents': 0} +== 1. прерванная загрузка == + ok - no object, no metadata, no journal row after interrupted upload ({'objects': 29, 'attachments': 29, 'upload_intents': 0}) +== 2. отказ БД после PUT: немедленная компенсация == + ok - upload answered 503, not 202 (got 503) + ok - object deleted by compensation, journal clean ({'objects': 29, 'attachments': 29, 'upload_intents': 0}) +== 3. процесс убит между PUT и COMMIT == + ok - client did NOT get 202 (got 502) + ok - orphan object + journal row left behind, no metadata ({'objects': 30, 'attachments': 29, 'upload_intents': 1}) + ok - default grace 900 s: fresh upload is NOT touched + ok - dry-run reports 1 object, changes nothing + ok - janitor removed orphan + journal row ({'grace_seconds': 5, 'dry_run': False, 'locked': True, 'intents_expired': 1, 'intents_removed': 1, 'objects_deleted': 1, 'would_delete_objects': 0, 'orphans_found': 0, 'errors': 0, 'duration_ms': 30.5}) + ok - second run is a no-op (idempotent) +== 4. объект без журнала (страховочный обход bucket) == + ok - untracked object planted + ok - scan with grace 1 h ignores a fresh object + ok - scan dry-run finds 1 + ok - scan deletes the untracked object ({'grace_seconds': 5, 'dry_run': False, 'locked': True, 'intents_expired': 0, 'intents_removed': 0, 'objects_deleted': 1, 'would_delete_objects': 0, 'orphans_found': 1, 'errors': 0, 'objects_scanned': 30, 'duration_ms': 126.7}) +== 5. готовые вложения не тронуты == + ok - committed attachment still downloads, sha256 matches + ok - published original + metadata, staging cleaned, no journal row ({'staging_object_exists': False, 'original_object_exists': True, 'attachment_row': True, 'attachment_status': 'ready', 'upload_intent': False}) + +18 passed, 0 failed +[exit 0] + +$ docker compose --env-file .env --profile fault stop api-fault + Container mtusi-lab5-api-fault-1 Stopping + Container mtusi-lab5-api-fault-1 Stopped +[exit 0] diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..3704d823 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,31 @@ +# persistence +# date: 2026-11-12T15:08:19 commit: 276dbda + +$ docker compose --env-file .env ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab5-api-1 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 4 minutes (healthy) 8080/tcp +mtusi-lab5-api-2 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 4 minutes (healthy) 8080/tcp +mtusi-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 15 hours ago Up 5 minutes (healthy) 6379/tcp +mtusi-lab5-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 15 hours ago Up 5 minutes (healthy) 127.0.0.1:8080-8082->8080-8082/tcp +mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 15 hours ago Up 5 minutes (healthy) 5432/tcp +mtusi-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 15 hours ago Up 5 minutes (healthy) 6379/tcp +mtusi-lab5-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z "/usr/bin/docker-ent…" s3 15 hours ago Up 3 minutes (healthy) 9000/tcp +mtusi-lab5-worker-1 mtusi-messenger:lab5 "python -m app.worker" worker 4 minutes ago Up 4 minutes (healthy) 8080/tcp +[exit 0] + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/resilience.py --mode persistence --action-script C:\Users\79152\Desktop\mtusi2026lab5\scripts\restart.cmd --base-url http://localhost:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +[exit 0] + +$ docker compose --env-file .env ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab5-api-1 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 5 seconds (health: starting) 8080/tcp +mtusi-lab5-api-2 mtusi-messenger:lab5 "python -m app" api 5 minutes ago Up 5 seconds (health: starting) 8080/tcp +mtusi-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 15 hours ago Up 6 seconds (healthy) 6379/tcp +mtusi-lab5-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 15 hours ago Up 4 seconds (health: starting) 127.0.0.1:8080-8082->8080-8082/tcp +mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 15 hours ago Up 6 seconds (health: starting) 5432/tcp +mtusi-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 15 hours ago Up 6 seconds (health: starting) 6379/tcp +mtusi-lab5-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z "/usr/bin/docker-ent…" s3 15 hours ago Up 8 seconds (healthy) 9000/tcp +mtusi-lab5-worker-1 mtusi-messenger:lab5 "python -m app.worker" worker 5 minutes ago Up 5 seconds (health: starting) 8080/tcp +[exit 0] diff --git a/evidence/queue-drain.txt b/evidence/queue-drain.txt new file mode 100644 index 00000000..c684bd61 --- /dev/null +++ b/evidence/queue-drain.txt @@ -0,0 +1,7 @@ +# queue-drain +# date: 2026-11-12T15:06:34 commit: 276dbda +queue drained in 102.6 s: {'ready': 332} + +$ docker compose --env-file .env exec -T api python -m app.tools queue +{"stream_length": 0, "groups": [{"name": "workers", "consumers": 6, "pending": 0, "lag": 0}], "dlq_length": 1, "outbox_unpublished": 0, "jobs_by_status": {"ready": 332}} +[exit 0] diff --git a/evidence/s3-bootstrap.txt b/evidence/s3-bootstrap.txt new file mode 100644 index 00000000..35953e70 --- /dev/null +++ b/evidence/s3-bootstrap.txt @@ -0,0 +1,30 @@ +# s3-bootstrap +# date: 2026-11-12T00:01:07 commit: 276dbda + +$ docker compose --env-file .env run --rm s3-init + Container mtusi-lab5-s3-1 Running + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-s3-init-run-a68a74b86415 Creating + Container mtusi-lab5-s3-init-run-a68a74b86415 Created +Bucket created successfully `root/messenger-attachments`. +Access permission for `root/messenger-attachments` is set to `private` +policy lab4-api-rw attached to API user +verified: API user has policy lab4-api-rw +Access permission for `root/messenger-attachments` is `private` +bootstrap done: bucket=messenger-attachments (private), API user restricted by policy lab4-api-rw +[exit 0] + +$ docker compose --env-file .env run --rm s3-init + Container mtusi-lab5-s3-1 Running + Container mtusi-lab5-s3-1 Waiting + Container mtusi-lab5-s3-1 Healthy + Container mtusi-lab5-s3-init-run-17a0af6b8ff2 Creating + Container mtusi-lab5-s3-init-run-17a0af6b8ff2 Created +Bucket created successfully `root/messenger-attachments`. +Access permission for `root/messenger-attachments` is set to `private` +policy lab4-api-rw attached to API user +verified: API user has policy lab4-api-rw +Access permission for `root/messenger-attachments` is `private` +bootstrap done: bucket=messenger-attachments (private), API user restricted by policy lab4-api-rw +[exit 0] diff --git a/evidence/s3-outage.txt b/evidence/s3-outage.txt new file mode 100644 index 00000000..3eae7c5c --- /dev/null +++ b/evidence/s3-outage.txt @@ -0,0 +1,22 @@ +# s3-outage +# date: 2026-11-12T15:04:18 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/s3_outage.py --base-url http://localhost:8080 + ok - upload before outage + ok - it is processed to ready before the outage (lab5: 202 → worker) +== MinIO остановлен == + ok - upload -> 503 unavailable (got 503) + ok - bounded by timeouts, not a hang (6.6 s) + ok - download -> 503 (got 503) + ok - metadata still served from PostgreSQL + ok - text messages still work + ok - message referencing an existing attachment still works (metadata only) + ok - /health/ready = 200 (S3 is not critical for text path) +== MinIO снова работает == + ok - file uploaded before outage downloads unchanged + ok - uploads work again on all replicas: 4 x 202 in a row after 4.7 s (codes during recovery: [503, 202, 503, 202, 503, 202, 503, 202, 503, 202, 503, 202, 503, 202, 202, 202, 202]) + stats: {'objects': 242, 'attachments': 242, 'upload_intents': 1} + ok - janitor cleaned journal rows left by failed uploads ({'grace_seconds': 0, 'dry_run': False, 'locked': True, 'intents_expired': 1, 'intents_removed': 1, 'objects_deleted': 1, 'would_delete_objects': 0, 'orphans_found': 0, 'errors': 0, 'duration_ms': 39.5}) + +12 passed, 0 failed +[exit 0] diff --git a/evidence/s3-private.txt b/evidence/s3-private.txt new file mode 100644 index 00000000..0c37e4c5 --- /dev/null +++ b/evidence/s3-private.txt @@ -0,0 +1,17 @@ +# s3-private +# date: 2026-11-12T00:01:16 commit: 276dbda + +$ docker compose --env-file .env exec -T api python -m app.tools private-check + ok API credentials: GET own object -> 200 + ok anonymous GET object -> 403 + ok anonymous LIST bucket -> 403 + ok anonymous PUT object -> 403 + ok API credentials: ListBuckets (no foreign buckets) -> only messenger-attachments + ok API credentials: CreateBucket -> AccessDenied + ok API credentials: PutBucketPolicy (make public) -> AccessDenied + ok API credentials: DeleteBucket -> AccessDenied + +8 passed, 0 failed +[exit 0] +127.0.0.1:9000 closed (TimeoutError) ok +127.0.0.1:9001 closed (TimeoutError) ok diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..10ba39e2 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke +# date: 2026-11-11T23:51:45 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe tests/smoke.py --base-url http://localhost:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. +[exit 0] diff --git a/evidence/stand.txt b/evidence/stand.txt new file mode 100644 index 00000000..f0e6f2c0 --- /dev/null +++ b/evidence/stand.txt @@ -0,0 +1,30 @@ +# stand +# date: 2026-11-11T23:51:42 commit: 276dbda + +$ docker compose --env-file .env ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab5-api-1 mtusi-messenger:lab5 "python -m app" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-lab5-api-2 mtusi-messenger:lab5 "python -m app" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-lab5-broker-1 redis:7.4.1-alpine "docker-entrypoint.s…" broker 2 minutes ago Up 2 minutes (healthy) 6379/tcp +mtusi-lab5-lb-1 haproxy:3.0.5-alpine "docker-entrypoint.s…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8080-8082->8080-8082/tcp +mtusi-lab5-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 2 minutes ago Up 2 minutes (healthy) 5432/tcp +mtusi-lab5-redis-1 redis:7.4.1-alpine "docker-entrypoint.s…" redis 2 minutes ago Up 2 minutes (healthy) 6379/tcp +mtusi-lab5-s3-1 pgsty/minio:RELEASE.2026-08-04T00-00-00Z "/usr/bin/docker-ent…" s3 2 minutes ago Up 2 minutes (healthy) 9000/tcp +mtusi-lab5-worker-1 mtusi-messenger:lab5 "python -m app.worker" worker 2 minutes ago Up 2 minutes (healthy) 8080/tcp +[exit 0] + +$ docker compose --env-file .env config --images +redis:7.4.1-alpine +haproxy:3.0.5-alpine +mtusi-messenger:lab5 +postgres:16.4-alpine +mtusi-messenger:lab5 +pgsty/mc:RELEASE.2026-09-16T00-00-00Z +mtusi-messenger:lab5 +redis:7.4.1-alpine +pgsty/minio:RELEASE.2026-08-04T00-00-00Z +[exit 0] + +$ docker compose --env-file .env exec -T api python -m app.migrate +applied: nothing (up to date) +[exit 0] diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..5be53ef9 --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,67 @@ +# student-tests +# date: 2026-11-11T23:52:05 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/run.py --base-url http://localhost:8080 +== аутентификация == + ok - wrong password -> 401 + Basic challenge + ok - unknown user -> 401 (same as wrong password) + ok - no token -> 401 + Bearer challenge + ok - X-User-Id alone gives no access + ok - unknown/forged token -> 401 + ok - Basic on business endpoint -> 401 + ok - X-User-Id cannot override bearer identity + ok - password shorter than 12 -> 400 +== logout == + ok - logout -> 204 + ok - token after logout -> 401 + ok - second logout -> 401 +== гонка регистрации одного username == + ok - 20 parallel registrations -> one 201, nineteen 409 (got [201, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409, 409]) +== ошибки и доступ (из lab1) == + ok - malformed JSON -> 400 + ok - duplicate username -> 409 + ok - invalid username 'review123\n' -> 400, not 500 (got 400) + ok - invalid username 'review123\r\n' -> 400, not 500 (got 400) + ok - invalid username ' review123' -> 400, not 500 (got 400) + ok - invalid username 'Review123' -> 400, not 500 (got 400) + ok - invalid username 'ab' -> 400, not 500 (got 400) + ok - blank message -> 400 + ok - sender_id in body is rejected + ok - outsider POST to foreign chat -> 404 + ok - outsider GET foreign history -> 404 + ok - invalid chat uuid -> 400 +== пагинация == + ok - 5 messages / limit 2 -> 3 pages, no duplicates + ok - cursor from another chat -> 400 +== Content-Type == + ok - valid JSON with text/plain -> 415 (got 415) + ok - message with text/plain -> 415 (got 415) + ok - text/plain without token -> 401 first (got 401) + ok - application/json; charset=utf-8 accepted (got 201) + +30 passed, 0 failed +[exit 0] + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/idempotency.py --base-url http://localhost:8080 +== 20 параллельных запросов с одним ключом == + ok - all parallel repeats -> 201 (got [201]) + ok - all responses carry the same message id (1 distinct) + ok - requests were served by >=2 replicas (2) + ok - exactly one message in history + ok - every replay returns the identical Message +== семантика ключа == + ok - same key, different body -> 409 (got 409) + ok - other JSON formatting = same request (got 201) + ok - same key in another chat is independent + ok - same key from another user is independent + ok - missing Idempotency-Key -> 400 (got 400) + ok - invalid key 'has space'... -> 400 (got 400) + ok - invalid key 'xxxxxxxxxxxx'... -> 400 (got 400) + ok - invalid key 'slash/key'... -> 400 (got 400) + ok - replay without session -> 401, access re-checked (got 401) +== неопределённый результат: клиент повторяет с тем же ключом == + ok - retry after uncertain result -> 201 + ok - still exactly one message + +16 passed, 0 failed +[exit 0] diff --git a/evidence/upload-memory-10.json b/evidence/upload-memory-10.json new file mode 100644 index 00000000..a8cbf022 --- /dev/null +++ b/evidence/upload-memory-10.json @@ -0,0 +1,391 @@ +{ + "file_bytes": 10026876, + "concurrency": 10, + "rounds": 3, + "uploads": 30, + "status_counts": { + "202": 30 + }, + "sha256_ok": 30, + "instances": [ + "215466c2b510", + "e6cf2ec7aa3b" + ], + "elapsed_s": 13.39, + "throughput_MiB_s": 21.4, + "latency_s": { + "p50": 4.36, + "max": 4.68 + }, + "memory_MiB": { + "idle": { + "e6cf2ec7aa3b": { + "usage": 73.8, + "anon": 68.5, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 73.0, + "anon": 68.3, + "peak": 149.5 + } + }, + "sampled_peak": { + "e6cf2ec7aa3b": { + "usage": 139.6, + "anon": 90.5 + }, + "215466c2b510": { + "usage": 141.4, + "anon": 93.7 + } + }, + "after": { + "e6cf2ec7aa3b": { + "usage": 84.1, + "anon": 78.2, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.3, + "anon": 82.1, + "peak": 151.6 + } + }, + "max_growth_usage": 68.4, + "max_growth_anon": 25.4, + "cgroup_peak_after": { + "e6cf2ec7aa3b": 146.6, + "215466c2b510": 151.6 + } + }, + "container_limit_MiB": 256, + "samples": 25, + "raw_samples": [ + { + "t": 0.35, + "e6cf2ec7aa3b": { + "usage": 77.7, + "anon": 72.3, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 75.4, + "anon": 70.8, + "peak": 149.5 + } + }, + { + "t": 1.03, + "e6cf2ec7aa3b": { + "usage": 88.3, + "anon": 81.9, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 76.6, + "anon": 71.4, + "peak": 149.5 + } + }, + { + "t": 1.82, + "e6cf2ec7aa3b": { + "usage": 89.1, + "anon": 83.2, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 90.2, + "anon": 75.7, + "peak": 149.5 + } + }, + { + "t": 2.55, + "e6cf2ec7aa3b": { + "usage": 88.2, + "anon": 80.5, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 130.1, + "anon": 81.4, + "peak": 149.5 + } + }, + { + "t": 3.22, + "e6cf2ec7aa3b": { + "usage": 96.7, + "anon": 72.2, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 95.6, + "anon": 81.2, + "peak": 149.5 + } + }, + { + "t": 3.8, + "e6cf2ec7aa3b": { + "usage": 108.5, + "anon": 75.9, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 86.7, + "anon": 81.2, + "peak": 149.5 + } + }, + { + "t": 4.31, + "e6cf2ec7aa3b": { + "usage": 90.8, + "anon": 75.8, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 86.6, + "anon": 81.2, + "peak": 149.5 + } + }, + { + "t": 5.23, + "e6cf2ec7aa3b": { + "usage": 83.2, + "anon": 78.9, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 93.2, + "anon": 87.4, + "peak": 149.5 + } + }, + { + "t": 6.23, + "e6cf2ec7aa3b": { + "usage": 84.6, + "anon": 79.3, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 96.5, + "anon": 91.4, + "peak": 149.5 + } + }, + { + "t": 7.1, + "e6cf2ec7aa3b": { + "usage": 99.1, + "anon": 75.9, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 100.5, + "anon": 93.7, + "peak": 149.5 + } + }, + { + "t": 7.73, + "e6cf2ec7aa3b": { + "usage": 119.8, + "anon": 78.0, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 100.2, + "anon": 81.4, + "peak": 149.5 + } + }, + { + "t": 8.32, + "e6cf2ec7aa3b": { + "usage": 139.6, + "anon": 87.8, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 122.5, + "anon": 81.4, + "peak": 149.5 + } + }, + { + "t": 8.81, + "e6cf2ec7aa3b": { + "usage": 123.9, + "anon": 90.5, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 132.4, + "anon": 81.5, + "peak": 149.5 + } + }, + { + "t": 9.53, + "e6cf2ec7aa3b": { + "usage": 87.1, + "anon": 81.0, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.0, + "anon": 81.6, + "peak": 149.5 + } + }, + { + "t": 10.22, + "e6cf2ec7aa3b": { + "usage": 89.1, + "anon": 82.9, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 88.0, + "anon": 82.1, + "peak": 149.5 + } + }, + { + "t": 10.83, + "e6cf2ec7aa3b": { + "usage": 89.4, + "anon": 82.8, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 88.4, + "anon": 82.1, + "peak": 149.5 + } + }, + { + "t": 11.42, + "e6cf2ec7aa3b": { + "usage": 90.0, + "anon": 82.9, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 88.6, + "anon": 82.0, + "peak": 149.5 + } + }, + { + "t": 12.12, + "e6cf2ec7aa3b": { + "usage": 107.6, + "anon": 82.9, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 107.8, + "anon": 82.1, + "peak": 149.5 + } + }, + { + "t": 12.73, + "e6cf2ec7aa3b": { + "usage": 134.3, + "anon": 82.8, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 141.4, + "anon": 91.3, + "peak": 149.7 + } + }, + { + "t": 13.21, + "e6cf2ec7aa3b": { + "usage": 134.4, + "anon": 82.9, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 135.1, + "anon": 93.2, + "peak": 151.6 + } + }, + { + "t": 13.61, + "e6cf2ec7aa3b": { + "usage": 83.9, + "anon": 78.0, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.4, + "anon": 82.1, + "peak": 151.6 + } + }, + { + "t": 14.0, + "e6cf2ec7aa3b": { + "usage": 87.6, + "anon": 81.7, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.2, + "anon": 82.1, + "peak": 151.6 + } + }, + { + "t": 14.51, + "e6cf2ec7aa3b": { + "usage": 94.4, + "anon": 88.6, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.5, + "anon": 82.1, + "peak": 151.6 + } + }, + { + "t": 14.91, + "e6cf2ec7aa3b": { + "usage": 84.2, + "anon": 78.1, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.4, + "anon": 82.1, + "peak": 151.6 + } + }, + { + "t": 15.28, + "e6cf2ec7aa3b": { + "usage": 83.9, + "anon": 78.1, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.3, + "anon": 82.1, + "peak": 151.6 + } + } + ] +} \ No newline at end of file diff --git a/evidence/upload-memory-20.json b/evidence/upload-memory-20.json new file mode 100644 index 00000000..8aab24a0 --- /dev/null +++ b/evidence/upload-memory-20.json @@ -0,0 +1,586 @@ +{ + "file_bytes": 10026613, + "concurrency": 20, + "rounds": 3, + "uploads": 60, + "status_counts": { + "202": 60 + }, + "sha256_ok": 60, + "instances": [ + "215466c2b510", + "e6cf2ec7aa3b" + ], + "elapsed_s": 25.05, + "throughput_MiB_s": 22.9, + "latency_s": { + "p50": 7.18, + "max": 8.63 + }, + "memory_MiB": { + "idle": { + "e6cf2ec7aa3b": { + "usage": 83.9, + "anon": 78.1, + "peak": 147.7 + }, + "215466c2b510": { + "usage": 99.6, + "anon": 94.4, + "peak": 151.6 + } + }, + "sampled_peak": { + "e6cf2ec7aa3b": { + "usage": 153.1, + "anon": 107.1 + }, + "215466c2b510": { + "usage": 146.2, + "anon": 101.9 + } + }, + "after": { + "e6cf2ec7aa3b": { + "usage": 93.7, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 95.5, + "anon": 90.3, + "peak": 212.6 + } + }, + "max_growth_usage": 69.2, + "max_growth_anon": 29.0, + "cgroup_peak_after": { + "e6cf2ec7aa3b": 203.8, + "215466c2b510": 212.6 + } + }, + "container_limit_MiB": 256, + "samples": 40, + "raw_samples": [ + { + "t": 0.6, + "e6cf2ec7aa3b": { + "usage": 87.7, + "anon": 82.9, + "peak": 147.7 + }, + "215466c2b510": { + "usage": 100.1, + "anon": 94.4, + "peak": 151.6 + } + }, + { + "t": 1.39, + "e6cf2ec7aa3b": { + "usage": 100.4, + "anon": 94.0, + "peak": 147.7 + }, + "215466c2b510": { + "usage": 89.9, + "anon": 84.6, + "peak": 151.6 + } + }, + { + "t": 2.34, + "e6cf2ec7aa3b": { + "usage": 103.4, + "anon": 96.9, + "peak": 147.7 + }, + "215466c2b510": { + "usage": 91.3, + "anon": 85.0, + "peak": 151.6 + } + }, + { + "t": 3.2, + "e6cf2ec7aa3b": { + "usage": 106.0, + "anon": 99.4, + "peak": 153.3 + }, + "215466c2b510": { + "usage": 92.2, + "anon": 85.1, + "peak": 155.1 + } + }, + { + "t": 3.79, + "e6cf2ec7aa3b": { + "usage": 93.4, + "anon": 87.0, + "peak": 166.3 + }, + "215466c2b510": { + "usage": 91.2, + "anon": 85.4, + "peak": 183.6 + } + }, + { + "t": 4.39, + "e6cf2ec7aa3b": { + "usage": 94.8, + "anon": 86.8, + "peak": 186.6 + }, + "215466c2b510": { + "usage": 97.8, + "anon": 81.0, + "peak": 189.1 + } + }, + { + "t": 5.0, + "e6cf2ec7aa3b": { + "usage": 98.6, + "anon": 82.7, + "peak": 190.6 + }, + "215466c2b510": { + "usage": 116.1, + "anon": 81.1, + "peak": 189.1 + } + }, + { + "t": 5.71, + "e6cf2ec7aa3b": { + "usage": 128.4, + "anon": 86.8, + "peak": 192.6 + }, + "215466c2b510": { + "usage": 119.8, + "anon": 81.0, + "peak": 189.1 + } + }, + { + "t": 6.63, + "e6cf2ec7aa3b": { + "usage": 153.1, + "anon": 86.8, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 146.2, + "anon": 93.0, + "peak": 189.1 + } + }, + { + "t": 7.2, + "e6cf2ec7aa3b": { + "usage": 137.7, + "anon": 86.9, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 133.1, + "anon": 94.5, + "peak": 189.1 + } + }, + { + "t": 7.59, + "e6cf2ec7aa3b": { + "usage": 138.2, + "anon": 86.8, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 115.8, + "anon": 83.2, + "peak": 189.1 + } + }, + { + "t": 8.05, + "e6cf2ec7aa3b": { + "usage": 92.7, + "anon": 86.7, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 88.4, + "anon": 83.2, + "peak": 189.1 + } + }, + { + "t": 8.91, + "e6cf2ec7aa3b": { + "usage": 94.9, + "anon": 88.3, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 93.5, + "anon": 87.8, + "peak": 189.1 + } + }, + { + "t": 9.8, + "e6cf2ec7aa3b": { + "usage": 107.2, + "anon": 100.4, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 90.6, + "anon": 84.3, + "peak": 189.1 + } + }, + { + "t": 10.62, + "e6cf2ec7aa3b": { + "usage": 112.1, + "anon": 105.3, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 92.0, + "anon": 85.4, + "peak": 189.1 + } + }, + { + "t": 11.42, + "e6cf2ec7aa3b": { + "usage": 101.0, + "anon": 92.8, + "peak": 194.6 + }, + "215466c2b510": { + "usage": 93.9, + "anon": 85.7, + "peak": 190.7 + } + }, + { + "t": 12.24, + "e6cf2ec7aa3b": { + "usage": 96.4, + "anon": 87.3, + "peak": 196.8 + }, + "215466c2b510": { + "usage": 112.5, + "anon": 86.1, + "peak": 191.2 + } + }, + { + "t": 12.92, + "e6cf2ec7aa3b": { + "usage": 117.1, + "anon": 87.5, + "peak": 196.8 + }, + "215466c2b510": { + "usage": 121.2, + "anon": 86.8, + "peak": 193.7 + } + }, + { + "t": 13.72, + "e6cf2ec7aa3b": { + "usage": 139.4, + "anon": 87.5, + "peak": 196.8 + }, + "215466c2b510": { + "usage": 140.0, + "anon": 97.5, + "peak": 193.7 + } + }, + { + "t": 14.39, + "e6cf2ec7aa3b": { + "usage": 150.9, + "anon": 89.5, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 135.7, + "anon": 99.2, + "peak": 193.7 + } + }, + { + "t": 14.99, + "e6cf2ec7aa3b": { + "usage": 131.9, + "anon": 89.4, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 133.7, + "anon": 100.6, + "peak": 193.7 + } + }, + { + "t": 15.37, + "e6cf2ec7aa3b": { + "usage": 131.9, + "anon": 89.5, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 130.2, + "anon": 88.3, + "peak": 193.7 + } + }, + { + "t": 15.86, + "e6cf2ec7aa3b": { + "usage": 132.2, + "anon": 89.6, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 130.4, + "anon": 88.3, + "peak": 193.7 + } + }, + { + "t": 16.21, + "e6cf2ec7aa3b": { + "usage": 123.1, + "anon": 90.1, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 130.0, + "anon": 88.3, + "peak": 193.7 + } + }, + { + "t": 17.13, + "e6cf2ec7aa3b": { + "usage": 105.8, + "anon": 99.8, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 93.6, + "anon": 88.2, + "peak": 193.7 + } + }, + { + "t": 18.01, + "e6cf2ec7aa3b": { + "usage": 113.1, + "anon": 106.9, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 93.5, + "anon": 87.6, + "peak": 193.7 + } + }, + { + "t": 18.83, + "e6cf2ec7aa3b": { + "usage": 114.2, + "anon": 107.1, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 96.2, + "anon": 90.2, + "peak": 193.7 + } + }, + { + "t": 19.73, + "e6cf2ec7aa3b": { + "usage": 101.4, + "anon": 94.7, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 97.1, + "anon": 90.1, + "peak": 193.7 + } + }, + { + "t": 20.63, + "e6cf2ec7aa3b": { + "usage": 102.9, + "anon": 94.7, + "peak": 198.8 + }, + "215466c2b510": { + "usage": 105.2, + "anon": 99.3, + "peak": 195.3 + } + }, + { + "t": 21.5, + "e6cf2ec7aa3b": { + "usage": 114.6, + "anon": 94.7, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 108.6, + "anon": 100.3, + "peak": 207.7 + } + }, + { + "t": 22.21, + "e6cf2ec7aa3b": { + "usage": 128.8, + "anon": 94.6, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 126.7, + "anon": 101.9, + "peak": 209.1 + } + }, + { + "t": 22.9, + "e6cf2ec7aa3b": { + "usage": 146.2, + "anon": 94.8, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 131.3, + "anon": 90.2, + "peak": 212.6 + } + }, + { + "t": 23.7, + "e6cf2ec7aa3b": { + "usage": 147.4, + "anon": 87.8, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 136.4, + "anon": 90.1, + "peak": 212.6 + } + }, + { + "t": 24.43, + "e6cf2ec7aa3b": { + "usage": 113.2, + "anon": 97.6, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 129.8, + "anon": 90.2, + "peak": 212.6 + } + }, + { + "t": 25.04, + "e6cf2ec7aa3b": { + "usage": 94.1, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 122.8, + "anon": 90.3, + "peak": 212.6 + } + }, + { + "t": 25.38, + "e6cf2ec7aa3b": { + "usage": 93.8, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 95.6, + "anon": 90.3, + "peak": 212.6 + } + }, + { + "t": 25.8, + "e6cf2ec7aa3b": { + "usage": 93.7, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 95.9, + "anon": 90.3, + "peak": 212.6 + } + }, + { + "t": 26.19, + "e6cf2ec7aa3b": { + "usage": 93.8, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 95.6, + "anon": 90.3, + "peak": 212.6 + } + }, + { + "t": 26.56, + "e6cf2ec7aa3b": { + "usage": 93.8, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 95.6, + "anon": 90.2, + "peak": 212.6 + } + }, + { + "t": 26.97, + "e6cf2ec7aa3b": { + "usage": 93.9, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 95.6, + "anon": 90.3, + "peak": 212.6 + } + } + ] +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 00000000..6ac77b05 --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,156 @@ +# upload-memory +# date: 2026-11-12T15:04:54 commit: 276dbda + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 10 --output evidence/upload-memory-10.json +file: 10026876 bytes (9.56 MiB), sha256 4582d3a7982b… +idle MiB: {'e6cf2ec7aa3b': {'usage': 73.8, 'anon': 68.5, 'peak': 141.4}, '215466c2b510': {'usage': 73.0, 'anon': 68.3, 'peak': 149.5}} +{ + "file_bytes": 10026876, + "concurrency": 10, + "rounds": 3, + "uploads": 30, + "status_counts": { + "202": 30 + }, + "sha256_ok": 30, + "instances": [ + "215466c2b510", + "e6cf2ec7aa3b" + ], + "elapsed_s": 13.39, + "throughput_MiB_s": 21.4, + "latency_s": { + "p50": 4.36, + "max": 4.68 + }, + "memory_MiB": { + "idle": { + "e6cf2ec7aa3b": { + "usage": 73.8, + "anon": 68.5, + "peak": 141.4 + }, + "215466c2b510": { + "usage": 73.0, + "anon": 68.3, + "peak": 149.5 + } + }, + "sampled_peak": { + "e6cf2ec7aa3b": { + "usage": 139.6, + "anon": 90.5 + }, + "215466c2b510": { + "usage": 141.4, + "anon": 93.7 + } + }, + "after": { + "e6cf2ec7aa3b": { + "usage": 84.1, + "anon": 78.2, + "peak": 146.6 + }, + "215466c2b510": { + "usage": 87.3, + "anon": 82.1, + "peak": 151.6 + } + }, + "max_growth_usage": 68.4, + "max_growth_anon": 25.4, + "cgroup_peak_after": { + "e6cf2ec7aa3b": 146.6, + "215466c2b510": 151.6 + } + }, + "container_limit_MiB": 256 +} + ok - download c8359ca8 sha256 matches + ok - download fcda2c0f sha256 matches + ok - download 6c733ea4 sha256 matches + ok - all 30 uploads -> 202 + ok - sha256 of every upload matches + ok - peak process memory (anon) growth per replica 25.4 MiB < 100 MiB (naive buffering: ≥ 5 × 9.6 MiB × 2) +saved evidence/upload-memory-10.json + +6 passed, 0 failed +[exit 0] + +$ C:\Users\79152\AppData\Local\Python\pythoncore-3.14-64\python.exe student-tests/upload_memory.py --base-url http://localhost:8080 --concurrency 20 --output evidence/upload-memory-20.json +file: 10026613 bytes (9.56 MiB), sha256 8bb6a49a0703… +idle MiB: {'e6cf2ec7aa3b': {'usage': 83.9, 'anon': 78.1, 'peak': 147.7}, '215466c2b510': {'usage': 99.6, 'anon': 94.4, 'peak': 151.6}} +{ + "file_bytes": 10026613, + "concurrency": 20, + "rounds": 3, + "uploads": 60, + "status_counts": { + "202": 60 + }, + "sha256_ok": 60, + "instances": [ + "215466c2b510", + "e6cf2ec7aa3b" + ], + "elapsed_s": 25.05, + "throughput_MiB_s": 22.9, + "latency_s": { + "p50": 7.18, + "max": 8.63 + }, + "memory_MiB": { + "idle": { + "e6cf2ec7aa3b": { + "usage": 83.9, + "anon": 78.1, + "peak": 147.7 + }, + "215466c2b510": { + "usage": 99.6, + "anon": 94.4, + "peak": 151.6 + } + }, + "sampled_peak": { + "e6cf2ec7aa3b": { + "usage": 153.1, + "anon": 107.1 + }, + "215466c2b510": { + "usage": 146.2, + "anon": 101.9 + } + }, + "after": { + "e6cf2ec7aa3b": { + "usage": 93.7, + "anon": 87.9, + "peak": 203.8 + }, + "215466c2b510": { + "usage": 95.5, + "anon": 90.3, + "peak": 212.6 + } + }, + "max_growth_usage": 69.2, + "max_growth_anon": 29.0, + "cgroup_peak_after": { + "e6cf2ec7aa3b": 203.8, + "215466c2b510": 212.6 + } + }, + "container_limit_MiB": 256 +} + ok - download 56cafe16 sha256 matches + ok - download 6a3c88ac sha256 matches + ok - download 795dd71a sha256 matches + ok - all 60 uploads -> 202 + ok - sha256 of every upload matches + ok - peak process memory (anon) growth per replica 29.0 MiB < 100 MiB (naive buffering: ≥ 10 × 9.6 MiB × 2) +saved evidence/upload-memory-20.json + +6 passed, 0 failed +[exit 0] diff --git a/evidence/worker-limits.txt b/evidence/worker-limits.txt new file mode 100644 index 00000000..3f97bd02 --- /dev/null +++ b/evidence/worker-limits.txt @@ -0,0 +1,12 @@ +# worker-limits +# date: 2026-11-11T23:54:14 commit: 276dbda + +$ docker compose --env-file .env exec -T api python -m app.tools limits-selftest + ok normal 320x200 with alpha: ok (3.3s) + ok pixel limit (36 Mp > limit 10 Mp): invalid_image: pixel limit (3.8s) + ok memory limit (36 Mp, RLIMIT_AS 120 MiB): invalid_image: memory limit (2.0s) + ok corrupt/truncated PNG: invalid_image: cannot decode: OSError (2.9s) + ok time limit (hangs 10 s, timeout 2 s): timeout(killed) (2.0s) + +5 passed, 0 failed +[exit 0] diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..285590d9 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,84 @@ +# Балансировщик lab3/lab4: единственная публичная точка входа (127.0.0.1:8080). +# Реплики API находятся через DNS Docker по имени сервиса "api" +# (server-template + resolvers): добавление/удаление контейнеров подхватывается +# без перезапуска LB. Активная проверка /health/ready исключает недоступную реплику. +global + log stdout format raw local0 info + maxconn 2048 + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + hold other 5s + hold refused 5s + hold nx 5s + hold timeout 5s + hold obsolete 5s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + # Короткий connect: остановленный контейнер не отвечает на SYN, и без этого + # запрос ждал бы 2 c до повтора на другой реплике. + timeout connect 500ms + timeout client 30s + timeout server 30s + timeout http-request 10s + # Повтор только при ошибке УСТАНОВКИ соединения (запрос ещё не отправлен) — + # безопасно даже для POST; redispatch отправляет повтор на другую реплику. + retries 3 + option redispatch + +frontend fe_api + bind :8080 + default_backend api + +# Отдельный вход для теста истечения сессии (профиль ttl): тоже только через LB. +frontend fe_api_ttl + bind :8081 + default_backend api_ttl + +# lab4: вход в реплику с внедрением отказов (профиль fault) — только для тестов +# компенсации/очистки (student-tests/orphans.py). Обычный трафик сюда не идёт. +frontend fe_api_fault + bind :8082 + default_backend api_fault + +# Проверка живости самого LB для healthcheck контейнера. +frontend fe_health + bind :8404 + monitor-uri /healthz + +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + # Проверка раз в 0,5 c; 2 неудачи подряд → реплика исключается (≈1–1,5 c), + # 2 успеха подряд → возвращается. + default-server inter 500ms fall 2 rise 2 resolvers docker init-addr none + server-template api 6 api:8080 check + # Запасной путь, только если НИ ОДНА реплика не готова (например, упала общая БД): + # тогда LB всё равно передаёт запрос в живой процесс, и API отвечает по контракту + # (401 анониму, 503 unavailable в формате ошибки, /health/live 200), а не голым 503 LB. + # Остановленный контейнер исчезает из DNS и сюда не попадает. + option allbackups + # allow-dup-ip: те же IP, что у основных серверов (по умолчанию HAProxy не даёт + # одному IP достаться двум серверам бэкенда, и резервные остались бы без адреса). + server-template fallback 6 api:8080 backup resolve-opts allow-dup-ip + +backend api_ttl + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 500ms fall 2 rise 2 resolvers docker init-addr none + server-template ttl 2 api-ttl:8080 + +backend api_fault + option httpchk GET /health/ready + http-check expect status 200 + default-server check inter 500ms fall 2 rise 2 resolvers docker init-addr none + server-template fault 1 api-fault:8080 diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..2ddab729 --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,56 @@ +#!/bin/sh +# Идемпотентный bootstrap MinIO (сервис s3-init, запускается при каждом `up`). +# 1. bucket (если уже есть — не ошибка); +# 2. bucket приватный: анонимный доступ «none»; +# 3. политика lab4-api-rw: только Get/Put/Delete объектов и List этого bucket; +# 4. отдельный пользователь API (S3_ACCESS_KEY_ID) с этой политикой. +# Root-пользователь MinIO нужен только здесь. Секреты не печатаются. +set -eu +: "${MINIO_ROOT_USER:?}" "${MINIO_ROOT_PASSWORD:?}" "${S3_BUCKET:?}" +: "${S3_ACCESS_KEY_ID:?}" "${S3_SECRET_ACCESS_KEY:?}" +if [ "$S3_ACCESS_KEY_ID" = "$MINIO_ROOT_USER" ]; then + echo "S3_ACCESS_KEY_ID must differ from MINIO_ROOT_USER (least privilege)" >&2 + exit 1 +fi + +mc alias set root http://s3:9000 "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" >/dev/null + +mc mb --ignore-existing "root/$S3_BUCKET" +mc anonymous set none "root/$S3_BUCKET" + +POLICY_NAME=lab4-api-rw +cat > /tmp/policy.json </dev/null +# add для существующего пользователя обновляет секрет — повтор безопасен. +mc admin user add root "$S3_ACCESS_KEY_ID" "$S3_SECRET_ACCESS_KEY" >/dev/null +# attach для уже привязанной политики возвращает ошибку — это не сбой, проверяем итог ниже. +if mc admin policy attach root "$POLICY_NAME" --user "$S3_ACCESS_KEY_ID" >/dev/null 2>&1; then + echo "policy $POLICY_NAME attached to API user" +else + echo "policy $POLICY_NAME was already attached (attach is a no-op)" +fi +info=$(mc admin user info root "$S3_ACCESS_KEY_ID" --json) +case "$info" in + *\"policyName\"*"$POLICY_NAME"*) echo "verified: API user has policy $POLICY_NAME" ;; + *) echo "API user has NO policy $POLICY_NAME: $info" >&2; exit 1 ;; +esac + +mc anonymous get "root/$S3_BUCKET" +echo "bootstrap done: bucket=$S3_BUCKET (private), API user restricted by policy $POLICY_NAME" diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..229ff8e4 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,54 @@ +-- 0001: начальная схема мессенджера (lab2). +-- Источник истины для пользователей, чатов, участников, сообщений и сессий. + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- Участники чата. position сохраняет порядок member_ids из запроса. +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id) +); +-- «Мои чаты»: найти чаты пользователя без полного скана. +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); +-- Keyset-пагинация истории: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) +-- ORDER BY created_at, id LIMIT n — индекс отдаёт страницу без скана всей истории. +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +-- Сессии. Храним только SHA-256 от токена: утечка таблицы не даёт рабочих токенов. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); +CREATE INDEX sessions_user_idx ON sessions (user_id); +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..9f4bf84f --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,17 @@ +-- 0002: общая идемпотентность POST /chats/{chat_id}/messages (lab3). +-- Область уникальности ключа: пользователь + чат (метод и путь фиксированы) + ключ. +-- Миграция только добавляет таблицу: старая версия API (lab2) с ней совместима. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, -- SHA-256 канонического тела запроса + -- Ключ вставляется ДО сообщения в той же транзакции: проверка FK — при COMMIT. + message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, idem_key), + CONSTRAINT idempotency_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$') +); +-- Для очистки ключей старше IDEMPOTENCY_TTL_SECONDS. +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 00000000..bc456a25 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,56 @@ +-- 0003: вложения (lab4). Метаданные — здесь, байты — в приватном S3 bucket. +-- Миграция только добавляет таблицы и ослабляет CHECK текста сообщения: +-- работающая версия lab3 с новой схемой совместима (rolling update возможен). + +-- Текст может быть пустым, если у сообщения есть вложения (проверяет API). +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users (id), + -- Только для отображения; в путь/ключ S3 не попадает. + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + status text NOT NULL, + error_code text, + -- Ключ объекта назначает сервер (originals/); уникален. + object_key text NOT NULL, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_dimensions CHECK (width > 0 AND height > 0), + CONSTRAINT attachments_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')) +); +CREATE INDEX attachments_chat_idx ON attachments (chat_id, created_at, id); + +-- Связь сообщение ↔ вложения. position сохраняет порядок attachment_ids из запроса. +-- Одно вложение чата можно приложить к нескольким сообщениям этого чата. +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id), + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position CHECK (position BETWEEN 0 AND 9) +); +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); + +-- Журнал загрузок (компенсация отказа между S3 и PostgreSQL). +-- Строка пишется и коммитится ДО PUT в S3, удаляется в одной транзакции +-- с INSERT в attachments. Строка старше grace period = брошенная загрузка: +-- janitor удаляет её объект в S3 (если он есть) и саму строку. +CREATE TABLE upload_intents ( + object_key text PRIMARY KEY, + attachment_id uuid NOT NULL, + created_at timestamptz NOT NULL +); +CREATE INDEX upload_intents_created_idx ON upload_intents (created_at); diff --git a/migrations/0004_jobs.sql b/migrations/0004_jobs.sql new file mode 100644 index 00000000..4ee95234 --- /dev/null +++ b/migrations/0004_jobs.sql @@ -0,0 +1,88 @@ +-- 0004: фоновая обработка вложений (lab5). +-- Только добавляет таблицы/колонки: старая версия API (lab4) с этой схемой работает. + +-- Где лежит принятый, но ещё не опубликованный оригинал (приватный prefix staging/). +-- object_key — КОНЕЧНЫЙ ключ originals/: объект там появляется, когда worker его опубликует. +-- После безопасной фиксации результата staging удаляется и колонка обнуляется. +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD CONSTRAINT attachments_staging_key_key UNIQUE (staging_key); + +-- Одна логическая работа на (вложение, версия pipeline). job_id не меняется при redelivery. +-- attempt — номер управляемой попытки (растёт при retry, lease expiry и replay). +-- lease_token — «забор» (fencing): зафиксировать результат может только владелец +-- текущей аренды; устаревший worker после пробуждения ничего не перезапишет. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + status text NOT NULL, + attempt integer NOT NULL DEFAULT 1, + max_attempts integer NOT NULL, + lease_token uuid, + lease_owner text, + lease_until timestamptz, + next_attempt_at timestamptz, + error_code text, + last_error text, + replays integer NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + finished_at timestamptz, + CONSTRAINT attachment_jobs_effect_key UNIQUE (attachment_id, pipeline_version), + CONSTRAINT attachment_jobs_status CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + CONSTRAINT attachment_jobs_attempt CHECK (attempt >= 1 AND attempt <= max_attempts), + CONSTRAINT attachment_jobs_pipeline CHECK (pipeline_version >= 1), + CONSTRAINT attachment_jobs_lease CHECK (status <> 'processing' OR (lease_token IS NOT NULL AND lease_until IS NOT NULL)) +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (finished_at) WHERE status = 'failed'; + +-- Transactional outbox: строка пишется в ТОЙ ЖЕ транзакции, что вложение и job. +-- Пока published_at IS NULL, relay опубликует её в брокер (повтор безопасен: worker идемпотентен). +-- available_at в будущем — отложенная попытка (backoff с jitter). +CREATE TABLE job_outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + attempt integer NOT NULL, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + created_at timestamptz NOT NULL, + published_at timestamptz, + stream_id text, + publish_attempts integer NOT NULL DEFAULT 0, + last_error text +); +CREATE INDEX job_outbox_pending_idx ON job_outbox (available_at) WHERE published_at IS NULL; + +-- Результаты. Ключ эффекта — (attachment_id, pipeline_version, variant): +-- повтор job делает ON CONFLICT DO NOTHING и не создаёт второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + variant text NOT NULL, + object_key text NOT NULL, + content_type text NOT NULL, + width integer NOT NULL, + height integer NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, variant), + CONSTRAINT attachment_variants_object_key UNIQUE (object_key), + CONSTRAINT attachment_variants_name CHECK (variant IN ('thumbnail', 'crop', 'optimized')), + CONSTRAINT attachment_variants_type CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + CONSTRAINT attachment_variants_dims CHECK (width > 0 AND height > 0 AND size_bytes > 0), + CONSTRAINT attachment_variants_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$') +); + +-- Журнал событий job: реестр для оператора (failed, replay, takeover) без секретов. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id) ON DELETE CASCADE, + at timestamptz NOT NULL DEFAULT now(), + event text NOT NULL, + attempt integer, + actor text, + detail text +); +CREATE INDEX job_events_job_idx ON job_events (job_id, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7e05721e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +# Рантайм-зависимости, версии закреплены для воспроизводимой сборки. +fastapi==0.141.1 +uvicorn[standard]==0.46.0 +pydantic==2.13.3 +psycopg[binary]==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==8.1.0 +# lab4: S3-клиент (MinIO) и проверка изображений +boto3==1.38.0 +botocore==1.38.0 +pillow==11.2.1 diff --git a/scripts/check-s3-private.cmd b/scripts/check-s3-private.cmd new file mode 100644 index 00000000..8ac03c77 --- /dev/null +++ b/scripts/check-s3-private.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows: same as check-s3-private.sh +cd /d "%~dp0.." +docker compose --env-file .env exec -T api python -m app.tools private-check %* diff --git a/scripts/check-s3-private.sh b/scripts/check-s3-private.sh new file mode 100644 index 00000000..0c00545f --- /dev/null +++ b/scripts/check-s3-private.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Проверка приватности bucket и ограниченных прав API ВНЕ API-приложения: +# прямые HTTP-запросы к MinIO без подписи и с ключами API-пользователя. +# Запросы идут из контейнера api (он в сети storage), но НЕ через код эндпоинтов. +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env exec -T api python -m app.tools private-check "$@" diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..377062d2 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +rem Restart hook (Windows): same as restart.sh, volumes are kept. +cd /d "%~dp0.." +docker compose --env-file .env restart postgres redis broker s3 api worker lb diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..2e665a5f --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Restart hook для make test-persistence: перезапускает ВСЕ реплики API, LB и +# хранилища (PostgreSQL, Redis, брокер, MinIO) и worker. Volumes сохраняются; ничего не удаляется и не сидится. +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres redis broker s3 api worker lb diff --git a/scripts/restore-api.cmd b/scripts/restore-api.cmd new file mode 100644 index 00000000..e63eb7a0 --- /dev/null +++ b/scripts/restore-api.cmd @@ -0,0 +1,4 @@ +@echo off +rem Restore stopped api replicas after the failover test. +cd /d "%~dp0.." +docker compose --env-file .env start api diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100644 index 00000000..f5d4a8a5 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Вернуть остановленные реплики API после failover-теста. +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env start api diff --git a/scripts/start-worker.cmd b/scripts/start-worker.cmd new file mode 100644 index 00000000..2be81f53 --- /dev/null +++ b/scripts/start-worker.cmd @@ -0,0 +1,4 @@ +@echo off +rem lab5: start the worker again; accepted jobs are processed to ready. +cd /d "%~dp0.." +docker compose --env-file .env start worker diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100644 index 00000000..ebf836e7 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# lab5: запустить worker; принятые задачи доходят до ready. +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env start worker diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..69f73d98 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,7 @@ +@echo off +rem Failover hook (Windows): stop ONE api container whose short ID = TARGET_INSTANCE_ID. +if "%TARGET_INSTANCE_ID%"=="" exit /b 2 +set CID= +for /f %%i in ('docker ps -q --filter "id=%TARGET_INSTANCE_ID%" --filter "label=com.docker.compose.service=api"') do set CID=%%i +if "%CID%"=="" exit /b 1 +docker stop -t 8 %CID% >nul diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..29224717 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,12 @@ +#!/bin/sh +# Failover hook для make test-failover. +# X-Instance-Id реплики = hostname контейнера = короткий ID контейнера Docker. +# Останавливаем РОВНО этот контейнер, и только если он относится к сервису api +# (БД, Redis и LB этот скрипт не трогает). Реплика остаётся остановленной; +# вернуть её: scripts/restore-api.sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +cid=$(docker ps -q --filter "id=${TARGET_INSTANCE_ID}" --filter "label=com.docker.compose.service=api") +[ -n "$cid" ] || { echo "no running api container for ${TARGET_INSTANCE_ID}" >&2; exit 1; } +docker stop -t 8 "$cid" >/dev/null diff --git a/scripts/stop-worker.cmd b/scripts/stop-worker.cmd new file mode 100644 index 00000000..8e50f040 --- /dev/null +++ b/scripts/stop-worker.cmd @@ -0,0 +1,4 @@ +@echo off +rem lab5: stop the worker (demo: upload is still accepted with 202 queued). +cd /d "%~dp0.." +docker compose --env-file .env stop worker diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100644 index 00000000..0e17fc02 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# lab5: остановить worker (демонстрация: upload всё равно 202 queued). +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env stop worker diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 00000000..a640dd2f --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +""" +Вложения (lab4, адаптировано под lab5): позитивный сценарий и негативный набор. +С lab5 успешный upload — 202 queued; байты и варианты доступны после ready. + + python student-tests/attachments.py --base-url http://localhost:8080 + +Проверяет (каждый пользовательский промах — ожидаемый 4xx, НИКОГДА не 500): + - PNG и JPEG: 202 queued → ready, sha256/size, скачивание побайтно вторым участником, + истинный Content-Type, Unicode filename, 404 постороннему (метаданные и байты); + - слишком большой файл: по Content-Length (тело не отправляется), chunked без + Content-Length больше лимита, файл ровно в лимит проходит; + - «бомба» по пикселям (маленький файл, > 40 млн пикселей) → 413; + - повреждённые изображения (усечённый PNG/JPEG, битый CRC, мусор) → 415; + - подмена MIME (PNG как image/jpeg, JPEG как image/png, текст/GIF/HTML как png) → 415; + - опасные filename (/, \\, NUL, управляющие, bidi, .., пусто, > 255) → 400; + - чужое attachment_id / вложение другого чата / несуществующее → 404; + дубликаты, > 10 id, не-UUID → 400; пустой текст без вложений → 400; + - идемпотентность с attachment_ids: отсутствие поля == [], другой набор → 409. +""" +import argparse +import uuid +from urllib.parse import quote + +from common import (SAMPLE_JPG, SAMPLE_PNG, Api, Report, make_png, raw_http, sha256, + upload, wait_ready) + +LIMIT = 10 * 1024 * 1024 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + args = ap.parse_args() + api, r = Api(args.base_url), Report() + statuses: list[int] = [] + + def expect(st, want, name): + statuses.append(st) + want = want if isinstance(want, tuple) else (want,) + r.check(st in want, f"{name} -> {'/'.join(map(str, want))} (got {st})") + + alice, bob, eve = api.new_user("att_a"), api.new_user("att_b"), api.new_user("att_e") + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "files", "member_ids": [alice["id"], bob["id"]]}, + token=alice["token"]) + st, other, _ = api.call("POST", "/api/v1/chats", {"title": "alice second", "member_ids": [alice["id"]]}, + token=alice["token"]) + st, evechat, _ = api.call("POST", "/api/v1/chats", {"title": "eve", "member_ids": [eve["id"]]}, + token=eve["token"]) + A, B, E = alice["token"], bob["token"], eve["token"] + mpath = f"/api/v1/chats/{chat['id']}/messages" + + print("== позитивный сценарий ==") + st, png, _ = upload(api, A, chat["id"], SAMPLE_PNG, "sample.png") + expect(st, 202, "upload PNG") + r.check(png and png["status"] == "queued" and png["variants"] == [] and png["error_code"] is None, + "202 queued, variants=[], error_code=null") + r.check(png and png["sha256"] == sha256(SAMPLE_PNG) and png["size_bytes"] == len(SAMPLE_PNG), + "sha256/size describe the original") + r.check(png and "object_key" not in png and "bucket" not in str(png), "no S3 key/bucket in public JSON") + st, jpg, _ = upload(api, B, chat["id"], SAMPLE_JPG, "фото 📷 отпуск.jpg", "image/jpeg") + expect(st, 202, "upload JPEG with Unicode filename") + r.check(jpg and jpg["filename"] == "фото 📷 отпуск.jpg" and jpg["content_type"] == "image/jpeg", + "filename returned as is (display only)") + png_ready, jpg_ready = wait_ready(api, B, png["id"]), wait_ready(api, A, jpg["id"]) + r.check(png_ready["status"] == jpg_ready["status"] == "ready", "both become ready (worker)") + st, raw, rh = api.raw("GET", f"/api/v1/attachments/{png['id']}/content", headers={"Authorization": "Bearer " + B}) + expect(st, 200, "member downloads PNG") + r.check(raw == SAMPLE_PNG and sha256(raw) == png["sha256"], "downloaded bytes are identical (sha256)") + r.check(rh.get("Content-Type", "").startswith("image/png"), "true Content-Type image/png") + r.check(rh.get("X-Content-Type-Options") == "nosniff", "nosniff header") + st, raw, rh = api.raw("GET", f"/api/v1/attachments/{jpg['id']}/content?variant=original", + headers={"Authorization": "Bearer " + A}) + expect(st, 200, "member downloads JPEG (variant=original)") + r.check(raw == SAMPLE_JPG and rh.get("Content-Type", "").startswith("image/jpeg"), "JPEG bytes and type") + st, msg, _ = api.call("POST", mpath, {"text": "", "attachment_ids": [png["id"], jpg["id"]]}, token=A) + expect(st, 201, "message with empty text + 2 attachments") + r.check(msg and msg["attachment_ids"] == [png["id"], jpg["id"]] and msg["text"] == "", + "attachment_ids kept in request order") + st, page, _ = api.call("GET", mpath, token=B) + got = [m for m in (page or {}).get("items", []) if m["id"] == (msg or {}).get("id")] + r.check(len(got) == 1 and got[0]["attachment_ids"] == [png["id"], jpg["id"]], "history shows attachment_ids") + st, plain, _ = api.call("POST", mpath, {"text": "просто текст"}, token=B) + r.check(st == 201 and plain["attachment_ids"] == [], "text-only POST (lab3) still valid, attachment_ids=[]") + st, meta, _ = api.call("GET", f"/api/v1/attachments/{png['id']}", token=B) + expect(st, 200, "member reads metadata") + st, raw, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}/content?variant=thumbnail", + headers={"Authorization": "Bearer " + B}) + expect(st, 200, "thumbnail variant exists in lab5") + st, _, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}/content?variant=bogus", + headers={"Authorization": "Bearer " + B}) + expect(st, 400, "unknown variant name") + + print("== доступ ==") + for suffix in ("", "/content"): + st, _, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}{suffix}", headers={"Authorization": "Bearer " + E}) + expect(st, 404, f"outsider GET attachment{suffix or ' metadata'}") + st, _, _ = api.raw("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", headers={"Authorization": "Bearer " + A}) + expect(st, 404, f"unknown attachment{suffix or ' metadata'}") + st, _, _ = api.raw("GET", f"/api/v1/attachments/{png['id']}/content") + expect(st, 401, "anonymous download") + st, _, _ = api.raw("GET", "/api/v1/attachments/not-a-uuid", headers={"Authorization": "Bearer " + A}) + expect(st, 400, "attachment id is not a UUID") + st, _, _ = upload(api, E, chat["id"], SAMPLE_PNG) + expect(st, 404, "outsider uploads into foreign chat") + st, _, _ = api.raw("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=a.png", SAMPLE_PNG, + {"Content-Type": "image/png"}) + expect(st, 401, "anonymous upload") + + print("== привязка к сообщению ==") + st, eve_att, _ = upload(api, E, evechat["id"], SAMPLE_PNG) + expect(st, 202, "eve uploads into her own chat") + st, other_att, _ = upload(api, A, other["id"], SAMPLE_PNG) + expect(st, 202, "alice uploads into her second chat") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [eve_att["id"]]}, token=A) + expect(st, 404, "foreign attachment_id (eve's chat) in message") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [other_att["id"]]}, token=A) + expect(st, 404, "attachment of ANOTHER chat of the same user") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, token=A) + expect(st, 404, "unknown attachment_id") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [png["id"], eve_att["id"]]}, token=A) + expect(st, 404, "one valid + one foreign -> whole message rejected") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [png["id"], png["id"]]}, token=A) + expect(st, 400, "duplicate attachment_ids") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [png["id"].upper(), png["id"]]}, token=A) + expect(st, 400, "same UUID in different case is a duplicate") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, token=A) + expect(st, 400, "more than 10 attachment_ids") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": ["nope"]}, token=A) + expect(st, 400, "attachment_id is not a UUID") + st, _, _ = api.call("POST", mpath, {"text": "x", "attachment_ids": None}, token=A) + expect(st, 400, "attachment_ids: null") + st, _, _ = api.call("POST", mpath, {"text": " ", "attachment_ids": []}, token=A) + expect(st, 400, "blank text without attachments") + st, _, _ = api.call("POST", mpath, {"attachment_ids": [png["id"]]}, token=A) + expect(st, 400, "text field is required") + st, m2, _ = api.call("POST", mpath, {"text": "ещё раз", "attachment_ids": [png["id"]]}, token=B) + expect(st, 201, "any member may reference a chat attachment again") + st, _, _ = api.call("POST", f"/api/v1/chats/{evechat['id']}/messages", + {"text": "x", "attachment_ids": [png["id"]]}, token=E) + expect(st, 404, "eve attaches alice's file to her own chat") + + print("== идемпотентность с вложениями ==") + key = uuid.uuid4().hex + st, first, _ = api.call("POST", mpath, {"text": "idem"}, token=A, headers={"Idempotency-Key": key}) + st2, again, _ = api.call("POST", mpath, {"attachment_ids": [], "text": "idem"}, token=A, + headers={"Idempotency-Key": key}) + r.check(st == st2 == 201 and first == again, "missing attachment_ids == [] (same Message)") + st, _, _ = api.call("POST", mpath, {"text": "idem", "attachment_ids": [png["id"]]}, token=A, + headers={"Idempotency-Key": key}) + expect(st, 409, "same key, different attachment_ids") + key2 = uuid.uuid4().hex + st, a1, _ = api.call("POST", mpath, {"text": "", "attachment_ids": [jpg["id"], png["id"]]}, token=A, + headers={"Idempotency-Key": key2}) + st2, a2, _ = api.call("POST", mpath, {"attachment_ids": [jpg["id"], png["id"]], "text": ""}, token=A, + headers={"Idempotency-Key": key2}) + r.check(st == st2 == 201 and a1 == a2, "replay with attachments returns the same Message") + + print("== размер ==") + base = f"/api/v1/chats/{chat['id']}/attachments?filename=big.png" + auth = {"Authorization": "Bearer " + A, "Content-Type": "image/png"} + st, _ = raw_http(args.base_url, "POST", base, {**auth, "Content-Length": str(LIMIT + 1)}) + expect(st, 413, "Content-Length = 10 MiB + 1, body never sent (limit before reading)") + st, _ = raw_http(args.base_url, "POST", base, {**auth, "Content-Length": str(50 * LIMIT)}) + expect(st, 413, "Content-Length = 500 MiB") + st, _ = raw_http(args.base_url, "POST", base, auth, chunks=(b"\0" * 65536 for _ in range(200)), chunked=True) + expect(st, 413, "chunked body > 10 MiB without Content-Length (stream cut at limit)") + big = make_png(1900, 1900, noise=True) # ≈10,8 МБ настоящего PNG + # Шлём кусками и следим за ранним ответом: сервер отвечает 413 по Content-Length, + # не дочитывая тело, и urllib получил бы обрыв соединения вместо ответа. + st, _ = raw_http(args.base_url, "POST", base, {**auth, "Content-Length": str(len(big))}, + chunks=(big[i:i + 65536] for i in range(0, len(big), 65536))) + expect(st, 413, f"real PNG of {len(big)} bytes") + fit = make_png(1860, 1870, noise=True) + if len(fit) <= LIMIT: + st, body, _ = upload(api, A, chat["id"], fit, "fits.png", timeout=120) + expect(st, 202, f"real PNG of {len(fit)} bytes (≤ 10 MiB) accepted") + if st == 202: + r.check(body["size_bytes"] == len(fit) and body["sha256"] == sha256(fit), "big file sha256/size") + st, _, _ = upload(api, A, chat["id"], b"") + expect(st, 400, "empty body") + + print("== пиксели ==") + bomb = make_png(8000, 8000, color_type=0, rows=(b"\0" * 8001) * 8000) # 64 млн пикс., ~60 КБ + st, _, _ = upload(api, A, chat["id"], bomb, "bomb.png") + expect(st, 413, f"decompression bomb 8000x8000 in {len(bomb)} bytes") + wide = make_png(40001, 1001, color_type=0, rows=(b"\0" * 40002) * 1001) # 40 040 001 пикс. + st, _, _ = upload(api, A, chat["id"], wide, "wide.png") + expect(st, 413, "40 040 001 pixels (just over the limit)") + + print("== повреждённые файлы ==") + corrupt = bytearray(SAMPLE_PNG) + corrupt[len(corrupt) // 2] ^= 0xFF + cases = [ + ("truncated PNG", SAMPLE_PNG[: len(SAMPLE_PNG) // 2], "image/png"), + ("PNG with flipped byte (CRC)", bytes(corrupt), "image/png"), + ("PNG signature only", SAMPLE_PNG[:8], "image/png"), + ("PNG header + garbage", SAMPLE_PNG[:33] + b"\xde\xad\xbe\xef" * 100, "image/png"), + ("truncated JPEG", SAMPLE_JPG[: len(SAMPLE_JPG) // 2], "image/jpeg"), + ("JPEG SOI + garbage", b"\xff\xd8\xff\xe0" + b"\x00" * 300, "image/jpeg"), + ] + for name, data, ctype in cases: + st, body, _ = upload(api, A, chat["id"], data, "x", ctype) + expect(st, 415, name) + r.check(not body or body["error"]["code"] == "unsupported_media_type", f"{name}: error code") + + print("== fuzz: случайные порчи байтов (только 202 или 415, никогда 500) ==") + import random + rnd = random.Random(2026) + fuzz_codes = [] + for data, ctype in ((SAMPLE_PNG, "image/png"), (SAMPLE_JPG, "image/jpeg")): + for _ in range(15): + d = bytearray(data) + for _ in range(rnd.randint(1, 4)): + d[rnd.randrange(8, len(d))] ^= 1 << rnd.randrange(8) + if rnd.random() < 0.3: + d = d[: rnd.randrange(16, len(d))] + st, _, _ = upload(api, A, chat["id"], bytes(d), "fuzz", ctype) + statuses.append(st) + fuzz_codes.append(st) + r.check(set(fuzz_codes) <= {202, 415}, + f"30 corrupted files -> only 202/415 (got {sorted(set(fuzz_codes), key=str)})") + + print("== подмена MIME ==") + spoof = [ + ("PNG bytes sent as image/jpeg", SAMPLE_PNG, "image/jpeg"), + ("JPEG bytes sent as image/png", SAMPLE_JPG, "image/png"), + ("text renamed to .png", b"just text, not an image\n" * 20, "image/png"), + ("GIF sent as image/png", b"GIF89a\x01\x00\x01\x00\x80\x00\x00" + b"\x00" * 40, "image/png"), + ("HTML sent as image/jpeg", b"", "image/jpeg"), + ("PNG with Content-Type image/gif", SAMPLE_PNG, "image/gif"), + ("PNG with Content-Type application/octet-stream", SAMPLE_PNG, "application/octet-stream"), + ("PNG with Content-Type text/html", SAMPLE_PNG, "text/html"), + ] + for name, data, ctype in spoof: + st, _, _ = upload(api, A, chat["id"], data, "evil.png", ctype) + expect(st, 415, name) + + print("== опасные filename ==") + bad_names = ["../../etc/passwd", "dir/a.png", "C:\\Windows\\a.png", "a\x00.png", "a\nb.png", + "tab\t.png", "\x1b[31mred.png", "photo\u202egnp.exe", "..", ".", " ", "x" * 256] + for name in bad_names: + st, _, _ = upload(api, A, chat["id"], SAMPLE_PNG, name) + expect(st, 400, f"filename {name[:20]!r}") + st, _, _ = api.raw("POST", f"/api/v1/chats/{chat['id']}/attachments", SAMPLE_PNG, + {"Authorization": "Bearer " + A, "Content-Type": "image/png"}) + expect(st, 400, "filename missing") + st, ok, _ = upload(api, A, chat["id"], SAMPLE_PNG, "x" * 255) + expect(st, 202, "filename of exactly 255 chars") + st, ok, _ = upload(api, A, chat["id"], SAMPLE_PNG, "отчёт (финал) v2 — 100%.png") + expect(st, 202, "filename with spaces, Unicode and punctuation") + st, _, _ = api.raw("POST", f"/api/v1/chats/{chat['id']}/attachments?filename={quote('a.png')}", SAMPLE_PNG, + {"Authorization": "Bearer " + A, "Content-Type": "image/png; charset=binary"}) + expect(st, 202, "media type parameters are ignored") + + r.check(500 not in statuses and all(s is not None for s in statuses), + f"no 500 and no dropped connection among {len(statuses)} checked responses") + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 00000000..2f944c77 --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +""" +Прогон ВСЕХ проверок lab5 (и унаследованных lab2–4) с сохранением вывода в evidence/ (Windows/macOS/Linux). +Стенд должен быть запущен: docker compose --env-file .env up -d --build + + python student-tests/collect_evidence.py --base-url http://localhost:8080 + python student-tests/collect_evidence.py --only smoke,attachments # выборочно + +Каждый шаг пишет evidence/<имя>.txt (команда, время, stdout/stderr, код выхода). +Ошибка одного шага не останавливает остальные; в конце — сводка PASS/FAIL. +Hook-и: на Windows scripts/*.cmd, иначе scripts/*.sh. Секреты не печатаются. +""" +import argparse +import datetime as dt +import hashlib +import os +import socket +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +EVIDENCE = ROOT / "evidence" +PY = sys.executable +WIN = os.name == "nt" +COMPOSE = ["docker", "compose", "--env-file", ".env"] +SAMPLE_SHA = hashlib.sha256((ROOT / "tests/fixtures/sample.png").read_bytes()).hexdigest() + + +def hook(name: str) -> str: + return str(ROOT / "scripts" / (name + (".cmd" if WIN else ".sh"))) + + +# Публичные tests/ читают JSON без явной кодировки: на Windows (cp1251) это падает. +# UTF-8 mode Python для всех дочерних процессов, сами тесты не меняем. +ENV = {**os.environ, "PYTHONUTF8": "1", "PYTHONIOENCODING": "utf-8"} + + +def run(cmd: list[str], log, timeout: int = 900) -> int: + log.write(f"\n$ {' '.join(cmd)}\n") + log.flush() + try: + p = subprocess.run(cmd, cwd=ROOT, env=ENV, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding="utf-8", errors="replace", timeout=timeout) + except subprocess.TimeoutExpired: + log.write("TIMEOUT\n") + return 124 + log.write(p.stdout) + log.write(f"[exit {p.returncode}]\n") + return p.returncode + + +def ports_closed(log) -> int: + """MinIO (9000 API, 9001 консоль) не опубликован на хосте.""" + bad = 0 + for port in (9000, 9001): + s = socket.socket() + s.settimeout(1) + try: + s.connect(("127.0.0.1", port)) + log.write(f"127.0.0.1:{port} OPEN <- не должно быть\n") + bad += 1 + except OSError as exc: + log.write(f"127.0.0.1:{port} closed ({type(exc).__name__}) ok\n") + finally: + s.close() + return bad + + +def drain(log, limit: int = 600) -> int: + """Ждать, пока в очереди не останется queued/processing (не дольше limit секунд).""" + import json + t0 = time.monotonic() + while time.monotonic() - t0 < limit: + out = subprocess.run(COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"], cwd=ROOT, + capture_output=True, text=True, encoding="utf-8", errors="replace").stdout + try: + jobs = json.loads(out.strip().splitlines()[-1])["jobs_by_status"] + except (ValueError, IndexError, KeyError): + jobs = {} + busy = jobs.get("queued", 0) + jobs.get("processing", 0) + if jobs and busy == 0: + log.write(f"queue drained in {time.monotonic() - t0:.1f} s: {jobs}\n") + return 0 + time.sleep(3) + log.write(f"queue NOT drained in {limit} s\n") + return 1 + + +def steps(base: str): + t = ["--base-url", base] + yield "check", [[PY, "scripts/check_contract.py"]] + yield "stand", [COMPOSE + ["ps"], COMPOSE + ["config", "--images"], + COMPOSE + ["exec", "-T", "api", "python", "-m", "app.migrate"]] + yield "smoke", [[PY, "tests/smoke.py", *t]] + yield "student-tests", [[PY, "student-tests/run.py", *t], [PY, "student-tests/idempotency.py", *t]] + yield "attachments", [[PY, "student-tests/attachments.py", *t]] + # lab5: брокер, worker, отказы и восстановление + yield "broker-config", [COMPOSE + ["exec", "-T", "broker", "redis-cli", "CONFIG", "GET", "append*"], + COMPOSE + ["exec", "-T", "broker", "redis-cli", "CONFIG", "GET", "maxmemory-policy"], + COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"]] + yield "worker-limits", [COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "limits-selftest"]] + for sc in ("worker-down", "broker-down", "redelivery", "crash-s3", "crash-ack", "outbox", "retry-replay", "lease"): + yield f"jobs-{sc}", [[PY, "student-tests/jobs.py", *t, "--scenario", sc]] + yield "jobs-registry", [COMPOSE + ["exec", "-T", "api", "python", "-m", "app.replay", "--list"], + COMPOSE + ["exec", "-T", "api", "python", "-m", "app.replay", "--dlq"], + COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"], + COMPOSE + ["--profile", "fault", "stop", "api-fault", "worker-fault"]] + yield "s3-bootstrap", [COMPOSE + ["run", "--rm", "s3-init"], COMPOSE + ["run", "--rm", "s3-init"]] + yield "s3-private", [COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "private-check"], + "PORTS"] + yield "no-local-file", [COMPOSE + ["exec", "-T", "--index", str(i), "api", "python", "-m", "app.tools", + "find-local", SAMPLE_SHA] for i in (1, 2)] + [ + COMPOSE + ["exec", "-T", "worker", "python", "-m", "app.tools", "find-local", SAMPLE_SHA]] + [ + ["docker", "inspect", "-f", "{{.Name}} mounts={{json .Mounts}}"] + "API_IDS".split()] + yield "orphans", [COMPOSE + ["--profile", "fault", "up", "-d", "api-fault"], "SLEEP", + [PY, "student-tests/orphans.py", *t, "--fault-url", "http://localhost:8082"], + COMPOSE + ["--profile", "fault", "stop", "api-fault"]] + yield "s3-outage", [[PY, "student-tests/s3_outage.py", *t]] + yield "upload-memory", [[PY, "student-tests/upload_memory.py", *t, "--concurrency", "10", + "--output", "evidence/upload-memory-10.json"], + [PY, "student-tests/upload_memory.py", *t, "--concurrency", "20", + "--output", "evidence/upload-memory-20.json"]] + # upload-memory ставит в очередь 90 больших картинок: ждём, пока worker их обработает, + # иначе вложение следующих тестов встанет в конец очереди (это и есть queue lag). + yield "queue-drain", ["DRAIN", COMPOSE + ["exec", "-T", "api", "python", "-m", "app.tools", "queue"]] + yield "persistence", [COMPOSE + ["ps"], [PY, "tests/resilience.py", "--mode", "persistence", + "--action-script", hook("restart"), *t], COMPOSE + ["ps"]] + yield "failover", [[PY, "tests/resilience.py", "--mode", "failover", "--action-script", hook("stop-one"), *t], + [hook("restore-api")], "SLEEP", + [PY, "tests/resilience.py", "--mode", "failover", "--action-script", hook("stop-one"), *t], + [hook("restore-api")]] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--only", default="") + args = ap.parse_args() + only = {x.strip() for x in args.only.split(",") if x.strip()} + EVIDENCE.mkdir(exist_ok=True) + commit = subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, + text=True).stdout.strip() or "no-git" + summary = [] + for name, cmds in steps(args.base_url): + if only and name not in only: + continue + path = EVIDENCE / f"{name}.txt" + print(f"== {name} -> {path.relative_to(ROOT)}") + with open(path, "w", encoding="utf-8") as log: + log.write(f"# {name}\n# date: {dt.datetime.now().isoformat(timespec='seconds')} commit: {commit}\n") + failed = 0 + for cmd in cmds: + if cmd == "SLEEP": + time.sleep(8) + continue + if cmd == "DRAIN": + failed += drain(log) + continue + if cmd == "PORTS": + failed += ports_closed(log) + continue + if "API_IDS" in cmd: + ids = subprocess.run(COMPOSE + ["ps", "-q", "api"], cwd=ROOT, capture_output=True, + text=True).stdout.split() + cmd = [c for c in cmd if c != "API_IDS"] + ids + code = run(cmd, log) + failed += 1 if code else 0 + status = "PASS" if not failed else f"FAIL ({failed})" + summary.append((name, status)) + print(" ", status) + print("\nSummary:") + for name, status in summary: + print(f" {name:18} {status}") + sys.exit(0 if all(s == "PASS" for _, s in summary) else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 00000000..6e1882e0 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,215 @@ +from __future__ import annotations +"""Общий HTTP-клиент для собственных тестов (только стандартная библиотека).""" +import base64 +import json +import urllib.error +import urllib.request +import uuid + +PASSWORD = "Test-password-" + uuid.uuid4().hex[:8] + + +class Api: + def __init__(self, base: str): + self.base = base.rstrip("/") + + def raw(self, method, path, data=None, headers=None, timeout=10): + req = urllib.request.Request(self.base + path, data=data, headers=headers or {}, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, r.read(), r.headers + except urllib.error.HTTPError as e: + return e.code, e.read(), e.headers + + def call(self, method, path, body=None, token=None, headers=None, timeout=10): + h = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode() + h["Content-Type"] = "application/json" + if token: + h["Authorization"] = "Bearer " + token + # С lab3 POST сообщения требует Idempotency-Key; если тест не задал свой — новый ключ. + if method == "POST" and path.split("?")[0].endswith("/messages"): + h.setdefault("Idempotency-Key", uuid.uuid4().hex) + st, raw, rh = self.raw(method, path, data, h, timeout=timeout) + try: + return st, json.loads(raw or b"null"), rh + except ValueError: + return st, None, rh + + def register(self, prefix: str, password: str = PASSWORD) -> dict: + name = (prefix + "_" + uuid.uuid4().hex[:10])[:32] + st, user, _ = self.call("POST", "/api/v1/users", + {"username": name, "display_name": prefix, "password": password}) + assert st == 201, f"register {name}: {st} {user}" + user["password"] = password + return user + + def login(self, username: str, password: str): + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + st, body, rh = self.call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + return st, body, rh + + def new_user(self, prefix: str) -> dict: + user = self.register(prefix) + st, sess, _ = self.login(user["username"], user["password"]) + assert st == 201, f"login: {st}" + user["token"] = sess["token"] + return user + + +class Report: + def __init__(self): + self.passed = self.failed = 0 + + def check(self, cond, name): + if cond: + self.passed += 1 + print(f" ok - {name}") + else: + self.failed += 1 + print(f" FAIL - {name}") + + def finish(self): + print(f"\n{self.passed} passed, {self.failed} failed") + raise SystemExit(1 if self.failed else 0) + + +# ---------------------------------------------------------------- lab4: файлы +import hashlib +import os +import re +import select +import socket +import struct +import subprocess +import zlib +from pathlib import Path +from urllib.parse import urlparse + +ROOT = Path(__file__).resolve().parent.parent +FIXTURES = Path(__file__).resolve().parent / "fixtures" +SAMPLE_PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +SAMPLE_JPG = (FIXTURES / "sample.jpg").read_bytes() +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def make_png(width: int, height: int, *, noise: bool = False, color_type: int = 2, + rows: bytes | None = None) -> bytes: + """Синтетический PNG только средствами stdlib. + noise=True — случайные пиксели (почти не сжимаются: так получаем файл нужного размера). + rows — готовые распакованные данные (для «бомб» и битых файлов).""" + channels = {0: 1, 2: 3, 6: 4}[color_type] + if rows is None: + line = width * channels + rows = b"".join(b"\x00" + (os.urandom(line) if noise else bytes(line)) for _ in range(height)) + + def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", width, height, 8, color_type, 0, 0, 0) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + + chunk(b"IDAT", zlib.compress(rows, 1 if noise else 9)) + chunk(b"IEND", b"")) + + +def upload(api, token: str, chat_id: str, data: bytes, filename: str = "image.png", + content_type: str = "image/png", headers: dict | None = None, timeout: int = 60): + from urllib.parse import quote + h = {"Authorization": "Bearer " + token, "Content-Type": content_type, **(headers or {})} + path = f"/api/v1/chats/{chat_id}/attachments?filename={quote(filename, safe='')}" + st, raw, rh = api.raw("POST", path, data, h, timeout=timeout) + try: + body = json.loads(raw or b"null") + except ValueError: + body = None + return st, body, rh + + +def wait_ready(api, token: str, attachment_id: str, seconds: float = 90) -> dict: + """lab5: upload отвечает 202 queued; ждём, пока worker доведёт до ready/failed.""" + import time + deadline = time.monotonic() + seconds + att = {} + while time.monotonic() < deadline: + st, att, _ = api.call("GET", f"/api/v1/attachments/{attachment_id}", token=token) + if st == 200 and att["status"] in ("ready", "failed"): + return att + time.sleep(0.3) + return att or {} + + +def raw_http(base: str, method: str, path: str, headers: dict, chunks=(), chunked: bool = False, + close_after_send: bool = False, timeout: float = 30): + """HTTP/1.1 «руками» через сокет — для сценариев, которые urllib не умеет: + заявить Content-Length и не прислать тело, прислать chunked больше лимита, + оборвать загрузку на середине. Пока шлём тело, следим, не ответил ли сервер + раньше (413 до конца тела) — тогда прекращаем отправку и читаем ответ. + Возвращает (status | None, body_bytes).""" + u = urlparse(base) + sock = socket.create_connection((u.hostname, u.port or 80), timeout=timeout) + try: + lines = [f"{method} {path} HTTP/1.1", f"Host: {u.hostname}:{u.port or 80}", "Connection: close"] + lines += [f"{k}: {v}" for k, v in headers.items()] + if chunked: + lines.append("Transfer-Encoding: chunked") + sock.sendall(("\r\n".join(lines) + "\r\n\r\n").encode("latin-1")) + for part in chunks: + ready, _, _ = select.select([sock], [], [], 0) + if ready: + break # сервер уже ответил — дальше не шлём + try: + sock.sendall(f"{len(part):x}\r\n".encode() + part + b"\r\n" if chunked else part) + except (BrokenPipeError, ConnectionResetError): + break + else: + if chunked and not close_after_send: + try: + sock.sendall(b"0\r\n\r\n") + except OSError: + pass + if close_after_send: + return None, b"" # обрыв загрузки: ответ не ждём + data = b"" + try: + while True: + piece = sock.recv(65536) + if not piece: + break + data += piece + except (ConnectionResetError, socket.timeout): + pass + finally: + sock.close() + m = re.match(rb"HTTP/1\.[01] (\d{3})", data) + return (int(m.group(1)) if m else None), data.split(b"\r\n\r\n", 1)[-1] + + +def compose(*args: str, check: bool = True) -> str: + out = subprocess.run(COMPOSE + list(args), cwd=ROOT, capture_output=True, text=True) + if check and out.returncode != 0: + raise RuntimeError(f"docker compose {' '.join(args)} failed: {out.stderr.strip()[-300:]}") + return out.stdout + + +def tool(*args: str) -> dict: + """python -m app.tools … внутри контейнера api → последняя строка JSON.""" + out = compose("exec", "-T", "api", "python", "-m", "app.tools", *args) + return json.loads(out.strip().splitlines()[-1]) + + +def janitor(*args: str, attempts: int = 10) -> dict: + """Один проход очистки внутри api. Если advisory lock занят фоновым + проходом другой реплики — повторяем.""" + import time + for _ in range(attempts): + out = compose("exec", "-T", "api", "python", "-m", "app.janitor", "--once", *args) + stats = json.loads(out.strip().splitlines()[-1]) + if stats.get("locked"): + return stats + time.sleep(1) + raise RuntimeError("janitor lock is busy") diff --git a/student-tests/expiry.py b/student-tests/expiry.py new file mode 100644 index 00000000..503cf37c --- /dev/null +++ b/student-tests/expiry.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +""" +Истечение сессии при коротком SESSION_TTL_SECONDS (=2). +Запускается против отдельного экземпляра API с коротким TTL: + docker compose --env-file .env --profile ttl up -d api-ttl + python student-tests/expiry.py --base-url http://localhost:8081 +""" +import argparse +import time +from datetime import datetime, timezone + +from common import Api, Report + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8081") + ap.add_argument("--ttl", type=float, default=2) + args = ap.parse_args() + api, r = Api(args.base_url), Report() + # Экземпляр с коротким TTL мог только что стартовать: ждём готовности до 60 с. + deadline = time.time() + 60 + while True: + try: + if api.raw("GET", "/health/ready", timeout=2)[0] == 200: + break + except OSError: + pass + if time.time() > deadline: + raise SystemExit("API is not ready at " + args.base_url) + time.sleep(1) + + user = api.register("ttl") + st, sess, _ = api.login(user["username"], user["password"]) + r.check(st == 201, "login -> 201") + exp = datetime.fromisoformat(sess["expires_at"].replace("Z", "+00:00")) + left = (exp - datetime.now(timezone.utc)).total_seconds() + r.check(0 < left <= args.ttl + 1, f"expires_at is about {args.ttl}s ahead (got {left:.2f}s)") + st, _, _ = api.call("GET", "/api/v1/users/me", token=sess["token"]) + r.check(st == 200, "token works before expiry") # заодно кладёт сессию в кэш Redis + time.sleep(args.ttl + 1.5) + st, _, rh = api.call("GET", "/api/v1/users/me", token=sess["token"]) + r.check(st == 401 and rh.get("WWW-Authenticate", "").startswith("Bearer"), + "expired token -> 401 (cache does not extend the session)") + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 00000000..235f1f9a --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,60 @@ +-- План запроса истории на ≥10 000 сообщениях (уровень «4»). +-- Всё внутри транзакции с ROLLBACK: учебные данные не остаются в базе. +-- Запуск (одинаково в PowerShell и sh): +-- docker compose cp student-tests/explain.sql postgres:/tmp/explain.sql +-- docker compose exec postgres psql -U messenger -d messenger -f /tmp/explain.sql +BEGIN; + +INSERT INTO users (id, username, display_name, password_hash, created_at) +VALUES ('00000000-0000-4000-8000-000000000001', 'explain_seed', 'Seed', 'x', now()); + +-- Чат, историю которого читаем, и 9 «соседних» чатов, чтобы в таблице были чужие сообщения. +INSERT INTO chats (id, title, created_by, created_at) +SELECT ('00000000-0000-4000-8000-0000000001' || lpad(g::text, 2, '0'))::uuid, 'seed ' || g, + '00000000-0000-4000-8000-000000000001', now() +FROM generate_series(0, 9) g; + +INSERT INTO chat_members (chat_id, user_id, position) +SELECT ('00000000-0000-4000-8000-0000000001' || lpad(g::text, 2, '0'))::uuid, + '00000000-0000-4000-8000-000000000001', 0 +FROM generate_series(0, 9) g; + +-- 12 000 сообщений в целевом чате + 48 000 в соседних = 60 000 строк. +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), + ('00000000-0000-4000-8000-0000000001' || lpad((CASE WHEN g <= 12000 THEN 0 ELSE g % 9 + 1 END)::text, 2, '0'))::uuid, + '00000000-0000-4000-8000-000000000001', + 'msg ' || g, + now() - interval '1 day' + g * interval '1 millisecond' +FROM generate_series(1, 60000) g; + +ANALYZE messages; + +SELECT count(*) AS messages_in_target_chat +FROM messages WHERE chat_id = '00000000-0000-4000-8000-000000000100'; + +\echo '=== 1. Первая страница (limit 50 + 1) ===' +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = '00000000-0000-4000-8000-000000000100' +ORDER BY created_at, id LIMIT 51; + +\echo '=== 2. Страница из середины истории (keyset-курсор после ~6000-го сообщения) ===' +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = '00000000-0000-4000-8000-000000000100' + AND (created_at, id) > (now() - interval '1 day' + 6000 * interval '1 millisecond', + '00000000-0000-0000-0000-000000000000') +ORDER BY created_at, id LIMIT 51; + +\echo '=== 3. Для сравнения: та же страница, если запретить индекс ===' +SET LOCAL enable_indexscan = off; +SET LOCAL enable_bitmapscan = off; +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = '00000000-0000-4000-8000-000000000100' + AND (created_at, id) > (now() - interval '1 day' + 6000 * interval '1 millisecond', + '00000000-0000-0000-0000-000000000000') +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; diff --git a/student-tests/failover_each.py b/student-tests/failover_each.py new file mode 100644 index 00000000..2bd2894c --- /dev/null +++ b/student-tests/failover_each.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +""" +Поочерёдная остановка КАЖДОЙ реплики API под нагрузкой (lab3, уровень «4»). + +Для каждой реплики: дождаться, что LB обслуживает все реплики → запустить +нагрузку записи → через --before секунд остановить реплику (docker stop, SIGTERM) +→ ещё --after секунд нагрузки → собрать RPS, ошибки, время восстановления → +проверить сохранность и отсутствие дублей → вернуть реплику и дождаться, +что LB снова её обслуживает. + + python student-tests/failover_each.py --base-url http://localhost:8080 --output evidence/failover-each.json +""" +import argparse +import json +import shlex +import subprocess +import time + +from common import Api, Report +from loadgen import LoadGen + +COMPOSE = "docker compose --env-file .env" + + +def run(cmd: str, **fmt) -> str: + out = subprocess.run(shlex.split(cmd.format(**fmt)), check=True, capture_output=True, text=True) + return out.stdout + + +def served_instances(api: Api, n: int = 30) -> set: + seen = set() + for _ in range(n): + try: + st, _, rh = api.call("GET", "/health/live", timeout=2) + if st == 200: + seen.add(rh.get("X-Instance-Id", "")) + except OSError: + pass + return seen + + +def wait_served(api: Api, iid: str, seconds: float = 60) -> float: + start = time.monotonic() + while time.monotonic() - start < seconds: + if iid in served_instances(api, 10): + return round(time.monotonic() - start, 2) + time.sleep(0.5) + return -1 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--workers", type=int, default=4) + ap.add_argument("--before", type=float, default=4, help="секунд нагрузки до остановки") + ap.add_argument("--after", type=float, default=12, help="секунд нагрузки после остановки") + ap.add_argument("--list-cmd", default=f"{COMPOSE} ps -q api") + ap.add_argument("--stop-cmd", default="docker stop -t 8 {id}") + ap.add_argument("--start-cmd", default="docker start {id}") + ap.add_argument("--output", default=None) + args = ap.parse_args() + api, r = Api(args.base_url), Report() + + replicas = [line.strip() for line in run(args.list_cmd).splitlines() if line.strip()] + r.check(len(replicas) >= 2, f"at least 2 API replicas running ({len(replicas)})") + results = [] + user = api.new_user("failover") + for cid in replicas: + iid = cid[:12] # X-Instance-Id = hostname контейнера = короткий ID + print(f"== реплика {iid} ==") + r.check(wait_served(api, iid) >= 0, f"LB serves {iid} before the test") + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": f"failover {iid}", "member_ids": [user["id"]]}, + token=user["token"]) + gen = LoadGen(api, user["token"], chat["id"], workers=args.workers) + gen.start() + time.sleep(args.before) + t_stop = time.monotonic() + run(args.stop_cmd, id=cid) + stop_cmd_seconds = round(time.monotonic() - t_stop, 2) + time.sleep(args.after) + gen.stop() + rel = t_stop - gen.t0 + later = gen.instances_after(t_stop + 3) + res = { + "stopped_instance": iid, + "stop_command_seconds": stop_cmd_seconds, + "before": gen.window(0, rel), + "first_10s_after_stop": gen.window(rel, rel + 10), + "first_success_after_stop_s": gen.first_success_after(t_stop), + "max_gap_without_success_after_stop_s": gen.max_gap_after(t_stop), + "stopped_instance_served_after_3s": later.count(iid), + "instances_after_stop": sorted(set(later)), + **gen.verify(), + } + t_start = time.monotonic() + run(args.start_cmd, id=cid) + res["rejoin_seconds"] = wait_served(api, iid) + res["rejoin_including_start_s"] = round(time.monotonic() - t_start, 2) + results.append(res) + print(json.dumps(res, ensure_ascii=False, indent=2)) + r.check(res["max_gap_without_success_after_stop_s"] <= 10, f"writes resumed within 10 s after stopping {iid}") + r.check(res["stopped_instance_served_after_3s"] == 0, f"stopped {iid} not served after detection") + r.check(res["lost_or_changed"] == 0, "confirmed messages preserved") + r.check(res["duplicate_texts"] == 0, "no duplicates despite retries with the same key") + r.check(res["rejoin_seconds"] >= 0, f"{iid} rejoined the LB after restart") + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + json.dump({"workload": f"{args.workers} closed-loop writers, retry same key on failure, " + f"{args.before}s before + {args.after}s after stop", + "replicas": results}, f, ensure_ascii=False, indent=2) + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/README.md b/student-tests/fixtures/README.md new file mode 100644 index 00000000..12ad8e3f --- /dev/null +++ b/student-tests/fixtures/README.md @@ -0,0 +1,7 @@ +# Собственные fixtures + +- `sample.jpg` — синтетический JPEG 240×160 (линии и рамка), создан программно, без EXIF и персональных данных. + +Остальные файлы для негативных тестов (oversized, битые, «бомба» по пикселям, +подмена MIME, шум ~9,5 МБ для эксперимента памяти) генерируются тестами +на лету (`student-tests/common.py: make_png`) — большие файлы в Git не храним. diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f6a486878608f3abe1baa07a65e63cf22d59c67b GIT binary patch literal 19177 zcmeIZby!v1+BZ63QPL${3QBjZMTY_+0s=}%xm$}cB&*T~~$DH@L$9@0q7%>OeU#}Md(uYbaN&p510AQeh zfa`6lw+|mF;>aEV`(5~dI+7$$CR&TB3z2;~)l3YhV}5D+!z6*3hT5iy4e3qj1q(S3np{$$7b z2Ro?Kf9aGD>f~gGaJI6CJE0o@wSP9a|GU8pd&Ivsp!n4{pB3B!VP$R z1}5ZWSy%09a;sA@`C$!B-iwD3%;@KKFm(G&D^r++xjn)T>TGRg2emNcw{|eIu(pCb zIYZ$vGnM;D9;BUvgq4-4gor6bTo@w$g4ax31jZ`@5f$WxLd8UQMTLZ6P$*2$TufZh z7GY)!{VlDfm4l@N6oD2)!yJw9@}Gjs|M!AJVZR&lxeEOyHA>uT=qv8nv(Z8JuKqe* zTQsADUs7jS{7LMaVgARR&2Qc99n9cnrq<|wp>}4@W(cPbHvfdf>2(Io1!0TibuzOx zXZ(u#-nlBv zl&nAHKr+JY?f9T@XG?^=gB6SqW(##OHRE$Y*t=Sp{<~;M=6@%(M_5@{!Fgf!aC0;% z8hi^ExDDLi4Q^v<<^)4nIsAeLX@2iFN0xtZWO25%a$+&JvNdCYI-@IQEC@447c(bk z7JG9RD2of)dsv`wQx-d@JBuaM)r*T3GJCx9~40`AnhC z(7z1qLtXtOg!CB}v*E)A)to&h^b4O~2F`{-Z7s~5ETKZ8VlW4&srs)L|GfS*3S~M& z);n^x0_nwoq;e=Z$7XoN+pibLVcyTiekk|Z(fpPg{@c=M{oz*TXjrVxkZ75meg)L) z40IieaHcf3HG^9?Thda!0{~2!KlNb$t%nfI_FqHncOv*@R4cnb{1lVqe-+ot$;tu_ zb#_6RA=UoH(|)@a|NmBo_`@6jV)q||@&(LHKpZR#;}tfAi13P-3PX6IU^B5_i3e;h z{zAwUDkzBf)8r=qbacP}gQ4&TYp#FM{Xa(8Kg0b`YknIY$sNT=W;(>Jt+_A~=^xzQ zQg9hg@9LKZCJq9zeg5Ff`9Jaf%gO#^t8hufsgVtcgw5T-S!ye#wTZ zF+$Tpm>8ItlyYn7_&-|3hZ}4B&Ttg4r7fOP00Q_pcw7Mw2Bon;AORIYYJrO5|zkx@Xfv>d)Zn=uABU9RXsr9E>Cw7}r=>AWU4J-`FtT!6FtQ z#KGsr!N9`s`(})Zfv%y+D1MP)3IGDkziO;lGy+sUl!KbfSg+Mh4Omet?L&z&$t{zH zt^bNR$N~^B#S{>9Vx&yPcsbhlG;C05Njfz+$reZ5+Y*^R$Ia2o=9Ye$zgC zqPg^0`NoG|nNOc6^`Cj~@4LSd_#1)05%?Q{zY+Kwfxi*>8-c$O_#1)05%?Q{|Njs; z)|0EZ8>B8b3UHZ;=TC`$Wr>lZF)K#ej=|VOfFQk&>70A651f&$4syj7=T+kfgO~Ee zJeHoe82%hw^s`)?S1r4q0qy}fSlBq&SU5P?IJmetc=)9F_&09gQxcO9l2TLB&`?uS zQPDBpVWy*JWuT&B;bCFD%g)KgNz2SDz{??Uhl7*j7ZVIzTwHuSd$({r!voTRJrKri0Fw*@M22zQ4xo37F|aXyU;n@v0~3Un7Z>jaJ^`Abk`%zi0D&;E zK-kz=SZHcb^m_n{44eFxpgazRCKUI!Bc;&m$Paj|3Z*SKwMO>XgkKj0|JBI z21iB5#Ky(HPe{zj%*xKm{g{_uR$ftARb5kC*V@+J(b?7A(>ppgKJjJp>(unp^2+Mk z`o_1-t^I?;pGU_hsME7wdSL*dKh#41{zJ3>MK3b6UYO{O1T5TNdSPI?p$mu%3;UKJ z4!OK0F4U3Yw$N)lN`=S|r7bsDg|+r>zCeuNQ?ZFG-P!-8+HcMNpDE`1|4OsJ6#J)M z(*Pj|18qDI86XSnx-jD+lbWf^tUhN1@E5|D($LX~^M&K-(iW_1ITyUHoOc2e@ z`|rObu=+WtzM-`L)b|?5m{D`AeDt0q1{4Pr5N!s~Uhx}p9XpqtynU7ZN^#wY_7@14H<3r}_Y_qtdUyf3c$E}1pD zSEYDYF7Ms13v4wZw_C4BS@~KQtx*vJeKWu1JTbXzA_vBy6bAnkDUjUq;5V!>D${Fhfpzgi0-hn6UNIJg^6 zRJ1^?#47iw5Kn_6KRHFPb2qOH9!GgUl!-IOcYcN(ZM20k1t{JcRVi1I-<^Gn<`<1g*Z@DQa&3RjR(%Q|z zZ%k|^6qg0f3l!-Gl_tWr)Px1}w6bq7aVEUHbeOsQ!kiAJ-fAX^hpalel6=uih~V@?Zf^h85h}8WPGn`-6`CLR_V(*|DHt#64W7fCAQa< z{5{gxU(i?{E2M~m!AE{6Mqrf`xoR3K+TtP9ShFV^!tcRY#9c`3wB}$(|Ca zGt=S4R5$06+X=swib>f+%2H(zp!0|dB#tG}sll*Cs@}Jlws;McG*5vio%K-VHe!*l zvXEoyYLj$^zT7e0Os0enI@RUY5|CQn?rBTKhnFOVBi#3QhbL4%D5)&+V>(%+#ue|GT%Ju7P+H!OQT3$ZO!1 zNm$ci-}&vf%Uv+yir#zj{Bprd%!_Bsd#VwqX^KBq^vjIpmC;N!bHYo<7?Po*4$})+ zmvOJ#$jQewq1fXTcQS;cfDkITQoKHxqdlu7y zQ3AHg9eoW*%VI9H$D7(l9S2`hUr?0p701*u2Z~%mu!ykaR7&4*MjWhC3=)ugGfi}& zz*k&N4AwFnUbeR7X1VA4eXi0p$pW(5j%jg~R~*~SQ;pP#qgaO}l{`}|c=QR?go60Q z>KM(PUgpN)d80NOm`L(jBbu0LIqK(5Z*y6q@*fQK&xEVvr~^0CR8JVhYYpQ&cx8t! z!Yr2OOQkQqT#=6~zj`;5qx}AscGuH;-;l&AodE6!$TlCJofO65*Tyl~#3~28%tHo0 zDIbSS$)FxaaeQfi1T%g`b+Yk)){-O(Snl%{~e>EhwO zLn6|9IbkeiL|;V%ywA#3MH$&S@so{$zjW+-=7rd0w+;K;@g$}gNUjoBlFGL$PM7E_z2e5K98Zg7wVi&dvZ}PDuU|3s zF?iY{hy`oIJri^Nns@ zbx6$SJbi@`Yg>LNZOT(QmZ+}RVU8EvbB!=Rqm6(I;T+UUqBzoZ?UYG!!0h>h#VT9| zVjNtihViaMZ<2+pd+md9j2sieW0riWPM#)_?-s-HhH?h$8q3|CWJG7E&UMuvp_gAF2-%KutG6c#X#t zU7)400w=0^pU0-IftJH(?{(T*i%XR1iiQ%tr_Ns;Z?t<8mN2VK-t_kT08tpl%+3;9 zTz!~HJ5|kU#lqiRX{v0oqNcRGtpJu`La_V|#;;XKi#kgdEv3 zUO>jAG$G@>Yo7eZN}Rt7 zOsO$wN_O|WT&9)u_!B>9b4_)464QN_(%qZOKB;%Ty6`@L^%kf*%ZTPKT?5li&GVkh zYN+U3#VllOw95WKfoJ0UvR*H5AK}%HMc23q}k8yDj zBD8w}Np74o$sYeSb)oGF5;e{EkcQGK@vPJ|<;ESU$!~i#QV(#Rgj6AZt_A9pjlheF z8dm1q!oJn$Y+VCO7inqcE=+Wj*?Sj&92$fs`Kbm-Z!w{S3Z(hk}kZI?TIf5Nh9#Ob6G@2-Imx#`dAQCbT+$Nk%6Yq zBh%{C09Z|SW+uA^mgD!U121&#&y(D<&Z#o4f!6MqR8LRe@65jnpUY(c+!shH8oBn4 zc73F@-dYrI2FsRLmxh$zCQV-?ZP8^2`JvQJOP{~dX7lwzcX3Xm`D`kYSK$lz{-NeU zP@!1#pqmnkUVX$TGnsBZkLvTO^_TELesgC9rIXHj{q!;CaMEe^1jkHco_f8|B>ng} z28zQ%nBQ>P$Lf7^10&TXm$~n^?)rZrc;`+lj*<09pu78yUbrDSw`|MZm3D8UN)w{3 z8}^rZxrKEww!*@jF)!{u&-BUlQ9{7j$gSovKiBNDw`|u#V%I(geDM^&hyA#p_3|rj zj#wFCGJ@DsgZrJYE_#M!zW+)XZe^ObL@^*0&Q@lrI5+oBSg^W>Ej4RQhQzx^d{lO) z`YP;Y>z-|C&)(e~SR`Xc*=J?^<`~;<2Tx)=Sq08{>NtoO=;7Fd$h@+8IXxc&;cnn8 zN{(s0JlQtcV}RGiUP$9LbK~B8wj2MDo9OvzIhdclJ8&JI8wVT8JIY2mUIP?EG^nKZ zmG?{VUE6?Wj_fC-?ko&0`8rWxACFshL{awp zI_0Gy>`qb-oyt#;Tf!ksg&Ok&!2{*_F^s1l8b>bmy}J_q2kSp}EDjNLfE4mGHY1?7 z@x7K-@C*FeX~kQ^l>Obrl?YoQ# z^>KUcJu3&rYA1^%M-8G^Cr*cSUSXq~)jtyPjO^Q^g-@%4i`ZNjqq08O!e2bNt90rh zw@PkvL*wkKE~T_>PIBARU?yqe^1YX*)J7bh$S$k?{&+n2tYRu^L5&BZ@jhIHxh^|= zj!~MVKhsHd?@P(@`Nv9G>OMBHky#eX zoCwL$yt{68Z-`aaJfz+pa8?uF&_IYIFJc(0CApu-=Eul@r<3U3qNgQF(5< zS><{bA7H;&{a}~x%NlzuQV8pyYNM1zkajzS*atS#wxRWg!A3dZb&U+5Lz@fJyq1wqR5Gw6lvDyee zQGa)TpE78yjd+J%M~U7@e1@PLCJ=>({pkfd5C{u4y!cQ?1*nIYb}m&14FjFDkzi#3 zTv{!X;l)KCCAk4;IHARriR=M2dY1p0VA+}vsX4Ok!m+z>aVFCx5lbVZ|75~lcwTKh z^{($18w0nh{j1=<&jymaQDi4opCoe?GP&GY!1GHwUrGvxkkj7nMuanW^Wc5<`Omn@ zgI3#acgt_mrxVQJ0_0Pyt=f#J1JMJJ7>;vhG32#Xnk;~>qOqip-csUM#uUA-M=)SahCbx)bq zQ`Bb^_KRbei=?Yi_U$8uZiYUehAuZ+@Cr$^RGE(^vX>~B0{3k?=(JfgwyfiadxWw1 zX3MzKp%H{Q%%2%>)!Zg2``bsH;|>HolwHxwr{3MBQ$Zar-3Rg!Q2 zbirQjO}Lpjg9;`8@@n_fTwd9WFGb@5r%gyPk1m=mT&b0zWUgm}^ab`OUOE!XV_#3w zFuc=IxLtJ&vvZr*fYqUqyHdZRjP_)q+Bh-Z1xsy1*yOQCS%4y9(uwBrM;byt#?#|g z=B}G7C*D{xL(3YZo3eWmKdyQ{+3bS9>m9mgh|RjE@h@~DUOMMTcrYB=NpXXZf5=v| zBgHPmFO{b5yPsbJV8i3Wtkt=ldW}7a7jwJtY8k)%j+{%`78&CA?&bBZd~{O>-I3`^ z&+!KFEN%s@Kv#-+w-36OOPCdNyEiZWKXBGIy2{pB;Y#WqWD&m+z{21tNxO?;$8tiw z52s{(W*MA6nuJ5dGjQ;Rpy_pk(2t()?zpT%+e`EvmkET=nvy)-aHrYc7<|-wyqafm z2fn~9d zyAF#E`8hMn;llrER&BY7%%DT2(!EMeYtXikX#_!#p|Pfv^NPB?|VzavRyf8DX| zZMU!+SDo@T&=Z9`kDpuyvd&EITy*r|0XGN*$chT1AJos0FWeSzte{%xb3tc0ie&#z zEmhZ-l$(?A-uSp}_`=2sQ;(HMojI(Wyl*+_^ZN3#4sE%1O0%HU10~g3*5}=e0k=i! z0{T>Kco`Esj!74`;2{`T$x83B3XxhG^l1z}IMe$d_Qqrnsjh(@`>Vlv^y$ws_`Mucj;^8EVTboL~;uY>wLq%I2_ifFxYk+)mZOYjQ>?WB; zgnYT(uF6{#N7>_kVjx}+QIVcUsUStEk9BY-|we1tG48N#+ z?U)u5oq!_`c<$E8z@kkQHH~VdJZ%|D)HWxaEo4{;G)OurFJv49yY{8|rR2cK4Vq*8s6V)d%3o2!mphYR$E)``b3m_4{4 zCPwdU5C)(1^GSq>Ka)@`zm!ZBeJJWsGC$#VRmDAHtVR;?+?#VLdLZmUv;Ixg2J*CK zT5t%L4aP2Am6X?3cn#R;Kl*f;sb8%c%N3w(@U`qK?KDQr>3ck4!rsi95~4iQ^MMw$X{P=q`qp7eGaMPTi>r2@mBC{ee>GlI%@6kCn7H32(oky%!BYzZFy zsTg?5sy|TUmwF9xtr>X`;KsIxJ2_5#RJ*u~V}cAeS`;k`(o!4id;sfiB?dG2 zr6OB~o_p5IkMM|ai&l@N>vaw@Dd3#q4e`tkv2Ugda0d&->bhl!cCI^r;*MhadG2i7 zw026vM6$;i5Kua6TA@@NdZdk#9RD%8ZPA+5@{4mscZNmqcA+8Bq%Q^O8%XeJifmLms;4B`ITYTt!Ay(_VJJ2`Z5ezv>ie%MkY^HSFBlvza1 z>6y+qbB5fjwX1eKR)X)rOZ| zT0Px~xwOwUuuT|+jaaYDd^lTtGCik(9um%xwS9_=yTjQQ*64wwGvq)L`@UvW`%l14 zbSw1+3O@R#VV~UYbu_0XO~LBMZe(Z;4Fp%}Fog8w5H^aduWdxpQK+c1pyEeb^4*tj z7z1rr*xAz|#Z?U)qX>@)nU!+-INdAX4{oFY;0ie4j@91%lwu>JXFX(t5u;xA z)aLxtg-Pydl9%ZI=N}DgdX+^%kFoO~Jb<IxX)R1WKy*uJ{`F**@~Q zwpr2Ml|=S5OD&dO-R+`X`AV3$Hn$z+Vy6AT*3%f5J%I7mBc-yP5%2e#Y(Mj(8m(2d zc;}Y()&}Ng6xz7h!VvXmS>pqEOJAEsg2#5nba)OQDwidEZE)8!J?F8hk6NzOm?>?^vUb{iUdVHsqams9KU6| z>aI)Fr(z2;GnuW0d&$$vd1)ZMbeMi_i5|0RK<&tA%Drh8w50ypfz*8*1b@XO#!WEb45 z*;(-U&celcs;9(}K#3Y`<|4fITS=M>OR#5Y)wD<9!Cqxe7?9CM#2)_+Ne$ay68=8KCgsK$guz_(Pd)s z=)1?v|2VoBTFCtxCjL|Tw=BfHsao)!m&NKwP8gmpeO9FJ$DZaMDHqWdSyGo*%=G9l zvpEU5(*8+lsP`tFA##xOqySw>mFZenf-zq%cFlWESqvyIR=d6ZIN-<)cQbv2{(R!;APK8< z{o+MB{G(`>FElhZ=PdyMH!pje9VM5G=YtVv8}&qt@_8u)t5!-dy)ts*h=^Xja!m7) zj!x?w!CcJ{md}mH>PD!mtree|s{9|}HY5Q>uj{fQ3u7i4C}uH-IB&8F%9tKEi;oXK z);{p%(1o`IdH8Rsi$j__-LUL~rs=G%9*MaXIr5)vn!Y7&Cc#ewGh*;gqLgsl&LDfm zk(pKz*5H?7k?zG?HDWrLgJ8W(kE@@Z)K=^)TriOI4J?q(o_TD}lpe}@tnJKm=xvF< zd7VoDCn0N8DQ8!=V*o6&3y~WN#i|m{Gr7rLICknfcgAwaEaJ2G1hIZjf726&q@R^Ux%;XvE1l4oQes~<$ z$5^N_F!FXgRa$Ge*_gh5Ung{ln~_3Q&}xJrMQhV`#Pt0l1G$_nLwH>~iGbt5(*+U~ zf9u&$``k;#VP|oXA|cUgRYw>dM@$ICCk`UtvXH8mAr>I|f+7vE>}MsqeDT)+f0I+c z{gnvPb*00YV|=tvhtxU|Hl>imal-w6L>{{Sobbap4LG+L$-8Hiu3q>-)7GylRP*lE z+=lFJjd6#j$+81SU83LSN%wLCU1V{Q3r5j$qsmFg;pK8)9IK~XI|s$YAlHB%F5uaM z(*C?>?l0Ti$EzjH{wVl(9>&^V<#-yEm#{KzLfh3(;$4S3mLav7?NxO_Y~{pDKcmsS zY~RRD9~quczlIZmLtH=IW3?OPSs(E2OVFo?pQqw{gRX?nmv9N zH50`DWu0Yz@QJPNtSOJGas|0qL|?7Cx^fwEvdx&LQUe>6&rD(&N9Wld$@>#?fbkK6 z!%N%zOY5nhD;QDpO^`2pv564rwruJef5kkSJuIb28_Z^N{~AzEd;JE&Q_JQ#m+qc< zo@}F@t+$n?jIx}Z%NyXxtz9ff_OpszmCwUK+G^GJZO5Yt!qAT7cB zt#RmXaD~S#Vg-tXY%aRo_e%J;Z`I5WJMC>#}B>6 z*pCf@9d7}w6J#$sU? zPx3s~I~O11mFAf=*@j;dv#Wr!5aNs z3VsWLWTZ?yYi;BUR4E^6U6}T1&2a}I0RSO8-!;7*8d!P{#V(10n;sM?TR|j%H-S8i(!|s>njwy10&O=Y zHYvWgnicD$xHOI!G#wb*nvj{8*igO8M`o{9j4TN0=_&NGr35v1%0oa=-zC7kGS9hX zw&v(KTs&PzY@Hw1ej0j_Zzl}okC1+wZXDQTOU4l>XD zCoaH#7qjqw@^ct7_evN#s`w~0NG`M`oa(qaIoGe)OygRkAk&R+eBDCSC1MlnqARPb zr%ZYq7If}?$c;v?*1|oYgvPIyexhi;lgli`6ZpQw##cvThs#aKw)HvM!#<)8+oK!C zgnHKEr=oDNZ0aQ87I$CUi3>nA+1o<*c>`M-qx@A)n%>I}fJsHX zWyNM$`QnkN1~E#Z>%_sZo2@pOc`o*~1PtYXSv%v`%g-oWW<<;tji@jVGkK#cKaW9S zl}a`#rRiy%X4_;{38ITr8~JGdv}F)OvRi7ed3D^I&Lv%=DV zET%HsEMYOTrpWM8+e6ldXrq}kvqtS(m7CVhfzW$ms^4q1PDWlDz&dTv51x)h>h ze*kj;gY8mEDsw` zi15d3y@RQ`liwI`r`csWa-ofiiF@xL6WR69a^OHY^dRQEq#RBgHUWL@3+C^Cf-#QL zzTz|qu4@X6KanucW1g|ikn%zGVG$gpFKc$=l9fmc4=vEjP+h2=KfBEzC>qC-{sPOU zRy86RA{ck7GL`Ideo$poJJ3 z+9lI)rM=@$c!S}S-FU@kE;XBb4g1PIliy#q8!U}iINxB?QrZH$YepbKdk$vFA`}k% zq?tpU3z`DvxixMg%{%CoQalR!98VsXBQnDyRq&sL__Q(NX;Z&*ewD&>>UG;gy6#*p zrE~MgvX;>USUj5x#{#+N2Ve zR)P16ceLf4Nhm(vde=qoUFaXgTu*CiSMU8Mz)H+6Pv2w~iBIhut8ttg2+yK`ztt}Xv`T19@L1pzLyo^WXJAB&})yixi zx6RNSs+${gv`j=3F7&iH<1$Q!h|!1Udjqrc=9D=*`(`l%t=@R#MCpvpQJ$I2CS(;F zG@Y!~(cfQ2G}~|oKK}0d8cug-`+|=Zt{?T z)C4Y7TODRltby?szo;&eS6fGwIp~JV(k>0pH@l;6%Xo$P{TQ)N@+0u3%W^YTeXkVxZ z^P_iDMG4!QC3JPXbaT=pFM>IWVb3#yH!gISi)7m(m@8!ykln`IrsqS0;?k5BwLj5^ z(6UxDiU&BgQmj5|>)L*8YN#IjzE=M^7Fzw*O|n`}Hg6|vyn*<_mm5x;M(+uI%FZaBt9A#D5glBc)1H|s<`E#liQMfcT zw~3VX(%K&@E;WJl`*7Yj)g^jn>sqnrt>#DEV7nOT?t1LL1{R(2;yDrcss~-m&K9lr zCX^kx0-4rT?|uIMlG)Gh%XvF_NwgEP&5@{wD-(jnq_O9d>AFnJGc~iBMiOUq&!ooa zUVC&w52d9l14T#xtM8P!>jo{rz%bJkt%S15_Ad{GS8fMY1$e9lcpVa6%+sQg%}@00#*UrY4t8G1sfer`(ETPSGAECDTcltY?-M z=uO(Q*om&-3$J;_LaFSE-X{PF@L`oh3VW;l5Pfv1^(?S`h?KR#&VDJ0Q=a3L3Cb8o z!l(S96}06A)2feN)(vJEE4;sNN%hFs#-E12Yo2BAMkh5(1ObB!D09EKmimo|sv3LS zaJwjN>CMVI%mKRO&~pbjawP3md#Zz?Wn|2jnhLkqei8Kp{zE|z_m8vNbs@%tdkqdE zT={$nxILuWCnPja4LKn_gOmt72_~V2(mfL2&VkEXTOtyg&oEo3YkmT(^cLuMrpfW_`*ftjTCXB%%f&^r%E$W?Ve z&a)+l8Ty6f-I|H+0ntcV#%m|;d2OL?6w?lV4?FH# zV{m0L*iC0`NcU_2Ph|tBMz`V+ea`fRpM`mD1C?(d{4BJ+ z>e!fWH0kQDM8#`*tg9ak7adi}KChC=DrB==5?xYK{WIEZDA>eY9iP`nqys`m;UM-X zjl8=B9|pvGItC5;4z8M5gPI~MheG1SkV;h}c8P9brZk5%Bnc9$7yC?(zFFbUs3m%1 zO9}NfxMm65g;Ed72NM6f-Kp`YEaWHcP4*Vl)ues#_5k50nLMu_WMR6`*+O$cvcfg& zm#JSxal_F^RLu6dTM|u~d5#b0yDXL-;t=^Li?{R6)NB-sghd-GOlO$%HkGvhe>pZ8 ze^Gb(-V=Rla&P)>+5Sn`4m?=d>CUJjn^}u_XrdA!AT6+&rq;}4)yNrJLGo^%X*}k( zf%S0GjW{;?RnKP3DMq2;!{jBr%@$Q}RP2R&?YZ*6Wzln~(Bz%zf|RLF*>TxNM@*e&nz-=|-tK!VB_6izZPPhU z<$a6tETe9gvtl6D{5glW+%hyA@e$*J!<)8T{X9u4()Md<$GeeJFzuK zi=={084Cu;cWrU;0T9#b#%dzAJ8$yAgd!HDExoCdOxyt|JCRhHeX)YY`b~z`5ctoo z$nYVu)sO77kPZBr**nsC@_o@7!E_gO5hM4OmDk^;OqlM3{IK4+-gv){9ZsoZbwawkr+d~?U@wS`-+pliyuJ$K-f|ZGu=Omw? z%$Cuo4wq`kE+k!h`MziNW?AImZCJ8J6+*OT9DPCH_Gv6H7GGv&50QkG@l4pAjtV(k z3()$h`{g=Ge{vu$tN7`O3?(YqYNd8h_Z>?BuTrO=J*6z}iYrW5ow}%d zug*1jE*pKP5uGJVbCsblDU|moGU|8`nH(?D+=-RB)YZohKM*>|NtUmT$Jr_I<9HZV zDm*u;de&O1t2KprkA0t$<}|4Du|1ZrZqfdUpPiIlt^qQ$<%FZIHNTH=udxL|oz5Ow z#z{lyAWv^VgYD!YTxi8i!J`}~P%V0g=0*!aYxRQBny**f(5IeV@~CA0c=_pS%_KK=dukQb8fGF^Fu zIAt|1#9i=Z*v5ut!0Pk;C>%xC$|Wixq|I{mRL 201 (got {sorted(set(codes))})") + r.check(len(ids) == 1, f"all responses carry the same message id ({len(ids)} distinct)") + r.check(len(instances) >= 2, f"requests were served by >=2 replicas ({len(instances)})") + r.check(sum(m["text"] == text for m in history(path, alice["token"])) == 1, "exactly one message in history") + first = res[0][1] + r.check(all(x[1] == first for x in res), "every replay returns the identical Message") + + print("== семантика ключа ==") + st, _, _ = api.call("POST", path, {"text": "другой текст"}, token=alice["token"], headers={"Idempotency-Key": key}) + r.check(st == 409, f"same key, different body -> 409 (got {st})") + raw = ('{ "text" : ' + json.dumps(text, ensure_ascii=False) + ' }').encode() + st, body, _ = api.raw("POST", path, raw, {"Content-Type": "application/json", "Idempotency-Key": key, + "Authorization": "Bearer " + alice["token"]}) + r.check(st == 201 and json.loads(body)["id"] == first["id"], f"other JSON formatting = same request (got {st})") + st, other, _ = api.call("POST", path2, {"text": text}, token=alice["token"], headers={"Idempotency-Key": key}) + r.check(st == 201 and other["id"] != first["id"], "same key in another chat is independent") + st, bobs, _ = api.call("POST", path, {"text": text}, token=bob["token"], headers={"Idempotency-Key": key}) + r.check(st == 201 and bobs["id"] != first["id"] and bobs["sender_id"] == bob["id"], + "same key from another user is independent") + st, _, _ = api.raw("POST", path, json.dumps({"text": "без ключа"}).encode(), + {"Content-Type": "application/json", "Authorization": "Bearer " + alice["token"]}) + r.check(st == 400, f"missing Idempotency-Key -> 400 (got {st})") + for bad in ("has space", "x" * 129, "slash/key"): + st, _, _ = api.call("POST", path, {"text": "bad key"}, token=alice["token"], + headers={"Idempotency-Key": bad}) + r.check(st == 400, f"invalid key {bad[:12]!r}... -> 400 (got {st})") + st, _, _ = api.call("POST", path, {"text": text}, headers={"Idempotency-Key": key}) + r.check(st == 401, f"replay without session -> 401, access re-checked (got {st})") + + print("== неопределённый результат: клиент повторяет с тем же ключом ==") + key2 = uuid.uuid4().hex + try: + api.call("POST", path, {"text": "таймаут?"}, token=alice["token"], headers={"Idempotency-Key": key2}, + timeout=0.001) + except OSError: + pass # ответ не дождались: записано или нет — неизвестно + st, again, _ = api.call("POST", path, {"text": "таймаут?"}, token=alice["token"], headers={"Idempotency-Key": key2}) + r.check(st == 201, "retry after uncertain result -> 201") + r.check(sum(m["text"] == "таймаут?" for m in history(path, alice["token"])) == 1, "still exactly one message") + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/jobs.py b/student-tests/jobs.py new file mode 100644 index 00000000..3457ae51 --- /dev/null +++ b/student-tests/jobs.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +""" +Фоновая обработка (lab5): отказы, повторная доставка, outbox, retry/DLQ/replay, аренда. + + python student-tests/jobs.py --base-url http://localhost:8080 [--scenario NAME|all] + +Сценарии (каждый печатает ok/FAIL и итог): + worker-down worker остановлен → 202 queued, content 409, сообщения работают; + рестарт брокера и API с volumes; worker запущен → ready (уровень 3) + broker-down брокер недоступен при upload → всё равно 202 (намерение в outbox), + брокер вернулся → relay публикует → ready + redelivery дубль сообщения уже ready-задачи → no-op, второго набора вариантов нет + crash-s3 worker-fault падает ПОСЛЕ записи в S3, ДО COMMIT/ack → та же задача доходит до ready + crash-ack worker-fault падает ПОСЛЕ COMMIT ready, ДО ack → повторная доставка = no-op + outbox api-fault падает ПОСЛЕ COMMIT, ДО publish → relay публикует, ready без нового upload + retry-replay временная ошибка на каждой попытке → backoff с jitter → failed retry_exhausted + → явный replay → ready; повторный replay ready-задачи отклонён + lease «долгая, но живая» задача (heartbeat) не перехватывается; + «замёрзший» worker теряет аренду → reaper → другая попытка → ready, + проснувшийся старый worker ничего не перезаписывает + +Docker-команды (по умолчанию) можно заменить файлом --hooks-file (JSON: имя → argv). +Тест сам переключает worker ↔ worker-fault и в конце возвращает обычный worker. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +import time +from pathlib import Path + +from common import Api, Report, upload + +ROOT = Path(__file__).resolve().parent.parent +FIXTURE = (ROOT / "tests/fixtures/sample.png").read_bytes() # 320×200 +C = ["docker", "compose", "--env-file", ".env"] +F = C + ["--profile", "fault"] +HOOKS = { + "stop_worker": C + ["stop", "worker"], + "start_worker": C + ["start", "worker"], + "start_fault_worker": F + ["up", "-d", "worker-fault"], + "stop_fault_worker": F + ["stop", "worker-fault"], + "start_fault_api": F + ["up", "-d", "api-fault"], + "restart_api": C + ["restart", "api"], + "restart_broker": C + ["restart", "broker"], + "stop_broker": C + ["stop", "broker"], + "start_broker": C + ["start", "broker"], + "tool": C + ["exec", "-T", "api", "python", "-m", "app.tools"], + "replay": C + ["exec", "-T", "api", "python", "-m", "app.replay"], + "redis": C + ["exec", "-T", "broker", "redis-cli"], +} +FAULT_BASE = "http://localhost:8082" + + +def sh(name: str, *args: str, check: bool = True) -> str: + out = subprocess.run(HOOKS[name] + list(args), cwd=ROOT, capture_output=True, text=True) + if check and out.returncode != 0: + raise RuntimeError(f"{name} {' '.join(args)}: {out.stderr.strip()[-300:]}") + return out.stdout + + +def tool(*args: str): + out = sh("tool", *args) + text = out.strip() + try: + return json.loads(text) + except ValueError: + return json.loads(text.splitlines()[-1]) + + +def fault(*args: str) -> None: + sh("redis", *args) + + +class Ctx: + def __init__(self, base: str): + self.api = Api(base) + self.r = Report() + self.alice = self.api.new_user("jobs_a") + st, chat, _ = self.api.call("POST", "/api/v1/chats", {"title": "jobs", "member_ids": [self.alice["id"]]}, + token=self.alice["token"]) + assert st == 201, st + self.chat = chat + + def upload(self, data: bytes = FIXTURE, base: str | None = None, headers: dict | None = None): + api = Api(base) if base else self.api + return upload(api, self.alice["token"], self.chat["id"], data, "sample.png", headers=headers) + + def get(self, att_id: str) -> dict: + st, att, _ = self.api.call("GET", f"/api/v1/attachments/{att_id}", token=self.alice["token"]) + assert st == 200, st + return att + + def wait(self, att_id: str, statuses=("ready",), seconds: float = 90) -> dict: + deadline = time.monotonic() + seconds + att = self.get(att_id) + while time.monotonic() < deadline and att["status"] not in statuses: + time.sleep(0.5) + att = self.get(att_id) + return att + + def content(self, att_id: str, variant: str = "original"): + return self.api.raw("GET", f"/api/v1/attachments/{att_id}/content?variant={variant}", + headers={"Authorization": "Bearer " + self.alice["token"]}) + + def check_ready_result(self, att: dict, label: str) -> None: + r = self.r + names = [v["name"] for v in att["variants"]] + r.check(att["status"] == "ready" and att["error_code"] is None, f"{label}: status ready, error_code null") + r.check(names == ["thumbnail", "crop", "optimized"], f"{label}: variants thumbnail+crop+optimized ({names})") + st, body, rh = self.content(att["id"]) + r.check(st == 200 and body == FIXTURE, f"{label}: original byte-for-byte") + th = {v["name"]: v for v in att["variants"]}.get("thumbnail") + st, tb, rh = self.content(att["id"], "thumbnail") + r.check(st == 200 and th and (th["width"], th["height"]) == (256, 160) + and hashlib.sha256(tb).hexdigest() == th["sha256"], f"{label}: thumbnail 256x160, sha256 matches") + + def job(self, att_id: str) -> dict: + return tool("job", att_id) + + def use_fault_worker(self) -> None: + sh("stop_worker") + sh("start_fault_worker") + time.sleep(3) + + def use_normal_worker(self) -> None: + sh("stop_fault_worker", check=False) + sh("start_worker") + + +# ======================================================================== сценарии +def worker_down(c: Ctx) -> None: + r = c.r + print("== worker остановлен: приём не зависит от обработки ==") + sh("stop_worker") + t0 = time.monotonic() + st, att, _ = c.upload() + took = time.monotonic() - t0 + r.check(st == 202 and att["status"] == "queued" and att["variants"] == [], + f"upload → 202 queued without worker ({took:.2f}s)") + aid = att["id"] + time.sleep(3) + r.check(c.get(aid)["status"] == "queued", "still queued after 3 s: API did not resize synchronously") + st, _, _ = c.content(aid) + r.check(st == 409, f"content of queued attachment → 409 (got {st})") + st, _, _ = c.content(aid, "thumbnail") + r.check(st == 409, f"thumbnail of queued attachment → 409 (got {st})") + st, msg, _ = c.api.call("POST", f"/api/v1/chats/{c.chat['id']}/messages", + {"text": "", "attachment_ids": [aid]}, token=c.alice["token"]) + r.check(st == 201, f"message with queued attachment → 201 (got {st})") + st, _, _ = c.api.call("POST", f"/api/v1/chats/{c.chat['id']}/messages", {"text": "другие сообщения работают"}, + token=c.alice["token"]) + r.check(st == 201, "other messages keep working") + q = tool("queue") + r.check(q["stream_length"] >= 1 and q["outbox_unpublished"] == 0, + f"job is in the broker stream (len={q['stream_length']}), outbox published") + + print("== рестарт брокера и API с сохранением volumes ==") + sh("restart_broker") + sh("restart_api") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + if c.api.raw("GET", "/health/ready")[0] == 200: + break + except OSError: + pass + time.sleep(1) + q = tool("queue") + r.check(q["stream_length"] >= 1, f"after broker restart the message is still there (len={q['stream_length']})") + st, same, _ = c.api.call("GET", f"/api/v1/attachments/{aid}", token=c.alice["token"]) + r.check(st == 200 and same["status"] == "queued", "same session, same attachment, still queued") + + print("== worker запущен ==") + sh("start_worker") + att = c.wait(aid, ("ready", "failed"), 60) + c.check_ready_result(att, "after restart") + time.sleep(3) + j = c.job(aid) + r.check(not j["staging_object_exists"] and not j["attachment"]["staging_present"], + "staging removed after ready (original lives in originals/)") + print(f" attachment {aid}, job {j['job']['job_id']}, events: {j['events']}") + + +def broker_down(c: Ctx) -> None: + r = c.r + print("== брокер недоступен: приём не зависит от брокера (transactional outbox) ==") + sh("stop_broker") + try: + r.check(c.api.raw("GET", "/health/ready")[0] == 200, "/health/ready = 200: broker is not critical for API") + t0 = time.monotonic() + st, att, _ = c.upload() + r.check(st == 202 and att["status"] == "queued", + f"upload → 202 queued while broker is down ({time.monotonic() - t0:.1f}s, publish timeout ≤ 2s)") + aid = att["id"] + j = c.job(aid) + r.check(len(j["outbox"]) == 1 and not j["outbox"][0]["published"], "intent kept in outbox, not published yet") + finally: + sh("start_broker") + att = c.wait(aid, ("ready", "failed"), 60) + c.check_ready_result(att, "after broker is back") + j = c.job(aid) + r.check(j["outbox"][0]["published"], "relay published the outbox row when broker came back") + + +def redelivery(c: Ctx) -> None: + r = c.r + print("== повторная доставка уже ready-задачи ==") + st, att, _ = c.upload() + att = c.wait(att["id"]) + before = c.job(att["id"]) + for _ in range(2): + print(" ", sh("tool", "republish", att["id"]).strip()[:120]) + time.sleep(4) + after = c.job(att["id"]) + same = c.get(att["id"]) + r.check(same["status"] == "ready" and same["variants"] == att["variants"], + "duplicates: status stays ready, variants unchanged (same sha256)") + r.check(len(after["variant_rows"]) == 3 and after["variant_objects_in_s3"] == 3, + f"no second set of variants: rows={len(after['variant_rows'])}, S3 objects={after['variant_objects_in_s3']}") + r.check(sum("ready" in e.split()[1:2] for e in after["events"]) == 1 + and after["job"]["attempt"] == before["job"]["attempt"], + "job finished exactly once, attempt unchanged") + q = tool("queue") + r.check(all(g["pending"] == 0 for g in q["groups"]), "duplicates acked (no pending messages)") + + +def crash_s3(c: Ctx) -> None: + r = c.r + print("== crash после записи в S3, до COMMIT и ack ==") + c.use_fault_worker() + try: + fault("SET", "fault:crash-after-s3-write", "1") + st, att, _ = c.upload() + t0 = time.monotonic() + att = c.wait(att["id"], ("ready", "failed"), 120) + print(f" recovered in {time.monotonic() - t0:.1f}s (worker restart + lease expiry)") + c.check_ready_result(att, "after crash before ack") + j = c.job(att["id"]) + r.check(any("lease_takeover" in e for e in j["events"]) and j["job"]["attempt"] == 1, + "same attempt taken over after lease expiry (redelivery, not a new job)") + r.check(len(j["variant_rows"]) == 3 and j["variant_objects_in_s3"] == 3, + "S3 objects from the crashed attempt reused: still exactly 3 variants") + print(f" events: {j['events']}") + finally: + fault("DEL", "fault:crash-after-s3-write") + c.use_normal_worker() + + +def crash_ack(c: Ctx) -> None: + r = c.r + print("== crash после COMMIT ready, до ack ==") + c.use_fault_worker() + try: + fault("SET", "fault:crash-after-commit-before-ack", "1") + st, att, _ = c.upload() + att = c.wait(att["id"], ("ready", "failed"), 60) + c.check_ready_result(att, "commit before crash") + deadline = time.monotonic() + 40 + while time.monotonic() < deadline: + q = tool("queue") + if all(g["pending"] == 0 for g in q["groups"]): + break + time.sleep(2) + r.check(all(g["pending"] == 0 for g in q["groups"]), + "unacked message redelivered after restart and acked as no-op") + j = c.job(att["id"]) + r.check(sum(e.split()[1] == "ready" for e in j["events"]) == 1 and len(j["variant_rows"]) == 3, + "ready recorded once, 3 variants") + finally: + fault("DEL", "fault:crash-after-commit-before-ack") + c.use_normal_worker() + + +def outbox(c: Ctx) -> None: + r = c.r + print("== crash API между COMMIT и publish (transactional outbox) ==") + sh("start_fault_api") + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + if Api(FAULT_BASE).raw("GET", "/health/ready")[0] == 200: + break + except OSError: + pass + time.sleep(1) + before = {a["id"] for a in tool("chat-attachments", c.chat["id"])} + try: + st, body, _ = c.upload(base=FAULT_BASE, headers={"X-Fault-Inject": "crash-after-commit-before-publish"}) + except OSError as exc: + st, body = None, str(exc) + r.check(st != 202, f"client did NOT get 202: API died after COMMIT (got {st})") + new = [a for a in tool("chat-attachments", c.chat["id"]) if a["id"] not in before] + r.check(len(new) == 1, f"attachment and job were committed before the crash ({len(new)} new)") + if new: + aid = new[0]["id"] + att = c.wait(aid, ("ready", "failed"), 60) + c.check_ready_result(att, "relay-published job") + j = c.job(aid) + r.check(all(o["published"] for o in j["outbox"]) and len(j["outbox"]) == 1, + "outbox row published by relay (no new job created)") + + +def retry_replay(c: Ctx) -> None: + r = c.r + print("== временные ошибки → backoff+jitter → failed → replay ==") + c.use_fault_worker() + try: + fault("SET", "fault:transient-error", "100") + st, att, _ = c.upload() + aid = att["id"] + t0 = time.monotonic() + att = c.wait(aid, ("failed", "ready"), 150) + r.check(att["status"] == "failed" and att["error_code"] == "retry_exhausted" and att["variants"] == [], + f"after {time.monotonic() - t0:.0f}s: failed, error_code=retry_exhausted") + j = c.job(aid) + delays = [float(e.split(" in ")[-1].rstrip("s")) for e in j["events"] if "retry_scheduled" in e] + r.check(j["job"]["attempt"] == j["job"]["max_attempts"] == 5 and len(delays) == 4, + f"bounded: 5 attempts, 4 scheduled retries, delays {delays}") + base = [1, 2, 4, 8] + r.check(all(b / 2 <= d <= b * 1.01 for b, d in zip(base, delays)), + "backoff grows 2^n with jitter in [d/2, d] (JOB_BACKOFF_BASE_SECONDS=1)") + st, _, _ = c.content(aid) + r.check(st == 409, f"content of failed → 409 (got {st})") + st, _, _ = c.api.call("POST", f"/api/v1/chats/{c.chat['id']}/messages", + {"text": "x", "attachment_ids": [aid]}, token=c.alice["token"]) + r.check(st == 409, f"failed attachment cannot be attached → 409 (got {st})") + listed = sh("replay", "--list") + r.check(aid in listed, "failed job is in the registry (python -m app.replay --list)") + + fault("DEL", "fault:transient-error") + out = sh("replay", "--attachment", aid, "--actor", "jobs-test") + r.check("replayed" in out, "explicit replay accepted") + att = c.wait(aid, ("ready", "failed"), 60) + c.check_ready_result(att, "after replay") + j = c.job(aid) + r.check(j["job"]["attempt"] == 6 and j["job"]["replays"] == 1, "same job_id, attempt 6, replays=1 (audit)") + out = sh("replay", "--attachment", aid) + r.check("refused" in out, "replay of a ready job is refused (no duplicate execution)") + finally: + fault("DEL", "fault:transient-error") + c.use_normal_worker() + + +def lease(c: Ctx) -> None: + r = c.r + print("== долгая, но живая задача: heartbeat продлевает аренду ==") + c.use_fault_worker() + try: + fault("SET", "fault:slow-seconds", "30") + t0 = time.monotonic() + st, att, _ = c.upload() + att = c.wait(att["id"], ("ready", "failed"), 90) + took = time.monotonic() - t0 + fault("DEL", "fault:slow-seconds") + j = c.job(att["id"]) + r.check(att["status"] == "ready" and took > 25 and j["job"]["attempt"] == 1 + and not any("takeover" in e or "lease_expired" in e for e in j["events"]), + f"job ran {took:.0f}s > lease 20s, no takeover, attempt 1") + + print("== замёрзший worker: аренда истекает, задачу забирают ==") + fault("SET", "fault:freeze-after-claim", "1") + st, att, _ = c.upload() + aid = att["id"] + time.sleep(3) + r.check(c.get(aid)["status"] == "processing", "claimed by worker-fault, then it froze (processing)") + sh("start_worker") # второй, здоровый worker + t0 = time.monotonic() + att = c.wait(aid, ("ready", "failed"), 120) + print(f" recovered in {time.monotonic() - t0:.1f}s") + c.check_ready_result(att, "after frozen worker") + j = c.job(aid) + r.check(j["job"]["attempt"] == 2 and any("lease_expired" in e for e in j["events"]), + "reaper: lease expired → attempt 2 → ready") + time.sleep(max(0, 65 - (time.monotonic() - t0))) # frozen worker wakes up (3×lease after claim) + j2 = c.job(aid) + r.check(sum(e.split()[1] == "ready" for e in j2["events"]) == 1 and len(j2["variant_rows"]) == 3 + and c.get(aid)["variants"] == att["variants"], + "woken stale worker could not commit (lease_token fencing), result unchanged") + finally: + fault("DEL", "fault:slow-seconds") + fault("DEL", "fault:freeze-after-claim") + c.use_normal_worker() + + +SCENARIOS = {"worker-down": worker_down, "broker-down": broker_down, "redelivery": redelivery, "crash-s3": crash_s3, + "crash-ack": crash_ack, "outbox": outbox, "retry-replay": retry_replay, "lease": lease} + + +def main() -> None: + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--scenario", default="all", choices=["all", *SCENARIOS]) + ap.add_argument("--hooks-file", help="JSON {hook: argv} вместо docker compose") + args = ap.parse_args() + if args.hooks_file: + HOOKS.update(json.loads(Path(args.hooks_file).read_text(encoding="utf-8"))) + c = Ctx(args.base_url) + names = list(SCENARIOS) if args.scenario == "all" else [args.scenario] + for name in names: + t0 = time.monotonic() + try: + SCENARIOS[name](c) + except Exception as exc: # noqa: BLE001 + c.r.check(False, f"{name}: crashed with {type(exc).__name__}: {exc}") + print(f" [{name}: {time.monotonic() - t0:.0f}s]") + c.r.finish() + + +if __name__ == "__main__": + sys.stdout.reconfigure(encoding="utf-8") + main() diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 00000000..b76e62dd --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,110 @@ +""" +Генератор записей для экспериментов отказа и rolling update (lab3, уровень «4»). + +Несколько потоков в замкнутом цикле отправляют сообщения через LB. У каждого +сообщения свой Idempotency-Key; если ответ не получен (ошибка/таймаут/5xx), +поток повторяет ТОТ ЖЕ запрос с ТЕМ ЖЕ ключом — так проверяется, что +неопределённая запись безопасно повторяется и не даёт дублей. +""" +import threading +import time +import uuid +from urllib.parse import quote + +from common import Api + + +class LoadGen: + def __init__(self, api: Api, token: str, chat_id: str, workers: int = 4, timeout: float = 2.0): + self.api, self.token, self.chat_id = api, token, chat_id + self.path = f"/api/v1/chats/{chat_id}/messages" + self.workers, self.timeout = workers, timeout + self.stop_event = threading.Event() + self.lock = threading.Lock() + self.ok = [] # (t, instance, message) + self.errors = [] # (t, reason) + self.retried = 0 # сколько раз повторяли тот же ключ + self.threads = [] + self.t0 = None + + def _worker(self, w: int): + i = 0 + while not self.stop_event.is_set(): + key = uuid.uuid4().hex + text = f"load-{w}-{i}-{key[:8]}" + first = True + while not self.stop_event.is_set(): + if not first: + with self.lock: + self.retried += 1 + first = False + try: + st, body, rh = self.api.call("POST", self.path, {"text": text}, token=self.token, + headers={"Idempotency-Key": key}, timeout=self.timeout) + except OSError as exc: + st, body, rh = None, None, None + reason = type(exc).__name__ + else: + reason = f"HTTP {st}" + now = time.monotonic() + if st == 201: + with self.lock: + self.ok.append((now, rh.get("X-Instance-Id", ""), body)) + break + with self.lock: + self.errors.append((now, reason)) + time.sleep(0.1) # повтор с тем же ключом + i += 1 + + def start(self): + self.t0 = time.monotonic() + for w in range(self.workers): + t = threading.Thread(target=self._worker, args=(w,), daemon=True) + t.start() + self.threads.append(t) + + def stop(self): + self.stop_event.set() + for t in self.threads: + t.join(timeout=self.timeout + 2) + + # ---------------------------------------------------------------- метрики + def window(self, a: float, b: float) -> dict: + """Успехи/ошибки в окне [a, b) секунд от старта нагрузки.""" + ok = [x for x in self.ok if a <= x[0] - self.t0 < b] + err = [x for x in self.errors if a <= x[0] - self.t0 < b] + dur = max(b - a, 1e-9) + return {"successful": len(ok), "errors": len(err), "successful_rps": round(len(ok) / dur, 1)} + + def max_gap_after(self, t_abs: float) -> float: + """Самый длинный промежуток без успешной записи после момента t_abs (сек).""" + times = [t_abs] + sorted(x[0] for x in self.ok if x[0] >= t_abs) + if len(times) == 1: + return float("inf") + return round(max(b - a for a, b in zip(times, times[1:])), 3) + + def first_success_after(self, t_abs: float): + later = sorted(x[0] for x in self.ok if x[0] >= t_abs) + return round(later[0] - t_abs, 3) if later else None + + def instances_after(self, t_abs: float) -> list: + return [x[1] for x in sorted(self.ok) if x[0] >= t_abs] + + def verify(self) -> dict: + """Все подтверждённые сообщения есть в истории без изменений, дублей нет.""" + items, cursor = [], None + while True: + q = "?limit=100" + (f"&cursor={quote(cursor, safe='')}" if cursor else "") + st, page, _ = self.api.call("GET", self.path + q, token=self.token) + assert st == 200, f"history read failed: {st}" + items += page["items"] + cursor = page["next_cursor"] + if cursor is None: + break + by_id = {m["id"]: m for m in items} + texts = [m["text"] for m in items] + confirmed = [x[2] for x in self.ok] + lost = [m for m in confirmed if by_id.get(m["id"]) != m] + return {"history_messages": len(items), "confirmed": len(confirmed), + "lost_or_changed": len(lost), "duplicate_texts": len(texts) - len(set(texts)), + "retries_with_same_key": self.retried} diff --git a/student-tests/logout_cache.py b/student-tests/logout_cache.py new file mode 100644 index 00000000..04ae3725 --- /dev/null +++ b/student-tests/logout_cache.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +""" +Logout при приостановленном Redis (замечание к app/sessions.py). + +Сценарий преподавателя: прогреть сессию в кэше, приостановить Redis, сделать +logout, восстановить Redis и проверить старый токен. +Требование: 204 только при гарантированном отзыве; после 204 старый токен +никогда не принимается, в том числе после восстановления Redis. + + python student-tests/logout_cache.py --base-url http://localhost:8080 +(по умолчанию сам делает docker compose pause/unpause redis) +""" +import argparse +import shlex +import subprocess +import time + +from common import Api, Report + +COMPOSE = "docker compose --env-file .env" + + +def sh(cmd: str): + print(f" $ {cmd}") + subprocess.run(shlex.split(cmd), check=True, stdout=subprocess.DEVNULL) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--pause-redis", default=f"{COMPOSE} pause redis") + ap.add_argument("--unpause-redis", default=f"{COMPOSE} unpause redis") + args = ap.parse_args() + api, r = Api(args.base_url), Report() + + user = api.new_user("logoutcache") + token = user["token"] + for _ in range(2): # первый запрос кладёт сессию в Redis, второй читает из кэша + st, _, _ = api.call("GET", "/api/v1/users/me", token=token) + r.check(st == 200, "session is warmed up in cache") + + print("== Redis приостановлен ==") + sh(args.pause_redis) + try: + st_logout, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=token) + r.check(st_logout != 204, + f"logout is NOT confirmed while cache cannot be invalidated (got {st_logout}, expected 503)") + finally: + sh(args.unpause_redis) + time.sleep(0.5) + + print("== Redis восстановлен ==") + st, _, _ = api.call("GET", "/api/v1/users/me", token=token) + if st_logout == 204: + r.check(st == 401, f"after a 204 the old token is rejected after Redis recovery (got {st})") + else: + r.check(st == 200, f"unconfirmed logout left the session consistent and valid (got {st})") + st, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=token) + r.check(st == 204, f"retry logout after Redis recovery -> 204 (got {st})") + st, _, _ = api.call("GET", "/api/v1/users/me", token=token) + r.check(st == 401, f"old token after confirmed logout -> 401, no stale cache copy (got {st})") + st, _, _ = api.call("GET", "/api/v1/users/me", token=token) + r.check(st == 401, f"old token still rejected on repeat (got {st})") + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/orphans.py b/student-tests/orphans.py new file mode 100644 index 00000000..c43d5f00 --- /dev/null +++ b/student-tests/orphans.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +Согласованность S3 ↔ PostgreSQL при сбоях (lab4, уровень «4»). +Сам управляет контейнерами через docker compose; запускать из корня репозитория: + + docker compose --env-file .env --profile fault up -d api-fault + python student-tests/orphans.py --base-url http://localhost:8080 --fault-url http://localhost:8082 + +Сценарии: + 1. Прерванная загрузка (клиент оборвал тело) → в S3 и журнале ничего не появилось. + 2. PUT в S3 прошёл, запись в БД упала (db-error-after-s3-put) → 503, API сразу + компенсирует: объект и строка журнала удалены, вложения нет. + 3. PUT прошёл, процесс убит до COMMIT (crash-after-s3-put, os._exit) → остаётся + объект + строка upload_intents (известная сирота). Janitor: + - с рабочим grace period (900 c) её НЕ трогает (может быть живая загрузка); + - dry-run показывает, что удалит, и ничего не меняет; + - после истечения grace удаляет объект и строку; повтор — без изменений. + 4. Объект без журнала и без метаданных (страховка) → находится обходом bucket + (--scan-bucket) и удаляется только после grace. + 5. Готовые вложения не удаляются никогда: после всех проходов скачиваются, sha256 совпадает. +""" +import argparse +import time + +from common import SAMPLE_PNG, Api, Report, compose, janitor, raw_http, sha256, tool, upload, wait_ready as att_ready + + +def wait_ready(api: Api, seconds: int = 60) -> bool: + deadline = time.time() + seconds + while time.time() < deadline: + try: + if api.raw("GET", "/health/ready", timeout=3)[0] == 200: + return True + except OSError: + pass + time.sleep(1) + return False + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--fault-url", default="http://localhost:8082") + ap.add_argument("--grace", type=int, default=5, help="маленький grace для теста (с)") + args = ap.parse_args() + api, fault, r = Api(args.base_url), Api(args.fault_url), Report() + r.check(wait_ready(fault, 60), "fault replica (profile fault) is ready on --fault-url") + + user = api.new_user("orphan") + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "orphans", "member_ids": [user["id"]]}, + token=user["token"]) + st, keep, _ = upload(api, user["token"], chat["id"], SAMPLE_PNG, "keep.png") + r.check(st == 202, "committed attachment that must survive every cleanup") + r.check(att_ready(api, user["token"], keep["id"]).get("status") == "ready", "it is processed to ready (lab5)") + # Сначала дочищаем всё старое, чтобы считать объекты от нуля. + janitor("--grace-seconds", "0", "--scan-bucket") + base = tool("stats") + print(" baseline:", base) + + print("== 1. прерванная загрузка ==") + path = f"/api/v1/chats/{chat['id']}/attachments?filename=cut.png" + raw_http(args.base_url, "POST", path, + {"Authorization": "Bearer " + user["token"], "Content-Type": "image/png", + "Content-Length": str(len(SAMPLE_PNG))}, + chunks=[SAMPLE_PNG[:20000]], close_after_send=True) + time.sleep(2) + now = tool("stats") + r.check(now == base, f"no object, no metadata, no journal row after interrupted upload ({now})") + + print("== 2. отказ БД после PUT: немедленная компенсация ==") + st, body, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "dbfail.png", + headers={"X-Fault-Inject": "db-error-after-s3-put"}) + r.check(st == 503, f"upload answered 503, not 202 (got {st})") + now = tool("stats") + r.check(now == base, f"object deleted by compensation, journal clean ({now})") + + print("== 3. процесс убит между PUT и COMMIT ==") + try: + st, _, _ = upload(fault, user["token"], chat["id"], SAMPLE_PNG, "crash.png", + headers={"X-Fault-Inject": "crash-after-s3-put"}, timeout=20) + except OSError: + st = None + r.check(st != 202, f"client did NOT get 202 (got {st})") + time.sleep(1) + now = tool("stats") + r.check(now["objects"] == base["objects"] + 1 and now["upload_intents"] == base["upload_intents"] + 1 + and now["attachments"] == base["attachments"], + f"orphan object + journal row left behind, no metadata ({now})") + s = janitor() # рабочий grace period + r.check(s["objects_deleted"] == 0 and tool("stats") == now, + f"default grace {s['grace_seconds']} s: fresh upload is NOT touched") + time.sleep(args.grace + 1) + s = janitor("--grace-seconds", str(args.grace), "--dry-run") + r.check(s["would_delete_objects"] == 1 and tool("stats") == now, "dry-run reports 1 object, changes nothing") + s = janitor("--grace-seconds", str(args.grace)) + after = tool("stats") + r.check(s["objects_deleted"] == 1 and after == base, f"janitor removed orphan + journal row ({s})") + s = janitor("--grace-seconds", str(args.grace)) + r.check(s["objects_deleted"] == 0 and tool("stats") == base, "second run is a no-op (idempotent)") + + print("== 4. объект без журнала (страховочный обход bucket) ==") + compose("exec", "-T", "api", "python", "-c", + "import uuid; from app import storage; storage.init(); " + "storage.client().put_object(Bucket=storage.settings.s3_bucket, " + "Key=storage.object_key(uuid.uuid4()), Body=b'lost')") + r.check(tool("stats")["objects"] == base["objects"] + 1, "untracked object planted") + s = janitor("--scan-bucket", "--grace-seconds", "3600") + r.check(s["orphans_found"] == 0, "scan with grace 1 h ignores a fresh object") + time.sleep(args.grace + 1) + s = janitor("--scan-bucket", "--grace-seconds", str(args.grace), "--dry-run") + r.check(s["orphans_found"] == 1 and tool("stats")["objects"] == base["objects"] + 1, "scan dry-run finds 1") + s = janitor("--scan-bucket", "--grace-seconds", str(args.grace)) + r.check(s["objects_deleted"] == 1 and tool("stats") == base, f"scan deletes the untracked object ({s})") + + print("== 5. готовые вложения не тронуты ==") + st, raw, _ = api.raw("GET", f"/api/v1/attachments/{keep['id']}/content", + headers={"Authorization": "Bearer " + user["token"]}) + r.check(st == 200 and sha256(raw) == keep["sha256"], "committed attachment still downloads, sha256 matches") + state = tool("state", keep["id"]) + r.check(state == {"staging_object_exists": False, "original_object_exists": True, + "attachment_row": True, "attachment_status": "ready", "upload_intent": False}, + f"published original + metadata, staging cleaned, no journal row ({state})") + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/outage.py b/student-tests/outage.py new file mode 100644 index 00000000..f6e73438 --- /dev/null +++ b/student-tests/outage.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +""" +Недоступность зависимостей (уровень «4»). Сам останавливает и запускает контейнеры: + python student-tests/outage.py --base-url http://localhost:8080 + +1. Redis остановлен → API работает в обход кэша (сессии читаются из PostgreSQL). +2. PostgreSQL остановлен → защищённые запросы получают ограниченную ошибку 503, + неизвестный токен НЕ принимается (никогда 200), /health/ready = 503. +3. PostgreSQL запущен снова → тот же токен снова работает, данные на месте. +""" +import argparse +import shlex +import subprocess +import time +import uuid + +from common import Api, Report + +COMPOSE = "docker compose --env-file .env" + + +def sh(cmd: str): + print(f" $ {cmd}") + subprocess.run(shlex.split(cmd), check=True, stdout=subprocess.DEVNULL) + + +def wait_ready(api: Api, seconds=90): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if api.raw("GET", "/health/ready", timeout=3)[0] == 200: + return True + except OSError: + pass + time.sleep(1) + return False + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--stop-redis", default=f"{COMPOSE} stop redis") + ap.add_argument("--start-redis", default=f"{COMPOSE} start redis") + ap.add_argument("--stop-db", default=f"{COMPOSE} stop postgres") + ap.add_argument("--start-db", default=f"{COMPOSE} start postgres") + args = ap.parse_args() + api, r = Api(args.base_url), Report() + + user = api.new_user("outage") + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "outage", "member_ids": [user["id"]]}, token=user["token"]) + path = f"/api/v1/chats/{chat['id']}/messages" + api.call("POST", path, {"text": "before outage"}, token=user["token"]) + + print("== 1. Redis недоступен ==") + sh(args.stop_redis) + try: + st, _, _ = api.call("GET", "/api/v1/users/me", token=user["token"]) + r.check(st == 200, "valid token still works without Redis (cache bypassed)") + st, _, _ = api.call("POST", path, {"text": "without redis"}, token=user["token"]) + r.check(st == 201, "writes still work without Redis") + fresh = api.register("noredis") + st, sess, _ = api.login(fresh["username"], fresh["password"]) + r.check(st == 201, "login works without Redis") + st, _, _ = api.call("GET", "/api/v1/users/me", token="forged-" + uuid.uuid4().hex) + r.check(st == 401, "forged token still rejected without Redis") + finally: + sh(args.start_redis) + + print("== 2. PostgreSQL недоступен ==") + sh(args.stop_db) + try: + time.sleep(1) + st, _, _ = api.call("GET", path, token=user["token"]) + r.check(st == 503, f"history without DB -> 503, not a hang (got {st})") + st, _, _ = api.call("GET", "/api/v1/users/me", token="forged-" + uuid.uuid4().hex) + r.check(st in (401, 503), f"unknown token is never trusted without DB (got {st})") + st, _, _ = api.call("GET", "/api/v1/users/me") + r.check(st == 401, "anonymous request -> 401") + st = api.raw("GET", "/health/ready")[0] + r.check(st == 503, "/health/ready -> 503") + st = api.raw("GET", "/health/live")[0] + r.check(st == 200, "/health/live -> 200 (process is alive)") + finally: + sh(args.start_db) + + print("== 3. восстановление ==") + r.check(wait_ready(api), "/health/ready back to 200") + st, page, _ = api.call("GET", path, token=user["token"]) + texts = [m["text"] for m in (page or {}).get("items", [])] + r.check(st == 200 and "before outage" in texts and "without redis" in texts, + "same token reads the same history after DB is back") + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/race.py b/student-tests/race.py new file mode 100644 index 00000000..d65b8ab7 --- /dev/null +++ b/student-tests/race.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +""" +Конкурентная запись: 10 клиентов параллельно отправляют по 10 сообщений. +Ожидаем ровно 100 уникальных сообщений в истории. + python student-tests/race.py --base-url http://localhost:8080 +""" +import argparse +from concurrent.futures import ThreadPoolExecutor + +from common import Api + +CLIENTS, PER_CLIENT = 10, 10 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + api = Api(ap.parse_args().base_url) + user = api.new_user("race") + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "race", "member_ids": [user["id"]]}, token=user["token"]) + mpath = f"/api/v1/chats/{chat['id']}/messages" + + def worker(w): + return [api.call("POST", mpath, {"text": f"c{w}-m{i}"}, token=user["token"])[0] for i in range(PER_CLIENT)] + + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + codes = [c for batch in pool.map(worker, range(CLIENTS)) for c in batch] + + ids, texts, cursor = [], [], None + while True: + st, page, _ = api.call("GET", f"{mpath}?limit=50" + (f"&cursor={cursor}" if cursor else ""), token=user["token"]) + ids += [m["id"] for m in page["items"]] + texts += [m["text"] for m in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + expected = {f"c{w}-m{i}" for w in range(CLIENTS) for i in range(PER_CLIENT)} + ok = codes.count(201) == 100 and len(ids) == 100 and len(set(ids)) == 100 and set(texts) == expected + print(f"sent=100 accepted={codes.count(201)} received={len(ids)} unique_ids={len(set(ids))} " + f"all_texts_present={set(texts) == expected}") + print("PASS" if ok else "FAIL") + raise SystemExit(0 if ok else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/rollback_check.py b/student-tests/rollback_check.py new file mode 100644 index 00000000..0a8a0994 --- /dev/null +++ b/student-tests/rollback_check.py @@ -0,0 +1,52 @@ +""" +Rollback составной операции «создать чат + участников» (уровень «4»). + +Создаём чат, где последний участник — несуществующий пользователь. Проверку +приложения нарочно обходим и вызываем запись напрямую: первый INSERT в chats +проходит, а вставка участника падает на внешнем ключе. Ожидаем, что транзакция +откатилась целиком: ни чата, ни участников в базе нет. + +Запуск внутри контейнера API (у него есть доступ к БД и коду): + docker compose cp student-tests/rollback_check.py api:/tmp/rollback_check.py + docker compose exec api python /tmp/rollback_check.py +Локально без Docker: python student-tests/rollback_check.py +""" +import asyncio +import sys +import uuid +from pathlib import Path + +sys.path.insert(0, str(Path.cwd())) +import psycopg # noqa: E402 + +from app import repo # noqa: E402 +from app.config import settings # noqa: E402 + + +async def main() -> int: + title = "rollback-" + uuid.uuid4().hex[:8] + async with await psycopg.AsyncConnection.connect(settings.database_url) as conn: + creator = await repo.create_user(conn, "rb_" + uuid.uuid4().hex[:10], "rb", "x") + await conn.commit() + ghost = uuid.uuid4() # такого пользователя нет + try: + async with conn.transaction(): + await repo.insert_chat(conn, title, [uuid.UUID(creator["id"]), ghost], creator["id"]) + print("FAIL - insert unexpectedly succeeded") + return 1 + except psycopg.errors.ForeignKeyViolation: + print(" ok - insert of unknown member failed on foreign key") + cur = await conn.execute( + "SELECT count(*) FROM chats WHERE title = %s", (title,)) + chats = (await cur.fetchone())[0] + cur = await conn.execute( + "SELECT count(*) FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE c.title = %s", (title,)) + members = (await cur.fetchone())[0] + ok = chats == 0 and members == 0 + print(f" {'ok ' if ok else 'FAIL'} - after rollback: chats={chats}, members={members} (expected 0, 0)") + print("PASS" if ok else "FAIL") + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 00000000..34a92f9a --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +Поочерёдная замена реплик API (rolling update) под нагрузкой (lab3, уровень «4»). + +Compose не делает rolling update сам, поэтому замена выполняется по шагам: + 1. собрать новый образ (docker compose build api); + 2. для каждой старой реплики: + поднять ОДНУ новую (up --scale api=N+1 --no-recreate) → дождаться healthy + и того, что LB её обслуживает → остановить старую (SIGTERM, graceful) → удалить; + 3. всё это время нагрузка пишет сообщения через LB и повторяет неудачные + запросы с тем же Idempotency-Key. +Итог: ошибки/RPS во время замены, все реплики на новом образе, нет потерь и дублей. + +Миграции: перед заменой выполняется сервис migrate (однократно). Новые миграции +должны быть совместимы с ещё работающей старой версией (только добавление). + + python student-tests/rolling_update.py --base-url http://localhost:8080 --output evidence/rolling-update.json +""" +import argparse +import json +import shlex +import subprocess +import time + +from common import Api, Report +from failover_each import served_instances +from loadgen import LoadGen + +COMPOSE = "docker compose --env-file .env" + + +def sh(cmd: str) -> str: + print(f" $ {cmd}") + return subprocess.run(shlex.split(cmd), check=True, capture_output=True, text=True).stdout + + +def api_ids() -> list: + return [x.strip() for x in sh(f"{COMPOSE} ps -q api").splitlines() if x.strip()] + + +def image_of(cid: str) -> str: + return sh(f"docker inspect -f {{{{.Image}}}} {cid}").strip() + + +def wait_healthy(cid: str, seconds: float = 90) -> bool: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if sh(f"docker inspect -f {{{{.State.Health.Status}}}} {cid}").strip() == "healthy": + return True + time.sleep(1) + return False + + +def wait_served(api: Api, iid: str, seconds: float = 60) -> bool: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + if iid in served_instances(api, 10): + return True + time.sleep(0.5) + return False + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--workers", type=int, default=4) + ap.add_argument("--no-build", action="store_true", help="не пересобирать образ") + ap.add_argument("--output", default=None) + args = ap.parse_args() + api, r = Api(args.base_url), Report() + + if not args.no_build: + sh(f"{COMPOSE} build api") + sh(f"{COMPOSE} run --rm migrate") # однократные миграции до замены + new_image = sh("docker image inspect -f {{.Id}} mtusi-messenger:lab4").strip() + old = api_ids() + n = len(old) + r.check(n >= 2, f"{n} replicas before update") + old_images = {cid[:12]: image_of(cid) for cid in old} + + user = api.new_user("rolling") + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "rolling", "member_ids": [user["id"]]}, + token=user["token"]) + gen = LoadGen(api, user["token"], chat["id"], workers=args.workers) + gen.start() + time.sleep(3) + t_begin = time.monotonic() + steps = [] + for cid in old: + before = set(api_ids()) + sh(f"{COMPOSE} up -d --no-deps --no-recreate --scale api={len(before) + 1} api") + new = sorted(set(api_ids()) - before) + r.check(len(new) == 1, f"one new replica started for {cid[:12]}") + new_id = new[0] + healthy = wait_healthy(new_id) + served = wait_served(api, new_id[:12]) + t0 = time.monotonic() + sh(f"docker stop -t 15 {cid}") + sh(f"docker rm {cid}") + steps.append({"old": cid[:12], "new": new_id[:12], "new_healthy": healthy, "new_served_by_lb": served, + "old_stop_seconds": round(time.monotonic() - t0, 2)}) + r.check(healthy and served, f"new replica {new_id[:12]} healthy and in LB before old {cid[:12]} stops") + t_end = time.monotonic() + time.sleep(3) + gen.stop() + + final = api_ids() + rel_a, rel_b = t_begin - gen.t0, t_end - gen.t0 + result = { + "replicas_before": sorted(old_images), "replicas_after": sorted(c[:12] for c in final), + "all_on_new_image": all(image_of(c) == new_image for c in final), + "steps": steps, + "update_duration_s": round(t_end - t_begin, 1), + "before_update": gen.window(0, rel_a), + "during_update": gen.window(rel_a, rel_b), + "after_update": gen.window(rel_b, rel_b + 3), + "max_gap_without_success_s": gen.max_gap_after(t_begin), + "instances_serving_at_end": sorted(set(gen.instances_after(t_end))), + **gen.verify(), + } + print(json.dumps(result, ensure_ascii=False, indent=2)) + r.check(len(final) == n, f"replica count restored to {n}") + r.check(result["all_on_new_image"], "all replicas run the new image") + r.check(not set(result["instances_serving_at_end"]) & set(old_images), "old replicas no longer serve traffic") + r.check(result["lost_or_changed"] == 0 and result["duplicate_texts"] == 0, "no lost or duplicated writes") + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 00000000..30fb8f96 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +""" +Собственные тесты lab2 (запуск одной командой): + python student-tests/run.py --base-url http://localhost:8080 + +Негативные сценарии lab1 на новой аутентификации + проверки уровня «4»: +неверный пароль, logout, X-User-Id/Basic не дают доступ, гонка регистрации. +""" +import argparse +import base64 +import json +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, Api, Report + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + api = Api(ap.parse_args().base_url) + r = Report() + + alice, bob, eve = api.new_user("alice"), api.new_user("bob"), api.new_user("eve") + st, chat, _ = api.call("POST", "/api/v1/chats", + {"title": "own tests", "member_ids": [alice["id"], bob["id"]]}, token=alice["token"]) + assert st == 201, st + mpath = f"/api/v1/chats/{chat['id']}/messages" + + print("== аутентификация ==") + st, _, rh = api.login(alice["username"], "Wrong-password-123") + r.check(st == 401 and rh.get("WWW-Authenticate", "").startswith("Basic"), "wrong password -> 401 + Basic challenge") + st, _, _ = api.login("no_such_user_" + uuid.uuid4().hex[:8], PASSWORD) + r.check(st == 401, "unknown user -> 401 (same as wrong password)") + st, _, rh = api.call("GET", "/api/v1/users/me") + r.check(st == 401 and rh.get("WWW-Authenticate", "").startswith("Bearer"), "no token -> 401 + Bearer challenge") + st, _, _ = api.call("GET", "/api/v1/users/me", headers={"X-User-Id": alice["id"]}) + r.check(st == 401, "X-User-Id alone gives no access") + st, _, _ = api.call("GET", "/api/v1/users/me", token="forged-" + uuid.uuid4().hex) + r.check(st == 401, "unknown/forged token -> 401") + basic = base64.b64encode(f"{alice['username']}:{alice['password']}".encode()).decode() + st, _, _ = api.call("GET", "/api/v1/users/me", headers={"Authorization": "Basic " + basic}) + r.check(st == 401, "Basic on business endpoint -> 401") + st, me, _ = api.call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": eve["id"]}) + r.check(st == 200 and me["id"] == alice["id"], "X-User-Id cannot override bearer identity") + st, _, _ = api.call("POST", "/api/v1/users", {"username": "short_pw_" + uuid.uuid4().hex[:6], + "display_name": "x", "password": "short"}) + r.check(st == 400, "password shorter than 12 -> 400") + + print("== logout ==") + tmp = api.new_user("logout") + st, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=tmp["token"]) + r.check(st == 204, "logout -> 204") + st, _, _ = api.call("GET", "/api/v1/users/me", token=tmp["token"]) + r.check(st == 401, "token after logout -> 401") + st, _, _ = api.call("DELETE", "/api/v1/auth/sessions/current", token=tmp["token"]) + r.check(st == 401, "second logout -> 401") + + print("== гонка регистрации одного username ==") + name = "race_" + uuid.uuid4().hex[:10] + + def reg(_): + st, _, _ = api.call("POST", "/api/v1/users", {"username": name, "display_name": "r", "password": PASSWORD}) + return st + + with ThreadPoolExecutor(max_workers=20) as pool: + codes = list(pool.map(reg, range(20))) + r.check(codes.count(201) == 1 and codes.count(409) == 19, + f"20 parallel registrations -> one 201, nineteen 409 (got {sorted(codes)})") + + print("== ошибки и доступ (из lab1) ==") + st, _, _ = api.raw("POST", "/api/v1/users", b"{not json", {"Content-Type": "application/json"}) + r.check(st == 400, "malformed JSON -> 400") + st, _, _ = api.call("POST", "/api/v1/users", {"username": alice["username"], "display_name": "d", "password": PASSWORD}) + r.check(st == 409, "duplicate username -> 409") + for bad in ("review123\n", "review123\r\n", " review123", "Review123", "ab"): + st, _, _ = api.call("POST", "/api/v1/users", {"username": bad, "display_name": "d", "password": PASSWORD}) + r.check(st == 400, f"invalid username {bad!r} -> 400, not 500 (got {st})") + st, _, _ = api.call("POST", mpath, {"text": " "}, token=alice["token"]) + r.check(st == 400, "blank message -> 400") + st, _, _ = api.call("POST", mpath, {"text": "x", "sender_id": eve["id"]}, token=alice["token"]) + r.check(st == 400, "sender_id in body is rejected") + st, _, _ = api.call("POST", mpath, {"text": "чужое"}, token=eve["token"]) + r.check(st == 404, "outsider POST to foreign chat -> 404") + st, _, _ = api.call("GET", mpath, token=eve["token"]) + r.check(st == 404, "outsider GET foreign history -> 404") + st, _, _ = api.call("GET", "/api/v1/chats/not-a-uuid", token=alice["token"]) + r.check(st == 400, "invalid chat uuid -> 400") + + print("== пагинация ==") + for i in range(5): + api.call("POST", mpath, {"text": f"m{i}"}, token=bob["token"]) + seen, cursor, pages = [], None, 0 + while pages < 10: + st, page, _ = api.call("GET", f"{mpath}?limit=2" + (f"&cursor={cursor}" if cursor else ""), token=alice["token"]) + pages += 1 + seen += [m["id"] for m in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + r.check(len(seen) == 5 and len(set(seen)) == 5 and pages == 3, "5 messages / limit 2 -> 3 pages, no duplicates") + st, other, _ = api.call("POST", "/api/v1/chats", {"title": "other", "member_ids": [alice["id"]]}, token=alice["token"]) + st, p, _ = api.call("GET", f"{mpath}?limit=1", token=alice["token"]) + st, _, _ = api.call("GET", f"/api/v1/chats/{other['id']}/messages?cursor={p['next_cursor']}", token=alice["token"]) + r.check(st == 400, "cursor from another chat -> 400") + + print("== Content-Type ==") + body = json.dumps({"username": "ct_" + uuid.uuid4().hex[:8], "display_name": "c", "password": PASSWORD}).encode() + st, _, _ = api.raw("POST", "/api/v1/users", body, {"Content-Type": "text/plain"}) + r.check(st == 415, f"valid JSON with text/plain -> 415 (got {st})") + st, _, _ = api.raw("POST", mpath, json.dumps({"text": "x"}).encode(), + {"Content-Type": "text/plain", "Authorization": "Bearer " + alice["token"]}) + r.check(st == 415, f"message with text/plain -> 415 (got {st})") + st, _, _ = api.raw("POST", "/api/v1/chats", b"{}", {"Content-Type": "text/plain"}) + r.check(st == 401, f"text/plain without token -> 401 first (got {st})") + st, _, _ = api.raw("POST", mpath, json.dumps({"text": "utf8"}).encode(), + {"Content-Type": "application/json; charset=utf-8", "Authorization": "Bearer " + alice["token"], + "Idempotency-Key": uuid.uuid4().hex}) + r.check(st == 201, f"application/json; charset=utf-8 accepted (got {st})") + + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/s3_outage.py b/student-tests/s3_outage.py new file mode 100644 index 00000000..90d0f2ed --- /dev/null +++ b/student-tests/s3_outage.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 +""" +Недоступность S3 (lab4). Сам останавливает и запускает MinIO; из корня репозитория: + + python student-tests/s3_outage.py --base-url http://localhost:8080 + +Ожидание: пока MinIO остановлен, загрузка и скачивание дают быстрый 503 в формате +контракта (не зависание и не 500), а текстовая часть мессенджера, метаданные +вложений и /health/ready работают (S3 не критичен для обязательного текстового пути). +После старта MinIO всё снова работает, ранее загруженный файл не изменился. +Брошенная во время отказа запись журнала дочищается janitor после grace period. +""" +import argparse +import time + +from common import wait_ready, SAMPLE_PNG, Api, Report, compose, janitor, sha256, tool, upload + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + args = ap.parse_args() + api, r = Api(args.base_url), Report() + u = api.new_user("s3down") + T = u["token"] + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "s3 outage", "member_ids": [u["id"]]}, token=T) + st, before, _ = upload(api, T, chat["id"], SAMPLE_PNG, "before.png") + r.check(st == 202, "upload before outage") + r.check(wait_ready(api, T, before["id"], seconds=120).get("status") == "ready", + "it is processed to ready before the outage (lab5: 202 → worker)") + content = f"/api/v1/attachments/{before['id']}/content" + + print("== MinIO остановлен ==") + compose("stop", "s3") + try: + t0 = time.monotonic() + st, body, _ = upload(api, T, chat["id"], SAMPLE_PNG, "during.png", timeout=30) + took = time.monotonic() - t0 + r.check(st == 503 and body and body["error"]["code"] == "unavailable", f"upload -> 503 unavailable (got {st})") + r.check(took < 15, f"bounded by timeouts, not a hang ({took:.1f} s)") + st, _, _ = api.raw("GET", content, headers={"Authorization": "Bearer " + T}, timeout=30) + r.check(st == 503, f"download -> 503 (got {st})") + st, meta, _ = api.call("GET", f"/api/v1/attachments/{before['id']}", token=T) + r.check(st == 200 and meta["sha256"] == before["sha256"], "metadata still served from PostgreSQL") + st, _, _ = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "S3 лежит, а чат работает"}, token=T) + r.check(st == 201, "text messages still work") + st, _, _ = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", + {"text": "с вложением", "attachment_ids": [before["id"]]}, token=T) + r.check(st == 201, "message referencing an existing attachment still works (metadata only)") + r.check(api.raw("GET", "/health/ready")[0] == 200, "/health/ready = 200 (S3 is not critical for text path)") + finally: + compose("start", "s3") + + print("== MinIO снова работает ==") + # После старта MinIO каждая реплика может ещё до S3_BREAKER_SECONDS (5 c) отвечать 503: + # предохранитель закрывается по таймеру. Поэтому ждём, пока ОБЕ реплики восстановятся: + # несколько успешных загрузок подряд (LB чередует реплики). + t0 = time.monotonic() + ok = False + for _ in range(40): + st, raw, _ = api.raw("GET", content, headers={"Authorization": "Bearer " + T}) + if st == 200: + ok = sha256(raw) == before["sha256"] + break + time.sleep(0.5) + r.check(ok, "file uploaded before outage downloads unchanged") + streak, codes = 0, [] + while time.monotonic() - t0 < 40 and streak < 4: + st, _, rh = upload(api, T, chat["id"], SAMPLE_PNG, "after.png") + codes.append(st) + streak = streak + 1 if st == 202 else 0 + if st != 202: + time.sleep(0.5) + recovered = time.monotonic() - t0 + r.check(streak >= 4, f"uploads work again on all replicas: 4 x 202 in a row after {recovered:.1f} s " + f"(codes during recovery: {codes})") + stats = tool("stats") + print(" stats:", stats) + s = janitor("--grace-seconds", "0") + r.check(s["errors"] == 0, f"janitor cleaned journal rows left by failed uploads ({s})") + r.finish() + + +if __name__ == "__main__": + main() diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..c81b2c12 --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +""" +Эксперимент: пик памяти API при N одновременных загрузках (lab4, уровень «4»). + + python student-tests/upload_memory.py --base-url http://localhost:8080 \ + --concurrency 10 --output evidence/upload-memory-10.json + +Гипотеза: память реплики почти не зависит от размера и числа одновременных +загрузок, потому что тело не копится в RAM (≤ UPLOAD_SPOOL_MEMORY_BYTES в памяти, +остальное во временном файле), проверка изображения потоковая, в S3 уходит поток. +Наивная реализация (`await request.body()` + `Image.open().load()`) держала бы +≥ размер файла × 2…4 на каждую загрузку: 10 × 9,5 МБ → 200–400 МБ, что больше +лимита контейнера 256 МБ (OOM). + +Методика: генерируется настоящий PNG из шума ≈9,5 МБ (почти не сжимается), +N потоков одновременно отправляют его через LB (несколько раундов), параллельно +каждые ~0,3 с читается память cgroup v2 всех контейнеров api (usage как в docker stats, anon, memory.peak). Фиксируются память +до нагрузки, пик во время и после, время ответа, коды ответов и sha256. +""" +import argparse +import json +import re +import subprocess +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +from common import ROOT, Api, Report, compose, make_png, sha256, upload, wait_ready + +# Печатаем «имя значение»: cgroup v2, иначе v1 (старые ядра/Docker Desktop с cgroup v1). +CGROUP = ( + "if [ -f /sys/fs/cgroup/memory.current ]; then " + "echo current $(cat /sys/fs/cgroup/memory.current); " + "grep -E '^(anon|inactive_file) ' /sys/fs/cgroup/memory.stat; " + "[ -f /sys/fs/cgroup/memory.peak ] && echo peak $(cat /sys/fs/cgroup/memory.peak); " + "else M=/sys/fs/cgroup/memory; " + "echo current $(cat $M/memory.usage_in_bytes); " + "grep -E '^total_(rss|inactive_file) ' $M/memory.stat | sed -e 's/^total_rss/anon/' -e 's/^total_//'; " + "echo peak $(cat $M/memory.max_usage_in_bytes); fi") +UNITS = {"B": 1, "KiB": 1024, "MiB": 1024 ** 2, "GiB": 1024 ** 3, "kB": 1000, "MB": 1000 ** 2, "GB": 1000 ** 3} + + +def api_containers() -> list[str]: + return [c[:12] for c in compose("ps", "-q", "api").split()] + + +def _docker_stats(cid: str) -> dict: + """Запасной путь: docker stats (usage без файлового кэша).""" + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", cid], + capture_output=True, text=True, cwd=ROOT).stdout + m = re.match(r"\s*([\d.]+)\s*([A-Za-z]+)", out) + usage = round(float(m.group(1)) * UNITS[m.group(2)] / 1024 ** 2, 1) if m else 0.0 + return {"usage": usage, "anon": usage, "peak": None} + + +_warned: list = [] + + +def read_cgroup(cid: str) -> dict: + """Память контейнера (MiB): usage как в docker stats (current − inactive_file), + anon — память процессов без файлового кэша, peak — максимум за жизнь cgroup.""" + p = subprocess.run(["docker", "exec", cid, "sh", "-c", CGROUP], + capture_output=True, text=True, cwd=ROOT) + vals = {} + for line in p.stdout.splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].isdigit(): + vals[parts[0]] = int(parts[1]) + if "current" not in vals: + if not _warned: + _warned.append(1) + print(f" cgroup read failed (exit {p.returncode}: {p.stderr.strip()[:200]!r}); using docker stats") + return _docker_stats(cid) + mib = 1024 ** 2 + return {"usage": round((vals["current"] - vals.get("inactive_file", 0)) / mib, 1), + "anon": round(vals.get("anon", vals["current"]) / mib, 1), + "peak": round(vals["peak"] / mib, 1) if "peak" in vals else None} + + +def sample_mem(ids: list[str]) -> dict: + with ThreadPoolExecutor(max_workers=len(ids) or 1) as pool: + return dict(zip(ids, pool.map(read_cgroup, ids))) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--base-url", default="http://localhost:8080") + ap.add_argument("--concurrency", type=int, default=10) + ap.add_argument("--rounds", type=int, default=3) + ap.add_argument("--width", type=int, default=1780, help="PNG width/height: 1780 → ≈9,5 МБ") + ap.add_argument("--output", default=None) + args = ap.parse_args() + api, r = Api(args.base_url), Report() + + data = make_png(args.width, args.width, noise=True) + digest = sha256(data) + print(f"file: {len(data)} bytes ({len(data) / 1024 ** 2:.2f} MiB), sha256 {digest[:12]}…") + u = api.new_user("memexp") + st, chat, _ = api.call("POST", "/api/v1/chats", {"title": "memory", "member_ids": [u["id"]]}, token=u["token"]) + + ids = api_containers() + idle = sample_mem(ids) + print("idle MiB:", idle) + peak = {k: {"usage": v["usage"], "anon": v["anon"]} for k, v in idle.items()} + samples = [] + stop = threading.Event() + + def sampler(): + t0 = time.monotonic() + while not stop.is_set(): + m = sample_mem(ids) + samples.append({"t": round(time.monotonic() - t0, 2), **m}) + for k, v in m.items(): + for f in ("usage", "anon"): + peak[k][f] = max(peak[k][f], v[f]) + time.sleep(0.2) + + th = threading.Thread(target=sampler, daemon=True) + th.start() + results = [] + + def one(i): + t = time.monotonic() + try: + st, body, rh = upload(api, u["token"], chat["id"], data, f"noise-{i}.png", timeout=120) + except OSError as exc: + return {"status": None, "error": type(exc).__name__, "seconds": round(time.monotonic() - t, 2)} + return {"status": st, "seconds": round(time.monotonic() - t, 2), + "sha_ok": bool(body) and body.get("sha256") == digest, + "instance": rh.get("X-Instance-Id"), "id": (body or {}).get("id")} + + t_start = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + for rnd in range(args.rounds): + results += list(pool.map(one, range(rnd * args.concurrency, (rnd + 1) * args.concurrency))) + elapsed = time.monotonic() - t_start + time.sleep(2) + stop.set() + th.join() + after = sample_mem(ids) + + ok = [x for x in results if x["status"] == 202] + lat = sorted(x["seconds"] for x in ok) + summary = { + "file_bytes": len(data), "concurrency": args.concurrency, "rounds": args.rounds, + "uploads": len(results), "status_counts": {str(s): sum(1 for x in results if x["status"] == s) + for s in sorted({x["status"] for x in results}, key=str)}, + "sha256_ok": sum(1 for x in ok if x["sha_ok"]), + "instances": sorted({x["instance"] for x in ok if x["instance"]}), + "elapsed_s": round(elapsed, 2), + "throughput_MiB_s": round(len(ok) * len(data) / 1024 ** 2 / elapsed, 1) if elapsed else None, + "latency_s": {"p50": lat[len(lat) // 2] if lat else None, "max": lat[-1] if lat else None}, + "memory_MiB": {"idle": idle, "sampled_peak": peak, "after": after, + "max_growth_usage": round(max(peak[k]["usage"] - idle[k]["usage"] for k in peak), 1), + "max_growth_anon": round(max(peak[k]["anon"] - idle[k]["anon"] for k in peak), 1), + "cgroup_peak_after": {k: v["peak"] for k, v in after.items()}}, + "container_limit_MiB": 256, "samples": len(samples), + } + print(json.dumps({k: v for k, v in summary.items() if k != "samples"}, ensure_ascii=False, indent=2)) + + # Проверка, что сохранены настоящие байты (выборочно). С lab5 байты отдаются после ready: + # 202 означает «принято», обработку делает worker (под нагрузкой — с очередью). + for x in ok[:3]: + wait_ready(api, u["token"], x["id"], seconds=300) + st, raw, _ = api.raw("GET", f"/api/v1/attachments/{x['id']}/content", + headers={"Authorization": "Bearer " + u["token"]}, timeout=60) + r.check(st == 200 and sha256(raw) == digest, f"download {x['id'][:8]} sha256 matches") + r.check(len(ok) == len(results), f"all {len(results)} uploads -> 202") + r.check(summary["sha256_ok"] == len(ok), "sha256 of every upload matches") + # Критерий — память ПРОЦЕССОВ (anon): именно туда попало бы накопление тела в RAM. + # usage дополнительно включает файловый кэш ядра для временных spool-файлов (вытесняемый), + # он печатается выше для полноты, но буферизацию не показывает. + growth = summary["memory_MiB"]["max_growth_anon"] + r.check(growth < 100, + f"peak process memory (anon) growth per replica {growth} MiB < 100 MiB " + f"(naive buffering: ≥ {args.concurrency // 2} × {len(data) / 1024 ** 2:.1f} MiB × 2)") + if args.output: + with open(args.output, "w", encoding="utf-8") as f: + json.dump({**summary, "raw_samples": samples}, f, ensure_ascii=False, indent=2) + print("saved", args.output) + r.finish() + + +if __name__ == "__main__": + main() -- 2.54.0 From ed4e626585dbf08855f6855a40ab6947df09413a Mon Sep 17 00:00:00 2001 From: LOSYASH Date: Thu, 12 Nov 2026 15:21:01 +0100 Subject: [PATCH 2/2] =?UTF-8?q?=D0=9B=D0=A05:=20=D0=BE=D1=82=D1=87=D1=91?= =?UTF-8?q?=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 76d1ce4d..954979bc 100644 --- a/REPORT.md +++ b/REPORT.md @@ -3,7 +3,7 @@ - Автор, группа: Санжаровский, БВТ2304 - Целевая оценка: **4** (трек 5 не заявлен) - Commit/PR: ветка `solution/bvt2304-sanzharovskiy` форка `Anatoly01/mtusi2026lab5`, PR в `mtusi2026/mtusi2026lab5` -- Источники и библиотеки: всё из lab4 (FastAPI, uvicorn, psycopg 3 + psycopg-pool, argon2-cffi, redis-py 8.1, boto3 1.38, Pillow 11.2, HAProxy 3.0, MinIO от Pigsty). Брокер — **Redis Streams** (тот же образ `redis:7.4.1-alpine`, отдельный экземпляр с AOF). Документация: Redis Streams (consumer groups, PEL, `XAUTOCLAIM`), Redis persistence (AOF, `appendfsync always`), RabbitMQ reliability/confirms (для сравнения границ confirm/ack), Pillow (`draft`, `ImageOps.exif_transpose`, WebP), Python `resource` (RLIMIT_AS/RLIMIT_CPU), `multiprocessing` forkserver; Transactional outbox (microservices.io). Помощь ИИ: Claude — код, тесты, отчёт; решения разобраны и проверены прогоном. +- Источники и библиотеки: всё из lab4 (FastAPI, uvicorn, psycopg 3 + psycopg-pool, argon2-cffi, redis-py 8.1, boto3 1.38, Pillow 11.2, HAProxy 3.0, MinIO от Pigsty). Брокер — **Redis Streams** (тот же образ `redis:7.4.1-alpine`, отдельный экземпляр с AOF). Документация: Redis Streams (consumer groups, PEL, `XAUTOCLAIM`), Redis persistence (AOF, `appendfsync always`), RabbitMQ reliability/confirms (для сравнения границ confirm/ack), Pillow (`draft`, `ImageOps.exif_transpose`, WebP), Python `resource` (RLIMIT_AS/RLIMIT_CPU), `multiprocessing` forkserver; Transactional outbox (microservices.io). Использование ИИ: ИИ-ассистент помогал с кодом, тестами и текстом отчёта; решения разобраны и проверены прогоном. ## Запуск из чистого clone -- 2.54.0