From fd6f85d94cc5bc7d31ef5a4ab940c60c95a30fb9 Mon Sep 17 00:00:00 2001 From: kalibr456 <145665723+kalibr456@users.noreply.github.com> Date: Sat, 24 Oct 2026 15:53:17 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=B0=D0=B1=D0=BE=D1=80=D0=B0=D1=82?= =?UTF-8?q?=D0=BE=D1=80=D0=BD=D0=B0=D1=8F=205:=20=D0=B1=D1=80=D0=BE=D0=BA?= =?UTF-8?q?=D0=B5=D1=80,=20worker,=20outbox,=20=D0=B2=D0=B0=D1=80=D0=B8?= =?UTF-8?q?=D0=B0=D0=BD=D1=82=D1=8B=20=D0=B8=D0=B7=D0=BE=D0=B1=D1=80=D0=B0?= =?UTF-8?q?=D0=B6=D0=B5=D0=BD=D0=B8=D0=B9,=20=D1=82=D0=B5=D1=81=D1=82?= =?UTF-8?q?=D1=8B=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D0=BE=D0=B2=20(=D0=BE?= =?UTF-8?q?=D1=86=D0=B5=D0=BD=D0=BA=D0=B0=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 --- .env.example | 44 +- .gitattributes | 6 + .github/pull_request_template.md | 16 +- Dockerfile | 31 + REPORT.md | 283 ++++++++- attachments.py | 80 +++ auth.py | 139 +++++ broker.py | 105 ++++ compose.yaml | 320 ++++++++++ db.py | 391 ++++++++++++ dispatcher.py | 150 +++++ errors.py | 40 ++ evidence/attachments.txt | 96 +++ evidence/balancing.txt | 18 + evidence/check.txt | 6 + evidence/consistency.txt | 47 ++ evidence/db.txt | 21 + evidence/explain-history.txt | 99 +++ evidence/explain.txt | 43 ++ evidence/failover.txt | 33 + evidence/failures.txt | 29 + evidence/idempotency.txt | 22 + evidence/jobs-concurrency.txt | 8 + evidence/jobs-crash.txt | 15 + evidence/jobs-limits.txt | 11 + evidence/jobs-outbox.txt | 9 + evidence/jobs-recovery.txt | 16 + evidence/jobs-retry.txt | 15 + evidence/jobs-variants.txt | 14 + evidence/load.json | 17 + evidence/load.txt | 22 + evidence/native-sandbox/attachments.txt | 96 +++ evidence/native-sandbox/check.txt | 6 + evidence/native-sandbox/jobs-concurrency.txt | 8 + evidence/native-sandbox/jobs-crash.txt | 15 + evidence/native-sandbox/jobs-limits.txt | 11 + evidence/native-sandbox/jobs-outbox.txt | 9 + evidence/native-sandbox/jobs-recovery.txt | 16 + evidence/native-sandbox/jobs-retry.txt | 15 + evidence/native-sandbox/jobs-variants.txt | 14 + evidence/native-sandbox/persistence.txt | 16 + evidence/native-sandbox/race.txt | 10 + evidence/native-sandbox/smoke.txt | 6 + evidence/native-sandbox/student-tests.txt | 26 + evidence/native-sandbox/summary.md | 40 ++ evidence/persistence.txt | 30 + evidence/race.txt | 10 + evidence/redis.txt | 14 + evidence/s3-security.txt | 47 ++ evidence/smoke.txt | 6 + evidence/stack.txt | 49 ++ evidence/student-tests.txt | 26 + evidence/summary.md | 78 +++ evidence/upload-memory.json | 124 ++++ evidence/upload-memory.txt | 13 + faults.py | 51 ++ imaging.py | 209 +++++++ infra/haproxy.cfg | 77 +++ infra/rabbitmq.conf | 22 + infra/redis.conf | 13 + janitor.py | 278 +++++++++ main.py | 603 +++++++++++++++++++ migrate.py | 8 + migrations/001_init.sql | 33 + migrations/002_text_as_bytes.sql | 11 + migrations/003_idempotency.sql | 11 + migrations/004_attachments.sql | 48 ++ migrations/005_jobs.sql | 133 ++++ ops.py | 194 ++++++ pipeline.py | 445 ++++++++++++++ requirements.txt | 30 + s3_init.py | 103 ++++ scripts/init_secrets.py | 153 +++++ scripts/restart.cmd | 14 + scripts/restart.sh | 13 + scripts/stop-one.cmd | 6 + scripts/stop-one.sh | 5 + scripts/stop_one.py | 29 + scripts/worker.cmd | 8 + scripts/worker.sh | 11 + settings.py | 111 ++++ storage.py | 183 ++++++ student-tests/check_stack.py | 91 +++ student-tests/collect_evidence.py | 359 +++++++++++ student-tests/common.py | 347 +++++++++++ student-tests/demo_instances.py | 16 + student-tests/explain.py | 28 + student-tests/explain.sql | 76 +++ student-tests/failover_metrics.py | 129 ++++ student-tests/fixtures/progressive.jpg | Bin 0 -> 1081 bytes student-tests/fixtures/sample.jpg | Bin 0 -> 1492 bytes student-tests/images.py | 93 +++ student-tests/loadgen.py | 102 ++++ student-tests/redis_info.py | 29 + student-tests/rolling_update.py | 88 +++ student-tests/run_failover.py | 55 ++ student-tests/run_persistence.py | 47 ++ student-tests/test_api.py | 258 ++++++++ student-tests/test_attachments.py | 338 +++++++++++ student-tests/test_balancing.py | 103 ++++ student-tests/test_consistency.py | 276 +++++++++ student-tests/test_db.py | 101 ++++ student-tests/test_expiry.py | 57 ++ student-tests/test_failures.py | 110 ++++ student-tests/test_idempotency.py | 174 ++++++ student-tests/test_jobs.py | 567 +++++++++++++++++ student-tests/test_race.py | 57 ++ student-tests/test_s3_security.py | 146 +++++ student-tests/upload_memory.py | 222 +++++++ student-tests/worker_probe.py | 148 +++++ uploads.py | 519 ++++++++++++++++ worker.py | 390 ++++++++++++ 112 files changed, 10533 insertions(+), 26 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 attachments.py create mode 100644 auth.py create mode 100644 broker.py create mode 100644 compose.yaml create mode 100644 db.py create mode 100644 dispatcher.py create mode 100644 errors.py create mode 100644 evidence/attachments.txt create mode 100644 evidence/balancing.txt create mode 100644 evidence/check.txt create mode 100644 evidence/consistency.txt create mode 100644 evidence/db.txt create mode 100644 evidence/explain-history.txt create mode 100644 evidence/explain.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/failures.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/jobs-concurrency.txt create mode 100644 evidence/jobs-crash.txt create mode 100644 evidence/jobs-limits.txt create mode 100644 evidence/jobs-outbox.txt create mode 100644 evidence/jobs-recovery.txt create mode 100644 evidence/jobs-retry.txt create mode 100644 evidence/jobs-variants.txt create mode 100644 evidence/load.json create mode 100644 evidence/load.txt create mode 100644 evidence/native-sandbox/attachments.txt create mode 100644 evidence/native-sandbox/check.txt create mode 100644 evidence/native-sandbox/jobs-concurrency.txt create mode 100644 evidence/native-sandbox/jobs-crash.txt create mode 100644 evidence/native-sandbox/jobs-limits.txt create mode 100644 evidence/native-sandbox/jobs-outbox.txt create mode 100644 evidence/native-sandbox/jobs-recovery.txt create mode 100644 evidence/native-sandbox/jobs-retry.txt create mode 100644 evidence/native-sandbox/jobs-variants.txt create mode 100644 evidence/native-sandbox/persistence.txt create mode 100644 evidence/native-sandbox/race.txt create mode 100644 evidence/native-sandbox/smoke.txt create mode 100644 evidence/native-sandbox/student-tests.txt create mode 100644 evidence/native-sandbox/summary.md create mode 100644 evidence/persistence.txt create mode 100644 evidence/race.txt create mode 100644 evidence/redis.txt create mode 100644 evidence/s3-security.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stack.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/summary.md create mode 100644 evidence/upload-memory.json create mode 100644 evidence/upload-memory.txt create mode 100644 faults.py create mode 100644 imaging.py create mode 100644 infra/haproxy.cfg create mode 100644 infra/rabbitmq.conf create mode 100644 infra/redis.conf create mode 100644 janitor.py create mode 100644 main.py create mode 100644 migrate.py create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_text_as_bytes.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_jobs.sql create mode 100644 ops.py create mode 100644 pipeline.py create mode 100644 requirements.txt create mode 100644 s3_init.py create mode 100644 scripts/init_secrets.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 scripts/worker.cmd create mode 100644 scripts/worker.sh create mode 100644 settings.py create mode 100644 storage.py create mode 100644 student-tests/check_stack.py create mode 100644 student-tests/collect_evidence.py create mode 100644 student-tests/common.py create mode 100644 student-tests/demo_instances.py create mode 100644 student-tests/explain.py create mode 100644 student-tests/explain.sql create mode 100644 student-tests/failover_metrics.py create mode 100644 student-tests/fixtures/progressive.jpg create mode 100644 student-tests/fixtures/sample.jpg create mode 100644 student-tests/images.py create mode 100644 student-tests/loadgen.py create mode 100644 student-tests/redis_info.py create mode 100644 student-tests/rolling_update.py create mode 100644 student-tests/run_failover.py create mode 100644 student-tests/run_persistence.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_balancing.py create mode 100644 student-tests/test_consistency.py create mode 100644 student-tests/test_db.py create mode 100644 student-tests/test_expiry.py create mode 100644 student-tests/test_failures.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_jobs.py create mode 100644 student-tests/test_race.py create mode 100644 student-tests/test_s3_security.py create mode 100644 student-tests/upload_memory.py create mode 100644 student-tests/worker_probe.py create mode 100644 uploads.py create mode 100644 worker.py diff --git a/.env.example b/.env.example index 119384b..818a90a 100644 --- a/.env.example +++ b/.env.example @@ -6,20 +6,56 @@ APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Optional on 3, required use on 4 in lab2; Redis OR Valkey (this solution: Redis since lab2): +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# SeaweedFS S3 (since lab4) +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url + +# lab5: broker and worker. The API has NO broker credentials (it writes the transactional outbox); +# compose.yaml passes per-service files: dispatcher -> /run/secrets/broker_url_dispatcher, +# worker -> /run/secrets/broker_url_worker. This variable is therefore empty for api/janitor. +BROKER_URL_FILE= WORKER_CONCURRENCY=2 +WORKER_PREFETCH=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=15 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=60 +WORKER_MEMORY_LIMIT_MB=512 STAGING_RETENTION_SECONDS=86400 +PIPELINE_VERSION=1 +# Fault injection (worker/dispatcher only, triggered by files in the faults volume): compose.yaml enables it on the +# training stand by default; set FAULT_INJECTION=0 here to disable. + +# Pools and timeouts (same as lab3) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_WAIT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_POOL_MAX=20 +REDIS_TIMEOUT_MS=500 +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=10 + +# lab4: S3 client and uploads +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=5 +S3_MAX_ATTEMPTS=2 +S3_POOL_MAX=20 +UPLOAD_TIMEOUT_SECONDS=60 +UPLOAD_IDLE_TIMEOUT_SECONDS=10 +DECODE_CONCURRENCY=2 +# Cleanup of abandoned uploads and staging (janitor service): grace period and pass interval +ATTACHMENT_GRACE_SECONDS=3600 +JANITOR_INTERVAL_SECONDS=300 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3fd8009 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.example text eol=lf +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a..3fb6285 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,14 +1,14 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Бабиченко Максим, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: из lab4 — весь API, сессии в Redis, балансировка, приём/проверка файлов, приватный S3, janitor. Новое: RabbitMQ (quorum-очередь + DLQ, definitions с минимальными правами), transactional outbox + dispatcher (publisher confirms), отдельный worker (2 потока, prefetch 1, ручной ack, аренда с fencing, backoff с jitter, постоянные/временные ошибки), варианты thumbnail/crop/optimized в дочернем процессе с RLIMIT, replay и разбор DLQ (`ops.py`), очистка staging. Upload: 201 → 202 queued. +Команды запуска из чистого clone: `Copy-Item .env.example .env; python scripts/init_secrets.py; docker compose up -d --build --wait; python student-tests/collect_evidence.py` (подробно — REPORT.md, раздел «Запуск»). +Результаты проверок и ссылка на REPORT.md: REPORT.md, раздел «Доказательства критериев»; `evidence/summary.md`. +Ограничения / известные ошибки: один узел RabbitMQ; worker использует ограниченный S3-ключ API; точки внедрения сбоев включены на учебном стенде (`FAULT_INJECTION=1`). См. REPORT.md, «Ограничения». +Источники и использование ИИ: REPORT.md, шапка. - [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [ ] Контракт и публичные тесты не ослаблены. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dd51df0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt + +COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py \ + pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py ./ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser \ + && mkdir -p /faults && chown appuser:appuser /faults +# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher); владелец appuser +# копируется в пустой named volume при первом монтировании. + +USER appuser + +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)" + +CMD ["python", "main.py"] diff --git a/REPORT.md b/REPORT.md index e98f75e..b91db7f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,292 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бабиченко Максим, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран +- Commit/PR: <<ссылка на MR в Gitea после публикации>> +- Использованные источники, библиотеки и помощь ИИ: всё из lab4 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py, boto3/botocore 1.43.116, Pillow 12.3.0; образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine, haproxy:3.2.25-alpine, chrislusf/seaweedfs:3.80). Новое: брокер RabbitMQ (образ `rabbitmq:4.1.4-alpine`), клиент pika 1.3.2. Документация: RabbitMQ — Reliability Guide, Consumer Acknowledgements and Publisher Confirms, Quorum Queues (`x-delivery-limit`, at-least-once dead-lettering, `x-overflow=reject-publish`), Definitions import, Access Control (configure/write/read); PostgreSQL `SELECT ... FOR UPDATE SKIP LOCKED`, `LISTEN/NOTIFY`; паттерн transactional outbox; W3C Trace Context (`traceparent`); Pillow (`Image.draft`, `ImageOps.exif_transpose`, параметры JPEG/WebP); `setrlimit(2)`. При написании кода и тестов использовался ИИ-ассистент (Claude); весь код проверен и прогнан мной, объяснить и изменить его могу на защите. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверял на Windows, все команды в PowerShell. Python нужен только для тестов (стандартная библиотека; Pillow на хосте не нужен — изображения для тестов генерирует и проверяет пробник внутри контейнера worker). Перед запуском остановить стенд lab4 (`docker compose down` в его папке), иначе занят порт 8080. Имя проекта Compose — `messenger-lab5`, поэтому volumes отдельные от lab4. + +```powershell +git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab5.git +cd mtusi2026lab5 +git checkout solution/bvt2303-babichenko +$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают +Copy-Item .env.example .env +python scripts/init_secrets.py # пароли PostgreSQL/Redis, ключи S3, пользователи RabbitMQ и + # secrets/rabbitmq_definitions.json (папка secrets/ в .gitignore) +docker compose --env-file .env config --quiet +docker compose up -d --build --wait # postgres, s3, rabbitmq -> migrate, s3-init -> api x2, janitor, + # dispatcher, worker -> lb +docker compose ps -a # migrate и s3-init: Exited (0), остальные healthy + +python student-tests/collect_evidence.py # все проверки lab1-5 по очереди (~15 мин), вывод в evidence/, + # таблицы замеров — в этот отчёт +``` + +Отдельные проверки (make на Windows нет, ниже то же самое без make): + +| Команда из README | То же в PowerShell | +| --- | --- | +| `make check` | `python scripts/check_contract.py` | +| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` | +| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` | +| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` | +| все сценарии lab5 | `python student-tests/test_jobs.py` | +| один сценарий lab5 | `python student-tests/test_jobs.py worker-down` (список — в начале файла) | +| состояние pipeline | `docker compose exec dispatcher python ops.py status` | +| реестр failed / журнал job | `docker compose exec dispatcher python ops.py failed`, `... ops.py events ` | +| replay failed / разбор DLQ | `docker compose exec dispatcher python ops.py replay `, `... ops.py dlq`, `... ops.py dlq-replay` | +| остановить / запустить worker | `scripts\worker.cmd stop`, `scripts\worker.cmd start` (Linux/macOS: `scripts/worker.sh`) | + +После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть: `docker compose up -d --wait`. + +Bootstrap: миграции — одноразовый сервис `migrate` (теперь и `005_jobs.sql`), bucket — `s3-init`, топология RabbitMQ (exchange, очереди, DLQ, пользователи и права) — декларативный файл `secrets/rabbitmq_definitions.json`, который брокер импортирует при каждом старте (`infra/rabbitmq.conf`). Всё идемпотентно: повторный `docker compose run --rm migrate` / `s3-init` и рестарт `rabbitmq` ничего не ломают. Сервисы не имеют права `configure` в брокере и только проверяют топологию пассивным declare — создать или удалить очередь они не могут. + +Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults` остаются). Безопасная остановка worker: `docker compose stop worker` — SIGTERM, потоки доделывают текущее задание (не дольше `JOB_TIMEOUT_SECONDS`, `stop_grace_period: 70s`), неподтверждённое сообщение брокер доставит снова. Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py` (новые пароли не подходят к старым volumes, поэтому удаляются вместе). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент --HTTP--> lb (HAProxy) --> api x2 --+--> postgres: метаданные, attachment_jobs, outbox, job_events, + | варианты (ИСТОЧНИК ИСТИНЫ, volume pgdata) + +--> s3 staging/: полный проверенный оригинал до 202 + +--> redis: сессии + COMMIT(queued + job + outbox) --NOTIFY--> dispatcher --publish+confirm--> rabbitmq + (outbox, (quorum queue attachments.process.v1, + аренды, DLQ attachments.process.v1.dlq, + сверка) volume rabbitmqdata) + | + worker x2 потока <-+ prefetch 1, ручной ack + | claim (аренда + fencing-токен) в postgres + | staging -> imaging.py (дочерний процесс, лимиты) + | PUT attachments//v1/{original,thumbnail.jpg,crop.jpg,optimized.webp} + | COMMIT ready (+варианты) -> ack -> удалить staging +janitor: брошенные uploading, staging ready/failed по retention, сироты в bucket, старый outbox +``` + +| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` | нет | edge; `127.0.0.1:8080`, `127.0.0.1:8404` | не нужны | +| API ×2 | `api` | python:3.11.13-slim | `/health/ready` (PostgreSQL + Redis; брокер и S3 не входят) | нет | edge + data | `database_url`, `cache_url`, ключ S3 API. **Доступа к брокеру нет** | +| Брокер | `rabbitmq` | rabbitmq:4.1.4-alpine, `hostname: rabbitmq` | узел запущен и очередь `attachments.process.v1` загружена | `rabbitmqdata` | только data, 5672 не публикуется | `rabbitmq_definitions` (хеши паролей, права) | +| Dispatcher | `dispatcher` (образ приложения) | python:3.11.13-slim | файл-пульс `/tmp/alive` не старше 30 с | `faults` | data | `database_url`, `broker_url_dispatcher` (write только в exchange `attachments`, read только DLQ) | +| Worker | `worker` (образ приложения), 2 потока | python:3.11.13-slim | файл-пульс | `faults`; `/tmp` — tmpfs 256 MiB | data | `database_url`, `broker_url_worker` (read только основной очереди, write нет), ключ S3 API | +| Janitor | `janitor` (образ приложения) | python:3.11.13-slim | файл-пульс `/tmp/alive` (обновляется перед проходом и каждые 10 с между проходами) | нет | data | как в lab4: `database_url`, ключ S3 API | +| S3, s3-init, migrate, postgres, redis | как в lab4 | | | | | | + +Почему RabbitMQ: durable quorum-очередь (Raft-журнал на диске, fsync до подтверждения издателю), publisher confirms, ручной ack, `x-delivery-limit` и dead-lettering «at-least-once» — ровно те границы, о которых спрашивают на защите. Почему dispatcher, а не publish из API: API фиксирует вложение, задание и строку outbox **одной транзакцией** и сразу отвечает 202; брокер в этом пути не участвует, поэтому его недоступность не мешает приёму, а API не нужны его пароли. + +Что перенесено из lab4 без изменений по смыслу: пользователи, сессии в Redis, чаты, сообщения, идемпотентность, балансировка, приём и потоковая проверка файла (лимиты bytes/пикселей до распаковки, MIME по содержимому), приватный bucket и ограниченный ключ, janitor. + +Что изменено в lab5: +- Новые файлы: `pipeline.py` (формат сообщения, outbox, переходы состояний, replay), `broker.py` (RabbitMQ, publisher confirms), `dispatcher.py`, `worker.py`, `imaging.py` (варианты, отдельный процесс), `faults.py` (учебное внедрение сбоев), `ops.py` (операторские команды), `migrations/005_jobs.sql`, `infra/rabbitmq.conf`. +- `attachments.py` / `db.py`: исходник пишется в `staging/`, затем одной транзакцией `state=queued` + `attachment_jobs` + `outbox` + `NOTIFY`; ответ `202` (переход 201 → 202 из COURSE.md). GET отдаёт `queued/processing/ready/failed`, варианты текущей версии pipeline. Сообщение можно создать с queued/processing-вложением, с failed — 409. +- `main.py`: `variant=original|thumbnail|crop|optimized`; для не-ready — 409, отсутствующий вариант ready — 404. +- `janitor.py`: очистка staging после безопасной фиксации и по retention для failed; сироты под `attachments//...`; очистка опубликованного outbox. +- `compose.yaml`: сервисы `rabbitmq`, `dispatcher`, `worker`, volumes `rabbitmqdata`, `faults`, три новых секрета; `scripts/init_secrets.py` генерирует пользователей RabbitMQ и definitions; restart-hook перезапускает ещё `rabbitmq`, `dispatcher`, `worker`. +- `.env.example`: в skeleton стояли `SESSION_STORE=postgres`, `S3_ENDPOINT=http://s3:9000` и `BROKER_URL_FILE=/run/secrets/broker_url`, с которыми API этого решения не стартует (сессии в Redis с lab2, S3 — SeaweedFS на 8333, у API нет брокера). Значения приведены к реальной конфигурации, добавлены переменные lab5. +- Тесты lab4: хелпер `upload()` ждёт `202`, затем опрашивает GET до `ready` — проверки «файл скачивается побайтно» сохранены; `test_consistency` имитирует падение в `enqueue_upload` вместо `finish_upload`. + +### Задание, сообщение и состояния + +Сообщение брокера — ровно `contracts/jobs.schema.json`: `schema_version`, `job_id`, `type=attachment.process.v1`, `attachment_id`, `pipeline_version=1`, `attempt`, `created_at`, `traceparent`. Ни байтов, ни URL, ни ключей S3, ни имени файла: worker берёт `source_key`, размер и SHA-256 из PostgreSQL. `traceparent` дублируется в native header AMQP; `message_id = :`; `delivery_mode=2` (persistent). Worker строго проверяет сообщение по схеме (лишние поля, типы, форматы) — не прошедшее уходит в DLQ. + +Таблицы (`005_jobs.sql`): `attachment_jobs` (одна строка на `(attachment_id, pipeline_version)`, `job_id` не меняется при повторах; `attempt`, `max_attempts`, `available_at`, `lease_token`/`lease_owner`/`lease_until`, `error_code`, `last_error`, `replays`), `attachment_variants` с **первичным ключом `(attachment_id, pipeline_version, name)`** — это и есть ключ эффекта из контракта, второй набор вариантов физически не вставится, `outbox`, `job_events` (журнал: created, claimed, retry_scheduled, lease_expired, duplicate_ignored, stale_result_rejected, ready, failed, replayed, staging_expired). CHECK-ограничения: `ready ⇔ object_key`, `failed ⇔ error_code`, `processing ⇔ lease`, до ready исходник обязан быть в staging. + +| Переход | Где | Что устойчиво сохранено | +| --- | --- | --- | +| нет → queued | API: PUT в staging, затем одна транзакция | исходник в `staging/` (SHA-256 сверяет S3 при PUT), строка вложения, job, outbox | +| queued → processing | worker `claim` под `FOR UPDATE` | attempt, новый `lease_token`, `lease_until = now()+15 с`, событие claimed | +| processing → queued | временная ошибка / истёкшая аренда | причина (код), `available_at = now()+backoff`, новая строка outbox | +| processing → ready | worker после PUT всех объектов | варианты + оригинал + ready одним COMMIT, **только если `lease_token` ещё наш** | +| processing → failed | постоянная ошибка или попытки исчерпаны | `error_code`, запись в журнале, исходник хранится 24 ч для replay | +| failed → queued | `ops.py replay` | событие replayed (audit), новая попытка того же job_id через outbox | + +Повторная доставка ready/failed — no-op с ack (событие `duplicate_ignored`). Устаревшее сообщение прежней попытки (job ещё ждёт backoff) — ack без работы: своё сообщение придёт из outbox. Ready никогда не откатывается: `finish_ready` обновляет только `state='processing'` и не перезаписывает более новую `pipeline_version`. + +### Где граница «202» + +К моменту ответа 202 сохранено: полный проверенный оригинал в S3 (staging — приватный prefix того же bucket, persistent volume `s3data`; это честная граница: оригинал уже в S3, worker «публикует» его копией в конечный prefix `attachments//v1/original`), строка вложения `queued`, задание и сообщение в outbox — всё в одной COMMIT PostgreSQL (`synchronous_commit=on`). Брокер на этот момент может ещё ничего не знать — это нормально: строку outbox опубликует dispatcher. Перезапуск/падение API после ответа не теряет работу. + +### Transactional outbox и окна отказа + +Dispatcher просыпается по `NOTIFY outbox` (сразу после COMMIT в API) или раз в 1 с, берёт строки `published_at IS NULL AND available_at <= now()` под `FOR UPDATE SKIP LOCKED`, публикует в канал с publisher confirms (`mandatory=True`: непривязанное сообщение — ошибка, nack при переполнении очереди — ошибка) и только после confirm ставит `published_at`. + +| Окно | Что происходит | Проверка | +| --- | --- | --- | +| после staging, до записи задания | клиент получил 503/обрыв, 202 не было; строка `uploading` невидима, janitor удалит объект и строку после grace | `test_consistency.py crash` (lab4, теперь с `enqueue_upload`) | +| после COMMIT, до publish | строка outbox не отмечена, после рестарта dispatcher публикует её | `test_jobs.py outbox-crash` (`dispatcher_crash_before_publish`) | +| после publish, до confirm / после confirm, до отметки | сообщение опубликовано, а отметки нет → публикуется ещё раз → дубль, worker видит ready и делает ack | `test_jobs.py outbox-crash` (`dispatcher_crash_after_confirm`) | +| после записи S3, до COMMIT результата | ack не отправлен, брокер доставляет снова; живая аренда → no-op; аренда истекает → новая попытка пишет **те же ключи** | `test_jobs.py crash-after-put` | +| после COMMIT результата, до ack | повторная доставка видит ready → ack, второго набора нет; staging доочищает janitor | `test_jobs.py crash-after-commit`, `staging` | + +Дополнительная страховка: раз в 30 с dispatcher ищет `queued`-задания старше `JOB_RECONCILE_SECONDS=120` без сообщения в полёте (например, сообщение ушло в DLQ после `x-delivery-limit`, пока PostgreSQL был недоступен) и публикует заново. Дубль безопасен, потеря — нет. + +**Publisher confirm и consumer ack — разные границы:** confirm значит «брокер принял сообщение и записал в quorum-журнал» (граница dispatcher → broker, после неё можно отметить outbox); ack значит «worker устойчиво выполнил эффект — объекты в S3 и COMMIT в PostgreSQL» (граница broker → worker, после неё брокер удаляет сообщение). At-least-once + идемпотентный эффект — это не exactly-once доставка: дубли сообщений происходят (и видны в журнале как `duplicate_ignored`), но не дают второго эффекта. + +### Повторы, аренда, DLQ, replay + +- Попытки: `JOB_MAX_ATTEMPTS=5`. Backoff «equal jitter»: `потолок = min(60, 2·2^(n-1))`, задержка = половина потолка + случайная половина: 1–2 с, 2–4 с, 4–8 с, 8–16 с. Задержка реализована через `available_at` строки outbox, а не через sleep в worker: поток не занят ожиданием. +- Постоянные ошибки (сразу failed, без повторов): `invalid_image`, `image_too_large`, `resource_limit` (не хватило памяти под RLIMIT_AS), `source_missing`, `source_corrupted` (staging не совпал по SHA-256 с принятым). Временные (повтор): `storage_unavailable`, `processing_timeout`, `processing_failed`/`processing_killed`, ошибки PostgreSQL. После последней попытки — `failed/retry_exhausted`. Наружу в `error_code` уходит только финальный код, без деталей. +- Аренда: `JOB_LEASE_SECONDS=15`, отдельный поток продлевает её каждые 5 с, пока идёт работа. **Долго работающий worker** продлевает аренду (проверено: зависание imaging 60 с — ни одного `lease_expired`), его ограничивает жёсткий `JOB_TIMEOUT_SECONDS=60`. **Зависший/умерший** перестаёт продлевать → через 15 с dispatcher возвращает job в queued (попытка засчитана) или worker сам перехватывает её при повторной доставке. Если «воскресший» исполнитель всё же дойдёт до COMMIT, fencing-проверка `lease_token` отклонит его результат (`stale_result_rejected`) — два неконтролируемых дубля не фиксируются. Его PUT в S3 перезаписал бы те же ключи теми же байтами (Pillow детерминирован), поэтому опасности для опубликованных объектов нет. +- RabbitMQ `consumer_timeout = 300 с`: канал, который не прислал ack дольше, закрывается брокером — последняя страховка от вечного зависания. +- DLQ `attachments.process.v1.dlq` (classic durable) получает: сообщения не по схеме, неизвестный `job_id`, превышение `x-delivery-limit=20` (бесконечный nack при недоступной БД). Реестр failed — `attachment_jobs` со `status='failed'` + `job_events` (`ops.py failed`, `ops.py events`). +- Replay (`ops.py replay `): только для `failed` с исходником в staging; та же job получает новую попытку и ещё 5 попыток, событие `replayed`; ready/queued/processing не трогаются («повторить» ≠ исполнить дубль успешной job). `ops.py dlq-replay`: для валидных сообщений DLQ решает по строке job в PostgreSQL (failed → replay, ready → удалить как no-op, queued/processing → удалить, работу ведёт outbox); невалидные и неизвестные остаются в DLQ. + +### Варианты изображения (pipeline_version = 1) + +| Вариант | Формат | Правило | Параметры | +| --- | --- | --- | --- | +| thumbnail | JPEG | вписан в 256×256, без увеличения, размеры вниз (минимум 1 px); 320×200 → 256×160 | quality 85, `optimize=True`, LANCZOS | +| crop | JPEG | центрированный квадрат 128×128 (маленькие увеличиваются) | quality 85, `optimize=True` | +| optimized | WebP | вписан в 1280×1280 без увеличения; прозрачность сохраняется | quality 80, method 4 | + +Во всех вариантах применяется EXIF orientation (1–8, как `ImageOps.exif_transpose`), альфа для JPEG компонуется на белый, EXIF/XMP/комментарии **не записываются** — в `save()` передаётся только ICC-профиль (цвет). Оригинал хранится побайтно как загружен (его EXIF не трогается — это требование контракта). Размер варианта считается от размеров исходника после поворота, а не от промежуточной копии. Большой JPEG декодируется сразу в уменьшенном масштабе (`draft`), сначала уменьшение до ≤1280, потом поворот. Не обещаю, что optimized меньше маленького исходника. + +### Лимиты worker + +- Bytes: API не принимает больше 10 MiB (413); worker скачивает staging не дальше `MAX_UPLOAD_BYTES` и сверяет размер и SHA-256 с метаданными. +- Пиксели: API проверяет по заголовку до распаковки (413); worker повторно — `Image.MAX_IMAGE_PIXELS` и явная проверка `w·h ≤ 40 000 000` (защита в глубину, `image_too_large`). +- Память: обработка идёт в дочернем процессе `imaging.py`, который **до импорта Pillow** ставит себе `RLIMIT_AS = WORKER_MEMORY_LIMIT_MB` (512 MiB), `RLIMIT_CPU`, `RLIMIT_FSIZE = 64 MiB`. Не хватило памяти → `MemoryError` → `resource_limit`, worker и его соединение с брокером живы. Контейнер worker: 2 CPU / 1536 MiB ≥ 2 потока × 512 MiB + сам worker + tmpfs. +- Время: worker убивает дочерний процесс по `JOB_TIMEOUT_SECONDS=60` (`processing_timeout`, временная ошибка). +- Конкуренция: `WORKER_CONCURRENCY=2` потока, у каждого своё соединение и `prefetch_count=1` → одновременно не больше 2 заданий на контейнер, остальное ждёт в очереди; очередь ограничена `x-max-length=100000` с `reject-publish` (переполнение — nack издателю, строка outbox ждёт, не потеря). + +### Очистка staging + +Worker удаляет `staging/` сразу после COMMIT ready и ack. Если не успел (упал, S3 недоступен) — janitor удаляет исходник ready-вложения, обновлённого больше 60 с назад. Исходник failed хранится `STAGING_RETENTION_SECONDS=86400` (для replay), потом удаляется с событием `staging_expired`. queued/processing janitor не трогает никогда. Объекты под `attachments//` удаляются, только если вложения нет, оно failed или ключ не входит в опубликованный набор ready. + +### Соответствие `.env.example` + +| Переменная | Что делает в этом решении | +| --- | --- | +| APP_*, SESSION_*, DATABASE_URL_FILE, CACHE_URL_FILE, API_REPLICAS, S3_*, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS, пулы и таймауты | как в lab4 | +| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) | +| BROKER_URL_FILE | пусто для api/janitor (брокера у них нет); compose задаёт свой файл dispatcher (`broker_url_dispatcher`) и worker (`broker_url_worker`) | +| WORKER_CONCURRENCY, WORKER_PREFETCH | 2 потока-потребителя, prefetch 1 на поток | +| JOB_MAX_ATTEMPTS, JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS | 5 попыток, 60 с на обработку изображения, аренда 15 с | +| JOB_BACKOFF_BASE_SECONDS, JOB_BACKOFF_MAX_SECONDS | 2 с и 60 с (equal jitter) | +| WORKER_MEMORY_LIMIT_MB | RLIMIT_AS процесса обработки, 512 MiB | +| STAGING_RETENTION_SECONDS | сколько хранится исходник failed-вложения для replay, 86400 с | +| PIPELINE_VERSION | 1 (версионирование нескольких pipeline — трек 5, не заявлен) | +| FAULT_INJECTION | учебные точки сбоев, на стенде включены по умолчанию (срабатывают только по файлу в volume `faults`, доступном лишь worker/dispatcher); `FAULT_INJECTION=0` — выключить | ## Доказательства критериев +`python student-tests/collect_evidence.py` создаёт `evidence/*.txt` (вывод тестов как есть, с датой, commit и кодом выхода) и `evidence/summary.md`. Сценарии lab5 — `student-tests/test_jobs.py`, сбои внедряются через `faults.py` (файл в volume `faults` → процесс в нужной точке делает `os._exit(137)`, как SIGKILL/OOM, или временную ошибку). Каждый сценарий проверяет инвариант «ровно 3 строки вариантов и ровно 4 объекта под `attachments//v1/`, SHA-256 совпадают». + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python tests/smoke.py` | PASS: публичный сценарий lab5 через балансировщик (202 → ready); размеры вариантов (thumbnail 256×160 для 320×200) — в jobs-variants.txt | smoke.txt | +| Контракт и публичные тесты не изменены | `python scripts/check_contract.py`; `git diff` по `contracts/`, `tests/` | PASS; не менялись | check.txt | +| 3: минимум lab4 сохранён (201 → 202) | `test_attachments.py`, `test_api.py`, `test_race.py`, `test_idempotency.py`, `test_balancing.py`, публичные persistence/failover | все PASS; upload ждёт 202 и затем ready | attachments.txt, student-tests.txt, race.txt, idempotency.txt, balancing.txt, persistence.txt, failover.txt | +| 3: 202 только после устойчивого staging и задания; остановка API после ответа не теряет работу | `test_jobs.py worker-down` | staging-объект есть до 202; job и outbox в БД; после рестарта api и rabbitmq тот же job_id → ready | jobs-recovery.txt | +| 3: отдельные broker и worker, durable, at-least-once, ack после эффекта, API не делает resize | `check_stack.py`, `test_jobs.py worker-down`, `broker-down` | rabbitmq/worker/dispatcher — отдельные сервисы; при остановленном worker вложение остаётся queued (API не обрабатывает); без брокера 202 и /health/ready 200 | stack.txt, jobs-recovery.txt | +| 3: queued/processing/ready/failed в API; оригинал неизменен; thumbnail по контракту | `test_jobs.py variants`, smoke | оригинал побайтно; 320×200 → 256×160; content 409 для queued и failed | jobs-variants.txt | +| 3: ограниченный retry, failed с безопасным error_code, повтор не создаёт второго набора | `test_jobs.py retry`, `exhausted-replay`, все сценарии сбоев | 5 попыток → `retry_exhausted`; в каждом сценарии ровно один набор | jobs-retry.txt, jobs-crash.txt | +| 3: демонстрация worker off → 202 → сообщение → рестарт API → worker → ready; брокер с volume не теряет задание | `test_jobs.py worker-down` | сообщение с вложением 201, другие сообщения работают, после `restart rabbitmq` в очереди 1 сообщение, затем ready и thumbnail | jobs-recovery.txt | +| 4: crop 128×128 и optimized WebP, параметры описаны | `test_jobs.py variants` | 6 исходников (PNG/JPEG, EXIF orientation 6, alpha, 40×30, 3000×2000, 5000×2): размеры по правилам, EXIF/XMP в вариантах 0, alpha на белом | jobs-variants.txt | +| 4: лимиты по bytes, пикселям, времени и памяти worker | `test_jobs.py limits` | 10 MiB+ → 413; 40 Мпикс PNG — пик RSS дочернего процесса 227 MiB при RLIMIT 512 MiB; лимит пикселей в worker — exit 4; «бомба памяти» → `resource_limit`; зависание → убито через 60 с, затем ready | jobs-limits.txt | +| 4: outbox, crash между COMMIT и publish, восстановление без ручного задания | `test_jobs.py outbox-crash` | dispatcher убит до publish → после рестарта тот же job опубликован и ready (attempt 1); убит после confirm → дубль, `duplicate_ignored`, один набор | jobs-outbox.txt | +| 4: backoff с jitter; постоянные/временные ошибки; DLQ/реестр failed; безопасный replay | `test_jobs.py retry exhausted-replay permanent dlq` | задержки в диапазонах 1–2 и 2–4 с; `invalid_image` — failed с attempt 1; replay → ready; replay ready — отказ без изменений; невалидное сообщение и неизвестный job → DLQ, dlq-replay их не исполняет | jobs-retry.txt | +| 4: остановка worker после записи объекта до ack; повтор завершает тот же результат | `test_jobs.py crash-after-put`, `crash-after-commit` | после PUT: аренда истекла, attempt 2 перезаписал те же ключи, 4 объекта; после COMMIT: redelivery → no-op ack, очередь пуста | jobs-crash.txt | +| 4: timeout/lease для зависшего processing | `test_jobs.py lease`, `limits` | зависший не продлевает аренду → через ~15 с job возвращена и выполнена другим потоком, поздний результат отклонён fencing; живой долгий — аренда продлевается | jobs-crash.txt, jobs-limits.txt | +| 4: ограниченная конкуренция/prefetch | `test_jobs.py concurrency` | 2 consumer, `prefetch_count=1`, ручной ack; из 8 накопленных заданий в processing одновременно ровно 2 | jobs-concurrency.txt | +| 4: очистка staging после безопасной фиксации | `test_jobs.py worker-down`, `staging` | staging удалён worker'ом после COMMIT; после crash до очистки — janitor | jobs-recovery.txt, jobs-crash.txt | +| Выбранный трек 5 | — | не выбран | — | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +**Где проверено сейчас.** Все строки таблицы выше подтверждены на Docker-стенде из этого репозитория (Windows 10, Docker Desktop, Compose 2.35, образы из `compose.yaml`, 2 реплики API за балансировщиком): 24.10.2026 один полный прогон `python student-tests/collect_evidence.py` — все 26 шагов PASS, сводка в `evidence/summary.md`, вывод каждого шага в `evidence/*.txt`, таблицы замеров ниже взяты из того же прогона. Перед ним на том же стенде отдельно прошли `docker compose up -d --build --wait` (migrate и s3-init — Exited (0), остальные healthy), `python tests/smoke.py` и `python student-tests/test_jobs.py worker-down`. Ранний прогон процессами без Docker (PostgreSQL 16, RabbitMQ 3.12, SeaweedFS 3.80, Redis) сохранён в `evidence/native-sandbox/` для сравнения. + +Что исправлено по итогам первого запуска в Docker (без Docker эти места не проявлялись): путь к definitions в `infra/rabbitmq.conf` (Compose монтирует секрет как `/run/secrets/rabbitmq_definitions`, без `.json`, — брокер не стартовал); у janitor появился healthcheck по файлу-пульсу (с `healthcheck: disable: true` команда `up --wait` завершалась ошибкой); в тестах — пробник `test_jobs.py` запускается одноразовым контейнером worker, когда сам worker остановлен, `upload_memory.py` ждёт 202 вместо 201 из lab4, а сценарий `staging` проверяет результат очистки независимо от того, какой проход janitor её сделал. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотезы: +1. Принятая (202) работа не теряется ни при каком одиночном отказе процесса (API, dispatcher, worker, broker) в любом из окон контракта, а повторная доставка не создаёт второго эффекта. +2. Память worker ограничена независимо от входа: худший допустимый вход (PNG 8000×5000) помещается в RLIMIT_AS 512 MiB, а вход сверх лимита ломает только свою попытку, не процесс worker. + +Методика: каждый сценарий `test_jobs.py` взводит одну точку сбоя (`faults.py`), загружает синтетический файл через балансировщик, ждёт конечного состояния через публичный GET и проверяет журнал `job_events`, строки outbox, глубину очередей (`rabbitmqctl list_queues`) и объекты в bucket (пробник в контейнере worker). Пик памяти обработки — `ru_maxrss` дочернего процесса `imaging.py` (`getrusage(RUSAGE_CHILDREN)`). + + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-24, commit 276dbda + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 5aa14958c864 | 2 | 123 | 0 | 4.406 | +| 2 | b9ad6d33d190 | 2 | 125 | 0 | 4.188 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.45 / 1.56 | 5aa14958c864: 5 | 96.0 | 109.7 | 13.7 | 149.6 | +| | | | | b9ad6d33d190: 5 | 94.5 | 108.9 | 14.4 | 142.1 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.04 / 1.31 | 5aa14958c864: 5 | 96.6 | 97.6 | 1.0 | 89.6 | +| | | | | b9ad6d33d190: 5 | 96.2 | 98.6 | 2.3 | 88.9 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 4.96 / 5.2 | 5aa14958c864: 10 | 94.7 | 114.9 | 20.3 | 155.4 | +| | | | | b9ad6d33d190: 10 | 97.0 | 121.8 | 24.8 | 176.9 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 243.885 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.406, 4.188 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 58.5 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19922 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14860 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | + + +Результаты прогона на нативном стенде (`evidence/native-sandbox/`, 1 API-реплика без балансировщика): + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) | - | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается: падение/рестарт любого одного процесса (api, dispatcher, worker, rabbitmq, postgres с volume) в любом окне отказа из контракта; недоступность брокера при приёме; зависание и нехватка памяти в обработке изображения; временные ошибки S3. +- Не выдерживается: потеря volume `pgdata` (источник истины) или `s3data` — нужен бэкап; один узел RabbitMQ (quorum-очередь из одного члена — durability есть, отказоустойчивости узла нет); при недоступном PostgreSQL новые загрузки и обработка останавливаются (это осознанно: без источника истины нельзя ни принять, ни зафиксировать). +- Учебные упрощения: worker использует тот же ограниченный S3-ключ, что и API (права только на свой bucket; отдельный ключ worker — следующий шаг); точки внедрения сбоев включены на стенде (`FAULT_INJECTION=1`), в эксплуатации выключаются; replay — CLI в контейнере dispatcher, а не API с авторизацией оператора. +- Подтверждено измерением: память обработки ограничена, конкуренция равна `WORKER_CONCURRENCY`, отказы не дают потерь и дублей эффекта. Гипотеза без измерения: пропускная способность pipeline под перегрузкой и выбор concurrency/prefetch — это трек 5.1, не заявлен. + +## Ответы на вопросы защиты (кратко) + +- **Publisher confirm vs consumer ack.** Confirm — брокер принял и сохранил сообщение (после него dispatcher отмечает outbox). Ack — worker устойчиво выполнил эффект (объекты в S3 + COMMIT ready); до ack сообщение остаётся у брокера и будет доставлено снова. +- **Что сохранено в момент 202.** Полный проверенный оригинал в S3-prefix `staging/`, строка вложения `queued`, задание `attachment_jobs` и сообщение в `outbox` — одной транзакцией PostgreSQL. Брокер может ещё ничего не знать. +- **Почему повторная доставка нормальна и где ключ идемпотентности.** At-least-once: ack может потеряться после эффекта, dispatcher может опубликовать дважды. Ключ — `job_id` (одна логическая работа) и ключ эффекта `(attachment_id, pipeline_version, variant)`: первичный ключ `attachment_variants` и детерминированные ключи S3 `attachments//v/`; состояние job в PostgreSQL решает, работать или подтвердить как no-op. +- **Зависший vs долго работающий.** Живой продлевает аренду каждые 5 с из отдельного потока, его ограничивает только `JOB_TIMEOUT_SECONDS`. Зависший/умерший перестаёт продлевать — через 15 с job перехватывают; если он «оживёт», fencing по `lease_token` не даст зафиксировать результат. diff --git a/attachments.py b/attachments.py new file mode 100644 index 0000000..dabbde2 --- /dev/null +++ b/attachments.py @@ -0,0 +1,80 @@ +"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202. + +Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого +объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение: + + 1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503. + 2. PUT исходника в staging/ — упало: удаляем объект (PUT мог пройти) + и строку, клиенту 503. Если и это не вышло — + uploading-строку уберёт janitor после grace. + 3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT + — упало/неизвестно: клиенту 503, строка + остаётся uploading -> janitor удалит объект + и строку. Если COMMIT всё же прошёл, задание + есть и будет обработано (повтор клиента даст + второе вложение — в контракте нет ключа). + 4. 202 Attachment(status=queued) — только после шага 3. + +Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный +брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно. +""" +import logging +from uuid import UUID, uuid4 + +import db +import settings +import storage +from errors import unavailable +from uploads import ImageInfo, Upload + +log = logging.getLogger("messenger.attachments") + + +def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict: + attachment_id = uuid4() + key = storage.new_key(uuid4().hex) + db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type, + upload.size, upload.sha256, image.width, image.height, key) + try: + storage.put(key, upload.file, upload.size, image.content_type, upload.sha256) + except storage.StorageUnavailable: + compensate(attachment_id, key) + raise unavailable("File storage is temporarily unavailable") from None + row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID) + if row is None: + # janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace). + log.warning("Upload %s was reclaimed before it was queued", attachment_id) + raise unavailable() + log.info("Attachment %s accepted into staging and queued", attachment_id) + return {**row, "variants": []} + + +def compensate(attachment_id: UUID, key: str) -> None: + """Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor.""" + try: + storage.delete(key, quick=True) + db.delete_upload_intent(attachment_id) + log.info("Upload %s rolled back after S3 failure", attachment_id) + except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту + log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__) + + +def public(row: dict) -> dict: + """Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей).""" + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["state"], + "variants": [ + {key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")} + for v in row.get("variants", []) + ], + "error_code": row["error_code"] if row["state"] == "failed" else None, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..c21212f --- /dev/null +++ b/auth.py @@ -0,0 +1,139 @@ +import base64 +import binascii +import hashlib +import logging +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from redis.backoff import NoBackoff +from redis.retry import Retry + +import settings +from errors import BASIC_CHALLENGE, unauthorized, unavailable + +log = logging.getLogger("messenger.auth") + +hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1) + +DUMMY_HASH = hasher.hash("there-is-no-such-user") + +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def password_matches(password_hash: str | None, password: str) -> bool: + try: + hasher.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """'Basic base64(username:password)' -> (username, password) или None. + + Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617). + """ + scheme, _, value = (header or "").strip().partition(" ") + if scheme.lower() != "basic" or not value or len(value) > 1024: + return None + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + username, colon, password = decoded.partition(":") + if not colon or not username or not username.isascii(): + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + scheme, _, value = (header or "").strip().partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value: + return None + return value + + + +KEY_PREFIX = "session:" + +_redis = redis.Redis( + connection_pool=redis.BlockingConnectionPool.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + max_connections=settings.REDIS_POOL_MAX, + timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) +) + + +@dataclass(frozen=True) +class Session: + user_id: UUID + key: str + + +def session_key(token: str) -> str: + return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: UUID) -> tuple[str, datetime]: + """Новая сессия: случайный токен и абсолютное время истечения.""" + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS) + try: + _redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000)) + except redis.RedisError as exc: + log.warning("Redis is unavailable on login: %s", type(exc).__name__) + raise unavailable() from None + return token, expires_at + + +def check_bearer(header: str | None) -> Session: + token = parse_bearer(header) + if token is None: + raise unauthorized("Bearer token is required") + key = session_key(token) + try: + user_id = _redis.get(key) + except redis.RedisError as exc: + log.warning("Redis is unavailable: %s", type(exc).__name__) + raise unavailable() from None + if user_id is None: + raise unauthorized("Session is invalid or expired") + return Session(UUID(user_id), key) + + +def delete_session(session: Session) -> None: + try: + _redis.delete(session.key) + except redis.RedisError: + raise unavailable() from None + + +def ping_redis() -> None: + try: + _redis.ping() + except redis.RedisError: + raise unavailable() from None + + +def login_failed(): + return unauthorized("Invalid username or password", BASIC_CHALLENGE) + + +def close() -> None: + _redis.close() + _redis.connection_pool.disconnect() diff --git a/broker.py b/broker.py new file mode 100644 index 0000000..504ce4b --- /dev/null +++ b/broker.py @@ -0,0 +1,105 @@ +"""RabbitMQ: подключение, проверка топологии, публикация с publisher confirms. + +Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует +scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure: +они только проверяют её пассивным declare и не могут создать/удалить очередь. + + exchange attachments (direct, durable) + --attachment.process.v1--> queue attachments.process.v1 (quorum, durable) + x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once), + x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря) + exchange attachments.dlx (direct, durable) + --attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py + через basic_get + nack не расходует delivery-limit) + +Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до +подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются. +""" +import json +import logging + +import pika +from pika.exceptions import AMQPError + +import pipeline +import settings + +log = logging.getLogger("messenger.broker") +logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал + +BrokerError = (AMQPError, OSError) + + +def parameters(url: str | None = None) -> pika.URLParameters: + params = pika.URLParameters(url or settings.BROKER_URL) + params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS + params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно + params.socket_timeout = 5 + params.connection_attempts = 1 + params.client_properties = {"connection_name": settings.INSTANCE_ID} + return params + + +def connect(url: str | None = None) -> pika.BlockingConnection: + return pika.BlockingConnection(parameters(url)) + + +def check_topology(channel) -> dict: + """Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях.""" + channel.exchange_declare(pipeline.EXCHANGE, passive=True) + main = channel.queue_declare(pipeline.QUEUE, passive=True).method + dlq = channel.queue_declare(pipeline.DLQ, passive=True).method + return {"queue": main.message_count, "consumers": main.consumer_count, "dlq": dlq.message_count} + + +def properties(message: dict) -> pika.BasicProperties: + headers = {} + if "traceparent" in message: + headers["traceparent"] = message["traceparent"] # тот же carrier и в native headers + return pika.BasicProperties( + delivery_mode=pika.DeliveryMode.Persistent, + content_type="application/json", + message_id=f"{message['job_id']}:{message['attempt']}", + type=message["type"], + headers=headers or None, + ) + + +class Publisher: + """Канал в режиме confirm: basic_publish возвращает управление только после ack брокера. + + mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError. + Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях + строка outbox остаётся неопубликованной и будет отправлена позже. + """ + + def __init__(self): + self.connection = None + self.channel = None + + def open(self): + if self.channel is None or self.channel.is_closed or self.connection.is_closed: + self.close() + self.connection = connect() + self.channel = self.connection.channel() + self.channel.confirm_delivery() + check_topology(self.channel) + log.info("connected to broker, publisher confirms enabled") + return self.channel + + def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY): + body = json.dumps(message, separators=(",", ":")).encode("utf-8") + self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True) + + def pump(self): + if self.connection is not None and self.connection.is_open: + self.connection.process_data_events(time_limit=0) # heartbeat в паузах + + def close(self): + try: + if self.connection is not None and self.connection.is_open: + self.connection.close() + except BrokerError: + pass + self.connection = None + self.channel = None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..056873b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,320 @@ +name: messenger-lab5 + +x-app-build: &app-build + context: . + args: + APP_VERSION: ${APP_VERSION:-dev} + +# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py. +x-pipeline-env: &pipeline-env + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} + JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + # Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults, + # доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0. + FAULT_INJECTION: ${FAULT_INJECTION:-1} + FAULTS_DIR: /faults + +x-alive-check: &alive-check + # Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад. + test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 20s + start_interval: 1s + +x-s3-api-secrets: &s3-api-secrets + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + migrate: + build: *app-build + command: ["python", "migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений. + # Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config + # (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket. + # Анонимного доступа нет: identity "anonymous" в конфиге не объявлена. + s3: + image: chrislusf/seaweedfs:3.80 + entrypoint: ["/usr/bin/weed"] + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0", + "-master.volumeSizeLimitMB=1024", "-volume.max=0", + "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: ["s3data:/data"] + networks: [data] + healthcheck: + # S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт») + test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 15s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки. + s3-init: + build: *app-build + command: ["python", "s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + api: + build: *app-build + env_file: .env + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL + <<: *s3-api-secrets + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "1.00", memory: 512M} + + # Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ. + janitor: + build: *app-build + command: ["python", "janitor.py"] + env_file: .env + environment: + CACHE_URL_FILE: "" # janitor не ходит в Redis + BROKER_URL_FILE: "" # и в брокер + <<: *s3-api-secrets + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + + # Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права — + # в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте. + # hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume. + rabbitmq: + image: rabbitmq:4.1.4-alpine + hostname: rabbitmq + volumes: + - ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro + - rabbitmqdata:/var/lib/rabbitmq + secrets: [rabbitmq_definitions] + networks: [data] + healthcheck: + # Узел запущен, и vhost с очередями загружен (а не только «порт открыт»). + test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"] + interval: 5s + timeout: 10s + retries: 24 + start_period: 20s + start_interval: 2s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает + # просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет. + dispatcher: + build: *app-build + command: ["python", "dispatcher.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_dispatcher + <<: *pipeline-env + secrets: [database_url, broker_url_dispatcher] + volumes: ["faults:/faults"] + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + # Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем + # процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS. + # Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker. + worker: + build: *app-build + command: ["python", "worker.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_worker + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512} + <<: [*pipeline-env, *s3-api-secrets] + secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key] + volumes: ["faults:/faults"] + tmpfs: + - /tmp:size=256m,mode=1777 + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + healthcheck: *alive-check + restart: unless-stopped + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят; + # если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова. + stop_grace_period: 70s + deploy: + resources: + limits: {cpus: "2.00", memory: 1536M} + + lb: + image: haproxy:3.2.25-alpine + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + postgres: + image: postgres:17.11-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] + secrets: [postgres_password] + volumes: ["pgdata:/var/lib/postgresql/data"] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + redis: + image: redis:7.4.11-alpine + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + - redisdata:/data + secrets: [redis_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + +networks: + edge: {} + data: + internal: true + +volumes: + pgdata: {} + redisdata: {} + s3data: {} + rabbitmqdata: {} + faults: {} + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + redis_auth: {file: ./secrets/redis_auth} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + s3_config: {file: ./secrets/s3_config.json} + broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher} + broker_url_worker: {file: ./secrets/broker_url_worker} + rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json} diff --git a/db.py b/db.py new file mode 100644 index 0000000..1530665 --- /dev/null +++ b/db.py @@ -0,0 +1,391 @@ +import logging +import time +from contextlib import contextmanager +from datetime import datetime +from uuid import UUID, uuid4 + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +import pipeline +import settings +from errors import bad_request, conflict, not_found, unavailable + +log = logging.getLogger("messenger.db") + + +pool = ConnectionPool( + conninfo=settings.DATABASE_URL, + min_size=settings.DB_POOL_MIN, + max_size=settings.DB_POOL_MAX, + timeout=settings.DB_POOL_WAIT_SECONDS, + kwargs={ + "connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS, + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" + " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", + }, + check=ConnectionPool.check_connection, + open=False, +) + + +@contextmanager +def connection(wait_seconds: float | None = None): + try: + with pool.connection(timeout=wait_seconds) as conn: + yield conn + except (psycopg.OperationalError, PoolTimeout) as exc: + log.warning("PostgreSQL is unavailable: %s", type(exc).__name__) + raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text", "filename") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} + + +def ping() -> None: + with connection(wait_seconds=1) as conn: + conn.execute("SELECT 1") + + +MIGRATIONS_LOCK = 20260202 + + +def migration_files() -> list: + return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) + + +def run_migrations() -> list[str]: + files = migration_files() + applied = [] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now())""") + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + applied.append(path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) + return applied + + +def wait_and_migrate(max_wait_seconds: int = 60) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + applied = run_migrations() + log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") + return + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + log.warning("PostgreSQL is not ready yet, retry in 1 s") + time.sleep(1) + + +def missing_migrations() -> list[str]: + expected = [path.name for path in migration_files()] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + try: + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + except pg_errors.UndefinedTable: + return expected + return [name for name in expected if name not in done] + + +def wait_for_schema(max_wait_seconds: int) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + missing = missing_migrations() + except psycopg.OperationalError: + missing = None + if missing == []: + log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) + return + if time.monotonic() > deadline: + raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") + log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None + else "missing " + ", ".join(missing)) + time.sleep(1) + + + +USER_COLUMNS = "id, username, display_name, created_at" + + +def create_user(username: str, display_name: str, password_hash: str) -> dict: + try: + with connection() as conn: + return from_db(conn.execute( + f"""INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) + except pg_errors.UniqueViolation: + raise conflict("Username already exists") from None + + +def get_user(user_id: UUID) -> dict | None: + with connection() as conn: + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) + + +def get_user_with_password(username: str) -> dict | None: + with connection() as conn: + return from_db(conn.execute( + f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) + ).fetchone()) + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" + + +def create_chat(title: str, member_ids: list[UUID]) -> dict: + chat_id = uuid4() + members = sorted(member_ids) + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, to_db(title)), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in members], + ) + except pg_errors.ForeignKeyViolation: + raise bad_request("Unknown member") from None + return {**from_db(row), "member_ids": members} + + +def get_chat(chat_id: UUID, user_id: UUID) -> dict: + with connection() as conn: + chat = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s""", + (user_id, chat_id), + ).fetchone() + if chat is None: + raise not_found("Chat not found") + return from_db(chat) + + +def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" + where, params = "", [user_id] + if after is not None: + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params += list(after) + with connection() as conn: + rows = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + {where} + ORDER BY c.created_at, c.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids""" + + +def is_member(conn, chat_id: UUID, user_id: UUID) -> bool: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() is not None + + +def check_member(chat_id: UUID, user_id: UUID) -> None: + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + + +def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID], + idempotency_key: str, request_hash: bytes) -> dict: + try: + with connection() as conn: + with conn.transaction(): + if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404 + raise not_found("Chat not found") + if attachment_ids: + # Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409). + # FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние. + found = conn.execute( + """SELECT id, state FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""", + (attachment_ids, chat_id), + ).fetchall() + if len(found) != len(attachment_ids): + raise not_found("Attachment not found") + failed = any(row["state"] == "failed" for row in found) + else: + failed = False + saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m + WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""" + if failed: + # Повтор уже созданного сообщения возвращает прежний 201, даже если вложение + # с тех пор стало failed. Новое сообщение с failed-вложением — 409. + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if saved is None: + raise conflict("Failed attachment cannot be attached") + created = None + else: + created = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), + ).fetchone() + if created is None: + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if created is not None: + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + """INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES (%s, %s, %s, %s)""", + [(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)], + ) + return {**from_db(created), "attachment_ids": list(attachment_ids)} + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен + raise unavailable() + if bytes(saved.pop("request_hash")) != request_hash: + raise conflict("Idempotency-Key was already used with a different request") + return from_db(saved) + + +def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """История чата после курсора (keyset-пагинация), до limit+1 строк.""" + where, params = "m.chat_id = %s", [chat_id] + if after is not None: + where += " AND (m.created_at, m.id) > (%s, %s)" + params += list(after) + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + rows = conn.execute( + f"""SELECT {MESSAGE_COLUMNS} FROM messages m + WHERE {where} + ORDER BY m.created_at, m.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + +# ---------- вложения ---------- + +ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, " + "pipeline_version, created_at, updated_at, object_key") +VISIBLE_STATES = ("queued", "processing", "ready", "failed") + + +def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, + size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None: + """Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging. + + Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт + и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД). + """ + try: + with connection() as conn: + conn.execute( + """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, source_key, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""", + (attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256, + width, height, source_key), + ) + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + + +def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None: + """Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией. + + После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL, + сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace). + """ + with connection() as conn: + with conn.transaction(): + row = conn.execute( + f"""UPDATE attachments SET state = 'queued', updated_at = now() + WHERE id = %s AND state = 'uploading' + RETURNING {ATTACHMENT_COLUMNS}""", + (attachment_id,), + ).fetchone() + if row is None: + return None + pipeline.enqueue_new(conn, attachment_id, actor) + return from_db(row) + + +def delete_upload_intent(attachment_id: UUID) -> None: + with connection(wait_seconds=1) as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,)) + + +def get_attachment(attachment_id: UUID, user_id: UUID) -> dict: + """Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404. + + Возвращает строку и варианты текущей версии pipeline (они есть только у ready). + """ + with connection() as conn: + row = conn.execute( + f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))} + FROM attachments a + JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s + WHERE a.id = %s AND a.state = ANY(%s)""", + (user_id, attachment_id, list(VISIBLE_STATES)), + ).fetchone() + if row is None: + raise not_found("Attachment not found") + variants = [] + if row["state"] == "ready": + variants = conn.execute( + """SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""", + (attachment_id, row["pipeline_version"]), + ).fetchall() + return {**from_db(row), "variants": variants} diff --git a/dispatcher.py b/dispatcher.py new file mode 100644 index 0000000..f231370 --- /dev/null +++ b/dispatcher.py @@ -0,0 +1,150 @@ +"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания. + +Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной +транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором +намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox +остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса, +ни ключей брокера, и недоступность брокера не мешает приёму файлов. + +Цикл: +1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP + LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT. + Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения, + который worker распознаёт по состоянию job (at-least-once, а не exactly-once). +2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed. +3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages). +Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру. +""" +import logging +import signal +import threading +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +import broker +import faults +import pipeline +import settings + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.dispatcher") + +STOP = threading.Event() +ACTOR = "dispatcher@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def publish_due(conn, publisher: broker.Publisher) -> int: + published = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, job_id, attempt, payload FROM outbox + WHERE published_at IS NULL AND available_at <= now() + ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", + (settings.DISPATCHER_BATCH,), + ).fetchall() + if not rows: + return published + for row in rows: + faults.crash("dispatcher_crash_before_publish") + try: + publisher.publish(row["payload"]) + except broker.BrokerError as exc: + log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__) + publisher.close() + raise + faults.crash("dispatcher_crash_after_confirm") + conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],)) + published += 1 + log.info("published %s message(s)", len(rows)) + if len(rows) < settings.DISPATCHER_BATCH: + return published + return published + + +def note_failure(conn) -> None: + """Счётчик неудачных публикаций для наблюдения (ops.py status).""" + try: + conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1 + WHERE published_at IS NULL AND available_at <= now()""") + except psycopg.Error: + pass + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def run() -> None: + publisher = broker.Publisher() + conn = None + last_leases = last_reconcile = 0.0 + pause = 1.0 + while not STOP.is_set(): + try: + if conn is None or conn.closed: + conn = connect_db() + conn.execute("LISTEN outbox") + log.info("connected to PostgreSQL, listening for outbox notifications") + touch_alive() + publish_due(conn, publisher) + now = time.monotonic() + if now - last_leases >= settings.DISPATCHER_POLL_SECONDS: + last_leases = now + if recovered := pipeline.recover_expired_leases(conn, ACTOR): + log.warning("recovered %s job(s) with expired lease", recovered) + if now - last_reconcile >= 30: + last_reconcile = now + if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS): + log.warning("re-published %s queued job(s) without a message in flight", lost) + pause = 1.0 + publisher.pump() + # Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса. + for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1): + pass + except broker.BrokerError: + if conn is not None and not conn.closed: + note_failure(conn) + log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause) + STOP.wait(pause) + pause = min(pause * 2, 15) + except psycopg.Error as exc: + log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause) + try: + if conn is not None: + conn.close() + except psycopg.Error: + pass + conn = None + STOP.wait(pause) + pause = min(pause * 2, 15) + publisher.close() + if conn is not None: + conn.close() + log.info("stopped") + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID, + "ON" if settings.FAULT_INJECTION else "off") + run() + + +if __name__ == "__main__": + main() diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..a641476 --- /dev/null +++ b/errors.py @@ -0,0 +1,40 @@ +# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт. +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions +BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "Service temporarily unavailable") -> ApiError: + return ApiError(503, "unavailable", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 0000000..e70352d --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-10-24 15:37:16, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_attachments.py +# exit code: 0, 10.2 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready f338bc0d +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/balancing.txt b/evidence/balancing.txt new file mode 100644 index 0000000..54f98f0 --- /dev/null +++ b/evidence/balancing.txt @@ -0,0 +1,18 @@ +# balancing: balancing, shared sessions/data/files, X-Instance-Id +# date: 2026-10-24 15:40:58, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_balancing.py +# exit code: 0, 2.1 s + +20 x GET /users/me with one token -> replicas: {'b9ad6d33d190': 10, '5aa14958c864': 10} +PASS: one session works on every replica, no second login (sessions are shared in Redis) +message written by 5aa14958c864; read back by ['5aa14958c864', 'b9ad6d33d190'] +PASS: a message written via one replica is read right away via another (shared PostgreSQL) +logout on b9ad6d33d190; old token then rejected by ['5aa14958c864', 'b9ad6d33d190'] +PASS: logout on one replica revokes the session on all of them +first POST on b9ad6d33d190 -> 201 af45ed44-ef90-4258-b144-77628e56bfed; same key on 5aa14958c864 -> 201 af45ed44-ef90-4258-b144-77628e56bfed +PASS: Idempotency-Key is shared: a retry through another replica returns the same message +file uploaded via b9ad6d33d190; downloaded byte for byte via ['5aa14958c864', 'b9ad6d33d190'] +PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local) +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['5aa14958c864', 'b9ad6d33d190']; api containers: ['5aa14958c864', 'b9ad6d33d190'] +PASS: X-Instance-Id is in every response and points to a real container +6 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..36d085e --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-10-24 15:30:17, commit: 276dbda + uncommitted changes +# command: python.exe scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 0000000..4d95004 --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,47 @@ +# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage +# date: 2026-10-24 15:40:02, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_consistency.py +# exit code: 0, 156.4 s + +declared 9 000 000 bytes, sent 211817, closed the connection +declared 9 000 000 bytes, sent 3111817, closed the connection +rows in attachments: 219 -> 219; objects in bucket: 860 -> 860 +temporary upload files in api containers: {'b9ad6d33d190': 0, '5aa14958c864': 0} (removed when the request ends) +api containers have no volumes: the original is never stored locally +PASS: an interrupted upload leaves no row, no object and no local file +crash after S3 PUT: row 3a083fe9 state=uploading, object staging/5bda5e392029444d915a233aef8e7af9 exists in S3 +the half-written attachment is invisible: GET -> 404, attach to a message -> 404 +janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 865, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.31} -> fresh row and object are kept (the upload might still be running) +row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 865, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.301} -> nothing deleted +janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 864, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.29} -> row and object deleted +janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 864, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.278} -> idempotent +the ready attachment of the same chat is untouched +PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period +object staging/bea3e5320fc84973a5b2b762fe96ddd8 put into the bucket without a row in PostgreSQL +janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.292} -> fresh object kept +after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.298} -> reported, kept +janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.295} -> orphan deleted +ready attachments older than the grace period are untouched (they have rows) +object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500) +PASS: orphans are found and deleted only after the grace period; missing objects are reported +--- S3 stopped (connection refused / name not resolved): docker compose stop s3 +upload -> 503 unavailable in 19922 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose start s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 879, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.29} +PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up +--- S3 frozen (no answer, timeouts): docker compose pause s3 +upload -> 503 unavailable in 14860 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose unpause s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 887, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.284} +PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up +5 passed, 0 failed diff --git a/evidence/db.txt b/evidence/db.txt new file mode 100644 index 0000000..f45a622 --- /dev/null +++ b/evidence/db.txt @@ -0,0 +1,21 @@ +# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations +# date: 2026-10-24 15:41:16, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_db.py +# exit code: 0, 10.7 s + +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 185 -> 186 +PASS: chat row was inserted and then rolled back together with members +direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" +PASS: the database itself rejects a message from a non-member +stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97) +PASS: password is stored as Argon2id hash +unique constraint on users: UNIQUE (username) +PASS: username is unique in the database +applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; running the migrate service once more +after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; log: Migrations applied: none (schema is up to date) +PASS: migrations are applied once, a repeated run does nothing +api replicas that checked the schema at start: 28; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago +PASS: only the migrate service applies migrations, replicas wait for the schema +unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) +PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database +7 passed, 0 failed diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 0000000..cf19e0d --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,99 @@ +=== Data volume + messages_total | messages_in_big_chat +----------------+---------------------- + 29169 | 15000 + + +=== History index + indexdef +---------------------------------------------------------------------------------------- + CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id) + + +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + QUERY PLAN +------------------------------------------------------------------------------------------------ + Limit (actual time=0.005..0.012 rows=51 loops=1) + Buffers: shared hit=14 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Index Cond: (chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) + Buffers: shared hit=14 + Planning: + Buffers: shared hit=6 + Planning Time: 0.058 ms + Execution Time: 0.020 ms + + +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.006..0.012 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.010 rows=51 loops=1) + Index Cond: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid))) + Buffers: shared hit=5 + Planning Time: 0.043 ms + Execution Time: 0.029 ms + + +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.020..0.155 rows=51 loops=1) + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.019..0.152 rows=51 loops=1) + Index Cond: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid))) + Buffers: shared hit=59 + SubPlan 1 + -> Sort (actual time=0.002..0.002 rows=0 loops=51) + Sort Key: a."position" + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Recheck Cond: (message_id = m.id) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Index Cond: (message_id = m.id) + Buffers: shared hit=51 + Planning: + Buffers: shared hit=63 + Planning Time: 0.205 ms + Execution Time: 0.172 ms + + +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Limit (actual time=0.832..0.838 rows=51 loops=1) + Buffers: shared hit=266 + -> Index Scan using messages_history on messages (actual time=0.007..0.645 rows=10051 loops=1) + Index Cond: (chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) + Buffers: shared hit=266 + Planning Time: 0.041 ms + Execution Time: 0.849 ms + + +=== 4. Query 2 without the index (index is dropped only inside this transaction) + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual time=1.257..1.261 rows=51 loops=1) + Buffers: shared hit=450 + -> Sort (actual time=1.257..1.258 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 31kB + Buffers: shared hit=450 + -> Seq Scan on messages (actual time=0.530..0.937 rows=5000 loops=1) + Filter: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid))) + Rows Removed by Filter: 24169 + Buffers: shared hit=450 + Planning: + Buffers: shared hit=4 + Planning Time: 0.082 ms + Execution Time: 1.274 ms + + +=== ROLLBACK: test data removed, index is back + explain_users_left +-------------------- + 0 + diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 0000000..0a7a8a8 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,43 @@ +# explain: EXPLAIN of the history page (full plans: explain-history.txt) +# date: 2026-10-24 15:41:20, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/explain.py +# exit code: 0, 0.7 s + +=== Data volume +=== History index +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=14 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Buffers: shared hit=14 + Buffers: shared hit=6 + Execution Time: 0.020 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.010 rows=51 loops=1) + Buffers: shared hit=5 + Execution Time: 0.029 ms +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.019..0.152 rows=51 loops=1) + Buffers: shared hit=59 + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + Buffers: shared hit=63 + Execution Time: 0.172 ms +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + Buffers: shared hit=266 + -> Index Scan using messages_history on messages (actual time=0.007..0.645 rows=10051 loops=1) + Buffers: shared hit=266 + Execution Time: 0.849 ms +=== 4. Query 2 without the index (index is dropped only inside this transaction) + Buffers: shared hit=450 + Buffers: shared hit=450 + -> Seq Scan on messages (actual time=0.530..0.937 rows=5000 loops=1) + Buffers: shared hit=450 + Buffers: shared hit=4 + Execution Time: 1.274 ms +=== ROLLBACK: test data removed, index is back +PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..79d541e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,33 @@ +# failover: public failover test for every replica, file downloaded via the other one (make test-failover) +# date: 2026-10-24 15:42:00, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/run_failover.py +# exit code: 0, 40.6 s + +hook: scripts\stop-one.cmd; api replicas: ['5aa14958c864', 'b9ad6d33d190'] +=== run 1: api containers before +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Up 44 seconds (healthy) +api 5aa14958c864 Up 43 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.406} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Up About a minute (healthy) +api 5aa14958c864 Exited (0) 11 seconds ago +the hook stopped 5aa14958c864 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 5aa14958c864 -> healthy and back in HAProxy +=== run 2: api containers before +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Up About a minute (healthy) +api 5aa14958c864 Up 2 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 4.188} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Exited (0) 11 seconds ago +api 5aa14958c864 Up 19 seconds (healthy) +the hook stopped b9ad6d33d190 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start b9ad6d33d190 -> healthy and back in HAProxy +PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 0000000..5ffffe6 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,29 @@ +# failures: Redis and PostgreSQL outages via the balancer +# date: 2026-10-24 15:43:26, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_failures.py +# exit code: 0, 55.0 s + +stopping redis +/health/live -> 200, /health/ready -> 503 (8016 ms) +valid token: GET /users/me -> 503 unavailable (8015 ms), session cannot be checked +unknown token -> 503, not 200: nobody is trusted without a check +no token -> 401 +login -> 503: there is nowhere to store the session +registration -> 201: it needs only PostgreSQL +starting redis +after start: the old token works, confirmed messages are in place +PASS: Redis outage gives 503 and does not let anyone in without a session check +killing redis with SIGKILL (docker compose kill redis): no graceful shutdown +right after the crash: GET /users/me -> 503 +starting redis +after start: the session issued before the crash still works (AOF, appendfsync always) +PASS: sessions survive a Redis crash, not only a clean restart +stopping postgres +/health/live -> 200, /health/ready -> 503 (1016 ms) +valid token: users/me, chats, history, new message -> 503 unavailable, no details +no token -> 401, unknown token -> 401: anonymous users are not let in +login -> 503, registration -> 503 +starting postgres +after start: the old token works, confirmed messages are in place +PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out +3 passed, 0 failed diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..769aa85 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-10-24 15:41:01, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_idempotency.py +# exit code: 0, 2.5 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'5aa14958c864': 10, 'b9ad6d33d190': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/jobs-concurrency.txt b/evidence/jobs-concurrency.txt new file mode 100644 index 0000000..1433d72 --- /dev/null +++ b/evidence/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-10-24 15:37:05, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py concurrency +# exit code: 0, 15.1 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) +1 passed, 0 failed diff --git a/evidence/jobs-crash.txt b/evidence/jobs-crash.txt new file mode 100644 index 0000000..81ab8d6 --- /dev/null +++ b/evidence/jobs-crash.txt @@ -0,0 +1,15 @@ +# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease +# date: 2026-10-24 15:35:40, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease +# exit code: 0, 156.3 s + +--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат. +worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] +--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack. +worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) +--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit). +staging source still present after the crash (worker died before cleanup); waiting 60 s window +janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone +--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; +frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] +4 passed, 0 failed diff --git a/evidence/jobs-limits.txt b/evidence/jobs-limits.txt new file mode 100644 index 0000000..a97da4f --- /dev/null +++ b/evidence/jobs-limits.txt @@ -0,0 +1,11 @@ +# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) +# date: 2026-10-24 15:31:54, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py limits +# exit code: 0, 73.6 s + +--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker. +10 MiB + 8 bytes -> 413 before any job is created +imaging.py on 8000x5000 PNG: exit 0, peak RSS 227.2 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large) +memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived +imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s +1 passed, 0 failed diff --git a/evidence/jobs-outbox.txt b/evidence/jobs-outbox.txt new file mode 100644 index 0000000..d844522 --- /dev/null +++ b/evidence/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-10-24 15:33:04, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py outbox-crash +# exit code: 0, 15.1 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/jobs-recovery.txt b/evidence/jobs-recovery.txt new file mode 100644 index 0000000..bcb15b0 --- /dev/null +++ b/evidence/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-10-24 15:32:49, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 54.9 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment cabbd2a3-fa8c-4805-aa76-6d59e2b5d68a +message with attachment 201, content 409, other messages 201; job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job 17043324-e0c6-4c0c-88df-6ae2a8eca8fd +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/jobs-retry.txt b/evidence/jobs-retry.txt new file mode 100644 index 0000000..d3f0e7e --- /dev/null +++ b/evidence/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-10-24 15:36:50, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 69.7 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] +4 passed, 0 failed diff --git a/evidence/jobs-variants.txt b/evidence/jobs-variants.txt new file mode 100644 index 0000000..ab1edfb --- /dev/null +++ b/evidence/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-10-24 15:30:40, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py variants +# exit code: 0, 19.5 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..080d981 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.047, + "concurrency": 4, + "successful_requests": 7328, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 243.885, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 0000000..e2747c8 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-10-24 15:42:31, commit: 276dbda + uncommitted changes +# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.3 s + +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.047, + "concurrency": 4, + "successful_requests": 7328, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 243.885, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/attachments.txt b/evidence/native-sandbox/attachments.txt new file mode 100644 index 0000000..4168063 --- /dev/null +++ b/evidence/native-sandbox/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-10-24 14:39:50, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_attachments.py +# exit code: 0, 10.5 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 4fea67fd +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/native-sandbox/check.txt b/evidence/native-sandbox/check.txt new file mode 100644 index 0000000..e5baf60 --- /dev/null +++ b/evidence/native-sandbox/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-10-24 14:34:04, commit: 81d8f37 + uncommitted changes +# command: python3 scripts/check_contract.py +# exit code: 0, 0.0 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/native-sandbox/jobs-concurrency.txt b/evidence/native-sandbox/jobs-concurrency.txt new file mode 100644 index 0000000..0c61a2c --- /dev/null +++ b/evidence/native-sandbox/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-10-24 14:39:39, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py concurrency +# exit code: 0, 7.5 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-crash.txt b/evidence/native-sandbox/jobs-crash.txt new file mode 100644 index 0000000..3a4fba0 --- /dev/null +++ b/evidence/native-sandbox/jobs-crash.txt @@ -0,0 +1,15 @@ +# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease +# date: 2026-10-24 14:38:43, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py crash-after-put crash-after-commit staging lease +# exit code: 0, 143.5 s + +--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат. +worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] +--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack. +worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) +--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit). +staging source still present after the crash (worker died before cleanup); waiting 60 s window +janitor removed the staging source of the ready attachment: 1 +--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; +frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] +4 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-limits.txt b/evidence/native-sandbox/jobs-limits.txt new file mode 100644 index 0000000..3311283 --- /dev/null +++ b/evidence/native-sandbox/jobs-limits.txt @@ -0,0 +1,11 @@ +# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) +# date: 2026-10-24 14:35:26, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py limits +# exit code: 0, 69.1 s + +--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker. +10 MiB + 8 bytes -> 413 before any job is created +imaging.py on 8000x5000 PNG: exit 0, peak RSS 223.4 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large) +memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived +imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-outbox.txt b/evidence/native-sandbox/jobs-outbox.txt new file mode 100644 index 0000000..869700c --- /dev/null +++ b/evidence/native-sandbox/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-10-24 14:36:19, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py outbox-crash +# exit code: 0, 7.9 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-recovery.txt b/evidence/native-sandbox/jobs-recovery.txt new file mode 100644 index 0000000..d75c922 --- /dev/null +++ b/evidence/native-sandbox/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-10-24 14:36:11, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 45.9 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment 5b41ee9a-207b-46d6-a0af-635082295231 +message with attachment 201, content 409, other messages 201; job d62da8f8-b127-4141-95c3-8f691c39bb72 published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job aab8166b-e551-4ec9-9566-0f8207527e36 +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-retry.txt b/evidence/native-sandbox/jobs-retry.txt new file mode 100644 index 0000000..abbbfdd --- /dev/null +++ b/evidence/native-sandbox/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-10-24 14:39:32, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 48.8 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] +4 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-variants.txt b/evidence/native-sandbox/jobs-variants.txt new file mode 100644 index 0000000..6ddec84 --- /dev/null +++ b/evidence/native-sandbox/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-10-24 14:34:16, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py variants +# exit code: 0, 11.4 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/native-sandbox/persistence.txt b/evidence/native-sandbox/persistence.txt new file mode 100644 index 0000000..21a02fb --- /dev/null +++ b/evidence/native-sandbox/persistence.txt @@ -0,0 +1,16 @@ +# persistence (native sandbox): public tests/resilience.py --mode persistence with scripts/restart.sh +# date: 2026-10-24 14:40:30; run_persistence.py (docker inspect proof of restarts) needs real Docker + +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# exit code: 0 + +# processes after the hook (new PIDs = real restarts): +postgres running 2971 +redis running 2964 +rabbitmq running 3077 +s3 running 2988 +api running 3504 +dispatcher running 3518 +worker running 3522 +janitor running 3508 diff --git a/evidence/native-sandbox/race.txt b/evidence/native-sandbox/race.txt new file mode 100644 index 0000000..ec304c7 --- /dev/null +++ b/evidence/native-sandbox/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-10-24 14:39:54, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_race.py +# exit code: 0, 1.8 s + +sent 100 messages from 10 clients in 0.50 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/native-sandbox/smoke.txt b/evidence/native-sandbox/smoke.txt new file mode 100644 index 0000000..429532c --- /dev/null +++ b/evidence/native-sandbox/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test), with attachment +# date: 2026-10-24 14:34:05, commit: 81d8f37 + uncommitted changes +# command: python3 tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 0.8 s + +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/native-sandbox/student-tests.txt b/evidence/native-sandbox/student-tests.txt new file mode 100644 index 0000000..fba8591 --- /dev/null +++ b/evidence/native-sandbox/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-10-24 14:39:52, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_api.py +# exit code: 0, 2.6 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/native-sandbox/summary.md b/evidence/native-sandbox/summary.md new file mode 100644 index 0000000..eeebfe1 --- /dev/null +++ b/evidence/native-sandbox/summary.md @@ -0,0 +1,40 @@ +# Сводка проверок + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/native-sandbox/check.txt | +| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/native-sandbox/smoke.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 11 | evidence/native-sandbox/jobs-variants.txt | +| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 69 | evidence/native-sandbox/jobs-limits.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 46 | evidence/native-sandbox/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 8 | evidence/native-sandbox/jobs-outbox.txt | +| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 143 | evidence/native-sandbox/jobs-crash.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 49 | evidence/native-sandbox/jobs-retry.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 7 | evidence/native-sandbox/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 11 | evidence/native-sandbox/attachments.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/native-sandbox/student-tests.txt | +| race | lab2 races via the balancer | PASS | 2 | evidence/native-sandbox/race.txt | +| persistence | public persistence test (make test-persistence) on the native stand; docker-inspect proof needs Docker | PASS | 40 | evidence/native-sandbox/persistence.txt | + +# Где прогнано + +Нативный стенд без Docker (в среде сборки нет доступа к Docker Hub): PostgreSQL 16, RabbitMQ 3.12.1, SeaweedFS 3.80, Redis — процессами с той же конфигурацией и секретами, что в compose.yaml; 1 реплика API на 8080 без HAProxy; Python 3.11. Команды `docker compose ...` тестов выполнялись через шим, переводящий их на эти процессы. Для защиты повторить на Docker: `python student-tests/collect_evidence.py` (вывод в evidence/). + +# Отказы lab5 + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) | - | diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..d339ffb --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,30 @@ +# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence) +# date: 2026-10-24 15:44:25, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/run_persistence.py +# exit code: 0, 59.6 s + +before: container, ID, started at + messenger-lab5-api-1 b9ad6d33d190 2026-10-24T12:41:59.057757842Z + messenger-lab5-api-2 5aa14958c864 2026-10-24T12:41:38.920683556Z + messenger-lab5-dispatcher-1 9c5531a487f2 2026-10-24T12:32:58.22608557Z + messenger-lab5-janitor-1 c412a9460051 2026-10-24T12:23:20.245209318Z + messenger-lab5-lb-1 0760689bc54d 2026-10-24T12:23:27.648064757Z + messenger-lab5-postgres-1 10fec15d4ea6 2026-10-24T12:43:25.148756098Z + messenger-lab5-rabbitmq-1 95b4214594a8 2026-10-24T12:32:33.167311219Z + messenger-lab5-redis-1 e7ed9903c342 2026-10-24T12:43:04.620093059Z + messenger-lab5-s3-1 56a0cdfb9bc8 2026-10-24T12:39:05.007533298Z + messenger-lab5-worker-1 6e040d05805f 2026-10-24T12:37:00.256852184Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after (hook + waiting for readiness took 58.5 s): container, ID, started at, restarted? + messenger-lab5-api-1 b9ad6d33d190 2026-10-24T12:44:20.339074469Z same container: True, restarted: True + messenger-lab5-api-2 5aa14958c864 2026-10-24T12:44:20.477467705Z same container: True, restarted: True + messenger-lab5-dispatcher-1 9c5531a487f2 2026-10-24T12:44:17.516290602Z same container: True, restarted: True + messenger-lab5-janitor-1 c412a9460051 2026-10-24T12:44:16.494020858Z same container: True, restarted: True + messenger-lab5-lb-1 0760689bc54d 2026-10-24T12:44:23.831977922Z same container: True, restarted: True + messenger-lab5-postgres-1 10fec15d4ea6 2026-10-24T12:43:40.513609095Z same container: True, restarted: True + messenger-lab5-rabbitmq-1 95b4214594a8 2026-10-24T12:43:41.265182826Z same container: True, restarted: True + messenger-lab5-redis-1 e7ed9903c342 2026-10-24T12:43:40.396962627Z same container: True, restarted: True + messenger-lab5-s3-1 56a0cdfb9bc8 2026-10-24T12:43:50.186903251Z same container: True, restarted: True + messenger-lab5-worker-1 6e040d05805f 2026-10-24T12:44:17.661567668Z same container: True, restarted: True +PASS: the hook restarted postgres, redis, s3 (SeaweedFS), rabbitmq, all api replicas, janitor, dispatcher, worker and lb (same containers, new start time); data, the attachment bytes and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 0000000..cf88f2f --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-10-24 15:41:05, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_race.py +# exit code: 0, 1.2 s + +sent 100 messages from 10 clients in 0.20 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/redis.txt b/evidence/redis.txt new file mode 100644 index 0000000..a7995f1 --- /dev/null +++ b/evidence/redis.txt @@ -0,0 +1,14 @@ +# redis: Redis as the session store +# date: 2026-10-24 15:41:19, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/redis_info.py +# exit code: 0, 3.1 s + +session key: session:5dc10133a5554a2f...; value is the user id: True +TTL of the key, seconds: 3599 +sessions in Redis: 299 +config appendonly = yes +config appendfsync = always +config maxmemory-policy = noeviction +aof_enabled:1 +aof_last_write_status:ok +without password: NOAUTH Authentication required. diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt new file mode 100644 index 0000000..505cd3e --- /dev/null +++ b/evidence/s3-security.txt @@ -0,0 +1,47 @@ +# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap +# date: 2026-10-24 15:37:26, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_s3_security.py +# exit code: 0, 10.3 s + +anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous head_object directly to S3 (from the data network) -> 403 +anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied +anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied +the same file through the API for a chat member -> 200 +PASS: the bucket is private: no anonymous read, list, write or policy read +API key: list buckets -> ['messenger-attachments'] +API key: get own object -> allowed +API key: put object -> allowed +API key: delete object -> allowed +API key: list bucket -> allowed +API key: create another bucket -> denied: AccessDenied +API key: make bucket public (put policy) -> denied: NotImplemented +API key: read bucket policy -> denied: NoSuchBucketPolicy +API key: delete bucket -> denied: AccessDenied +API key: change versioning -> denied: NotImplemented +PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied +compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand) +127.0.0.1:8333 from the host -> connection refused +127.0.0.1:9333 from the host -> connection refused +127.0.0.1:8888 from the host -> connection refused +127.0.0.1:19333 from the host -> connection refused +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init +s3-init: s3-init: bucket messenger-attachments exists +s3-init: s3-init: api key can put/get/list/delete objects of its bucket +s3-init: s3-init: api key cannot create buckets (AccessDenied) +s3-init: s3-init: bucket is private (anonymous list/get/put -> [403]) +after the repeated bootstrap: old file downloads, new upload works +PASS: bucket/user/policy bootstrap can be repeated safely +4 passed, 0 failed diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..1e91652 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test), with attachment +# date: 2026-10-24 15:30:21, commit: 276dbda + uncommitted changes +# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 1.0 s + +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 0000000..6a29a62 --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,49 @@ +# stack: stand layout: replicas, ports, limits, networks, secrets, S3 +# date: 2026-10-24 15:30:20, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/check_stack.py +# exit code: 0, 3.0 s + +host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 +Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop + +SERVICE CONTAINER ID STATUS PORTS +api b9ad6d33d190 Up 4 minutes (healthy) 8080/tcp +api 5aa14958c864 Up 4 minutes (healthy) 8080/tcp +dispatcher 9c5531a487f2 Up 3 minutes (healthy) +janitor c412a9460051 Up 6 minutes (healthy) +lb 0760689bc54d Up 6 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +migrate 72d1affe9c8e Exited (0) 3 seconds ago +postgres 10fec15d4ea6 Up 7 minutes (healthy) +rabbitmq 95b4214594a8 Up 4 minutes (healthy) +redis e7ed9903c342 Up 7 minutes (healthy) +s3 56a0cdfb9bc8 Up 7 minutes (healthy) +s3-init 39d4358fe2ba Exited (0) 2 seconds ago +worker 6e040d05805f Up 3 minutes (healthy) + +container image cpus memory host ports networks +messenger-lab5-api-1 messenger-lab5-api 1.00 512 MiB - data,edge +messenger-lab5-api-2 messenger-lab5-api 1.00 512 MiB - data,edge +messenger-lab5-dispatcher-1 messenger-lab5-dispatcher 0.25 128 MiB - data +messenger-lab5-janitor-1 messenger-lab5-janitor 0.25 192 MiB - data +messenger-lab5-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge +messenger-lab5-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data +messenger-lab5-rabbitmq-1 rabbitmq:4.1.4-alpine 1.00 512 MiB - data +messenger-lab5-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data +messenger-lab5-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data +messenger-lab5-worker-1 messenger-lab5-worker 2.00 1536 MiB - data + +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] + +through http://127.0.0.1:8080 one session is served by 2 replicas: ['5aa14958c864', 'b9ad6d33d190'] +/health/ready -> 200 from b9ad6d33d190 +PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor/rabbitmq/worker/dispatcher only in the internal data network, S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 0000000..fee036b --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-10-24 15:41:04, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_api.py +# exit code: 0, 2.7 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/summary.md b/evidence/summary.md new file mode 100644 index 0000000..71fc1f1 --- /dev/null +++ b/evidence/summary.md @@ -0,0 +1,78 @@ +# Сводка проверок + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/check.txt | +| stack | stand layout: replicas, ports, limits, networks, secrets, S3 | PASS | 3 | evidence/stack.txt | +| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/smoke.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 20 | evidence/jobs-variants.txt | +| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 74 | evidence/jobs-limits.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 55 | evidence/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 15 | evidence/jobs-outbox.txt | +| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 156 | evidence/jobs-crash.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 70 | evidence/jobs-retry.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 15 | evidence/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 10 | evidence/attachments.txt | +| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt | +| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 156 | evidence/consistency.txt | +| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 54 | evidence/upload-memory.txt | +| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt | +| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt | +| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 11 | evidence/db.txt | +| redis | Redis as the session store | PASS | 3 | evidence/redis.txt | +| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt | +| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 41 | evidence/failover.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| failures | Redis and PostgreSQL outages via the balancer | PASS | 55 | evidence/failures.txt | +| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 60 | evidence/persistence.txt | + +# Замеры + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-24, commit 276dbda + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 5aa14958c864 | 2 | 123 | 0 | 4.406 | +| 2 | b9ad6d33d190 | 2 | 125 | 0 | 4.188 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.45 / 1.56 | 5aa14958c864: 5 | 96.0 | 109.7 | 13.7 | 149.6 | +| | | | | b9ad6d33d190: 5 | 94.5 | 108.9 | 14.4 | 142.1 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.04 / 1.31 | 5aa14958c864: 5 | 96.6 | 97.6 | 1.0 | 89.6 | +| | | | | b9ad6d33d190: 5 | 96.2 | 98.6 | 2.3 | 88.9 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 4.96 / 5.2 | 5aa14958c864: 10 | 94.7 | 114.9 | 20.3 | 155.4 | +| | | | | b9ad6d33d190: 10 | 97.0 | 121.8 | 24.8 | 176.9 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 243.885 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.406, 4.188 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 58.5 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19922 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14860 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..4ba7f24 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,124 @@ +{ + "date": "2026-10-24 15:40", + "api_limit_mib": 512, + "api_limit_cpus": 1.0, + "runs": [ + { + "scenario": "10 x ~9.5 MiB PNG (noise)", + "concurrency": 10, + "file_bytes": [ + 9953047 + ], + "total_mib": 94.9, + "all_202": true, + "wall_seconds": 1.56, + "latency_s": { + "p50": 1.45, + "max": 1.56 + }, + "sampler_rounds": 9, + "replicas": [ + { + "replica": "5aa14958c864", + "uploads_served": 5, + "rss_before_mib": 96.0, + "rss_peak_mib": 109.7, + "rss_growth_mib": 13.7, + "rss_peak_sampled_mib": 109.7, + "cgroup_peak_mib": 149.6, + "cgroup_peak_sampled_mib": 145.2 + }, + { + "replica": "b9ad6d33d190", + "uploads_served": 5, + "rss_before_mib": 94.5, + "rss_peak_mib": 108.9, + "rss_growth_mib": 14.4, + "rss_peak_sampled_mib": 108.9, + "cgroup_peak_mib": 142.1, + "cgroup_peak_sampled_mib": 139.3 + } + ] + }, + { + "scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", + "concurrency": 10, + "file_bytes": [ + 116714 + ], + "total_mib": 1.1, + "all_202": true, + "wall_seconds": 1.31, + "latency_s": { + "p50": 1.04, + "max": 1.31 + }, + "sampler_rounds": 8, + "replicas": [ + { + "replica": "5aa14958c864", + "uploads_served": 5, + "rss_before_mib": 96.6, + "rss_peak_mib": 97.6, + "rss_growth_mib": 1.0, + "rss_peak_sampled_mib": 97.6, + "cgroup_peak_mib": 89.6, + "cgroup_peak_sampled_mib": 89.2 + }, + { + "replica": "b9ad6d33d190", + "uploads_served": 5, + "rss_before_mib": 96.2, + "rss_peak_mib": 98.6, + "rss_growth_mib": 2.3, + "rss_peak_sampled_mib": 98.6, + "cgroup_peak_mib": 88.9, + "cgroup_peak_sampled_mib": 86.2 + } + ] + }, + { + "scenario": "20 mixed: large + 40 Mpx + JPEG", + "concurrency": 20, + "file_bytes": [ + 1492, + 116714, + 9953047 + ], + "total_mib": 133.2, + "all_202": true, + "wall_seconds": 5.2, + "latency_s": { + "p50": 4.96, + "max": 5.2 + }, + "sampler_rounds": 27, + "replicas": [ + { + "replica": "5aa14958c864", + "uploads_served": 10, + "rss_before_mib": 94.7, + "rss_peak_mib": 114.9, + "rss_growth_mib": 20.3, + "rss_peak_sampled_mib": 114.9, + "cgroup_peak_mib": 155.4, + "cgroup_peak_sampled_mib": 154.4 + }, + { + "replica": "b9ad6d33d190", + "uploads_served": 10, + "rss_before_mib": 97.0, + "rss_peak_mib": 121.8, + "rss_growth_mib": 24.8, + "rss_peak_sampled_mib": 121.8, + "cgroup_peak_mib": 176.9, + "cgroup_peak_sampled_mib": 173.1 + } + ] + } + ], + "single_file_extra_rss_mib": { + "naive": 161, + "streaming": 0 + } +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 0000000..ac7213b --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,13 @@ +# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas +# date: 2026-10-24 15:40:56, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json +# exit code: 0, 53.8 s + +client: Python 3.11.1 on Windows 10 +preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)... +large: 9953047 bytes; 40 Mpx: 116714 bytes +{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_202": true, "wall_seconds": 1.56, "latency_s": {"p50": 1.45, "max": 1.56}, "sampler_rounds": 9, "replicas": [{"replica": "5aa14958c864", "uploads_served": 5, "rss_before_mib": 96.0, "rss_peak_mib": 109.7, "rss_growth_mib": 13.7, "rss_peak_sampled_mib": 109.7, "cgroup_peak_mib": 149.6, "cgroup_peak_sampled_mib": 145.2}, {"replica": "b9ad6d33d190", "uploads_served": 5, "rss_before_mib": 94.5, "rss_peak_mib": 108.9, "rss_growth_mib": 14.4, "rss_peak_sampled_mib": 108.9, "cgroup_peak_mib": 142.1, "cgroup_peak_sampled_mib": 139.3}]} +{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_202": true, "wall_seconds": 1.31, "latency_s": {"p50": 1.04, "max": 1.31}, "sampler_rounds": 8, "replicas": [{"replica": "5aa14958c864", "uploads_served": 5, "rss_before_mib": 96.6, "rss_peak_mib": 97.6, "rss_growth_mib": 1.0, "rss_peak_sampled_mib": 97.6, "cgroup_peak_mib": 89.6, "cgroup_peak_sampled_mib": 89.2}, {"replica": "b9ad6d33d190", "uploads_served": 5, "rss_before_mib": 96.2, "rss_peak_mib": 98.6, "rss_growth_mib": 2.3, "rss_peak_sampled_mib": 98.6, "cgroup_peak_mib": 88.9, "cgroup_peak_sampled_mib": 86.2}]} +{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_202": true, "wall_seconds": 5.2, "latency_s": {"p50": 4.96, "max": 5.2}, "sampler_rounds": 27, "replicas": [{"replica": "5aa14958c864", "uploads_served": 10, "rss_before_mib": 94.7, "rss_peak_mib": 114.9, "rss_growth_mib": 20.3, "rss_peak_sampled_mib": 114.9, "cgroup_peak_mib": 155.4, "cgroup_peak_sampled_mib": 154.4}, {"replica": "b9ad6d33d190", "uploads_served": 10, "rss_before_mib": 97.0, "rss_peak_mib": 121.8, "rss_growth_mib": 24.8, "rss_peak_sampled_mib": 121.8, "cgroup_peak_mib": 176.9, "cgroup_peak_sampled_mib": 173.1}]} +one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB +PASS: all uploads succeeded; largest RSS growth of one replica: 24.8 MiB (limit 512 MiB); results in evidence/upload-memory.json diff --git a/faults.py b/faults.py new file mode 100644 index 0000000..2bb92a7 --- /dev/null +++ b/faults.py @@ -0,0 +1,51 @@ +"""Учебное внедрение сбоев для тестов отказов (только при FAULT_INJECTION=1). + +Тест создаёт в общем volume /faults файлы <точка>., по одному на срабатывание: + + docker compose exec -T worker sh -c 'touch /faults/worker_crash_after_put.1' + +Процесс, дошедший до точки, атомарно удаляет один файл (os.remove успешен ровно у одного процесса) +и выполняет сбой. Так сбой случается заданное число раз и не зацикливает перезапуски. + +Точки: + dispatcher_crash_before_publish — COMMIT задания уже был, publish ещё нет (окно COMMIT -> publish) + dispatcher_crash_after_confirm — брокер подтвердил публикацию, отметка в outbox ещё не записана + worker_crash_after_put — объекты записаны в S3, COMMIT результата ещё нет + worker_crash_after_commit — COMMIT результата был, ack ещё нет + worker_freeze — worker «завис»: не продлевает аренду и спит 3 x lease + worker_s3_error — временная ошибка S3 при записи результата + imaging_hang / imaging_invalid / imaging_oom — сбой внутри процесса обработки изображения +""" +import logging +import os +import sys +from pathlib import Path + +import settings + +log = logging.getLogger("messenger.faults") + + +def hit(point: str) -> bool: + if not settings.FAULT_INJECTION: + return False + directory = Path(settings.FAULTS_DIR) + try: + candidates = sorted(directory.glob(point + ".*")) + except OSError: + return False + for path in candidates: + try: + path.unlink() + except FileNotFoundError: + continue # этот файл забрал другой процесс + log.warning("FAULT INJECTION: %s", point) + return True + return False + + +def crash(point: str) -> None: + """Аварийное завершение всего процесса без очистки (как SIGKILL/OOM): без ack, без закрытия соединений.""" + if hit(point): + sys.stderr.flush() + os._exit(137) diff --git a/imaging.py b/imaging.py new file mode 100644 index 0000000..a08fbb0 --- /dev/null +++ b/imaging.py @@ -0,0 +1,209 @@ +"""Преобразование изображения, pipeline_version=1. Запускается worker'ом ОТДЕЛЬНЫМ процессом с лимитами. + + python imaging.py + +Почему отдельный процесс: декодер Pillow работает в C, его нельзя прервать изнутри Python и нельзя +ограничить по памяти отдельно от остального worker. Процесс сам ставит себе RLIMIT_AS (память, +IMAGING_MEMORY_MB), RLIMIT_CPU (процессорное время) и RLIMIT_FSIZE до импорта Pillow, а worker +убивает его по таймауту JOB_TIMEOUT_SECONDS. +Падение, зависание или нехватка памяти здесь не роняют worker и не ломают подключение к брокеру. + +Результат: файлы thumbnail.jpg, crop.jpg, optimized.webp в out_dir и одна строка JSON в stdout. +Код выхода: 0 — успех; 3 — невалидное изображение; 4 — слишком много пикселей; 5 — не хватило памяти; +прочее (в том числе убит сигналом) — неожиданная ошибка, worker считает её временной. + +Правила вариантов (contracts/README.md): +- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, размеры округляются вниз + (минимум 1 px), EXIF orientation применяется, прозрачность накладывается на белый фон. 320x200 -> 256x160. +- crop: JPEG 128x128, центрированный квадрат (маленькие исходники увеличиваются). +- optimized: WebP, вписан в 1280x1280 без увеличения, quality=80, method=4; прозрачность сохраняется. +Во всех вариантах нет EXIF/XMP/комментариев: в save() передаётся только ICC-профиль (цвет), остальное +отбрасывается. Pillow при одинаковой версии даёт побайтно одинаковый результат, поэтому повтор job +пишет в те же ключи S3 те же байты. +""" +import json +import os +import resource +import sys +import time + + +def apply_limits() -> None: + """Лимиты на себя, ДО импорта Pillow: память (RLIMIT_AS), CPU, размер записываемых файлов.""" + memory = int(os.environ.get("IMAGING_MEMORY_MB", "640")) * 1024 * 1024 + cpu = int(os.environ.get("IMAGING_CPU_SECONDS", "60")) + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024)) + + +if __name__ == "__main__": + apply_limits() + +from PIL import Image # noqa: E402 — после apply_limits: память декодера тоже под лимитом + +PIPELINE_VERSION = 1 +THUMB_BOX = 256 +CROP_SIZE = 128 +OPTIMIZED_BOX = 1280 +THUMB_QUALITY = 85 +CROP_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +EXIT_INVALID = 3 +EXIT_TOO_LARGE = 4 +EXIT_MEMORY = 5 + +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +# EXIF orientation -> преобразование (как в PIL.ImageOps.exif_transpose) +ORIENTATION = { + 2: [Image.Transpose.FLIP_LEFT_RIGHT], + 3: [Image.Transpose.ROTATE_180], + 4: [Image.Transpose.FLIP_TOP_BOTTOM], + 5: [Image.Transpose.TRANSPOSE], + 6: [Image.Transpose.ROTATE_270], + 7: [Image.Transpose.TRANSVERSE], + 8: [Image.Transpose.ROTATE_90], +} + + +class TooLarge(Exception): + pass + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + """Вписать в box x box без увеличения; размеры вниз, минимум 1 px. Целочисленно, без float.""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def orientation_of(image: Image.Image) -> int: + try: + value = image.getexif().get(0x0112, 1) + except Exception: # noqa: BLE001 — битый EXIF не делает картинку невалидной, ориентацию не трогаем + return 1 + return value if value in ORIENTATION else 1 + + +def normalize(image: Image.Image) -> Image.Image: + """Любой режим PNG/JPEG -> L, RGB или RGBA (RGBA только если есть прозрачность).""" + mode = image.mode + transparent = "transparency" in image.info + if mode in ("I;16", "I;16B", "I;16L", "I"): + image = image.convert("I").point(lambda value: value * (1 / 256)).convert("L") + mode = "L" + if mode == "1": + return image.convert("L") + if mode in ("P", "PA"): + return image.convert("RGBA" if transparent or mode == "PA" else "RGB") + if mode in ("LA", "La", "RGBa"): + return image.convert("RGBA") + if mode in ("L", "RGB") and transparent: + return image.convert("RGBA") + if mode in ("L", "RGB", "RGBA"): + return image + return image.convert("RGB") # CMYK, YCbCr, LAB, ... + + +def on_white(image: Image.Image) -> Image.Image: + """Прозрачность -> белый фон (для JPEG).""" + if image.mode != "RGBA": + return image + background = Image.new("RGB", image.size, (255, 255, 255)) + background.paste(image, mask=image.getchannel("A")) + return background + + +def process(source: str, out_dir: str, content_type: str, max_pixels: int) -> dict: + Image.MAX_IMAGE_PIXELS = max_pixels + with Image.open(source) as image: + if image.format != FORMATS.get(content_type): + raise ValueError("format does not match stored content type") + width, height = image.size + if width * height > max_pixels: + raise TooLarge() + orientation = orientation_of(image) + icc = image.info.get("icc_profile") + swap = orientation in (5, 6, 7, 8) + oriented = (height, width) if swap else (width, height) + + # Сначала уменьшаем до optimized (<= 1280), потом поворачиваем: квадратные рамки 1280/256/128 + # не зависят от поворота, а поворот маленькой картинки дешевле, чем копия 40-мегапиксельной. + target = fit(*oriented, OPTIMIZED_BOX) + stored_target = (target[1], target[0]) if swap else target + if image.format == "JPEG": + image.draft(None, stored_target) # декодирование сразу в масштабе 1/2..1/8, если хватает + image.load() + work = normalize(image) + if work.size != stored_target: + work = work.resize(stored_target, Image.Resampling.LANCZOS, reducing_gap=3.0) + elif work is image: + work = work.copy() + for step in ORIENTATION.get(orientation, []): + work = work.transpose(step) + + extra = {"icc_profile": icc} if icc else {} + results = [] + + optimized = work if work.mode != "L" else work.convert("RGB") + path = os.path.join(out_dir, "optimized.webp") + optimized.save(path, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, **extra) + results.append({"name": "optimized", "content_type": "image/webp", "width": optimized.width, + "height": optimized.height, "file": "optimized.webp"}) + + thumb_size = fit(*oriented, THUMB_BOX) # от размеров ИСХОДНИКА, а не промежуточной копии + thumb = work if work.size == thumb_size else work.resize(thumb_size, Image.Resampling.LANCZOS) + thumb = on_white(thumb) + path = os.path.join(out_dir, "thumbnail.jpg") + thumb.save(path, format="JPEG", quality=THUMB_QUALITY, optimize=True, **extra) + results.append({"name": "thumbnail", "content_type": "image/jpeg", "width": thumb.width, + "height": thumb.height, "file": "thumbnail.jpg"}) + + side = min(work.size) + left, top = (work.width - side) // 2, (work.height - side) // 2 + crop = work.crop((left, top, left + side, top + side)) + crop = on_white(crop.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS)) + path = os.path.join(out_dir, "crop.jpg") + crop.save(path, format="JPEG", quality=CROP_QUALITY, optimize=True, **extra) + results.append({"name": "crop", "content_type": "image/jpeg", "width": CROP_SIZE, "height": CROP_SIZE, + "file": "crop.jpg"}) + + return {"pipeline_version": PIPELINE_VERSION, "source": {"width": width, "height": height, + "orientation": orientation}, "variants": results} + + +def injected_fault() -> None: + """Учебные сбои (FAULT_IMAGING задаёт worker, только при FAULT_INJECTION=1).""" + fault = os.environ.get("FAULT_IMAGING", "") + if fault == "hang": + time.sleep(3600) + elif fault == "invalid": + raise ValueError("injected invalid image") + elif fault == "oom": + blocks = [] + while True: + blocks.append(bytearray(64 * 1024 * 1024)) + + +def main() -> int: + source, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) + try: + injected_fault() + result = process(source, out_dir, content_type, max_pixels) + except (TooLarge, Image.DecompressionBombError): + return EXIT_TOO_LARGE + except MemoryError: + return EXIT_MEMORY + except (OSError, SyntaxError, ValueError, IndexError, TypeError, EOFError, Image.UnidentifiedImageError): + # Pillow сообщает о битом файле разными исключениями; подробности не нужны ни клиенту, ни логу. + return EXIT_INVALID + print(json.dumps(result), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..1605eed --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,77 @@ +# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики). +# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить +# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky). + +global + log stdout format raw local0 info + maxconn 4096 + # При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с. + hard-stop-after 10s + +# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик. +# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s # перечитывать список реплик раз в 2 с + hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlog-normal # в лог только ошибки, а не каждый запрос + timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос + timeout client 30s + # Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка + # при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт, + # сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT). + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода. + # Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key. + retries 2 + option redispatch 1 + +frontend api_in + bind :8080 + option forwardfor + default_backend api + +backend api + balance roundrobin + # Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis. + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT. + # Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP. + # observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки. + server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + # Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis + # или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате + # контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает. + # Пока готова хоть одна реплика, эти записи не используются. + # allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend). + server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + option allbackups + +# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1). +# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда. +frontend stats + bind :8404 + monitor-uri /lb-health + monitor fail if { nbsrv(api) lt 1 } + stats enable + stats uri /stats + stats refresh 2s diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 0000000..f4f41e0 --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,22 @@ +# RabbitMQ 4.1 (сервис rabbitmq). Топология, пользователи и права — в secrets/rabbitmq_definitions.json +# (генерирует scripts/init_secrets.py), файл импортируется при каждом старте узла. Пользователя guest нет: +# при импорте определений на старте пользователь по умолчанию не создаётся. +definitions.import_backend = local_filesystem +definitions.local.path = /run/secrets/rabbitmq_definitions +definitions.skip_if_unchanged = true + +listeners.tcp.default = 5672 +loopback_users = none + +# Ресурсы: при нехватке памяти/диска брокер блокирует издателей (dispatcher ждёт и повторяет), +# сообщения не теряются. Абсолютное значение: ниже лимита контейнера 512M (доля считалась бы от RAM хоста). +vm_memory_high_watermark.absolute = 320MB +disk_free_limit.absolute = 256MB + +# Consumer, не приславший ack за это время, считается зависшим: канал закрывается, сообщение +# возвращается в очередь. Должно быть больше JOB_TIMEOUT_SECONDS + времени записи в S3. +consumer_timeout = 300000 + +log.console = true +log.console.level = info +log.file = false diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 0000000..33c8c08 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,13 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +appendonly yes +appendfsync always +save "" + +maxmemory 64mb +maxmemory-policy noeviction + +include /run/secrets/redis_auth diff --git a/janitor.py b/janitor.py new file mode 100644 index 0000000..559c87d --- /dev/null +++ b/janitor.py @@ -0,0 +1,278 @@ +"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor). + + python janitor.py --once [--dry-run] [--grace-seconds N] + +Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock): + +1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен + в очередь. Удалить объект staging (идемпотентно), затем строку. +2. Staging после безопасной фиксации: + - ready: исходник уже опубликован в attachments//v/original -> удалить staging-объект + (обычно это делает worker сразу после ack; здесь — если он не успел); + - failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется. + queued/processing не трогаются никогда: это текущая работа. +3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются. +4. Обход bucket по всем prefix (объекты старше grace): + - staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key; + - attachments//...: сирота, если вложения нет, оно failed, или оно ready, но ключ не входит + в его опубликованный набор (оригинал + варианты). У queued/processing объекты не трогаются: + их перезапишет следующая попытка теми же ключами. + Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных. + +Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому +janitor не удаляет то, что ещё пишется. +""" +import argparse +import json +import logging +import re +import signal +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import pipeline +import settings +import storage + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.janitor") + +JANITOR_LOCK = 20260404 +BATCH = 100 +STOP = threading.Event() +ACTOR = "janitor" +FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/") + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def idle(seconds: float) -> bool: + """Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки.""" + deadline = time.monotonic() + seconds + while (left := deadline - time.monotonic()) > 0: + touch_alive() + if STOP.wait(min(left, 10)): + return True + return False + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]: + """Шаг 1. Возвращает (найдено, удалено).""" + found = deleted = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, source_key FROM attachments + WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + FOR UPDATE SKIP LOCKED""", + (grace, BATCH), + ).fetchall() + found += len(rows) + if dry_run or not rows: + return found, deleted + for row in rows: + storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],)) + deleted += 1 + log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"]) + if len(rows) < BATCH: + return found, deleted + return found, deleted + + +def cleanup_staging(conn, dry_run: bool) -> dict: + """Шаг 2. Исходники ready (сразу) и failed (после retention).""" + report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0} + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a + LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.source_key IS NOT NULL + AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds') + OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s))) + ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""", + (settings.STAGING_RETENTION_SECONDS, BATCH), + ).fetchall() + if dry_run or not rows: + if dry_run: + report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows) + report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows) + break + for row in rows: + storage.delete(row["source_key"]) + conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],)) + if row["state"] == "ready": + report["staging_ready_deleted"] += 1 + else: + report["staging_failed_expired"] += 1 + if row["job_id"] is not None: + pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR, + "replay no longer possible") + if len(rows) < BATCH: + break + report["staging_kept_for_replay"] = conn.execute( + "SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"] + return report + + +def cleanup_outbox(conn, dry_run: bool) -> int: + if dry_run: + return conn.execute( + """SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"] + return conn.execute( + """DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).rowcount + + +def classify_final(conn, keys: list[str]) -> set[str]: + """Какие ключи attachments//... не принадлежат ни одному живому вложению (кандидаты в сироты).""" + ids = {} + for key in keys: + match = FINAL_KEY_RE.match(key) + if match: + try: + ids[key] = UUID(match.group(1)) + except ValueError: + pass + unique = list(set(ids.values())) + rows = {r["id"]: r for r in conn.execute( + """SELECT a.id, a.state, a.object_key, + ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()} + orphans = set() + for key in keys: + attachment_id = ids.get(key) + row = rows.get(attachment_id) if attachment_id else None + if row is None or row["state"] == "failed": + orphans.add(key) + elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]: + orphans.add(key) + return orphans + + +def sweep_objects(conn, grace: int, dry_run: bool) -> dict: + """Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000.""" + started = datetime.now(timezone.utc) + cutoff = started - timedelta(seconds=grace) + scanned = orphans = orphans_deleted = 0 + listed = set() + + def orphan_keys(prefix, items): + keys = [key for key, _ in items] + if prefix == storage.FINAL_PREFIX: + return classify_final(conn, keys) + known = {r["source_key"] for r in conn.execute( + "SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()} + return {key for key in keys if key not in known} + + def still_orphan(prefix, key): + if prefix == storage.FINAL_PREFIX: + return key in classify_final(conn, [key]) + return conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is None + + def check(prefix, items): + nonlocal orphans, orphans_deleted + candidates = orphan_keys(prefix, items) + for key, modified in items: + if key not in candidates or modified >= cutoff: + continue + orphans += 1 + if not dry_run and still_orphan(prefix, key): # повторная проверка прямо перед удалением + storage.delete(key) + orphans_deleted += 1 + log.info("deleted orphan object %s", key) + + for prefix in storage.ALL_PREFIXES: + page = [] + for key, modified, _size in storage.list_objects(prefix): + scanned += 1 + listed.add(key) + page.append((key, modified)) + if len(page) >= 1000: + check(prefix, page) + page = [] + if page: + check(prefix, page) + + missing = [] + for r in conn.execute( + """SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall(): + if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]): + missing.append(str(r["id"])) + if missing: + log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s", + ", ".join(missing[:20])) + return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted, + "missing_objects": len(missing)} + + +def run_once(grace: int, dry_run: bool) -> dict | None: + started = time.monotonic() + with connect() as conn: + if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]: + log.info("another janitor is running, skip") + return None + try: + found, deleted = reclaim_stale_intents(conn, grace, dry_run) + report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found, + "stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run), + "outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)} + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,)) + report["seconds"] = round(time.monotonic() - started, 3) + return report + + +def main(): + parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects") + parser.add_argument("--once", action="store_true", help="one pass and exit") + parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing") + parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS) + args = parser.parse_args() + if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS: + parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} " + "(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted") + + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + while True: + touch_alive() + try: + report = run_once(args.grace_seconds, args.dry_run) + if report is not None: + print(json.dumps(report), flush=True) + failed = False + except (psycopg.Error, storage.StorageUnavailable) as exc: + log.warning("pass failed, will retry: %s", type(exc).__name__) + failed = True + if args.once: + raise SystemExit(1 if failed else 0) + if idle(settings.JANITOR_INTERVAL_SECONDS): + return + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py new file mode 100644 index 0000000..bacd111 --- /dev/null +++ b/main.py @@ -0,0 +1,603 @@ +import base64 +import binascii +import hashlib +import json +import logging +import re +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path +from typing import Literal +from urllib.parse import quote +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +import attachments +import auth +import db +import settings +import storage +import uploads +from errors import ApiError, bad_request, conflict, not_found, unauthorized, unavailable + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger") + +DRAINING = threading.Event() + + +class Server(uvicorn.Server): + """Остановка по SIGTERM в два шага. + + 1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS + HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы. + 2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы + (не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM, + получал бы 502 от балансировщика. + Повторный сигнал останавливает сразу. + """ + + def handle_exit(self, sig, frame): + if DRAINING.is_set(): + return super().handle_exit(sig, frame) + DRAINING.set() + log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s", + sig, settings.SHUTDOWN_DRAIN_SECONDS) + threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings.check() + # Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему. + await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS) + # Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3. + Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True) + db.pool.open() + log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s, " + "processing via outbox (pipeline v%s)", settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, + settings.SESSION_TTL_SECONDS, settings.S3_BUCKET, settings.PIPELINE_VERSION) + yield + db.pool.close() + auth.close() + log.info("Stopped: connections closed, data stays in PostgreSQL and Redis") + + +app = FastAPI(lifespan=lifespan) + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH) + + @field_validator("display_name", "password") + @classmethod + def validate_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("value cannot be blank") + return value + + +class User(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(BaseModel): + token: str + token_type: str + expires_at: datetime + user: User + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title cannot be empty") + return value + + +class Chat(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class ChatPage(BaseModel): + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + # С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("attachment_ids must be unique") + if not self.attachment_ids and not self.text.strip(): + raise ValueError("text cannot be empty without attachments") + return self + + +class Message(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: list[UUID] + + +class Variant(BaseModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + +class MessagePage(BaseModel): + items: list[Message] + next_cursor: str | None + + +def error_response( + request: Request, + code: str, + message: str, + status_code: int, + headers: dict | None = None, +): + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers={"X-Request-Id": request_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + return error_response( + request, + "invalid_request", + "Invalid request data", + 400, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Not found", 404) + return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def general_exception_handler( + request: Request, + exc: Exception, +): + return error_response( + request, + "internal_error", + "Internal server error", + 500, + ) + +PUBLIC_ENDPOINTS = { + ("POST", "/api/v1/users"), + ("POST", "/api/v1/auth/sessions"), +} + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + path = request.url.path + + try: + if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS: + request.state.session = await run_in_threadpool( + auth.check_bearer, request.headers.get("Authorization") + ) + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + except Exception: + log.exception("Unhandled error, request_id=%s", request_id) + response = error_response(request, "internal_error", "Internal server error", 500) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила + # Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки. + level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO + log.log( + level, + "%s %s -> %s (%.1f ms) request_id=%s", + request.method, path, response.status_code, + (time.perf_counter() - started) * 1000, request_id, + ) + return response + + +def current_user_id(request: Request) -> UUID: + session = getattr(request.state, "session", None) + if session is None: + raise unauthorized("Bearer token is required") + return session.user_id + + +def encode_cursor( + created_at: datetime, + object_id: UUID, + *, + chat_id: UUID | None = None, + user_id: UUID | None = None, +) -> str: + data = { + "created_at": created_at.isoformat(), + "id": str(object_id), + } + + if chat_id is not None: + data["chat_id"] = str(chat_id) + + if user_id is not None: + data["user_id"] = str(user_id) + + raw = json.dumps(data).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode_cursor(cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + data = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(data["created_at"]) + + if created_at.tzinfo is None: + raise ValueError("Invalid cursor") + + if not isinstance(data["id"], str): + raise ValueError("Invalid cursor") + + object_id = UUID(data["id"]) + + chat_id = UUID(data["chat_id"]) if "chat_id" in data else None + user_id = UUID(data["user_id"]) if "user_id" in data else None + + return created_at, object_id, chat_id, user_id + + except ( + ValueError, + KeyError, + TypeError, + AttributeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("Invalid cursor") + + +def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): + if cursor is None: + return None + try: + created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) + except ValueError: + raise bad_request("Invalid cursor") from None + if cursor_chat_id != chat_id or cursor_user_id != user_id: + raise bad_request("Invalid cursor") + return created_at, object_id + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + if DRAINING.is_set(): + raise unavailable("Instance is shutting down") + db.ping() + auth.ping_redis() + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=User, status_code=201) +def create_user(user_data: CreateUser): + password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется + return db.create_user(user_data.username, user_data.display_name, password_hash) + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +def create_session( + response: Response, + authorization: str | None = Header(default=None), +): + credentials = auth.parse_basic(authorization) + if credentials is None: + raise auth.login_failed() + username, password = credentials + + found = None + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: + found = db.get_user_with_password(username) + if not auth.password_matches(found["password_hash"] if found else None, password): + raise auth.login_failed() + + token, expires_at = auth.create_session(found["id"]) + response.headers["Cache-Control"] = "no-store" + user = {key: found[key] for key in ("id", "username", "display_name", "created_at")} + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + current_user_id(request) + auth.delete_session(request.state.session) + return Response(status_code=204) + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(request: Request): + user = db.get_user(current_user_id(request)) + if user is None: + raise unauthorized("Session is invalid or expired") + return user + + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, chat_data: CreateChat): + user_id = current_user_id(request) + + if user_id not in chat_data.member_ids: + raise bad_request("Creator must be a chat member") + + if len(chat_data.member_ids) != len(set(chat_data.member_ids)): + raise bad_request("Duplicate member IDs") + return db.create_chat(chat_data.title, chat_data.member_ids) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, user_id=user_id) + + rows = db.list_chats(user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id) + + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID): + return db.get_chat(chat_id, current_user_id(request)) + + +def request_fingerprint(data: BaseModel) -> bytes: + """SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют. + + Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []} + равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними. + """ + body = data.model_dump(mode="json") + if not body.get("attachment_ids"): + body.pop("attachment_ids", None) + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + response_model=Message, + status_code=201, +) +def create_message( + request: Request, + message_data: CreateMessage, + chat_id: UUID, + idempotency_key: str | None = Header(default=None), +): + # С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику) + # вернёт то же сообщение, а не создаст второе. См. db.add_message. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -") + return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids, + idempotency_key, request_fingerprint(message_data)) + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +def list_messages( + request: Request, + chat_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, chat_id=chat_id) + + rows = db.list_messages(chat_id, user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id) + + return MessagePage(items=items, next_cursor=next_cursor) + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=Attachment, + status_code=202, +) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: str | None = Query(default=None), +): + """Приём PNG/JPEG сырыми байтами. 202 queued — когда исходник в staging (S3), а задание и сообщение + для брокера зафиксированы в PostgreSQL одной транзакцией (attachments.store). Обработку делает worker. + + Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415) + -> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413), + проверка содержимого (415/413) и запись. + """ + user_id = current_user_id(request) + try: + await run_in_threadpool(db.check_member, chat_id, user_id) + name = uploads.validate_filename(filename) + declared = uploads.declared_type(request.headers.get("content-type")) + uploads.declared_length(request.headers.get("content-length")) + except ApiError: + await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке + raise + upload = await uploads.receive(request) + try: + image = await run_in_threadpool(uploads.inspect, upload, declared) + row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image) + finally: + upload.file.close() # временный файл удаляется сразу; исходник остаётся только в staging (S3) + return attachments.public(row) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public(db.get_attachment(attachment_id, current_user_id(request))) + + +VARIANT_SUFFIX = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp"} + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + request: Request, + attachment_id: UUID, + variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(default="original"), +): + """Байты оригинала или варианта через API (без redirect): сначала участник чата, потом S3. + + queued/processing/failed -> 409 (ещё/уже нечего отдавать); варианта нет у ready -> 404. + """ + row = db.get_attachment(attachment_id, current_user_id(request)) + if row["state"] != "ready": + raise conflict("Attachment is not ready") + if variant == "original": + key, content_type, size, sha256 = row["object_key"], row["content_type"], row["size_bytes"], row["sha256"] + name = row["filename"] + else: + found = [v for v in row["variants"] if v["name"] == variant] + if not found: + raise not_found("Variant not found") + v = found[0] + key, content_type, size, sha256 = v["object_key"], v["content_type"], v["size_bytes"], v["sha256"] + stem = row["filename"].rsplit(".", 1)[0] or "image" + name = f"{stem}.{variant}{VARIANT_SUFFIX[content_type]}" + try: + stream = storage.open_read(key) + except storage.ObjectMissing: + log.error("Object %s of ready attachment %s is missing in S3", variant, row["id"]) + raise unavailable("File is temporarily unavailable") from None + except storage.StorageUnavailable: + raise unavailable("File storage is temporarily unavailable") from None + headers = { + "Content-Length": str(size), + "Content-Disposition": "inline; filename*=UTF-8''" + quote(name, safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + sha256 + '"', + } + return StreamingResponse(storage.iter_chunks(stream), media_type=content_type, headers=headers) + + +if __name__ == "__main__": + Server(uvicorn.Config( + app, + host=settings.APP_HOST, + port=settings.APP_PORT, + timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS, + access_log=False, + )).run() diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..e95f414 --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +import logging + +import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + +if __name__ == "__main__": + db.wait_and_migrate() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..8b981d0 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,33 @@ + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 0000000..73f0c84 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..5770e1d --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,11 @@ +ALTER TABLE messages ADD COLUMN idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); + +ALTER TABLE messages ADD COLUMN request_hash bytea + CHECK (octet_length(request_hash) = 32); + +ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together + CHECK ((idempotency_key IS NULL) = (request_hash IS NULL)); + +ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key + UNIQUE (chat_id, sender_id, idempotency_key); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..35697dd --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,48 @@ +-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением. +-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать. + +-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата. +ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + -- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит. + object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'), + -- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна; + -- ready: объект в S3 сохранён, можно читать и прикреплять. + state text NOT NULL CHECK (state IN ('uploading', 'ready')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (width::bigint * height <= 40000000), + UNIQUE (id, chat_id), + -- владелец на момент загрузки был участником именно этого чата + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); + +-- janitor ищет старые незавершённые загрузки +CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + -- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id); + +-- С lab4 текст может быть пустым, если есть вложение (проверяет API). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/migrations/005_jobs.sql b/migrations/005_jobs.sql new file mode 100644 index 0000000..0751798 --- /dev/null +++ b/migrations/005_jobs.sql @@ -0,0 +1,133 @@ +-- lab5: асинхронная обработка вложений. Задания, outbox, варианты, журнал событий. +-- +-- Состояния attachments.state: +-- uploading — строка-намерение до записи в staging (как в lab4), клиенту не видна; +-- queued / processing / ready / failed — публичные состояния из контракта. +-- source_key — исходник в staging (staging/; у старых вложений lab4 — originals/); +-- object_key — опубликованный оригинал attachments//v/original, есть только у ready. + +ALTER TABLE attachments DROP CONSTRAINT attachments_state_check; +ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_check; +ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL; +ALTER TABLE attachments ADD COLUMN source_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer; + +-- Вложения lab4 были приняты синхронно и не имеют вариантов: переносим их в staging-модель и ставим +-- в очередь (ниже), worker опубликует оригинал в конечный prefix и сделает варианты. +UPDATE attachments SET source_key = object_key, object_key = NULL; +UPDATE attachments SET state = 'queued', updated_at = now() WHERE state = 'ready'; + +ALTER TABLE attachments ADD CONSTRAINT attachments_state_check + CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD CONSTRAINT attachments_source_key_check + CHECK (source_key ~ '^(staging|originals)/[a-f0-9]{32}$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_check + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/original$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code_check + CHECK (error_code ~ '^[a-z_]{1,64}$'); +-- ready <=> опубликованный оригинал; failed <=> код ошибки; до ready исходник обязан быть в staging. +ALTER TABLE attachments ADD CONSTRAINT attachments_ready_has_object + CHECK ((state = 'ready') = (object_key IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_failed_has_code + CHECK ((state = 'failed') = (error_code IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_pending_has_source + CHECK (state NOT IN ('uploading', 'queued', 'processing') OR source_key IS NOT NULL); + +-- janitor: исходники ready/failed, которые пора удалить из staging +CREATE INDEX attachments_staging ON attachments (updated_at) + WHERE source_key IS NOT NULL AND state IN ('ready', 'failed'); + +-- Варианты. Первичный ключ = ключ эффекта (attachment_id, pipeline_version, variant): +-- повтор job не может создать второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/[a-z]+\.(jpg|png|webp)$'), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- Задание: одна логическая работа на (вложение, версия pipeline). job_id не меняется при повторах. +-- attempt — управляемая попытка (claim worker'ом); lease_* — аренда processing с fencing-токеном. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + max_attempts integer NOT NULL CHECK (max_attempts >= 1), + available_at timestamptz NOT NULL DEFAULT now(), + lease_token uuid, + lease_owner text, + lease_until timestamptz, + error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'), + last_error text CHECK (last_error ~ '^[a-z_]{1,64}$'), + replays integer NOT NULL DEFAULT 0, + traceparent text CHECK (traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + UNIQUE (attachment_id, pipeline_version), + CHECK (attempt <= max_attempts), + CHECK ((status = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)), + CHECK ((status = 'failed') = (error_code IS NOT NULL)) +); +CREATE INDEX attachment_jobs_leases ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_queued ON attachment_jobs (available_at) WHERE status = 'queued'; +CREATE INDEX attachment_jobs_failed ON attachment_jobs (finished_at) WHERE status = 'failed'; + +-- Журнал (audit): claim, повторы, истечение аренды, дубли, replay. Только коды, без секретов и путей. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + event text NOT NULL CHECK (event ~ '^[a-z_]{1,64}$'), + detail text CHECK (char_length(detail) <= 200), + actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX job_events_by_job ON job_events (job_id, id); + +-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и задание. +-- dispatcher публикует строки с available_at <= now() и отмечает published_at после publisher confirm. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + publish_failures integer NOT NULL DEFAULT 0 +); +CREATE INDEX outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_by_job ON outbox (job_id); + +-- Задания для вложений, принятых в lab4 (миграция = их producer). JOB_MAX_ATTEMPTS по умолчанию 5. +INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, traceparent) +SELECT gen_random_uuid(), id, 1, 'queued', 1, 5, + '00-' || md5(random()::text || id::text) || '-' || substr(md5(id::text), 1, 16) || '-01' +FROM attachments WHERE state = 'queued'; + +INSERT INTO job_events (job_id, attempt, event, detail, actor) +SELECT job_id, 1, 'created', 'lab4_backfill', 'migration 005' FROM attachment_jobs; + +INSERT INTO outbox (job_id, attempt, payload) +SELECT job_id, 1, jsonb_build_object( + 'schema_version', 1, + 'job_id', job_id, + 'type', 'attachment.process.v1', + 'attachment_id', attachment_id, + 'pipeline_version', pipeline_version, + 'attempt', 1, + 'created_at', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'traceparent', traceparent) +FROM attachment_jobs; diff --git a/ops.py b/ops.py new file mode 100644 index 0000000..ee815fe --- /dev/null +++ b/ops.py @@ -0,0 +1,194 @@ +"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher). + + docker compose exec dispatcher python ops.py status + docker compose exec dispatcher python ops.py failed [--limit N] + docker compose exec dispatcher python ops.py events + docker compose exec dispatcher python ops.py replay [--reason TEXT] + docker compose exec dispatcher python ops.py dlq [--limit N] + docker compose exec dispatcher python ops.py dlq-replay [--limit N] + +Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code, +число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя +обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit. + +replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready, +queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка +идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant). + +dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины): +failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу +ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения +остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3. +""" +import argparse +import json +import sys +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import broker +import pipeline +import settings + +ACTOR = "ops@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options="-c TimeZone=UTC") + + +def text(value): + if isinstance(value, (UUID,)): + return str(value) + if hasattr(value, "isoformat"): + return pipeline.utc_text(value) + return value + + +def rows_json(rows): + return [{key: text(value) for key, value in row.items()} for row in rows] + + +def cmd_status(conn, _args): + jobs = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()} + outbox = conn.execute( + """SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending, + count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due, + coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures, + extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL + AND available_at <= now())) AS oldest_due_s + FROM outbox""").fetchone() + leases = conn.execute( + """SELECT count(*) FILTER (WHERE lease_until >= now()) AS active, + count(*) FILTER (WHERE lease_until < now()) AS expired + FROM attachment_jobs WHERE status = 'processing'""").fetchone() + staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"] + report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v) + for k, v in outbox.items()}, + "leases": leases, "attachments_with_staging_source": staging} + try: + connection = broker.connect() + try: + report["broker"] = broker.check_topology(connection.channel()) + finally: + connection.close() + except broker.BrokerError as exc: + report["broker"] = {"error": type(exc).__name__} + return report + + +def cmd_failed(conn, args): + return rows_json(conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, + j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall()) + + +def cmd_events(conn, args): + return rows_json(conn.execute( + "SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id", + (args.job_id,)).fetchall()) + + +def cmd_replay(conn, args): + try: + return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)} + except pipeline.ReplayRefused as exc: + return {"refused": str(exc), "job_id": str(args.job_id)} + + +def dlq_messages(limit, handler): + """Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'.""" + connection = broker.connect() + results = [] + try: + channel = connection.channel() + keep = [] + for _ in range(limit): + method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False) + if method is None: + break + death = (properties.headers or {}).get("x-death") or [{}] + reason = death[0].get("reason") + reason = reason.decode() if isinstance(reason, bytes) else reason + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + message, problem = None, str(exc) + else: + problem = None + verdict, note = handler(message) + results.append({"job_id": message["job_id"] if message else None, "invalid": problem, + "dead_letter_reason": reason, "action": verdict, "note": note}) + if verdict == "ack": + channel.basic_ack(method.delivery_tag) + else: + keep.append(method.delivery_tag) + for tag in keep: # вернуть непрочитанными в DLQ + channel.basic_nack(tag, requeue=True) + finally: + connection.close() + return results + + +def cmd_dlq(_conn, args): + return dlq_messages(args.limit, lambda message: ("keep", "peek")) + + +def cmd_dlq_replay(conn, args): + def handle(message): + if message is None: + return "keep", "invalid message is kept for analysis" + job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s", + (UUID(message["job_id"]),)).fetchone() + if job is None: + return "keep", "unknown job" + if job["status"] == "failed": + try: + pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay") + return "ack", "replayed" + except pipeline.ReplayRefused as exc: + return "keep", str(exc) + if job["status"] == "ready": + return "ack", "already ready, no-op" + # queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново + conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s " + "AND status = 'queued'", (UUID(message["job_id"]),)) + return "ack", f"job is {job['status']}, handled by outbox/lease recovery" + return dlq_messages(args.limit, handle) + + +def main(): + parser = argparse.ArgumentParser(description="Attachment pipeline operator commands") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("status") + p = sub.add_parser("failed") + p.add_argument("--limit", type=int, default=50) + p = sub.add_parser("events") + p.add_argument("job_id", type=UUID) + p = sub.add_parser("replay") + p.add_argument("job_id", type=UUID) + p.add_argument("--reason", default="operator replay") + p = sub.add_parser("dlq") + p.add_argument("--limit", type=int, default=20) + p = sub.add_parser("dlq-replay") + p.add_argument("--limit", type=int, default=100) + args = parser.parse_args() + if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()): + parser.error("--reason: up to 100 printable characters") + commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay, + "dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay} + with connect_db() as conn: + result = commands[args.command](conn, args) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 0000000..78b6b5a --- /dev/null +++ b/pipeline.py @@ -0,0 +1,445 @@ +"""Задания обработки вложений: формат сообщения, outbox, переходы состояний. + +Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат) +и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции +вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT. + +Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать, +решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение +безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op. + +Переходы (contracts/attachment-states.md): + (нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202 + queued -> processing claim worker, аренда lease_until + fencing-токен lease_token + processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter) + recover_lease аренда истекла (worker умер/завис), попытка засчитана + processing -> ready finish_ready только при совпадении lease_token (fencing) + processing -> failed finish_failed постоянная ошибка или попытки исчерпаны + failed -> queued replay явная операция оператора (ops.py), с записью в журнал +""" +import json +import random +import re +import secrets +from dataclasses import dataclass +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from psycopg.types.json import Jsonb + +import settings + +SCHEMA_VERSION = 1 +JOB_TYPE = "attachment.process.v1" +EXCHANGE = "attachments" +ROUTING_KEY = "attachment.process.v1" +QUEUE = "attachments.process.v1" +DLX = "attachments.dlx" +DLQ = "attachments.process.v1.dlq" +DLQ_ROUTING_KEY = "attachment.process.v1.dlq" + +MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", + "created_at", "traceparent", "tracestate"} +REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"} +TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") +UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") +CODE_RE = re.compile(r"^[a-z_]{1,64}$") + +# Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES. +PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"} +FINAL_CODES = PERMANENT | {"retry_exhausted"} + + +class InvalidMessage(Exception): + """Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ.""" + + +def utc_text(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def new_traceparent() -> str: + """W3C trace context: trace-id на всё задание, span-id новый для каждой публикации.""" + return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01" + + +def child_traceparent(parent: str | None) -> str: + if parent and TRACEPARENT_RE.match(parent): + return f"00-{parent[3:35]}-{secrets.token_hex(8)}-01" + return new_traceparent() + + +def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str | None) -> dict: + """Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3.""" + message = { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": utc_text(datetime.now(timezone.utc)), + } + if traceparent: + message["traceparent"] = child_traceparent(traceparent) + return message + + +def parse_message(body: bytes) -> dict: + """Строгая проверка по схеме (additionalProperties: false, const, типы, форматы).""" + try: + message = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise InvalidMessage("not json") from None + if not isinstance(message, dict): + raise InvalidMessage("not an object") + if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message): + raise InvalidMessage("fields") + if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE: + raise InvalidMessage("schema_version/type") + for name in ("job_id", "attachment_id"): + if not isinstance(message[name], str) or not UUID_RE.match(message[name]): + raise InvalidMessage(name) + for name in ("pipeline_version", "attempt"): + value = message[name] + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidMessage(name) + try: + created = message["created_at"] + if not isinstance(created, str): + raise ValueError + datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError: + raise InvalidMessage("created_at") from None + if "traceparent" in message and (not isinstance(message["traceparent"], str) + or not TRACEPARENT_RE.match(message["traceparent"])): + raise InvalidMessage("traceparent") + if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512): + raise InvalidMessage("tracestate") + return message + + +def backoff_seconds(failed_attempt: int) -> float: + """Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна. + + Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS. + Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу). + """ + ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS, + settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1)) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None: + conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)", + (job_id, attempt, name, detail[:200] if detail else None, actor[:200])) + + +def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str | None, delay_seconds: float = 0) -> None: + """Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT.""" + payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent) + conn.execute( + """INSERT INTO outbox (job_id, attempt, payload, available_at) + VALUES (%s, %s, %s, now() + make_interval(secs => %s))""", + (job_id, attempt, Jsonb(payload), delay_seconds), + ) + conn.execute("NOTIFY outbox") + + +def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID: + """Новое задание для вложения, уже переведённого в queued (в транзакции API).""" + job_id = uuid4() + traceparent = new_traceparent() + conn.execute( + """INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, + traceparent) + VALUES (%s, %s, %s, 'queued', 1, %s, %s)""", + (job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent), + ) + event(conn, job_id, 1, "created", actor) + add_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent) + return job_id + + +# ---------- worker: захват и результат ---------- + +@dataclass +class Claim: + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + max_attempts: int + lease_token: UUID + source_key: str + content_type: str + size_bytes: int + sha256: str + traceparent: str | None + + +@dataclass +class Skip: + """Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker).""" + reason: str + + +@dataclass +class Reject: + """Сообщение не относится ни к одной job -> DLQ.""" + reason: str + + +def claim(conn, message: dict, owner: str, redelivered: bool): + """queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject.""" + job_id = UUID(message["job_id"]) + with conn.transaction(): + job = conn.execute( + """SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired + FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""", + (job_id,), + ).fetchone() + if job is None: + return Reject("unknown_job") + if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]: + return Reject("job_mismatch") + status = job["status"] + if status in ("ready", "failed"): + event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}") + return Skip(f"already_{status}") + attempt = job["attempt"] + if status == "queued": + if not job["due"]: + return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox + else: # processing + if not job["lease_expired"]: + event(conn, job_id, attempt, "duplicate_ignored", owner, "leased") + return Skip("leased") + # Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана. + event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker") + if attempt >= job["max_attempts"]: + _fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired") + return Skip("exhausted") + attempt += 1 + token = uuid4() + conn.execute( + """UPDATE attachment_jobs + SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, + lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id), + ) + att = conn.execute( + """UPDATE attachments SET state = 'processing', updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing') + RETURNING source_key, content_type, size_bytes, sha256""", + (job["attachment_id"],), + ).fetchone() + if att is None or att["source_key"] is None: + raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат + event(conn, job_id, attempt, "claimed", owner, + f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}") + return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token, + att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"]) + + +def renew_lease(conn, c: Claim) -> bool: + """Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить.""" + row = conn.execute( + """UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""", + (settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def _fenced(conn, c: Claim, owner: str) -> bool: + """Fencing: результат принимается, только если lease_token всё ещё наш.""" + row = conn.execute( + """SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing' + FOR UPDATE""", + (c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> bool: + """processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT.""" + with conn.transaction(): + if not _fenced(conn, c, owner): + stale = True + else: + stale = False + for v in variants: + conn.execute( + """INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""", + (c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"], + v["width"], v["height"], v["sha256"], v["object_key"]), + ) + # Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна). + updated = conn.execute( + """UPDATE attachments + SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now() + WHERE id = %s AND state = 'processing' + AND (pipeline_version IS NULL OR pipeline_version <= %s) + RETURNING id""", + (original_key, c.pipeline_version, c.attachment_id, c.pipeline_version), + ).fetchone() + if updated is None: + raise RuntimeError("attachment changed state during processing") + conn.execute( + """UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s""", + (c.job_id,), + ) + event(conn, c.job_id, c.attempt, "ready", owner) + if stale: + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost") + return not stale + + +def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None): + conn.execute( + """UPDATE attachment_jobs + SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s""", + (code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id), + ) + conn.execute( + """UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing')""", + (code, attachment_id), + ) + event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}") + + +def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int, + traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None: + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (next_attempt, reason, delay, job_id), + ) + conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'", + (attachment_id,)) + add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay) + event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s") + + +def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str: + """Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'.""" + with conn.transaction(): + if not _fenced(conn, c, owner): + outcome = "stale" + elif permanent: + _fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None) + outcome = "failed" + elif c.attempt >= c.max_attempts: + _fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code) + outcome = "failed" + else: + _requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent, + backoff_seconds(c.attempt), code, owner, "retry_scheduled") + outcome = "retry" + if outcome == "stale": + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code) + return outcome + + +# ---------- dispatcher: аренды и сверка ---------- + +def recover_expired_leases(conn, actor: str, limit: int = 100) -> int: + """processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted).""" + recovered = 0 + with conn.transaction(): + rows = conn.execute( + """SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent + FROM attachment_jobs WHERE status = 'processing' AND lease_until < now() + ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""", + (limit,), + ).fetchall() + for job in rows: + if job["attempt"] >= job["max_attempts"]: + _fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor, + "lease_expired") + else: + _requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1, + job["traceparent"], backoff_seconds(job["attempt"]), "lease_expired", actor, + "lease_expired") + recovered += 1 + return recovered + + +def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int: + """Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет. + + Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его + подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет. + """ + with conn.transaction(): + rows = conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent + FROM attachment_jobs j + WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s) + AND j.updated_at < now() - make_interval(secs => %s) + AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND + (o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s))) + ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""", + (older_than, older_than, older_than, limit), + ).fetchall() + for job in rows: + add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], + job["traceparent"]) + conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],)) + event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight") + return len(rows) + + +# ---------- оператор: replay ---------- + +class ReplayRefused(Exception): + pass + + +def replay(conn, job_id: UUID, actor: str, reason: str) -> dict: + """failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются.""" + with conn.transaction(): + job = conn.execute( + """SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""", + (job_id,), + ).fetchone() + if job is None: + raise ReplayRefused("job not found") + if job["status"] != "failed": + # «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается. + raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs") + if job["source_key"] is None: + raise ReplayRefused("source file is no longer in staging (retention expired)") + attempt = job["attempt"] + 1 + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL, + available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now() + WHERE job_id = %s""", + (attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id), + ) + conn.execute( + "UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'", + (job["attachment_id"],), + ) + event(conn, job_id, attempt, "replayed", actor, reason or "operator replay") + add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, job["traceparent"]) + return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d213edc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,30 @@ +fastapi==0.141.1 +uvicorn==0.54.0 +pydantic==2.13.5 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==6.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +pycparser==3.1 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +boto3==1.43.116 +botocore==1.43.116 +s3transfer==0.19.3 +jmespath==1.1.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +urllib3==2.8.0 +pillow==12.3.0 +pika==1.3.2 diff --git a/s3_init.py b/s3_init.py new file mode 100644 index 0000000..99489bc --- /dev/null +++ b/s3_init.py @@ -0,0 +1,103 @@ +"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init. + +Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает. +Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py): + admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init; + messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor. +Значения ключей не печатаются. +""" +import logging +import sys +import time +import urllib.error +import urllib.request +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +logging.basicConfig(level=logging.WARNING) + + +def client(key_id: str, secret: str): + return boto3.client( + "s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION, + aws_access_key_id=key_id, aws_secret_access_key=secret, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2, + read_timeout=10, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", response_checksum_validation="when_required"), + ) + + +def code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "?") + + +def anonymous_status(url: str, method: str = "GET") -> int: + try: + with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply: + return reply.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + bucket = settings.S3_BUCKET + admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"), + settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY) + + deadline = time.monotonic() + 90 + while True: + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if time.monotonic() > deadline: + print(f"s3-init: S3 is not ready: {type(exc).__name__}") + return 1 + time.sleep(1) + + try: + admin.create_bucket(Bucket=bucket) + print(f"s3-init: bucket {bucket} created") + except ClientError as exc: + if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + print(f"s3-init: cannot create bucket: {code(exc)}") + return 1 + print(f"s3-init: bucket {bucket} exists") + + # Ключ API работает со своим bucket ... + probe = "originals/s3-init-probe-" + uuid.uuid4().hex + api.put_object(Bucket=bucket, Key=probe, Body=b"probe") + api.get_object(Bucket=bucket, Key=probe)["Body"].read() + api.list_objects_v2(Bucket=bucket, MaxKeys=1) + api.delete_object(Bucket=bucket, Key=probe) + print("s3-init: api key can put/get/list/delete objects of its bucket") + # ... и не может администрировать хранилище. + intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8] + try: + api.create_bucket(Bucket=intruder) + admin.delete_bucket(Bucket=intruder) + print("s3-init: api key must NOT be able to create buckets") + return 1 + except ClientError as exc: + print(f"s3-init: api key cannot create buckets ({code(exc)})") + + # Приватность: анонимные запросы без подписи. + base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket + statuses = {"list": anonymous_status(base + "?list-type=2"), + "get": anonymous_status(base + "/originals/any"), + "put": anonymous_status(base + "/originals/any", "PUT")} + if any(status not in (401, 403) for status in statuses.values()): + print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}") + return 1 + print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..0db2b0a --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,153 @@ +import base64 +import hashlib +import json +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + with open(SECRETS / name, "w", encoding="utf-8", newline="") as file: + file.write(value) + (SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore + + +def bucket_name() -> str: + """S3_BUCKET из .env (если задан), иначе значение по умолчанию.""" + env = ROOT / ".env" + if env.exists(): + for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("S3_BUCKET="): + return line.split("=", 1)[1].strip() + return "messenger-attachments" + + +def s3_config(bucket: str) -> str: + """Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket. + + Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects) + с суффиксом : действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет. + Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются. + """ + def read(name): + return (SECRETS / name).read_text(encoding="utf-8").strip() + config = {"identities": [ + {"name": "admin", + "credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ]} + return json.dumps(config, indent=2) + + +BROKER_VHOST = "messenger" +# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей). +EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1" +DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq" + + +def rabbit_hash(password: str) -> str: + """Формат rabbit_password_hashing_sha256: base64(salt(4 байта) + SHA-256(salt + пароль)).""" + salt = secrets.token_bytes(4) + return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii") + + +def rabbit_definitions(passwords: dict) -> str: + """Топология и пользователи RabbitMQ; брокер загружает файл при каждом старте (infra/rabbitmq.conf). + + Права (configure / write / read, регулярные выражения по именам ресурсов): + dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет; + worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет. + Ни у кого из сервисов нет тега administrator и права создать/удалить очередь. + """ + queue_args = { + "x-queue-type": "quorum", + "x-delivery-limit": 20, # ядовитое сообщение не крутится бесконечно -> DLQ + "x-dead-letter-exchange": DLX, + "x-dead-letter-routing-key": DLQ_ROUTING_KEY, + "x-dead-letter-strategy": "at-least-once", # перенос в DLQ без потери (требует reject-publish) + "x-overflow": "reject-publish", # переполнение -> nack издателю, outbox повторит + "x-max-length": 100000, + } + definitions = { + "users": [{"name": name, "password_hash": rabbit_hash(password), + "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} + for name, password in passwords.items()], + "vhosts": [{"name": BROKER_VHOST}], + "permissions": [ + {"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$", + "write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"}, + {"user": "worker", "vhost": BROKER_VHOST, "configure": "^$", + "write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"}, + ], + "exchanges": [ + {"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + {"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + ], + "queues": [ + {"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args}, + # DLQ — classic durable: у quorum в 4.x delivery-limit по умолчанию 20, и многократный + # просмотр (basic_get + nack) в ops.py мог бы выбросить сообщение. Здесь потерь от просмотра нет. + {"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, + "arguments": {"x-queue-type": "classic"}}, + ], + "bindings": [ + {"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue", + "routing_key": ROUTING_KEY, "arguments": {}}, + {"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue", + "routing_key": DLQ_ROUTING_KEY, "arguments": {}}, + ], + } + return json.dumps(definitions, indent=2) + + +def broker_secrets(created: list, host: str = "rabbitmq") -> None: + if (SECRETS / "rabbitmq_definitions.json").exists(): + return + passwords = {"dispatcher": secrets.token_hex(24), "worker": secrets.token_hex(24)} + for name, password in passwords.items(): + write(f"broker_url_{name}", f"amqp://{name}:{password}@{host}:5672/{BROKER_VHOST}") + write("rabbitmq_definitions.json", rabbit_definitions(passwords)) + created += ["broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions.json"] + + +def main() -> None: + """Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes).""" + SECRETS.mkdir(exist_ok=True) + created = [] + if not (SECRETS / "postgres_password").exists(): + db_password = secrets.token_hex(24) + redis_password = secrets.token_hex(24) + write("postgres_password", db_password) + write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger") + write("cache_url", f"redis://:{redis_password}@redis:6379/0") + write("redis_auth", f"requirepass {redis_password}\n") + created += ["postgres_password", "database_url", "cache_url", "redis_auth"] + if not (SECRETS / "s3_root_password").exists(): + # admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init + write("s3_root_user", "root-" + secrets.token_hex(6)) + write("s3_root_password", secrets.token_hex(24)) + created += ["s3_root_user", "s3_root_password"] + if not (SECRETS / "s3_secret_access_key").exists(): + # отдельный ключ API с политикой только на свой bucket (создаёт s3-init) + write("s3_access_key_id", "api-" + secrets.token_hex(8)) + write("s3_secret_access_key", secrets.token_hex(24)) + created += ["s3_access_key_id", "s3_secret_access_key"] + if not (SECRETS / "s3_config.json").exists(): + write("s3_config.json", s3_config(bucket_name())) + created.append("s3_config.json") + broker_secrets(created) + if created: + print("Created in secrets/: " + ", ".join(created) + " (values are not printed).") + else: + print("secrets/ is complete, nothing changed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..6a5dfb4 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,14 @@ +@echo off +rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ), then api, janitor, dispatcher, worker, +rem then the balancer. Volumes are kept. +rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +cd /d "%~dp0.." +docker compose restart postgres redis s3 rabbitmq || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq || exit /b 1 +docker compose restart api janitor dispatcher worker || exit /b 1 +docker compose restart lb || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps api lb dispatcher worker || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..98d2561 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Restart hook for tests/resilience.py --mode persistence (Linux/macOS). +# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ), then api, janitor, dispatcher and +# worker, then the balancer. Volumes are kept (no down -v), nothing is re-seeded. +# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres redis s3 rabbitmq +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq +docker compose restart api janitor dispatcher worker +docker compose restart lb +docker compose up -d --wait --no-recreate --no-build --no-deps api lb dispatcher worker diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..8ffd0fa --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py. +python "%~dp0stop_one.py" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..cf4d724 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID. +# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd. +set -eu +exec python3 "$(dirname "$0")/stop_one.py" diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 0000000..5cdcce7 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STOP_TIMEOUT_SECONDS = 8 + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{12,64}", target): + print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr) + return 2 + running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + matches = [container for container in running if container.startswith(target)] + if len(matches) != 1: + print(f"no running api container with id {target}", file=sys.stderr) + return 1 + subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + print(f"stopped api container {matches[0][:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/worker.cmd b/scripts/worker.cmd new file mode 100644 index 0000000..9ffab68 --- /dev/null +++ b/scripts/worker.cmd @@ -0,0 +1,8 @@ +@echo off +rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept). +cd /d "%~dp0.." +if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% ) +if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% ) +if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% ) +echo usage: %0 stop^|start^|status +exit /b 2 diff --git a/scripts/worker.sh b/scripts/worker.sh new file mode 100644 index 0000000..cab7c76 --- /dev/null +++ b/scripts/worker.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Demo hook for the defense: stop or start the worker without touching volumes. +# scripts/worker.sh stop | start | status +set -eu +cd "$(dirname "$0")/.." +case "${1:-}" in + stop) docker compose stop worker ;; + start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;; + status) docker compose ps worker ;; + *) echo "usage: $0 stop|start|status" >&2; exit 2 ;; +esac diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..e4cf3eb --- /dev/null +++ b/settings.py @@ -0,0 +1,111 @@ +import os +import socket +from pathlib import Path + + +def env_str(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def env_int(name: str, default: int) -> int: + value = int(env_str(name, str(default))) + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value + + +def read_secret(file_variable: str, plain_variable: str) -> str: + path = env_str(file_variable, "") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return env_str(plain_variable, "") + + +APP_HOST = env_str("APP_HOST", "0.0.0.0") +APP_PORT = env_int("APP_PORT", 8080) +APP_ENV = env_str("APP_ENV", "lab") +APP_VERSION = env_str("APP_VERSION", "dev") + +INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname() + +SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600) +SESSION_STORE = env_str("SESSION_STORE", "redis") + +DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL") +DB_POOL_MIN = env_int("DB_POOL_MIN", 1) +DB_POOL_MAX = env_int("DB_POOL_MAX", 10) +DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3) +DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3) +DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + +REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL") +REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20) +REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500) + +MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations"))) +SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60) + +# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket. +S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333") +S3_REGION = env_str("S3_REGION", "us-east-1") +S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1) +S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5) +S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2) +S3_POOL_MAX = env_int("S3_POOL_MAX", 20) + +# Вложения: лимиты из контракта и защита памяти/времени. +MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10) +DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2) +UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads") + +# Очистка незавершённых загрузок (janitor.py). +ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600) +JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300) + +SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3) +SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10) + +# lab5: брокер (RabbitMQ) и фоновая обработка вложений. +# У API адреса брокера нет вообще: API пишет задание в outbox (PostgreSQL), публикует dispatcher. +BROKER_URL = read_secret("BROKER_URL_FILE", "BROKER_URL") +BROKER_HEARTBEAT_SECONDS = env_int("BROKER_HEARTBEAT_SECONDS", 30) +PIPELINE_VERSION = env_int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = env_int("WORKER_CONCURRENCY", 2) # потоков-потребителей в одном worker +WORKER_PREFETCH = env_int("WORKER_PREFETCH", 1) # неподтверждённых сообщений на поток +JOB_MAX_ATTEMPTS = env_int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = env_int("JOB_TIMEOUT_SECONDS", 60) # жёсткий предел одной обработки изображения +JOB_LEASE_SECONDS = env_int("JOB_LEASE_SECONDS", 15) # аренда processing, продлевается каждые lease/3 +JOB_BACKOFF_BASE_SECONDS = env_int("JOB_BACKOFF_BASE_SECONDS", 2) +JOB_BACKOFF_MAX_SECONDS = env_int("JOB_BACKOFF_MAX_SECONDS", 60) +JOB_RECONCILE_SECONDS = env_int("JOB_RECONCILE_SECONDS", 120) # queued без сообщения дольше -> опубликовать заново +WORKER_MEMORY_LIMIT_MB = env_int("WORKER_MEMORY_LIMIT_MB", 640) # RLIMIT_AS процесса обработки +WORKER_TMP_DIR = env_str("WORKER_TMP_DIR", "/tmp/work") +DISPATCHER_POLL_SECONDS = env_int("DISPATCHER_POLL_SECONDS", 1) +DISPATCHER_BATCH = env_int("DISPATCHER_BATCH", 100) +STAGING_RETENTION_SECONDS = env_int("STAGING_RETENTION_SECONDS", 86400) # исходник failed-вложения для replay +OUTBOX_RETENTION_SECONDS = env_int("OUTBOX_RETENTION_SECONDS", 7 * 86400) +ALIVE_FILE = env_str("ALIVE_FILE", "/tmp/alive") # healthcheck фоновых процессов + +# Учебное внедрение сбоев (faults.py). На стенде включено, в эксплуатации FAULT_INJECTION=0. +FAULT_INJECTION = env_str("FAULT_INJECTION", "0") == "1" +FAULTS_DIR = env_str("FAULTS_DIR", "/faults") + + +def check() -> None: + """Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе.""" + if not DATABASE_URL: + raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE") + if not REDIS_URL: + raise RuntimeError("Redis address is not set: CACHE_URL_FILE") + if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY: + raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE") + if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS: + raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS") + if SESSION_STORE != "redis": + raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution") diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..86e90bc --- /dev/null +++ b/storage.py @@ -0,0 +1,183 @@ +"""Приватное S3-хранилище (SeaweedFS) для байтов вложений. + +Раскладка ключей (lab5): + staging/<32 hex> — исходник, принятый API до ответа 202 (устойчивый staging); + attachments//v/original — опубликованный worker'ом оригинал (побайтно исходный файл); + attachments//v/thumbnail.jpg, crop.jpg, optimized.webp — варианты pipeline версии N; + originals/<32 hex> — исходники вложений lab4, worker обрабатывает их как staging. + +API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket +(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер, +имя файла пользователя в ключ не попадает. +""" +import base64 +import logging +from datetime import datetime +from typing import BinaryIO, Iterator + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +log = logging.getLogger("messenger.storage") + +ORIGINALS_PREFIX = "originals/" # lab4 +STAGING_PREFIX = "staging/" +FINAL_PREFIX = "attachments/" +ALL_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, ORIGINALS_PREFIX) +READ_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей.""" + + +class ObjectMissing(Exception): + """Объекта нет в bucket (NoSuchKey / 404).""" + + +def _make_client(connect_timeout: float, read_timeout: float, attempts: int): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT, + region_name=settings.S3_REGION, + aws_access_key_id=settings.S3_ACCESS_KEY_ID or None, + aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, # http://s3:8333// + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую + max_pool_connections=settings.S3_POOL_MAX, + # Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно. + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + +_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS) +# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor. +_quick_client = _make_client(1, 2, 1) + + +def _is_missing(exc: ClientError) -> bool: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("NoSuchKey", "404", "NotFound") or status == 404 + + +def _unavailable(action: str, exc: Exception) -> StorageUnavailable: + detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else "" + log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail) + return StorageUnavailable() + + +def new_key(object_id: str) -> str: + """Ключ staging для нового исходника (имя файла пользователя в ключ не попадает).""" + return STAGING_PREFIX + object_id + + +def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + """Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект.""" + body.seek(0) + try: + _client.put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("put", exc) from None + + +def open_read(key: str): + """GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после.""" + try: + return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"] + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing(key) from None + raise _unavailable("get", exc) from None + except BotoCoreError as exc: + raise _unavailable("get", exc) from None + + +def download(key: str, path, max_bytes: int) -> int: + """GET объекта в файл кусками. Дальше max_bytes не читаем: размер не совпадёт, worker отклонит исходник.""" + stream = open_read(key) + size = 0 + try: + with open(path, "wb") as file: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return size + size += len(chunk) + file.write(chunk) + if size > max_bytes: + return size + except (BotoCoreError, OSError) as exc: + raise _unavailable("get", exc) from None + finally: + stream.close() + + +def iter_chunks(stream) -> Iterator[bytes]: + """Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание.""" + try: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return + yield chunk + except (BotoCoreError, OSError) as exc: + # Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length). + log.warning("S3 stream interrupted: %s", type(exc).__name__) + raise + finally: + stream.close() + + +def delete(key: str, quick: bool = False) -> None: + """Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа.""" + try: + (_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("delete", exc) from None + + +def exists(key: str) -> bool: + try: + _client.head_object(Bucket=settings.S3_BUCKET, Key=key) + return True + except ClientError as exc: + if _is_missing(exc): + return False + raise _unavailable("head", exc) from None + except BotoCoreError as exc: + raise _unavailable("head", exc) from None + + +def list_objects(prefix: str) -> Iterator[tuple[str, datetime, int]]: + """Все объекты под prefix: (key, last_modified, size). Постранично, по 1000.""" + paginator = _client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"], item["Size"] + except (BotoCoreError, ClientError) as exc: + raise _unavailable("list", exc) from None + + +def ping() -> None: + try: + _client.head_bucket(Bucket=settings.S3_BUCKET) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("head_bucket", exc) from None diff --git a/student-tests/check_stack.py b/student-tests/check_stack.py new file mode 100644 index 0000000..3f4258e --- /dev/null +++ b/student-tests/check_stack.py @@ -0,0 +1,91 @@ + +import json +import platform +import sys + +from common import call, check, compose_output, docker, new_user, seen_instances + + +def main(): + info = json.loads(docker("info", "--format", "{{json .}}")) + print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"client Python {platform.python_version()}") + print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, " + f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, " + f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}") + + print() + print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip()) + for one_shot in ("migrate", "s3-init"): + status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip() + check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0") + + config = json.loads(compose_output("config", "--format", "json")) + services = config["services"] + containers = compose_output("ps", "-q").split() + rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()] + print() + print("container image cpus memory host ports networks") + api_count = 0 + for row in sorted(rows, key=lambda r: r["Name"]): + labels = row["Config"]["Labels"] + service = labels.get("com.docker.compose.service") + limits = row["HostConfig"] + cpus = limits.get("NanoCpus", 0) / 1e9 + memory = limits.get("Memory", 0) // 2**20 + ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values() + for b in (bindings or [])}) + networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"]) + print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB " + f"{','.join(ports) or '-':<23} {','.join(networks)}") + check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit") + if service == "api": + api_count += 1 + check(not ports, "api must not publish ports: everything goes through the balancer") + if service == "lb": + check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only") + if service in ("postgres", "redis", "s3", "janitor", "rabbitmq", "worker", "dispatcher"): + check(not ports and networks == ["data"], f"{service} must be only in the internal data network") + check(api_count >= 2, "need at least 2 api replicas") + running = {r["Config"]["Labels"].get("com.docker.compose.service") for r in rows if r["State"]["Running"]} + check({"rabbitmq", "worker", "dispatcher"} <= running, "lab5: broker, worker and dispatcher must be running") + check(any(v.get("target") == "/var/lib/rabbitmq" and v.get("type") == "volume" + for v in services["rabbitmq"].get("volumes", [])), "rabbitmq needs a persistent volume") + check(services["rabbitmq"].get("hostname") == "rabbitmq", "rabbitmq needs a fixed hostname (node data dir)") + check(all("container_name" not in s for s in services.values()), "container_name prevents scaling") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis") + + print() + for name, service in sorted(services.items()): + secrets = sorted(s["source"] for s in service.get("secrets", [])) + print(f"secrets of {name}: {secrets or '-'}") + check(not services["lb"].get("secrets"), "lb gets no secrets") + check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url") + for name in ("api", "janitor", "worker"): + names = {s["source"] for s in services[name]["secrets"]} + check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys") + check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key") + broker_secrets = {"broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions"} + for name in ("api", "janitor", "lb", "migrate"): + names = {s["source"] for s in services[name].get("secrets", [])} + check(not names & broker_secrets, f"{name} must not get broker credentials (API uses the outbox)") + check({s["source"] for s in services["dispatcher"]["secrets"]} == {"database_url", "broker_url_dispatcher"}, + "dispatcher: only database_url and its own broker user") + check(not {s["source"] for s in services["dispatcher"]["secrets"]} & {"s3_access_key_id", "s3_secret_access_key"}, + "dispatcher needs no S3 key") + + user = new_user("stack") + served = seen_instances(user["token"]) + print() + print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}") + check(len(served) >= 2, "requests are not balanced") + health = call("GET", "/health/ready", 200) + print(f"/health/ready -> {health.status} from {health.instance}") + print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every " + f"container, postgres/redis/s3/janitor/rabbitmq/worker/dispatcher only in the internal data network, " + f"S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 0000000..c44bb68 --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,359 @@ + +import json +import os +import platform +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence") # по умолчанию evidence/ +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PY = sys.executable + +STEPS = [ + ("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]), + ("stack", "stand layout: replicas, ports, limits, networks, secrets, S3", [PY, "student-tests/check_stack.py"]), + ("smoke", "public smoke via the balancer (make test), with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]), + ("jobs-variants", "lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped", + [PY, "student-tests/test_jobs.py", "variants"]), + ("jobs-limits", "lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)", + [PY, "student-tests/test_jobs.py", "limits"]), + ("jobs-recovery", "lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits", + [PY, "student-tests/test_jobs.py", "worker-down", "broker-down"]), + ("jobs-outbox", "lab5: dispatcher crash between COMMIT and publish / after confirm", + [PY, "student-tests/test_jobs.py", "outbox-crash"]), + ("jobs-crash", "lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease", + [PY, "student-tests/test_jobs.py", "crash-after-put", "crash-after-commit", "staging", "lease"]), + ("jobs-retry", "lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ", + [PY, "student-tests/test_jobs.py", "retry", "exhausted-replay", "permanent", "dlq"]), + ("jobs-concurrency", "lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing", + [PY, "student-tests/test_jobs.py", "concurrency"]), + ("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids", + [PY, "student-tests/test_attachments.py"]), + ("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap", + [PY, "student-tests/test_s3_security.py"]), + ("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage", + [PY, "student-tests/test_consistency.py"]), + ("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas", + [PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]), + ("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]), + ("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]), + ("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]), + ("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]), + ("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]), + ("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]), + ("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]), + ("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)", + [PY, "student-tests/run_failover.py"]), + ("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL, + "--duration", "30", "--concurrency", "4", + "--output", "evidence/load.json"]), + ("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]), + ("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)", + [PY, "student-tests/run_persistence.py"]), +] + + +def git(*args): + try: + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "" + + +def commit(): + head = git("rev-parse", "--short", "HEAD") or "unknown" + changed = [line for line in git("status", "--porcelain").splitlines() + if not line[3:].startswith(("evidence/", "REPORT.md"))] + return head + (" + uncommitted changes" if changed else "") + + +def run_step(name, description, command): + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL) + print(f"=== {name}: {description}", flush=True) + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + seconds = time.monotonic() - started + output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n") + shown = " ".join(Path(part).name if part == PY else part for part in command) + header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n" + f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n") + (EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8") + tail = [line for line in output.splitlines() if line.strip()][-3:] + for line in tail: + print(" " + line) + print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True) + return result.returncode == 0, seconds + + +def load_json(name): + path = EVIDENCE / name + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + +def measurements(): + """Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md).""" + lines = [] + try: + info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True, + text=True, check=True).stdout) + docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM" + except (OSError, ValueError, subprocess.CalledProcessError): + docker_vm = "unknown" + lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона " + f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB " + f"каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, " + f"janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB.") + lines.append("") + + failover = EVIDENCE / "failover.txt" + results = [] + if failover.exists(): + text = failover.read_text(encoding="utf-8") + stopped = re.findall(r"the hook stopped (\w+)", text) + results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)] + if results: + lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):") + lines.append("") + lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток " + "(повторены с тем же ключом) | Первая запись после hook, с |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for number, (result, container) in enumerate(zip(results, stopped), start=1): + lines.append(f"| {number} | {container} | {result['observed_instances_before']} | " + f"{result['successful_writes']} | {result['failed_attempts']} | " + f"{result['first_recovery_seconds']} |") + lines.append("") + + memory = load_json("upload-memory.json") + if memory: + lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики " + f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). " + f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:") + lines.append("") + lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | " + "Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for run in memory["runs"]: + for n, r in enumerate(run["replicas"]): + head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | " + f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |" + cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-" + lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | " + f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |") + naive = memory["single_file_extra_rss_mib"] + lines.append("") + lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в " + f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка " + f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в " + f"{memory['api_limit_mib']} MiB.") + lines.append("") + + metrics = load_json("failover-metrics.json") + if metrics: + lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи " + "без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):") + lines.append("") + lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок " + "(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | " + "Вернулась в балансировку после docker start, с |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for r in metrics["runs"]: + how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)" + codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-" + lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | " + f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | " + f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |") + lines.append("") + lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика " + "после остановки не ответила ни разу.") + lines.append("") + + rolling = load_json("rolling-update.json") + if rolling: + during = rolling["during"] + codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-" + lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> " + f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за " + f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, " + f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); " + f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено " + f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.") + lines.append("") + + load = load_json("load.json") + if load: + latency = load["successful_latency_ms"] + lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, " + f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных " + f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных " + f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.") + lines.append("") + + persistence = EVIDENCE / "persistence.txt" + restart_seconds = None + if persistence.exists(): + found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8")) + restart_seconds = found.group(1) if found else None + + impacts = [] + if metrics: + for mode, what, expected in ( + ("stop", "`docker stop` реплики (SIGTERM) под нагрузкой", + "ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"), + ("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой", + "единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")): + runs = [r for r in metrics["runs"] if r["mode"] == mode] + if runs: + impacts.append( + f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); " + f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | " + f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; " + f"потерь 0, дублей 0; реплика вернулась через " + f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |") + if failover.exists() and results: + impacts.append( + f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, " + f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: " + f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через " + f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |") + if rolling: + impacts.append( + f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | " + f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |") + if restart_seconds: + impacts.append( + f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и " + f"прежний токен живы | " + f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |") + consistency = EVIDENCE / "consistency.txt" + if consistency.exists(): + text = consistency.read_text(encoding="utf-8") + timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text) + if len(timings) >= 2: + impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, " + f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | " + f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |") + impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за " + f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |") + if "PASS: S3 written but DB not committed" in text: + impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | " + "вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | " + "GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | " + "мусора не осталось; повторный проход ничего не делает |") + if "PASS: an interrupted upload leaves no row" in text: + impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | " + "строк и объектов не прибавилось, временных файлов 0 | мгновенно |") + impacts += lab5_impacts() + if impacts: + lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |") + lines.append("| --- | --- | --- | --- |") + lines.extend(impacts) + lines.append("") + return "\n".join(lines).strip() + + +def lab5_line(name, pattern): + """Строка вывода test_jobs.py из evidence/.txt, содержащая pattern (или None).""" + path = EVIDENCE / f"{name}.txt" + if not path.exists(): + return None + for line in path.read_text(encoding="utf-8").splitlines(): + if pattern in line and not line.startswith(("#", "---")): + return line.strip() + return None + + +def lab5_impacts(): + """Строки таблицы отказов lab5: наблюдение — дословно строка вывода теста (числа из прогона).""" + rows = [ + ("jobs-recovery", "worker started", "worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker", + "202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready", + "потерь нет; staging удалён после COMMIT"), + ("jobs-recovery", "rabbitmq started", "rabbitmq остановлен во время upload", + "202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется", "потерь нет"), + ("jobs-outbox", "killed between COMMIT and publish", "crash dispatcher между COMMIT задания и publish", + "строка outbox не отмечена -> публикуется после рестарта процесса", "без ручного действия"), + ("jobs-outbox", "after publisher confirm", "crash dispatcher после confirm, до отметки outbox", + "дубль сообщения, worker распознаёт по состоянию job", "один набор вариантов"), + ("jobs-crash", "killed after S3 PUT", "crash worker после PUT в S3, до COMMIT и ack", + "повтор после истечения аренды перезаписывает те же ключи", "один набор, 4 объекта"), + ("jobs-crash", "after COMMIT, before ack", "crash worker после COMMIT, до ack", + "redelivery ready-задачи -> no-op ack", "очередь пуста, дублей нет"), + ("jobs-crash", "frozen attempt", "worker завис (не продлевает аренду)", + "dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing", "ready не откатан"), + ("jobs-limits", "imaging hang", "зависание обработки изображения", + "процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась", "повтор -> ready"), + ("jobs-limits", "memory bomb", "нехватка памяти в процессе обработки (RLIMIT_AS)", + "failed/resource_limit, worker жив", "постоянная ошибка, без повторов"), + ("jobs-retry", "storage_unavailable ->", "2 временные ошибки S3", "backoff с jitter, затем ready", "-"), + ("jobs-retry", "retry_exhausted (public", "5 временных ошибок подряд", + "failed/retry_exhausted, реестр failed, content 409", "replay оператора"), + ("jobs-retry", "ops replay", "replay failed и повторный replay ready", "тот же job_id -> ready; повтор отклонён", + "дубля нет"), + ("jobs-retry", "-> reject -> DLQ", "невалидное сообщение и неизвестный job_id", "reject -> DLQ, dlq-replay не исполняет", + "сообщения сохранены"), + ("jobs-concurrency", "peak processing", "8 заданий в очереди, старт worker", + "WORKER_CONCURRENCY=2, prefetch 1", "-"), + ] + out = [] + for name, pattern, what, expected, recovery in rows: + line = lab5_line(name, pattern) + if line: + out.append(f"| {what} | {expected} | {line.replace('|', '/')} | {recovery} |") + return out + + +def update_report(block): + report = ROOT / "REPORT.md" + text = report.read_text(encoding="utf-8") + begin, end = "", "" + if begin not in text or end not in text: + print("REPORT.md has no measurement markers, only evidence/summary.md was written") + return + head, rest = text.split(begin, 1) + _, tail = rest.split(end, 1) + report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8") + print("REPORT.md: measurement tables updated") + + +def main(): + args = sys.argv[1:] + names = [name for name, _, _ in STEPS] + if args[:1] == ["--from"]: + if len(args) < 2 or args[1] not in names: + raise SystemExit("usage: --from ; steps: " + ", ".join(names)) + selected = names[names.index(args[1]):] + else: + unknown = [a for a in args if a not in names] + if unknown: + raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names)) + selected = args or names + EVIDENCE.mkdir(exist_ok=True) + results = [] + for name, description, command in STEPS: + if name not in selected: + continue + ok, seconds = run_step(name, description, command) + results.append((name, description, ok, seconds)) + if not ok: + print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.") + print(f"After fixing: python student-tests/collect_evidence.py --from {name}") + break + block = measurements() + summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |", + "| --- | --- | --- | --- | --- |"] + summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results] + (EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8") + print("evidence/summary.md written") + passed = all(ok for _, _, ok, _ in results) + if passed: + update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 0000000..66cc2a8 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,347 @@ + +import base64 +import csv +import hashlib +import io +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +PASSWORD = "Test-password-123" + + +class Response: + def __init__(self, status, headers, data): + self.status = status + self.headers = {name.lower(): value for name, value in headers.items()} + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except ValueError: + self.json = None + + @property + def instance(self): + """X-Instance-Id: какая реплика API ответила (ID её контейнера).""" + return self.headers.get("x-instance-id") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30): + """HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError. + + Заодно проверяет то, что контракт требует от любого ответа: + X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401. + """ + all_headers = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + if data is not None: + all_headers.setdefault("Content-Type", "application/json") + if token is not None: + all_headers["Authorization"] = "Bearer " + token + + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + response = Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + response = Response(error.code, error.headers, error.read()) + + expected = expected if isinstance(expected, tuple) else (expected,) + check(response.status in expected, + f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}") + request_id = response.headers.get("x-request-id") + check(request_id, f"{method} {path}: no X-Request-Id header") + check(response.instance, f"{method} {path}: no X-Instance-Id header") + if response.status >= 400: + error = (response.json or {}).get("error") or {} + check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format") + check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header") + if response.status == 401: + check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate") + return response + + +def try_call(method, path, *, body=None, token=None, headers=None, timeout=2): + """Запрос без проверок и без исключений — для нагрузки во время отказа. + + Возвращает Response или None, если соединение оборвалось / вышел timeout. + """ + all_headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + all_headers["Content-Type"] = "application/json" + if token is not None: + all_headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + return Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + try: + return Response(error.code, error.headers, error.read()) + except OSError: + return None + except (urllib.error.URLError, OSError): + return None + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8.""" + return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + + +def login(username, password, expected=201): + return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)}) + + +def new_user(prefix="user", password=PASSWORD): + """Регистрирует пользователя и входит. Возвращает dict: id, username, password, token.""" + username = f"{prefix}_{uuid.uuid4().hex[:10]}" + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": prefix.title(), "password": password}).json + session = login(username, password).json + return {**user, "password": password, "token": session["token"]} + + +def new_chat(owner, *others, title="Test chat"): + member_ids = [owner["id"]] + [user["id"] for user in others] + return call("POST", "/api/v1/chats", 201, token=owner["token"], + body={"title": title, "member_ids": member_ids}).json + + +def new_key(): + return uuid.uuid4().hex + + +def send(user, chat_id, text, expected=201, key=None): + """POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body={"text": text}, headers={"Idempotency-Key": key or new_key()}) + + +def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None): + """POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся.""" + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body=body, headers={"Idempotency-Key": key or new_key()}) + + +def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=202, timeout=60, + wait=True): + """POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра. + + Lab5: успешный приём — 202 со снимком queued (переход 201 -> 202 из COURSE.md). При wait=True + ответ дополняется ожиданием готовности: .json — метаданные после GET в состоянии ready/failed, + .accepted — исходный снимок 202. Так проверки lab4 («файл скачивается побайтно») сохраняются. + """ + query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="") + headers = {"Content-Type": content_type} if content_type is not None else {} + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data, + token=user["token"] if user else None, headers=headers, timeout=timeout) + if reply.status == 202: + check(reply.json["status"] == "queued" and reply.json["variants"] == [], "202 must be a queued snapshot") + reply.accepted = reply.json + if wait: + reply.json = wait_attachment(user, reply.json["id"]) + return reply + + +def wait_attachment(user, attachment_id, statuses=("ready", "failed"), seconds=60): + """GET метаданных, пока вложение не перейдёт в одно из statuses (smoke ждёт ready до 60 с).""" + deadline = time.monotonic() + seconds + while True: + att = call("GET", f"/api/v1/attachments/{attachment_id}", 200, token=user["token"]).json + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"attachment {attachment_id} is {att['status']} after {seconds} s") + time.sleep(0.2) + + +def read_history(user, chat_id, limit=100): + """Вся история чата по страницам. Возвращает (сообщения, число_страниц).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "") + page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json + items += page["items"] + pages += 1 + cursor = page["next_cursor"] + if cursor is None: + return items, pages + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while True: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply: + if reply.status == 200: + return + except (urllib.error.URLError, OSError): + pass + check(time.monotonic() < deadline, f"API is not ready after {seconds} s") + time.sleep(0.5) + + +def compose(*args, env=None): + """docker compose в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам.""" + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def compose_output(*args, env=None): + """docker compose и его вывод (для logs, ps).""" + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env) + return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace") + + +def docker(*args, check_exit=True): + """docker , возвращает stdout.""" + result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit) + return result.stdout.decode("utf-8", "replace").strip() + + +def api_containers(include_stopped=False): + """Полные ID контейнеров сервиса api (по умолчанию только запущенные).""" + args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"] + return sorted(compose_output(*args).split()) + + +def health(container): + """Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''.""" + return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container, + check_exit=False) + + +def lb_servers(): + """Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}.""" + with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply: + rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace")))) + header = [name.lstrip("# ") for name in rows[0]] + servers = {} + for row in rows[1:]: + item = dict(zip(header, row)) + if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"): + servers[item["svname"]] = item.get("status", "") + return servers + + +def lb_up_count(): + """Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP).""" + return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP")) + + +def seen_instances(token, requests=20): + """Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me.""" + return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)} + + +def wait_balanced(replicas, seconds=60): + """Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке.""" + deadline = time.monotonic() + seconds + while True: + running = api_containers() + healthy = [container for container in running if health(container) == "healthy"] + try: + up = lb_up_count() + except OSError: + up = 0 + if len(healthy) >= replicas and up >= replicas: + return running + check(time.monotonic() < deadline, + f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}") + time.sleep(0.5) + + +def redis_cli(*args): + """redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод.""" + command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args) + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command], + cwd=ROOT, capture_output=True, check=True) + return result.stdout.decode("utf-8", "replace").strip() + + +def session_key(token): + """Ключ сессии в Redis, как его считает auth.py: session:.""" + return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def psql(sql): + """Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки). + + Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя. + """ + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, check=False, + ) + output = result.stdout.decode("utf-8", "replace").strip() + error = result.stderr.decode("utf-8", "replace").strip() + return result.returncode, output, error + + +def app_python(code, *args, service="api", env=None): + """Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API). + + Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON). + """ + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return output + + +def janitor(*args, env=None): + """Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт.""" + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += ["janitor", "python", "janitor.py", "--once", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return json.loads(output.splitlines()[-1]) + + +def sql_value(query): + """Одно значение из PostgreSQL (psql -A -t).""" + code, output, error = psql(query) + check(code == 0, f"psql failed: {error[-300:]}") + return output + + +def run_tests(tests): + """Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал.""" + failed = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста + failed += 1 + print(f"FAIL: {test.__name__}: {exc}") + print(f"{len(tests) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/demo_instances.py b/student-tests/demo_instances.py new file mode 100644 index 0000000..8f97f1c --- /dev/null +++ b/student-tests/demo_instances.py @@ -0,0 +1,16 @@ + +from common import call, compose_output, new_user + + +def main(): + user = new_user("demo") + print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:") + for number in range(1, 9): + reply = call("GET", "/api/v1/users/me", 200, token=user["token"]) + print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}") + print() + print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 0000000..81416de --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,28 @@ + +import subprocess + +from common import ROOT + +OUTPUT = ROOT / "evidence" / "explain-history.txt" + + +def main(): + sql = (ROOT / "student-tests" / "explain.sql").read_bytes() + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + cwd=ROOT, input=sql, capture_output=True, check=False, + ) + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_bytes(result.stdout + result.stderr) + if result.returncode != 0: + print(result.stderr.decode("utf-8", "replace")) + raise SystemExit("FAIL: psql returned an error, see output above") + for line in result.stdout.decode("utf-8", "replace").splitlines(): + if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line: + print(line) + print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 0000000..01cd9e7 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,76 @@ + +\pset footer off +BEGIN; + +CREATE TEMP TABLE seed_chats AS + SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size + FROM generate_series(0, 50) AS n; +CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; + +INSERT INTO users (id, username, display_name, password_hash) + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; +INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), + timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' + FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; +ANALYZE messages; + +SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset +SELECT created_at AS cursor_time, id AS cursor_id FROM messages + WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== Data volume' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat +FROM messages; + +\echo '' +\echo '=== History index' +SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history'; + +\echo '' +\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY m.created_at, m.id LIMIT 51; + +\echo '' +\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id OFFSET 10000 LIMIT 51; + +\echo '' +\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)' +DROP INDEX messages_history; +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; +\echo '' +\echo '=== ROLLBACK: test data removed, index is back' +SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%'; diff --git a/student-tests/failover_metrics.py b/student-tests/failover_metrics.py new file mode 100644 index 0000000..19421a1 --- /dev/null +++ b/student-tests/failover_metrics.py @@ -0,0 +1,129 @@ +import json +import platform +import sys +import threading +import time + +from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced +from loadgen import Load, summary + +WARMUP_SECONDS = 3 +AFTER_SECONDS = 10 +OUTPUT = ROOT / "evidence" / "failover-metrics.json" + + +class LbWatch: + """Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP).""" + + def __init__(self): + self.points, self.stopping = [], threading.Event() + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stopping.is_set(): + try: + self.points.append((time.monotonic(), lb_up_count())) + except OSError: + pass + time.sleep(0.1) + + def start(self): + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join(timeout=3) + + def first_drop(self, after, replicas): + return next((t for t, up in self.points if t >= after and up < replicas), None) + + +def measure(container, mode, replicas): + wait_balanced(replicas) + load, watch = Load(), LbWatch() + watch.start() + load.start() + time.sleep(WARMUP_SECONDS) + t0 = time.monotonic() + if mode == "stop": + docker("stop", "--time", "8", container) + else: + docker("kill", container) + command_seconds = time.monotonic() - t0 + time.sleep(max(0.0, AFTER_SECONDS - command_seconds)) + end = t0 + AFTER_SECONDS + load.stop() + watch.stop() + + errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]] + errors_end = max((e[0] + e[5] for e in errors), default=None) + drop = watch.first_drop(t0, replicas) + served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]] + target_hits = sum(1 for instance in served_after if instance and container.startswith(instance)) + integrity = load.verify() + + started = time.monotonic() + docker("start", container) + wait_balanced(replicas) + rejoin_seconds = time.monotonic() - started + + return { + "replica": container[:12], + "mode": mode, + "command_seconds": round(command_seconds, 2), + "lb_excluded_after_seconds": round(drop - t0, 2) if drop else None, + "errors": len(errors), + "error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"], + "errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0, + "before": summary(load, t0 - WARMUP_SECONDS, t0), + "after": summary(load, t0, end), + "stopped_replica_answers_after_stop": target_hits, + "integrity": integrity, + "rejoin_seconds": round(rejoin_seconds, 2), + } + + +def main(): + modes = sys.argv[1:] or ["stop", "kill"] + check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]") + containers = api_containers() + replicas = len(containers) + check(replicas >= 2, "need at least 2 running api replicas") + print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, " + f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and " + f"{AFTER_SECONDS} s after the failure") + runs = [] + for container in containers: + for mode in modes: + print(f"--- {mode} {container[:12]} ...") + run = measure(container, mode, replicas) + runs.append(run) + print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after " + f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, " + f"stopped after {run['errors_stopped_after_seconds']} s") + print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; " + f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; " + f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}") + print(f" confirmed {run['integrity']['confirmed_writes']}, in history " + f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; " + f"back in the balancer {run['rejoin_seconds']} s after docker start") + check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered") + check(run["after"]["write_rps"] > 0, "no successful writes after the failure") + print() + print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s") + for r in runs: + print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} " + f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} " + f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS, + "after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()}, + "runs": runs, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; " + f"saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/progressive.jpg b/student-tests/fixtures/progressive.jpg new file mode 100644 index 0000000000000000000000000000000000000000..38faa2bd4a2bb00b572233684dd5474d46572d38 GIT binary patch literal 1081 zcmex=^lOiET&UP@Y7ModgWM?qOlT~kX_QeM|USHnP6LsJ7}2qQZ?I~NC+Fc+7w zhLo6;2Fc+6AqGJXh5&{NW=16jCP7AKLB{__7$kv?WCpnw4wxC3SlHNsQWA_{0GD83 zU}0ipXa0YSfrpuqfk}W_0O-e0Us7g@q%7gR_iLh?m&0PUC4!SCXq;RjxhlxgB!%5l zB;^Y4y)P5pd>t06DhN&v(m3fMxhj9ZT2I4sr=BT|yYB~Rob&`LVloP1l{8g0_4@wg z;slu|fs=#Gt1kgXlzOH#fHkY6t=#u#mJ#!uE~6l}_q85C?}^ISER2f9~O z0L335cMAd?%K;Q+WM%509st@UCI}20q>y1?U}9hdTLuhFdqzD7I4%wW$=wjJ@hAk` zcmx4&%0bQs8iisK%-KvJlU88?Kusb7=z0pEUS+dq62l%C5cdHC!(Oo`S>l)>t%&2W z6vDs^^q0V|se8Fk9r(1>j{TMMuhu{Ow;I1Ktz-Vh1QNe>2U zW@aX!`+*(+s;&jfGq4D<3Mm>ovIz$!vMUve7&T5@$f4}C@t|nX#SbdRNkvVZTw>x9 zl2WQ_>Kd9_CZ=ZQ7M51dF0O9w9-dyoA)#U65s^{JDXD4c8JStdC8cHM6_r)ZEv;?s z9i3g1CQq3GGAU*RJ2V zdF$b$$4{OPfBE|D`;VW$K>lK6V1{@LNJ2b@<}X2@znFm0!om*n7b88f2KE_o9%~}Y zXK;@p{B?_ghnW!=dCYp^Nk zI`iL#Ki&>f`#Y>_;hTV?BFa#mKfu=BySiva)fccMR;=xKyLZJ}+l8;3MHfC61R6dA zsN_%Cy{lsJIJc*Fss0-!LA1S;hP4?4;WTe zNi6EY^g|NJ;Qc@!feePYdTl7gV2F?4&Tusb`z-gZYYxzbu2CzhCLGPE06H8TNU&hp z7a3yT2M$D#1O7sTygm!&S4fcV2U`gY#9&*Xv# bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes: + """PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой). + + pad_to — дополнить файл текстовым чанком tEXt после IDAT до точного размера в байтах. + idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое). + """ + channels = 1 if gray else 3 + header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace) + if idat is None: + compressor = zlib.compressobj(6) + parts = [] + # Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход. + passes = [(width, height)] if not interlace else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + for pass_width, pass_height in passes: + if pass_width <= 0 or pass_height <= 0: + continue + row = b"\x00" + bytes(pass_width * channels) + for _ in range(pass_height): + parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row)) + parts.append(compressor.flush()) + idat = b"".join(parts) + body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat) + if pad_to is not None: + need = pad_to - len(body) - 12 - 12 - 8 # заголовок чанка-заполнителя, IEND, ключ "Comment\0" + assert need >= 0, "image is already larger than pad_to" + body += chunk(b"tEXt", b"Comment\x00" + b"a" * need) + return body + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями).""" + side = int(((target_bytes - 4096) / 3) ** 0.5) + return png(side, side, noise=True) + + +def sample_png() -> bytes: + return (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (FIXTURES / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (FIXTURES / "progressive.jpg").read_bytes() + + +def flip_byte(data: bytes, position: int) -> bytes: + return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:] + + +GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + + +# --- сборка PNG из произвольных чанков (для проверок смысла чанков при верных CRC) --- + +def ihdr(width, height, depth=8, color=2, interlace=0): + return b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace) + + +def zero_idat(width, height, depth=8, channels=3): + """Сжатые пиксели: каждая строка = байт фильтра 0 + нули (индекс палитры 0 / чёрный).""" + row_bytes = (width * depth * channels + 7) // 8 + return b"IDAT", zlib.compress((b"\x00" + bytes(row_bytes)) * height) + + +def assemble(*chunks): + """PNG из чанков (kind, data) с ВЕРНЫМИ CRC; IEND добавляется, если не передан.""" + if not chunks or chunks[-1][0] != b"IEND": + chunks = chunks + ((b"IEND", b""),) + return b"\x89PNG\r\n\x1a\n" + b"".join(chunk(kind, data) for kind, data in chunks) diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 0000000..998548c --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,102 @@ +import math +import threading +import time + +from common import check, new_chat, new_key, new_user, read_history, try_call + + +class Load: + def __init__(self, writers=4, timeout=2): + self.alice, self.bob = new_user("load_a"), new_user("load_b") + self.chat = new_chat(self.alice, self.bob, title="load") + self.writers, self.timeout = writers, timeout + self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с) + self.confirmed = [] # сообщения, на которые пришёл 201 + self.lock = threading.Lock() + self.stopping = threading.Event() + self.threads = [] + + def _record(self, kind, started, reply, ok): + code = reply.status if reply is not None else "conn" + instance = reply.instance if reply is not None else None + with self.lock: + self.events.append((started, kind, ok, code, instance, time.monotonic() - started)) + + def _writer(self, number): + path = f"/api/v1/chats/{self.chat['id']}/messages" + sent = 0 + key, text = new_key(), f"load-{number}-{sent}" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("POST", path, body={"text": text}, token=self.alice["token"], + headers={"Idempotency-Key": key}, timeout=self.timeout) + ok = reply is not None and reply.status == 201 + self._record("write", started, reply, ok) + if ok: + with self.lock: + self.confirmed.append(reply.json) + sent += 1 + key, text = new_key(), f"load-{number}-{sent}" + else: + time.sleep(0.05) # и повторить тот же text с тем же ключом + + def _reader(self): + path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout) + self._record("read", started, reply, reply is not None and reply.status == 200) + time.sleep(0.05) + + def start(self): + self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)] + self.threads.append(threading.Thread(target=self._reader, daemon=True)) + for thread in self.threads: + thread.start() + + def stop(self): + self.stopping.set() + for thread in self.threads: + thread.join(timeout=self.timeout + 5) + + def window(self, start, end, kind=None): + with self.lock: + return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)] + + def verify(self): + """Все подтверждённые сообщения есть в истории, id и тексты не повторяются.""" + history, _ = read_history(self.bob, self.chat["id"]) + ids = [message["id"] for message in history] + texts = [message["text"] for message in history] + by_id = {message["id"]: message for message in history} + check(len(ids) == len(set(ids)), "duplicated message ids in history") + check(len(texts) == len(set(texts)), "a retried request created a duplicate message") + lost = [m for m in self.confirmed if by_id.get(m["id"]) != m] + check(not lost, f"{len(lost)} confirmed messages are lost or changed") + return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0} + + +def summary(load, start, end): + """Успешные RPS, ошибки и задержки на отрезке [start, end).""" + events = load.window(start, end) + writes = [e for e in events if e[1] == "write"] + ok_writes = sorted(e[5] for e in writes if e[2]) + errors = [e for e in events if not e[2]] + seconds = max(end - start, 1e-6) + + def percentile(p): + return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None + + codes = {} + for e in errors: + codes[str(e[3])] = codes.get(str(e[3]), 0) + 1 + return { + "seconds": round(seconds, 2), + "write_rps": round(len(ok_writes) / seconds, 1), + "reads": len([e for e in events if e[1] == "read"]), + "errors": len(errors), + "error_codes": codes, + "write_p50_ms": percentile(0.50), + "write_p95_ms": percentile(0.95), + "write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None, + } diff --git a/student-tests/redis_info.py b/student-tests/redis_info.py new file mode 100644 index 0000000..6535eae --- /dev/null +++ b/student-tests/redis_info.py @@ -0,0 +1,29 @@ +import subprocess + +from common import ROOT, login, new_user, redis_cli, session_key + + +def without_password(): + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"], + cwd=ROOT, capture_output=True, check=False) + return (result.stdout + result.stderr).decode("utf-8", "replace").strip() + + +def main(): + user = new_user("info") + token = login(user["username"], user["password"]).json["token"] + key = session_key(token) + print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}") + print(f"TTL of the key, seconds: {redis_cli('ttl', key)}") + keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines() + print(f"sessions in Redis: {len(keys)}") + for name in ("appendonly", "appendfsync", "maxmemory-policy"): + print("config", " = ".join(redis_cli("config", "get", name).split())) + for line in redis_cli("info", "persistence").splitlines(): + if line.startswith(("aof_enabled", "aof_last_write_status")): + print(line) + print(f"without password: {without_password()}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 0000000..f2f3ffb --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,88 @@ +import json +import os +import time + +from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced +from loadgen import Load, summary + +OUTPUT = ROOT / "evidence" / "rolling-update.json" + + +def env_of(container): + lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +def wait_new_replica(container, up_needed, seconds=90): + deadline = time.monotonic() + seconds + while health(container) != "healthy" or lb_up_count() < up_needed: + check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") + time.sleep(0.2) + + +def main(): + old = api_containers() + replicas = len(old) + check(replicas >= 2, "need at least 2 running api replicas") + wait_balanced(replicas) + old_version = env_of(old[0]).get("APP_VERSION") + version = time.strftime("rolling-%Y%m%d-%H%M%S") + env = dict(os.environ, APP_VERSION=version) + + print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") + started = time.monotonic() + compose("build", "api", "migrate", env=env) + print(f"image built in {time.monotonic() - started:.1f} s") + migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) + print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] + .split("messenger.db: ")[-1]) + + load = Load() + load.start() + time.sleep(3) + t0 = time.monotonic() + steps = [] + for container in old: + step_start = time.monotonic() + before = set(api_containers()) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) + new = sorted(set(api_containers()) - before) + check(len(new) == 1, f"expected one new container, got {len(new)}") + wait_new_replica(new[0], replicas + 1) + ready = time.monotonic() - step_start + docker("stop", "--time", "15", container) + docker("rm", container) + steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), + "step_seconds": round(time.monotonic() - step_start, 2)}) + print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " + f"step {steps[-1]['step_seconds']} s") + t1 = time.monotonic() + time.sleep(3) + load.stop() + + final = api_containers() + versions = sorted({env_of(c).get("APP_VERSION") for c in final}) + integrity = load.verify() + during = summary(load, t0, t1) + before_stats = summary(load, t0 - 3, t0) + print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") + print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " + f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " + f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") + print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " + f"lost 0, duplicates 0") + check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") + check(versions == [version], "not every replica runs the new version") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "from_version": old_version, "to_version": version, "replicas": replicas, + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, + "steps": steps, "update_seconds": round(t1 - t0, 2), + "before": before_stats, "during": during, "integrity": integrity, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: all {replicas} replicas replaced one by one under load, " + f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_failover.py b/student-tests/run_failover.py new file mode 100644 index 0000000..0f28ec8 --- /dev/null +++ b/student-tests/run_failover.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys + +from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced + +HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + + +def ps(): + return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() + + +def aim(wanted): + """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" + probe = call("GET", "/health/live", 200).instance + if not any(container.startswith(probe) for container in wanted): + call("GET", "/health/live", 200) # теперь последней ответила другая реплика + + +def main(): + containers = api_containers() + check(len(containers) >= 2, "need at least 2 running api replicas") + print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") + stopped_once = set() + for attempt in range(1, 7): + wait_balanced(len(containers)) + if len(containers) == 2: + aim(set(containers) - stopped_once) + print(f"=== run {attempt}: api containers before") + print(ps()) + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public failover test failed") + print("api containers after") + print(ps()) + stopped = [container for container in containers if container not in api_containers()] + check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") + print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " + f"other api containers kept running, postgres/redis/lb were not touched") + stopped_once.add(stopped[0]) + docker("start", stopped[0]) + wait_balanced(len(containers)) + print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") + if stopped_once == set(containers): + break + check(stopped_once == set(containers), "not every replica was stopped by the public test") + print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_persistence.py b/student-tests/run_persistence.py new file mode 100644 index 0000000..fada376 --- /dev/null +++ b/student-tests/run_persistence.py @@ -0,0 +1,47 @@ +import os +import subprocess +import sys +import time + +from common import BASE_URL, ROOT, check, compose_output, docker + +HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + + +def started(): + containers = compose_output("ps", "-q").split() + lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines() + return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)} + + +def main(): + before = started() + print("before: container, ID, started at") + for name, (cid, at) in sorted(before.items()): + print(f" {name:<28} {cid[:12]} {at}") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + begin = time.monotonic() + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + seconds = time.monotonic() - begin + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public persistence test failed") + after = started() + print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?") + restarted = [] + for name, (cid, at) in sorted(after.items()): + same_id = before.get(name, ("", ""))[0] == cid + new_start = before.get(name, ("", ""))[1] != at + print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}") + check(same_id, f"{name}: container was recreated, volumes/IDs must stay") + if new_start: + restarted.append(name.split("-")[-2]) + for service in ("postgres", "redis", "s3", "rabbitmq", "api", "janitor", "dispatcher", "worker", "lb"): + check(service in restarted, f"{service} was not restarted by the hook") + print("PASS: the hook restarted postgres, redis, s3 (SeaweedFS), rabbitmq, all api replicas, janitor, dispatcher, worker and lb (same containers, " + "new start time); data, the attachment bytes and the old token survived") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..7c770d9 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,258 @@ +import base64 +import datetime as dt +import uuid + +from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send + + +def parse_time(value): + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + + +def test_duplicate_username(): + body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} + call("POST", "/api/v1/users", 201, body=body) + call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user")) + print("PASS: duplicate username returns 409") + + +def test_invalid_json(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", "/api/v1/users", 400, raw=b'{"username":') + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json", + headers={"Idempotency-Key": new_key()}) + print("PASS: invalid JSON returns 400") + + +def test_wrong_fields(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD}) + call("POST", "/api/v1/users", 400, + body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True}) + # Отправителя нельзя подставить: он берётся из сессии. + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()}) + print("PASS: wrong and extra fields return 400") + + +def test_foreign_chat(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + chat = new_chat(alice, bob) + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"]) + send(eve, chat["id"], "not my chat", expected=404) + # Несуществующий чат выглядит так же, как чужой. + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"]) + print("PASS: foreign chat returns 404") + + +def test_empty_message(): + alice = new_user("alice") + chat = new_chat(alice) + for text in ("", " ", "\n\t"): + send(alice, chat["id"], text, expected=400) + send(alice, chat["id"], "я" * 4001, expected=400) + send(alice, chat["id"], "я" * 4000) + print("PASS: empty and too long messages return 400, 4000 chars is ok") + + +def test_pagination(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)] + + history, pages = read_history(bob, chat["id"], limit=2) + check(pages == 4, f"expected 4 pages, got {pages}") + check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages") + + other = new_chat(alice) + for i in range(3): + send(alice, other["id"], f"Other {i}") + first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) + print("PASS: pagination works across multiple pages, foreign cursor returns 400") + + +def test_empty_cursor(): + alice = new_user("alice") + chat = new_chat(alice) + send(alice, chat["id"], "one message") + for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"): + for query in ("?cursor=", "?cursor", "?limit=10&cursor="): + call("GET", path + query, 400, token=alice["token"]) + call("GET", path, 200, token=alice["token"]) + print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page") + + +def test_chat_list_pagination(): + alice, bob = new_user("alice"), new_user("bob") + created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] + seen, cursor = [], None + while True: + query = "?limit=1" + (f"&cursor={cursor}" if cursor else "") + page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json + seen += [chat["id"] for chat in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + check(seen == created, "chat list pages are wrong") + print("PASS: chat list pagination") + + +def test_unicode(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob, title="Чат 👋") + text = "Привет 👋 你好" + check(send(alice, chat["id"], text).json["text"] == text, "text changed on write") + history, _ = read_history(bob, chat["id"]) + check(history[-1]["text"] == text, "text changed on read") + print("PASS: unicode text is stored as is") + + +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}, + headers={"Idempotency-Key": new_key()}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + + +def test_user_has_no_password(): + body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} + user = call("POST", "/api/v1/users", 201, body=body).json + check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}") + print("PASS: user response has no password or hash") + + +def test_password_rules(): + def register(password, expected): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password} + call("POST", "/api/v1/users", expected, body=body) + + register("x" * 11, 400) + register("x" * 129, 400) + register(" " * 12, 400) + register("x" * 12, 201) + register("я" * 128, 201) + print("PASS: password length 12..128, blank password is rejected") + + +def test_login_gives_token(): + username = "lg_" + uuid.uuid4().hex[:10] + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Login", "password": PASSWORD}).json + reply = login(username, PASSWORD) + session = reply.json + check(session["token_type"] == "Bearer", "token_type must be Bearer") + check(session["user"] == user, "session user differs from registered user") + check(len(session["token"]) >= 43, "token is too short") + check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque") + check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached") + left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s") + me = call("GET", "/api/v1/users/me", token=session["token"]).json + check(me == user, "/users/me returned another user") + print("PASS: Basic login returns opaque Bearer token with expiry") + + +def test_wrong_password(): + user = new_user("wp") + wrong = login(user["username"], "Wrong-password-000", expected=401) + unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) + check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") + check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") + print("PASS: wrong password and unknown user both return the same 401") + + +def test_malformed_basic(): + no_colon = base64.b64encode(b"just_username").decode() + for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"): + headers = {} if header is None else {"Authorization": header} + call("POST", "/api/v1/auth/sessions", 401, headers=headers) + print("PASS: malformed Basic header returns 401") + + +def test_password_with_colon_and_unicode(): + password = "пароль:с:двоеточием 👋" + user = new_user("colon", password=password) + call("GET", "/api/v1/users/me", token=user["token"]) + print("PASS: password with colon and unicode works") + + +def test_bearer_required(): + user = new_user("br") + for headers in ({}, {"Authorization": basic(user["username"], user["password"])}, + {"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}): + reply = call("GET", "/api/v1/users/me", 401, headers=headers) + check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer") + call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]}) + print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401") + + +def test_x_user_id_ignored(): + alice, bob = new_user("alice"), new_user("bob") + me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json + check(me["id"] == alice["id"], "X-User-Id changed the current user") + print("PASS: X-User-Id does not change the current user") + + +def test_auth_before_body(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) + print("PASS: 401 comes before 400 and 404") + + +def test_logout(): + user = new_user("lo") + second = login(user["username"], user["password"]).json["token"] + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + call("GET", "/api/v1/users/me", 401, token=user["token"]) + call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива + again = login(user["username"], user["password"]).json["token"] + check(again != user["token"], "new login must give a new token") + call("GET", "/api/v1/users/me", 200, token=again) + print("PASS: logout revokes only the current session") + + +if __name__ == "__main__": + run_tests([ + test_duplicate_username, + test_invalid_json, + test_wrong_fields, + test_foreign_chat, + test_empty_message, + test_pagination, + test_empty_cursor, + test_chat_list_pagination, + test_unicode, + test_nul_character_is_stored, + test_user_has_no_password, + test_password_rules, + test_login_gives_token, + test_wrong_password, + test_malformed_basic, + test_password_with_colon_and_unicode, + test_bearer_required, + test_x_user_id_ignored, + test_auth_before_body, + test_logout, + ]) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..e57fe28 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,338 @@ +"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ. + +Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа: +пользовательская ошибка не должна становиться 500. +""" +import hashlib +import http.client +import socket +import struct +import urllib.parse +import uuid +import zlib + +import images +from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload + +URL = urllib.parse.urlsplit(BASE_URL) + + +def setup(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + return alice, bob, eve, new_chat(alice, bob) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def check_error(reply, code): + check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}") + + +def test_png_and_jpeg_roundtrip(): + alice, bob, _, chat = setup() + for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"), + (images.sample_jpeg(), "image/jpeg", "sample.jpg"), + (images.progressive_jpeg(), "image/jpeg", "progressive.jpg"), + (images.png(33, 17, interlace=1), "image/png", "interlaced.png"), + (images.png(1, 1, gray=True), "image/png", "1x1.png")): + att = upload(alice, chat["id"], data, ctype, name).json + check(att["status"] == "ready" and att["error_code"] is None, "not ready") + check([v["name"] for v in att["variants"]] == ["thumbnail", "crop", "optimized"], "lab5 variants missing") + check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ") + check(att["content_type"] == ctype and att["filename"] == name, "metadata differs") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta == att, "GET metadata differs from POST response") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "downloaded bytes differ") + check(got.headers["content-type"] == ctype, "wrong Content-Type on download") + check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length") + check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing") + print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes") + print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte") + + +def test_size_limit_exact_boundary(): + alice, _, _, chat = setup() + exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES) + check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB") + att = upload(alice, chat["id"], exact).json + check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed") + print(f"exactly {len(exact)} bytes (10 MiB) -> 202, then ready") + over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1) + reply = upload(alice, chat["id"], over, expected=413) + check_error(reply, "payload_too_large") + print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body") + print("PASS: the 10 MiB limit is exact") + + +def chunked_upload(user, chat_id, total, piece=64 * 1024): + """Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении.""" + conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60) + + def body(): + sent = 0 + head = images.sample_png()[:64] + yield head + sent += len(head) + while sent < total: + part = bytes(min(piece, total - sent)) + sent += len(part) + yield part + try: + conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(), + headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]}, + encode_chunked=True) + reply = conn.getresponse() + return reply.status, reply.read() + finally: + conn.close() + + +def test_size_limit_streaming(): + alice, _, _, chat = setup() + status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024) + check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}") + print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed") + call("GET", "/health/ready", 200) + print("PASS: oversized streamed body is cut off, the API keeps working") + + +def test_pixel_limit(): + alice, _, _, chat = setup() + at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей + att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json + print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 202, ready {att['id'][:8]}") + over = images.png(8001, 5000, gray=True) + check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large") + print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413") + bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01") + check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large") + print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded") + print("PASS: pixel limit is checked before decompression") + + +def test_damaged_images(): + alice, _, _, chat = setup() + png, jpeg = images.sample_png(), images.sample_jpeg() + cases = [ + ("PNG cut in half", png[:len(png) // 2], "image/png"), + ("PNG without IEND", png[:-12], "image/png"), + ("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"), + ("PNG with garbage after IEND", png + b"garbage", "image/png"), + ("PNG signature only", png[:8], "image/png"), + ("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"), + ("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"), + ("JPEG header only", jpeg[:20], "image/jpeg"), + ("random bytes", uuid.uuid4().bytes * 100, "image/png"), + ("plain text", "это не картинка".encode(), "image/jpeg"), + ] + for name, data, ctype in cases: + reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415) + check_error(reply, "unsupported_media_type") + print(f"{name} ({len(data)} bytes, {ctype}) -> 415") + check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request") + print("empty body -> 400") + print("PASS: damaged and foreign files are rejected with 415, never 500") + + +def test_png_chunk_semantics(): + """CRC у всех чанков верные, но содержимое нарушает спецификацию PNG -> 415, а не 201. + + Регрессия: PNG 1x1, глубина 8, палитровый, PLTE из 771 байта (257 записей) раньше получал 201 ready, + хотя Pillow его отклоняет (invalid palette size). + """ + alice, bob, _, chat = setup() + from images import assemble, ihdr, zero_idat + pal = lambda entries: (b"PLTE", bytes(3 * entries)) + idat_p = zero_idat(1, 1, 8, 1) + bad = [ + ("PLTE 771 bytes = 257 entries (review case)", + assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(771)), (b"IDAT", zlib.compress(b"\0\0")))), + ("PLTE length not a multiple of 3", assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(4)), idat_p)), + ("empty PLTE", assemble(ihdr(1, 1, 8, 3), (b"PLTE", b""), idat_p)), + ("palette larger than bit depth allows (depth 1, 3 entries)", + assemble(ihdr(1, 1, 1, 3), pal(3), zero_idat(1, 1, 1, 1))), + ("palette image without PLTE", assemble(ihdr(1, 1, 8, 3), idat_p)), + ("PLTE in a grayscale image", assemble(ihdr(1, 1, 8, 0), pal(2), zero_idat(1, 1, 8, 1))), + ("PLTE after IDAT", assemble(ihdr(1, 1, 8, 3), idat_p, pal(2))), + ("two PLTE chunks", assemble(ihdr(1, 1, 8, 3), pal(2), pal(2), idat_p)), + ("tRNS longer than the palette", assemble(ihdr(1, 1, 8, 3), pal(2), (b"tRNS", bytes(3)), idat_p)), + ("tRNS before PLTE", assemble(ihdr(1, 1, 8, 3), (b"tRNS", bytes(1)), pal(2), idat_p)), + ("tRNS of wrong size for RGB (4 bytes instead of 6)", + assemble(ihdr(1, 1, 8, 2), (b"tRNS", bytes(4)), zero_idat(1, 1, 8, 3))), + ("tRNS in an image with alpha channel", + assemble(ihdr(1, 1, 8, 6), (b"tRNS", bytes(2)), zero_idat(1, 1, 8, 4))), + ("IEND with data", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"IEND", b"x"))), + ("unknown critical chunk", assemble(ihdr(1, 1, 8, 2), (b"ABCD", b""), zero_idat(1, 1, 8, 3))), + ("IDAT chunks not consecutive", + assemble(ihdr(1, 2, 8, 2), (b"IDAT", zero_idat(1, 2, 8, 3)[1][:5]), (b"tEXt", b"k\0v"), + (b"IDAT", zero_idat(1, 2, 8, 3)[1][5:]))), + ("gAMA after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"gAMA", bytes(4)))), + ("gAMA of 2 bytes instead of 4", assemble(ihdr(1, 1, 8, 2), (b"gAMA", bytes(2)), zero_idat(1, 1, 8, 3))), + ("3 MiB metadata chunk before IDAT", assemble(ihdr(1, 1, 8, 2), (b"prVt", bytes(3 * 1024 * 1024)), + zero_idat(1, 1, 8, 3))), + ("private chunk after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"prVt", bytes(100)))), + ("animated PNG frame (fdAT)", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"fdAT", bytes(10)))), + ("tIME of 2 bytes instead of 7", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tIME", bytes(2)))), + ("tEXt without keyword separator", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tEXt", b"nokey"))), + ("zTXt with broken zlib after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0garbage"))), + ("zTXt text bomb: 50 MB of text in ~50 KB after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0" + zlib.compress(bytes(50_000_000))))), + ] + for name, data in bad: + check_error(upload(alice, chat["id"], data, filename="bad.png", expected=415), "unsupported_media_type") + print(f"{name} ({len(data)} bytes, valid CRCs) -> 415") + good = [ + ("palette 256 entries + tRNS + tEXt and tIME after IDAT", + assemble(ihdr(2, 2, 8, 3), pal(256), (b"tRNS", bytes(256)), zero_idat(2, 2, 8, 1), + (b"tEXt", b"Comment\0ok"), (b"tIME", struct.pack(">HBBBBB", 2026, 10, 22, 12, 0, 0)))), + ("palette depth 2 with 4 entries", assemble(ihdr(3, 1, 2, 3), pal(4), zero_idat(3, 1, 2, 1))), + ("RGB with 6-byte tRNS and gAMA", assemble(ihdr(1, 1, 8, 2), (b"gAMA", struct.pack(">I", 45455)), + (b"tRNS", bytes(6)), zero_idat(1, 1, 8, 3))), + ("compressed zTXt (100 KB of text) and iTXt after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"Comment\0\0" + zlib.compress(b"a" * 100_000)), + (b"iTXt", b"Title\0\1\0ru\0\0" + zlib.compress("Привет".encode())))), + ] + for name, data in good: + att = upload(alice, chat["id"], data, filename="good.png").json + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "valid PNG changed") + print(f"{name} -> 201, downloaded byte for byte") + print("PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted") + + +def test_mime_substitution(): + alice, _, _, chat = setup() + cases = [ + ("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"), + ("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"), + ("GIF bytes sent as image/png", images.GIF, "image/png"), + ("PNG sent as image/gif", images.sample_png(), "image/gif"), + ("PNG sent as text/html", images.sample_png(), "text/html"), + ("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"), + ("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"), + ] + for name, data, ctype in cases: + check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type") + print(f"{name} -> 415") + att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json + check(att["content_type"] == "image/png", "content type must be normalised") + print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png") + print("PASS: the type is taken from the content; a declared type that does not match is refused") + + +def test_filenames(): + alice, _, _, chat = setup() + bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png", + "bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256] + for name in bad: + check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request") + print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400") + check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request") + print("no filename -> 400") + for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"): + att = upload(alice, chat["id"], images.sample_png(), filename=name).json + check(att["filename"] == name, "display name changed") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded") + print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only") + print("PASS: filename never becomes a path; dangerous names are refused") + + +def test_access_control(): + alice, bob, eve, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + for suffix in ("", "/content"): + reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + check_error(reply, "not_found") + call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400") + check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found") + check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found") + upload(None, chat["id"], images.sample_png(), expected=401) + print("upload into someone else's / unknown chat -> 404, without token -> 401") + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=preview", 400, token=bob["token"]) + for variant in ("original", "thumbnail", "crop", "optimized"): + call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}", 200, token=bob["token"]) + print("variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)") + print("PASS: only chat members can see an attachment") + + +def test_message_attachments(): + alice, bob, eve, chat = setup() + first = upload(alice, chat["id"], images.sample_png()).json + second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json + msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json + check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed") + print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member") + plain = post_message(bob, chat["id"], "без вложений").json + check(plain["attachment_ids"] == [], "default attachment_ids must be []") + reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json + history, _ = read_history(bob, chat["id"]) + check(history == [msg, plain, reused], "history differs from the confirmed messages") + print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed") + + other_chat = new_chat(alice, eve) + foreign = upload(alice, other_chat["id"], images.sample_png()).json + for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"), + ([first["id"], foreign["id"]], "own + foreign attachment"), + ([str(uuid.uuid4())], "unknown attachment_id")): + check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found") + print(f"{what} -> 404") + check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found") + print("outsider referencing an attachment of the chat -> 404") + for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"), + ([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"), + (["not-a-uuid"], "x", "bad UUID"), + ([], "", "empty text without attachments"), + ([], " ", "whitespace text without attachments")): + check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request") + print(f"{what} -> 400") + history_after, _ = read_history(bob, chat["id"]) + check(history_after == history, "a rejected message was stored") + print("PASS: only attachments of the same chat can be linked; rejected requests change nothing") + + +def test_idempotency_with_attachments(): + alice, bob, _, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json + key = new_key() + first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + check(again == first, "replay returned a different message") + check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict") + check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict") + key2 = new_key() + plain = post_message(alice, chat["id"], "текст", None, key=key2).json + check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field") + history, _ = read_history(bob, chat["id"]) + check(len(history) == 2, "replays created extra messages") + print("same key + same attachments -> same 201; other attachments -> 409; missing field == []") + print("PASS: Idempotency-Key covers attachment_ids") + + +def test_aborted_upload(): + alice, _, _, chat = setup() + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000)) + sock.close() # клиент пропал на середине тела + print("sent 1.1 MB of a declared 5 MB body and closed the connection") + att = upload(alice, chat["id"], images.sample_png()).json + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + print("the API keeps serving uploads; nothing of the aborted request is visible " + "(DB/S3 side is checked by test_consistency.py)") + print("PASS: an interrupted upload is dropped") + + +if __name__ == "__main__": + run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming, + test_pixel_limit, test_damaged_images, test_png_chunk_semantics, test_mime_substitution, test_filenames, test_access_control, + test_message_attachments, test_idempotency_with_attachments, test_aborted_upload]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 0000000..01f2251 --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,103 @@ +from collections import Counter + +import images +from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload + + +def test_requests_spread_over_replicas(): + alice = new_user("alice") + replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)] + served = Counter(reply.instance for reply in replies) + print(f"20 x GET /users/me with one token -> replicas: {dict(served)}") + check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned") + check(len(served) >= 2, "all requests went to one replica: balancing does not work") + print("PASS: one session works on every replica, no second login (sessions are shared in Redis)") + + +def test_write_on_one_read_on_other(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = send(alice, chat["id"], "written on one replica") + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]) + check(sent.json in reply.json["items"], "confirmed message is not visible on another replica") + readers.add(reply.instance) + print(f"message written by {sent.instance}; read back by {sorted(readers)}") + check(readers - {sent.instance}, "no read went to another replica") + print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)") + + +def test_logout_seen_by_all_replicas(): + user = new_user("logout") + out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)] + print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}") + check(len(set(after)) >= 2, "only one replica answered") + print("PASS: logout on one replica revokes the session on all of them") + + +def test_idempotent_retry_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "retry me", key=key) + for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику + retry = send(alice, chat["id"], "retry me", key=key) + if retry.instance != first.instance: + break + print(f"first POST on {first.instance} -> 201 {first.json['id']}; " + f"same key on {retry.instance} -> {retry.status} {retry.json['id']}") + check(retry.instance != first.instance, "retry never reached another replica") + check(retry.json == first.json, "retry on another replica returned a different message") + items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"] + check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate") + print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message") + + +def test_attachment_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + sent = upload(alice, chat["id"], data) + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"]) + check(reply.data == data, "another replica returned different bytes") + readers.add(reply.instance) + print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}") + check(readers - {sent.instance}, "no download went to another replica") + print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)") + + +def test_instance_header_everywhere(): + containers = api_containers() + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + replies = [ + call("GET", "/health/live", 200), + call("GET", "/health/ready", 200), + send(alice, chat["id"], "hello"), + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}), + call("GET", "/api/v1/users/me", 401), + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]), + call("POST", "/api/v1/users", 409, + body={"username": alice["username"], "display_name": "x", "password": alice["password"]}), + ] + for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер + check(any(container.startswith(reply.instance) for container in containers), + f"X-Instance-Id {reply.instance} is not a running api container") + print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container " + f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}") + print("PASS: X-Instance-Id is in every response and points to a real container") + + +if __name__ == "__main__": + run_tests([ + test_requests_spread_over_replicas, + test_write_on_one_read_on_other, + test_logout_seen_by_all_replicas, + test_idempotent_retry_on_other_replica, + test_attachment_on_other_replica, + test_instance_header_everywhere, + ]) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 0000000..ff3d4ab --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,276 @@ +"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка). + +Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`), +выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает +janitor вручную. Данные синтетические, volumes не удаляются. + + python student-tests/test_consistency.py [s3|crash|orphan|aborted|all] +""" +import json +import socket +import sys +import time +import urllib.parse + +import images +from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat, + new_user, post_message, run_tests, sql_value, upload, wait_ready) + +URL = urllib.parse.urlsplit(BASE_URL) + +# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но enqueue_upload +# (lab5: UPDATE ... state='queued' + задание + outbox, COMMIT) падает — как если бы реплика умерла +# сразу после PUT в staging, до фиксации задания. +CRASH_AFTER_PUT = r''' +import json, struct, sys, tempfile, zlib +from uuid import UUID +import attachments, db, storage, uploads +chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2]) +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b"")) +f = tempfile.SpooledTemporaryFile(); f.write(data) +import hashlib +upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest()) +image = uploads.inspect(upload, "image/png") +created = {} +real_intent = db.create_upload_intent +def remember(attachment_id, *args): + created["id"], created["key"] = str(attachment_id), args[-1] + return real_intent(attachment_id, *args) +def crash(attachment_id, *args): + raise RuntimeError("simulated crash between S3 PUT and DB COMMIT") +db.create_upload_intent = remember +db.enqueue_upload = crash +db.pool.open() +try: + attachments.store(chat_id, owner_id, "crash.png", upload, image) + outcome = "stored" +except RuntimeError as exc: + outcome = str(exc) +print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"], + "object_exists": storage.exists(created["key"])})) +''' + +OBJECT_EXISTS = r''' +import json, sys, storage +print(json.dumps({"exists": storage.exists(sys.argv[1])})) +''' + +PUT_ORPHAN = r''' +import hashlib, io, json, uuid, storage +data = b"orphan object without a row in PostgreSQL" +key = storage.new_key(uuid.uuid4().hex) +storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest()) +print(json.dumps({"key": key})) +''' + +DELETE_OBJECT = r''' +import json, sys, storage +storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]})) +''' + +COUNT_OBJECTS = r''' +import json, storage +print(json.dumps({"objects": sum(1 for prefix in storage.ALL_PREFIXES for _ in storage.list_objects(prefix))})) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def object_exists(key): + return last_json(app_python(OBJECT_EXISTS, key))["exists"] + + +def backdate(where): + sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}") + + +def uploading_rows(chat_id): + return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'")) + + +def wait_download(user, att, seconds=60): + deadline = time.monotonic() + seconds + while True: + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"]) + if reply.status == 200: + return reply + check(time.monotonic() < deadline, "download did not recover after S3 start") + time.sleep(1) + + +def s3_outage(stop_action, start_action, label): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + att = upload(alice, chat["id"], data).json + before = uploading_rows(chat["id"]) + print(f"--- S3 {label}: docker compose {stop_action} s3") + compose(stop_action, "s3") + try: + started = time.monotonic() + reply = upload(alice, chat["id"], data, expected=503) + ms = (time.monotonic() - started) * 1000 + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + body = reply.data.decode().lower() + check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")), + "error leaks storage details") + print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body") + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + print("download of an existing file -> 503") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta["sha256"] == att["sha256"], "metadata changed") + post_message(alice, chat["id"], "текст без файла") + post_message(alice, chat["id"], "", [att["id"]]) + call("GET", "/health/ready", 200) + print("metadata, messages (also with an already stored attachment) and /health/ready keep working: " + "S3 is needed only for file bytes") + after = uploading_rows(chat["id"]) + print(f"uploading rows of this chat: before {before}, after the failed upload {after} " + "(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API") + finally: + print(f"docker compose {start_action} s3") + compose(start_action, "s3") + got = wait_download(bob, att) + check(got.data == data, "file changed after S3 recovery") + print("after S3 recovery the old file downloads byte for byte") + upload(alice, chat["id"], data) + print("new upload -> 201") + leftover = uploading_rows(chat["id"]) + if leftover: + backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'") + report = janitor() + print(f"janitor after grace period: {json.dumps(report)}") + check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain") + print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up") + + +def test_s3_stopped(): + s3_outage("stop", "start", "stopped (connection refused / name not resolved)") + + +def test_s3_frozen(): + s3_outage("pause", "unpause", "frozen (no answer, timeouts)") + + +def test_crash_between_s3_and_db(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"])) + check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}") + att_id, key = result["attachment_id"], result["object_key"] + state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'") + check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3") + print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3") + call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"]) + post_message(alice, chat["id"], "x", [att_id], expected=404) + print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404") + + report = janitor() + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key), + "fresh upload was deleted inside the grace period") + print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> " + "fresh row and object are kept (the upload might still be running)") + backdate(f"id = '{att_id}'") + report = janitor("--dry-run") + check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row") + print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted") + report = janitor() + check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists") + check(not object_exists(key), "object still exists") + print(f"janitor: {json.dumps(report)} -> row and object deleted") + report = janitor() + print(f"janitor again: {json.dumps(report)} -> idempotent") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment damaged by cleanup") + print("the ready attachment of the same chat is untouched") + print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period") + + +def test_orphan_objects(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + key = last_json(app_python(PUT_ORPHAN))["key"] + print(f"object {key} put into the bucket without a row in PostgreSQL") + # Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с. + short = {"UPLOAD_TIMEOUT_SECONDS": "1"} + report = janitor("--grace-seconds", "3", env=short) + check(object_exists(key), "fresh orphan deleted inside grace period") + print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept") + time.sleep(4) + report = janitor("--grace-seconds", "3", "--dry-run", env=short) + check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key), + "dry run must report and keep the orphan") + print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept") + report = janitor("--grace-seconds", "3", env=short) + check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted") + print(f"janitor: {json.dumps(report)} -> orphan deleted") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment was deleted with orphans") + print("ready attachments older than the grace period are untouched (they have rows)") + + victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json + victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'") + app_python(DELETE_OBJECT, victim_key) + report = janitor("--grace-seconds", "3", env=short) + check(report["missing_objects"] >= 1, "missing object not reported") + check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted") + reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"]) + print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects=" + f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)") + print("PASS: orphans are found and deleted only after the grace period; missing objects are reported") + + +def api_tmp_files(): + counts = {} + for container in compose_output("ps", "-q", "api").split(): + counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0) + return counts + + +def test_aborted_upload_leaves_nothing(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + objects = last_json(app_python(COUNT_OBJECTS))["objects"] + rows = int(sql_value("SELECT count(*) FROM attachments")) + for sent in (100_000, 3_000_000): + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n" + f"Content-Length: 9000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(sent)) + sock.close() + print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection") + time.sleep(2) + check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row") + check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object") + print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}") + tmp = api_tmp_files() + check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}") + print(f"temporary upload files in api containers: {tmp} (removed when the request ends)") + for container in compose_output("ps", "-q", "api").split(): + mounts = docker("inspect", "-f", "{{json .Mounts}}", container) + # файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных + storage = [m for m in json.loads(mounts) + if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)] + check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}") + print("api containers have no volumes: the original is never stored locally") + print("PASS: an interrupted upload leaves no row, no object and no local file") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db], + "orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]} + groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"] + check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]") + wait_ready() + run_tests(groups[target]) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 0000000..3d71fcc --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,101 @@ +import time +import uuid + +from common import call, check, compose_output, new_user, psql, run_tests + + +def sql_value(sql): + code, output, error = psql(sql) + check(code == 0, f"psql failed: {error}") + return output + + +def test_create_chat_rolls_back(): + alice = new_user("rollback") + title = "rollback-" + uuid.uuid4().hex[:8] + inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + call("POST", "/api/v1/chats", 400, token=alice["token"], + body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) + + left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") + orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") + inserted_after = inserted_before + deadline = time.monotonic() + 15 + while inserted_after == inserted_before and time.monotonic() < deadline: + time.sleep(0.5) + inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; " + f"chats insert counter: {inserted_before} -> {inserted_after}") + check(left == "0" and orphans == "0", "chat row survived the failed transaction") + check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown") + print("PASS: chat row was inserted and then rolled back together with members") + + +def test_message_needs_membership_in_db(): + code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") + print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") + check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") + print("PASS: the database itself rejects a message from a non-member") + + +def test_password_is_argon2id(): + user = new_user("hash") + stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'") + print(f"stored password: {stored[:31]}... (length {len(stored)})") + check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash") + print("PASS: password is stored as Argon2id hash") + + +def test_unique_username_constraint(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'users'::regclass AND contype = 'u'") + print(f"unique constraint on users: {constraint}") + check(constraint == "UNIQUE (username)", "no unique constraint on username") + print("PASS: username is unique in the database") + + +def test_migrations_are_not_repeated(): + versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + print(f"applied migrations: {versions}; running the migrate service once more") + output = compose_output("run", "--rm", "--no-deps", "-T", "migrate") + again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1] + print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}") + check(again == versions and "none" in last_log, "migrations were applied again") + print("PASS: migrations are applied once, a repeated run does nothing") + + +def test_replicas_do_not_migrate(): + """lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему.""" + logs = compose_output("logs", "api").splitlines() + checked = [line for line in logs if "Schema is up to date" in line] + migrated = [line for line in logs if "Migrations applied" in line] + migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': " + f"{len(migrated)}; migrate service: {migrate_exit}") + check(checked and not migrated, "api replicas must not apply migrations") + check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0") + print("PASS: only the migrate service applies migrations, replicas wait for the schema") + + +def test_idempotency_key_is_unique_in_db(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conname = 'messages_idempotency_key'") + print(f"unique constraint for Idempotency-Key: {constraint}") + check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys") + print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database") + + +if __name__ == "__main__": + run_tests([ + test_create_chat_rolls_back, + test_message_needs_membership_in_db, + test_password_is_argon2id, + test_unique_username_constraint, + test_migrations_are_not_repeated, + test_replicas_do_not_migrate, + test_idempotency_key_is_unique_in_db, + ]) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..e6017ce --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,57 @@ +import datetime as dt +import os +import time + +from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready + +SHORT_TTL = 2 + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env + else: + env["SESSION_TTL_SECONDS"] = str(ttl) + compose("up", "-d", "--no-deps", "--wait", "api", env=env) + wait_ready() + + +def seconds_left(expires_at): + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def test_session_expires(): + print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}") + restart_api(SHORT_TTL) + try: + user = new_user("ttl") + session = login(user["username"], user["password"]).json + token = session["token"] + left = seconds_left(session["expires_at"]) + print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: " + f"{redis_cli('pttl', session_key(token))} ms") + check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions") + + call("GET", "/api/v1/users/me", 200, token=token) + print("before expiry: GET /users/me -> 200") + + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0) + call("GET", "/api/v1/users/me", 401, token=token) + exists = redis_cli("exists", session_key(token)) + print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}") + check(exists == "0", "Redis must delete the expired session key") + finally: + print("recreating api with SESSION_TTL_SECONDS from .env") + restart_api() + + user = new_user("ttl_back") + left = seconds_left(login(user["username"], user["password"]).json["expires_at"]) + print(f"normal API again: new session expires in {left:.0f} s") + check(left > 300, "normal TTL was not restored") + print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401") + + +if __name__ == "__main__": + run_tests([test_session_expires]) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 0000000..8eaef38 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,110 @@ +import sys +import time +import uuid + +from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready + +FAKE_TOKEN = "A" * 43 + + +def timed(method, path, expected, **kwargs): + started = time.monotonic() + reply = call(method, path, expected, **kwargs) + return reply, (time.monotonic() - started) * 1000 + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + before = send(alice, chat["id"], "confirmed before outage").json + return alice, bob, chat, before + + +def recovered(alice, chat, before): + wait_ready() + history, _ = read_history(alice, chat["id"]) + check(before in history, "message confirmed before the outage is lost or changed") + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the old token works, confirmed messages are in place") + + +def test_redis_down(): + alice, bob, chat, before = prepare() + print("stopping redis") + compose("stop", "redis") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") + call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) + print("unknown token -> 503, not 200: nobody is trusted without a check") + call("GET", "/api/v1/chats", 401) + print("no token -> 401") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + print("login -> 503: there is nowhere to store the session") + call("POST", "/api/v1/users", 201, + body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("registration -> 201: it needs only PostgreSQL") + finally: + print("starting redis") + compose("start", "redis") + recovered(alice, chat, before) + print("PASS: Redis outage gives 503 and does not let anyone in without a session check") + + +def test_redis_crash(): + alice = new_user("crash") + print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") + compose("kill", "redis") + try: + # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. + status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status + print(f"right after the crash: GET /users/me -> {status}") + finally: + print("starting redis") + compose("start", "redis") + wait_ready() + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the session issued before the crash still works (AOF, appendfsync always)") + print("PASS: sessions survive a Redis crash, not only a clean restart") + + +def test_postgres_down(): + alice, bob, chat, before = prepare() + print("stopping postgres") + compose("stop", "postgres") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + for method, path, body in (("GET", "/api/v1/users/me", None), + ("GET", "/api/v1/chats", None), + ("GET", f"/api/v1/chats/{chat['id']}/messages", None), + ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): + reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + check("postgres" not in reply.data.decode().lower(), "error must not show database details") + print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") + call("GET", "/api/v1/chats", 401) + call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) + print("no token -> 401, unknown token -> 401: anonymous users are not let in") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + call("POST", "/api/v1/users", 503, + body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("login -> 503, registration -> 503") + finally: + print("starting postgres") + compose("start", "postgres") + recovered(alice, chat, before) + print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], + "all": [test_redis_down, test_redis_crash, test_postgres_down]} + check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") + run_tests(tests[target]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..407dea3 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,174 @@ +import json +import socket +import threading +import time +import urllib.parse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send + +PARALLEL = 20 + + +def post_raw(user, chat_id, raw, key): + """POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"], + raw=raw, headers={"Idempotency-Key": key}) + + +def texts_in(user, chat_id): + history, _ = read_history(user, chat_id) + return [message["text"] for message in history] + + +def test_replay_returns_same_message(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "Привет 👋", key=key) + # То же JSON-значение, записанное иначе: пробелы и \u-экранирование. + same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key) + again = send(alice, chat["id"], "Привет 👋", key=key) + print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}") + check(same.status == 201 and same.json == first.json and again.json == first.json, + "retry must return the original 201 and the same message") + check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate") + print("PASS: a retry with the same key returns the original 201 and message, no duplicate") + + +def test_same_key_parallel_through_replicas(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(_): + barrier.wait() # все запросы стартуют одновременно + return send(alice, chat["id"], "one message for 20 requests", key=key) + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + replies = list(pool.map(post, range(PARALLEL))) + bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies} + served = Counter(reply.instance for reply in replies) + count = texts_in(bob, chat["id"]).count("one message for 20 requests") + print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, " + f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}") + check(len(served) >= 2, "requests did not go through different replicas") + check(len(bodies) == 1 and count == 1, "parallel retries created more than one message") + print("PASS: parallel requests with one key through different replicas create exactly one message") + + +def test_same_key_different_bodies_parallel(): + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(index): + text = "body A" if index % 2 == 0 else "body B" + barrier.wait() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"], + body={"text": text}, headers={"Idempotency-Key": key}) + return text, reply + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(post, range(PARALLEL))) + texts = texts_in(alice, chat["id"]) + winner = [t for t in texts if t in ("body A", "body B")] + print(f"{PARALLEL} parallel POSTs, one key, two different bodies: " + f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}") + check(len(winner) == 1, "exactly one message must be created") + for text, reply in results: + expected = 201 if text == winner[0] else 409 + check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}") + print("PASS: the first body wins, the same body gets 201, the other body gets 409") + + +def test_key_scope(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + other = new_chat(alice, bob) + key = new_key() + a1 = send(alice, chat["id"], "scope", key=key).json + b1 = send(bob, chat["id"], "scope", key=key).json + a2 = send(alice, other["id"], "scope", key=key).json + print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages") + check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat") + call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"], + body={"text": "other text"}, headers={"Idempotency-Key": key}) + print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409") + + +def test_key_validation(): + alice = new_user("alice") + chat = new_chat(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": "no key"}) + for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"): + call("POST", path, 400, token=alice["token"], body={"text": "bad key"}, + headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8 + longest = ("Az09._:-" * 16)[:128] + call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest}) + print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201") + print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}") + + +def test_failed_request_does_not_burn_key(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key}) + call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key}) + created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key}) + print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}") + print("PASS: rejected requests (400, 404) do not store a result for the key") + + +def test_replay_checks_session_and_access(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + send(alice, chat["id"], "private", key=key) + stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}") + print("PASS: a retry checks the session and chat access again, the saved answer is not leaked") + + +def test_lost_response_then_retry(): + """Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен.""" + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + body = json.dumps({"text": "answer was lost"}).encode("utf-8") + url = urllib.parse.urlsplit(BASE_URL) + request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection: + connection.sendall(request + body) # и сразу закрываем, ответ не читаем + time.sleep(1) + retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"], + body={"text": "answer was lost"}, headers={"Idempotency-Key": key}) + count = texts_in(alice, chat["id"]).count("answer was lost") + print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}") + check(count == 1, "the retry after a lost answer created a duplicate") + print("PASS: an unconfirmed POST can be safely retried with the same key") + + +if __name__ == "__main__": + run_tests([ + test_replay_returns_same_message, + test_same_key_parallel_through_replicas, + test_same_key_different_bodies_parallel, + test_key_scope, + test_key_validation, + test_failed_request_does_not_burn_key, + test_replay_checks_session_and_access, + test_lost_response_then_retry, + ]) diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 0000000..b382ee2 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,567 @@ +"""Lab5: асинхронная обработка вложений — варианты, сохранность, отказы, повторы, DLQ, replay. + +Нужен запущенный стенд из compose.yaml с FAULT_INJECTION=1 (по умолчанию на учебном стенде). +Тест управляет только своими синтетическими данными и сервисами через `docker compose stop|start|restart`, +volumes не удаляет. Сбои внедряются файлами в volume faults (faults.py), SQL — только чтение. + + python student-tests/test_jobs.py [сценарий ...] # без аргументов — все по порядку + +Сценарии: variants limits worker-down broker-down outbox-crash crash-after-put crash-after-commit + lease retry exhausted-replay permanent concurrency dlq +""" +import base64 +import json +import sys +import time +import urllib.parse +from pathlib import Path + +from common import (call, check, compose, compose_output, new_chat, new_user, post_message, psql, run_tests, + send, sql_value, wait_ready) + +HERE = Path(__file__).resolve().parent +PROBE = (HERE / "worker_probe.py").read_text(encoding="utf-8") +FINAL_KEYS = ["v1/crop.jpg", "v1/optimized.webp", "v1/original", "v1/thumbnail.jpg"] +LEASE = 15 # JOB_LEASE_SECONDS стенда +TIMEOUT = 60 # JOB_TIMEOUT_SECONDS стенда +MAX_ATTEMPTS = 5 # JOB_MAX_ATTEMPTS стенда + +STATE = {} + + +# ---------- инфраструктура ---------- + +def in_container(service, code, *args): + """python - в контейнере service, код через stdin. Последняя строка stdout — JSON. + Если service остановлен (сценарии с выключенным worker) — в одноразовом контейнере того же сервиса.""" + import subprocess + from common import ROOT + running = subprocess.run(["docker", "compose", "ps", "--status", "running", "-q", service], cwd=ROOT, + capture_output=True, check=False).stdout.strip() + launch = (["exec", "-T", service, "python"] if running + else ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service]) + result = subprocess.run(["docker", "compose", *launch, "-", *map(str, args)], cwd=ROOT, + input=code.encode("utf-8"), capture_output=True, check=False) + check(result.returncode == 0, f"{service}: {result.stderr.decode('utf-8', 'replace')[-500:]}") + return json.loads(result.stdout.decode("utf-8").strip().splitlines()[-1]) + + +def probe(*args): + return in_container("worker", PROBE, *args) + + +def ops(*args): + import subprocess + from common import ROOT + result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)], + cwd=ROOT, capture_output=True, check=False) + check(result.returncode == 0, f"ops {args[0]}: {result.stderr.decode('utf-8', 'replace')[-400:]}") + return json.loads(result.stdout.decode("utf-8")) + + +def fault(point, times=1): + """Взвести точку сбоя: times файлов /faults/. (каждое срабатывание удаляет один файл).""" + code = ("import pathlib, sys, time\n" + "for n in range(int(sys.argv[2])):\n" + " pathlib.Path('/faults', f'{sys.argv[1]}.{time.time_ns()}_{n}').touch()\n" + "print('{}')\n") + in_container("worker", code, point, times) + + +def clear_faults(): + code = "import pathlib\nfor p in pathlib.Path('/faults').glob('*'):\n p.unlink()\nprint('{}')\n" + in_container("worker", code) + + +def image(kind): + return base64.b64decode(probe("make", kind)["b64"]) + + +def user_and_chat(): + if "alice" not in STATE: + STATE["alice"] = new_user("alice") + STATE["bob"] = new_user("bob") + STATE["chat"] = new_chat(STATE["alice"], STATE["bob"], title="Lab5 jobs") + return STATE["alice"], STATE["bob"], STATE["chat"] + + +def upload(data, content_type="image/png", filename="probe.png", expected=202): + alice, _bob, chat = user_and_chat() + query = "?filename=" + urllib.parse.quote(filename, safe="") + att = call("POST", f"/api/v1/chats/{chat['id']}/attachments{query}", expected, raw=data, token=alice["token"], + headers={"Content-Type": content_type}, timeout=60).json + if expected == 202: + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, + f"202 must be a queued snapshot, got {att['status']}") + return att + + +def get(att_id, user=None): + user = user or STATE["alice"] + return call("GET", f"/api/v1/attachments/{att_id}", 200, token=user["token"]).json + + +def wait_status(att_id, statuses, seconds=60): + statuses = (statuses,) if isinstance(statuses, str) else statuses + deadline = time.monotonic() + seconds + while True: + att = get(att_id) + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"{att_id[:8]}: still {att['status']} after {seconds} s") + time.sleep(0.3) + + +def content(att_id, variant="original", expected=200): + return call("GET", f"/api/v1/attachments/{att_id}/content?variant={variant}", expected, + token=STATE["bob"]["token"]) + + +def job(att_id): + rows = sql_value(f"SELECT job_id || '|' || status || '|' || attempt || '|' || coalesce(error_code, '-') " + f"FROM attachment_jobs WHERE attachment_id = '{att_id}'") + check(rows and "\n" not in rows, f"expected exactly one job for {att_id}") + job_id, status, attempt, error = rows.split("|") + return {"job_id": job_id, "status": status, "attempt": int(attempt), "error_code": error} + + +def events(job_id): + out = sql_value(f"SELECT attempt || ':' || event || ':' || coalesce(detail, '') FROM job_events " + f"WHERE job_id = '{job_id}' ORDER BY id") + return out.splitlines() if out else [] + + +def names(evs): + return [e.split(":")[1] for e in evs] + + +def assert_one_result(att_id, label=""): + """Инвариант идемпотентности: ровно один набор вариантов и ровно 4 объекта под конечным prefix.""" + info = probe("inspect", att_id) + check(info["state"] == "ready", f"{label}: not ready in DB") + check(info["variant_rows"] == 3, f"{label}: {info['variant_rows']} variant rows, expected 3") + check(info["final_keys"] == FINAL_KEYS, f"{label}: final objects {info['final_keys']}") + check(info["original_sha_ok"] and info["original_size_ok"], f"{label}: original differs from upload") + check(all(v["sha_ok"] and v["meta_size_ok"] for v in info["variants"].values()), f"{label}: variant mismatch") + return info + + +def outbox_rows(job_id): + out = sql_value(f"SELECT attempt || ':' || (published_at IS NOT NULL) FROM outbox WHERE job_id = '{job_id}' " + f"ORDER BY id") + return out.splitlines() if out else [] + + +def queue_depth(): + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_queues", "-p", "messenger", + "name", "messages", "messages_unacknowledged") + depth = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) == 3 and parts[1].isdigit(): + depth[parts[0]] = (int(parts[1]), int(parts[2])) + return depth + + +def log(text): + print(text, flush=True) + + +# ---------- сценарии ---------- + +def test_variants(): + """thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.""" + alice, bob, chat = user_and_chat() + cases = { + "thumb320": ("image/png", {"thumbnail": [256, 160], "crop": [128, 128], "optimized": [320, 200]}), + "exif6": ("image/jpeg", {"thumbnail": [192, 256], "crop": [128, 128], "optimized": [300, 400]}), + "alpha": ("image/png", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [300, 200]}), + "tiny": ("image/png", {"thumbnail": [40, 30], "crop": [128, 128], "optimized": [40, 30]}), + "large": ("image/jpeg", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [1280, 853]}), + "strip": ("image/png", {"thumbnail": [256, 1], "crop": [128, 128], "optimized": [1280, 1]}), + } + for kind, (ctype, sizes) in cases.items(): + data = image(kind) + att = upload(data, ctype, f"{kind}.{'jpg' if ctype == 'image/jpeg' else 'png'}") + ready = wait_status(att["id"], "ready") + meta = {v["name"]: v for v in ready["variants"]} + check(set(meta) == {"thumbnail", "crop", "optimized"}, f"{kind}: variants {sorted(meta)}") + for name, size in sizes.items(): + check([meta[name]["width"], meta[name]["height"]] == size, + f"{kind}/{name}: {meta[name]['width']}x{meta[name]['height']}, expected {size}") + check(meta["thumbnail"]["content_type"] == "image/jpeg" and meta["crop"]["content_type"] == "image/jpeg" + and meta["optimized"]["content_type"] == "image/webp", f"{kind}: wrong variant content types") + original = content(att["id"]) + check(original.data == data, f"{kind}: original is not byte-identical") + for name in ("thumbnail", "crop", "optimized"): + reply = content(att["id"], name) + check(reply.headers["content-type"] == meta[name]["content_type"], f"{kind}/{name}: Content-Type") + info = assert_one_result(att["id"], kind) + for name, v in info["variants"].items(): + check(v["exif_tags"] == 0 and not v["has_xmp"], f"{kind}/{name}: metadata not stripped") + if kind == "exif6": + top = info["variants"]["thumbnail"]["pixel_top_center"] + check(top[0] > 200 and top[1] < 60, f"exif6: orientation not applied, top pixel {top}") + if kind == "alpha": + left = info["variants"]["thumbnail"]["pixel_left_center"] + right = info["variants"]["thumbnail"]["pixel_right_center"] + check(left[:3] == [255, 255, 255] and right[2] > 200, f"alpha: not composited on white {left} {right}") + check(info["variants"]["optimized"]["mode"] == "RGBA", "alpha: optimized WebP lost transparency") + log(f"{kind}: " + ", ".join(f"{n} {s[0]}x{s[1]}" for n, s in sizes.items()) + + f"; original byte-identical; EXIF/XMP in variants: 0; S3 keys {info['final_keys']}") + # Отсутствующий вариант у ready -> 404 (нет такого имени в контракте -> 400), queued -> 409 проверяется ниже. + content(att["id"], "unknown", expected=400) + log("unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload") + + +def test_limits(): + """Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.""" + alice, _bob, chat = user_and_chat() + big = b"\x89PNG\r\n\x1a\n" + b"\0" * (10 * 1024 * 1024) + call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=big.png", 413, raw=big, token=alice["token"], + headers={"Content-Type": "image/png"}, timeout=60) + log("10 MiB + 8 bytes -> 413 before any job is created") + report = probe("imaging-limits") + at, over = report["at_limit_40mpx"], report["over_limit_max_10mpx"] + check(at["exit_code"] == 0, f"40 Mpx PNG at the limit must be processed, exit {at['exit_code']}") + check(over["exit_code"] == 4, f"worker-side pixel limit must reject (exit 4), got {over['exit_code']}") + check(at["peak_rss_mib_max_children"] < report["memory_limit_mib"], + f"peak RSS {at['peak_rss_mib_max_children']} MiB exceeds RLIMIT_AS {report['memory_limit_mib']} MiB") + log(f"imaging.py on 8000x5000 PNG: exit 0, peak RSS {at['peak_rss_mib_max_children']} MiB " + f"(RLIMIT_AS {report['memory_limit_mib']} MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)") + # Память: процесс обработки упирается в RLIMIT_AS -> MemoryError -> постоянная ошибка resource_limit. + fault("imaging_oom") + att = upload(image("thumb320"), filename="oom.png") + failed = wait_status(att["id"], ("failed", "ready"), 60) + check(failed["status"] == "failed" and failed["error_code"] == "resource_limit", + f"oom: expected failed/resource_limit, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == 1, "resource_limit is permanent: no retries expected") + log(f"memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived") + STATE["oom_job"] = j["job_id"] + # Время: зависший процесс обработки убивается по JOB_TIMEOUT_SECONDS -> временная ошибка -> retry -> ready. + fault("imaging_hang") + att = upload(image("thumb320"), filename="hang.png") + started = time.monotonic() + ready = wait_status(att["id"], "ready", TIMEOUT + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check("lease_expired" not in names(evs), f"a long-running (alive) attempt must keep its lease: {evs}") + check(any("processing_timeout" in e for e in evs) and j["attempt"] == 2, f"timeout not handled as retry: {evs}") + log(f"imaging hang -> killed after {TIMEOUT} s (processing_timeout, temporary), lease renewed meanwhile " + f"(no lease_expired), attempt 2 ready after {time.monotonic() - started:.0f} s") + + +def test_worker_down(): + """Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.""" + alice, bob, chat = user_and_chat() + compose("stop", "worker") + log("worker stopped") + data = image("thumb320") + att = upload(data, filename="offline.png") + log(f"upload while worker is down -> 202 queued, attachment {att['id']}") + j = job(att["id"]) + source_key = sql_value(f"SELECT source_key FROM attachments WHERE id = '{att['id']}'") + check(source_key.startswith("staging/"), "source must be in the durable staging prefix") + check(probe("exists", source_key)["exists"], "staging object must exist before 202") + msg = post_message(alice, chat["id"], "photo while worker is down", [att["id"]]).json + check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment") + content(att["id"], expected=409) + content(att["id"], "thumbnail", expected=409) + send(bob, chat["id"], "other messages keep working") + deadline = time.monotonic() + 10 + while outbox_rows(j["job_id"]) != ["1:true"]: + check(time.monotonic() < deadline, "dispatcher did not publish the job") + time.sleep(0.2) + deadline = time.monotonic() + 15 # счётчики quorum-очереди в rabbitmqctl обновляются раз в несколько секунд + while (queue_depth().get("attachments.process.v1") or (0, 0))[0] < 1: + check(time.monotonic() < deadline, "published message is not visible in the queue") + time.sleep(0.5) + depth = queue_depth().get("attachments.process.v1") + log(f"message with attachment 201, content 409, other messages 201; job {j['job_id']} published, " + f"queue messages/unacked = {depth}") + compose("restart", "rabbitmq") + compose("restart", "api") + wait_ready() + depth_after = queue_depth().get("attachments.process.v1") + check(depth_after and depth_after[0] >= 1, f"message lost after broker restart: {depth_after}") + check(get(att["id"])["status"] == "queued", "job must still be queued after restarts") + log(f"rabbitmq and api restarted with volumes; queue still holds {depth_after[0]} message(s); status queued") + compose("start", "worker") + ready = wait_status(att["id"], "ready", 60) + j2 = job(att["id"]) + check(j2["job_id"] == j["job_id"] and j2["status"] == "ready", "same job_id must finish") + thumb = content(att["id"], "thumbnail") + check(len(thumb.data) == {v["name"]: v for v in ready["variants"]}["thumbnail"]["size_bytes"], "thumbnail size") + info = assert_one_result(att["id"], "worker-down") + check(not info["staging_present"] and not probe("exists", source_key)["exists"], + "staging must be cleaned after the safe commit") + log(f"worker started -> same job {j['job_id']} ready, thumbnail downloadable; staging object deleted") + + +def test_broker_down(): + """Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.""" + compose("stop", "rabbitmq") + log("rabbitmq stopped") + att = upload(image("thumb320"), filename="no-broker.png") + call("GET", "/health/ready", 200) + j = job(att["id"]) + time.sleep(3) + check(outbox_rows(j["job_id"]) == ["1:false"], "outbox row must wait while the broker is down") + log(f"upload -> 202 and /health/ready 200 without broker; outbox row pending for job {j['job_id']}") + compose("start", "rabbitmq") + wait_status(att["id"], "ready", 90) + check(outbox_rows(j["job_id"]) == ["1:true"], "outbox row must be published after recovery") + assert_one_result(att["id"], "broker-down") + log("rabbitmq started -> dispatcher published the pending row -> ready") + + +def test_outbox_crash(): + """Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.""" + fault("dispatcher_crash_before_publish") + att = upload(image("thumb320"), filename="crash-before-publish.png") + ready = wait_status(att["id"], "ready", 60) + j = job(att["id"]) + logs = compose_output("logs", "--tail", "400", "dispatcher") + check("FAULT INJECTION: dispatcher_crash_before_publish" in logs, "fault did not fire") + check(outbox_rows(j["job_id"]) == ["1:true"] and j["attempt"] == 1, "outbox must publish exactly the same job") + assert_one_result(att["id"], "crash-before-publish") + log(f"dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job " + f"{j['job_id']} ready, attempt 1, no manual action") + fault("dispatcher_crash_after_confirm") + att = upload(image("thumb320"), filename="crash-after-confirm.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + deadline = time.monotonic() + 20 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"duplicate delivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + assert_one_result(att["id"], "crash-after-confirm") + log(f"dispatcher killed after publisher confirm, before marking outbox -> message published twice; " + f"worker: {names(events(j['job_id']))} -> one result set") + + +def test_crash_after_put(): + """Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.""" + fault("worker_crash_after_put") + att = upload(image("thumb320"), filename="crash-after-put.png") + wait_status(att["id"], "ready", LEASE + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check(names(evs).count("claimed") == 2 and "lease_expired" in names(evs), f"unexpected history {evs}") + assert_one_result(att["id"], "crash-after-put") + log(f"worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired " + f"after {LEASE} s, attempt {j['attempt']} rewrote the same keys -> one set; events {names(evs)}") + + +def test_crash_after_commit(): + """Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.""" + fault("worker_crash_after_commit") + att = upload(image("thumb320"), filename="crash-after-commit.png") + wait_status(att["id"], "ready", 30) + j = job(att["id"]) + deadline = time.monotonic() + 30 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"redelivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + evs = events(j["job_id"]) + check(names(evs).count("ready") == 1 and j["attempt"] == 1, f"ready must be recorded once: {evs}") + assert_one_result(att["id"], "crash-after-commit") + deadline = time.monotonic() + 20 # счётчики quorum-очереди обновляются с задержкой + while queue_depth().get("attachments.process.v1") != (0, 0): + check(time.monotonic() < deadline, f"message left in the queue: {queue_depth()}") + time.sleep(0.5) + log(f"worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op ({evs[-1]}); " + f"queue empty (0 ready, 0 unacked)") + STATE["staging_check"] = att["id"] + + +def test_lease(): + """Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; + результат зависшего отклоняется fencing-токеном, ready не откатывается.""" + fault("worker_freeze") + att = upload(image("thumb320"), filename="frozen.png") + started = time.monotonic() + wait_status(att["id"], "ready", LEASE + 40) + took = time.monotonic() - started + j = job(att["id"]) + deadline = time.monotonic() + 3 * LEASE + 20 + while "stale_result_rejected" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"stale result of the frozen attempt not seen: {events(j['job_id'])}") + time.sleep(0.5) + evs = events(j["job_id"]) + check(get(att["id"])["status"] == "ready", "late result must not roll ready back") + check(names(evs).count("ready") == 1, f"exactly one ready: {evs}") + assert_one_result(att["id"], "lease") + log(f"frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after " + f"{took:.0f} s; frozen attempt woke up -> stale_result_rejected; events {names(evs)}") + + +def test_retry(): + """Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.""" + fault("worker_s3_error", 2) + att = upload(image("thumb320"), filename="retry.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + evs = [e for e in events(j["job_id"]) if ":retry_scheduled:" in e] + check(len(evs) == 2 and j["attempt"] == 3, f"expected 2 retries, got {evs}") + delays = [float(e.split("delay=")[1].rstrip("s")) for e in evs] + check(1.0 <= delays[0] <= 2.0 and 2.0 <= delays[1] <= 4.0, f"backoff out of range: {delays}") + assert_one_result(att["id"], "retry") + log(f"2 x storage_unavailable -> retries after {delays[0]:.2f} s and {delays[1]:.2f} s (equal jitter: " + f"1..2 s, 2..4 s), attempt 3 ready") + + +def test_exhausted_replay(): + """Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.""" + alice, _bob, chat = user_and_chat() + fault("worker_s3_error", MAX_ATTEMPTS) + att = upload(image("thumb320"), filename="exhausted.png") + failed = wait_status(att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", + f"expected failed/retry_exhausted, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == MAX_ATTEMPTS, f"attempts {j['attempt']}") + content(att["id"], expected=409) + post_message(alice, chat["id"], "", [att["id"]], expected=409) + registry = {row["job_id"]: row for row in ops("failed")} + check(registry.get(j["job_id"], {}).get("replayable") is True, "failed job must be in the registry") + log(f"{MAX_ATTEMPTS} temporary errors -> failed/retry_exhausted (public error_code), content 409, " + f"attach 409, listed by ops.py failed (replayable)") + replay = ops("replay", j["job_id"], "--reason", "s3 recovered") + check("replayed" in replay, f"replay refused: {replay}") + wait_status(att["id"], "ready", 60) + assert_one_result(att["id"], "replay") + j2 = job(att["id"]) + refused = ops("replay", j["job_id"]) + check("refused" in refused, "replay of a ready job must be refused") + again = job(att["id"]) + check(again == j2, "refused replay must not change the job") + check(names(events(j["job_id"])).count("replayed") == 1, "audit must contain the replay once") + log(f"ops replay -> same job_id {j['job_id']} attempt {j2['attempt']} ready; second replay refused: " + f"\"{refused['refused']}\"") + + +def test_permanent(): + """Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.""" + fault("imaging_invalid") + att = upload(image("thumb320"), filename="invalid.png") + failed = wait_status(att["id"], ("failed", "ready"), 30) + check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", f"got {failed}") + j = job(att["id"]) + check(j["attempt"] == 1 and "retry_scheduled" not in names(events(j["job_id"])), "no retries for permanent") + log("invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)") + + +def test_concurrency(): + """Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.""" + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_consumers", "-p", "messenger", + "queue_name", "prefetch_count", "ack_required") + lines = [line.split() for line in out.splitlines() if line.strip()] + header = lines[0] if lines and lines[0][0] == "queue_name" else ["queue_name", "prefetch_count", "ack_required"] + consumers = [dict(zip(header, line)) for line in lines if line[0] == "attachments.process.v1"] + check(len(consumers) == 2 and all(c["prefetch_count"] == "1" and c["ack_required"] == "true" + for c in consumers), f"consumers {out}") + # Копим очередь при остановленном worker, затем запускаем: работы больше, чем потоков. + compose("stop", "worker") + data = image("large") + ids = [upload(data, "image/jpeg", f"burst{n}.jpg")["id"] for n in range(8)] + id_list = ", ".join(f"'{i}'" for i in ids) + compose("start", "worker") + started = time.monotonic() + peak, samples = 0, 0 + while True: + busy, ready = map(int, sql_value( + f"SELECT count(*) FILTER (WHERE status = 'processing') || '|' || count(*) FILTER (WHERE status = 'ready') " + f"FROM attachment_jobs WHERE attachment_id IN ({id_list})").split("|")) + peak, samples = max(peak, busy), samples + 1 + if ready == len(ids): + break + check(time.monotonic() - started < 120, "burst not processed in 120 s") + time.sleep(0.05) + took = time.monotonic() - started + check(peak == 2, f"peak {peak} jobs in processing, expected exactly WORKER_CONCURRENCY=2") + log(f"2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in {took:.1f} s after " + f"worker start, peak processing {peak} of 8 ({samples} samples)") + + +def test_dlq(): + """Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.""" + publish = r''' +import json, sys, uuid +import broker, pipeline +p = broker.Publisher() +p.open().basic_publish(pipeline.EXCHANGE, pipeline.ROUTING_KEY, b'{"not":"a job"}', mandatory=True) +p.publish(pipeline.build_message(uuid.uuid4(), uuid.uuid4(), 1, 1, None)) +p.close() +print("{}") +''' + before = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + in_container("dispatcher", publish) + deadline = time.monotonic() + 15 + while queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] < before + 2: + check(time.monotonic() < deadline, "messages did not reach the DLQ") + time.sleep(0.3) + peek = ops("dlq", "--limit", "50") + reasons = {(m["invalid"] is not None, m["dead_letter_reason"]) for m in peek} + check((True, "rejected") in reasons, f"invalid message not in DLQ: {peek}") + replayed = ops("dlq-replay", "--limit", "50") + kept = [m for m in replayed if m["action"] == "keep"] + check(len(kept) >= 2, "invalid/unknown messages must be kept, not executed") + after = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + check(after >= before + 2, "dlq-replay must not drop messages it cannot handle") + log(f"schema-invalid message and unknown job_id -> reject -> DLQ ({after} message(s)); " + f"dlq-replay kept them: {[m['note'] for m in kept][:2]}") + + +def test_staging_cleanup(): + """Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).""" + att_id = STATE.get("staging_check") + if not att_id: + log("skip: run crash-after-commit first") + return + key = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + if key: + log(f"staging source still present after the crash (worker died before cleanup); waiting 60 s window") + time.sleep(62) + from common import janitor + report = janitor() + # фоновый janitor (проход раз в JANITOR_INTERVAL_SECONDS) мог успеть первым в окне 62 с — проверяем результат + left = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + check(not left, f"staging source still referenced: {report}") + check(not probe("exists", key)["exists"], "staging object must be gone") + by = "this pass" if report["staging_ready_deleted"] else "the background janitor pass" + log(f"janitor removed the staging source of the ready attachment ({by}); source_key NULL, object gone") + else: + log("staging already cleaned by the worker") + + +SCENARIOS = { + "variants": test_variants, "limits": test_limits, "worker-down": test_worker_down, + "broker-down": test_broker_down, "outbox-crash": test_outbox_crash, "crash-after-put": test_crash_after_put, + "crash-after-commit": test_crash_after_commit, "staging": test_staging_cleanup, "lease": test_lease, + "retry": test_retry, "exhausted-replay": test_exhausted_replay, "permanent": test_permanent, + "concurrency": test_concurrency, "dlq": test_dlq, +} + + +def main(): + chosen = sys.argv[1:] or list(SCENARIOS) + wait_ready() + clear_faults() + tests = [] + for name in chosen: + def run(name=name): + log(f"--- {name}: {SCENARIOS[name].__doc__.splitlines()[0]}") + try: + SCENARIOS[name]() + finally: + compose("start", "worker") # идемпотентно: стенд остаётся рабочим даже после FAIL + compose("start", "rabbitmq") + run.__name__ = name + tests.append(run) + run_tests(tests) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 0000000..4af44fa --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,57 @@ +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send + +CLIENTS = 10 +MESSAGES_PER_CLIENT = 10 +REGISTRATIONS = 20 + + +def test_concurrent_messages(): + users = [new_user(f"race{i}") for i in range(CLIENTS)] + chat = new_chat(*users, title="race") + barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно + + def client(index): + barrier.wait() + for n in range(MESSAGES_PER_CLIENT): + send(users[index], chat["id"], f"client-{index}-message-{n}") + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков + elapsed = time.monotonic() - started + + history, pages = read_history(users[0], chat["id"], limit=30) + ids = {message["id"] for message in history} + expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)} + print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; " + f"read {len(history)} in {pages} pages, unique ids: {len(ids)}") + check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages") + check(len(ids) == len(history), "duplicate message ids") + check({message["text"] for message in history} == expected, "lost or changed message texts") + print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates") + + +def test_concurrent_registration(): + username = "race_" + uuid.uuid4().hex[:10] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + barrier.wait() + body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD} + return call("POST", "/api/v1/users", (201, 409), body=body).status + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + statuses = list(pool.map(register, range(REGISTRATIONS))) + created, conflicts = statuses.count(201), statuses.count(409) + print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}") + check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win") + print("PASS: concurrent registration created exactly one user") + + +if __name__ == "__main__": + run_tests([test_concurrent_messages, test_concurrent_registration]) diff --git a/student-tests/test_s3_security.py b/student-tests/test_s3_security.py new file mode 100644 index 0000000..ece71a3 --- /dev/null +++ b/student-tests/test_s3_security.py @@ -0,0 +1,146 @@ +"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен. + +Нужен запущенный стенд из compose.yaml. +""" +import json +import socket + +import images +from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload + +# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API. +ANONYMOUS = r''' +import json, sys, urllib.request, urllib.error +import settings +base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET +def status(method, url, data=None): + try: + with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r: + return r.status, r.read(300).decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read(300).decode("utf-8", "replace") +out = {} +for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None), + ("head_object", "HEAD", base + "/" + sys.argv[1], None), + ("list_bucket", "GET", base + "?list-type=2", None), + ("put_object", "PUT", base + "/originals/anonymous-test", b"x"), + ("get_bucket_policy", "GET", base + "?policy", None)): + code, body = status(method, url, data) + out[name] = {"status": code, "s3_code": body.split("")[1].split("")[0] if "" in body else ""} +print(json.dumps(out)) +''' + +# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json). +API_KEY_RIGHTS = r''' +import io, json, sys, uuid +from botocore.exceptions import ClientError +import settings, storage +c, bucket = storage._client, settings.S3_BUCKET +def attempt(name, fn): + try: + fn(); return "allowed" + except ClientError as e: + return "denied: " + e.response.get("Error", {}).get("Code", "?") +public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) +key = "originals/" + uuid.uuid4().hex +result = { + "get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()), + "put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")), + "delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)), + "list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)), + "create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])), + "make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)), + "read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)), + "delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)), + "change versioning": attempt("versioning", lambda: c.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})), +} +try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket + result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", [])) +except ClientError as e: + result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?") +print(json.dumps(result)) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + att = upload(alice, chat["id"], images.sample_png()).json + key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + return alice, bob, att, key + + +def test_bucket_is_private(): + alice, bob, att, key = prepare() + result = last_json(app_python(ANONYMOUS, key)) + for name, reply in result.items(): + print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}") + check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}") + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + print("the same file through the API for a chat member -> 200") + print("PASS: the bucket is private: no anonymous read, list, write or policy read") + + +def test_api_key_is_limited(): + alice, bob, att, key = prepare() + result = last_json(app_python(API_KEY_RIGHTS, key)) + allowed = {"get own object", "put object", "delete object", "list bucket"} + visible = result.pop("visible buckets") + print(f"API key: {'list buckets':<32} -> {visible}") + check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets") + for name, outcome in result.items(): + print(f"API key: {name:<32} -> {outcome}") + if name in allowed: + check(outcome == "allowed", f"API key must be allowed to {name}") + else: + check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}") + print("PASS: the API key can only get/put/delete/list objects of its own bucket; " + "bucket admin, policy and other buckets are denied") + + +def test_no_open_ports_or_console(): + config = json.loads(compose_output("config", "--format", "json")) + s3 = config["services"]["s3"] + check(not s3.get("ports"), "s3 must not publish ports") + check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, " + "filer UI 8888 are reachable only inside the stand)") + for port in (8333, 9333, 8888, 19333): + sock = socket.socket() + sock.settimeout(1) + refused = sock.connect_ex(("127.0.0.1", port)) != 0 + sock.close() + check(refused, f"127.0.0.1:{port} is reachable from the host") + print(f"127.0.0.1:{port} from the host -> connection refused") + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()} + for name, items in sorted(secrets.items()): + print(f"secrets of {name}: {items or '-'}") + for name in ("api", "janitor"): + check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"}, + f"{name} must not get S3 admin keys or the key config") + check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key") + print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init") + + +def test_bootstrap_is_idempotent(): + alice, bob, att, key = prepare() + output = compose_output("run", "--rm", "--no-deps", "s3-init") + print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip())) + check("bucket is private" in output, "bootstrap did not confirm a private bucket") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "file changed after repeated bootstrap") + upload(alice, att["chat_id"], images.sample_png()) + print("after the repeated bootstrap: old file downloads, new upload works") + print("PASS: bucket/user/policy bootstrap can be repeated safely") + + +if __name__ == "__main__": + run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console, + test_bootstrap_is_idempotent]) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..01f1ced --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,222 @@ +"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик. + +Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что +тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково +(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше +DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование) +при тех же 10 загрузках не помещается в лимит 512 MiB. + +Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков +одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status +(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера +(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с. + + python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json] +""" +import argparse +import hashlib +import json +import platform +import statistics +import threading +import time + +import images +from common import (ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_attachment, + wait_balanced, wait_ready) + + +def proc_status(container): + """VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты).""" + script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; " + "for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current " + "/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; " + "do [ -r $f ] && echo \"$f $(cat $f)\"; done; true") + out = docker("exec", container, "sh", "-c", script) + values = {} + for line in out.splitlines(): + parts = line.replace(":", " ").split() + if parts[0] in ("VmRSS", "VmHWM"): + values[parts[0]] = int(parts[1]) + elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")): + values["cgroup_peak"] = int(parts[1]) + elif parts[0].endswith(("memory.current", "usage_in_bytes")): + values["cgroup_current"] = int(parts[1]) + return values + + +class Sampler(threading.Thread): + def __init__(self, containers): + super().__init__(daemon=True) + self.containers = containers + self.stop = threading.Event() + self.rss = {c: 0 for c in containers} + self.cgroup = {c: 0 for c in containers} + self.samples = 0 + + def run(self): + while not self.stop.is_set(): + for c in self.containers: + try: + v = proc_status(c) + except Exception: # noqa: BLE001 — один пропущенный замер не важен + continue + self.rss[c] = max(self.rss[c], v.get("VmRSS", 0)) + self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0)) + self.samples += 1 + + +def api_containers(): + from common import api_containers as running + return running() + + +def fresh_replicas(): + compose("restart", "api") + containers = wait_balanced(2, seconds=90) + wait_ready() + return containers + + +def warm_up(user, chat): + """По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п.""" + for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4: + upload(user, chat["id"], data, ctype) + + +def scenario(name, payloads, concurrency): + user = new_user("mem") + chat = new_chat(user) + containers = fresh_replicas() + warm_up(user, chat) + before = {c: proc_status(c) for c in containers} + sampler = Sampler(containers) + sampler.start() + barrier = threading.Barrier(concurrency) + results = [None] * concurrency + + def worker(i): + data, ctype = payloads[i % len(payloads)] + barrier.wait() + started = time.monotonic() + try: + # lab5: замеряем только приём (202 queued); обработку worker'ом ждём ниже, вне замера + reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180, wait=False) + results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data) + except AssertionError as exc: + results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)] + wall = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - wall + time.sleep(0.5) + sampler.stop.set() + sampler.join() + after = {c: proc_status(c) for c in containers} + + ok = [r for r in results if r[0] == 202] + check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 202]}") + for status, _, _, att, data in ok[:3]: + att = wait_attachment(user, att["id"], seconds=120) + check(att["status"] == "ready", f"{name}: attachment {att['id']} is {att['status']}, expected ready") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"]) + check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs") + latencies = sorted(r[1] for r in results) + per_replica = [] + for c in containers: + b, a = before[c], after[c] + per_replica.append({ + "replica": c[:12], + "uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])), + "rss_before_mib": round(b["VmRSS"] / 1024, 1), + "rss_peak_mib": round(a["VmHWM"] / 1024, 1), + "rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1), + "rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1), + "cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None, + "cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None, + }) + sizes = sorted({len(d) for d, _ in payloads}) + summary = { + "scenario": name, + "concurrency": concurrency, + "file_bytes": sizes, + "total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1), + "all_202": True, + "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)}, + "sampler_rounds": sampler.samples, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False)) + return summary + + +# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование. +NAIVE_VS_STREAMING = r''' +import json, os, subprocess, sys +code = r""" +import io, os, resource, struct, sys, zlib +sys.path.insert(0, os.getcwd()) +os.environ.setdefault("S3_ACCESS_KEY_ID", "x") +from PIL import Image +import uploads # импорт до замера: сравниваем только саму проверку файла +Image.MAX_IMAGE_PIXELS = None +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +w, h = 8000, 5000 +comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3) +idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush() +data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") +del idat, comp, row +base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +if sys.argv[1] == "naive": + body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB) + im = Image.open(io.BytesIO(data)); im.load() +else: + f = io.BytesIO(data); uploads.check_png(f) +print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024) +""" +out = {} +for mode in ("naive", "streaming"): + r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd()) + out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:] +print(json.dumps(out)) +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default="evidence/upload-memory.json") + args = parser.parse_args() + check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads") + + print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}") + print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...") + large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)] + pixels = [(images.png(8000, 5000), "image/png")] + jpeg = [(images.sample_jpeg(), "image/jpeg")] + print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes") + + runs = [ + scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency), + scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency), + scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency), + ] + naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1]) + print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) " + f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB") + limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split() + result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20, + "api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive} + (ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"]) + print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB " + f"(limit {result['api_limit_mib']} MiB); results in {args.output}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/worker_probe.py b/student-tests/worker_probe.py new file mode 100644 index 0000000..7b75584 --- /dev/null +++ b/student-tests/worker_probe.py @@ -0,0 +1,148 @@ +"""Пробник, который выполняется ВНУТРИ контейнера worker (там Pillow, модули приложения, доступ к БД и S3). + +Тест передаёт этот файл через stdin: `docker compose exec -T worker python - <команда> [аргументы]`. +Секреты остаются в контейнере; наружу выходит одна строка JSON (без ключей доступа, только факты). + + make — тестовое изображение, base64 (kind: thumb320, exif6, alpha, tiny, large, strip, big40) + inspect — метаданные из БД + декодированные объекты из S3: размеры, формат, EXIF, + цвета контрольных точек, SHA-256, список ключей под attachments// + exists — есть ли объект в bucket + imaging-limits — imaging.py на 40-Мп PNG: пик RSS дочернего процесса и отказ по лимиту пикселей +""" +import base64 +import hashlib +import io +import json +import os +import resource +import subprocess +import sys +import tempfile +from uuid import UUID + +from PIL import Image + +import psycopg +from psycopg.rows import dict_row + +import settings +import storage + + +def make(kind: str) -> bytes: + out = io.BytesIO() + if kind == "thumb320": # 320x200 -> thumbnail 256x160 (контракт) + img = Image.new("RGB", (320, 200), (200, 30, 30)) + img.paste((30, 30, 200), (0, 0, 160, 200)) + img.save(out, "PNG") + elif kind == "exif6": # JPEG 400x300 с EXIF Orientation=6 и Make/Model/GPS + img = Image.new("RGB", (400, 300), (0, 160, 0)) + img.paste((255, 0, 0), (0, 0, 100, 300)) # левая полоса красная: после поворота на 90° по часовой — сверху + exif = Image.Exif() + exif[0x0112] = 6 + exif[0x010F] = "ProbeCamera" + exif[0x0110] = "Model-X" + exif[0x8825] = {1: "N", 2: (55.0, 45.0, 0.0)} + img.save(out, "JPEG", quality=95, exif=exif.tobytes()) + elif kind == "alpha": # RGBA 300x200: левая половина прозрачная + img = Image.new("RGBA", (300, 200), (0, 0, 255, 255)) + img.paste((0, 0, 0, 0), (0, 0, 150, 200)) + img.save(out, "PNG") + elif kind == "tiny": # 40x30: без увеличения для thumbnail/optimized + Image.new("RGB", (40, 30), (10, 120, 220)).save(out, "PNG") + elif kind == "large": # 3000x2000 JPEG: optimized 1280x853, thumbnail 256x170 + img = Image.radial_gradient("L").resize((3000, 2000)).convert("RGB") + img.save(out, "JPEG", quality=90) + elif kind == "strip": # 5000x2: thumbnail 256x1 (минимум 1 px) + Image.new("RGB", (5000, 2), (90, 90, 90)).save(out, "PNG") + elif kind == "big40": # 8000x5000 = 40 Мп ровно на лимите + Image.new("RGB", (8000, 5000), (120, 60, 30)).save(out, "PNG") + else: + raise SystemExit("unknown kind") + return out.getvalue() + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row) + + +def read(key: str) -> bytes: + stream = storage.open_read(key) + try: + return b"".join(storage.iter_chunks(stream)) + finally: + stream.close() + + +def inspect(attachment_id: str) -> dict: + with connect() as conn: + a = conn.execute("SELECT state, object_key, source_key, pipeline_version, sha256, size_bytes " + "FROM attachments WHERE id = %s", (UUID(attachment_id),)).fetchone() + variants = conn.execute("SELECT name, content_type, width, height, sha256, size_bytes, object_key " + "FROM attachment_variants WHERE attachment_id = %s ORDER BY name", + (UUID(attachment_id),)).fetchall() + result = {"state": a["state"], "staging_present": a["source_key"] is not None and storage.exists(a["source_key"]), + "variant_rows": len(variants), "variants": {}} + if a["object_key"]: + original = read(a["object_key"]) + result["original_sha_ok"] = hashlib.sha256(original).hexdigest() == a["sha256"] + result["original_size_ok"] = len(original) == a["size_bytes"] + for v in variants: + data = read(v["object_key"]) + img = Image.open(io.BytesIO(data)) + info = {"format": img.format, "size": list(img.size), "mode": img.mode, + "meta_size_ok": [v["width"], v["height"]] == list(img.size), + "sha_ok": hashlib.sha256(data).hexdigest() == v["sha256"] and len(data) == v["size_bytes"], + "exif_tags": len(img.getexif()), "has_xmp": "xmp" in img.info or b"http://ns.adobe.com/xap" in data, + "quality_hint": None} + rgb = img.convert("RGBA") + w, h = rgb.size + info["pixel_top_center"] = list(rgb.getpixel((w // 2, 0))) + info["pixel_left_center"] = list(rgb.getpixel((0, h // 2))) + info["pixel_right_center"] = list(rgb.getpixel((w - 1, h // 2))) + result["variants"][v["name"]] = info + prefix = f"attachments/{attachment_id}/" + result["final_keys"] = sorted(key[len(prefix):] for key, _m, _s in storage.list_objects(prefix)) + return result + + +def imaging_limits() -> dict: + """Пик памяти процесса обработки на худшем допустимом входе и отказ при превышении лимита пикселей.""" + here = os.path.dirname(os.path.abspath(storage.__file__)) + report = {} + with tempfile.TemporaryDirectory(dir=settings.WORKER_TMP_DIR) as tmp: + source = os.path.join(tmp, "big.png") + with open(source, "wb") as f: + f.write(make("big40")) + env = {"PATH": os.environ.get("PATH", ""), "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for label, max_pixels in (("at_limit_40mpx", 40_000_000), ("over_limit_max_10mpx", 10_000_000)): + out = os.path.join(tmp, label) + os.mkdir(out) + before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + proc = subprocess.run([sys.executable, os.path.join(here, "imaging.py"), source, out, "image/png", + str(max_pixels)], env=env, capture_output=True, timeout=120) + peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + report[label] = {"exit_code": proc.returncode, "peak_rss_mib_max_children": round(peak / 1024, 1), + "rss_grew": peak > before} + report["memory_limit_mib"] = settings.WORKER_MEMORY_LIMIT_MB + report["timeout_s"] = settings.JOB_TIMEOUT_SECONDS + return report + + +def main(): + command = sys.argv[1] + if command == "make": + print(json.dumps({"b64": base64.b64encode(make(sys.argv[2])).decode("ascii")})) + elif command == "inspect": + print(json.dumps(inspect(sys.argv[2]))) + elif command == "exists": + print(json.dumps({"exists": storage.exists(sys.argv[2])})) + elif command == "imaging-limits": + print(json.dumps(imaging_limits())) + else: + raise SystemExit("unknown command") + + +if __name__ == "__main__": + main() diff --git a/uploads.py b/uploads.py new file mode 100644 index 0000000..c01bfc1 --- /dev/null +++ b/uploads.py @@ -0,0 +1,519 @@ +"""Приём и проверка загружаемого изображения с ограниченной памятью. + +1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше + MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном + файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса. +2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях + читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково + (CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма), + JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение. +""" +import asyncio +import hashlib +import logging +import struct +import tempfile +import threading +import time +import unicodedata +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image +from starlette.requests import ClientDisconnect, Request + +import settings +from errors import bad_request, payload_too_large, unsupported_media_type + +log = logging.getLogger("messenger.uploads") + +PNG = "image/png" +JPEG = "image/jpeg" +ALLOWED_TYPES = (PNG, JPEG) +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" + +SPOOL_IN_MEMORY = 256 * 1024 +INFLATE_CHUNK = 64 * 1024 +CRC_CHUNK = 64 * 1024 + +# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке). +Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS + +# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY. +_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY) + + +@dataclass +class Upload: + file: BinaryIO + size: int + sha256: str + + +@dataclass +class ImageInfo: + content_type: str + width: int + height: int + + +class Damaged(Exception): + pass + + +class TooManyPixels(Exception): + pass + + +# ---------- имя файла и заголовки ---------- + +def validate_filename(value: str | None) -> str: + """filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов.""" + if value is None: + raise bad_request("Query parameter filename is required") + if not 1 <= len(value) <= 255 or not value.strip(): + raise bad_request("filename must be 1-255 characters") + if value in (".", ".."): + raise bad_request("filename is not allowed") + for char in value: + if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"): + raise bad_request("filename contains forbidden characters") + return value + + +def declared_type(header: str | None) -> str: + base = (header or "").split(";", 1)[0].strip().lower() + if base not in ALLOWED_TYPES: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return base + + +def declared_length(header: str | None) -> int | None: + if header is None: + return None + if not header.isdigit(): + raise bad_request("Invalid Content-Length") + length = int(header) + if length > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + return length + + +# ---------- приём тела ---------- + +async def receive(request: Request) -> Upload: + """Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется.""" + spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR) + digest = hashlib.sha256() + size = 0 + deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS + stream = request.stream().__aiter__() + try: + while True: + wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic()) + if wait <= 0: + raise bad_request("Upload took too long") + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except StopAsyncIteration: + break + except asyncio.TimeoutError: + raise bad_request("Upload stalled or took too long") from None + if not chunk: + continue + size += len(chunk) + if size > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + digest.update(chunk) + spool.write(chunk) + except ClientDisconnect: + spool.close() + log.info("Upload aborted by the client after %s bytes, nothing stored", size) + raise bad_request("Upload was interrupted") from None + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise bad_request("Empty body") + return Upload(spool, size, digest.hexdigest()) + + +async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None: + """Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв. + + Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается. + """ + read = 0 + deadline = time.monotonic() + seconds + try: + stream = request.stream().__aiter__() + while read <= limit: + wait = deadline - time.monotonic() + if wait <= 0: + return + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except (StopAsyncIteration, asyncio.TimeoutError): + return + read += len(chunk) + except (ClientDisconnect, RuntimeError): + return + + +# ---------- проверка содержимого ---------- + +def sniff(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return PNG + if head.startswith(JPEG_SIGNATURE): + return JPEG + return None + + +def inspect(upload: Upload, declared: str) -> ImageInfo: + upload.file.seek(0) + detected = sniff(upload.file.read(16)) + if detected is None: + raise unsupported_media_type("File is not a PNG or JPEG image") + if detected != declared: + raise unsupported_media_type("Content-Type does not match the file content") + upload.file.seek(0) + try: + with _decode_slots: + if detected == PNG: + width, height = check_png(upload.file) + upload.file.seek(0) + check_png_metadata(upload.file) + else: + width, height = check_jpeg(upload.file) + except TooManyPixels: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except Damaged as exc: + log.info("Rejected damaged %s: %s", detected, exc) + raise unsupported_media_type("Image is damaged or not supported") from None + except Image.DecompressionBombError: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc: + log.info("Rejected damaged %s: %s", detected, type(exc).__name__) + raise unsupported_media_type("Image is damaged or not supported") from None + finally: + upload.file.seek(0) + return ImageInfo(detected, width, height) + + +def check_pixels(width: int, height: int) -> None: + if width < 1 or height < 1: + raise Damaged("zero dimension") + if width * height > settings.MAX_IMAGE_PIXELS: + raise TooManyPixels() + + +def check_jpeg(file: BinaryIO) -> tuple[int, int]: + with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS + width, height = image.size + check_pixels(width, height) + # Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8: + # 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB. + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + return width, height + + +# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR). +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +FILTER_BYTES = bytes(range(5)) + + +class PngRows: + """Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки.""" + + def __init__(self, width, height, depth, color, interlace): + bits = depth * PNG_CHANNELS[color] + + def row_bytes(w): + return (w * bits + 7) // 8 + + passes = [(width, height)] if interlace == 0 else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + # (длина строки с байтом фильтра, байт в проходе) + self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0] + self.expected = sum(total for _, total in self.passes) + self.index, self.pos = 0, 0 + self.total = 0 + + def feed(self, data: bytes) -> None: + view = memoryview(data) + self.total += len(data) + if self.total > self.expected: + raise Damaged("more pixel data than the header says") + offset = 0 + while offset < len(view): + stride, size = self.passes[self.index] + take = min(len(view) - offset, size - self.pos) + first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride + if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES): + raise Damaged("invalid filter type") + self.pos += take + offset += take + if self.pos == size: + self.index, self.pos = self.index + 1, 0 + + def done(self) -> bool: + return self.total == self.expected + + +# Чанки, содержимое которых проверяется (держим в памяти; размер ограничен спецификацией): +# IHDR — 13 байт, PLTE — до 256 x 3 = 768, tRNS — до 256 (палитра) / 6 (RGB). +SMALL_CHUNK_LIMIT = {b"IHDR": 13, b"PLTE": 768, b"tRNS": 256} +CRITICAL = (b"IHDR", b"PLTE", b"IDAT", b"IEND") +# После данных изображения допустимы только время и текст (строгий список). +AFTER_IDAT = {b"IEND", b"tIME", b"tEXt", b"zTXt", b"iTXt"} +TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt") +# Анимированные PNG (APNG) не поддерживаются: кадры — отдельная сущность, превью строится по одному кадру. +APNG_CHUNKS = (b"acTL", b"fcTL", b"fdAT") +# Как PngImagePlugin.MAX_TEXT_CHUNK в Pillow: больше 1 MiB распакованного текста в одном чанке — отказ. +TEXT_LIMIT = 1024 * 1024 +# Чанки метаданных до IDAT Pillow (Image.open) читает целиком: ограничиваем, чтобы память не росла. +# ICC-профили и EXIF реальных картинок — единицы-сотни KiB. +PRE_IDAT_CHUNK_LIMIT = 2 * 1024 * 1024 +# Чанки, которые по спецификации обязаны стоять ДО первого IDAT (PNG 3rd ed., 5.6 «Chunk ordering»). +# После IDAT допустимы только tIME, tEXt, zTXt, iTXt, кадры APNG и частные чанки. +BEFORE_IDAT = {b"PLTE", b"tRNS", b"cHRM", b"cICP", b"gAMA", b"iCCP", b"mDCV", b"cLLI", b"sBIT", b"sRGB", + b"bKGD", b"hIST", b"pHYs", b"sPLT", b"eXIf"} + + +def check_png(file: BinaryIO) -> tuple[int, int]: + """Структура PNG по спецификации (PNG 3rd ed., гл. 5 и 11) без декодирования пикселей в память. + + Кроме CRC каждого чанка проверяется СМЫСЛ критичных чанков и tRNS: CRC защищает только от + случайной порчи, а намеренно собранный файл с верными CRC может быть некорректным. + """ + if file.read(8) != PNG_SIGNATURE: + raise Damaged("signature") + rows = inflater = None + width = height = depth = 0 + color = None + palette_entries = 0 + seen = set() # какие чанки уже встречались + idat_started = idat_finished = False + first = True + while True: + header = file.read(8) + if len(header) < 8: + raise Damaged("truncated: no IEND") + length, kind = struct.unpack(">I4s", header) + if length > 0x7FFFFFFF: + raise Damaged("chunk length") + if not kind.isalpha() or not kind.isascii(): + raise Damaged("chunk type") + if first and kind != b"IHDR": + raise Damaged("IHDR must be first") + if kind[:1].isupper() and kind not in CRITICAL: + raise Damaged(f"unknown critical chunk {kind!r}") # такой чанк декодер обязан отвергнуть + if kind in SMALL_CHUNK_LIMIT and length > SMALL_CHUNK_LIMIT[kind]: + raise Damaged(f"{kind!r} is too long") + if kind == b"IDAT": + if rows is None or idat_finished: + raise Damaged("IDAT chunks must be consecutive") + idat_started = True + elif idat_started: + idat_finished = True + if kind in APNG_CHUNKS: + raise Damaged("animated PNG is not supported") + if kind in BEFORE_IDAT and idat_started: + raise Damaged(f"{kind!r} after IDAT") + if idat_finished and kind not in AFTER_IDAT: + raise Damaged(f"{kind!r} after IDAT") + if not idat_started and kind != b"IDAT" and length > PRE_IDAT_CHUNK_LIMIT: + raise Damaged(f"{kind!r} metadata chunk is too large") + if kind == b"tIME" and length != 7: + raise Damaged("tIME size") + if kind in (b"IHDR", b"PLTE", b"tRNS", b"IEND") and kind in seen: + raise Damaged(f"duplicate {kind!r}") + seen.add(kind) + + crc = zlib.crc32(kind) + kept = b"" + text = TextChunk(kind) if kind in TEXT_CHUNKS else None + left = length + while left: + piece = file.read(min(CRC_CHUNK, left)) + if not piece: + raise Damaged("truncated chunk") + left -= len(piece) + crc = zlib.crc32(piece, crc) + if kind in SMALL_CHUNK_LIMIT: + kept += piece + elif kind == b"IDAT": + feed_inflate(inflater, rows, piece) + elif text is not None: + text.feed(piece) + if text is not None: + text.finish() + stored = file.read(4) + if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: + raise Damaged(f"CRC mismatch in {kind!r}") + + if kind == b"IHDR": + if length != 13: + raise Damaged("IHDR") + width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) + if width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise Damaged("dimensions") + check_pixels(width, height) # до распаковки + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]: + raise Damaged("color type / bit depth") + if compression != 0 or filter_method != 0 or interlace not in (0, 1): + raise Damaged("IHDR methods") + rows = PngRows(width, height, depth, color, interlace) + inflater = zlib.decompressobj() + elif kind == b"PLTE": + # 1-256 записей по 3 байта; для палитровых — не больше 2^глубина; у серых палитры быть не может. + if color in (0, 4): + raise Damaged("PLTE in a grayscale image") + if length == 0 or length % 3: + raise Damaged("palette size") + palette_entries = length // 3 + if color == 3 and palette_entries > 1 << depth: + raise Damaged("palette larger than the bit depth allows") + elif kind == b"tRNS": + # Прозрачность: серое — 2 байта, RGB — 6, палитра — не больше записей PLTE; при альфа-канале запрещён. + if color in (4, 6): + raise Damaged("tRNS with an alpha channel") + if color == 3 and not palette_entries: + raise Damaged("tRNS before PLTE") + expected_ok = {0: length == 2, 2: length == 6, 3: 1 <= length <= palette_entries}[color] + if not expected_ok: + raise Damaged("tRNS size") + elif kind == b"IEND": + if length != 0: + raise Damaged("IEND with data") + break + first = False + + if file.read(1): + raise Damaged("data after IEND") + if color == 3 and not palette_entries: + raise Damaged("palette image without PLTE") + if not idat_started: + raise Damaged("no IDAT") + tail = inflater.flush(INFLATE_CHUNK) + if tail: + rows.feed(tail) + if not inflater.eof or inflater.unused_data or not rows.done(): + raise Damaged("pixel data is incomplete") + return width, height + + +def check_png_metadata(file: BinaryIO) -> None: + """Второе мнение: стандартный парсер Pillow разбирает метаданные до IDAT (gAMA, iCCP, pHYs, tEXt…), + НЕ читая пиксели. Только open, без verify(): verify читает каждый чанк, включая IDAT до 10 MiB, + целиком в память, а CRC и данные после IDAT уже потоково проверил check_png. + Нужен для вспомогательных чанков: по спецификации их можно игнорировать, но если Pillow + не может их разобрать, файл нельзя будет обработать (превью в lab5) — лучше отклонить сразу. + """ + with Image.open(file, formats=["PNG"]): + pass # open разбирает все чанки до первого IDAT; после IDAT check_png пускает только время и текст + + +class TextChunk: + """Потоковая проверка tEXt / zTXt / iTXt: ключ 1-79 байт, метод сжатия 0, сжатый текст — + корректный zlib не больше TEXT_LIMIT после распаковки (распаковка кусками, текст не хранится).""" + + def __init__(self, kind: bytes): + self.kind = kind + self.head = b"" + self.done_header = False + self.inflater = None + self.size = 0 + + def feed(self, piece: bytes) -> None: + if not self.done_header: + self.head += piece + rest = self._header() + if rest is None: + if len(self.head) > 4096: + raise Damaged("text chunk header") + return + piece = rest + if self.inflater is not None and piece: + self._inflate(piece) + + def _keyword_end(self) -> int | None: + end = self.head.find(b"\x00") + if end == -1: + return None + if not 1 <= end <= 79: + raise Damaged("text keyword") + return end + + def _header(self) -> bytes | None: + end = self._keyword_end() + if end is None: + return None + h = self.head + if self.kind == b"tEXt": + self.done_header = True + return b"" + if self.kind == b"zTXt": + if len(h) < end + 2: + return None + if h[end + 1] != 0: + raise Damaged("zTXt compression method") + self.done_header, self.inflater = True, zlib.decompressobj() + return h[end + 2:] + # iTXt: keyword \0 flag method language \0 translated-keyword \0 text + if len(h) < end + 3: + return None + flag, method = h[end + 1], h[end + 2] + if flag not in (0, 1) or (flag and method != 0): + raise Damaged("iTXt compression") + lang_end = h.find(b"\x00", end + 3) + tkey_end = h.find(b"\x00", lang_end + 1) if lang_end != -1 else -1 + if tkey_end == -1: + return None + self.done_header = True + if flag: + self.inflater = zlib.decompressobj() + return h[tkey_end + 1:] + + def _inflate(self, data: bytes) -> None: + if self.inflater.eof: + raise Damaged("data after compressed text") + out = self.inflater.decompress(data, INFLATE_CHUNK) + self.size += len(out) + while self.inflater.unconsumed_tail and not self.inflater.eof and self.size <= TEXT_LIMIT: + out = self.inflater.decompress(self.inflater.unconsumed_tail, INFLATE_CHUNK) + self.size += len(out) + if self.size > TEXT_LIMIT: + raise Damaged("compressed text is too large") + if self.inflater.eof and self.inflater.unused_data: + raise Damaged("data after compressed text") + + def finish(self) -> None: + if not self.done_header: + raise Damaged("text chunk header") + if self.inflater is not None: + self.size += len(self.inflater.flush(INFLATE_CHUNK)) + if not self.inflater.eof or self.size > TEXT_LIMIT: + raise Damaged("compressed text") + + +def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: + """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" + if inflater.eof: + if data: + raise Damaged("data after the end of zlib stream") + return + out = inflater.decompress(data, INFLATE_CHUNK) + rows.feed(out) + while inflater.unconsumed_tail and not inflater.eof: + out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK) + rows.feed(out) + if inflater.eof and inflater.unused_data: + raise Damaged("data after the end of zlib stream") diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..cbab0ad --- /dev/null +++ b/worker.py @@ -0,0 +1,390 @@ +"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose). + +Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1): +одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди. + +Обработка одного сообщения: + 1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ. + 2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый + lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op. + 3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает -> + аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued. + Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения. + 4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД. + 5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь). + 6. PUT в конечный prefix: attachments//v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp. + Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи. + 7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT. + 8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД. + 9. Удаление исходника из staging (если не вышло — доделает janitor). +Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД, +таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed. +""" +import hashlib +import json +import logging +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +import broker +import faults +import imaging +import pipeline +import settings +import storage + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.worker") + +STOP = threading.Event() +HERE = Path(__file__).resolve().parent +VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"} +IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large", + imaging.EXIT_MEMORY: "resource_limit"} + + +class JobError(Exception): + def __init__(self, code: str, permanent: bool): + super().__init__(code) + self.code = code + self.permanent = permanent + + +class LeaseLost(Exception): + pass + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def final_prefix(claim: pipeline.Claim) -> str: + return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/" + + +class LeaseKeeper(threading.Thread): + """Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку.""" + + def __init__(self, claim: pipeline.Claim): + super().__init__(daemon=True) + self.claim = claim + self.done = threading.Event() + self.lost = threading.Event() + self.frozen = threading.Event() # fault worker_freeze: перестать продлевать + + def run(self): + interval = max(1.0, settings.JOB_LEASE_SECONDS / 3) + conn = None + while not self.done.wait(interval): + if self.frozen.is_set(): + continue + try: + if conn is None: + conn = connect_db() + if not pipeline.renew_lease(conn, self.claim): + log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id) + self.lost.set() + return + except psycopg.Error as exc: + # Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT). + log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__) + if conn is not None: + conn.close() + conn = None + if conn is not None: + conn.close() + + def check(self): + if self.lost.is_set(): + raise LeaseLost() + + +def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: + # Дочернему процессу — только лимиты, никаких секретов окружения. + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1", + "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for fault in ("hang", "invalid", "oom"): + if faults.hit("imaging_" + fault): + env["FAULT_IMAGING"] = fault + process = subprocess.Popen( + [sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type, + str(settings.MAX_IMAGE_PIXELS)], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir), + ) + deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS + try: + while True: + try: + output, _ = process.communicate(timeout=0.5) + break + except subprocess.TimeoutExpired: + if time.monotonic() > deadline: + raise JobError("processing_timeout", permanent=False) from None + keeper.check() + finally: + if process.poll() is None: + process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку + process.wait() + code = process.returncode + if code in IMAGING_EXIT: + raise JobError(IMAGING_EXIT[code], permanent=True) + if code != 0: + # Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены. + raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False) + return json.loads(output.decode("utf-8").strip().splitlines()[-1]) + + +def sha256_file(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + with open(path, "rb") as file: + while chunk := file.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None: + if faults.hit("worker_s3_error"): + raise JobError("storage_unavailable", permanent=False) + with open(path, "rb") as file: + try: + storage.put(key, file, size, content_type, sha256) + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + + +def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]: + """Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов.""" + source = workdir / "source" + try: + storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES) + except storage.ObjectMissing: + raise JobError("source_missing", permanent=True) from None + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + size, digest = sha256_file(source) + if size != claim.size_bytes or digest != claim.sha256: + # Staging не совпадает с тем, что принял API: публиковать нельзя. + raise JobError("source_corrupted", permanent=True) + keeper.check() + + out_dir = workdir / "out" + out_dir.mkdir() + result = run_imaging(source, out_dir, claim, keeper) + keeper.check() + + prefix = final_prefix(claim) + original_key = prefix + "original" + put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes) + variants = [] + for item in result["variants"]: + path = out_dir / item["file"] + v_size, v_sha = sha256_file(path) + key = prefix + VARIANT_KEYS[item["name"]] + put_file(key, path, item["content_type"], v_sha, v_size) + variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size, + "width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key}) + keeper.check() + return original_key, variants + + +@dataclass +class Outcome: + ack: bool = True # False -> nack с requeue (не смогли даже записать claim) + reject: bool = False # True -> reject без requeue -> DLQ + cleanup_key: str | None = None + attachment_id: object = None + + +class Consumer(threading.Thread): + def __init__(self, number: int): + super().__init__(name=f"consumer-{number}", daemon=True) + self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}" + self.db = None + + def database(self): + if self.db is None or self.db.closed: + self.db = connect_db() + return self.db + + def handle(self, body: bytes, redelivered: bool) -> Outcome: + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + log.warning("invalid message -> DLQ (%s)", exc) + return Outcome(reject=True) + try: + result = pipeline.claim(self.database(), message, self.owner, redelivered) + except psycopg.Error as exc: + log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__) + self.db = None + STOP.wait(2) + return Outcome(ack=False) + if isinstance(result, pipeline.Reject): + log.warning("job %s: %s -> DLQ", message["job_id"], result.reason) + return Outcome(reject=True) + if isinstance(result, pipeline.Skip): + log.info("job %s: no-op (%s), ack", message["job_id"], result.reason) + return Outcome() + return self.execute(result) + + def execute(self, claim: pipeline.Claim) -> Outcome: + started = time.monotonic() + log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt, + claim.max_attempts, claim.attachment_id) + keeper = LeaseKeeper(claim) + keeper.start() + workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR)) + try: + try: + original_key, variants = process_job(claim, keeper, workdir) + faults.crash("worker_crash_after_put") + if faults.hit("worker_freeze"): + keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят + time.sleep(settings.JOB_LEASE_SECONDS * 3) + committed = pipeline.finish_ready(self.database(), claim, original_key, variants, self.owner) + except JobError as exc: + outcome = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner) + log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code, + "permanent" if exc.permanent else "temporary", outcome) + return Outcome() + except LeaseLost: + log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt) + return Outcome() + except psycopg.Error: + raise + except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены + log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt) + pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner) + return Outcome() + except psycopg.Error as exc: + # Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и + # dispatcher вернёт job в очередь; если прошёл — повтор увидит ready. + log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__) + self.db = None + return Outcome() + finally: + keeper.done.set() + shutil.rmtree(workdir, ignore_errors=True) + if not committed: + log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id, + claim.attempt) + return Outcome() + faults.crash("worker_crash_after_commit") + log.info("job %s attempt %s: ready in %.0f ms", claim.job_id, claim.attempt, + (time.monotonic() - started) * 1000) + return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id) + + def cleanup_staging(self, outcome: Outcome) -> None: + """После COMMIT ready и ack: исходник в staging больше не нужен.""" + try: + storage.delete(outcome.cleanup_key) + self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' " + "AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key)) + except (storage.StorageUnavailable, psycopg.Error) as exc: + log.info("staging cleanup postponed to janitor: %s", type(exc).__name__) + self.db = None + + def consume_once(self) -> None: + connection = broker.connect() + try: + channel = connection.channel() + channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH) + broker.check_topology(channel) + log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH) + for method, _properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1): + if STOP.is_set(): + break + if method is None: + continue + box = {} + runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle( + body, method.redelivered)), daemon=True) + runner.start() + while runner.is_alive(): + connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа + outcome = box.get("outcome", Outcome(ack=False)) + if outcome.reject: + channel.basic_reject(method.delivery_tag, requeue=False) + elif outcome.ack: + channel.basic_ack(method.delivery_tag) + else: + channel.basic_nack(method.delivery_tag, requeue=True) + if outcome.cleanup_key: + self.cleanup_staging(outcome) + try: + channel.cancel() + except broker.BrokerError: + pass + finally: + try: + connection.close() + except broker.BrokerError: + pass + + def safe_handle(self, body, redelivered) -> Outcome: + try: + return self.handle(body, redelivered) + except Exception: # noqa: BLE001 + log.exception("unexpected error while handling a message; requeue") + self.db = None + STOP.wait(2) + return Outcome(ack=False) + + def run(self): + pause = 1.0 + while not STOP.is_set(): + try: + self.consume_once() + pause = 1.0 + except broker.BrokerError as exc: + log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name, + type(exc).__name__, pause) + STOP.wait(pause) + pause = min(pause * 2, 15) + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True) + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)] + for consumer in consumers: + consumer.start() + log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, " + "fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH, + settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB, + "ON" if settings.FAULT_INJECTION else "off") + while not STOP.wait(2): + if not all(consumer.is_alive() for consumer in consumers): + log.error("a consumer thread died, exiting so that Docker restarts the worker") + os._exit(1) + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят. + for consumer in consumers: + consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10) + log.info("stopped") + + +if __name__ == "__main__": + main()