diff --git a/.env.example b/.env.example index 119384b..818a90a 100644 --- a/.env.example +++ b/.env.example @@ -6,20 +6,56 @@ APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Optional on 3, required use on 4 in lab2; Redis OR Valkey (this solution: Redis since lab2): +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +# SeaweedFS S3 (since lab4) +S3_ENDPOINT=http://s3:8333 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 -BROKER_URL_FILE=/run/secrets/broker_url + +# lab5: broker and worker. The API has NO broker credentials (it writes the transactional outbox); +# compose.yaml passes per-service files: dispatcher -> /run/secrets/broker_url_dispatcher, +# worker -> /run/secrets/broker_url_worker. This variable is therefore empty for api/janitor. +BROKER_URL_FILE= WORKER_CONCURRENCY=2 +WORKER_PREFETCH=1 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=15 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=60 +WORKER_MEMORY_LIMIT_MB=512 STAGING_RETENTION_SECONDS=86400 +PIPELINE_VERSION=1 +# Fault injection (worker/dispatcher only, triggered by files in the faults volume): compose.yaml enables it on the +# training stand by default; set FAULT_INJECTION=0 here to disable. + +# Pools and timeouts (same as lab3) +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_WAIT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_POOL_MAX=20 +REDIS_TIMEOUT_MS=500 +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=10 + +# lab4: S3 client and uploads +S3_CONNECT_TIMEOUT_SECONDS=1 +S3_READ_TIMEOUT_SECONDS=5 +S3_MAX_ATTEMPTS=2 +S3_POOL_MAX=20 +UPLOAD_TIMEOUT_SECONDS=60 +UPLOAD_IDLE_TIMEOUT_SECONDS=10 +DECODE_CONCURRENCY=2 +# Cleanup of abandoned uploads and staging (janitor service): grace period and pass interval +ATTACHMENT_GRACE_SECONDS=3600 +JANITOR_INTERVAL_SECONDS=300 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..3fd8009 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.example text eol=lf +*.cmd text eol=crlf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a..3fb6285 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,14 +1,14 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Бабиченко Максим, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: из lab4 — весь API, сессии в Redis, балансировка, приём/проверка файлов, приватный S3, janitor. Новое: RabbitMQ (quorum-очередь + DLQ, definitions с минимальными правами), transactional outbox + dispatcher (publisher confirms), отдельный worker (2 потока, prefetch 1, ручной ack, аренда с fencing, backoff с jitter, постоянные/временные ошибки), варианты thumbnail/crop/optimized в дочернем процессе с RLIMIT, replay и разбор DLQ (`ops.py`), очистка staging. Upload: 201 → 202 queued. +Команды запуска из чистого clone: `Copy-Item .env.example .env; python scripts/init_secrets.py; docker compose up -d --build --wait; python student-tests/collect_evidence.py` (подробно — REPORT.md, раздел «Запуск»). +Результаты проверок и ссылка на REPORT.md: REPORT.md, раздел «Доказательства критериев»; `evidence/summary.md`. +Ограничения / известные ошибки: один узел RabbitMQ; worker использует ограниченный S3-ключ API; точки внедрения сбоев включены на учебном стенде (`FAULT_INJECTION=1`). См. REPORT.md, «Ограничения». +Источники и использование ИИ: REPORT.md, шапка. - [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. - [ ] Контракт и публичные тесты не ослаблены. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..dd51df0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.11.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt + +COPY main.py settings.py errors.py db.py auth.py migrate.py storage.py uploads.py attachments.py janitor.py s3_init.py \ + pipeline.py broker.py dispatcher.py worker.py imaging.py faults.py ops.py ./ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser \ + && mkdir -p /faults && chown appuser:appuser /faults +# /faults — точка монтирования volume faults (учебное внедрение сбоев worker/dispatcher); владелец appuser +# копируется в пустой named volume при первом монтировании. + +USER appuser + +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)" + +CMD ["python", "main.py"] diff --git a/REPORT.md b/REPORT.md index e98f75e..b91db7f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,292 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бабиченко Максим, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран +- Commit/PR: <<ссылка на MR в Gitea после публикации>> +- Использованные источники, библиотеки и помощь ИИ: всё из lab4 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py, boto3/botocore 1.43.116, Pillow 12.3.0; образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine, haproxy:3.2.25-alpine, chrislusf/seaweedfs:3.80). Новое: брокер RabbitMQ (образ `rabbitmq:4.1.4-alpine`), клиент pika 1.3.2. Документация: RabbitMQ — Reliability Guide, Consumer Acknowledgements and Publisher Confirms, Quorum Queues (`x-delivery-limit`, at-least-once dead-lettering, `x-overflow=reject-publish`), Definitions import, Access Control (configure/write/read); PostgreSQL `SELECT ... FOR UPDATE SKIP LOCKED`, `LISTEN/NOTIFY`; паттерн transactional outbox; W3C Trace Context (`traceparent`); Pillow (`Image.draft`, `ImageOps.exif_transpose`, параметры JPEG/WebP); `setrlimit(2)`. При написании кода и тестов использовался ИИ-ассистент (Claude); весь код проверен и прогнан мной, объяснить и изменить его могу на защите. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверял на Windows, все команды в PowerShell. Python нужен только для тестов (стандартная библиотека; Pillow на хосте не нужен — изображения для тестов генерирует и проверяет пробник внутри контейнера worker). Перед запуском остановить стенд lab4 (`docker compose down` в его папке), иначе занят порт 8080. Имя проекта Compose — `messenger-lab5`, поэтому volumes отдельные от lab4. + +```powershell +git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab5.git +cd mtusi2026lab5 +git checkout solution/bvt2303-babichenko +$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают +Copy-Item .env.example .env +python scripts/init_secrets.py # пароли PostgreSQL/Redis, ключи S3, пользователи RabbitMQ и + # secrets/rabbitmq_definitions.json (папка secrets/ в .gitignore) +docker compose --env-file .env config --quiet +docker compose up -d --build --wait # postgres, s3, rabbitmq -> migrate, s3-init -> api x2, janitor, + # dispatcher, worker -> lb +docker compose ps -a # migrate и s3-init: Exited (0), остальные healthy + +python student-tests/collect_evidence.py # все проверки lab1-5 по очереди (~15 мин), вывод в evidence/, + # таблицы замеров — в этот отчёт +``` + +Отдельные проверки (make на Windows нет, ниже то же самое без make): + +| Команда из README | То же в PowerShell | +| --- | --- | +| `make check` | `python scripts/check_contract.py` | +| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` | +| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` | +| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` | +| все сценарии lab5 | `python student-tests/test_jobs.py` | +| один сценарий lab5 | `python student-tests/test_jobs.py worker-down` (список — в начале файла) | +| состояние pipeline | `docker compose exec dispatcher python ops.py status` | +| реестр failed / журнал job | `docker compose exec dispatcher python ops.py failed`, `... ops.py events ` | +| replay failed / разбор DLQ | `docker compose exec dispatcher python ops.py replay `, `... ops.py dlq`, `... ops.py dlq-replay` | +| остановить / запустить worker | `scripts\worker.cmd stop`, `scripts\worker.cmd start` (Linux/macOS: `scripts/worker.sh`) | + +После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть: `docker compose up -d --wait`. + +Bootstrap: миграции — одноразовый сервис `migrate` (теперь и `005_jobs.sql`), bucket — `s3-init`, топология RabbitMQ (exchange, очереди, DLQ, пользователи и права) — декларативный файл `secrets/rabbitmq_definitions.json`, который брокер импортирует при каждом старте (`infra/rabbitmq.conf`). Всё идемпотентно: повторный `docker compose run --rm migrate` / `s3-init` и рестарт `rabbitmq` ничего не ломают. Сервисы не имеют права `configure` в брокере и только проверяют топологию пассивным declare — создать или удалить очередь они не могут. + +Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata`, `s3data`, `rabbitmqdata`, `faults` остаются). Безопасная остановка worker: `docker compose stop worker` — SIGTERM, потоки доделывают текущее задание (не дольше `JOB_TIMEOUT_SECONDS`, `stop_grace_period: 70s`), неподтверждённое сообщение брокер доставит снова. Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py` (новые пароли не подходят к старым volumes, поэтому удаляются вместе). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент --HTTP--> lb (HAProxy) --> api x2 --+--> postgres: метаданные, attachment_jobs, outbox, job_events, + | варианты (ИСТОЧНИК ИСТИНЫ, volume pgdata) + +--> s3 staging/: полный проверенный оригинал до 202 + +--> redis: сессии + COMMIT(queued + job + outbox) --NOTIFY--> dispatcher --publish+confirm--> rabbitmq + (outbox, (quorum queue attachments.process.v1, + аренды, DLQ attachments.process.v1.dlq, + сверка) volume rabbitmqdata) + | + worker x2 потока <-+ prefetch 1, ручной ack + | claim (аренда + fencing-токен) в postgres + | staging -> imaging.py (дочерний процесс, лимиты) + | PUT attachments//v1/{original,thumbnail.jpg,crop.jpg,optimized.webp} + | COMMIT ready (+варианты) -> ack -> удалить staging +janitor: брошенные uploading, staging ready/failed по retention, сироты в bucket, старый outbox +``` + +| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` | нет | edge; `127.0.0.1:8080`, `127.0.0.1:8404` | не нужны | +| API ×2 | `api` | python:3.11.13-slim | `/health/ready` (PostgreSQL + Redis; брокер и S3 не входят) | нет | edge + data | `database_url`, `cache_url`, ключ S3 API. **Доступа к брокеру нет** | +| Брокер | `rabbitmq` | rabbitmq:4.1.4-alpine, `hostname: rabbitmq` | узел запущен и очередь `attachments.process.v1` загружена | `rabbitmqdata` | только data, 5672 не публикуется | `rabbitmq_definitions` (хеши паролей, права) | +| Dispatcher | `dispatcher` (образ приложения) | python:3.11.13-slim | файл-пульс `/tmp/alive` не старше 30 с | `faults` | data | `database_url`, `broker_url_dispatcher` (write только в exchange `attachments`, read только DLQ) | +| Worker | `worker` (образ приложения), 2 потока | python:3.11.13-slim | файл-пульс | `faults`; `/tmp` — tmpfs 256 MiB | data | `database_url`, `broker_url_worker` (read только основной очереди, write нет), ключ S3 API | +| Janitor | `janitor` (образ приложения) | python:3.11.13-slim | файл-пульс `/tmp/alive` (обновляется перед проходом и каждые 10 с между проходами) | нет | data | как в lab4: `database_url`, ключ S3 API | +| S3, s3-init, migrate, postgres, redis | как в lab4 | | | | | | + +Почему RabbitMQ: durable quorum-очередь (Raft-журнал на диске, fsync до подтверждения издателю), publisher confirms, ручной ack, `x-delivery-limit` и dead-lettering «at-least-once» — ровно те границы, о которых спрашивают на защите. Почему dispatcher, а не publish из API: API фиксирует вложение, задание и строку outbox **одной транзакцией** и сразу отвечает 202; брокер в этом пути не участвует, поэтому его недоступность не мешает приёму, а API не нужны его пароли. + +Что перенесено из lab4 без изменений по смыслу: пользователи, сессии в Redis, чаты, сообщения, идемпотентность, балансировка, приём и потоковая проверка файла (лимиты bytes/пикселей до распаковки, MIME по содержимому), приватный bucket и ограниченный ключ, janitor. + +Что изменено в lab5: +- Новые файлы: `pipeline.py` (формат сообщения, outbox, переходы состояний, replay), `broker.py` (RabbitMQ, publisher confirms), `dispatcher.py`, `worker.py`, `imaging.py` (варианты, отдельный процесс), `faults.py` (учебное внедрение сбоев), `ops.py` (операторские команды), `migrations/005_jobs.sql`, `infra/rabbitmq.conf`. +- `attachments.py` / `db.py`: исходник пишется в `staging/`, затем одной транзакцией `state=queued` + `attachment_jobs` + `outbox` + `NOTIFY`; ответ `202` (переход 201 → 202 из COURSE.md). GET отдаёт `queued/processing/ready/failed`, варианты текущей версии pipeline. Сообщение можно создать с queued/processing-вложением, с failed — 409. +- `main.py`: `variant=original|thumbnail|crop|optimized`; для не-ready — 409, отсутствующий вариант ready — 404. +- `janitor.py`: очистка staging после безопасной фиксации и по retention для failed; сироты под `attachments//...`; очистка опубликованного outbox. +- `compose.yaml`: сервисы `rabbitmq`, `dispatcher`, `worker`, volumes `rabbitmqdata`, `faults`, три новых секрета; `scripts/init_secrets.py` генерирует пользователей RabbitMQ и definitions; restart-hook перезапускает ещё `rabbitmq`, `dispatcher`, `worker`. +- `.env.example`: в skeleton стояли `SESSION_STORE=postgres`, `S3_ENDPOINT=http://s3:9000` и `BROKER_URL_FILE=/run/secrets/broker_url`, с которыми API этого решения не стартует (сессии в Redis с lab2, S3 — SeaweedFS на 8333, у API нет брокера). Значения приведены к реальной конфигурации, добавлены переменные lab5. +- Тесты lab4: хелпер `upload()` ждёт `202`, затем опрашивает GET до `ready` — проверки «файл скачивается побайтно» сохранены; `test_consistency` имитирует падение в `enqueue_upload` вместо `finish_upload`. + +### Задание, сообщение и состояния + +Сообщение брокера — ровно `contracts/jobs.schema.json`: `schema_version`, `job_id`, `type=attachment.process.v1`, `attachment_id`, `pipeline_version=1`, `attempt`, `created_at`, `traceparent`. Ни байтов, ни URL, ни ключей S3, ни имени файла: worker берёт `source_key`, размер и SHA-256 из PostgreSQL. `traceparent` дублируется в native header AMQP; `message_id = :`; `delivery_mode=2` (persistent). Worker строго проверяет сообщение по схеме (лишние поля, типы, форматы) — не прошедшее уходит в DLQ. + +Таблицы (`005_jobs.sql`): `attachment_jobs` (одна строка на `(attachment_id, pipeline_version)`, `job_id` не меняется при повторах; `attempt`, `max_attempts`, `available_at`, `lease_token`/`lease_owner`/`lease_until`, `error_code`, `last_error`, `replays`), `attachment_variants` с **первичным ключом `(attachment_id, pipeline_version, name)`** — это и есть ключ эффекта из контракта, второй набор вариантов физически не вставится, `outbox`, `job_events` (журнал: created, claimed, retry_scheduled, lease_expired, duplicate_ignored, stale_result_rejected, ready, failed, replayed, staging_expired). CHECK-ограничения: `ready ⇔ object_key`, `failed ⇔ error_code`, `processing ⇔ lease`, до ready исходник обязан быть в staging. + +| Переход | Где | Что устойчиво сохранено | +| --- | --- | --- | +| нет → queued | API: PUT в staging, затем одна транзакция | исходник в `staging/` (SHA-256 сверяет S3 при PUT), строка вложения, job, outbox | +| queued → processing | worker `claim` под `FOR UPDATE` | attempt, новый `lease_token`, `lease_until = now()+15 с`, событие claimed | +| processing → queued | временная ошибка / истёкшая аренда | причина (код), `available_at = now()+backoff`, новая строка outbox | +| processing → ready | worker после PUT всех объектов | варианты + оригинал + ready одним COMMIT, **только если `lease_token` ещё наш** | +| processing → failed | постоянная ошибка или попытки исчерпаны | `error_code`, запись в журнале, исходник хранится 24 ч для replay | +| failed → queued | `ops.py replay` | событие replayed (audit), новая попытка того же job_id через outbox | + +Повторная доставка ready/failed — no-op с ack (событие `duplicate_ignored`). Устаревшее сообщение прежней попытки (job ещё ждёт backoff) — ack без работы: своё сообщение придёт из outbox. Ready никогда не откатывается: `finish_ready` обновляет только `state='processing'` и не перезаписывает более новую `pipeline_version`. + +### Где граница «202» + +К моменту ответа 202 сохранено: полный проверенный оригинал в S3 (staging — приватный prefix того же bucket, persistent volume `s3data`; это честная граница: оригинал уже в S3, worker «публикует» его копией в конечный prefix `attachments//v1/original`), строка вложения `queued`, задание и сообщение в outbox — всё в одной COMMIT PostgreSQL (`synchronous_commit=on`). Брокер на этот момент может ещё ничего не знать — это нормально: строку outbox опубликует dispatcher. Перезапуск/падение API после ответа не теряет работу. + +### Transactional outbox и окна отказа + +Dispatcher просыпается по `NOTIFY outbox` (сразу после COMMIT в API) или раз в 1 с, берёт строки `published_at IS NULL AND available_at <= now()` под `FOR UPDATE SKIP LOCKED`, публикует в канал с publisher confirms (`mandatory=True`: непривязанное сообщение — ошибка, nack при переполнении очереди — ошибка) и только после confirm ставит `published_at`. + +| Окно | Что происходит | Проверка | +| --- | --- | --- | +| после staging, до записи задания | клиент получил 503/обрыв, 202 не было; строка `uploading` невидима, janitor удалит объект и строку после grace | `test_consistency.py crash` (lab4, теперь с `enqueue_upload`) | +| после COMMIT, до publish | строка outbox не отмечена, после рестарта dispatcher публикует её | `test_jobs.py outbox-crash` (`dispatcher_crash_before_publish`) | +| после publish, до confirm / после confirm, до отметки | сообщение опубликовано, а отметки нет → публикуется ещё раз → дубль, worker видит ready и делает ack | `test_jobs.py outbox-crash` (`dispatcher_crash_after_confirm`) | +| после записи S3, до COMMIT результата | ack не отправлен, брокер доставляет снова; живая аренда → no-op; аренда истекает → новая попытка пишет **те же ключи** | `test_jobs.py crash-after-put` | +| после COMMIT результата, до ack | повторная доставка видит ready → ack, второго набора нет; staging доочищает janitor | `test_jobs.py crash-after-commit`, `staging` | + +Дополнительная страховка: раз в 30 с dispatcher ищет `queued`-задания старше `JOB_RECONCILE_SECONDS=120` без сообщения в полёте (например, сообщение ушло в DLQ после `x-delivery-limit`, пока PostgreSQL был недоступен) и публикует заново. Дубль безопасен, потеря — нет. + +**Publisher confirm и consumer ack — разные границы:** confirm значит «брокер принял сообщение и записал в quorum-журнал» (граница dispatcher → broker, после неё можно отметить outbox); ack значит «worker устойчиво выполнил эффект — объекты в S3 и COMMIT в PostgreSQL» (граница broker → worker, после неё брокер удаляет сообщение). At-least-once + идемпотентный эффект — это не exactly-once доставка: дубли сообщений происходят (и видны в журнале как `duplicate_ignored`), но не дают второго эффекта. + +### Повторы, аренда, DLQ, replay + +- Попытки: `JOB_MAX_ATTEMPTS=5`. Backoff «equal jitter»: `потолок = min(60, 2·2^(n-1))`, задержка = половина потолка + случайная половина: 1–2 с, 2–4 с, 4–8 с, 8–16 с. Задержка реализована через `available_at` строки outbox, а не через sleep в worker: поток не занят ожиданием. +- Постоянные ошибки (сразу failed, без повторов): `invalid_image`, `image_too_large`, `resource_limit` (не хватило памяти под RLIMIT_AS), `source_missing`, `source_corrupted` (staging не совпал по SHA-256 с принятым). Временные (повтор): `storage_unavailable`, `processing_timeout`, `processing_failed`/`processing_killed`, ошибки PostgreSQL. После последней попытки — `failed/retry_exhausted`. Наружу в `error_code` уходит только финальный код, без деталей. +- Аренда: `JOB_LEASE_SECONDS=15`, отдельный поток продлевает её каждые 5 с, пока идёт работа. **Долго работающий worker** продлевает аренду (проверено: зависание imaging 60 с — ни одного `lease_expired`), его ограничивает жёсткий `JOB_TIMEOUT_SECONDS=60`. **Зависший/умерший** перестаёт продлевать → через 15 с dispatcher возвращает job в queued (попытка засчитана) или worker сам перехватывает её при повторной доставке. Если «воскресший» исполнитель всё же дойдёт до COMMIT, fencing-проверка `lease_token` отклонит его результат (`stale_result_rejected`) — два неконтролируемых дубля не фиксируются. Его PUT в S3 перезаписал бы те же ключи теми же байтами (Pillow детерминирован), поэтому опасности для опубликованных объектов нет. +- RabbitMQ `consumer_timeout = 300 с`: канал, который не прислал ack дольше, закрывается брокером — последняя страховка от вечного зависания. +- DLQ `attachments.process.v1.dlq` (classic durable) получает: сообщения не по схеме, неизвестный `job_id`, превышение `x-delivery-limit=20` (бесконечный nack при недоступной БД). Реестр failed — `attachment_jobs` со `status='failed'` + `job_events` (`ops.py failed`, `ops.py events`). +- Replay (`ops.py replay `): только для `failed` с исходником в staging; та же job получает новую попытку и ещё 5 попыток, событие `replayed`; ready/queued/processing не трогаются («повторить» ≠ исполнить дубль успешной job). `ops.py dlq-replay`: для валидных сообщений DLQ решает по строке job в PostgreSQL (failed → replay, ready → удалить как no-op, queued/processing → удалить, работу ведёт outbox); невалидные и неизвестные остаются в DLQ. + +### Варианты изображения (pipeline_version = 1) + +| Вариант | Формат | Правило | Параметры | +| --- | --- | --- | --- | +| thumbnail | JPEG | вписан в 256×256, без увеличения, размеры вниз (минимум 1 px); 320×200 → 256×160 | quality 85, `optimize=True`, LANCZOS | +| crop | JPEG | центрированный квадрат 128×128 (маленькие увеличиваются) | quality 85, `optimize=True` | +| optimized | WebP | вписан в 1280×1280 без увеличения; прозрачность сохраняется | quality 80, method 4 | + +Во всех вариантах применяется EXIF orientation (1–8, как `ImageOps.exif_transpose`), альфа для JPEG компонуется на белый, EXIF/XMP/комментарии **не записываются** — в `save()` передаётся только ICC-профиль (цвет). Оригинал хранится побайтно как загружен (его EXIF не трогается — это требование контракта). Размер варианта считается от размеров исходника после поворота, а не от промежуточной копии. Большой JPEG декодируется сразу в уменьшенном масштабе (`draft`), сначала уменьшение до ≤1280, потом поворот. Не обещаю, что optimized меньше маленького исходника. + +### Лимиты worker + +- Bytes: API не принимает больше 10 MiB (413); worker скачивает staging не дальше `MAX_UPLOAD_BYTES` и сверяет размер и SHA-256 с метаданными. +- Пиксели: API проверяет по заголовку до распаковки (413); worker повторно — `Image.MAX_IMAGE_PIXELS` и явная проверка `w·h ≤ 40 000 000` (защита в глубину, `image_too_large`). +- Память: обработка идёт в дочернем процессе `imaging.py`, который **до импорта Pillow** ставит себе `RLIMIT_AS = WORKER_MEMORY_LIMIT_MB` (512 MiB), `RLIMIT_CPU`, `RLIMIT_FSIZE = 64 MiB`. Не хватило памяти → `MemoryError` → `resource_limit`, worker и его соединение с брокером живы. Контейнер worker: 2 CPU / 1536 MiB ≥ 2 потока × 512 MiB + сам worker + tmpfs. +- Время: worker убивает дочерний процесс по `JOB_TIMEOUT_SECONDS=60` (`processing_timeout`, временная ошибка). +- Конкуренция: `WORKER_CONCURRENCY=2` потока, у каждого своё соединение и `prefetch_count=1` → одновременно не больше 2 заданий на контейнер, остальное ждёт в очереди; очередь ограничена `x-max-length=100000` с `reject-publish` (переполнение — nack издателю, строка outbox ждёт, не потеря). + +### Очистка staging + +Worker удаляет `staging/` сразу после COMMIT ready и ack. Если не успел (упал, S3 недоступен) — janitor удаляет исходник ready-вложения, обновлённого больше 60 с назад. Исходник failed хранится `STAGING_RETENTION_SECONDS=86400` (для replay), потом удаляется с событием `staging_expired`. queued/processing janitor не трогает никогда. Объекты под `attachments//` удаляются, только если вложения нет, оно failed или ключ не входит в опубликованный набор ready. + +### Соответствие `.env.example` + +| Переменная | Что делает в этом решении | +| --- | --- | +| APP_*, SESSION_*, DATABASE_URL_FILE, CACHE_URL_FILE, API_REPLICAS, S3_*, MAX_UPLOAD_BYTES, MAX_IMAGE_PIXELS, пулы и таймауты | как в lab4 | +| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) | +| BROKER_URL_FILE | пусто для api/janitor (брокера у них нет); compose задаёт свой файл dispatcher (`broker_url_dispatcher`) и worker (`broker_url_worker`) | +| WORKER_CONCURRENCY, WORKER_PREFETCH | 2 потока-потребителя, prefetch 1 на поток | +| JOB_MAX_ATTEMPTS, JOB_TIMEOUT_SECONDS, JOB_LEASE_SECONDS | 5 попыток, 60 с на обработку изображения, аренда 15 с | +| JOB_BACKOFF_BASE_SECONDS, JOB_BACKOFF_MAX_SECONDS | 2 с и 60 с (equal jitter) | +| WORKER_MEMORY_LIMIT_MB | RLIMIT_AS процесса обработки, 512 MiB | +| STAGING_RETENTION_SECONDS | сколько хранится исходник failed-вложения для replay, 86400 с | +| PIPELINE_VERSION | 1 (версионирование нескольких pipeline — трек 5, не заявлен) | +| FAULT_INJECTION | учебные точки сбоев, на стенде включены по умолчанию (срабатывают только по файлу в volume `faults`, доступном лишь worker/dispatcher); `FAULT_INJECTION=0` — выключить | ## Доказательства критериев +`python student-tests/collect_evidence.py` создаёт `evidence/*.txt` (вывод тестов как есть, с датой, commit и кодом выхода) и `evidence/summary.md`. Сценарии lab5 — `student-tests/test_jobs.py`, сбои внедряются через `faults.py` (файл в volume `faults` → процесс в нужной точке делает `os._exit(137)`, как SIGKILL/OOM, или временную ошибку). Каждый сценарий проверяет инвариант «ровно 3 строки вариантов и ровно 4 объекта под `attachments//v1/`, SHA-256 совпадают». + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python tests/smoke.py` | PASS: публичный сценарий lab5 через балансировщик (202 → ready); размеры вариантов (thumbnail 256×160 для 320×200) — в jobs-variants.txt | smoke.txt | +| Контракт и публичные тесты не изменены | `python scripts/check_contract.py`; `git diff` по `contracts/`, `tests/` | PASS; не менялись | check.txt | +| 3: минимум lab4 сохранён (201 → 202) | `test_attachments.py`, `test_api.py`, `test_race.py`, `test_idempotency.py`, `test_balancing.py`, публичные persistence/failover | все PASS; upload ждёт 202 и затем ready | attachments.txt, student-tests.txt, race.txt, idempotency.txt, balancing.txt, persistence.txt, failover.txt | +| 3: 202 только после устойчивого staging и задания; остановка API после ответа не теряет работу | `test_jobs.py worker-down` | staging-объект есть до 202; job и outbox в БД; после рестарта api и rabbitmq тот же job_id → ready | jobs-recovery.txt | +| 3: отдельные broker и worker, durable, at-least-once, ack после эффекта, API не делает resize | `check_stack.py`, `test_jobs.py worker-down`, `broker-down` | rabbitmq/worker/dispatcher — отдельные сервисы; при остановленном worker вложение остаётся queued (API не обрабатывает); без брокера 202 и /health/ready 200 | stack.txt, jobs-recovery.txt | +| 3: queued/processing/ready/failed в API; оригинал неизменен; thumbnail по контракту | `test_jobs.py variants`, smoke | оригинал побайтно; 320×200 → 256×160; content 409 для queued и failed | jobs-variants.txt | +| 3: ограниченный retry, failed с безопасным error_code, повтор не создаёт второго набора | `test_jobs.py retry`, `exhausted-replay`, все сценарии сбоев | 5 попыток → `retry_exhausted`; в каждом сценарии ровно один набор | jobs-retry.txt, jobs-crash.txt | +| 3: демонстрация worker off → 202 → сообщение → рестарт API → worker → ready; брокер с volume не теряет задание | `test_jobs.py worker-down` | сообщение с вложением 201, другие сообщения работают, после `restart rabbitmq` в очереди 1 сообщение, затем ready и thumbnail | jobs-recovery.txt | +| 4: crop 128×128 и optimized WebP, параметры описаны | `test_jobs.py variants` | 6 исходников (PNG/JPEG, EXIF orientation 6, alpha, 40×30, 3000×2000, 5000×2): размеры по правилам, EXIF/XMP в вариантах 0, alpha на белом | jobs-variants.txt | +| 4: лимиты по bytes, пикселям, времени и памяти worker | `test_jobs.py limits` | 10 MiB+ → 413; 40 Мпикс PNG — пик RSS дочернего процесса 227 MiB при RLIMIT 512 MiB; лимит пикселей в worker — exit 4; «бомба памяти» → `resource_limit`; зависание → убито через 60 с, затем ready | jobs-limits.txt | +| 4: outbox, crash между COMMIT и publish, восстановление без ручного задания | `test_jobs.py outbox-crash` | dispatcher убит до publish → после рестарта тот же job опубликован и ready (attempt 1); убит после confirm → дубль, `duplicate_ignored`, один набор | jobs-outbox.txt | +| 4: backoff с jitter; постоянные/временные ошибки; DLQ/реестр failed; безопасный replay | `test_jobs.py retry exhausted-replay permanent dlq` | задержки в диапазонах 1–2 и 2–4 с; `invalid_image` — failed с attempt 1; replay → ready; replay ready — отказ без изменений; невалидное сообщение и неизвестный job → DLQ, dlq-replay их не исполняет | jobs-retry.txt | +| 4: остановка worker после записи объекта до ack; повтор завершает тот же результат | `test_jobs.py crash-after-put`, `crash-after-commit` | после PUT: аренда истекла, attempt 2 перезаписал те же ключи, 4 объекта; после COMMIT: redelivery → no-op ack, очередь пуста | jobs-crash.txt | +| 4: timeout/lease для зависшего processing | `test_jobs.py lease`, `limits` | зависший не продлевает аренду → через ~15 с job возвращена и выполнена другим потоком, поздний результат отклонён fencing; живой долгий — аренда продлевается | jobs-crash.txt, jobs-limits.txt | +| 4: ограниченная конкуренция/prefetch | `test_jobs.py concurrency` | 2 consumer, `prefetch_count=1`, ручной ack; из 8 накопленных заданий в processing одновременно ровно 2 | jobs-concurrency.txt | +| 4: очистка staging после безопасной фиксации | `test_jobs.py worker-down`, `staging` | staging удалён worker'ом после COMMIT; после crash до очистки — janitor | jobs-recovery.txt, jobs-crash.txt | +| Выбранный трек 5 | — | не выбран | — | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +**Где проверено сейчас.** Все строки таблицы выше подтверждены на Docker-стенде из этого репозитория (Windows 10, Docker Desktop, Compose 2.35, образы из `compose.yaml`, 2 реплики API за балансировщиком): 24.10.2026 один полный прогон `python student-tests/collect_evidence.py` — все 26 шагов PASS, сводка в `evidence/summary.md`, вывод каждого шага в `evidence/*.txt`, таблицы замеров ниже взяты из того же прогона. Перед ним на том же стенде отдельно прошли `docker compose up -d --build --wait` (migrate и s3-init — Exited (0), остальные healthy), `python tests/smoke.py` и `python student-tests/test_jobs.py worker-down`. Ранний прогон процессами без Docker (PostgreSQL 16, RabbitMQ 3.12, SeaweedFS 3.80, Redis) сохранён в `evidence/native-sandbox/` для сравнения. + +Что исправлено по итогам первого запуска в Docker (без Docker эти места не проявлялись): путь к definitions в `infra/rabbitmq.conf` (Compose монтирует секрет как `/run/secrets/rabbitmq_definitions`, без `.json`, — брокер не стартовал); у janitor появился healthcheck по файлу-пульсу (с `healthcheck: disable: true` команда `up --wait` завершалась ошибкой); в тестах — пробник `test_jobs.py` запускается одноразовым контейнером worker, когда сам worker остановлен, `upload_memory.py` ждёт 202 вместо 201 из lab4, а сценарий `staging` проверяет результат очистки независимо от того, какой проход janitor её сделал. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотезы: +1. Принятая (202) работа не теряется ни при каком одиночном отказе процесса (API, dispatcher, worker, broker) в любом из окон контракта, а повторная доставка не создаёт второго эффекта. +2. Память worker ограничена независимо от входа: худший допустимый вход (PNG 8000×5000) помещается в RLIMIT_AS 512 MiB, а вход сверх лимита ломает только свою попытку, не процесс worker. + +Методика: каждый сценарий `test_jobs.py` взводит одну точку сбоя (`faults.py`), загружает синтетический файл через балансировщик, ждёт конечного состояния через публичный GET и проверяет журнал `job_events`, строки outbox, глубину очередей (`rabbitmqctl list_queues`) и объекты в bucket (пробник в контейнере worker). Пик памяти обработки — `ru_maxrss` дочернего процесса `imaging.py` (`getrusage(RUSAGE_CHILDREN)`). + + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-24, commit 276dbda + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 5aa14958c864 | 2 | 123 | 0 | 4.406 | +| 2 | b9ad6d33d190 | 2 | 125 | 0 | 4.188 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.45 / 1.56 | 5aa14958c864: 5 | 96.0 | 109.7 | 13.7 | 149.6 | +| | | | | b9ad6d33d190: 5 | 94.5 | 108.9 | 14.4 | 142.1 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.04 / 1.31 | 5aa14958c864: 5 | 96.6 | 97.6 | 1.0 | 89.6 | +| | | | | b9ad6d33d190: 5 | 96.2 | 98.6 | 2.3 | 88.9 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 4.96 / 5.2 | 5aa14958c864: 10 | 94.7 | 114.9 | 20.3 | 155.4 | +| | | | | b9ad6d33d190: 10 | 97.0 | 121.8 | 24.8 | 176.9 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 243.885 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.406, 4.188 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 58.5 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19922 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14860 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | + + +Результаты прогона на нативном стенде (`evidence/native-sandbox/`, 1 API-реплика без балансировщика): + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) | - | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается: падение/рестарт любого одного процесса (api, dispatcher, worker, rabbitmq, postgres с volume) в любом окне отказа из контракта; недоступность брокера при приёме; зависание и нехватка памяти в обработке изображения; временные ошибки S3. +- Не выдерживается: потеря volume `pgdata` (источник истины) или `s3data` — нужен бэкап; один узел RabbitMQ (quorum-очередь из одного члена — durability есть, отказоустойчивости узла нет); при недоступном PostgreSQL новые загрузки и обработка останавливаются (это осознанно: без источника истины нельзя ни принять, ни зафиксировать). +- Учебные упрощения: worker использует тот же ограниченный S3-ключ, что и API (права только на свой bucket; отдельный ключ worker — следующий шаг); точки внедрения сбоев включены на стенде (`FAULT_INJECTION=1`), в эксплуатации выключаются; replay — CLI в контейнере dispatcher, а не API с авторизацией оператора. +- Подтверждено измерением: память обработки ограничена, конкуренция равна `WORKER_CONCURRENCY`, отказы не дают потерь и дублей эффекта. Гипотеза без измерения: пропускная способность pipeline под перегрузкой и выбор concurrency/prefetch — это трек 5.1, не заявлен. + +## Ответы на вопросы защиты (кратко) + +- **Publisher confirm vs consumer ack.** Confirm — брокер принял и сохранил сообщение (после него dispatcher отмечает outbox). Ack — worker устойчиво выполнил эффект (объекты в S3 + COMMIT ready); до ack сообщение остаётся у брокера и будет доставлено снова. +- **Что сохранено в момент 202.** Полный проверенный оригинал в S3-prefix `staging/`, строка вложения `queued`, задание `attachment_jobs` и сообщение в `outbox` — одной транзакцией PostgreSQL. Брокер может ещё ничего не знать. +- **Почему повторная доставка нормальна и где ключ идемпотентности.** At-least-once: ack может потеряться после эффекта, dispatcher может опубликовать дважды. Ключ — `job_id` (одна логическая работа) и ключ эффекта `(attachment_id, pipeline_version, variant)`: первичный ключ `attachment_variants` и детерминированные ключи S3 `attachments//v/`; состояние job в PostgreSQL решает, работать или подтвердить как no-op. +- **Зависший vs долго работающий.** Живой продлевает аренду каждые 5 с из отдельного потока, его ограничивает только `JOB_TIMEOUT_SECONDS`. Зависший/умерший перестаёт продлевать — через 15 с job перехватывают; если он «оживёт», fencing по `lease_token` не даст зафиксировать результат. diff --git a/attachments.py b/attachments.py new file mode 100644 index 0000000..dabbde2 --- /dev/null +++ b/attachments.py @@ -0,0 +1,80 @@ +"""Приём вложения (lab5): staging в S3 + задание в PostgreSQL до ответа 202. + +Распределённой транзакции между S3 и PostgreSQL нет, порядок записи выбран так, чтобы у каждого +объекта в bucket была строка-указатель в БД, а у каждого 202 — устойчивый исходник и намерение: + + 1. INSERT attachments (state=uploading, source_key) и COMMIT — упало: в S3 ничего нет, клиенту 503. + 2. PUT исходника в staging/ — упало: удаляем объект (PUT мог пройти) + и строку, клиенту 503. Если и это не вышло — + uploading-строку уберёт janitor после grace. + 3. ОДНА транзакция: state=queued + attachment_jobs + outbox + NOTIFY, COMMIT + — упало/неизвестно: клиенту 503, строка + остаётся uploading -> janitor удалит объект + и строку. Если COMMIT всё же прошёл, задание + есть и будет обработано (повтор клиента даст + второе вложение — в контракте нет ключа). + 4. 202 Attachment(status=queued) — только после шага 3. + +Брокер в этом пути не участвует: сообщение из outbox публикует dispatcher. Поэтому недоступный +брокер или упавший worker не мешают приёму, а API никогда не делает resize синхронно. +""" +import logging +from uuid import UUID, uuid4 + +import db +import settings +import storage +from errors import unavailable +from uploads import ImageInfo, Upload + +log = logging.getLogger("messenger.attachments") + + +def store(chat_id: UUID, owner_id: UUID, filename: str, upload: Upload, image: ImageInfo) -> dict: + attachment_id = uuid4() + key = storage.new_key(uuid4().hex) + db.create_upload_intent(attachment_id, chat_id, owner_id, filename, image.content_type, + upload.size, upload.sha256, image.width, image.height, key) + try: + storage.put(key, upload.file, upload.size, image.content_type, upload.sha256) + except storage.StorageUnavailable: + compensate(attachment_id, key) + raise unavailable("File storage is temporarily unavailable") from None + row = db.enqueue_upload(attachment_id, "api@" + settings.INSTANCE_ID) + if row is None: + # janitor уже признал загрузку брошенной и удалил объект (возможно только при зависании дольше grace). + log.warning("Upload %s was reclaimed before it was queued", attachment_id) + raise unavailable() + log.info("Attachment %s accepted into staging and queued", attachment_id) + return {**row, "variants": []} + + +def compensate(attachment_id: UUID, key: str) -> None: + """Откат после неудачного PUT. Любая ошибка здесь не страшна: остаток уберёт janitor.""" + try: + storage.delete(key, quick=True) + db.delete_upload_intent(attachment_id) + log.info("Upload %s rolled back after S3 failure", attachment_id) + except Exception as exc: # noqa: BLE001 — откат best-effort, исходную ошибку отдаём клиенту + log.warning("Rollback of upload %s postponed to janitor: %s", attachment_id, type(exc).__name__) + + +def public(row: dict) -> dict: + """Строка БД -> Attachment из контракта (без ключей S3, размеров в пикселях и служебных полей).""" + return { + "id": row["id"], + "chat_id": row["chat_id"], + "owner_id": row["owner_id"], + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["state"], + "variants": [ + {key: v[key] for key in ("name", "content_type", "size_bytes", "width", "height", "sha256")} + for v in row.get("variants", []) + ], + "error_code": row["error_code"] if row["state"] == "failed" else None, + "created_at": row["created_at"], + "updated_at": row["updated_at"], + } diff --git a/auth.py b/auth.py new file mode 100644 index 0000000..c21212f --- /dev/null +++ b/auth.py @@ -0,0 +1,139 @@ +import base64 +import binascii +import hashlib +import logging +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from redis.backoff import NoBackoff +from redis.retry import Retry + +import settings +from errors import BASIC_CHALLENGE, unauthorized, unavailable + +log = logging.getLogger("messenger.auth") + +hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1) + +DUMMY_HASH = hasher.hash("there-is-no-such-user") + +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def password_matches(password_hash: str | None, password: str) -> bool: + try: + hasher.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """'Basic base64(username:password)' -> (username, password) или None. + + Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617). + """ + scheme, _, value = (header or "").strip().partition(" ") + if scheme.lower() != "basic" or not value or len(value) > 1024: + return None + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + username, colon, password = decoded.partition(":") + if not colon or not username or not username.isascii(): + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + scheme, _, value = (header or "").strip().partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value: + return None + return value + + + +KEY_PREFIX = "session:" + +_redis = redis.Redis( + connection_pool=redis.BlockingConnectionPool.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + max_connections=settings.REDIS_POOL_MAX, + timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) +) + + +@dataclass(frozen=True) +class Session: + user_id: UUID + key: str + + +def session_key(token: str) -> str: + return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: UUID) -> tuple[str, datetime]: + """Новая сессия: случайный токен и абсолютное время истечения.""" + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS) + try: + _redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000)) + except redis.RedisError as exc: + log.warning("Redis is unavailable on login: %s", type(exc).__name__) + raise unavailable() from None + return token, expires_at + + +def check_bearer(header: str | None) -> Session: + token = parse_bearer(header) + if token is None: + raise unauthorized("Bearer token is required") + key = session_key(token) + try: + user_id = _redis.get(key) + except redis.RedisError as exc: + log.warning("Redis is unavailable: %s", type(exc).__name__) + raise unavailable() from None + if user_id is None: + raise unauthorized("Session is invalid or expired") + return Session(UUID(user_id), key) + + +def delete_session(session: Session) -> None: + try: + _redis.delete(session.key) + except redis.RedisError: + raise unavailable() from None + + +def ping_redis() -> None: + try: + _redis.ping() + except redis.RedisError: + raise unavailable() from None + + +def login_failed(): + return unauthorized("Invalid username or password", BASIC_CHALLENGE) + + +def close() -> None: + _redis.close() + _redis.connection_pool.disconnect() diff --git a/broker.py b/broker.py new file mode 100644 index 0000000..504ce4b --- /dev/null +++ b/broker.py @@ -0,0 +1,105 @@ +"""RabbitMQ: подключение, проверка топологии, публикация с publisher confirms. + +Топология объявлена декларативно в definitions (secrets/rabbitmq_definitions.json, генерирует +scripts/init_secrets.py) и загружается брокером при каждом старте. У сервисов нет права configure: +они только проверяют её пассивным declare и не могут создать/удалить очередь. + + exchange attachments (direct, durable) + --attachment.process.v1--> queue attachments.process.v1 (quorum, durable) + x-delivery-limit=20, dead-letter -> attachments.dlx (стратегия at-least-once), + x-max-length=100000 + x-overflow=reject-publish (переполнение -> nack издателю, не потеря) + exchange attachments.dlx (direct, durable) + --attachment.process.v1.dlq--> queue attachments.process.v1.dlq (classic, durable: просмотр в ops.py + через basic_get + nack не расходует delivery-limit) + +Сообщения persistent (delivery_mode=2). Quorum queue пишет их в журнал Raft на диск (fsync) до +подтверждения издателю; при рестарте брокера с volume сообщения и неподтверждённые доставки сохраняются. +""" +import json +import logging + +import pika +from pika.exceptions import AMQPError + +import pipeline +import settings + +log = logging.getLogger("messenger.broker") +logging.getLogger("pika").setLevel(logging.WARNING) # pika пишет INFO на каждое соединение/канал + +BrokerError = (AMQPError, OSError) + + +def parameters(url: str | None = None) -> pika.URLParameters: + params = pika.URLParameters(url or settings.BROKER_URL) + params.heartbeat = settings.BROKER_HEARTBEAT_SECONDS + params.blocked_connection_timeout = 30 # брокер в memory/disk alarm блокирует издателя: не ждать вечно + params.socket_timeout = 5 + params.connection_attempts = 1 + params.client_properties = {"connection_name": settings.INSTANCE_ID} + return params + + +def connect(url: str | None = None) -> pika.BlockingConnection: + return pika.BlockingConnection(parameters(url)) + + +def check_topology(channel) -> dict: + """Пассивная проверка: exchange и очереди существуют. Возвращает число сообщений в очередях.""" + channel.exchange_declare(pipeline.EXCHANGE, passive=True) + main = channel.queue_declare(pipeline.QUEUE, passive=True).method + dlq = channel.queue_declare(pipeline.DLQ, passive=True).method + return {"queue": main.message_count, "consumers": main.consumer_count, "dlq": dlq.message_count} + + +def properties(message: dict) -> pika.BasicProperties: + headers = {} + if "traceparent" in message: + headers["traceparent"] = message["traceparent"] # тот же carrier и в native headers + return pika.BasicProperties( + delivery_mode=pika.DeliveryMode.Persistent, + content_type="application/json", + message_id=f"{message['job_id']}:{message['attempt']}", + type=message["type"], + headers=headers or None, + ) + + +class Publisher: + """Канал в режиме confirm: basic_publish возвращает управление только после ack брокера. + + mandatory=True: если сообщение некуда положить (нет привязки), брокер вернёт его -> UnroutableError. + Nack (например, очередь переполнена и x-overflow=reject-publish) -> NackError. В обоих случаях + строка outbox остаётся неопубликованной и будет отправлена позже. + """ + + def __init__(self): + self.connection = None + self.channel = None + + def open(self): + if self.channel is None or self.channel.is_closed or self.connection.is_closed: + self.close() + self.connection = connect() + self.channel = self.connection.channel() + self.channel.confirm_delivery() + check_topology(self.channel) + log.info("connected to broker, publisher confirms enabled") + return self.channel + + def publish(self, message: dict, exchange: str = pipeline.EXCHANGE, routing_key: str = pipeline.ROUTING_KEY): + body = json.dumps(message, separators=(",", ":")).encode("utf-8") + self.open().basic_publish(exchange, routing_key, body, properties(message), mandatory=True) + + def pump(self): + if self.connection is not None and self.connection.is_open: + self.connection.process_data_events(time_limit=0) # heartbeat в паузах + + def close(self): + try: + if self.connection is not None and self.connection.is_open: + self.connection.close() + except BrokerError: + pass + self.connection = None + self.channel = None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..056873b --- /dev/null +++ b/compose.yaml @@ -0,0 +1,320 @@ +name: messenger-lab5 + +x-app-build: &app-build + context: . + args: + APP_VERSION: ${APP_VERSION:-dev} + +# Общие настройки фоновой обработки (worker и dispatcher); значения по умолчанию — из settings.py. +x-pipeline-env: &pipeline-env + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-1} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + JOB_LEASE_SECONDS: ${JOB_LEASE_SECONDS:-15} + JOB_BACKOFF_BASE_SECONDS: ${JOB_BACKOFF_BASE_SECONDS:-2} + JOB_BACKOFF_MAX_SECONDS: ${JOB_BACKOFF_MAX_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + # Учебный стенд: точки внедрения сбоев включены (срабатывают только по файлу в volume faults, + # доступном лишь worker/dispatcher). В эксплуатации FAULT_INJECTION=0. + FAULT_INJECTION: ${FAULT_INJECTION:-1} + FAULTS_DIR: /faults + +x-alive-check: &alive-check + # Фоновые процессы без HTTP: жив, если обновлял файл не позже 30 с назад. + test: ["CMD", "python", "-c", "import os, sys, time; sys.exit(time.time() - os.stat('/tmp/alive').st_mtime > 30)"] + interval: 10s + timeout: 3s + retries: 3 + start_period: 20s + start_interval: 1s + +x-s3-api-secrets: &s3-api-secrets + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + migrate: + build: *app-build + command: ["python", "migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + # S3-совместимое хранилище SeaweedFS (master + volume + filer + S3 в одном процессе): байты вложений. + # Только во внутренней сети data, порты не публикуются. Ключи и их права — в секрете s3_config + # (генерирует scripts/init_secrets.py): admin-ключ для bootstrap и ключ API с правами только на свой bucket. + # Анонимного доступа нет: identity "anonymous" в конфиге не объявлена. + s3: + image: chrislusf/seaweedfs:3.80 + entrypoint: ["/usr/bin/weed"] + command: ["server", "-dir=/data", "-ip=127.0.0.1", "-ip.bind=0.0.0.0", + "-master.volumeSizeLimitMB=1024", "-volume.max=0", + "-s3", "-s3.port=8333", "-s3.config=/run/secrets/s3_config"] + secrets: [s3_config] + volumes: ["s3data:/data"] + networks: [data] + healthcheck: + # S3-порт отвечает по HTTP (анонимный запрос получает 403 — это и есть «жив и закрыт») + test: ["CMD-SHELL", "wget -q -S -O /dev/null http://127.0.0.1:8333/ 2>&1 | grep -q 'HTTP/'"] + interval: 5s + timeout: 3s + retries: 24 + start_period: 15s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Одноразовый идемпотентный bootstrap (s3_init.py, образ приложения): приватный bucket + проверки. + s3-init: + build: *app-build + command: ["python", "s3_init.py"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_user + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: [s3_root_user, s3_root_password, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + s3: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + api: + build: *app-build + env_file: .env + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + BROKER_URL_FILE: "" # у API нет доступа к брокеру: задание уходит через outbox в PostgreSQL + <<: *s3-api-secrets + secrets: [database_url, cache_url, s3_access_key_id, s3_secret_access_key] + networks: [edge, data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "1.00", memory: 512M} + + # Очистка брошенных загрузок и сверка БД <-> S3 (janitor.py). Тот же образ и тот же ограниченный S3-ключ. + janitor: + build: *app-build + command: ["python", "janitor.py"] + env_file: .env + environment: + CACHE_URL_FILE: "" # janitor не ходит в Redis + BROKER_URL_FILE: "" # и в брокер + <<: *s3-api-secrets + secrets: [database_url, s3_access_key_id, s3_secret_access_key] + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 192M} + + # Брокер RabbitMQ: durable quorum-очередь с журналом на volume. Топология/пользователи/права — + # в секрете rabbitmq_definitions (scripts/init_secrets.py), импорт при каждом старте. + # hostname фиксирован: имя узла rabbit@rabbitmq определяет каталог данных в volume. + rabbitmq: + image: rabbitmq:4.1.4-alpine + hostname: rabbitmq + volumes: + - ./infra/rabbitmq.conf:/etc/rabbitmq/conf.d/90-messenger.conf:ro + - rabbitmqdata:/var/lib/rabbitmq + secrets: [rabbitmq_definitions] + networks: [data] + healthcheck: + # Узел запущен, и vhost с очередями загружен (а не только «порт открыт»). + test: ["CMD-SHELL", "rabbitmq-diagnostics -q check_running && rabbitmqctl -q list_queues -p messenger name | grep -q '^attachments.process.v1$$'"] + interval: 5s + timeout: 10s + retries: 24 + start_period: 20s + start_interval: 2s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + # Dispatcher: публикует transactional outbox в RabbitMQ (publisher confirms), восстанавливает + # просроченные аренды processing и сверяет queued без сообщения. У API доступа к брокеру нет. + dispatcher: + build: *app-build + command: ["python", "dispatcher.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_dispatcher + <<: *pipeline-env + secrets: [database_url, broker_url_dispatcher] + volumes: ["faults:/faults"] + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + healthcheck: *alive-check + restart: unless-stopped + stop_grace_period: 10s + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + + # Worker: потребитель очереди, отдельный процесс и сервис. Обработка изображения — в дочернем + # процессе imaging.py с RLIMIT_AS=WORKER_MEMORY_LIMIT_MB, RLIMIT_CPU и таймаутом JOB_TIMEOUT_SECONDS. + # Лимит контейнера >= WORKER_CONCURRENCY x WORKER_MEMORY_LIMIT_MB + сам worker. + worker: + build: *app-build + command: ["python", "worker.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + BROKER_URL_FILE: /run/secrets/broker_url_worker + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + WORKER_MEMORY_LIMIT_MB: ${WORKER_MEMORY_LIMIT_MB:-512} + <<: [*pipeline-env, *s3-api-secrets] + secrets: [database_url, broker_url_worker, s3_access_key_id, s3_secret_access_key] + volumes: ["faults:/faults"] + tmpfs: + - /tmp:size=256m,mode=1777 + networks: [data] + depends_on: + migrate: {condition: service_completed_successfully} + s3-init: {condition: service_completed_successfully} + rabbitmq: {condition: service_healthy} + healthcheck: *alive-check + restart: unless-stopped + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят; + # если не успели — SIGKILL, ack не отправлен, брокер доставит задание снова. + stop_grace_period: 70s + deploy: + resources: + limits: {cpus: "2.00", memory: 1536M} + + lb: + image: haproxy:3.2.25-alpine + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + postgres: + image: postgres:17.11-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] + secrets: [postgres_password] + volumes: ["pgdata:/var/lib/postgresql/data"] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + redis: + image: redis:7.4.11-alpine + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + - redisdata:/data + secrets: [redis_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + +networks: + edge: {} + data: + internal: true + +volumes: + pgdata: {} + redisdata: {} + s3data: {} + rabbitmqdata: {} + faults: {} + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + redis_auth: {file: ./secrets/redis_auth} + s3_root_user: {file: ./secrets/s3_root_user} + s3_root_password: {file: ./secrets/s3_root_password} + s3_access_key_id: {file: ./secrets/s3_access_key_id} + s3_secret_access_key: {file: ./secrets/s3_secret_access_key} + s3_config: {file: ./secrets/s3_config.json} + broker_url_dispatcher: {file: ./secrets/broker_url_dispatcher} + broker_url_worker: {file: ./secrets/broker_url_worker} + rabbitmq_definitions: {file: ./secrets/rabbitmq_definitions.json} diff --git a/db.py b/db.py new file mode 100644 index 0000000..1530665 --- /dev/null +++ b/db.py @@ -0,0 +1,391 @@ +import logging +import time +from contextlib import contextmanager +from datetime import datetime +from uuid import UUID, uuid4 + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +import pipeline +import settings +from errors import bad_request, conflict, not_found, unavailable + +log = logging.getLogger("messenger.db") + + +pool = ConnectionPool( + conninfo=settings.DATABASE_URL, + min_size=settings.DB_POOL_MIN, + max_size=settings.DB_POOL_MAX, + timeout=settings.DB_POOL_WAIT_SECONDS, + kwargs={ + "connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS, + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" + " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", + }, + check=ConnectionPool.check_connection, + open=False, +) + + +@contextmanager +def connection(wait_seconds: float | None = None): + try: + with pool.connection(timeout=wait_seconds) as conn: + yield conn + except (psycopg.OperationalError, PoolTimeout) as exc: + log.warning("PostgreSQL is unavailable: %s", type(exc).__name__) + raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text", "filename") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} + + +def ping() -> None: + with connection(wait_seconds=1) as conn: + conn.execute("SELECT 1") + + +MIGRATIONS_LOCK = 20260202 + + +def migration_files() -> list: + return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) + + +def run_migrations() -> list[str]: + files = migration_files() + applied = [] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now())""") + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + applied.append(path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) + return applied + + +def wait_and_migrate(max_wait_seconds: int = 60) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + applied = run_migrations() + log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") + return + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + log.warning("PostgreSQL is not ready yet, retry in 1 s") + time.sleep(1) + + +def missing_migrations() -> list[str]: + expected = [path.name for path in migration_files()] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + try: + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + except pg_errors.UndefinedTable: + return expected + return [name for name in expected if name not in done] + + +def wait_for_schema(max_wait_seconds: int) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + missing = missing_migrations() + except psycopg.OperationalError: + missing = None + if missing == []: + log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) + return + if time.monotonic() > deadline: + raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") + log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None + else "missing " + ", ".join(missing)) + time.sleep(1) + + + +USER_COLUMNS = "id, username, display_name, created_at" + + +def create_user(username: str, display_name: str, password_hash: str) -> dict: + try: + with connection() as conn: + return from_db(conn.execute( + f"""INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) + except pg_errors.UniqueViolation: + raise conflict("Username already exists") from None + + +def get_user(user_id: UUID) -> dict | None: + with connection() as conn: + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) + + +def get_user_with_password(username: str) -> dict | None: + with connection() as conn: + return from_db(conn.execute( + f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) + ).fetchone()) + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" + + +def create_chat(title: str, member_ids: list[UUID]) -> dict: + chat_id = uuid4() + members = sorted(member_ids) + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, to_db(title)), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in members], + ) + except pg_errors.ForeignKeyViolation: + raise bad_request("Unknown member") from None + return {**from_db(row), "member_ids": members} + + +def get_chat(chat_id: UUID, user_id: UUID) -> dict: + with connection() as conn: + chat = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s""", + (user_id, chat_id), + ).fetchone() + if chat is None: + raise not_found("Chat not found") + return from_db(chat) + + +def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" + where, params = "", [user_id] + if after is not None: + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params += list(after) + with connection() as conn: + rows = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + {where} + ORDER BY c.created_at, c.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + + +MESSAGE_COLUMNS = """m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids""" + + +def is_member(conn, chat_id: UUID, user_id: UUID) -> bool: + return conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() is not None + + +def check_member(chat_id: UUID, user_id: UUID) -> None: + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + + +def add_message(chat_id: UUID, sender_id: UUID, text: str, attachment_ids: list[UUID], + idempotency_key: str, request_hash: bytes) -> dict: + try: + with connection() as conn: + with conn.transaction(): + if not is_member(conn, chat_id, sender_id): # проверяется и при повторе: чужой чат — 404 + raise not_found("Chat not found") + if attachment_ids: + # Только принятые вложения ЭТОГО чата (lab5: queued/processing/ready можно, failed — 409). + # FOR KEY SHARE: строку не удалят до конца транзакции, а worker может менять её состояние. + found = conn.execute( + """SELECT id, state FROM attachments + WHERE id = ANY(%s) AND chat_id = %s AND state <> 'uploading' FOR KEY SHARE""", + (attachment_ids, chat_id), + ).fetchall() + if len(found) != len(attachment_ids): + raise not_found("Attachment not found") + failed = any(row["state"] == "failed" for row in found) + else: + failed = False + saved_query = f"""SELECT {MESSAGE_COLUMNS}, m.request_hash FROM messages m + WHERE m.chat_id = %s AND m.sender_id = %s AND m.idempotency_key = %s""" + if failed: + # Повтор уже созданного сообщения возвращает прежний 201, даже если вложение + # с тех пор стало failed. Новое сообщение с failed-вложением — 409. + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if saved is None: + raise conflict("Failed attachment cannot be attached") + created = None + else: + created = conn.execute( + """INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING + RETURNING id, chat_id, sender_id, text, created_at""", + (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), + ).fetchone() + if created is None: + saved = conn.execute(saved_query, (chat_id, sender_id, idempotency_key)).fetchone() + if created is not None: + if attachment_ids: + with conn.cursor() as cur: + cur.executemany( + """INSERT INTO message_attachments (message_id, chat_id, attachment_id, position) + VALUES (%s, %s, %s, %s)""", + [(created["id"], chat_id, aid, n) for n, aid in enumerate(attachment_ids)], + ) + return {**from_db(created), "attachment_ids": list(attachment_ids)} + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен + raise unavailable() + if bytes(saved.pop("request_hash")) != request_hash: + raise conflict("Idempotency-Key was already used with a different request") + return from_db(saved) + + +def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """История чата после курсора (keyset-пагинация), до limit+1 строк.""" + where, params = "m.chat_id = %s", [chat_id] + if after is not None: + where += " AND (m.created_at, m.id) > (%s, %s)" + params += list(after) + with connection() as conn: + if not is_member(conn, chat_id, user_id): + raise not_found("Chat not found") + rows = conn.execute( + f"""SELECT {MESSAGE_COLUMNS} FROM messages m + WHERE {where} + ORDER BY m.created_at, m.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + +# ---------- вложения ---------- + +ATTACHMENT_COLUMNS = ("id, chat_id, owner_id, filename, content_type, size_bytes, sha256, state, error_code, " + "pipeline_version, created_at, updated_at, object_key") +VISIBLE_STATES = ("queued", "processing", "ready", "failed") + + +def create_upload_intent(attachment_id: UUID, chat_id: UUID, owner_id: UUID, filename: str, content_type: str, + size_bytes: int, sha256: str, width: int, height: int, source_key: str) -> None: + """Шаг 1 из 3: строка-намерение (state=uploading) фиксируется ДО записи в staging. + + Поэтому у любого объекта, который API кладёт в bucket, уже есть строка в БД: janitor найдёт + и удалит его, если загрузка не дошла до queued (обрыв, падение реплики, отказ S3 или БД). + """ + try: + with connection() as conn: + conn.execute( + """INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, source_key, state) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading')""", + (attachment_id, chat_id, owner_id, to_db(filename), content_type, size_bytes, sha256, + width, height, source_key), + ) + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + + +def enqueue_upload(attachment_id: UUID, actor: str) -> dict | None: + """Шаг 3 из 3: исходник уже в staging -> queued + задание + outbox ОДНОЙ транзакцией. + + После этого COMMIT принятая работа не может потеряться: исходник в S3, намерение в PostgreSQL, + сообщение для брокера в outbox. None, если строку успел убрать janitor (зависание дольше grace). + """ + with connection() as conn: + with conn.transaction(): + row = conn.execute( + f"""UPDATE attachments SET state = 'queued', updated_at = now() + WHERE id = %s AND state = 'uploading' + RETURNING {ATTACHMENT_COLUMNS}""", + (attachment_id,), + ).fetchone() + if row is None: + return None + pipeline.enqueue_new(conn, attachment_id, actor) + return from_db(row) + + +def delete_upload_intent(attachment_id: UUID) -> None: + with connection(wait_seconds=1) as conn: + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (attachment_id,)) + + +def get_attachment(attachment_id: UUID, user_id: UUID) -> dict: + """Вложение (queued/processing/ready/failed), если user_id — участник его чата. Иначе 404. + + Возвращает строку и варианты текущей версии pipeline (они есть только у ready). + """ + with connection() as conn: + row = conn.execute( + f"""SELECT {", ".join("a." + c.strip() for c in ATTACHMENT_COLUMNS.split(","))} + FROM attachments a + JOIN chat_members me ON me.chat_id = a.chat_id AND me.user_id = %s + WHERE a.id = %s AND a.state = ANY(%s)""", + (user_id, attachment_id, list(VISIBLE_STATES)), + ).fetchone() + if row is None: + raise not_found("Attachment not found") + variants = [] + if row["state"] == "ready": + variants = conn.execute( + """SELECT name, content_type, size_bytes, width, height, sha256, object_key + FROM attachment_variants WHERE attachment_id = %s AND pipeline_version = %s + ORDER BY array_position(ARRAY['thumbnail', 'crop', 'optimized'], name)""", + (attachment_id, row["pipeline_version"]), + ).fetchall() + return {**from_db(row), "variants": variants} diff --git a/dispatcher.py b/dispatcher.py new file mode 100644 index 0000000..f231370 --- /dev/null +++ b/dispatcher.py @@ -0,0 +1,150 @@ +"""Dispatcher: публикует outbox в RabbitMQ и восстанавливает зависшие задания. + +Почему отдельный процесс, а не publish прямо в API: API фиксирует задание и строку outbox одной +транзакцией PostgreSQL и сразу отвечает 202. Между COMMIT и публикацией нет окна, в котором +намерение может потеряться: если брокер недоступен, dispatcher упал или процесс убит, строка outbox +остаётся неопубликованной и будет отправлена после восстановления. Поэтому у API нет ни адреса, +ни ключей брокера, и недоступность брокера не мешает приёму файлов. + +Цикл: +1. publish_due: строки outbox с published_at IS NULL и available_at <= now() (под FOR UPDATE SKIP + LOCKED) -> basic_publish с publisher confirm -> published_at = now() -> COMMIT. + Падение после confirm, но до COMMIT -> строка будет опубликована ещё раз: дубль сообщения, + который worker распознаёт по состоянию job (at-least-once, а не exactly-once). +2. Раз в DISPATCHER_POLL_SECONDS: processing с истёкшей арендой -> queued (+ outbox) или failed. +3. Раз в 30 с: сверка «queued без сообщения в полёте» (pipeline.reconcile_lost_messages). +Просыпается по NOTIFY outbox (сразу после COMMIT в API) или по таймеру. +""" +import logging +import signal +import threading +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +import broker +import faults +import pipeline +import settings + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.dispatcher") + +STOP = threading.Event() +ACTOR = "dispatcher@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def publish_due(conn, publisher: broker.Publisher) -> int: + published = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, job_id, attempt, payload FROM outbox + WHERE published_at IS NULL AND available_at <= now() + ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED""", + (settings.DISPATCHER_BATCH,), + ).fetchall() + if not rows: + return published + for row in rows: + faults.crash("dispatcher_crash_before_publish") + try: + publisher.publish(row["payload"]) + except broker.BrokerError as exc: + log.warning("publish of outbox %s failed: %s", row["id"], type(exc).__name__) + publisher.close() + raise + faults.crash("dispatcher_crash_after_confirm") + conn.execute("UPDATE outbox SET published_at = now() WHERE id = %s", (row["id"],)) + published += 1 + log.info("published %s message(s)", len(rows)) + if len(rows) < settings.DISPATCHER_BATCH: + return published + return published + + +def note_failure(conn) -> None: + """Счётчик неудачных публикаций для наблюдения (ops.py status).""" + try: + conn.execute("""UPDATE outbox SET publish_failures = publish_failures + 1 + WHERE published_at IS NULL AND available_at <= now()""") + except psycopg.Error: + pass + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def run() -> None: + publisher = broker.Publisher() + conn = None + last_leases = last_reconcile = 0.0 + pause = 1.0 + while not STOP.is_set(): + try: + if conn is None or conn.closed: + conn = connect_db() + conn.execute("LISTEN outbox") + log.info("connected to PostgreSQL, listening for outbox notifications") + touch_alive() + publish_due(conn, publisher) + now = time.monotonic() + if now - last_leases >= settings.DISPATCHER_POLL_SECONDS: + last_leases = now + if recovered := pipeline.recover_expired_leases(conn, ACTOR): + log.warning("recovered %s job(s) with expired lease", recovered) + if now - last_reconcile >= 30: + last_reconcile = now + if lost := pipeline.reconcile_lost_messages(conn, ACTOR, settings.JOB_RECONCILE_SECONDS): + log.warning("re-published %s queued job(s) without a message in flight", lost) + pause = 1.0 + publisher.pump() + # Ждём NOTIFY (COMMIT нового задания) не дольше интервала опроса. + for _ in conn.notifies(timeout=settings.DISPATCHER_POLL_SECONDS, stop_after=1): + pass + except broker.BrokerError: + if conn is not None and not conn.closed: + note_failure(conn) + log.warning("broker is unavailable, outbox is kept, retry in %.0f s", pause) + STOP.wait(pause) + pause = min(pause * 2, 15) + except psycopg.Error as exc: + log.warning("PostgreSQL error %s, reconnect in %.0f s", type(exc).__name__, pause) + try: + if conn is not None: + conn.close() + except psycopg.Error: + pass + conn = None + STOP.wait(pause) + pause = min(pause * 2, 15) + publisher.close() + if conn is not None: + conn.close() + log.info("stopped") + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + log.info("dispatcher %s started, fault injection %s", settings.INSTANCE_ID, + "ON" if settings.FAULT_INJECTION else "off") + run() + + +if __name__ == "__main__": + main() diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..a641476 --- /dev/null +++ b/errors.py @@ -0,0 +1,40 @@ +# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт. +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions +BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "Service temporarily unavailable") -> ApiError: + return ApiError(503, "unavailable", message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, "unsupported_media_type", message) diff --git a/evidence/attachments.txt b/evidence/attachments.txt new file mode 100644 index 0000000..e70352d --- /dev/null +++ b/evidence/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-10-24 15:37:16, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_attachments.py +# exit code: 0, 10.2 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready f338bc0d +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/balancing.txt b/evidence/balancing.txt new file mode 100644 index 0000000..54f98f0 --- /dev/null +++ b/evidence/balancing.txt @@ -0,0 +1,18 @@ +# balancing: balancing, shared sessions/data/files, X-Instance-Id +# date: 2026-10-24 15:40:58, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_balancing.py +# exit code: 0, 2.1 s + +20 x GET /users/me with one token -> replicas: {'b9ad6d33d190': 10, '5aa14958c864': 10} +PASS: one session works on every replica, no second login (sessions are shared in Redis) +message written by 5aa14958c864; read back by ['5aa14958c864', 'b9ad6d33d190'] +PASS: a message written via one replica is read right away via another (shared PostgreSQL) +logout on b9ad6d33d190; old token then rejected by ['5aa14958c864', 'b9ad6d33d190'] +PASS: logout on one replica revokes the session on all of them +first POST on b9ad6d33d190 -> 201 af45ed44-ef90-4258-b144-77628e56bfed; same key on 5aa14958c864 -> 201 af45ed44-ef90-4258-b144-77628e56bfed +PASS: Idempotency-Key is shared: a retry through another replica returns the same message +file uploaded via b9ad6d33d190; downloaded byte for byte via ['5aa14958c864', 'b9ad6d33d190'] +PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local) +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['5aa14958c864', 'b9ad6d33d190']; api containers: ['5aa14958c864', 'b9ad6d33d190'] +PASS: X-Instance-Id is in every response and points to a real container +6 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 0000000..36d085e --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-10-24 15:30:17, commit: 276dbda + uncommitted changes +# command: python.exe scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/consistency.txt b/evidence/consistency.txt new file mode 100644 index 0000000..4d95004 --- /dev/null +++ b/evidence/consistency.txt @@ -0,0 +1,47 @@ +# consistency: lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage +# date: 2026-10-24 15:40:02, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_consistency.py +# exit code: 0, 156.4 s + +declared 9 000 000 bytes, sent 211817, closed the connection +declared 9 000 000 bytes, sent 3111817, closed the connection +rows in attachments: 219 -> 219; objects in bucket: 860 -> 860 +temporary upload files in api containers: {'b9ad6d33d190': 0, '5aa14958c864': 0} (removed when the request ends) +api containers have no volumes: the original is never stored locally +PASS: an interrupted upload leaves no row, no object and no local file +crash after S3 PUT: row 3a083fe9 state=uploading, object staging/5bda5e392029444d915a233aef8e7af9 exists in S3 +the half-written attachment is invisible: GET -> 404, attach to a message -> 404 +janitor with default grace (3600 s) right away: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 865, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.31} -> fresh row and object are kept (the upload might still be running) +row made 2 h old; janitor --dry-run: {"dry_run": true, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 865, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.301} -> nothing deleted +janitor: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 864, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.29} -> row and object deleted +janitor again: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 864, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.278} -> idempotent +the ready attachment of the same chat is untouched +PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period +object staging/bea3e5320fc84973a5b2b762fe96ddd8 put into the bucket without a row in PostgreSQL +janitor --grace-seconds 3 immediately: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.292} -> fresh object kept +after 4 s, --dry-run: {"dry_run": true, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 1, "orphan_objects_deleted": 0, "missing_objects": 1, "seconds": 0.298} -> reported, kept +janitor: {"dry_run": false, "grace_seconds": 3, "stale_uploads": 0, "stale_uploads_deleted": 0, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 869, "orphan_objects": 1, "orphan_objects_deleted": 1, "missing_objects": 1, "seconds": 0.295} -> orphan deleted +ready attachments older than the grace period are untouched (they have rows) +object of a ready attachment deleted behind the API's back: janitor reports missing_objects=2 and keeps the row; download -> 503 (not 500) +PASS: orphans are found and deleted only after the grace period; missing objects are reported +--- S3 stopped (connection refused / name not resolved): docker compose stop s3 +upload -> 503 unavailable in 19922 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose start s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 879, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.29} +PASS: S3 stopped (connection refused / name not resolved) -> 503 for file operations only, no visible half-uploads, everything cleaned up +--- S3 frozen (no answer, timeouts): docker compose pause s3 +upload -> 503 unavailable in 14860 ms, no storage details in the body +download of an existing file -> 503 +metadata, messages (also with an already stored attachment) and /health/ready keep working: S3 is needed only for file bytes +uploading rows of this chat: before 0, after the failed upload 1 (kept as a pointer for janitor if the rollback could not reach S3); not visible through the API +docker compose unpause s3 +after S3 recovery the old file downloads byte for byte +new upload -> 201 +janitor after grace period: {"dry_run": false, "grace_seconds": 3600, "stale_uploads": 1, "stale_uploads_deleted": 1, "staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 5, "outbox_rows_deleted": 0, "objects_scanned": 887, "orphan_objects": 0, "orphan_objects_deleted": 0, "missing_objects": 2, "seconds": 0.284} +PASS: S3 frozen (no answer, timeouts) -> 503 for file operations only, no visible half-uploads, everything cleaned up +5 passed, 0 failed diff --git a/evidence/db.txt b/evidence/db.txt new file mode 100644 index 0000000..f45a622 --- /dev/null +++ b/evidence/db.txt @@ -0,0 +1,21 @@ +# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations +# date: 2026-10-24 15:41:16, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_db.py +# exit code: 0, 10.7 s + +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 185 -> 186 +PASS: chat row was inserted and then rolled back together with members +direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" +PASS: the database itself rejects a message from a non-member +stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97) +PASS: password is stored as Argon2id hash +unique constraint on users: UNIQUE (username) +PASS: username is unique in the database +applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; running the migrate service once more +after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql,004_attachments.sql,005_jobs.sql; log: Migrations applied: none (schema is up to date) +PASS: migrations are applied once, a repeated run does nothing +api replicas that checked the schema at start: 28; api log lines 'Migrations applied': 0; migrate service: Exited (0) 4 minutes ago +PASS: only the migrate service applies migrations, replicas wait for the schema +unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) +PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database +7 passed, 0 failed diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 0000000..cf19e0d --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,99 @@ +=== Data volume + messages_total | messages_in_big_chat +----------------+---------------------- + 29169 | 15000 + + +=== History index + indexdef +---------------------------------------------------------------------------------------- + CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id) + + +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + QUERY PLAN +------------------------------------------------------------------------------------------------ + Limit (actual time=0.005..0.012 rows=51 loops=1) + Buffers: shared hit=14 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Index Cond: (chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) + Buffers: shared hit=14 + Planning: + Buffers: shared hit=6 + Planning Time: 0.058 ms + Execution Time: 0.020 ms + + +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.006..0.012 rows=51 loops=1) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.010 rows=51 loops=1) + Index Cond: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid))) + Buffers: shared hit=5 + Planning Time: 0.043 ms + Execution Time: 0.029 ms + + +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.020..0.155 rows=51 loops=1) + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.019..0.152 rows=51 loops=1) + Index Cond: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid))) + Buffers: shared hit=59 + SubPlan 1 + -> Sort (actual time=0.002..0.002 rows=0 loops=51) + Sort Key: a."position" + Sort Method: quicksort Memory: 25kB + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Recheck Cond: (message_id = m.id) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Index Cond: (message_id = m.id) + Buffers: shared hit=51 + Planning: + Buffers: shared hit=63 + Planning Time: 0.205 ms + Execution Time: 0.172 ms + + +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Limit (actual time=0.832..0.838 rows=51 loops=1) + Buffers: shared hit=266 + -> Index Scan using messages_history on messages (actual time=0.007..0.645 rows=10051 loops=1) + Index Cond: (chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) + Buffers: shared hit=266 + Planning Time: 0.041 ms + Execution Time: 0.849 ms + + +=== 4. Query 2 without the index (index is dropped only inside this transaction) + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual time=1.257..1.261 rows=51 loops=1) + Buffers: shared hit=450 + -> Sort (actual time=1.257..1.258 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 31kB + Buffers: shared hit=450 + -> Seq Scan on messages (actual time=0.530..0.937 rows=5000 loops=1) + Filter: ((chat_id = 'ee34a051-241a-4a88-a3cd-c6d8cc5d2152'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'b3d18836-9770-4887-8b0d-12b3e59970c1'::uuid))) + Rows Removed by Filter: 24169 + Buffers: shared hit=450 + Planning: + Buffers: shared hit=4 + Planning Time: 0.082 ms + Execution Time: 1.274 ms + + +=== ROLLBACK: test data removed, index is back + explain_users_left +-------------------- + 0 + diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 0000000..0a7a8a8 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,43 @@ +# explain: EXPLAIN of the history page (full plans: explain-history.txt) +# date: 2026-10-24 15:41:20, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/explain.py +# exit code: 0, 0.7 s + +=== Data volume +=== History index +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=14 + -> Index Scan using messages_history on messages (actual time=0.005..0.009 rows=51 loops=1) + Buffers: shared hit=14 + Buffers: shared hit=6 + Execution Time: 0.020 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + Buffers: shared hit=5 + -> Index Scan using messages_history on messages (actual time=0.005..0.010 rows=51 loops=1) + Buffers: shared hit=5 + Execution Time: 0.029 ms +=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message + Buffers: shared hit=59 + -> Index Scan using messages_history on messages m (actual time=0.019..0.152 rows=51 loops=1) + Buffers: shared hit=59 + Buffers: shared hit=54 + -> Bitmap Heap Scan on message_attachments a (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + -> Bitmap Index Scan on message_attachments_message_id_attachment_id_key (actual time=0.000..0.000 rows=0 loops=51) + Buffers: shared hit=51 + Buffers: shared hit=63 + Execution Time: 0.172 ms +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + Buffers: shared hit=266 + -> Index Scan using messages_history on messages (actual time=0.007..0.645 rows=10051 loops=1) + Buffers: shared hit=266 + Execution Time: 0.849 ms +=== 4. Query 2 without the index (index is dropped only inside this transaction) + Buffers: shared hit=450 + Buffers: shared hit=450 + -> Seq Scan on messages (actual time=0.530..0.937 rows=5000 loops=1) + Buffers: shared hit=450 + Buffers: shared hit=4 + Execution Time: 1.274 ms +=== ROLLBACK: test data removed, index is back +PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 0000000..79d541e --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,33 @@ +# failover: public failover test for every replica, file downloaded via the other one (make test-failover) +# date: 2026-10-24 15:42:00, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/run_failover.py +# exit code: 0, 40.6 s + +hook: scripts\stop-one.cmd; api replicas: ['5aa14958c864', 'b9ad6d33d190'] +=== run 1: api containers before +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Up 44 seconds (healthy) +api 5aa14958c864 Up 43 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 123, "failed_attempts": 0, "first_recovery_seconds": 4.406} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Up About a minute (healthy) +api 5aa14958c864 Exited (0) 11 seconds ago +the hook stopped 5aa14958c864 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 5aa14958c864 -> healthy and back in HAProxy +=== run 2: api containers before +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Up About a minute (healthy) +api 5aa14958c864 Up 2 seconds (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 125, "failed_attempts": 0, "first_recovery_seconds": 4.188} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api b9ad6d33d190 Exited (0) 11 seconds ago +api 5aa14958c864 Up 19 seconds (healthy) +the hook stopped b9ad6d33d190 (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start b9ad6d33d190 -> healthy and back in HAProxy +PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 0000000..5ffffe6 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,29 @@ +# failures: Redis and PostgreSQL outages via the balancer +# date: 2026-10-24 15:43:26, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_failures.py +# exit code: 0, 55.0 s + +stopping redis +/health/live -> 200, /health/ready -> 503 (8016 ms) +valid token: GET /users/me -> 503 unavailable (8015 ms), session cannot be checked +unknown token -> 503, not 200: nobody is trusted without a check +no token -> 401 +login -> 503: there is nowhere to store the session +registration -> 201: it needs only PostgreSQL +starting redis +after start: the old token works, confirmed messages are in place +PASS: Redis outage gives 503 and does not let anyone in without a session check +killing redis with SIGKILL (docker compose kill redis): no graceful shutdown +right after the crash: GET /users/me -> 503 +starting redis +after start: the session issued before the crash still works (AOF, appendfsync always) +PASS: sessions survive a Redis crash, not only a clean restart +stopping postgres +/health/live -> 200, /health/ready -> 503 (1016 ms) +valid token: users/me, chats, history, new message -> 503 unavailable, no details +no token -> 401, unknown token -> 401: anonymous users are not let in +login -> 503, registration -> 503 +starting postgres +after start: the old token works, confirmed messages are in place +PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out +3 passed, 0 failed diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 0000000..769aa85 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-10-24 15:41:01, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_idempotency.py +# exit code: 0, 2.5 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'5aa14958c864': 10, 'b9ad6d33d190': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 201): 10, ('body B', 409): 10}; saved: ['body A'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/jobs-concurrency.txt b/evidence/jobs-concurrency.txt new file mode 100644 index 0000000..1433d72 --- /dev/null +++ b/evidence/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-10-24 15:37:05, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py concurrency +# exit code: 0, 15.1 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) +1 passed, 0 failed diff --git a/evidence/jobs-crash.txt b/evidence/jobs-crash.txt new file mode 100644 index 0000000..81ab8d6 --- /dev/null +++ b/evidence/jobs-crash.txt @@ -0,0 +1,15 @@ +# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease +# date: 2026-10-24 15:35:40, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py crash-after-put crash-after-commit staging lease +# exit code: 0, 156.3 s + +--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат. +worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] +--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack. +worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) +--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit). +staging source still present after the crash (worker died before cleanup); waiting 60 s window +janitor removed the staging source of the ready attachment (this pass); source_key NULL, object gone +--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; +frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] +4 passed, 0 failed diff --git a/evidence/jobs-limits.txt b/evidence/jobs-limits.txt new file mode 100644 index 0000000..a97da4f --- /dev/null +++ b/evidence/jobs-limits.txt @@ -0,0 +1,11 @@ +# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) +# date: 2026-10-24 15:31:54, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py limits +# exit code: 0, 73.6 s + +--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker. +10 MiB + 8 bytes -> 413 before any job is created +imaging.py on 8000x5000 PNG: exit 0, peak RSS 227.2 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large) +memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived +imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s +1 passed, 0 failed diff --git a/evidence/jobs-outbox.txt b/evidence/jobs-outbox.txt new file mode 100644 index 0000000..d844522 --- /dev/null +++ b/evidence/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-10-24 15:33:04, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py outbox-crash +# exit code: 0, 15.1 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/jobs-recovery.txt b/evidence/jobs-recovery.txt new file mode 100644 index 0000000..bcb15b0 --- /dev/null +++ b/evidence/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-10-24 15:32:49, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 54.9 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment cabbd2a3-fa8c-4805-aa76-6d59e2b5d68a +message with attachment 201, content 409, other messages 201; job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job 17043324-e0c6-4c0c-88df-6ae2a8eca8fd +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/jobs-retry.txt b/evidence/jobs-retry.txt new file mode 100644 index 0000000..d3f0e7e --- /dev/null +++ b/evidence/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-10-24 15:36:50, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 69.7 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] +4 passed, 0 failed diff --git a/evidence/jobs-variants.txt b/evidence/jobs-variants.txt new file mode 100644 index 0000000..ab1edfb --- /dev/null +++ b/evidence/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-10-24 15:30:40, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_jobs.py variants +# exit code: 0, 19.5 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 0000000..080d981 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.047, + "concurrency": 4, + "successful_requests": 7328, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 243.885, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 0000000..e2747c8 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-10-24 15:42:31, commit: 276dbda + uncommitted changes +# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.3 s + +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.047, + "concurrency": 4, + "successful_requests": 7328, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 243.885, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 32.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/native-sandbox/attachments.txt b/evidence/native-sandbox/attachments.txt new file mode 100644 index 0000000..4168063 --- /dev/null +++ b/evidence/native-sandbox/attachments.txt @@ -0,0 +1,96 @@ +# attachments: lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids +# date: 2026-10-24 14:39:50, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_attachments.py +# exit code: 0, 10.5 s + +sample.png: 111817 bytes, sha256 99ea7533c441..., downloaded by the other member, same bytes +sample.jpg: 1492 bytes, sha256 c310b9a1c208..., downloaded by the other member, same bytes +progressive.jpg: 1081 bytes, sha256 913df01aec84..., downloaded by the other member, same bytes +interlaced.png: 79 bytes, sha256 38149222f78f..., downloaded by the other member, same bytes +1x1.png: 67 bytes, sha256 eaa4a94ea300..., downloaded by the other member, same bytes +PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte +exactly 10485760 bytes (10 MiB) -> 202, then ready +10485761 bytes (10 MiB + 1) with Content-Length -> 413 before reading the body +PASS: the 10 MiB limit is exact +12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed +PASS: oversized streamed body is cut off, the API keeps working +8000x5000 = 40 000 000 pixels (38957 bytes compressed) -> 202, ready 4fea67fd +8001x5000 = 40 005 000 pixels (38962 bytes) -> 413 +header claims 100000x100000 in 65 bytes -> 413 from the header, nothing is decoded +PASS: pixel limit is checked before decompression +PNG cut in half (55908 bytes, image/png) -> 415 +PNG without IEND (111805 bytes, image/png) -> 415 +PNG with a flipped byte in IDAT (CRC mismatch) (111817 bytes, image/png) -> 415 +PNG with garbage after IEND (111824 bytes, image/png) -> 415 +PNG signature only (8 bytes, image/png) -> 415 +PNG with less pixel data than the header says (68 bytes, image/png) -> 415 +JPEG cut in half (746 bytes, image/jpeg) -> 415 +JPEG header only (20 bytes, image/jpeg) -> 415 +random bytes (1600 bytes, image/png) -> 415 +plain text (28 bytes, image/jpeg) -> 415 +empty body -> 400 +PASS: damaged and foreign files are rejected with 415, never 500 +PLTE 771 bytes = 257 entries (review case) (850 bytes, valid CRCs) -> 415 +PLTE length not a multiple of 3 (83 bytes, valid CRCs) -> 415 +empty PLTE (79 bytes, valid CRCs) -> 415 +palette larger than bit depth allows (depth 1, 3 entries) (88 bytes, valid CRCs) -> 415 +palette image without PLTE (67 bytes, valid CRCs) -> 415 +PLTE in a grayscale image (85 bytes, valid CRCs) -> 415 +PLTE after IDAT (85 bytes, valid CRCs) -> 415 +two PLTE chunks (103 bytes, valid CRCs) -> 415 +tRNS longer than the palette (100 bytes, valid CRCs) -> 415 +tRNS before PLTE (98 bytes, valid CRCs) -> 415 +tRNS of wrong size for RGB (4 bytes instead of 6) (85 bytes, valid CRCs) -> 415 +tRNS in an image with alpha channel (82 bytes, valid CRCs) -> 415 +IEND with data (70 bytes, valid CRCs) -> 415 +unknown critical chunk (81 bytes, valid CRCs) -> 415 +IDAT chunks not consecutive (95 bytes, valid CRCs) -> 415 +gAMA after IDAT (85 bytes, valid CRCs) -> 415 +gAMA of 2 bytes instead of 4 (83 bytes, valid CRCs) -> 415 +3 MiB metadata chunk before IDAT (3145809 bytes, valid CRCs) -> 415 +private chunk after IDAT (181 bytes, valid CRCs) -> 415 +animated PNG frame (fdAT) (91 bytes, valid CRCs) -> 415 +tIME of 2 bytes instead of 7 (83 bytes, valid CRCs) -> 415 +tEXt without keyword separator (86 bytes, valid CRCs) -> 415 +zTXt with broken zlib after IDAT (91 bytes, valid CRCs) -> 415 +zTXt text bomb: 50 MB of text in ~50 KB after IDAT (48694 bytes, valid CRCs) -> 415 +palette 256 entries + tRNS + tEXt and tIME after IDAT -> 201, downloaded byte for byte +palette depth 2 with 4 entries -> 201, downloaded byte for byte +RGB with 6-byte tRNS and gAMA -> 201, downloaded byte for byte +compressed zTXt (100 KB of text) and iTXt after IDAT -> 201, downloaded byte for byte +PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted +PNG bytes sent as image/jpeg -> 415 +JPEG bytes sent as image/png -> 415 +GIF bytes sent as image/png -> 415 +PNG sent as image/gif -> 415 +PNG sent as text/html -> 415 +PNG sent as application/octet-stream -> 415 +PNG sent as multipart/form-data -> 415 +PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png +PASS: the type is taken from the content; a declared type that does not match is refused +13 dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400 +no filename -> 400 +Unicode, 255 chars, leading dots, %20 and ? are kept as display names only +PASS: filename never becomes a path; dangerous names are refused +outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400 +upload into someone else's / unknown chat -> 404, without token -> 401 +variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5) +PASS: only chat members can see an attachment +text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member +history returns attachment_ids for every message, [] for a message without files; reuse is allowed +attachment of another chat (author is a member of both) -> 404 +own + foreign attachment -> 404 +unknown attachment_id -> 404 +outsider referencing an attachment of the chat -> 404 +duplicate ids -> 400 +11 ids -> 400 +bad UUID -> 400 +empty text without attachments -> 400 +whitespace text without attachments -> 400 +PASS: only attachments of the same chat can be linked; rejected requests change nothing +same key + same attachments -> same 201; other attachments -> 409; missing field == [] +PASS: Idempotency-Key covers attachment_ids +sent 1.1 MB of a declared 5 MB body and closed the connection +the API keeps serving uploads; nothing of the aborted request is visible (DB/S3 side is checked by test_consistency.py) +PASS: an interrupted upload is dropped +12 passed, 0 failed diff --git a/evidence/native-sandbox/check.txt b/evidence/native-sandbox/check.txt new file mode 100644 index 0000000..e5baf60 --- /dev/null +++ b/evidence/native-sandbox/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-10-24 14:34:04, commit: 81d8f37 + uncommitted changes +# command: python3 scripts/check_contract.py +# exit code: 0, 0.0 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/native-sandbox/jobs-concurrency.txt b/evidence/native-sandbox/jobs-concurrency.txt new file mode 100644 index 0000000..0c61a2c --- /dev/null +++ b/evidence/native-sandbox/jobs-concurrency.txt @@ -0,0 +1,8 @@ +# jobs-concurrency: lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing +# date: 2026-10-24 14:39:39, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py concurrency +# exit code: 0, 7.5 s + +--- concurrency: Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2. +2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-crash.txt b/evidence/native-sandbox/jobs-crash.txt new file mode 100644 index 0000000..3a4fba0 --- /dev/null +++ b/evidence/native-sandbox/jobs-crash.txt @@ -0,0 +1,15 @@ +# jobs-crash: lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease +# date: 2026-10-24 14:38:43, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py crash-after-put crash-after-commit staging lease +# exit code: 0, 143.5 s + +--- crash-after-put: Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат. +worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] +--- crash-after-commit: Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack. +worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) +--- staging: Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit). +staging source still present after the crash (worker died before cleanup); waiting 60 s window +janitor removed the staging source of the ready attachment: 1 +--- lease: Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; +frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] +4 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-limits.txt b/evidence/native-sandbox/jobs-limits.txt new file mode 100644 index 0000000..3311283 --- /dev/null +++ b/evidence/native-sandbox/jobs-limits.txt @@ -0,0 +1,11 @@ +# jobs-limits: lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) +# date: 2026-10-24 14:35:26, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py limits +# exit code: 0, 69.1 s + +--- limits: Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker. +10 MiB + 8 bytes -> 413 before any job is created +imaging.py on 8000x5000 PNG: exit 0, peak RSS 223.4 MiB (RLIMIT_AS 512 MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large) +memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived +imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-outbox.txt b/evidence/native-sandbox/jobs-outbox.txt new file mode 100644 index 0000000..869700c --- /dev/null +++ b/evidence/native-sandbox/jobs-outbox.txt @@ -0,0 +1,9 @@ +# jobs-outbox: lab5: dispatcher crash between COMMIT and publish / after confirm +# date: 2026-10-24 14:36:19, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py outbox-crash +# exit code: 0, 7.9 s + +--- outbox-crash: Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки. +dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action +dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set +1 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-recovery.txt b/evidence/native-sandbox/jobs-recovery.txt new file mode 100644 index 0000000..d75c922 --- /dev/null +++ b/evidence/native-sandbox/jobs-recovery.txt @@ -0,0 +1,16 @@ +# jobs-recovery: lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits +# date: 2026-10-24 14:36:11, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py worker-down broker-down +# exit code: 0, 45.9 s + +--- worker-down: Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready. +worker stopped +upload while worker is down -> 202 queued, attachment 5b41ee9a-207b-46d6-a0af-635082295231 +message with attachment 201, content 409, other messages 201; job d62da8f8-b127-4141-95c3-8f691c39bb72 published, queue messages/unacked = (1, 0) +rabbitmq and api restarted with volumes; queue still holds 1 message(s); status queued +worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted +--- broker-down: Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления. +rabbitmq stopped +upload -> 202 and /health/ready 200 without broker; outbox row pending for job aab8166b-e551-4ec9-9566-0f8207527e36 +rabbitmq started -> dispatcher published the pending row -> ready +2 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-retry.txt b/evidence/native-sandbox/jobs-retry.txt new file mode 100644 index 0000000..abbbfdd --- /dev/null +++ b/evidence/native-sandbox/jobs-retry.txt @@ -0,0 +1,15 @@ +# jobs-retry: lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ +# date: 2026-10-24 14:39:32, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py retry exhausted-replay permanent dlq +# exit code: 0, 48.8 s + +--- retry: Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале. +2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready +--- exhausted-replay: Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ. +5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) +ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" +--- permanent: Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов. +invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error) +--- dlq: Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет. +schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] +4 passed, 0 failed diff --git a/evidence/native-sandbox/jobs-variants.txt b/evidence/native-sandbox/jobs-variants.txt new file mode 100644 index 0000000..6ddec84 --- /dev/null +++ b/evidence/native-sandbox/jobs-variants.txt @@ -0,0 +1,14 @@ +# jobs-variants: lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped +# date: 2026-10-24 14:34:16, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_jobs.py variants +# exit code: 0, 11.4 s + +--- variants: thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных. +thumb320: thumbnail 256x160, crop 128x128, optimized 320x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +exif6: thumbnail 192x256, crop 128x128, optimized 300x400; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +alpha: thumbnail 256x170, crop 128x128, optimized 300x200; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +tiny: thumbnail 40x30, crop 128x128, optimized 40x30; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +large: thumbnail 256x170, crop 128x128, optimized 1280x853; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +strip: thumbnail 256x1, crop 128x128, optimized 1280x1; original byte-identical; EXIF/XMP in variants: 0; S3 keys ['v1/crop.jpg', 'v1/optimized.webp', 'v1/original', 'v1/thumbnail.jpg'] +unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload +1 passed, 0 failed diff --git a/evidence/native-sandbox/persistence.txt b/evidence/native-sandbox/persistence.txt new file mode 100644 index 0000000..21a02fb --- /dev/null +++ b/evidence/native-sandbox/persistence.txt @@ -0,0 +1,16 @@ +# persistence (native sandbox): public tests/resilience.py --mode persistence with scripts/restart.sh +# date: 2026-10-24 14:40:30; run_persistence.py (docker inspect proof of restarts) needs real Docker + +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +# exit code: 0 + +# processes after the hook (new PIDs = real restarts): +postgres running 2971 +redis running 2964 +rabbitmq running 3077 +s3 running 2988 +api running 3504 +dispatcher running 3518 +worker running 3522 +janitor running 3508 diff --git a/evidence/native-sandbox/race.txt b/evidence/native-sandbox/race.txt new file mode 100644 index 0000000..ec304c7 --- /dev/null +++ b/evidence/native-sandbox/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-10-24 14:39:54, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_race.py +# exit code: 0, 1.8 s + +sent 100 messages from 10 clients in 0.50 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/native-sandbox/smoke.txt b/evidence/native-sandbox/smoke.txt new file mode 100644 index 0000000..429532c --- /dev/null +++ b/evidence/native-sandbox/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test), with attachment +# date: 2026-10-24 14:34:05, commit: 81d8f37 + uncommitted changes +# command: python3 tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 0.8 s + +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/native-sandbox/student-tests.txt b/evidence/native-sandbox/student-tests.txt new file mode 100644 index 0000000..fba8591 --- /dev/null +++ b/evidence/native-sandbox/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-10-24 14:39:52, commit: 81d8f37 + uncommitted changes +# command: python3 student-tests/test_api.py +# exit code: 0, 2.6 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/native-sandbox/summary.md b/evidence/native-sandbox/summary.md new file mode 100644 index 0000000..eeebfe1 --- /dev/null +++ b/evidence/native-sandbox/summary.md @@ -0,0 +1,40 @@ +# Сводка проверок + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/native-sandbox/check.txt | +| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/native-sandbox/smoke.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 11 | evidence/native-sandbox/jobs-variants.txt | +| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 69 | evidence/native-sandbox/jobs-limits.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 46 | evidence/native-sandbox/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 8 | evidence/native-sandbox/jobs-outbox.txt | +| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 143 | evidence/native-sandbox/jobs-crash.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 49 | evidence/native-sandbox/jobs-retry.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 7 | evidence/native-sandbox/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 11 | evidence/native-sandbox/attachments.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/native-sandbox/student-tests.txt | +| race | lab2 races via the balancer | PASS | 2 | evidence/native-sandbox/race.txt | +| persistence | public persistence test (make test-persistence) on the native stand; docker-inspect proof needs Docker | PASS | 40 | evidence/native-sandbox/persistence.txt | + +# Где прогнано + +Нативный стенд без Docker (в среде сборки нет доступа к Docker Hub): PostgreSQL 16, RabbitMQ 3.12.1, SeaweedFS 3.80, Redis — процессами с той же конфигурацией и секретами, что в compose.yaml; 1 реплика API на 8080 без HAProxy; Python 3.11. Команды `docker compose ...` тестов выполнялись через шим, переводящий их на эти процессы. Для защиты повторить на Docker: `python student-tests/collect_evidence.py` (вывод в evidence/). + +# Отказы lab5 + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job d62da8f8-b127-4141-95c3-8f691c39bb72 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 21f39e5f-676b-43a2-92ca-532e1a932fb0 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.40 s and 2.40 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id 43d09c9b-f72f-4918-91d1-3407fbb12848 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.9 s after worker start, peak processing 2 of 8 (16 samples) | - | diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 0000000..d339ffb --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,30 @@ +# persistence: public persistence test incl. S3 + proof of real restarts (make test-persistence) +# date: 2026-10-24 15:44:25, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/run_persistence.py +# exit code: 0, 59.6 s + +before: container, ID, started at + messenger-lab5-api-1 b9ad6d33d190 2026-10-24T12:41:59.057757842Z + messenger-lab5-api-2 5aa14958c864 2026-10-24T12:41:38.920683556Z + messenger-lab5-dispatcher-1 9c5531a487f2 2026-10-24T12:32:58.22608557Z + messenger-lab5-janitor-1 c412a9460051 2026-10-24T12:23:20.245209318Z + messenger-lab5-lb-1 0760689bc54d 2026-10-24T12:23:27.648064757Z + messenger-lab5-postgres-1 10fec15d4ea6 2026-10-24T12:43:25.148756098Z + messenger-lab5-rabbitmq-1 95b4214594a8 2026-10-24T12:32:33.167311219Z + messenger-lab5-redis-1 e7ed9903c342 2026-10-24T12:43:04.620093059Z + messenger-lab5-s3-1 56a0cdfb9bc8 2026-10-24T12:39:05.007533298Z + messenger-lab5-worker-1 6e040d05805f 2026-10-24T12:37:00.256852184Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after (hook + waiting for readiness took 58.5 s): container, ID, started at, restarted? + messenger-lab5-api-1 b9ad6d33d190 2026-10-24T12:44:20.339074469Z same container: True, restarted: True + messenger-lab5-api-2 5aa14958c864 2026-10-24T12:44:20.477467705Z same container: True, restarted: True + messenger-lab5-dispatcher-1 9c5531a487f2 2026-10-24T12:44:17.516290602Z same container: True, restarted: True + messenger-lab5-janitor-1 c412a9460051 2026-10-24T12:44:16.494020858Z same container: True, restarted: True + messenger-lab5-lb-1 0760689bc54d 2026-10-24T12:44:23.831977922Z same container: True, restarted: True + messenger-lab5-postgres-1 10fec15d4ea6 2026-10-24T12:43:40.513609095Z same container: True, restarted: True + messenger-lab5-rabbitmq-1 95b4214594a8 2026-10-24T12:43:41.265182826Z same container: True, restarted: True + messenger-lab5-redis-1 e7ed9903c342 2026-10-24T12:43:40.396962627Z same container: True, restarted: True + messenger-lab5-s3-1 56a0cdfb9bc8 2026-10-24T12:43:50.186903251Z same container: True, restarted: True + messenger-lab5-worker-1 6e040d05805f 2026-10-24T12:44:17.661567668Z same container: True, restarted: True +PASS: the hook restarted postgres, redis, s3 (SeaweedFS), rabbitmq, all api replicas, janitor, dispatcher, worker and lb (same containers, new start time); data, the attachment bytes and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 0000000..cf88f2f --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-10-24 15:41:05, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_race.py +# exit code: 0, 1.2 s + +sent 100 messages from 10 clients in 0.20 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/redis.txt b/evidence/redis.txt new file mode 100644 index 0000000..a7995f1 --- /dev/null +++ b/evidence/redis.txt @@ -0,0 +1,14 @@ +# redis: Redis as the session store +# date: 2026-10-24 15:41:19, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/redis_info.py +# exit code: 0, 3.1 s + +session key: session:5dc10133a5554a2f...; value is the user id: True +TTL of the key, seconds: 3599 +sessions in Redis: 299 +config appendonly = yes +config appendfsync = always +config maxmemory-policy = noeviction +aof_enabled:1 +aof_last_write_status:ok +without password: NOAUTH Authentication required. diff --git a/evidence/s3-security.txt b/evidence/s3-security.txt new file mode 100644 index 0000000..505cd3e --- /dev/null +++ b/evidence/s3-security.txt @@ -0,0 +1,47 @@ +# s3-security: lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap +# date: 2026-10-24 15:37:26, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_s3_security.py +# exit code: 0, 10.3 s + +anonymous get_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous head_object directly to S3 (from the data network) -> 403 +anonymous list_bucket directly to S3 (from the data network) -> 403 AccessDenied +anonymous put_object directly to S3 (from the data network) -> 403 AccessDenied +anonymous get_bucket_policy directly to S3 (from the data network) -> 403 AccessDenied +the same file through the API for a chat member -> 200 +PASS: the bucket is private: no anonymous read, list, write or policy read +API key: list buckets -> ['messenger-attachments'] +API key: get own object -> allowed +API key: put object -> allowed +API key: delete object -> allowed +API key: list bucket -> allowed +API key: create another bucket -> denied: AccessDenied +API key: make bucket public (put policy) -> denied: NotImplemented +API key: read bucket policy -> denied: NoSuchBucketPolicy +API key: delete bucket -> denied: AccessDenied +API key: change versioning -> denied: NotImplemented +PASS: the API key can only get/put/delete/list objects of its own bucket; bucket admin, policy and other buckets are denied +compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, filer UI 8888 are reachable only inside the stand) +127.0.0.1:8333 from the host -> connection refused +127.0.0.1:9333 from the host -> connection refused +127.0.0.1:8888 from the host -> connection refused +127.0.0.1:19333 from the host -> connection refused +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init +s3-init: s3-init: bucket messenger-attachments exists +s3-init: s3-init: api key can put/get/list/delete objects of its bucket +s3-init: s3-init: api key cannot create buckets (AccessDenied) +s3-init: s3-init: bucket is private (anonymous list/get/put -> [403]) +after the repeated bootstrap: old file downloads, new upload works +PASS: bucket/user/policy bootstrap can be repeated safely +4 passed, 0 failed diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 0000000..1e91652 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test), with attachment +# date: 2026-10-24 15:30:21, commit: 276dbda + uncommitted changes +# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 1.0 s + +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 0000000..6a29a62 --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,49 @@ +# stack: stand layout: replicas, ports, limits, networks, secrets, S3 +# date: 2026-10-24 15:30:20, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/check_stack.py +# exit code: 0, 3.0 s + +host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 +Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop + +SERVICE CONTAINER ID STATUS PORTS +api b9ad6d33d190 Up 4 minutes (healthy) 8080/tcp +api 5aa14958c864 Up 4 minutes (healthy) 8080/tcp +dispatcher 9c5531a487f2 Up 3 minutes (healthy) +janitor c412a9460051 Up 6 minutes (healthy) +lb 0760689bc54d Up 6 minutes (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +migrate 72d1affe9c8e Exited (0) 3 seconds ago +postgres 10fec15d4ea6 Up 7 minutes (healthy) +rabbitmq 95b4214594a8 Up 4 minutes (healthy) +redis e7ed9903c342 Up 7 minutes (healthy) +s3 56a0cdfb9bc8 Up 7 minutes (healthy) +s3-init 39d4358fe2ba Exited (0) 2 seconds ago +worker 6e040d05805f Up 3 minutes (healthy) + +container image cpus memory host ports networks +messenger-lab5-api-1 messenger-lab5-api 1.00 512 MiB - data,edge +messenger-lab5-api-2 messenger-lab5-api 1.00 512 MiB - data,edge +messenger-lab5-dispatcher-1 messenger-lab5-dispatcher 0.25 128 MiB - data +messenger-lab5-janitor-1 messenger-lab5-janitor 0.25 192 MiB - data +messenger-lab5-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge +messenger-lab5-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data +messenger-lab5-rabbitmq-1 rabbitmq:4.1.4-alpine 1.00 512 MiB - data +messenger-lab5-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data +messenger-lab5-s3-1 chrislusf/seaweedfs:3.80 1.00 512 MiB - data +messenger-lab5-worker-1 messenger-lab5-worker 2.00 1536 MiB - data + +secrets of api: ['cache_url', 'database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of dispatcher: ['broker_url_dispatcher', 'database_url'] +secrets of janitor: ['database_url', 's3_access_key_id', 's3_secret_access_key'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of rabbitmq: ['rabbitmq_definitions'] +secrets of redis: ['redis_auth'] +secrets of s3: ['s3_config'] +secrets of s3-init: ['s3_access_key_id', 's3_root_password', 's3_root_user', 's3_secret_access_key'] +secrets of worker: ['broker_url_worker', 'database_url', 's3_access_key_id', 's3_secret_access_key'] + +through http://127.0.0.1:8080 one session is served by 2 replicas: ['5aa14958c864', 'b9ad6d33d190'] +/health/ready -> 200 from b9ad6d33d190 +PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, postgres/redis/s3/janitor/rabbitmq/worker/dispatcher only in the internal data network, S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 0000000..fee036b --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,26 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-10-24 15:41:04, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/test_api.py +# exit code: 0, 2.7 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +20 passed, 0 failed diff --git a/evidence/summary.md b/evidence/summary.md new file mode 100644 index 0000000..71fc1f1 --- /dev/null +++ b/evidence/summary.md @@ -0,0 +1,78 @@ +# Сводка проверок + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/check.txt | +| stack | stand layout: replicas, ports, limits, networks, secrets, S3 | PASS | 3 | evidence/stack.txt | +| smoke | public smoke via the balancer (make test), with attachment | PASS | 1 | evidence/smoke.txt | +| jobs-variants | lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped | PASS | 20 | evidence/jobs-variants.txt | +| jobs-limits | lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS) | PASS | 74 | evidence/jobs-limits.txt | +| jobs-recovery | lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits | PASS | 55 | evidence/jobs-recovery.txt | +| jobs-outbox | lab5: dispatcher crash between COMMIT and publish / after confirm | PASS | 15 | evidence/jobs-outbox.txt | +| jobs-crash | lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease | PASS | 156 | evidence/jobs-crash.txt | +| jobs-retry | lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ | PASS | 70 | evidence/jobs-retry.txt | +| jobs-concurrency | lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing | PASS | 15 | evidence/jobs-concurrency.txt | +| attachments | lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids | PASS | 10 | evidence/attachments.txt | +| s3-security | lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap | PASS | 10 | evidence/s3-security.txt | +| consistency | lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage | PASS | 156 | evidence/consistency.txt | +| upload-memory | lab4: >=10 concurrent uploads, peak memory of api replicas | PASS | 54 | evidence/upload-memory.txt | +| balancing | balancing, shared sessions/data/files, X-Instance-Id | PASS | 2 | evidence/balancing.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 3 | evidence/student-tests.txt | +| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt | +| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 11 | evidence/db.txt | +| redis | Redis as the session store | PASS | 3 | evidence/redis.txt | +| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt | +| failover | public failover test for every replica, file downloaded via the other one (make test-failover) | PASS | 41 | evidence/failover.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| failures | Redis and PostgreSQL outages via the balancer | PASS | 55 | evidence/failures.txt | +| persistence | public persistence test incl. S3 + proof of real restarts (make test-persistence) | PASS | 60 | evidence/persistence.txt | + +# Замеры + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-24, commit 276dbda + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 5aa14958c864 | 2 | 123 | 0 | 4.406 | +| 2 | b9ad6d33d190 | 2 | 125 | 0 | 4.188 | + +Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики перезапускаются, лимит api 512 MiB / 1 CPU). Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов: + +| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 10 x ~9.5 MiB PNG (noise) | 10 | 94.9 | 1.45 / 1.56 | 5aa14958c864: 5 | 96.0 | 109.7 | 13.7 | 149.6 | +| | | | | b9ad6d33d190: 5 | 94.5 | 108.9 | 14.4 | 142.1 | +| 10 x 40 Mpx PNG (8000x5000 RGB) | 10 | 1.1 | 1.04 / 1.31 | 5aa14958c864: 5 | 96.6 | 97.6 | 1.0 | 89.6 | +| | | | | b9ad6d33d190: 5 | 96.2 | 98.6 | 2.3 | 88.9 | +| 20 mixed: large + 40 Mpx + JPEG | 20 | 133.2 | 4.96 / 5.2 | 5aa14958c864: 10 | 94.7 | 114.9 | 20.3 | 155.4 | +| | | | | b9ad6d33d190: 10 | 97.0 | 121.8 | 24.8 | 176.9 | + +Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в памяти + полное декодирование Pillow) 161 MiB, потоковая проверка 0 MiB. 10 таких наивных загрузок одновременно не поместились бы в 512 MiB. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 243.885 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 16.0 мс, p95 32.0 мс, p99 32.0 мс. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.406, 4.188 с (hook ждёт остановки ~4 с); потерь нет | +| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и прежний токен живы | PASS, те же контейнеры с новым временем старта | 58.5 с до готовности; потерь нет | +| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, сообщения и /health/ready работают | upload 503 за 19922 мс, download 503, остальное 200/201 | после start файлы на месте; uploading-строку убрал janitor; потерь нет | +| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за 14860 мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет | +| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | мусора не осталось; повторный проход ничего не делает | +| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | строк и объектов не прибавилось, временных файлов 0 | мгновенно | +| worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker | 202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready | worker started -> same job 9ab3ca59-e6f4-4f6e-928b-7163577d6139 ready, thumbnail downloadable; staging object deleted | потерь нет; staging удалён после COMMIT | +| rabbitmq остановлен во время upload | 202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется | rabbitmq started -> dispatcher published the pending row -> ready | потерь нет | +| crash dispatcher между COMMIT задания и publish | строка outbox не отмечена -> публикуется после рестарта процесса | dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job 9d62ceee-578c-46df-9916-1a7c4c6d8318 ready, attempt 1, no manual action | без ручного действия | +| crash dispatcher после confirm, до отметки outbox | дубль сообщения, worker распознаёт по состоянию job | dispatcher killed after publisher confirm, before marking outbox -> message published twice; worker: ['created', 'claimed', 'ready', 'duplicate_ignored'] -> one result set | один набор вариантов | +| crash worker после PUT в S3, до COMMIT и ack | повтор после истечения аренды перезаписывает те же ключи | worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired after 15 s, attempt 2 rewrote the same keys -> one set; events ['created', 'claimed', 'duplicate_ignored', 'lease_expired', 'claimed', 'ready'] | один набор, 4 объекта | +| crash worker после COMMIT, до ack | redelivery ready-задачи -> no-op ack | worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op (1:duplicate_ignored:status=ready); queue empty (0 ready, 0 unacked) | очередь пуста, дублей нет | +| worker завис (не продлевает аренду) | dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing | frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after 17 s; frozen attempt woke up -> stale_result_rejected; events ['created', 'claimed', 'lease_expired', 'claimed', 'ready', 'stale_result_rejected'] | ready не откатан | +| зависание обработки изображения | процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась | imaging hang -> killed after 60 s (processing_timeout, temporary), lease renewed meanwhile (no lease_expired), attempt 2 ready after 63 s | повтор -> ready | +| нехватка памяти в процессе обработки (RLIMIT_AS) | failed/resource_limit, worker жив | memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived | постоянная ошибка, без повторов | +| 2 временные ошибки S3 | backoff с jitter, затем ready | 2 x storage_unavailable -> retries after 1.00 s and 3.30 s (equal jitter: 1..2 s, 2..4 s), attempt 3 ready | - | +| 5 временных ошибок подряд | failed/retry_exhausted, реестр failed, content 409 | 5 temporary errors -> failed/retry_exhausted (public error_code), content 409, attach 409, listed by ops.py failed (replayable) | replay оператора | +| replay failed и повторный replay ready | тот же job_id -> ready; повтор отклонён | ops replay -> same job_id f7e15ba3-2d82-414f-8d88-3038061dbc51 attempt 6 ready; second replay refused: "job is ready, replay is only for failed jobs" | дубля нет | +| невалидное сообщение и неизвестный job_id | reject -> DLQ, dlq-replay не исполняет | schema-invalid message and unknown job_id -> reject -> DLQ (3 message(s)); dlq-replay kept them: ['unknown job', 'invalid message is kept for analysis'] | сообщения сохранены | +| 8 заданий в очереди, старт worker | WORKER_CONCURRENCY=2, prefetch 1 | 2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in 2.4 s after worker start, peak processing 2 of 8 (6 samples) | - | diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 0000000..4ba7f24 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,124 @@ +{ + "date": "2026-10-24 15:40", + "api_limit_mib": 512, + "api_limit_cpus": 1.0, + "runs": [ + { + "scenario": "10 x ~9.5 MiB PNG (noise)", + "concurrency": 10, + "file_bytes": [ + 9953047 + ], + "total_mib": 94.9, + "all_202": true, + "wall_seconds": 1.56, + "latency_s": { + "p50": 1.45, + "max": 1.56 + }, + "sampler_rounds": 9, + "replicas": [ + { + "replica": "5aa14958c864", + "uploads_served": 5, + "rss_before_mib": 96.0, + "rss_peak_mib": 109.7, + "rss_growth_mib": 13.7, + "rss_peak_sampled_mib": 109.7, + "cgroup_peak_mib": 149.6, + "cgroup_peak_sampled_mib": 145.2 + }, + { + "replica": "b9ad6d33d190", + "uploads_served": 5, + "rss_before_mib": 94.5, + "rss_peak_mib": 108.9, + "rss_growth_mib": 14.4, + "rss_peak_sampled_mib": 108.9, + "cgroup_peak_mib": 142.1, + "cgroup_peak_sampled_mib": 139.3 + } + ] + }, + { + "scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", + "concurrency": 10, + "file_bytes": [ + 116714 + ], + "total_mib": 1.1, + "all_202": true, + "wall_seconds": 1.31, + "latency_s": { + "p50": 1.04, + "max": 1.31 + }, + "sampler_rounds": 8, + "replicas": [ + { + "replica": "5aa14958c864", + "uploads_served": 5, + "rss_before_mib": 96.6, + "rss_peak_mib": 97.6, + "rss_growth_mib": 1.0, + "rss_peak_sampled_mib": 97.6, + "cgroup_peak_mib": 89.6, + "cgroup_peak_sampled_mib": 89.2 + }, + { + "replica": "b9ad6d33d190", + "uploads_served": 5, + "rss_before_mib": 96.2, + "rss_peak_mib": 98.6, + "rss_growth_mib": 2.3, + "rss_peak_sampled_mib": 98.6, + "cgroup_peak_mib": 88.9, + "cgroup_peak_sampled_mib": 86.2 + } + ] + }, + { + "scenario": "20 mixed: large + 40 Mpx + JPEG", + "concurrency": 20, + "file_bytes": [ + 1492, + 116714, + 9953047 + ], + "total_mib": 133.2, + "all_202": true, + "wall_seconds": 5.2, + "latency_s": { + "p50": 4.96, + "max": 5.2 + }, + "sampler_rounds": 27, + "replicas": [ + { + "replica": "5aa14958c864", + "uploads_served": 10, + "rss_before_mib": 94.7, + "rss_peak_mib": 114.9, + "rss_growth_mib": 20.3, + "rss_peak_sampled_mib": 114.9, + "cgroup_peak_mib": 155.4, + "cgroup_peak_sampled_mib": 154.4 + }, + { + "replica": "b9ad6d33d190", + "uploads_served": 10, + "rss_before_mib": 97.0, + "rss_peak_mib": 121.8, + "rss_growth_mib": 24.8, + "rss_peak_sampled_mib": 121.8, + "cgroup_peak_mib": 176.9, + "cgroup_peak_sampled_mib": 173.1 + } + ] + } + ], + "single_file_extra_rss_mib": { + "naive": 161, + "streaming": 0 + } +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 0000000..ac7213b --- /dev/null +++ b/evidence/upload-memory.txt @@ -0,0 +1,13 @@ +# upload-memory: lab4: >=10 concurrent uploads, peak memory of api replicas +# date: 2026-10-24 15:40:56, commit: 276dbda + uncommitted changes +# command: python.exe student-tests/upload_memory.py --output evidence/upload-memory.json +# exit code: 0, 53.8 s + +client: Python 3.11.1 on Windows 10 +preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)... +large: 9953047 bytes; 40 Mpx: 116714 bytes +{"scenario": "10 x ~9.5 MiB PNG (noise)", "concurrency": 10, "file_bytes": [9953047], "total_mib": 94.9, "all_202": true, "wall_seconds": 1.56, "latency_s": {"p50": 1.45, "max": 1.56}, "sampler_rounds": 9, "replicas": [{"replica": "5aa14958c864", "uploads_served": 5, "rss_before_mib": 96.0, "rss_peak_mib": 109.7, "rss_growth_mib": 13.7, "rss_peak_sampled_mib": 109.7, "cgroup_peak_mib": 149.6, "cgroup_peak_sampled_mib": 145.2}, {"replica": "b9ad6d33d190", "uploads_served": 5, "rss_before_mib": 94.5, "rss_peak_mib": 108.9, "rss_growth_mib": 14.4, "rss_peak_sampled_mib": 108.9, "cgroup_peak_mib": 142.1, "cgroup_peak_sampled_mib": 139.3}]} +{"scenario": "10 x 40 Mpx PNG (8000x5000 RGB)", "concurrency": 10, "file_bytes": [116714], "total_mib": 1.1, "all_202": true, "wall_seconds": 1.31, "latency_s": {"p50": 1.04, "max": 1.31}, "sampler_rounds": 8, "replicas": [{"replica": "5aa14958c864", "uploads_served": 5, "rss_before_mib": 96.6, "rss_peak_mib": 97.6, "rss_growth_mib": 1.0, "rss_peak_sampled_mib": 97.6, "cgroup_peak_mib": 89.6, "cgroup_peak_sampled_mib": 89.2}, {"replica": "b9ad6d33d190", "uploads_served": 5, "rss_before_mib": 96.2, "rss_peak_mib": 98.6, "rss_growth_mib": 2.3, "rss_peak_sampled_mib": 98.6, "cgroup_peak_mib": 88.9, "cgroup_peak_sampled_mib": 86.2}]} +{"scenario": "20 mixed: large + 40 Mpx + JPEG", "concurrency": 20, "file_bytes": [1492, 116714, 9953047], "total_mib": 133.2, "all_202": true, "wall_seconds": 5.2, "latency_s": {"p50": 4.96, "max": 5.2}, "sampler_rounds": 27, "replicas": [{"replica": "5aa14958c864", "uploads_served": 10, "rss_before_mib": 94.7, "rss_peak_mib": 114.9, "rss_growth_mib": 20.3, "rss_peak_sampled_mib": 114.9, "cgroup_peak_mib": 155.4, "cgroup_peak_sampled_mib": 154.4}, {"replica": "b9ad6d33d190", "uploads_served": 10, "rss_before_mib": 97.0, "rss_peak_mib": 121.8, "rss_growth_mib": 24.8, "rss_peak_sampled_mib": 121.8, "cgroup_peak_mib": 176.9, "cgroup_peak_sampled_mib": 173.1}]} +one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) 161 MiB, streaming check 0 MiB +PASS: all uploads succeeded; largest RSS growth of one replica: 24.8 MiB (limit 512 MiB); results in evidence/upload-memory.json diff --git a/faults.py b/faults.py new file mode 100644 index 0000000..2bb92a7 --- /dev/null +++ b/faults.py @@ -0,0 +1,51 @@ +"""Учебное внедрение сбоев для тестов отказов (только при FAULT_INJECTION=1). + +Тест создаёт в общем volume /faults файлы <точка>., по одному на срабатывание: + + docker compose exec -T worker sh -c 'touch /faults/worker_crash_after_put.1' + +Процесс, дошедший до точки, атомарно удаляет один файл (os.remove успешен ровно у одного процесса) +и выполняет сбой. Так сбой случается заданное число раз и не зацикливает перезапуски. + +Точки: + dispatcher_crash_before_publish — COMMIT задания уже был, publish ещё нет (окно COMMIT -> publish) + dispatcher_crash_after_confirm — брокер подтвердил публикацию, отметка в outbox ещё не записана + worker_crash_after_put — объекты записаны в S3, COMMIT результата ещё нет + worker_crash_after_commit — COMMIT результата был, ack ещё нет + worker_freeze — worker «завис»: не продлевает аренду и спит 3 x lease + worker_s3_error — временная ошибка S3 при записи результата + imaging_hang / imaging_invalid / imaging_oom — сбой внутри процесса обработки изображения +""" +import logging +import os +import sys +from pathlib import Path + +import settings + +log = logging.getLogger("messenger.faults") + + +def hit(point: str) -> bool: + if not settings.FAULT_INJECTION: + return False + directory = Path(settings.FAULTS_DIR) + try: + candidates = sorted(directory.glob(point + ".*")) + except OSError: + return False + for path in candidates: + try: + path.unlink() + except FileNotFoundError: + continue # этот файл забрал другой процесс + log.warning("FAULT INJECTION: %s", point) + return True + return False + + +def crash(point: str) -> None: + """Аварийное завершение всего процесса без очистки (как SIGKILL/OOM): без ack, без закрытия соединений.""" + if hit(point): + sys.stderr.flush() + os._exit(137) diff --git a/imaging.py b/imaging.py new file mode 100644 index 0000000..a08fbb0 --- /dev/null +++ b/imaging.py @@ -0,0 +1,209 @@ +"""Преобразование изображения, pipeline_version=1. Запускается worker'ом ОТДЕЛЬНЫМ процессом с лимитами. + + python imaging.py + +Почему отдельный процесс: декодер Pillow работает в C, его нельзя прервать изнутри Python и нельзя +ограничить по памяти отдельно от остального worker. Процесс сам ставит себе RLIMIT_AS (память, +IMAGING_MEMORY_MB), RLIMIT_CPU (процессорное время) и RLIMIT_FSIZE до импорта Pillow, а worker +убивает его по таймауту JOB_TIMEOUT_SECONDS. +Падение, зависание или нехватка памяти здесь не роняют worker и не ломают подключение к брокеру. + +Результат: файлы thumbnail.jpg, crop.jpg, optimized.webp в out_dir и одна строка JSON в stdout. +Код выхода: 0 — успех; 3 — невалидное изображение; 4 — слишком много пикселей; 5 — не хватило памяти; +прочее (в том числе убит сигналом) — неожиданная ошибка, worker считает её временной. + +Правила вариантов (contracts/README.md): +- thumbnail: JPEG, вписан в 256x256 с сохранением пропорций, без увеличения, размеры округляются вниз + (минимум 1 px), EXIF orientation применяется, прозрачность накладывается на белый фон. 320x200 -> 256x160. +- crop: JPEG 128x128, центрированный квадрат (маленькие исходники увеличиваются). +- optimized: WebP, вписан в 1280x1280 без увеличения, quality=80, method=4; прозрачность сохраняется. +Во всех вариантах нет EXIF/XMP/комментариев: в save() передаётся только ICC-профиль (цвет), остальное +отбрасывается. Pillow при одинаковой версии даёт побайтно одинаковый результат, поэтому повтор job +пишет в те же ключи S3 те же байты. +""" +import json +import os +import resource +import sys +import time + + +def apply_limits() -> None: + """Лимиты на себя, ДО импорта Pillow: память (RLIMIT_AS), CPU, размер записываемых файлов.""" + memory = int(os.environ.get("IMAGING_MEMORY_MB", "640")) * 1024 * 1024 + cpu = int(os.environ.get("IMAGING_CPU_SECONDS", "60")) + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + resource.setrlimit(resource.RLIMIT_FSIZE, (64 * 1024 * 1024, 64 * 1024 * 1024)) + + +if __name__ == "__main__": + apply_limits() + +from PIL import Image # noqa: E402 — после apply_limits: память декодера тоже под лимитом + +PIPELINE_VERSION = 1 +THUMB_BOX = 256 +CROP_SIZE = 128 +OPTIMIZED_BOX = 1280 +THUMB_QUALITY = 85 +CROP_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 4 + +EXIT_INVALID = 3 +EXIT_TOO_LARGE = 4 +EXIT_MEMORY = 5 + +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +# EXIF orientation -> преобразование (как в PIL.ImageOps.exif_transpose) +ORIENTATION = { + 2: [Image.Transpose.FLIP_LEFT_RIGHT], + 3: [Image.Transpose.ROTATE_180], + 4: [Image.Transpose.FLIP_TOP_BOTTOM], + 5: [Image.Transpose.TRANSPOSE], + 6: [Image.Transpose.ROTATE_270], + 7: [Image.Transpose.TRANSVERSE], + 8: [Image.Transpose.ROTATE_90], +} + + +class TooLarge(Exception): + pass + + +def fit(width: int, height: int, box: int) -> tuple[int, int]: + """Вписать в box x box без увеличения; размеры вниз, минимум 1 px. Целочисленно, без float.""" + if width <= box and height <= box: + return width, height + if width >= height: + return box, max(1, height * box // width) + return max(1, width * box // height), box + + +def orientation_of(image: Image.Image) -> int: + try: + value = image.getexif().get(0x0112, 1) + except Exception: # noqa: BLE001 — битый EXIF не делает картинку невалидной, ориентацию не трогаем + return 1 + return value if value in ORIENTATION else 1 + + +def normalize(image: Image.Image) -> Image.Image: + """Любой режим PNG/JPEG -> L, RGB или RGBA (RGBA только если есть прозрачность).""" + mode = image.mode + transparent = "transparency" in image.info + if mode in ("I;16", "I;16B", "I;16L", "I"): + image = image.convert("I").point(lambda value: value * (1 / 256)).convert("L") + mode = "L" + if mode == "1": + return image.convert("L") + if mode in ("P", "PA"): + return image.convert("RGBA" if transparent or mode == "PA" else "RGB") + if mode in ("LA", "La", "RGBa"): + return image.convert("RGBA") + if mode in ("L", "RGB") and transparent: + return image.convert("RGBA") + if mode in ("L", "RGB", "RGBA"): + return image + return image.convert("RGB") # CMYK, YCbCr, LAB, ... + + +def on_white(image: Image.Image) -> Image.Image: + """Прозрачность -> белый фон (для JPEG).""" + if image.mode != "RGBA": + return image + background = Image.new("RGB", image.size, (255, 255, 255)) + background.paste(image, mask=image.getchannel("A")) + return background + + +def process(source: str, out_dir: str, content_type: str, max_pixels: int) -> dict: + Image.MAX_IMAGE_PIXELS = max_pixels + with Image.open(source) as image: + if image.format != FORMATS.get(content_type): + raise ValueError("format does not match stored content type") + width, height = image.size + if width * height > max_pixels: + raise TooLarge() + orientation = orientation_of(image) + icc = image.info.get("icc_profile") + swap = orientation in (5, 6, 7, 8) + oriented = (height, width) if swap else (width, height) + + # Сначала уменьшаем до optimized (<= 1280), потом поворачиваем: квадратные рамки 1280/256/128 + # не зависят от поворота, а поворот маленькой картинки дешевле, чем копия 40-мегапиксельной. + target = fit(*oriented, OPTIMIZED_BOX) + stored_target = (target[1], target[0]) if swap else target + if image.format == "JPEG": + image.draft(None, stored_target) # декодирование сразу в масштабе 1/2..1/8, если хватает + image.load() + work = normalize(image) + if work.size != stored_target: + work = work.resize(stored_target, Image.Resampling.LANCZOS, reducing_gap=3.0) + elif work is image: + work = work.copy() + for step in ORIENTATION.get(orientation, []): + work = work.transpose(step) + + extra = {"icc_profile": icc} if icc else {} + results = [] + + optimized = work if work.mode != "L" else work.convert("RGB") + path = os.path.join(out_dir, "optimized.webp") + optimized.save(path, format="WEBP", quality=WEBP_QUALITY, method=WEBP_METHOD, **extra) + results.append({"name": "optimized", "content_type": "image/webp", "width": optimized.width, + "height": optimized.height, "file": "optimized.webp"}) + + thumb_size = fit(*oriented, THUMB_BOX) # от размеров ИСХОДНИКА, а не промежуточной копии + thumb = work if work.size == thumb_size else work.resize(thumb_size, Image.Resampling.LANCZOS) + thumb = on_white(thumb) + path = os.path.join(out_dir, "thumbnail.jpg") + thumb.save(path, format="JPEG", quality=THUMB_QUALITY, optimize=True, **extra) + results.append({"name": "thumbnail", "content_type": "image/jpeg", "width": thumb.width, + "height": thumb.height, "file": "thumbnail.jpg"}) + + side = min(work.size) + left, top = (work.width - side) // 2, (work.height - side) // 2 + crop = work.crop((left, top, left + side, top + side)) + crop = on_white(crop.resize((CROP_SIZE, CROP_SIZE), Image.Resampling.LANCZOS)) + path = os.path.join(out_dir, "crop.jpg") + crop.save(path, format="JPEG", quality=CROP_QUALITY, optimize=True, **extra) + results.append({"name": "crop", "content_type": "image/jpeg", "width": CROP_SIZE, "height": CROP_SIZE, + "file": "crop.jpg"}) + + return {"pipeline_version": PIPELINE_VERSION, "source": {"width": width, "height": height, + "orientation": orientation}, "variants": results} + + +def injected_fault() -> None: + """Учебные сбои (FAULT_IMAGING задаёт worker, только при FAULT_INJECTION=1).""" + fault = os.environ.get("FAULT_IMAGING", "") + if fault == "hang": + time.sleep(3600) + elif fault == "invalid": + raise ValueError("injected invalid image") + elif fault == "oom": + blocks = [] + while True: + blocks.append(bytearray(64 * 1024 * 1024)) + + +def main() -> int: + source, out_dir, content_type, max_pixels = sys.argv[1], sys.argv[2], sys.argv[3], int(sys.argv[4]) + try: + injected_fault() + result = process(source, out_dir, content_type, max_pixels) + except (TooLarge, Image.DecompressionBombError): + return EXIT_TOO_LARGE + except MemoryError: + return EXIT_MEMORY + except (OSError, SyntaxError, ValueError, IndexError, TypeError, EOFError, Image.UnidentifiedImageError): + # Pillow сообщает о битом файле разными исключениями; подробности не нужны ни клиенту, ни логу. + return EXIT_INVALID + print(json.dumps(result), flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 0000000..1605eed --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,77 @@ +# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики). +# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить +# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky). + +global + log stdout format raw local0 info + maxconn 4096 + # При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с. + hard-stop-after 10s + +# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик. +# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s # перечитывать список реплик раз в 2 с + hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlog-normal # в лог только ошибки, а не каждый запрос + timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос + timeout client 30s + # Ожидание ответа API после того, как запрос (включая тело файла) передан. Самый долгий ответ — загрузка + # при зависшем S3: PUT 2 попытки x (1 + 5 с) + откат 3 с + БД до 8 с ~ 24 с. Пока тело файла идёт, + # сервер «активен», этот таймаут не срабатывает; приём тела ограничивает сам API (UPLOAD_*_TIMEOUT). + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода. + # Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key. + retries 2 + option redispatch 1 + +frontend api_in + bind :8080 + option forwardfor + default_backend api + +backend api + balance roundrobin + # Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis. + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT. + # Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP. + # observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки. + server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + # Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis + # или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате + # контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает. + # Пока готова хоть одна реплика, эти записи не используются. + # allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend). + server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + option allbackups + +# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1). +# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда. +frontend stats + bind :8404 + monitor-uri /lb-health + monitor fail if { nbsrv(api) lt 1 } + stats enable + stats uri /stats + stats refresh 2s diff --git a/infra/rabbitmq.conf b/infra/rabbitmq.conf new file mode 100644 index 0000000..f4f41e0 --- /dev/null +++ b/infra/rabbitmq.conf @@ -0,0 +1,22 @@ +# RabbitMQ 4.1 (сервис rabbitmq). Топология, пользователи и права — в secrets/rabbitmq_definitions.json +# (генерирует scripts/init_secrets.py), файл импортируется при каждом старте узла. Пользователя guest нет: +# при импорте определений на старте пользователь по умолчанию не создаётся. +definitions.import_backend = local_filesystem +definitions.local.path = /run/secrets/rabbitmq_definitions +definitions.skip_if_unchanged = true + +listeners.tcp.default = 5672 +loopback_users = none + +# Ресурсы: при нехватке памяти/диска брокер блокирует издателей (dispatcher ждёт и повторяет), +# сообщения не теряются. Абсолютное значение: ниже лимита контейнера 512M (доля считалась бы от RAM хоста). +vm_memory_high_watermark.absolute = 320MB +disk_free_limit.absolute = 256MB + +# Consumer, не приславший ack за это время, считается зависшим: канал закрывается, сообщение +# возвращается в очередь. Должно быть больше JOB_TIMEOUT_SECONDS + времени записи в S3. +consumer_timeout = 300000 + +log.console = true +log.console.level = info +log.file = false diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 0000000..33c8c08 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,13 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +appendonly yes +appendfsync always +save "" + +maxmemory 64mb +maxmemory-policy noeviction + +include /run/secrets/redis_auth diff --git a/janitor.py b/janitor.py new file mode 100644 index 0000000..559c87d --- /dev/null +++ b/janitor.py @@ -0,0 +1,278 @@ +"""Очистка и сверка PostgreSQL <-> S3 (сервис janitor). + + python janitor.py --once [--dry-run] [--grace-seconds N] + +Проход (раз в JANITOR_INTERVAL_SECONDS, один janitor за счёт pg_try_advisory_lock): + +1. Брошенные намерения (как в lab4): state=uploading старше grace — исходник так и не был поставлен + в очередь. Удалить объект staging (идемпотентно), затем строку. +2. Staging после безопасной фиксации: + - ready: исходник уже опубликован в attachments//v/original -> удалить staging-объект + (обычно это делает worker сразу после ack; здесь — если он не успел); + - failed: исходник хранится STAGING_RETENTION_SECONDS после ошибки (для replay), потом удаляется. + queued/processing не трогаются никогда: это текущая работа. +3. Outbox: опубликованные строки старше OUTBOX_RETENTION_SECONDS удаляются. +4. Обход bucket по всем prefix (объекты старше grace): + - staging/, originals/: сирота, если ни одна строка не ссылается на ключ как source_key; + - attachments//...: сирота, если вложения нет, оно failed, или оно ready, но ключ не входит + в его опубликованный набор (оригинал + варианты). У queued/processing объекты не трогаются: + их перезапишет следующая попытка теми же ключами. + Отчёт о ready-вложениях без объекта в bucket (missing) — только отчёт, это потеря данных. + +Grace (ATTACHMENT_GRACE_SECONDS, по умолчанию 1 ч) больше любой живой загрузки/попытки, поэтому +janitor не удаляет то, что ещё пишется. +""" +import argparse +import json +import logging +import re +import signal +import threading +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import pipeline +import settings +import storage + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.janitor") + +JANITOR_LOCK = 20260404 +BATCH = 100 +STOP = threading.Event() +ACTOR = "janitor" +FINAL_KEY_RE = re.compile(r"^attachments/([0-9a-f-]{36})/") + + +def touch_alive() -> None: + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + + +def idle(seconds: float) -> bool: + """Пауза между проходами с обновлением файла живости (healthcheck). True — пришёл сигнал остановки.""" + deadline = time.monotonic() + seconds + while (left := deadline - time.monotonic()) > 0: + touch_alive() + if STOP.wait(min(left, 10)): + return True + return False + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def reclaim_stale_intents(conn, grace: int, dry_run: bool) -> tuple[int, int]: + """Шаг 1. Возвращает (найдено, удалено).""" + found = deleted = 0 + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT id, source_key FROM attachments + WHERE state = 'uploading' AND created_at < now() - make_interval(secs => %s) + ORDER BY created_at LIMIT %s + FOR UPDATE SKIP LOCKED""", + (grace, BATCH), + ).fetchall() + found += len(rows) + if dry_run or not rows: + return found, deleted + for row in rows: + storage.delete(row["source_key"]) # недоступен S3 -> исключение, транзакция откатится + conn.execute("DELETE FROM attachments WHERE id = %s AND state = 'uploading'", (row["id"],)) + deleted += 1 + log.info("reclaimed abandoned upload %s (object and row deleted)", row["id"]) + if len(rows) < BATCH: + return found, deleted + return found, deleted + + +def cleanup_staging(conn, dry_run: bool) -> dict: + """Шаг 2. Исходники ready (сразу) и failed (после retention).""" + report = {"staging_ready_deleted": 0, "staging_failed_expired": 0, "staging_kept_for_replay": 0} + while not STOP.is_set(): + with conn.transaction(): + rows = conn.execute( + """SELECT a.id, a.state, a.source_key, j.job_id, j.attempt FROM attachments a + LEFT JOIN attachment_jobs j ON j.attachment_id = a.id AND j.pipeline_version = a.pipeline_version + WHERE a.source_key IS NOT NULL + AND ((a.state = 'ready' AND a.updated_at < now() - interval '60 seconds') + OR (a.state = 'failed' AND a.updated_at < now() - make_interval(secs => %s))) + ORDER BY a.updated_at LIMIT %s FOR UPDATE OF a SKIP LOCKED""", + (settings.STAGING_RETENTION_SECONDS, BATCH), + ).fetchall() + if dry_run or not rows: + if dry_run: + report["staging_ready_deleted"] = sum(r["state"] == "ready" for r in rows) + report["staging_failed_expired"] = sum(r["state"] == "failed" for r in rows) + break + for row in rows: + storage.delete(row["source_key"]) + conn.execute("UPDATE attachments SET source_key = NULL WHERE id = %s", (row["id"],)) + if row["state"] == "ready": + report["staging_ready_deleted"] += 1 + else: + report["staging_failed_expired"] += 1 + if row["job_id"] is not None: + pipeline.event(conn, row["job_id"], row["attempt"], "staging_expired", ACTOR, + "replay no longer possible") + if len(rows) < BATCH: + break + report["staging_kept_for_replay"] = conn.execute( + "SELECT count(*) AS n FROM attachments WHERE state = 'failed' AND source_key IS NOT NULL").fetchone()["n"] + return report + + +def cleanup_outbox(conn, dry_run: bool) -> int: + if dry_run: + return conn.execute( + """SELECT count(*) AS n FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).fetchone()["n"] + return conn.execute( + """DELETE FROM outbox WHERE published_at < now() - make_interval(secs => %s)""", + (settings.OUTBOX_RETENTION_SECONDS,)).rowcount + + +def classify_final(conn, keys: list[str]) -> set[str]: + """Какие ключи attachments//... не принадлежат ни одному живому вложению (кандидаты в сироты).""" + ids = {} + for key in keys: + match = FINAL_KEY_RE.match(key) + if match: + try: + ids[key] = UUID(match.group(1)) + except ValueError: + pass + unique = list(set(ids.values())) + rows = {r["id"]: r for r in conn.execute( + """SELECT a.id, a.state, a.object_key, + ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.id = ANY(%s)""", (unique,)).fetchall()} + orphans = set() + for key in keys: + attachment_id = ids.get(key) + row = rows.get(attachment_id) if attachment_id else None + if row is None or row["state"] == "failed": + orphans.add(key) + elif row["state"] == "ready" and key != row["object_key"] and key not in row["variant_keys"]: + orphans.add(key) + return orphans + + +def sweep_objects(conn, grace: int, dry_run: bool) -> dict: + """Шаг 4. Обход bucket постранично; сверка ключей с БД пачками по 1000.""" + started = datetime.now(timezone.utc) + cutoff = started - timedelta(seconds=grace) + scanned = orphans = orphans_deleted = 0 + listed = set() + + def orphan_keys(prefix, items): + keys = [key for key, _ in items] + if prefix == storage.FINAL_PREFIX: + return classify_final(conn, keys) + known = {r["source_key"] for r in conn.execute( + "SELECT source_key FROM attachments WHERE source_key = ANY(%s)", (keys,)).fetchall()} + return {key for key in keys if key not in known} + + def still_orphan(prefix, key): + if prefix == storage.FINAL_PREFIX: + return key in classify_final(conn, [key]) + return conn.execute("SELECT 1 FROM attachments WHERE source_key = %s", (key,)).fetchone() is None + + def check(prefix, items): + nonlocal orphans, orphans_deleted + candidates = orphan_keys(prefix, items) + for key, modified in items: + if key not in candidates or modified >= cutoff: + continue + orphans += 1 + if not dry_run and still_orphan(prefix, key): # повторная проверка прямо перед удалением + storage.delete(key) + orphans_deleted += 1 + log.info("deleted orphan object %s", key) + + for prefix in storage.ALL_PREFIXES: + page = [] + for key, modified, _size in storage.list_objects(prefix): + scanned += 1 + listed.add(key) + page.append((key, modified)) + if len(page) >= 1000: + check(prefix, page) + page = [] + if page: + check(prefix, page) + + missing = [] + for r in conn.execute( + """SELECT a.id, a.object_key, ARRAY(SELECT v.object_key FROM attachment_variants v + WHERE v.attachment_id = a.id AND v.pipeline_version = a.pipeline_version) AS variant_keys + FROM attachments a WHERE a.state = 'ready' AND a.updated_at < %s""", (started,)).fetchall(): + if r["object_key"] not in listed or any(k not in listed for k in r["variant_keys"]): + missing.append(str(r["id"])) + if missing: + log.error("ready attachments with objects missing in S3 (not deleted, needs restore): %s", + ", ".join(missing[:20])) + return {"objects_scanned": scanned, "orphan_objects": orphans, "orphan_objects_deleted": orphans_deleted, + "missing_objects": len(missing)} + + +def run_once(grace: int, dry_run: bool) -> dict | None: + started = time.monotonic() + with connect() as conn: + if not conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (JANITOR_LOCK,)).fetchone()["ok"]: + log.info("another janitor is running, skip") + return None + try: + found, deleted = reclaim_stale_intents(conn, grace, dry_run) + report = {"dry_run": dry_run, "grace_seconds": grace, "stale_uploads": found, + "stale_uploads_deleted": deleted, **cleanup_staging(conn, dry_run), + "outbox_rows_deleted": cleanup_outbox(conn, dry_run), **sweep_objects(conn, grace, dry_run)} + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (JANITOR_LOCK,)) + report["seconds"] = round(time.monotonic() - started, 3) + return report + + +def main(): + parser = argparse.ArgumentParser(description="Reclaim abandoned uploads, clean staging, sweep orphan objects") + parser.add_argument("--once", action="store_true", help="one pass and exit") + parser.add_argument("--dry-run", action="store_true", help="only report, delete nothing") + parser.add_argument("--grace-seconds", type=int, default=settings.ATTACHMENT_GRACE_SECONDS) + args = parser.parse_args() + if args.grace_seconds < 2 * settings.UPLOAD_TIMEOUT_SECONDS: + parser.error(f"--grace-seconds must be >= {2 * settings.UPLOAD_TIMEOUT_SECONDS} " + "(2 x UPLOAD_TIMEOUT_SECONDS), otherwise a running upload could be deleted") + + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + while True: + touch_alive() + try: + report = run_once(args.grace_seconds, args.dry_run) + if report is not None: + print(json.dumps(report), flush=True) + failed = False + except (psycopg.Error, storage.StorageUnavailable) as exc: + log.warning("pass failed, will retry: %s", type(exc).__name__) + failed = True + if args.once: + raise SystemExit(1 if failed else 0) + if idle(settings.JANITOR_INTERVAL_SECONDS): + return + + +if __name__ == "__main__": + main() diff --git a/main.py b/main.py new file mode 100644 index 0000000..bacd111 --- /dev/null +++ b/main.py @@ -0,0 +1,603 @@ +import base64 +import binascii +import hashlib +import json +import logging +import re +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime +from pathlib import Path +from typing import Literal +from urllib.parse import quote +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +import attachments +import auth +import db +import settings +import storage +import uploads +from errors import ApiError, bad_request, conflict, not_found, unauthorized, unavailable + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger") + +DRAINING = threading.Event() + + +class Server(uvicorn.Server): + """Остановка по SIGTERM в два шага. + + 1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS + HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы. + 2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы + (не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM, + получал бы 502 от балансировщика. + Повторный сигнал останавливает сразу. + """ + + def handle_exit(self, sig, frame): + if DRAINING.is_set(): + return super().handle_exit(sig, frame) + DRAINING.set() + log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s", + sig, settings.SHUTDOWN_DRAIN_SECONDS) + threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings.check() + # Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему. + await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS) + # Только для временных файлов текущих загрузок (удаляются в конце запроса); оригиналы живут в S3. + Path(settings.UPLOAD_TMP_DIR).mkdir(parents=True, exist_ok=True) + db.pool.open() + log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis, files in S3 bucket %s, " + "processing via outbox (pipeline v%s)", settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, + settings.SESSION_TTL_SECONDS, settings.S3_BUCKET, settings.PIPELINE_VERSION) + yield + db.pool.close() + auth.close() + log.info("Stopped: connections closed, data stays in PostgreSQL and Redis") + + +app = FastAPI(lifespan=lifespan) + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH) + + @field_validator("display_name", "password") + @classmethod + def validate_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("value cannot be blank") + return value + + +class User(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(BaseModel): + token: str + token_type: str + expires_at: datetime + user: User + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title cannot be empty") + return value + + +class Chat(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class ChatPage(BaseModel): + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + # С lab4: 0-10 уникальных вложений этого чата; пустой текст допустим только с вложением. + attachment_ids: list[UUID] = Field(default_factory=list, max_length=10) + + @model_validator(mode="after") + def validate_content(self): + if len(set(self.attachment_ids)) != len(self.attachment_ids): + raise ValueError("attachment_ids must be unique") + if not self.attachment_ids and not self.text.strip(): + raise ValueError("text cannot be empty without attachments") + return self + + +class Message(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: list[UUID] + + +class Variant(BaseModel): + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int + width: int + height: int + sha256: str + + +class Attachment(BaseModel): + id: UUID + chat_id: UUID + owner_id: UUID + filename: str + content_type: str + size_bytes: int + sha256: str + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] + error_code: str | None + created_at: datetime + updated_at: datetime + + +class MessagePage(BaseModel): + items: list[Message] + next_cursor: str | None + + +def error_response( + request: Request, + code: str, + message: str, + status_code: int, + headers: dict | None = None, +): + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers={"X-Request-Id": request_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + return error_response( + request, + "invalid_request", + "Invalid request data", + 400, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Not found", 404) + return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def general_exception_handler( + request: Request, + exc: Exception, +): + return error_response( + request, + "internal_error", + "Internal server error", + 500, + ) + +PUBLIC_ENDPOINTS = { + ("POST", "/api/v1/users"), + ("POST", "/api/v1/auth/sessions"), +} + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + path = request.url.path + + try: + if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS: + request.state.session = await run_in_threadpool( + auth.check_bearer, request.headers.get("Authorization") + ) + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + except Exception: + log.exception("Unhandled error, request_id=%s", request_id) + response = error_response(request, "internal_error", "Internal server error", 500) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила + # Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки. + level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO + log.log( + level, + "%s %s -> %s (%.1f ms) request_id=%s", + request.method, path, response.status_code, + (time.perf_counter() - started) * 1000, request_id, + ) + return response + + +def current_user_id(request: Request) -> UUID: + session = getattr(request.state, "session", None) + if session is None: + raise unauthorized("Bearer token is required") + return session.user_id + + +def encode_cursor( + created_at: datetime, + object_id: UUID, + *, + chat_id: UUID | None = None, + user_id: UUID | None = None, +) -> str: + data = { + "created_at": created_at.isoformat(), + "id": str(object_id), + } + + if chat_id is not None: + data["chat_id"] = str(chat_id) + + if user_id is not None: + data["user_id"] = str(user_id) + + raw = json.dumps(data).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode_cursor(cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + data = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(data["created_at"]) + + if created_at.tzinfo is None: + raise ValueError("Invalid cursor") + + if not isinstance(data["id"], str): + raise ValueError("Invalid cursor") + + object_id = UUID(data["id"]) + + chat_id = UUID(data["chat_id"]) if "chat_id" in data else None + user_id = UUID(data["user_id"]) if "user_id" in data else None + + return created_at, object_id, chat_id, user_id + + except ( + ValueError, + KeyError, + TypeError, + AttributeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("Invalid cursor") + + +def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): + if cursor is None: + return None + try: + created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) + except ValueError: + raise bad_request("Invalid cursor") from None + if cursor_chat_id != chat_id or cursor_user_id != user_id: + raise bad_request("Invalid cursor") + return created_at, object_id + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + if DRAINING.is_set(): + raise unavailable("Instance is shutting down") + db.ping() + auth.ping_redis() + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=User, status_code=201) +def create_user(user_data: CreateUser): + password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется + return db.create_user(user_data.username, user_data.display_name, password_hash) + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +def create_session( + response: Response, + authorization: str | None = Header(default=None), +): + credentials = auth.parse_basic(authorization) + if credentials is None: + raise auth.login_failed() + username, password = credentials + + found = None + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: + found = db.get_user_with_password(username) + if not auth.password_matches(found["password_hash"] if found else None, password): + raise auth.login_failed() + + token, expires_at = auth.create_session(found["id"]) + response.headers["Cache-Control"] = "no-store" + user = {key: found[key] for key in ("id", "username", "display_name", "created_at")} + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + current_user_id(request) + auth.delete_session(request.state.session) + return Response(status_code=204) + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(request: Request): + user = db.get_user(current_user_id(request)) + if user is None: + raise unauthorized("Session is invalid or expired") + return user + + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, chat_data: CreateChat): + user_id = current_user_id(request) + + if user_id not in chat_data.member_ids: + raise bad_request("Creator must be a chat member") + + if len(chat_data.member_ids) != len(set(chat_data.member_ids)): + raise bad_request("Duplicate member IDs") + return db.create_chat(chat_data.title, chat_data.member_ids) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, user_id=user_id) + + rows = db.list_chats(user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id) + + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID): + return db.get_chat(chat_id, current_user_id(request)) + + +def request_fingerprint(data: BaseModel) -> bytes: + """SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют. + + Пустой attachment_ids не входит в отпечаток: {"text": "x"} и {"text": "x", "attachment_ids": []} + равны (контракт), и хеши сообщений, сохранённых до lab4, остаются прежними. + """ + body = data.model_dump(mode="json") + if not body.get("attachment_ids"): + body.pop("attachment_ids", None) + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + response_model=Message, + status_code=201, +) +def create_message( + request: Request, + message_data: CreateMessage, + chat_id: UUID, + idempotency_key: str | None = Header(default=None), +): + # С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику) + # вернёт то же сообщение, а не создаст второе. См. db.add_message. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -") + return db.add_message(chat_id, current_user_id(request), message_data.text, message_data.attachment_ids, + idempotency_key, request_fingerprint(message_data)) + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +def list_messages( + request: Request, + chat_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, chat_id=chat_id) + + rows = db.list_messages(chat_id, user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id) + + return MessagePage(items=items, next_cursor=next_cursor) + + +@app.post( + "/api/v1/chats/{chat_id}/attachments", + response_model=Attachment, + status_code=202, +) +async def upload_attachment( + request: Request, + chat_id: UUID, + filename: str | None = Query(default=None), +): + """Приём PNG/JPEG сырыми байтами. 202 queued — когда исходник в staging (S3), а задание и сообщение + для брокера зафиксированы в PostgreSQL одной транзакцией (attachments.store). Обработку делает worker. + + Порядок проверок: сессия (middleware) -> участник чата (404) -> filename (400) -> Content-Type (415) + -> Content-Length (413) — всё до чтения тела; затем потоковое чтение с лимитом (413), + проверка содержимого (415/413) и запись. + """ + user_id = current_user_id(request) + try: + await run_in_threadpool(db.check_member, chat_id, user_id) + name = uploads.validate_filename(filename) + declared = uploads.declared_type(request.headers.get("content-type")) + uploads.declared_length(request.headers.get("content-length")) + except ApiError: + await uploads.drain(request) # чтобы клиент получил ответ с ошибкой, а не обрыв на отправке + raise + upload = await uploads.receive(request) + try: + image = await run_in_threadpool(uploads.inspect, upload, declared) + row = await run_in_threadpool(attachments.store, chat_id, user_id, name, upload, image) + finally: + upload.file.close() # временный файл удаляется сразу; исходник остаётся только в staging (S3) + return attachments.public(row) + + +@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment) +def get_attachment(request: Request, attachment_id: UUID): + return attachments.public(db.get_attachment(attachment_id, current_user_id(request))) + + +VARIANT_SUFFIX = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp"} + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + request: Request, + attachment_id: UUID, + variant: Literal["original", "thumbnail", "crop", "optimized"] = Query(default="original"), +): + """Байты оригинала или варианта через API (без redirect): сначала участник чата, потом S3. + + queued/processing/failed -> 409 (ещё/уже нечего отдавать); варианта нет у ready -> 404. + """ + row = db.get_attachment(attachment_id, current_user_id(request)) + if row["state"] != "ready": + raise conflict("Attachment is not ready") + if variant == "original": + key, content_type, size, sha256 = row["object_key"], row["content_type"], row["size_bytes"], row["sha256"] + name = row["filename"] + else: + found = [v for v in row["variants"] if v["name"] == variant] + if not found: + raise not_found("Variant not found") + v = found[0] + key, content_type, size, sha256 = v["object_key"], v["content_type"], v["size_bytes"], v["sha256"] + stem = row["filename"].rsplit(".", 1)[0] or "image" + name = f"{stem}.{variant}{VARIANT_SUFFIX[content_type]}" + try: + stream = storage.open_read(key) + except storage.ObjectMissing: + log.error("Object %s of ready attachment %s is missing in S3", variant, row["id"]) + raise unavailable("File is temporarily unavailable") from None + except storage.StorageUnavailable: + raise unavailable("File storage is temporarily unavailable") from None + headers = { + "Content-Length": str(size), + "Content-Disposition": "inline; filename*=UTF-8''" + quote(name, safe=""), + "X-Content-Type-Options": "nosniff", + "Cache-Control": "private, no-store", + "ETag": '"' + sha256 + '"', + } + return StreamingResponse(storage.iter_chunks(stream), media_type=content_type, headers=headers) + + +if __name__ == "__main__": + Server(uvicorn.Config( + app, + host=settings.APP_HOST, + port=settings.APP_PORT, + timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS, + access_log=False, + )).run() diff --git a/migrate.py b/migrate.py new file mode 100644 index 0000000..e95f414 --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +import logging + +import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + +if __name__ == "__main__": + db.wait_and_migrate() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 0000000..8b981d0 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,33 @@ + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 0000000..73f0c84 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 0000000..5770e1d --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,11 @@ +ALTER TABLE messages ADD COLUMN idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); + +ALTER TABLE messages ADD COLUMN request_hash bytea + CHECK (octet_length(request_hash) = 32); + +ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together + CHECK ((idempotency_key IS NULL) = (request_hash IS NULL)); + +ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key + UNIQUE (chat_id, sender_id, idempotency_key); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 0000000..35697dd --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,48 @@ +-- lab4: вложения. Байты лежат в S3, здесь только метаданные и связь с чатом/сообщением. +-- Миграция только добавляет таблицы и ослабляет одно ограничение, API lab3 продолжает работать. + +-- Нужны для составных внешних ключей ниже: вложение и сообщение связываются только в пределах одного чата. +ALTER TABLE messages ADD CONSTRAINT messages_id_chat UNIQUE (id, chat_id); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + owner_id uuid NOT NULL, + filename bytea NOT NULL CHECK (octet_length(filename) BETWEEN 1 AND 1020), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + -- Ключ объекта назначает сервер (originals/<случайный hex>), имя файла в него не входит. + object_key text NOT NULL UNIQUE CHECK (object_key ~ '^originals/[a-f0-9]{32}$'), + -- uploading: строка-намерение записана ДО PUT в S3, клиенту не видна; + -- ready: объект в S3 сохранён, можно читать и прикреплять. + state text NOT NULL CHECK (state IN ('uploading', 'ready')), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (width::bigint * height <= 40000000), + UNIQUE (id, chat_id), + -- владелец на момент загрузки был участником именно этого чата + FOREIGN KEY (chat_id, owner_id) REFERENCES chat_members (chat_id, user_id) +); + +-- janitor ищет старые незавершённые загрузки +CREATE INDEX attachments_uploading ON attachments (created_at) WHERE state = 'uploading'; + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + chat_id uuid NOT NULL, + attachment_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, position), + UNIQUE (message_id, attachment_id), + -- один chat_id в обоих ключах: вложение другого чата прикрепить нельзя даже в обход API + FOREIGN KEY (message_id, chat_id) REFERENCES messages (id, chat_id), + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments (id, chat_id) +); +CREATE INDEX message_attachments_by_attachment ON message_attachments (attachment_id); + +-- С lab4 текст может быть пустым, если есть вложение (проверяет API). +ALTER TABLE messages DROP CONSTRAINT messages_text_check; +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) <= 16000); diff --git a/migrations/005_jobs.sql b/migrations/005_jobs.sql new file mode 100644 index 0000000..0751798 --- /dev/null +++ b/migrations/005_jobs.sql @@ -0,0 +1,133 @@ +-- lab5: асинхронная обработка вложений. Задания, outbox, варианты, журнал событий. +-- +-- Состояния attachments.state: +-- uploading — строка-намерение до записи в staging (как в lab4), клиенту не видна; +-- queued / processing / ready / failed — публичные состояния из контракта. +-- source_key — исходник в staging (staging/; у старых вложений lab4 — originals/); +-- object_key — опубликованный оригинал attachments//v/original, есть только у ready. + +ALTER TABLE attachments DROP CONSTRAINT attachments_state_check; +ALTER TABLE attachments DROP CONSTRAINT attachments_object_key_check; +ALTER TABLE attachments ALTER COLUMN object_key DROP NOT NULL; +ALTER TABLE attachments ADD COLUMN source_key text UNIQUE; +ALTER TABLE attachments ADD COLUMN error_code text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer; + +-- Вложения lab4 были приняты синхронно и не имеют вариантов: переносим их в staging-модель и ставим +-- в очередь (ниже), worker опубликует оригинал в конечный prefix и сделает варианты. +UPDATE attachments SET source_key = object_key, object_key = NULL; +UPDATE attachments SET state = 'queued', updated_at = now() WHERE state = 'ready'; + +ALTER TABLE attachments ADD CONSTRAINT attachments_state_check + CHECK (state IN ('uploading', 'queued', 'processing', 'ready', 'failed')); +ALTER TABLE attachments ADD CONSTRAINT attachments_source_key_check + CHECK (source_key ~ '^(staging|originals)/[a-f0-9]{32}$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_object_key_check + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/original$'); +ALTER TABLE attachments ADD CONSTRAINT attachments_error_code_check + CHECK (error_code ~ '^[a-z_]{1,64}$'); +-- ready <=> опубликованный оригинал; failed <=> код ошибки; до ready исходник обязан быть в staging. +ALTER TABLE attachments ADD CONSTRAINT attachments_ready_has_object + CHECK ((state = 'ready') = (object_key IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_failed_has_code + CHECK ((state = 'failed') = (error_code IS NOT NULL)); +ALTER TABLE attachments ADD CONSTRAINT attachments_pending_has_source + CHECK (state NOT IN ('uploading', 'queued', 'processing') OR source_key IS NOT NULL); + +-- janitor: исходники ready/failed, которые пора удалить из staging +CREATE INDEX attachments_staging ON attachments (updated_at) + WHERE source_key IS NOT NULL AND state IN ('ready', 'failed'); + +-- Варианты. Первичный ключ = ключ эффекта (attachment_id, pipeline_version, variant): +-- повтор job не может создать второй набор вариантов. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + name text NOT NULL CHECK (name IN ('thumbnail', 'crop', 'optimized')), + content_type text NOT NULL CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CHECK (size_bytes >= 1), + width integer NOT NULL CHECK (width >= 1), + height integer NOT NULL CHECK (height >= 1), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL UNIQUE + CHECK (object_key ~ '^attachments/[0-9a-f-]{36}/v[0-9]{1,4}/[a-z]+\.(jpg|png|webp)$'), + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- Задание: одна логическая работа на (вложение, версия pipeline). job_id не меняется при повторах. +-- attempt — управляемая попытка (claim worker'ом); lease_* — аренда processing с fencing-токеном. +CREATE TABLE attachment_jobs ( + job_id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id), + pipeline_version integer NOT NULL CHECK (pipeline_version >= 1), + status text NOT NULL CHECK (status IN ('queued', 'processing', 'ready', 'failed')), + attempt integer NOT NULL DEFAULT 1 CHECK (attempt >= 1), + max_attempts integer NOT NULL CHECK (max_attempts >= 1), + available_at timestamptz NOT NULL DEFAULT now(), + lease_token uuid, + lease_owner text, + lease_until timestamptz, + error_code text CHECK (error_code ~ '^[a-z_]{1,64}$'), + last_error text CHECK (last_error ~ '^[a-z_]{1,64}$'), + replays integer NOT NULL DEFAULT 0, + traceparent text CHECK (traceparent ~ '^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + finished_at timestamptz, + UNIQUE (attachment_id, pipeline_version), + CHECK (attempt <= max_attempts), + CHECK ((status = 'processing') = (lease_token IS NOT NULL AND lease_until IS NOT NULL)), + CHECK ((status = 'failed') = (error_code IS NOT NULL)) +); +CREATE INDEX attachment_jobs_leases ON attachment_jobs (lease_until) WHERE status = 'processing'; +CREATE INDEX attachment_jobs_queued ON attachment_jobs (available_at) WHERE status = 'queued'; +CREATE INDEX attachment_jobs_failed ON attachment_jobs (finished_at) WHERE status = 'failed'; + +-- Журнал (audit): claim, повторы, истечение аренды, дубли, replay. Только коды, без секретов и путей. +CREATE TABLE job_events ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + event text NOT NULL CHECK (event ~ '^[a-z_]{1,64}$'), + detail text CHECK (char_length(detail) <= 200), + actor text NOT NULL CHECK (char_length(actor) BETWEEN 1 AND 200), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX job_events_by_job ON job_events (job_id, id); + +-- Transactional outbox: сообщение для брокера пишется в той же транзакции, что и задание. +-- dispatcher публикует строки с available_at <= now() и отмечает published_at после publisher confirm. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (job_id), + attempt integer NOT NULL, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL DEFAULT now(), + created_at timestamptz NOT NULL DEFAULT now(), + published_at timestamptz, + publish_failures integer NOT NULL DEFAULT 0 +); +CREATE INDEX outbox_pending ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_by_job ON outbox (job_id); + +-- Задания для вложений, принятых в lab4 (миграция = их producer). JOB_MAX_ATTEMPTS по умолчанию 5. +INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, traceparent) +SELECT gen_random_uuid(), id, 1, 'queued', 1, 5, + '00-' || md5(random()::text || id::text) || '-' || substr(md5(id::text), 1, 16) || '-01' +FROM attachments WHERE state = 'queued'; + +INSERT INTO job_events (job_id, attempt, event, detail, actor) +SELECT job_id, 1, 'created', 'lab4_backfill', 'migration 005' FROM attachment_jobs; + +INSERT INTO outbox (job_id, attempt, payload) +SELECT job_id, 1, jsonb_build_object( + 'schema_version', 1, + 'job_id', job_id, + 'type', 'attachment.process.v1', + 'attachment_id', attachment_id, + 'pipeline_version', pipeline_version, + 'attempt', 1, + 'created_at', to_char(now() AT TIME ZONE 'UTC', 'YYYY-MM-DD"T"HH24:MI:SS"Z"'), + 'traceparent', traceparent) +FROM attachment_jobs; diff --git a/ops.py b/ops.py new file mode 100644 index 0000000..ee815fe --- /dev/null +++ b/ops.py @@ -0,0 +1,194 @@ +"""Операторские команды pipeline вложений (запускаются в контейнере dispatcher). + + docker compose exec dispatcher python ops.py status + docker compose exec dispatcher python ops.py failed [--limit N] + docker compose exec dispatcher python ops.py events + docker compose exec dispatcher python ops.py replay [--reason TEXT] + docker compose exec dispatcher python ops.py dlq [--limit N] + docker compose exec dispatcher python ops.py dlq-replay [--limit N] + +Реестр failed — таблица attachment_jobs (status='failed') + журнал job_events: там error_code, +число попыток, время. DLQ (attachments.process.v1.dlq) получает только сообщения, которые нельзя +обработать как сообщение: не по схеме, неизвестный job_id, превышен x-delivery-limit. + +replay безопасен: переводит в queued только job со статусом failed и исходником в staging; ready, +queued и processing не трогает («повторить» не значит исполнить дубль успешной job). Новая попытка +идёт через outbox тем же job_id, эффект защищён ключом (attachment_id, pipeline_version, variant). + +dlq-replay разбирает DLQ: для валидного сообщения смотрит строку job в PostgreSQL (источник истины): +failed -> replay; ready -> сообщение удаляется как no-op; queued/processing -> удаляется, работу +ведёт outbox/аренда (при необходимости dispatcher опубликует заново). Невалидные сообщения +остаются в DLQ для разбора. Вывод — JSON без секретов и ключей S3. +""" +import argparse +import json +import sys +from uuid import UUID + +import psycopg +from psycopg.rows import dict_row + +import broker +import pipeline +import settings + +ACTOR = "ops@" + settings.INSTANCE_ID + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options="-c TimeZone=UTC") + + +def text(value): + if isinstance(value, (UUID,)): + return str(value) + if hasattr(value, "isoformat"): + return pipeline.utc_text(value) + return value + + +def rows_json(rows): + return [{key: text(value) for key, value in row.items()} for row in rows] + + +def cmd_status(conn, _args): + jobs = {r["status"]: r["n"] for r in conn.execute( + "SELECT status, count(*) AS n FROM attachment_jobs GROUP BY status").fetchall()} + outbox = conn.execute( + """SELECT count(*) FILTER (WHERE published_at IS NULL) AS pending, + count(*) FILTER (WHERE published_at IS NULL AND available_at <= now()) AS due, + coalesce(max(publish_failures) FILTER (WHERE published_at IS NULL), 0) AS max_publish_failures, + extract(epoch FROM now() - min(available_at) FILTER (WHERE published_at IS NULL + AND available_at <= now())) AS oldest_due_s + FROM outbox""").fetchone() + leases = conn.execute( + """SELECT count(*) FILTER (WHERE lease_until >= now()) AS active, + count(*) FILTER (WHERE lease_until < now()) AS expired + FROM attachment_jobs WHERE status = 'processing'""").fetchone() + staging = conn.execute("SELECT count(*) AS n FROM attachments WHERE source_key IS NOT NULL").fetchone()["n"] + report = {"jobs": jobs, "outbox": {k: (round(float(v), 1) if k == "oldest_due_s" and v is not None else v) + for k, v in outbox.items()}, + "leases": leases, "attachments_with_staging_source": staging} + try: + connection = broker.connect() + try: + report["broker"] = broker.check_topology(connection.channel()) + finally: + connection.close() + except broker.BrokerError as exc: + report["broker"] = {"error": type(exc).__name__} + return report + + +def cmd_failed(conn, args): + return rows_json(conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.max_attempts, j.error_code, + j.last_error, j.replays, j.finished_at, (a.source_key IS NOT NULL) AS replayable + FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id + WHERE j.status = 'failed' ORDER BY j.finished_at DESC LIMIT %s""", (args.limit,)).fetchall()) + + +def cmd_events(conn, args): + return rows_json(conn.execute( + "SELECT id, attempt, event, detail, actor, created_at FROM job_events WHERE job_id = %s ORDER BY id", + (args.job_id,)).fetchall()) + + +def cmd_replay(conn, args): + try: + return {"replayed": pipeline.replay(conn, args.job_id, ACTOR, args.reason)} + except pipeline.ReplayRefused as exc: + return {"refused": str(exc), "job_id": str(args.job_id)} + + +def dlq_messages(limit, handler): + """Читает до limit сообщений DLQ (basic_get, ручной ack). handler(message|None) -> 'ack' | 'keep'.""" + connection = broker.connect() + results = [] + try: + channel = connection.channel() + keep = [] + for _ in range(limit): + method, properties, body = channel.basic_get(pipeline.DLQ, auto_ack=False) + if method is None: + break + death = (properties.headers or {}).get("x-death") or [{}] + reason = death[0].get("reason") + reason = reason.decode() if isinstance(reason, bytes) else reason + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + message, problem = None, str(exc) + else: + problem = None + verdict, note = handler(message) + results.append({"job_id": message["job_id"] if message else None, "invalid": problem, + "dead_letter_reason": reason, "action": verdict, "note": note}) + if verdict == "ack": + channel.basic_ack(method.delivery_tag) + else: + keep.append(method.delivery_tag) + for tag in keep: # вернуть непрочитанными в DLQ + channel.basic_nack(tag, requeue=True) + finally: + connection.close() + return results + + +def cmd_dlq(_conn, args): + return dlq_messages(args.limit, lambda message: ("keep", "peek")) + + +def cmd_dlq_replay(conn, args): + def handle(message): + if message is None: + return "keep", "invalid message is kept for analysis" + job = conn.execute("SELECT status FROM attachment_jobs WHERE job_id = %s", + (UUID(message["job_id"]),)).fetchone() + if job is None: + return "keep", "unknown job" + if job["status"] == "failed": + try: + pipeline.replay(conn, UUID(message["job_id"]), ACTOR, "dlq replay") + return "ack", "replayed" + except pipeline.ReplayRefused as exc: + return "keep", str(exc) + if job["status"] == "ready": + return "ack", "already ready, no-op" + # queued/processing: работу ведёт PostgreSQL; если сообщения в полёте нет — опубликовать заново + conn.execute("UPDATE attachment_jobs SET updated_at = now() - interval '1 day' WHERE job_id = %s " + "AND status = 'queued'", (UUID(message["job_id"]),)) + return "ack", f"job is {job['status']}, handled by outbox/lease recovery" + return dlq_messages(args.limit, handle) + + +def main(): + parser = argparse.ArgumentParser(description="Attachment pipeline operator commands") + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("status") + p = sub.add_parser("failed") + p.add_argument("--limit", type=int, default=50) + p = sub.add_parser("events") + p.add_argument("job_id", type=UUID) + p = sub.add_parser("replay") + p.add_argument("job_id", type=UUID) + p.add_argument("--reason", default="operator replay") + p = sub.add_parser("dlq") + p.add_argument("--limit", type=int, default=20) + p = sub.add_parser("dlq-replay") + p.add_argument("--limit", type=int, default=100) + args = parser.parse_args() + if args.command == "replay" and (len(args.reason) > 100 or not args.reason.isprintable()): + parser.error("--reason: up to 100 printable characters") + commands = {"status": cmd_status, "failed": cmd_failed, "events": cmd_events, "replay": cmd_replay, + "dlq": cmd_dlq, "dlq-replay": cmd_dlq_replay} + with connect_db() as conn: + result = commands[args.command](conn, args) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/pipeline.py b/pipeline.py new file mode 100644 index 0000000..78b6b5a --- /dev/null +++ b/pipeline.py @@ -0,0 +1,445 @@ +"""Задания обработки вложений: формат сообщения, outbox, переходы состояний. + +Общий код для API (создание задания), dispatcher (публикация, аренды), worker (claim/результат) +и ops.py (replay). Все функции получают открытое соединение psycopg и работают внутри транзакции +вызывающего: переход состояния, запись в outbox и в журнал фиксируются ОДНИМ COMMIT. + +Источник истины — PostgreSQL. Сообщение в брокере — только «разбуди worker для job_id»: что делать, +решает строка attachment_jobs. Поэтому повторная доставка, дубль публикации и устаревшее сообщение +безопасны: worker сверяет состояние job и либо берёт работу, либо подтверждает сообщение как no-op. + +Переходы (contracts/attachment-states.md): + (нет) -> queued enqueue_new вместе с вложением, в транзакции API перед 202 + queued -> processing claim worker, аренда lease_until + fencing-токен lease_token + processing -> queued schedule_retry временная ошибка, остались попытки (backoff + jitter) + recover_lease аренда истекла (worker умер/завис), попытка засчитана + processing -> ready finish_ready только при совпадении lease_token (fencing) + processing -> failed finish_failed постоянная ошибка или попытки исчерпаны + failed -> queued replay явная операция оператора (ops.py), с записью в журнал +""" +import json +import random +import re +import secrets +from dataclasses import dataclass +from datetime import datetime, timezone +from uuid import UUID, uuid4 + +from psycopg.types.json import Jsonb + +import settings + +SCHEMA_VERSION = 1 +JOB_TYPE = "attachment.process.v1" +EXCHANGE = "attachments" +ROUTING_KEY = "attachment.process.v1" +QUEUE = "attachments.process.v1" +DLX = "attachments.dlx" +DLQ = "attachments.process.v1.dlq" +DLQ_ROUTING_KEY = "attachment.process.v1.dlq" + +MESSAGE_FIELDS = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", + "created_at", "traceparent", "tracestate"} +REQUIRED_FIELDS = MESSAGE_FIELDS - {"traceparent", "tracestate"} +TRACEPARENT_RE = re.compile(r"^00-[0-9a-f]{32}-[0-9a-f]{16}-[0-9a-f]{2}$") +UUID_RE = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$") +CODE_RE = re.compile(r"^[a-z_]{1,64}$") + +# Коды ошибок. Наружу (Attachment.error_code) уходят только финальные коды из FINAL_CODES. +PERMANENT = {"invalid_image", "image_too_large", "resource_limit", "source_missing", "source_corrupted"} +FINAL_CODES = PERMANENT | {"retry_exhausted"} + + +class InvalidMessage(Exception): + """Сообщение не соответствует contracts/jobs.schema.json -> reject без requeue -> DLQ.""" + + +def utc_text(moment: datetime) -> str: + return moment.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def new_traceparent() -> str: + """W3C trace context: trace-id на всё задание, span-id новый для каждой публикации.""" + return f"00-{secrets.token_hex(16)}-{secrets.token_hex(8)}-01" + + +def child_traceparent(parent: str | None) -> str: + if parent and TRACEPARENT_RE.match(parent): + return f"00-{parent[3:35]}-{secrets.token_hex(8)}-01" + return new_traceparent() + + +def build_message(job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str | None) -> dict: + """Тело сообщения по jobs.schema.json. Только ID и служебные поля: ни байтов, ни URL, ни ключей S3.""" + message = { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": utc_text(datetime.now(timezone.utc)), + } + if traceparent: + message["traceparent"] = child_traceparent(traceparent) + return message + + +def parse_message(body: bytes) -> dict: + """Строгая проверка по схеме (additionalProperties: false, const, типы, форматы).""" + try: + message = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise InvalidMessage("not json") from None + if not isinstance(message, dict): + raise InvalidMessage("not an object") + if set(message) - MESSAGE_FIELDS or REQUIRED_FIELDS - set(message): + raise InvalidMessage("fields") + if message["schema_version"] != SCHEMA_VERSION or message["type"] != JOB_TYPE: + raise InvalidMessage("schema_version/type") + for name in ("job_id", "attachment_id"): + if not isinstance(message[name], str) or not UUID_RE.match(message[name]): + raise InvalidMessage(name) + for name in ("pipeline_version", "attempt"): + value = message[name] + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise InvalidMessage(name) + try: + created = message["created_at"] + if not isinstance(created, str): + raise ValueError + datetime.fromisoformat(created.replace("Z", "+00:00")) + except ValueError: + raise InvalidMessage("created_at") from None + if "traceparent" in message and (not isinstance(message["traceparent"], str) + or not TRACEPARENT_RE.match(message["traceparent"])): + raise InvalidMessage("traceparent") + if "tracestate" in message and (not isinstance(message["tracestate"], str) or len(message["tracestate"]) > 512): + raise InvalidMessage("tracestate") + return message + + +def backoff_seconds(failed_attempt: int) -> float: + """Экспоненциальная задержка с jitter («equal jitter»): половина фиксирована, половина случайна. + + Попытка 1 -> 1..2 с, 2 -> 2..4 с, 3 -> 4..8 с, ... не больше JOB_BACKOFF_MAX_SECONDS. + Случайная половина разводит повторы многих заданий после общего сбоя (S3 вернулся — не все сразу). + """ + ceiling = min(settings.JOB_BACKOFF_MAX_SECONDS, + settings.JOB_BACKOFF_BASE_SECONDS * 2 ** (max(failed_attempt, 1) - 1)) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +def event(conn, job_id: UUID, attempt: int, name: str, actor: str, detail: str | None = None) -> None: + conn.execute("INSERT INTO job_events (job_id, attempt, event, detail, actor) VALUES (%s, %s, %s, %s, %s)", + (job_id, attempt, name, detail[:200] if detail else None, actor[:200])) + + +def add_outbox(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, attempt: int, + traceparent: str | None, delay_seconds: float = 0) -> None: + """Сообщение в outbox (та же транзакция) + NOTIFY: dispatcher просыпается сразу после COMMIT.""" + payload = build_message(job_id, attachment_id, pipeline_version, attempt, traceparent) + conn.execute( + """INSERT INTO outbox (job_id, attempt, payload, available_at) + VALUES (%s, %s, %s, now() + make_interval(secs => %s))""", + (job_id, attempt, Jsonb(payload), delay_seconds), + ) + conn.execute("NOTIFY outbox") + + +def enqueue_new(conn, attachment_id: UUID, actor: str) -> UUID: + """Новое задание для вложения, уже переведённого в queued (в транзакции API).""" + job_id = uuid4() + traceparent = new_traceparent() + conn.execute( + """INSERT INTO attachment_jobs (job_id, attachment_id, pipeline_version, status, attempt, max_attempts, + traceparent) + VALUES (%s, %s, %s, 'queued', 1, %s, %s)""", + (job_id, attachment_id, settings.PIPELINE_VERSION, settings.JOB_MAX_ATTEMPTS, traceparent), + ) + event(conn, job_id, 1, "created", actor) + add_outbox(conn, job_id, attachment_id, settings.PIPELINE_VERSION, 1, traceparent) + return job_id + + +# ---------- worker: захват и результат ---------- + +@dataclass +class Claim: + job_id: UUID + attachment_id: UUID + pipeline_version: int + attempt: int + max_attempts: int + lease_token: UUID + source_key: str + content_type: str + size_bytes: int + sha256: str + traceparent: str | None + + +@dataclass +class Skip: + """Сообщение подтверждается без работы (дубль, ещё не время, работа у другого worker).""" + reason: str + + +@dataclass +class Reject: + """Сообщение не относится ни к одной job -> DLQ.""" + reason: str + + +def claim(conn, message: dict, owner: str, redelivered: bool): + """queued -> processing (или перехват processing с истёкшей арендой). Возвращает Claim | Skip | Reject.""" + job_id = UUID(message["job_id"]) + with conn.transaction(): + job = conn.execute( + """SELECT j.*, (j.available_at <= now()) AS due, (j.lease_until < now()) AS lease_expired + FROM attachment_jobs j WHERE j.job_id = %s FOR UPDATE""", + (job_id,), + ).fetchone() + if job is None: + return Reject("unknown_job") + if str(job["attachment_id"]) != message["attachment_id"] or job["pipeline_version"] != message["pipeline_version"]: + return Reject("job_mismatch") + status = job["status"] + if status in ("ready", "failed"): + event(conn, job_id, job["attempt"], "duplicate_ignored", owner, f"status={status}") + return Skip(f"already_{status}") + attempt = job["attempt"] + if status == "queued": + if not job["due"]: + return Skip("not_due") # устаревшее сообщение прежней попытки; своё придёт из outbox + else: # processing + if not job["lease_expired"]: + event(conn, job_id, attempt, "duplicate_ignored", owner, "leased") + return Skip("leased") + # Прежний исполнитель не продлил аренду (умер/завис): его попытка засчитана. + event(conn, job_id, attempt, "lease_expired", owner, "taken_over_by_worker") + if attempt >= job["max_attempts"]: + _fail(conn, job_id, job["attachment_id"], attempt, "retry_exhausted", owner, "lease_expired") + return Skip("exhausted") + attempt += 1 + token = uuid4() + conn.execute( + """UPDATE attachment_jobs + SET status = 'processing', attempt = %s, lease_token = %s, lease_owner = %s, + lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (attempt, token, owner, settings.JOB_LEASE_SECONDS, job_id), + ) + att = conn.execute( + """UPDATE attachments SET state = 'processing', updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing') + RETURNING source_key, content_type, size_bytes, sha256""", + (job["attachment_id"],), + ).fetchone() + if att is None or att["source_key"] is None: + raise RuntimeError("attachment is not in a processable state") # нарушен инвариант: откат + event(conn, job_id, attempt, "claimed", owner, + f"message_attempt={message['attempt']} redelivered={str(redelivered).lower()}") + return Claim(job_id, job["attachment_id"], job["pipeline_version"], attempt, job["max_attempts"], token, + att["source_key"], att["content_type"], att["size_bytes"], att["sha256"], job["traceparent"]) + + +def renew_lease(conn, c: Claim) -> bool: + """Продление аренды. False — аренду уже забрали (истекла и перехвачена): работу надо бросить.""" + row = conn.execute( + """UPDATE attachment_jobs SET lease_until = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s AND lease_token = %s AND status = 'processing' RETURNING 1""", + (settings.JOB_LEASE_SECONDS, c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def _fenced(conn, c: Claim, owner: str) -> bool: + """Fencing: результат принимается, только если lease_token всё ещё наш.""" + row = conn.execute( + """SELECT 1 FROM attachment_jobs WHERE job_id = %s AND lease_token = %s AND status = 'processing' + FOR UPDATE""", + (c.job_id, c.lease_token), + ).fetchone() + return row is not None + + +def finish_ready(conn, c: Claim, original_key: str, variants: list[dict], owner: str) -> bool: + """processing -> ready атомарно: варианты + оригинал + состояние job/вложения одним COMMIT.""" + with conn.transaction(): + if not _fenced(conn, c, owner): + stale = True + else: + stale = False + for v in variants: + conn.execute( + """INSERT INTO attachment_variants (attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, object_key) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING""", + (c.attachment_id, c.pipeline_version, v["name"], v["content_type"], v["size_bytes"], + v["width"], v["height"], v["sha256"], v["object_key"]), + ) + # Не откатываем более новую версию pipeline старой (трек 5; в курсе версия одна). + updated = conn.execute( + """UPDATE attachments + SET state = 'ready', object_key = %s, pipeline_version = %s, error_code = NULL, updated_at = now() + WHERE id = %s AND state = 'processing' + AND (pipeline_version IS NULL OR pipeline_version <= %s) + RETURNING id""", + (original_key, c.pipeline_version, c.attachment_id, c.pipeline_version), + ).fetchone() + if updated is None: + raise RuntimeError("attachment changed state during processing") + conn.execute( + """UPDATE attachment_jobs + SET status = 'ready', lease_token = NULL, lease_owner = NULL, lease_until = NULL, + error_code = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s""", + (c.job_id,), + ) + event(conn, c.job_id, c.attempt, "ready", owner) + if stale: + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, "lease_lost") + return not stale + + +def _fail(conn, job_id: UUID, attachment_id: UUID, attempt: int, code: str, actor: str, detail: str | None): + conn.execute( + """UPDATE attachment_jobs + SET status = 'failed', error_code = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, finished_at = now(), updated_at = now() + WHERE job_id = %s""", + (code, code if detail is None else (detail if CODE_RE.match(detail) else code), job_id), + ) + conn.execute( + """UPDATE attachments SET state = 'failed', error_code = %s, updated_at = now() + WHERE id = %s AND state IN ('queued', 'processing')""", + (code, attachment_id), + ) + event(conn, job_id, attempt, "failed", actor, code if detail is None else f"{code} after {detail}") + + +def _requeue(conn, job_id: UUID, attachment_id: UUID, pipeline_version: int, next_attempt: int, + traceparent: str | None, delay: float, reason: str, actor: str, event_name: str) -> None: + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, last_error = %s, lease_token = NULL, lease_owner = NULL, + lease_until = NULL, available_at = now() + make_interval(secs => %s), updated_at = now() + WHERE job_id = %s""", + (next_attempt, reason, delay, job_id), + ) + conn.execute("UPDATE attachments SET state = 'queued', updated_at = now() WHERE id = %s AND state = 'processing'", + (attachment_id,)) + add_outbox(conn, job_id, attachment_id, pipeline_version, next_attempt, traceparent, delay) + event(conn, job_id, next_attempt - 1, event_name, actor, f"{reason} delay={delay:.1f}s") + + +def finish_error(conn, c: Claim, code: str, permanent: bool, owner: str) -> str: + """Ошибка попытки. Возвращает 'retry' | 'failed' | 'stale'.""" + with conn.transaction(): + if not _fenced(conn, c, owner): + outcome = "stale" + elif permanent: + _fail(conn, c.job_id, c.attachment_id, c.attempt, code, owner, None) + outcome = "failed" + elif c.attempt >= c.max_attempts: + _fail(conn, c.job_id, c.attachment_id, c.attempt, "retry_exhausted", owner, code) + outcome = "failed" + else: + _requeue(conn, c.job_id, c.attachment_id, c.pipeline_version, c.attempt + 1, c.traceparent, + backoff_seconds(c.attempt), code, owner, "retry_scheduled") + outcome = "retry" + if outcome == "stale": + with conn.transaction(): + event(conn, c.job_id, c.attempt, "stale_result_rejected", owner, code) + return outcome + + +# ---------- dispatcher: аренды и сверка ---------- + +def recover_expired_leases(conn, actor: str, limit: int = 100) -> int: + """processing с истёкшей арендой -> queued (попытка засчитана) или failed(retry_exhausted).""" + recovered = 0 + with conn.transaction(): + rows = conn.execute( + """SELECT job_id, attachment_id, pipeline_version, attempt, max_attempts, traceparent + FROM attachment_jobs WHERE status = 'processing' AND lease_until < now() + ORDER BY lease_until LIMIT %s FOR UPDATE SKIP LOCKED""", + (limit,), + ).fetchall() + for job in rows: + if job["attempt"] >= job["max_attempts"]: + _fail(conn, job["job_id"], job["attachment_id"], job["attempt"], "retry_exhausted", actor, + "lease_expired") + else: + _requeue(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"] + 1, + job["traceparent"], backoff_seconds(job["attempt"]), "lease_expired", actor, + "lease_expired") + recovered += 1 + return recovered + + +def reconcile_lost_messages(conn, actor: str, older_than: int, limit: int = 100) -> int: + """Страховка: queued-задание давно «созрело», но неопубликованного сообщения для него нет. + + Так бывает, если сообщение пропало в брокере (очередь удалили/потеряли volume) или его + подтвердили как устаревшее. Публикуем новое: дубль безопасен, потеря — нет. + """ + with conn.transaction(): + rows = conn.execute( + """SELECT j.job_id, j.attachment_id, j.pipeline_version, j.attempt, j.traceparent + FROM attachment_jobs j + WHERE j.status = 'queued' AND j.available_at < now() - make_interval(secs => %s) + AND j.updated_at < now() - make_interval(secs => %s) + AND NOT EXISTS (SELECT 1 FROM outbox o WHERE o.job_id = j.job_id AND + (o.published_at IS NULL OR o.published_at > now() - make_interval(secs => %s))) + ORDER BY j.available_at LIMIT %s FOR UPDATE OF j SKIP LOCKED""", + (older_than, older_than, older_than, limit), + ).fetchall() + for job in rows: + add_outbox(conn, job["job_id"], job["attachment_id"], job["pipeline_version"], job["attempt"], + job["traceparent"]) + conn.execute("UPDATE attachment_jobs SET updated_at = now() WHERE job_id = %s", (job["job_id"],)) + event(conn, job["job_id"], job["attempt"], "republished", actor, "no_message_in_flight") + return len(rows) + + +# ---------- оператор: replay ---------- + +class ReplayRefused(Exception): + pass + + +def replay(conn, job_id: UUID, actor: str, reason: str) -> dict: + """failed -> queued. Новая попытка того же job_id; ready/queued/processing не трогаются.""" + with conn.transaction(): + job = conn.execute( + """SELECT j.*, a.source_key, a.state AS attachment_state FROM attachment_jobs j + JOIN attachments a ON a.id = j.attachment_id WHERE j.job_id = %s FOR UPDATE OF j, a""", + (job_id,), + ).fetchone() + if job is None: + raise ReplayRefused("job not found") + if job["status"] != "failed": + # «Повторить» не означает исполнить дубль: успешная или ещё идущая job не перезапускается. + raise ReplayRefused(f"job is {job['status']}, replay is only for failed jobs") + if job["source_key"] is None: + raise ReplayRefused("source file is no longer in staging (retention expired)") + attempt = job["attempt"] + 1 + conn.execute( + """UPDATE attachment_jobs + SET status = 'queued', attempt = %s, max_attempts = %s, error_code = NULL, + available_at = now(), replays = replays + 1, finished_at = NULL, updated_at = now() + WHERE job_id = %s""", + (attempt, attempt + settings.JOB_MAX_ATTEMPTS - 1, job_id), + ) + conn.execute( + "UPDATE attachments SET state = 'queued', error_code = NULL, updated_at = now() WHERE id = %s AND state = 'failed'", + (job["attachment_id"],), + ) + event(conn, job_id, attempt, "replayed", actor, reason or "operator replay") + add_outbox(conn, job_id, job["attachment_id"], job["pipeline_version"], attempt, job["traceparent"]) + return {"job_id": str(job_id), "attachment_id": str(job["attachment_id"]), "attempt": attempt} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..d213edc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,30 @@ +fastapi==0.141.1 +uvicorn==0.54.0 +pydantic==2.13.5 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==6.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +pycparser==3.1 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +boto3==1.43.116 +botocore==1.43.116 +s3transfer==0.19.3 +jmespath==1.1.0 +python-dateutil==2.9.0.post0 +six==1.17.0 +urllib3==2.8.0 +pillow==12.3.0 +pika==1.3.2 diff --git a/s3_init.py b/s3_init.py new file mode 100644 index 0000000..99489bc --- /dev/null +++ b/s3_init.py @@ -0,0 +1,103 @@ +"""Bootstrap S3 (SeaweedFS): приватный bucket и проверка прав. Запускается одноразовым сервисом s3-init. + +Идемпотентен: повторный запуск (docker compose run --rm s3-init) ничего не ломает. +Ключи и их права заданы статически в secrets/s3_config.json (scripts/init_secrets.py): + admin — Admin/Read/Write/List на всё, есть только у s3 и s3-init; + messenger-api — Read/Write/List только на bucket S3_BUCKET, его получают api и janitor. +Значения ключей не печатаются. +""" +import logging +import sys +import time +import urllib.error +import urllib.request +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +logging.basicConfig(level=logging.WARNING) + + +def client(key_id: str, secret: str): + return boto3.client( + "s3", endpoint_url=settings.S3_ENDPOINT, region_name=settings.S3_REGION, + aws_access_key_id=key_id, aws_secret_access_key=secret, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, connect_timeout=2, + read_timeout=10, retries={"total_max_attempts": 1}, + request_checksum_calculation="when_required", response_checksum_validation="when_required"), + ) + + +def code(exc: ClientError) -> str: + return exc.response.get("Error", {}).get("Code", "?") + + +def anonymous_status(url: str, method: str = "GET") -> int: + try: + with urllib.request.urlopen(urllib.request.Request(url, method=method), timeout=5) as reply: + return reply.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + bucket = settings.S3_BUCKET + admin = client(settings.read_secret("S3_ADMIN_ACCESS_KEY_ID_FILE", "S3_ADMIN_ACCESS_KEY_ID"), + settings.read_secret("S3_ADMIN_SECRET_ACCESS_KEY_FILE", "S3_ADMIN_SECRET_ACCESS_KEY")) + api = client(settings.S3_ACCESS_KEY_ID, settings.S3_SECRET_ACCESS_KEY) + + deadline = time.monotonic() + 90 + while True: + try: + admin.list_buckets() + break + except (BotoCoreError, ClientError) as exc: + if time.monotonic() > deadline: + print(f"s3-init: S3 is not ready: {type(exc).__name__}") + return 1 + time.sleep(1) + + try: + admin.create_bucket(Bucket=bucket) + print(f"s3-init: bucket {bucket} created") + except ClientError as exc: + if code(exc) not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + print(f"s3-init: cannot create bucket: {code(exc)}") + return 1 + print(f"s3-init: bucket {bucket} exists") + + # Ключ API работает со своим bucket ... + probe = "originals/s3-init-probe-" + uuid.uuid4().hex + api.put_object(Bucket=bucket, Key=probe, Body=b"probe") + api.get_object(Bucket=bucket, Key=probe)["Body"].read() + api.list_objects_v2(Bucket=bucket, MaxKeys=1) + api.delete_object(Bucket=bucket, Key=probe) + print("s3-init: api key can put/get/list/delete objects of its bucket") + # ... и не может администрировать хранилище. + intruder = "s3-init-intruder-" + uuid.uuid4().hex[:8] + try: + api.create_bucket(Bucket=intruder) + admin.delete_bucket(Bucket=intruder) + print("s3-init: api key must NOT be able to create buckets") + return 1 + except ClientError as exc: + print(f"s3-init: api key cannot create buckets ({code(exc)})") + + # Приватность: анонимные запросы без подписи. + base = settings.S3_ENDPOINT.rstrip("/") + "/" + bucket + statuses = {"list": anonymous_status(base + "?list-type=2"), + "get": anonymous_status(base + "/originals/any"), + "put": anonymous_status(base + "/originals/any", "PUT")} + if any(status not in (401, 403) for status in statuses.values()): + print(f"s3-init: bucket is NOT private, anonymous statuses: {statuses}") + return 1 + print(f"s3-init: bucket is private (anonymous list/get/put -> {sorted(set(statuses.values()))})") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 0000000..0db2b0a --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,153 @@ +import base64 +import hashlib +import json +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + with open(SECRETS / name, "w", encoding="utf-8", newline="") as file: + file.write(value) + (SECRETS / name).chmod(0o644) # контейнеры читают секрет не от root; папка secrets/ в .gitignore + + +def bucket_name() -> str: + """S3_BUCKET из .env (если задан), иначе значение по умолчанию.""" + env = ROOT / ".env" + if env.exists(): + for line in env.read_text(encoding="utf-8").splitlines(): + if line.strip().startswith("S3_BUCKET="): + return line.split("=", 1)[1].strip() + return "messenger-attachments" + + +def s3_config(bucket: str) -> str: + """Конфиг ключей SeaweedFS: admin — только для s3-init, messenger-api — один bucket. + + Права SeaweedFS: Read (GetObject/HeadObject), Write (PutObject/DeleteObject), List (ListObjects) + с суффиксом : действуют только в этом bucket. Admin (создание/удаление bucket и т.п.) у API нет. + Identity "anonymous" не объявлена, поэтому запросы без подписи отклоняются. + """ + def read(name): + return (SECRETS / name).read_text(encoding="utf-8").strip() + config = {"identities": [ + {"name": "admin", + "credentials": [{"accessKey": read("s3_root_user"), "secretKey": read("s3_root_password")}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"]}, + {"name": "messenger-api", + "credentials": [{"accessKey": read("s3_access_key_id"), "secretKey": read("s3_secret_access_key")}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"]}, + ]} + return json.dumps(config, indent=2) + + +BROKER_VHOST = "messenger" +# Имена из pipeline.py (скрипт не импортирует код приложения, чтобы работать без зависимостей). +EXCHANGE, QUEUE, ROUTING_KEY = "attachments", "attachments.process.v1", "attachment.process.v1" +DLX, DLQ, DLQ_ROUTING_KEY = "attachments.dlx", "attachments.process.v1.dlq", "attachment.process.v1.dlq" + + +def rabbit_hash(password: str) -> str: + """Формат rabbit_password_hashing_sha256: base64(salt(4 байта) + SHA-256(salt + пароль)).""" + salt = secrets.token_bytes(4) + return base64.b64encode(salt + hashlib.sha256(salt + password.encode("utf-8")).digest()).decode("ascii") + + +def rabbit_definitions(passwords: dict) -> str: + """Топология и пользователи RabbitMQ; брокер загружает файл при каждом старте (infra/rabbitmq.conf). + + Права (configure / write / read, регулярные выражения по именам ресурсов): + dispatcher — публикует только в exchange attachments; читает DLQ (ops.py dlq-replay); configure нет; + worker — читает только основную очередь; не публикует (повторы идут через outbox); configure нет. + Ни у кого из сервисов нет тега administrator и права создать/удалить очередь. + """ + queue_args = { + "x-queue-type": "quorum", + "x-delivery-limit": 20, # ядовитое сообщение не крутится бесконечно -> DLQ + "x-dead-letter-exchange": DLX, + "x-dead-letter-routing-key": DLQ_ROUTING_KEY, + "x-dead-letter-strategy": "at-least-once", # перенос в DLQ без потери (требует reject-publish) + "x-overflow": "reject-publish", # переполнение -> nack издателю, outbox повторит + "x-max-length": 100000, + } + definitions = { + "users": [{"name": name, "password_hash": rabbit_hash(password), + "hashing_algorithm": "rabbit_password_hashing_sha256", "tags": []} + for name, password in passwords.items()], + "vhosts": [{"name": BROKER_VHOST}], + "permissions": [ + {"user": "dispatcher", "vhost": BROKER_VHOST, "configure": "^$", + "write": f"^{EXCHANGE}$", "read": "^" + DLQ.replace(".", r"\.") + "$"}, + {"user": "worker", "vhost": BROKER_VHOST, "configure": "^$", + "write": "^$", "read": "^" + QUEUE.replace(".", r"\.") + "$"}, + ], + "exchanges": [ + {"name": EXCHANGE, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + {"name": DLX, "vhost": BROKER_VHOST, "type": "direct", "durable": True, "auto_delete": False, + "internal": False, "arguments": {}}, + ], + "queues": [ + {"name": QUEUE, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, "arguments": queue_args}, + # DLQ — classic durable: у quorum в 4.x delivery-limit по умолчанию 20, и многократный + # просмотр (basic_get + nack) в ops.py мог бы выбросить сообщение. Здесь потерь от просмотра нет. + {"name": DLQ, "vhost": BROKER_VHOST, "durable": True, "auto_delete": False, + "arguments": {"x-queue-type": "classic"}}, + ], + "bindings": [ + {"source": EXCHANGE, "vhost": BROKER_VHOST, "destination": QUEUE, "destination_type": "queue", + "routing_key": ROUTING_KEY, "arguments": {}}, + {"source": DLX, "vhost": BROKER_VHOST, "destination": DLQ, "destination_type": "queue", + "routing_key": DLQ_ROUTING_KEY, "arguments": {}}, + ], + } + return json.dumps(definitions, indent=2) + + +def broker_secrets(created: list, host: str = "rabbitmq") -> None: + if (SECRETS / "rabbitmq_definitions.json").exists(): + return + passwords = {"dispatcher": secrets.token_hex(24), "worker": secrets.token_hex(24)} + for name, password in passwords.items(): + write(f"broker_url_{name}", f"amqp://{name}:{password}@{host}:5672/{BROKER_VHOST}") + write("rabbitmq_definitions.json", rabbit_definitions(passwords)) + created += ["broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions.json"] + + +def main() -> None: + """Создаёт недостающие секреты. Уже существующие не меняет (иначе разошлись бы с данными в volumes).""" + SECRETS.mkdir(exist_ok=True) + created = [] + if not (SECRETS / "postgres_password").exists(): + db_password = secrets.token_hex(24) + redis_password = secrets.token_hex(24) + write("postgres_password", db_password) + write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger") + write("cache_url", f"redis://:{redis_password}@redis:6379/0") + write("redis_auth", f"requirepass {redis_password}\n") + created += ["postgres_password", "database_url", "cache_url", "redis_auth"] + if not (SECRETS / "s3_root_password").exists(): + # admin-ключ S3: только для сервисов s3 (в конфиге) и s3-init + write("s3_root_user", "root-" + secrets.token_hex(6)) + write("s3_root_password", secrets.token_hex(24)) + created += ["s3_root_user", "s3_root_password"] + if not (SECRETS / "s3_secret_access_key").exists(): + # отдельный ключ API с политикой только на свой bucket (создаёт s3-init) + write("s3_access_key_id", "api-" + secrets.token_hex(8)) + write("s3_secret_access_key", secrets.token_hex(24)) + created += ["s3_access_key_id", "s3_secret_access_key"] + if not (SECRETS / "s3_config.json").exists(): + write("s3_config.json", s3_config(bucket_name())) + created.append("s3_config.json") + broker_secrets(created) + if created: + print("Created in secrets/: " + ", ".join(created) + " (values are not printed).") + else: + print("secrets/ is complete, nothing changed.") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 0000000..6a5dfb4 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,14 @@ +@echo off +rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ), then api, janitor, dispatcher, worker, +rem then the balancer. Volumes are kept. +rem "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +rem ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +cd /d "%~dp0.." +docker compose restart postgres redis s3 rabbitmq || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq || exit /b 1 +docker compose restart api janitor dispatcher worker || exit /b 1 +docker compose restart lb || exit /b 1 +docker compose up -d --wait --no-recreate --no-build --no-deps api lb dispatcher worker || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 0000000..98d2561 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Restart hook for tests/resilience.py --mode persistence (Linux/macOS). +# Restarts the stores first (PostgreSQL, Redis, SeaweedFS S3, RabbitMQ), then api, janitor, dispatcher and +# worker, then the balancer. Volumes are kept (no down -v), nothing is re-seeded. +# "restart" does not wait for healthchecks; SeaweedFS needs a few seconds to bring up S3, so wait explicitly +# ("up --wait --no-recreate" keeps the same containers and only waits until they are healthy). +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres redis s3 rabbitmq +docker compose up -d --wait --no-recreate --no-build --no-deps postgres redis s3 rabbitmq +docker compose restart api janitor dispatcher worker +docker compose restart lb +docker compose up -d --wait --no-recreate --no-build --no-deps api lb dispatcher worker diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 0000000..8ffd0fa --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py. +python "%~dp0stop_one.py" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 0000000..cf4d724 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID. +# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd. +set -eu +exec python3 "$(dirname "$0")/stop_one.py" diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 0000000..5cdcce7 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STOP_TIMEOUT_SECONDS = 8 + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{12,64}", target): + print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr) + return 2 + running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + matches = [container for container in running if container.startswith(target)] + if len(matches) != 1: + print(f"no running api container with id {target}", file=sys.stderr) + return 1 + subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + print(f"stopped api container {matches[0][:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/worker.cmd b/scripts/worker.cmd new file mode 100644 index 0000000..9ffab68 --- /dev/null +++ b/scripts/worker.cmd @@ -0,0 +1,8 @@ +@echo off +rem Demo hook for the defense on Windows: scripts\worker.cmd stop ^| start ^| status (volumes are kept). +cd /d "%~dp0.." +if "%1"=="stop" ( docker compose stop worker & exit /b %ERRORLEVEL% ) +if "%1"=="start" ( docker compose up -d --wait --no-recreate --no-build --no-deps worker & exit /b %ERRORLEVEL% ) +if "%1"=="status" ( docker compose ps worker & exit /b %ERRORLEVEL% ) +echo usage: %0 stop^|start^|status +exit /b 2 diff --git a/scripts/worker.sh b/scripts/worker.sh new file mode 100644 index 0000000..cab7c76 --- /dev/null +++ b/scripts/worker.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Demo hook for the defense: stop or start the worker without touching volumes. +# scripts/worker.sh stop | start | status +set -eu +cd "$(dirname "$0")/.." +case "${1:-}" in + stop) docker compose stop worker ;; + start) docker compose up -d --wait --no-recreate --no-build --no-deps worker ;; + status) docker compose ps worker ;; + *) echo "usage: $0 stop|start|status" >&2; exit 2 ;; +esac diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..e4cf3eb --- /dev/null +++ b/settings.py @@ -0,0 +1,111 @@ +import os +import socket +from pathlib import Path + + +def env_str(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def env_int(name: str, default: int) -> int: + value = int(env_str(name, str(default))) + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value + + +def read_secret(file_variable: str, plain_variable: str) -> str: + path = env_str(file_variable, "") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return env_str(plain_variable, "") + + +APP_HOST = env_str("APP_HOST", "0.0.0.0") +APP_PORT = env_int("APP_PORT", 8080) +APP_ENV = env_str("APP_ENV", "lab") +APP_VERSION = env_str("APP_VERSION", "dev") + +INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname() + +SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600) +SESSION_STORE = env_str("SESSION_STORE", "redis") + +DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL") +DB_POOL_MIN = env_int("DB_POOL_MIN", 1) +DB_POOL_MAX = env_int("DB_POOL_MAX", 10) +DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3) +DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3) +DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + +REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL") +REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20) +REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500) + +MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations"))) +SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60) + +# S3 (SeaweedFS): адрес, bucket и отдельный ключ API с правами только на этот bucket. +S3_ENDPOINT = env_str("S3_ENDPOINT", "http://s3:8333") +S3_REGION = env_str("S3_REGION", "us-east-1") +S3_BUCKET = env_str("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") +S3_CONNECT_TIMEOUT_SECONDS = env_int("S3_CONNECT_TIMEOUT_SECONDS", 1) +S3_READ_TIMEOUT_SECONDS = env_int("S3_READ_TIMEOUT_SECONDS", 5) +S3_MAX_ATTEMPTS = env_int("S3_MAX_ATTEMPTS", 2) +S3_POOL_MAX = env_int("S3_POOL_MAX", 20) + +# Вложения: лимиты из контракта и защита памяти/времени. +MAX_UPLOAD_BYTES = env_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024) +MAX_IMAGE_PIXELS = env_int("MAX_IMAGE_PIXELS", 40_000_000) +UPLOAD_TIMEOUT_SECONDS = env_int("UPLOAD_TIMEOUT_SECONDS", 60) +UPLOAD_IDLE_TIMEOUT_SECONDS = env_int("UPLOAD_IDLE_TIMEOUT_SECONDS", 10) +DECODE_CONCURRENCY = env_int("DECODE_CONCURRENCY", 2) +UPLOAD_TMP_DIR = env_str("UPLOAD_TMP_DIR", "/tmp/uploads") + +# Очистка незавершённых загрузок (janitor.py). +ATTACHMENT_GRACE_SECONDS = env_int("ATTACHMENT_GRACE_SECONDS", 3600) +JANITOR_INTERVAL_SECONDS = env_int("JANITOR_INTERVAL_SECONDS", 300) + +SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3) +SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10) + +# lab5: брокер (RabbitMQ) и фоновая обработка вложений. +# У API адреса брокера нет вообще: API пишет задание в outbox (PostgreSQL), публикует dispatcher. +BROKER_URL = read_secret("BROKER_URL_FILE", "BROKER_URL") +BROKER_HEARTBEAT_SECONDS = env_int("BROKER_HEARTBEAT_SECONDS", 30) +PIPELINE_VERSION = env_int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = env_int("WORKER_CONCURRENCY", 2) # потоков-потребителей в одном worker +WORKER_PREFETCH = env_int("WORKER_PREFETCH", 1) # неподтверждённых сообщений на поток +JOB_MAX_ATTEMPTS = env_int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = env_int("JOB_TIMEOUT_SECONDS", 60) # жёсткий предел одной обработки изображения +JOB_LEASE_SECONDS = env_int("JOB_LEASE_SECONDS", 15) # аренда processing, продлевается каждые lease/3 +JOB_BACKOFF_BASE_SECONDS = env_int("JOB_BACKOFF_BASE_SECONDS", 2) +JOB_BACKOFF_MAX_SECONDS = env_int("JOB_BACKOFF_MAX_SECONDS", 60) +JOB_RECONCILE_SECONDS = env_int("JOB_RECONCILE_SECONDS", 120) # queued без сообщения дольше -> опубликовать заново +WORKER_MEMORY_LIMIT_MB = env_int("WORKER_MEMORY_LIMIT_MB", 640) # RLIMIT_AS процесса обработки +WORKER_TMP_DIR = env_str("WORKER_TMP_DIR", "/tmp/work") +DISPATCHER_POLL_SECONDS = env_int("DISPATCHER_POLL_SECONDS", 1) +DISPATCHER_BATCH = env_int("DISPATCHER_BATCH", 100) +STAGING_RETENTION_SECONDS = env_int("STAGING_RETENTION_SECONDS", 86400) # исходник failed-вложения для replay +OUTBOX_RETENTION_SECONDS = env_int("OUTBOX_RETENTION_SECONDS", 7 * 86400) +ALIVE_FILE = env_str("ALIVE_FILE", "/tmp/alive") # healthcheck фоновых процессов + +# Учебное внедрение сбоев (faults.py). На стенде включено, в эксплуатации FAULT_INJECTION=0. +FAULT_INJECTION = env_str("FAULT_INJECTION", "0") == "1" +FAULTS_DIR = env_str("FAULTS_DIR", "/faults") + + +def check() -> None: + """Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе.""" + if not DATABASE_URL: + raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE") + if not REDIS_URL: + raise RuntimeError("Redis address is not set: CACHE_URL_FILE") + if not S3_ACCESS_KEY_ID or not S3_SECRET_ACCESS_KEY: + raise RuntimeError("S3 credentials are not set: S3_ACCESS_KEY_ID_FILE, S3_SECRET_ACCESS_KEY_FILE") + if ATTACHMENT_GRACE_SECONDS < 2 * UPLOAD_TIMEOUT_SECONDS: + raise RuntimeError("ATTACHMENT_GRACE_SECONDS must be at least twice UPLOAD_TIMEOUT_SECONDS") + if SESSION_STORE != "redis": + raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution") diff --git a/storage.py b/storage.py new file mode 100644 index 0000000..86e90bc --- /dev/null +++ b/storage.py @@ -0,0 +1,183 @@ +"""Приватное S3-хранилище (SeaweedFS) для байтов вложений. + +Раскладка ключей (lab5): + staging/<32 hex> — исходник, принятый API до ответа 202 (устойчивый staging); + attachments//v/original — опубликованный worker'ом оригинал (побайтно исходный файл); + attachments//v/thumbnail.jpg, crop.jpg, optimized.webp — варианты pipeline версии N; + originals/<32 hex> — исходники вложений lab4, worker обрабатывает их как staging. + +API ходит в S3 с отдельным ключом, у которого есть только Read/Write/List в одном bucket +(identity messenger-api в secrets/s3_config.json, см. scripts/init_secrets.py). Ключ объекта назначает сервер, +имя файла пользователя в ключ не попадает. +""" +import base64 +import logging +from datetime import datetime +from typing import BinaryIO, Iterator + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +import settings + +log = logging.getLogger("messenger.storage") + +ORIGINALS_PREFIX = "originals/" # lab4 +STAGING_PREFIX = "staging/" +FINAL_PREFIX = "attachments/" +ALL_PREFIXES = (STAGING_PREFIX, FINAL_PREFIX, ORIGINALS_PREFIX) +READ_CHUNK = 64 * 1024 + + +class StorageUnavailable(Exception): + """S3 не ответил, ответил 5xx или отказал в доступе: клиенту 503, без деталей.""" + + +class ObjectMissing(Exception): + """Объекта нет в bucket (NoSuchKey / 404).""" + + +def _make_client(connect_timeout: float, read_timeout: float, attempts: int): + return boto3.client( + "s3", + endpoint_url=settings.S3_ENDPOINT, + region_name=settings.S3_REGION, + aws_access_key_id=settings.S3_ACCESS_KEY_ID or None, + aws_secret_access_key=settings.S3_SECRET_ACCESS_KEY or None, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, # http://s3:8333// + connect_timeout=connect_timeout, + read_timeout=read_timeout, + retries={"total_max_attempts": attempts, "mode": "standard"}, # всего попыток, включая первую + max_pool_connections=settings.S3_POOL_MAX, + # Контрольную сумму считаем сами (SHA-256 всего файла) и передаём её в PUT явно. + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), + ) + + +_client = _make_client(settings.S3_CONNECT_TIMEOUT_SECONDS, settings.S3_READ_TIMEOUT_SECONDS, settings.S3_MAX_ATTEMPTS) +# Для отката в запросе пользователя: одна короткая попытка, остальное доделает janitor. +_quick_client = _make_client(1, 2, 1) + + +def _is_missing(exc: ClientError) -> bool: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return code in ("NoSuchKey", "404", "NotFound") or status == 404 + + +def _unavailable(action: str, exc: Exception) -> StorageUnavailable: + detail = exc.response.get("Error", {}).get("Code", "") if isinstance(exc, ClientError) else "" + log.warning("S3 %s failed: %s %s", action, type(exc).__name__, detail) + return StorageUnavailable() + + +def new_key(object_id: str) -> str: + """Ключ staging для нового исходника (имя файла пользователя в ключ не попадает).""" + return STAGING_PREFIX + object_id + + +def put(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + """Однократный PUT (файл не больше 10 MiB). S3 сам сверяет SHA-256 и не сохранит искажённый объект.""" + body.seek(0) + try: + _client.put_object( + Bucket=settings.S3_BUCKET, + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + ChecksumSHA256=base64.b64encode(bytes.fromhex(sha256_hex)).decode("ascii"), + ) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("put", exc) from None + + +def open_read(key: str): + """GET объекта. Возвращает поток botocore; читать через iter_chunks, закрыть после.""" + try: + return _client.get_object(Bucket=settings.S3_BUCKET, Key=key)["Body"] + except ClientError as exc: + if _is_missing(exc): + raise ObjectMissing(key) from None + raise _unavailable("get", exc) from None + except BotoCoreError as exc: + raise _unavailable("get", exc) from None + + +def download(key: str, path, max_bytes: int) -> int: + """GET объекта в файл кусками. Дальше max_bytes не читаем: размер не совпадёт, worker отклонит исходник.""" + stream = open_read(key) + size = 0 + try: + with open(path, "wb") as file: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return size + size += len(chunk) + file.write(chunk) + if size > max_bytes: + return size + except (BotoCoreError, OSError) as exc: + raise _unavailable("get", exc) from None + finally: + stream.close() + + +def iter_chunks(stream) -> Iterator[bytes]: + """Отдаёт тело кусками по 64 KiB: в памяти API не больше одного куска на скачивание.""" + try: + while True: + chunk = stream.read(READ_CHUNK) + if not chunk: + return + yield chunk + except (BotoCoreError, OSError) as exc: + # Заголовки уже отправлены: обрываем соединение, клиент увидит неполный ответ (Content-Length). + log.warning("S3 stream interrupted: %s", type(exc).__name__) + raise + finally: + stream.close() + + +def delete(key: str, quick: bool = False) -> None: + """Удаление идемпотентно: S3 отвечает 204 и для отсутствующего ключа.""" + try: + (_quick_client if quick else _client).delete_object(Bucket=settings.S3_BUCKET, Key=key) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("delete", exc) from None + + +def exists(key: str) -> bool: + try: + _client.head_object(Bucket=settings.S3_BUCKET, Key=key) + return True + except ClientError as exc: + if _is_missing(exc): + return False + raise _unavailable("head", exc) from None + except BotoCoreError as exc: + raise _unavailable("head", exc) from None + + +def list_objects(prefix: str) -> Iterator[tuple[str, datetime, int]]: + """Все объекты под prefix: (key, last_modified, size). Постранично, по 1000.""" + paginator = _client.get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.S3_BUCKET, Prefix=prefix): + for item in page.get("Contents", []): + yield item["Key"], item["LastModified"], item["Size"] + except (BotoCoreError, ClientError) as exc: + raise _unavailable("list", exc) from None + + +def ping() -> None: + try: + _client.head_bucket(Bucket=settings.S3_BUCKET) + except (BotoCoreError, ClientError) as exc: + raise _unavailable("head_bucket", exc) from None diff --git a/student-tests/check_stack.py b/student-tests/check_stack.py new file mode 100644 index 0000000..3f4258e --- /dev/null +++ b/student-tests/check_stack.py @@ -0,0 +1,91 @@ + +import json +import platform +import sys + +from common import call, check, compose_output, docker, new_user, seen_instances + + +def main(): + info = json.loads(docker("info", "--format", "{{json .}}")) + print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"client Python {platform.python_version()}") + print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, " + f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, " + f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}") + + print() + print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip()) + for one_shot in ("migrate", "s3-init"): + status = compose_output("ps", "-a", "--format", "{{.Status}}", one_shot).strip() + check(status.startswith("Exited (0)"), f"{one_shot} service must finish with exit code 0") + + config = json.loads(compose_output("config", "--format", "json")) + services = config["services"] + containers = compose_output("ps", "-q").split() + rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()] + print() + print("container image cpus memory host ports networks") + api_count = 0 + for row in sorted(rows, key=lambda r: r["Name"]): + labels = row["Config"]["Labels"] + service = labels.get("com.docker.compose.service") + limits = row["HostConfig"] + cpus = limits.get("NanoCpus", 0) / 1e9 + memory = limits.get("Memory", 0) // 2**20 + ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values() + for b in (bindings or [])}) + networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"]) + print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB " + f"{','.join(ports) or '-':<23} {','.join(networks)}") + check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit") + if service == "api": + api_count += 1 + check(not ports, "api must not publish ports: everything goes through the balancer") + if service == "lb": + check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only") + if service in ("postgres", "redis", "s3", "janitor", "rabbitmq", "worker", "dispatcher"): + check(not ports and networks == ["data"], f"{service} must be only in the internal data network") + check(api_count >= 2, "need at least 2 api replicas") + running = {r["Config"]["Labels"].get("com.docker.compose.service") for r in rows if r["State"]["Running"]} + check({"rabbitmq", "worker", "dispatcher"} <= running, "lab5: broker, worker and dispatcher must be running") + check(any(v.get("target") == "/var/lib/rabbitmq" and v.get("type") == "volume" + for v in services["rabbitmq"].get("volumes", [])), "rabbitmq needs a persistent volume") + check(services["rabbitmq"].get("hostname") == "rabbitmq", "rabbitmq needs a fixed hostname (node data dir)") + check(all("container_name" not in s for s in services.values()), "container_name prevents scaling") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis") + + print() + for name, service in sorted(services.items()): + secrets = sorted(s["source"] for s in service.get("secrets", [])) + print(f"secrets of {name}: {secrets or '-'}") + check(not services["lb"].get("secrets"), "lb gets no secrets") + check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url") + for name in ("api", "janitor", "worker"): + names = {s["source"] for s in services[name]["secrets"]} + check(not names & {"s3_root_user", "s3_root_password", "s3_config"}, f"{name} must not get S3 admin keys") + check({"s3_access_key_id", "s3_secret_access_key"} <= names, f"{name} needs the limited S3 key") + broker_secrets = {"broker_url_dispatcher", "broker_url_worker", "rabbitmq_definitions"} + for name in ("api", "janitor", "lb", "migrate"): + names = {s["source"] for s in services[name].get("secrets", [])} + check(not names & broker_secrets, f"{name} must not get broker credentials (API uses the outbox)") + check({s["source"] for s in services["dispatcher"]["secrets"]} == {"database_url", "broker_url_dispatcher"}, + "dispatcher: only database_url and its own broker user") + check(not {s["source"] for s in services["dispatcher"]["secrets"]} & {"s3_access_key_id", "s3_secret_access_key"}, + "dispatcher needs no S3 key") + + user = new_user("stack") + served = seen_instances(user["token"]) + print() + print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}") + check(len(served) >= 2, "requests are not balanced") + health = call("GET", "/health/ready", 200) + print(f"/health/ready -> {health.status} from {health.instance}") + print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every " + f"container, postgres/redis/s3/janitor/rabbitmq/worker/dispatcher only in the internal data network, " + f"S3 admin keys only in s3 and s3-init, broker credentials only in dispatcher/worker") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 0000000..c44bb68 --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,359 @@ + +import json +import os +import platform +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / os.environ.get("EVIDENCE_DIR", "evidence") # по умолчанию evidence/ +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PY = sys.executable + +STEPS = [ + ("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]), + ("stack", "stand layout: replicas, ports, limits, networks, secrets, S3", [PY, "student-tests/check_stack.py"]), + ("smoke", "public smoke via the balancer (make test), with attachment", [PY, "tests/smoke.py", "--base-url", BASE_URL]), + ("jobs-variants", "lab5: thumbnail/crop/optimized, EXIF orientation, alpha, no upscale, metadata stripped", + [PY, "student-tests/test_jobs.py", "variants"]), + ("jobs-limits", "lab5: worker limits - bytes, pixels, memory (RLIMIT_AS), time (JOB_TIMEOUT_SECONDS)", + [PY, "student-tests/test_jobs.py", "limits"]), + ("jobs-recovery", "lab5: worker down -> 202 -> restart api+broker -> ready; broker down -> outbox waits", + [PY, "student-tests/test_jobs.py", "worker-down", "broker-down"]), + ("jobs-outbox", "lab5: dispatcher crash between COMMIT and publish / after confirm", + [PY, "student-tests/test_jobs.py", "outbox-crash"]), + ("jobs-crash", "lab5: worker crash after S3 PUT / after COMMIT before ack, staging cleanup, frozen worker lease", + [PY, "student-tests/test_jobs.py", "crash-after-put", "crash-after-commit", "staging", "lease"]), + ("jobs-retry", "lab5: backoff+jitter, retry_exhausted, replay, permanent errors, DLQ", + [PY, "student-tests/test_jobs.py", "retry", "exhausted-replay", "permanent", "dlq"]), + ("jobs-concurrency", "lab5: WORKER_CONCURRENCY consumers, prefetch 1, bounded processing", + [PY, "student-tests/test_jobs.py", "concurrency"]), + ("attachments", "lab4: files, limits, damaged images, MIME, filenames, access, attachment_ids", + [PY, "student-tests/test_attachments.py"]), + ("s3-security", "lab4: private bucket outside API, limited API key, no console/ports, idempotent bootstrap", + [PY, "student-tests/test_s3_security.py"]), + ("consistency", "lab4: aborted upload, crash between S3 and DB, janitor, orphans, S3 outage", + [PY, "student-tests/test_consistency.py"]), + ("upload-memory", "lab4: >=10 concurrent uploads, peak memory of api replicas", + [PY, "student-tests/upload_memory.py", "--output", "evidence/upload-memory.json"]), + ("balancing", "balancing, shared sessions/data/files, X-Instance-Id", [PY, "student-tests/test_balancing.py"]), + ("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]), + ("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]), + ("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]), + ("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]), + ("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]), + ("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]), + ("failover", "public failover test for every replica, file downloaded via the other one (make test-failover)", + [PY, "student-tests/run_failover.py"]), + ("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL, + "--duration", "30", "--concurrency", "4", + "--output", "evidence/load.json"]), + ("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]), + ("persistence", "public persistence test incl. S3 + proof of real restarts (make test-persistence)", + [PY, "student-tests/run_persistence.py"]), +] + + +def git(*args): + try: + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "" + + +def commit(): + head = git("rev-parse", "--short", "HEAD") or "unknown" + changed = [line for line in git("status", "--porcelain").splitlines() + if not line[3:].startswith(("evidence/", "REPORT.md"))] + return head + (" + uncommitted changes" if changed else "") + + +def run_step(name, description, command): + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL) + print(f"=== {name}: {description}", flush=True) + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + seconds = time.monotonic() - started + output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n") + shown = " ".join(Path(part).name if part == PY else part for part in command) + header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n" + f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n") + (EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8") + tail = [line for line in output.splitlines() if line.strip()][-3:] + for line in tail: + print(" " + line) + print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True) + return result.returncode == 0, seconds + + +def load_json(name): + path = EVIDENCE / name + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + +def measurements(): + """Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md).""" + lines = [] + try: + info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True, + text=True, check=True).stdout) + docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM" + except (OSError, ValueError, subprocess.CalledProcessError): + docker_vm = "unknown" + lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона " + f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB " + f"каждая, postgres, s3 (SeaweedFS) и rabbitmq по 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB, " + f"janitor 0,25 CPU / 192 MiB, dispatcher 0,25 CPU / 128 MiB, worker 2 CPU / 1536 MiB.") + lines.append("") + + failover = EVIDENCE / "failover.txt" + results = [] + if failover.exists(): + text = failover.read_text(encoding="utf-8") + stopped = re.findall(r"the hook stopped (\w+)", text) + results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)] + if results: + lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):") + lines.append("") + lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток " + "(повторены с тем же ключом) | Первая запись после hook, с |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for number, (result, container) in enumerate(zip(results, stopped), start=1): + lines.append(f"| {number} | {container} | {result['observed_instances_before']} | " + f"{result['successful_writes']} | {result['failed_attempts']} | " + f"{result['first_recovery_seconds']} |") + lines.append("") + + memory = load_json("upload-memory.json") + if memory: + lines.append(f"Одновременные загрузки (`student-tests/upload_memory.py`, перед каждым сценарием реплики " + f"перезапускаются, лимит api {memory['api_limit_mib']} MiB / {memory['api_limit_cpus']:g} CPU). " + f"Пик RSS — VmHWM процесса API; cgroup — память контейнера, включая page cache временных файлов:") + lines.append("") + lines.append("| Сценарий | Одновременно | Объём, MiB | Время, с (p50 / max) | Реплика: загрузок | RSS до, MiB | " + "Пик RSS, MiB | Рост, MiB | Пик cgroup, MiB |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for run in memory["runs"]: + for n, r in enumerate(run["replicas"]): + head = (f"| {run['scenario']} | {run['concurrency']} | {run['total_mib']} | " + f"{run['latency_s']['p50']} / {run['latency_s']['max']} |") if n == 0 else "| | | | |" + cgroup = r["cgroup_peak_mib"] or r["cgroup_peak_sampled_mib"] or "-" + lines.append(f"{head} {r['replica']}: {r['uploads_served']} | {r['rss_before_mib']} | " + f"{r['rss_peak_mib']} | {r['rss_growth_mib']} | {cgroup} |") + naive = memory["single_file_extra_rss_mib"] + lines.append("") + lines.append(f"Один PNG 8000x5000 RGB внутри контейнера api, дополнительный пик RSS: наивно (тело целиком в " + f"памяти + полное декодирование Pillow) {naive['naive']} MiB, потоковая проверка " + f"{naive['streaming']} MiB. 10 таких наивных загрузок одновременно не поместились бы в " + f"{memory['api_limit_mib']} MiB.") + lines.append("") + + metrics = load_json("failover-metrics.json") + if metrics: + lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи " + "без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):") + lines.append("") + lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок " + "(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | " + "Вернулась в балансировку после docker start, с |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for r in metrics["runs"]: + how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)" + codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-" + lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | " + f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | " + f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |") + lines.append("") + lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика " + "после остановки не ответила ни разу.") + lines.append("") + + rolling = load_json("rolling-update.json") + if rolling: + during = rolling["during"] + codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-" + lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> " + f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за " + f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, " + f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); " + f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено " + f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.") + lines.append("") + + load = load_json("load.json") + if load: + latency = load["successful_latency_ms"] + lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, " + f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных " + f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных " + f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.") + lines.append("") + + persistence = EVIDENCE / "persistence.txt" + restart_seconds = None + if persistence.exists(): + found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8")) + restart_seconds = found.group(1) if found else None + + impacts = [] + if metrics: + for mode, what, expected in ( + ("stop", "`docker stop` реплики (SIGTERM) под нагрузкой", + "ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"), + ("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой", + "единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")): + runs = [r for r in metrics["runs"] if r["mode"] == mode] + if runs: + impacts.append( + f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); " + f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | " + f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; " + f"потерь 0, дублей 0; реплика вернулась через " + f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |") + if failover.exists() and results: + impacts.append( + f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, " + f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: " + f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через " + f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |") + if rolling: + impacts.append( + f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | " + f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |") + if restart_seconds: + impacts.append( + f"| перезапуск всего стенда (`restart` hook: postgres, redis, s3, api, janitor, lb) | данные, файл и " + f"прежний токен живы | " + f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |") + consistency = EVIDENCE / "consistency.txt" + if consistency.exists(): + text = consistency.read_text(encoding="utf-8") + timings = re.findall(r"upload -> 503 unavailable in (\d+) ms", text) + if len(timings) >= 2: + impacts.append(f"| `docker compose stop s3` (SeaweedFS остановлен) | загрузка и скачивание -> 503, метаданные, " + f"сообщения и /health/ready работают | upload 503 за {timings[0]} мс, download 503, остальное 200/201 | " + f"после start файлы на месте; uploading-строку убрал janitor; потерь нет |") + impacts.append(f"| `docker compose pause s3` (SeaweedFS завис) | то же, но через таймауты S3-клиента | upload 503 за " + f"{timings[1]} мс (2 попытки x 5 с + откат 2 с) | после unpause файлы на месте; потерь нет |") + if "PASS: S3 written but DB not committed" in text: + impacts.append("| падение между PUT в S3 и COMMIT в БД (код API в контейнере, finish_upload падает) | " + "вложение не видно, janitor не трогает его до конца grace period, потом удаляет объект и строку | " + "GET/прикрепление -> 404; janitor: свежая строка сохранена, dry-run только отчёт, затем удалено | " + "мусора не осталось; повторный проход ничего не делает |") + if "PASS: an interrupted upload leaves no row" in text: + impacts.append("| обрыв загрузки клиентом (9 МБ заявлено, отправлено 0,2 и 3 МБ) | ничего не сохраняется | " + "строк и объектов не прибавилось, временных файлов 0 | мгновенно |") + impacts += lab5_impacts() + if impacts: + lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |") + lines.append("| --- | --- | --- | --- |") + lines.extend(impacts) + lines.append("") + return "\n".join(lines).strip() + + +def lab5_line(name, pattern): + """Строка вывода test_jobs.py из evidence/.txt, содержащая pattern (или None).""" + path = EVIDENCE / f"{name}.txt" + if not path.exists(): + return None + for line in path.read_text(encoding="utf-8").splitlines(): + if pattern in line and not line.startswith(("#", "---")): + return line.strip() + return None + + +def lab5_impacts(): + """Строки таблицы отказов lab5: наблюдение — дословно строка вывода теста (числа из прогона).""" + rows = [ + ("jobs-recovery", "worker started", "worker остановлен, upload, рестарт rabbitmq и api с volumes, start worker", + "202 queued, сообщение с вложением 201, content 409; после рестартов job в очереди; тот же job_id -> ready", + "потерь нет; staging удалён после COMMIT"), + ("jobs-recovery", "rabbitmq started", "rabbitmq остановлен во время upload", + "202 и /health/ready 200 без брокера; строка outbox ждёт; после start публикуется", "потерь нет"), + ("jobs-outbox", "killed between COMMIT and publish", "crash dispatcher между COMMIT задания и publish", + "строка outbox не отмечена -> публикуется после рестарта процесса", "без ручного действия"), + ("jobs-outbox", "after publisher confirm", "crash dispatcher после confirm, до отметки outbox", + "дубль сообщения, worker распознаёт по состоянию job", "один набор вариантов"), + ("jobs-crash", "killed after S3 PUT", "crash worker после PUT в S3, до COMMIT и ack", + "повтор после истечения аренды перезаписывает те же ключи", "один набор, 4 объекта"), + ("jobs-crash", "after COMMIT, before ack", "crash worker после COMMIT, до ack", + "redelivery ready-задачи -> no-op ack", "очередь пуста, дублей нет"), + ("jobs-crash", "frozen attempt", "worker завис (не продлевает аренду)", + "dispatcher возвращает job, другой поток завершает; поздний результат отклонён fencing", "ready не откатан"), + ("jobs-limits", "imaging hang", "зависание обработки изображения", + "процесс убит по JOB_TIMEOUT_SECONDS, временная ошибка, аренда продлевалась", "повтор -> ready"), + ("jobs-limits", "memory bomb", "нехватка памяти в процессе обработки (RLIMIT_AS)", + "failed/resource_limit, worker жив", "постоянная ошибка, без повторов"), + ("jobs-retry", "storage_unavailable ->", "2 временные ошибки S3", "backoff с jitter, затем ready", "-"), + ("jobs-retry", "retry_exhausted (public", "5 временных ошибок подряд", + "failed/retry_exhausted, реестр failed, content 409", "replay оператора"), + ("jobs-retry", "ops replay", "replay failed и повторный replay ready", "тот же job_id -> ready; повтор отклонён", + "дубля нет"), + ("jobs-retry", "-> reject -> DLQ", "невалидное сообщение и неизвестный job_id", "reject -> DLQ, dlq-replay не исполняет", + "сообщения сохранены"), + ("jobs-concurrency", "peak processing", "8 заданий в очереди, старт worker", + "WORKER_CONCURRENCY=2, prefetch 1", "-"), + ] + out = [] + for name, pattern, what, expected, recovery in rows: + line = lab5_line(name, pattern) + if line: + out.append(f"| {what} | {expected} | {line.replace('|', '/')} | {recovery} |") + return out + + +def update_report(block): + report = ROOT / "REPORT.md" + text = report.read_text(encoding="utf-8") + begin, end = "", "" + if begin not in text or end not in text: + print("REPORT.md has no measurement markers, only evidence/summary.md was written") + return + head, rest = text.split(begin, 1) + _, tail = rest.split(end, 1) + report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8") + print("REPORT.md: measurement tables updated") + + +def main(): + args = sys.argv[1:] + names = [name for name, _, _ in STEPS] + if args[:1] == ["--from"]: + if len(args) < 2 or args[1] not in names: + raise SystemExit("usage: --from ; steps: " + ", ".join(names)) + selected = names[names.index(args[1]):] + else: + unknown = [a for a in args if a not in names] + if unknown: + raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names)) + selected = args or names + EVIDENCE.mkdir(exist_ok=True) + results = [] + for name, description, command in STEPS: + if name not in selected: + continue + ok, seconds = run_step(name, description, command) + results.append((name, description, ok, seconds)) + if not ok: + print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.") + print(f"After fixing: python student-tests/collect_evidence.py --from {name}") + break + block = measurements() + summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |", + "| --- | --- | --- | --- | --- |"] + summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results] + (EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8") + print("evidence/summary.md written") + passed = all(ok for _, _, ok, _ in results) + if passed: + update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 0000000..66cc2a8 --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,347 @@ + +import base64 +import csv +import hashlib +import io +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +PASSWORD = "Test-password-123" + + +class Response: + def __init__(self, status, headers, data): + self.status = status + self.headers = {name.lower(): value for name, value in headers.items()} + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except ValueError: + self.json = None + + @property + def instance(self): + """X-Instance-Id: какая реплика API ответила (ID её контейнера).""" + return self.headers.get("x-instance-id") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30): + """HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError. + + Заодно проверяет то, что контракт требует от любого ответа: + X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401. + """ + all_headers = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + if data is not None: + all_headers.setdefault("Content-Type", "application/json") + if token is not None: + all_headers["Authorization"] = "Bearer " + token + + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + response = Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + response = Response(error.code, error.headers, error.read()) + + expected = expected if isinstance(expected, tuple) else (expected,) + check(response.status in expected, + f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}") + request_id = response.headers.get("x-request-id") + check(request_id, f"{method} {path}: no X-Request-Id header") + check(response.instance, f"{method} {path}: no X-Instance-Id header") + if response.status >= 400: + error = (response.json or {}).get("error") or {} + check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format") + check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header") + if response.status == 401: + check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate") + return response + + +def try_call(method, path, *, body=None, token=None, headers=None, timeout=2): + """Запрос без проверок и без исключений — для нагрузки во время отказа. + + Возвращает Response или None, если соединение оборвалось / вышел timeout. + """ + all_headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + all_headers["Content-Type"] = "application/json" + if token is not None: + all_headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + return Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + try: + return Response(error.code, error.headers, error.read()) + except OSError: + return None + except (urllib.error.URLError, OSError): + return None + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8.""" + return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + + +def login(username, password, expected=201): + return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)}) + + +def new_user(prefix="user", password=PASSWORD): + """Регистрирует пользователя и входит. Возвращает dict: id, username, password, token.""" + username = f"{prefix}_{uuid.uuid4().hex[:10]}" + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": prefix.title(), "password": password}).json + session = login(username, password).json + return {**user, "password": password, "token": session["token"]} + + +def new_chat(owner, *others, title="Test chat"): + member_ids = [owner["id"]] + [user["id"] for user in others] + return call("POST", "/api/v1/chats", 201, token=owner["token"], + body={"title": title, "member_ids": member_ids}).json + + +def new_key(): + return uuid.uuid4().hex + + +def send(user, chat_id, text, expected=201, key=None): + """POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body={"text": text}, headers={"Idempotency-Key": key or new_key()}) + + +def post_message(user, chat_id, text="", attachment_ids=None, expected=201, key=None): + """POST сообщения с вложениями (lab4). attachment_ids=None — поле не передаётся.""" + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body=body, headers={"Idempotency-Key": key or new_key()}) + + +def upload(user, chat_id, data, content_type="image/png", filename="photo.png", expected=202, timeout=60, + wait=True): + """POST /chats/{id}/attachments сырыми байтами. filename=None — без параметра. + + Lab5: успешный приём — 202 со снимком queued (переход 201 -> 202 из COURSE.md). При wait=True + ответ дополняется ожиданием готовности: .json — метаданные после GET в состоянии ready/failed, + .accepted — исходный снимок 202. Так проверки lab4 («файл скачивается побайтно») сохраняются. + """ + query = "" if filename is None else "?filename=" + urllib.parse.quote(filename, safe="") + headers = {"Content-Type": content_type} if content_type is not None else {} + reply = call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, raw=data, + token=user["token"] if user else None, headers=headers, timeout=timeout) + if reply.status == 202: + check(reply.json["status"] == "queued" and reply.json["variants"] == [], "202 must be a queued snapshot") + reply.accepted = reply.json + if wait: + reply.json = wait_attachment(user, reply.json["id"]) + return reply + + +def wait_attachment(user, attachment_id, statuses=("ready", "failed"), seconds=60): + """GET метаданных, пока вложение не перейдёт в одно из statuses (smoke ждёт ready до 60 с).""" + deadline = time.monotonic() + seconds + while True: + att = call("GET", f"/api/v1/attachments/{attachment_id}", 200, token=user["token"]).json + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"attachment {attachment_id} is {att['status']} after {seconds} s") + time.sleep(0.2) + + +def read_history(user, chat_id, limit=100): + """Вся история чата по страницам. Возвращает (сообщения, число_страниц).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "") + page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json + items += page["items"] + pages += 1 + cursor = page["next_cursor"] + if cursor is None: + return items, pages + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while True: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply: + if reply.status == 200: + return + except (urllib.error.URLError, OSError): + pass + check(time.monotonic() < deadline, f"API is not ready after {seconds} s") + time.sleep(0.5) + + +def compose(*args, env=None): + """docker compose в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам.""" + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def compose_output(*args, env=None): + """docker compose и его вывод (для logs, ps).""" + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env) + return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace") + + +def docker(*args, check_exit=True): + """docker , возвращает stdout.""" + result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit) + return result.stdout.decode("utf-8", "replace").strip() + + +def api_containers(include_stopped=False): + """Полные ID контейнеров сервиса api (по умолчанию только запущенные).""" + args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"] + return sorted(compose_output(*args).split()) + + +def health(container): + """Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''.""" + return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container, + check_exit=False) + + +def lb_servers(): + """Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}.""" + with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply: + rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace")))) + header = [name.lstrip("# ") for name in rows[0]] + servers = {} + for row in rows[1:]: + item = dict(zip(header, row)) + if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"): + servers[item["svname"]] = item.get("status", "") + return servers + + +def lb_up_count(): + """Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP).""" + return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP")) + + +def seen_instances(token, requests=20): + """Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me.""" + return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)} + + +def wait_balanced(replicas, seconds=60): + """Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке.""" + deadline = time.monotonic() + seconds + while True: + running = api_containers() + healthy = [container for container in running if health(container) == "healthy"] + try: + up = lb_up_count() + except OSError: + up = 0 + if len(healthy) >= replicas and up >= replicas: + return running + check(time.monotonic() < deadline, + f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}") + time.sleep(0.5) + + +def redis_cli(*args): + """redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод.""" + command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args) + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command], + cwd=ROOT, capture_output=True, check=True) + return result.stdout.decode("utf-8", "replace").strip() + + +def session_key(token): + """Ключ сессии в Redis, как его считает auth.py: session:.""" + return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def psql(sql): + """Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки). + + Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя. + """ + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, check=False, + ) + output = result.stdout.decode("utf-8", "replace").strip() + error = result.stderr.decode("utf-8", "replace").strip() + return result.returncode, output, error + + +def app_python(code, *args, service="api", env=None): + """Выполнить Python-скрипт внутри контейнера приложения (там есть модули API и S3-ключи API). + + Скрипт передаётся через stdin, секреты остаются в контейнере. Возвращает stdout (последняя строка — JSON). + """ + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += [service, "python", "-", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, input=code.encode("utf-8"), capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"script in {service} failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return output + + +def janitor(*args, env=None): + """Один проход janitor.py в контейнере janitor. Возвращает его JSON-отчёт.""" + command = ["docker", "compose", "exec", "-T"] + for name, value in (env or {}).items(): + command += ["-e", f"{name}={value}"] + command += ["janitor", "python", "janitor.py", "--once", *map(str, args)] + result = subprocess.run(command, cwd=ROOT, capture_output=True, check=False) + output = result.stdout.decode("utf-8", "replace").strip() + check(result.returncode == 0, f"janitor failed: {result.stderr.decode('utf-8', 'replace')[-600:]}") + return json.loads(output.splitlines()[-1]) + + +def sql_value(query): + """Одно значение из PostgreSQL (psql -A -t).""" + code, output, error = psql(query) + check(code == 0, f"psql failed: {error[-300:]}") + return output + + +def run_tests(tests): + """Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал.""" + failed = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста + failed += 1 + print(f"FAIL: {test.__name__}: {exc}") + print(f"{len(tests) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/demo_instances.py b/student-tests/demo_instances.py new file mode 100644 index 0000000..8f97f1c --- /dev/null +++ b/student-tests/demo_instances.py @@ -0,0 +1,16 @@ + +from common import call, compose_output, new_user + + +def main(): + user = new_user("demo") + print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:") + for number in range(1, 9): + reply = call("GET", "/api/v1/users/me", 200, token=user["token"]) + print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}") + print() + print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 0000000..81416de --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,28 @@ + +import subprocess + +from common import ROOT + +OUTPUT = ROOT / "evidence" / "explain-history.txt" + + +def main(): + sql = (ROOT / "student-tests" / "explain.sql").read_bytes() + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + cwd=ROOT, input=sql, capture_output=True, check=False, + ) + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_bytes(result.stdout + result.stderr) + if result.returncode != 0: + print(result.stderr.decode("utf-8", "replace")) + raise SystemExit("FAIL: psql returned an error, see output above") + for line in result.stdout.decode("utf-8", "replace").splitlines(): + if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line: + print(line) + print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 0000000..01cd9e7 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,76 @@ + +\pset footer off +BEGIN; + +CREATE TEMP TABLE seed_chats AS + SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size + FROM generate_series(0, 50) AS n; +CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; + +INSERT INTO users (id, username, display_name, password_hash) + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; +INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), + timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' + FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; +ANALYZE messages; + +SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset +SELECT created_at AS cursor_time, id AS cursor_id FROM messages + WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== Data volume' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat +FROM messages; + +\echo '' +\echo '=== History index' +SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history'; + +\echo '' +\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2b. The same page exactly as the app sends it since lab4: with attachment_ids of every message' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT a.attachment_id FROM message_attachments a + WHERE a.message_id = m.id ORDER BY a.position) AS attachment_ids +FROM messages m +WHERE m.chat_id = :'big_chat' AND (m.created_at, m.id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY m.created_at, m.id LIMIT 51; + +\echo '' +\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id OFFSET 10000 LIMIT 51; + +\echo '' +\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)' +DROP INDEX messages_history; +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; +\echo '' +\echo '=== ROLLBACK: test data removed, index is back' +SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%'; diff --git a/student-tests/failover_metrics.py b/student-tests/failover_metrics.py new file mode 100644 index 0000000..19421a1 --- /dev/null +++ b/student-tests/failover_metrics.py @@ -0,0 +1,129 @@ +import json +import platform +import sys +import threading +import time + +from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced +from loadgen import Load, summary + +WARMUP_SECONDS = 3 +AFTER_SECONDS = 10 +OUTPUT = ROOT / "evidence" / "failover-metrics.json" + + +class LbWatch: + """Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP).""" + + def __init__(self): + self.points, self.stopping = [], threading.Event() + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stopping.is_set(): + try: + self.points.append((time.monotonic(), lb_up_count())) + except OSError: + pass + time.sleep(0.1) + + def start(self): + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join(timeout=3) + + def first_drop(self, after, replicas): + return next((t for t, up in self.points if t >= after and up < replicas), None) + + +def measure(container, mode, replicas): + wait_balanced(replicas) + load, watch = Load(), LbWatch() + watch.start() + load.start() + time.sleep(WARMUP_SECONDS) + t0 = time.monotonic() + if mode == "stop": + docker("stop", "--time", "8", container) + else: + docker("kill", container) + command_seconds = time.monotonic() - t0 + time.sleep(max(0.0, AFTER_SECONDS - command_seconds)) + end = t0 + AFTER_SECONDS + load.stop() + watch.stop() + + errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]] + errors_end = max((e[0] + e[5] for e in errors), default=None) + drop = watch.first_drop(t0, replicas) + served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]] + target_hits = sum(1 for instance in served_after if instance and container.startswith(instance)) + integrity = load.verify() + + started = time.monotonic() + docker("start", container) + wait_balanced(replicas) + rejoin_seconds = time.monotonic() - started + + return { + "replica": container[:12], + "mode": mode, + "command_seconds": round(command_seconds, 2), + "lb_excluded_after_seconds": round(drop - t0, 2) if drop else None, + "errors": len(errors), + "error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"], + "errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0, + "before": summary(load, t0 - WARMUP_SECONDS, t0), + "after": summary(load, t0, end), + "stopped_replica_answers_after_stop": target_hits, + "integrity": integrity, + "rejoin_seconds": round(rejoin_seconds, 2), + } + + +def main(): + modes = sys.argv[1:] or ["stop", "kill"] + check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]") + containers = api_containers() + replicas = len(containers) + check(replicas >= 2, "need at least 2 running api replicas") + print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, " + f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and " + f"{AFTER_SECONDS} s after the failure") + runs = [] + for container in containers: + for mode in modes: + print(f"--- {mode} {container[:12]} ...") + run = measure(container, mode, replicas) + runs.append(run) + print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after " + f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, " + f"stopped after {run['errors_stopped_after_seconds']} s") + print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; " + f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; " + f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}") + print(f" confirmed {run['integrity']['confirmed_writes']}, in history " + f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; " + f"back in the balancer {run['rejoin_seconds']} s after docker start") + check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered") + check(run["after"]["write_rps"] > 0, "no successful writes after the failure") + print() + print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s") + for r in runs: + print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} " + f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} " + f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS, + "after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()}, + "runs": runs, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; " + f"saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/fixtures/progressive.jpg b/student-tests/fixtures/progressive.jpg new file mode 100644 index 0000000..38faa2b Binary files /dev/null and b/student-tests/fixtures/progressive.jpg differ diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 0000000..6816ff5 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/images.py b/student-tests/images.py new file mode 100644 index 0000000..e49bfc6 --- /dev/null +++ b/student-tests/images.py @@ -0,0 +1,93 @@ +"""Синтетические изображения для тестов вложений: только стандартная библиотека. + +PNG собирается вручную (сигнатура, IHDR, IDAT, IEND), поэтому можно получить любой размер +в пикселях и байтах, а также испортить конкретный байт. JPEG берётся из fixtures/. +""" +import os +import struct +import zlib +from pathlib import Path + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +ROOT = Path(__file__).resolve().parents[1] +MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +MAX_PIXELS = 40_000_000 +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) + + +def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png(width, height, *, gray=False, noise=False, interlace=0, pad_to=None, idat=None) -> bytes: + """PNG width x height. noise=True — случайные пиксели (почти не сжимаются, файл большой). + + pad_to — дополнить файл текстовым чанком tEXt после IDAT до точного размера в байтах. + idat — готовые сжатые данные (для «бомбы»: заголовок обещает одно, данных другое). + """ + channels = 1 if gray else 3 + header = struct.pack(">IIBBBBB", width, height, 8, 0 if gray else 2, 0, 0, interlace) + if idat is None: + compressor = zlib.compressobj(6) + parts = [] + # Adam7: 7 проходов, у каждого свои ширина и высота; без interlace — один проход. + passes = [(width, height)] if not interlace else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + for pass_width, pass_height in passes: + if pass_width <= 0 or pass_height <= 0: + continue + row = b"\x00" + bytes(pass_width * channels) + for _ in range(pass_height): + parts.append(compressor.compress(b"\x00" + os.urandom(pass_width * channels) if noise else row)) + parts.append(compressor.flush()) + idat = b"".join(parts) + body = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + chunk(b"IDAT", idat) + if pad_to is not None: + need = pad_to - len(body) - 12 - 12 - 8 # заголовок чанка-заполнителя, IEND, ключ "Comment\0" + assert need >= 0, "image is already larger than pad_to" + body += chunk(b"tEXt", b"Comment\x00" + b"a" * need) + return body + chunk(b"IEND", b"") + + +def noise_png(target_bytes: int) -> bytes: + """Почти несжимаемый PNG размером чуть меньше target_bytes (квадрат RGB со случайными пикселями).""" + side = int(((target_bytes - 4096) / 3) ** 0.5) + return png(side, side, noise=True) + + +def sample_png() -> bytes: + return (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def sample_jpeg() -> bytes: + return (FIXTURES / "sample.jpg").read_bytes() + + +def progressive_jpeg() -> bytes: + return (FIXTURES / "progressive.jpg").read_bytes() + + +def flip_byte(data: bytes, position: int) -> bytes: + return data[:position] + bytes([data[position] ^ 0xFF]) + data[position + 1:] + + +GIF = b"GIF89a\x01\x00\x01\x00\x80\x00\x00\x00\x00\x00\xff\xff\xff!\xf9\x04\x01\x00\x00\x00\x00,\x00\x00\x00\x00\x01\x00\x01\x00\x00\x02\x02D\x01\x00;" + + +# --- сборка PNG из произвольных чанков (для проверок смысла чанков при верных CRC) --- + +def ihdr(width, height, depth=8, color=2, interlace=0): + return b"IHDR", struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace) + + +def zero_idat(width, height, depth=8, channels=3): + """Сжатые пиксели: каждая строка = байт фильтра 0 + нули (индекс палитры 0 / чёрный).""" + row_bytes = (width * depth * channels + 7) // 8 + return b"IDAT", zlib.compress((b"\x00" + bytes(row_bytes)) * height) + + +def assemble(*chunks): + """PNG из чанков (kind, data) с ВЕРНЫМИ CRC; IEND добавляется, если не передан.""" + if not chunks or chunks[-1][0] != b"IEND": + chunks = chunks + ((b"IEND", b""),) + return b"\x89PNG\r\n\x1a\n" + b"".join(chunk(kind, data) for kind, data in chunks) diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 0000000..998548c --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,102 @@ +import math +import threading +import time + +from common import check, new_chat, new_key, new_user, read_history, try_call + + +class Load: + def __init__(self, writers=4, timeout=2): + self.alice, self.bob = new_user("load_a"), new_user("load_b") + self.chat = new_chat(self.alice, self.bob, title="load") + self.writers, self.timeout = writers, timeout + self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с) + self.confirmed = [] # сообщения, на которые пришёл 201 + self.lock = threading.Lock() + self.stopping = threading.Event() + self.threads = [] + + def _record(self, kind, started, reply, ok): + code = reply.status if reply is not None else "conn" + instance = reply.instance if reply is not None else None + with self.lock: + self.events.append((started, kind, ok, code, instance, time.monotonic() - started)) + + def _writer(self, number): + path = f"/api/v1/chats/{self.chat['id']}/messages" + sent = 0 + key, text = new_key(), f"load-{number}-{sent}" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("POST", path, body={"text": text}, token=self.alice["token"], + headers={"Idempotency-Key": key}, timeout=self.timeout) + ok = reply is not None and reply.status == 201 + self._record("write", started, reply, ok) + if ok: + with self.lock: + self.confirmed.append(reply.json) + sent += 1 + key, text = new_key(), f"load-{number}-{sent}" + else: + time.sleep(0.05) # и повторить тот же text с тем же ключом + + def _reader(self): + path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout) + self._record("read", started, reply, reply is not None and reply.status == 200) + time.sleep(0.05) + + def start(self): + self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)] + self.threads.append(threading.Thread(target=self._reader, daemon=True)) + for thread in self.threads: + thread.start() + + def stop(self): + self.stopping.set() + for thread in self.threads: + thread.join(timeout=self.timeout + 5) + + def window(self, start, end, kind=None): + with self.lock: + return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)] + + def verify(self): + """Все подтверждённые сообщения есть в истории, id и тексты не повторяются.""" + history, _ = read_history(self.bob, self.chat["id"]) + ids = [message["id"] for message in history] + texts = [message["text"] for message in history] + by_id = {message["id"]: message for message in history} + check(len(ids) == len(set(ids)), "duplicated message ids in history") + check(len(texts) == len(set(texts)), "a retried request created a duplicate message") + lost = [m for m in self.confirmed if by_id.get(m["id"]) != m] + check(not lost, f"{len(lost)} confirmed messages are lost or changed") + return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0} + + +def summary(load, start, end): + """Успешные RPS, ошибки и задержки на отрезке [start, end).""" + events = load.window(start, end) + writes = [e for e in events if e[1] == "write"] + ok_writes = sorted(e[5] for e in writes if e[2]) + errors = [e for e in events if not e[2]] + seconds = max(end - start, 1e-6) + + def percentile(p): + return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None + + codes = {} + for e in errors: + codes[str(e[3])] = codes.get(str(e[3]), 0) + 1 + return { + "seconds": round(seconds, 2), + "write_rps": round(len(ok_writes) / seconds, 1), + "reads": len([e for e in events if e[1] == "read"]), + "errors": len(errors), + "error_codes": codes, + "write_p50_ms": percentile(0.50), + "write_p95_ms": percentile(0.95), + "write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None, + } diff --git a/student-tests/redis_info.py b/student-tests/redis_info.py new file mode 100644 index 0000000..6535eae --- /dev/null +++ b/student-tests/redis_info.py @@ -0,0 +1,29 @@ +import subprocess + +from common import ROOT, login, new_user, redis_cli, session_key + + +def without_password(): + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"], + cwd=ROOT, capture_output=True, check=False) + return (result.stdout + result.stderr).decode("utf-8", "replace").strip() + + +def main(): + user = new_user("info") + token = login(user["username"], user["password"]).json["token"] + key = session_key(token) + print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}") + print(f"TTL of the key, seconds: {redis_cli('ttl', key)}") + keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines() + print(f"sessions in Redis: {len(keys)}") + for name in ("appendonly", "appendfsync", "maxmemory-policy"): + print("config", " = ".join(redis_cli("config", "get", name).split())) + for line in redis_cli("info", "persistence").splitlines(): + if line.startswith(("aof_enabled", "aof_last_write_status")): + print(line) + print(f"without password: {without_password()}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 0000000..f2f3ffb --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,88 @@ +import json +import os +import time + +from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced +from loadgen import Load, summary + +OUTPUT = ROOT / "evidence" / "rolling-update.json" + + +def env_of(container): + lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +def wait_new_replica(container, up_needed, seconds=90): + deadline = time.monotonic() + seconds + while health(container) != "healthy" or lb_up_count() < up_needed: + check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") + time.sleep(0.2) + + +def main(): + old = api_containers() + replicas = len(old) + check(replicas >= 2, "need at least 2 running api replicas") + wait_balanced(replicas) + old_version = env_of(old[0]).get("APP_VERSION") + version = time.strftime("rolling-%Y%m%d-%H%M%S") + env = dict(os.environ, APP_VERSION=version) + + print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") + started = time.monotonic() + compose("build", "api", "migrate", env=env) + print(f"image built in {time.monotonic() - started:.1f} s") + migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) + print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] + .split("messenger.db: ")[-1]) + + load = Load() + load.start() + time.sleep(3) + t0 = time.monotonic() + steps = [] + for container in old: + step_start = time.monotonic() + before = set(api_containers()) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) + new = sorted(set(api_containers()) - before) + check(len(new) == 1, f"expected one new container, got {len(new)}") + wait_new_replica(new[0], replicas + 1) + ready = time.monotonic() - step_start + docker("stop", "--time", "15", container) + docker("rm", container) + steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), + "step_seconds": round(time.monotonic() - step_start, 2)}) + print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " + f"step {steps[-1]['step_seconds']} s") + t1 = time.monotonic() + time.sleep(3) + load.stop() + + final = api_containers() + versions = sorted({env_of(c).get("APP_VERSION") for c in final}) + integrity = load.verify() + during = summary(load, t0, t1) + before_stats = summary(load, t0 - 3, t0) + print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") + print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " + f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " + f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") + print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " + f"lost 0, duplicates 0") + check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") + check(versions == [version], "not every replica runs the new version") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "from_version": old_version, "to_version": version, "replicas": replicas, + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, + "steps": steps, "update_seconds": round(t1 - t0, 2), + "before": before_stats, "during": during, "integrity": integrity, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: all {replicas} replicas replaced one by one under load, " + f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_failover.py b/student-tests/run_failover.py new file mode 100644 index 0000000..0f28ec8 --- /dev/null +++ b/student-tests/run_failover.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys + +from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced + +HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + + +def ps(): + return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() + + +def aim(wanted): + """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" + probe = call("GET", "/health/live", 200).instance + if not any(container.startswith(probe) for container in wanted): + call("GET", "/health/live", 200) # теперь последней ответила другая реплика + + +def main(): + containers = api_containers() + check(len(containers) >= 2, "need at least 2 running api replicas") + print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") + stopped_once = set() + for attempt in range(1, 7): + wait_balanced(len(containers)) + if len(containers) == 2: + aim(set(containers) - stopped_once) + print(f"=== run {attempt}: api containers before") + print(ps()) + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public failover test failed") + print("api containers after") + print(ps()) + stopped = [container for container in containers if container not in api_containers()] + check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") + print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " + f"other api containers kept running, postgres/redis/lb were not touched") + stopped_once.add(stopped[0]) + docker("start", stopped[0]) + wait_balanced(len(containers)) + print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") + if stopped_once == set(containers): + break + check(stopped_once == set(containers), "not every replica was stopped by the public test") + print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_persistence.py b/student-tests/run_persistence.py new file mode 100644 index 0000000..fada376 --- /dev/null +++ b/student-tests/run_persistence.py @@ -0,0 +1,47 @@ +import os +import subprocess +import sys +import time + +from common import BASE_URL, ROOT, check, compose_output, docker + +HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + + +def started(): + containers = compose_output("ps", "-q").split() + lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines() + return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)} + + +def main(): + before = started() + print("before: container, ID, started at") + for name, (cid, at) in sorted(before.items()): + print(f" {name:<28} {cid[:12]} {at}") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + begin = time.monotonic() + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + seconds = time.monotonic() - begin + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public persistence test failed") + after = started() + print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?") + restarted = [] + for name, (cid, at) in sorted(after.items()): + same_id = before.get(name, ("", ""))[0] == cid + new_start = before.get(name, ("", ""))[1] != at + print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}") + check(same_id, f"{name}: container was recreated, volumes/IDs must stay") + if new_start: + restarted.append(name.split("-")[-2]) + for service in ("postgres", "redis", "s3", "rabbitmq", "api", "janitor", "dispatcher", "worker", "lb"): + check(service in restarted, f"{service} was not restarted by the hook") + print("PASS: the hook restarted postgres, redis, s3 (SeaweedFS), rabbitmq, all api replicas, janitor, dispatcher, worker and lb (same containers, " + "new start time); data, the attachment bytes and the old token survived") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 0000000..7c770d9 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,258 @@ +import base64 +import datetime as dt +import uuid + +from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send + + +def parse_time(value): + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + + +def test_duplicate_username(): + body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} + call("POST", "/api/v1/users", 201, body=body) + call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user")) + print("PASS: duplicate username returns 409") + + +def test_invalid_json(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", "/api/v1/users", 400, raw=b'{"username":') + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json", + headers={"Idempotency-Key": new_key()}) + print("PASS: invalid JSON returns 400") + + +def test_wrong_fields(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD}) + call("POST", "/api/v1/users", 400, + body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True}) + # Отправителя нельзя подставить: он берётся из сессии. + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()}) + print("PASS: wrong and extra fields return 400") + + +def test_foreign_chat(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + chat = new_chat(alice, bob) + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"]) + send(eve, chat["id"], "not my chat", expected=404) + # Несуществующий чат выглядит так же, как чужой. + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"]) + print("PASS: foreign chat returns 404") + + +def test_empty_message(): + alice = new_user("alice") + chat = new_chat(alice) + for text in ("", " ", "\n\t"): + send(alice, chat["id"], text, expected=400) + send(alice, chat["id"], "я" * 4001, expected=400) + send(alice, chat["id"], "я" * 4000) + print("PASS: empty and too long messages return 400, 4000 chars is ok") + + +def test_pagination(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)] + + history, pages = read_history(bob, chat["id"], limit=2) + check(pages == 4, f"expected 4 pages, got {pages}") + check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages") + + other = new_chat(alice) + for i in range(3): + send(alice, other["id"], f"Other {i}") + first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) + print("PASS: pagination works across multiple pages, foreign cursor returns 400") + + +def test_empty_cursor(): + alice = new_user("alice") + chat = new_chat(alice) + send(alice, chat["id"], "one message") + for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"): + for query in ("?cursor=", "?cursor", "?limit=10&cursor="): + call("GET", path + query, 400, token=alice["token"]) + call("GET", path, 200, token=alice["token"]) + print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page") + + +def test_chat_list_pagination(): + alice, bob = new_user("alice"), new_user("bob") + created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] + seen, cursor = [], None + while True: + query = "?limit=1" + (f"&cursor={cursor}" if cursor else "") + page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json + seen += [chat["id"] for chat in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + check(seen == created, "chat list pages are wrong") + print("PASS: chat list pagination") + + +def test_unicode(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob, title="Чат 👋") + text = "Привет 👋 你好" + check(send(alice, chat["id"], text).json["text"] == text, "text changed on write") + history, _ = read_history(bob, chat["id"]) + check(history[-1]["text"] == text, "text changed on read") + print("PASS: unicode text is stored as is") + + +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}, + headers={"Idempotency-Key": new_key()}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + + +def test_user_has_no_password(): + body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} + user = call("POST", "/api/v1/users", 201, body=body).json + check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}") + print("PASS: user response has no password or hash") + + +def test_password_rules(): + def register(password, expected): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password} + call("POST", "/api/v1/users", expected, body=body) + + register("x" * 11, 400) + register("x" * 129, 400) + register(" " * 12, 400) + register("x" * 12, 201) + register("я" * 128, 201) + print("PASS: password length 12..128, blank password is rejected") + + +def test_login_gives_token(): + username = "lg_" + uuid.uuid4().hex[:10] + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Login", "password": PASSWORD}).json + reply = login(username, PASSWORD) + session = reply.json + check(session["token_type"] == "Bearer", "token_type must be Bearer") + check(session["user"] == user, "session user differs from registered user") + check(len(session["token"]) >= 43, "token is too short") + check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque") + check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached") + left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s") + me = call("GET", "/api/v1/users/me", token=session["token"]).json + check(me == user, "/users/me returned another user") + print("PASS: Basic login returns opaque Bearer token with expiry") + + +def test_wrong_password(): + user = new_user("wp") + wrong = login(user["username"], "Wrong-password-000", expected=401) + unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) + check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") + check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") + print("PASS: wrong password and unknown user both return the same 401") + + +def test_malformed_basic(): + no_colon = base64.b64encode(b"just_username").decode() + for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"): + headers = {} if header is None else {"Authorization": header} + call("POST", "/api/v1/auth/sessions", 401, headers=headers) + print("PASS: malformed Basic header returns 401") + + +def test_password_with_colon_and_unicode(): + password = "пароль:с:двоеточием 👋" + user = new_user("colon", password=password) + call("GET", "/api/v1/users/me", token=user["token"]) + print("PASS: password with colon and unicode works") + + +def test_bearer_required(): + user = new_user("br") + for headers in ({}, {"Authorization": basic(user["username"], user["password"])}, + {"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}): + reply = call("GET", "/api/v1/users/me", 401, headers=headers) + check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer") + call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]}) + print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401") + + +def test_x_user_id_ignored(): + alice, bob = new_user("alice"), new_user("bob") + me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json + check(me["id"] == alice["id"], "X-User-Id changed the current user") + print("PASS: X-User-Id does not change the current user") + + +def test_auth_before_body(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) + print("PASS: 401 comes before 400 and 404") + + +def test_logout(): + user = new_user("lo") + second = login(user["username"], user["password"]).json["token"] + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + call("GET", "/api/v1/users/me", 401, token=user["token"]) + call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива + again = login(user["username"], user["password"]).json["token"] + check(again != user["token"], "new login must give a new token") + call("GET", "/api/v1/users/me", 200, token=again) + print("PASS: logout revokes only the current session") + + +if __name__ == "__main__": + run_tests([ + test_duplicate_username, + test_invalid_json, + test_wrong_fields, + test_foreign_chat, + test_empty_message, + test_pagination, + test_empty_cursor, + test_chat_list_pagination, + test_unicode, + test_nul_character_is_stored, + test_user_has_no_password, + test_password_rules, + test_login_gives_token, + test_wrong_password, + test_malformed_basic, + test_password_with_colon_and_unicode, + test_bearer_required, + test_x_user_id_ignored, + test_auth_before_body, + test_logout, + ]) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 0000000..e57fe28 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,338 @@ +"""Вложения через балансировщик: корректные файлы, лимиты, битые файлы, подмена MIME, доступ. + +Только HTTP, Docker не нужен. Каждый негативный случай проверяет точный код ответа: +пользовательская ошибка не должна становиться 500. +""" +import hashlib +import http.client +import socket +import struct +import urllib.parse +import uuid +import zlib + +import images +from common import BASE_URL, call, check, new_chat, new_key, new_user, post_message, read_history, run_tests, upload + +URL = urllib.parse.urlsplit(BASE_URL) + + +def setup(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + return alice, bob, eve, new_chat(alice, bob) + + +def sha(data): + return hashlib.sha256(data).hexdigest() + + +def check_error(reply, code): + check(reply.json["error"]["code"] == code, f"expected error code {code}, got {reply.json['error']['code']}") + + +def test_png_and_jpeg_roundtrip(): + alice, bob, _, chat = setup() + for data, ctype, name in ((images.sample_png(), "image/png", "sample.png"), + (images.sample_jpeg(), "image/jpeg", "sample.jpg"), + (images.progressive_jpeg(), "image/jpeg", "progressive.jpg"), + (images.png(33, 17, interlace=1), "image/png", "interlaced.png"), + (images.png(1, 1, gray=True), "image/png", "1x1.png")): + att = upload(alice, chat["id"], data, ctype, name).json + check(att["status"] == "ready" and att["error_code"] is None, "not ready") + check([v["name"] for v in att["variants"]] == ["thumbnail", "crop", "optimized"], "lab5 variants missing") + check(att["sha256"] == sha(data) and att["size_bytes"] == len(data), "checksum/size differ") + check(att["content_type"] == ctype and att["filename"] == name, "metadata differs") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta == att, "GET metadata differs from POST response") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "downloaded bytes differ") + check(got.headers["content-type"] == ctype, "wrong Content-Type on download") + check(got.headers.get("content-length") == str(len(data)), "wrong Content-Length") + check(got.headers.get("x-content-type-options") == "nosniff", "nosniff is missing") + print(f"{name}: {len(data)} bytes, sha256 {att['sha256'][:12]}..., downloaded by the other member, same bytes") + print("PASS: PNG, JPEG, progressive JPEG, interlaced PNG and 1x1 PNG are stored and returned byte for byte") + + +def test_size_limit_exact_boundary(): + alice, _, _, chat = setup() + exact = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES) + check(len(exact) == images.MAX_UPLOAD_BYTES, "fixture must be exactly 10 MiB") + att = upload(alice, chat["id"], exact).json + check(att["size_bytes"] == images.MAX_UPLOAD_BYTES and att["sha256"] == sha(exact), "10 MiB file changed") + print(f"exactly {len(exact)} bytes (10 MiB) -> 202, then ready") + over = images.png(64, 64, pad_to=images.MAX_UPLOAD_BYTES + 1) + reply = upload(alice, chat["id"], over, expected=413) + check_error(reply, "payload_too_large") + print(f"{len(over)} bytes (10 MiB + 1) with Content-Length -> 413 before reading the body") + print("PASS: the 10 MiB limit is exact") + + +def chunked_upload(user, chat_id, total, piece=64 * 1024): + """Тело без Content-Length (Transfer-Encoding: chunked): сервер узнаёт размер только при чтении.""" + conn = http.client.HTTPConnection(URL.hostname, URL.port or 80, timeout=60) + + def body(): + sent = 0 + head = images.sample_png()[:64] + yield head + sent += len(head) + while sent < total: + part = bytes(min(piece, total - sent)) + sent += len(part) + yield part + try: + conn.request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=big.png", body=body(), + headers={"Content-Type": "image/png", "Authorization": "Bearer " + user["token"]}, + encode_chunked=True) + reply = conn.getresponse() + return reply.status, reply.read() + finally: + conn.close() + + +def test_size_limit_streaming(): + alice, _, _, chat = setup() + status, data = chunked_upload(alice, chat["id"], 12 * 1024 * 1024) + check(status == 413 and b"payload_too_large" in data, f"chunked 12 MiB: expected 413, got {status}") + print("12 MiB without Content-Length (chunked) -> 413 as soon as the 10 MiB limit is passed") + call("GET", "/health/ready", 200) + print("PASS: oversized streamed body is cut off, the API keeps working") + + +def test_pixel_limit(): + alice, _, _, chat = setup() + at_limit = images.png(8000, 5000, gray=True) # ровно 40 000 000 пикселей + att = upload(alice, chat["id"], at_limit, filename="40mpx.png").json + print(f"8000x5000 = 40 000 000 pixels ({len(at_limit)} bytes compressed) -> 202, ready {att['id'][:8]}") + over = images.png(8001, 5000, gray=True) + check_error(upload(alice, chat["id"], over, filename="over.png", expected=413), "payload_too_large") + print(f"8001x5000 = 40 005 000 pixels ({len(over)} bytes) -> 413") + bomb = images.png(100_000, 100_000, idat=b"x\x9c\x03\x00\x00\x00\x00\x01") + check_error(upload(alice, chat["id"], bomb, filename="bomb.png", expected=413), "payload_too_large") + print(f"header claims 100000x100000 in {len(bomb)} bytes -> 413 from the header, nothing is decoded") + print("PASS: pixel limit is checked before decompression") + + +def test_damaged_images(): + alice, _, _, chat = setup() + png, jpeg = images.sample_png(), images.sample_jpeg() + cases = [ + ("PNG cut in half", png[:len(png) // 2], "image/png"), + ("PNG without IEND", png[:-12], "image/png"), + ("PNG with a flipped byte in IDAT (CRC mismatch)", images.flip_byte(png, len(png) // 2), "image/png"), + ("PNG with garbage after IEND", png + b"garbage", "image/png"), + ("PNG signature only", png[:8], "image/png"), + ("PNG with less pixel data than the header says", images.png(10, 10, idat=zlib.compress(b"\x00" * 5)), "image/png"), + ("JPEG cut in half", jpeg[:len(jpeg) // 2], "image/jpeg"), + ("JPEG header only", jpeg[:20], "image/jpeg"), + ("random bytes", uuid.uuid4().bytes * 100, "image/png"), + ("plain text", "это не картинка".encode(), "image/jpeg"), + ] + for name, data, ctype in cases: + reply = upload(alice, chat["id"], data, ctype, "broken.bin", expected=415) + check_error(reply, "unsupported_media_type") + print(f"{name} ({len(data)} bytes, {ctype}) -> 415") + check_error(upload(alice, chat["id"], b"", expected=400), "invalid_request") + print("empty body -> 400") + print("PASS: damaged and foreign files are rejected with 415, never 500") + + +def test_png_chunk_semantics(): + """CRC у всех чанков верные, но содержимое нарушает спецификацию PNG -> 415, а не 201. + + Регрессия: PNG 1x1, глубина 8, палитровый, PLTE из 771 байта (257 записей) раньше получал 201 ready, + хотя Pillow его отклоняет (invalid palette size). + """ + alice, bob, _, chat = setup() + from images import assemble, ihdr, zero_idat + pal = lambda entries: (b"PLTE", bytes(3 * entries)) + idat_p = zero_idat(1, 1, 8, 1) + bad = [ + ("PLTE 771 bytes = 257 entries (review case)", + assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(771)), (b"IDAT", zlib.compress(b"\0\0")))), + ("PLTE length not a multiple of 3", assemble(ihdr(1, 1, 8, 3), (b"PLTE", bytes(4)), idat_p)), + ("empty PLTE", assemble(ihdr(1, 1, 8, 3), (b"PLTE", b""), idat_p)), + ("palette larger than bit depth allows (depth 1, 3 entries)", + assemble(ihdr(1, 1, 1, 3), pal(3), zero_idat(1, 1, 1, 1))), + ("palette image without PLTE", assemble(ihdr(1, 1, 8, 3), idat_p)), + ("PLTE in a grayscale image", assemble(ihdr(1, 1, 8, 0), pal(2), zero_idat(1, 1, 8, 1))), + ("PLTE after IDAT", assemble(ihdr(1, 1, 8, 3), idat_p, pal(2))), + ("two PLTE chunks", assemble(ihdr(1, 1, 8, 3), pal(2), pal(2), idat_p)), + ("tRNS longer than the palette", assemble(ihdr(1, 1, 8, 3), pal(2), (b"tRNS", bytes(3)), idat_p)), + ("tRNS before PLTE", assemble(ihdr(1, 1, 8, 3), (b"tRNS", bytes(1)), pal(2), idat_p)), + ("tRNS of wrong size for RGB (4 bytes instead of 6)", + assemble(ihdr(1, 1, 8, 2), (b"tRNS", bytes(4)), zero_idat(1, 1, 8, 3))), + ("tRNS in an image with alpha channel", + assemble(ihdr(1, 1, 8, 6), (b"tRNS", bytes(2)), zero_idat(1, 1, 8, 4))), + ("IEND with data", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"IEND", b"x"))), + ("unknown critical chunk", assemble(ihdr(1, 1, 8, 2), (b"ABCD", b""), zero_idat(1, 1, 8, 3))), + ("IDAT chunks not consecutive", + assemble(ihdr(1, 2, 8, 2), (b"IDAT", zero_idat(1, 2, 8, 3)[1][:5]), (b"tEXt", b"k\0v"), + (b"IDAT", zero_idat(1, 2, 8, 3)[1][5:]))), + ("gAMA after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"gAMA", bytes(4)))), + ("gAMA of 2 bytes instead of 4", assemble(ihdr(1, 1, 8, 2), (b"gAMA", bytes(2)), zero_idat(1, 1, 8, 3))), + ("3 MiB metadata chunk before IDAT", assemble(ihdr(1, 1, 8, 2), (b"prVt", bytes(3 * 1024 * 1024)), + zero_idat(1, 1, 8, 3))), + ("private chunk after IDAT", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"prVt", bytes(100)))), + ("animated PNG frame (fdAT)", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"fdAT", bytes(10)))), + ("tIME of 2 bytes instead of 7", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tIME", bytes(2)))), + ("tEXt without keyword separator", assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"tEXt", b"nokey"))), + ("zTXt with broken zlib after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0garbage"))), + ("zTXt text bomb: 50 MB of text in ~50 KB after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"k\0\0" + zlib.compress(bytes(50_000_000))))), + ] + for name, data in bad: + check_error(upload(alice, chat["id"], data, filename="bad.png", expected=415), "unsupported_media_type") + print(f"{name} ({len(data)} bytes, valid CRCs) -> 415") + good = [ + ("palette 256 entries + tRNS + tEXt and tIME after IDAT", + assemble(ihdr(2, 2, 8, 3), pal(256), (b"tRNS", bytes(256)), zero_idat(2, 2, 8, 1), + (b"tEXt", b"Comment\0ok"), (b"tIME", struct.pack(">HBBBBB", 2026, 10, 22, 12, 0, 0)))), + ("palette depth 2 with 4 entries", assemble(ihdr(3, 1, 2, 3), pal(4), zero_idat(3, 1, 2, 1))), + ("RGB with 6-byte tRNS and gAMA", assemble(ihdr(1, 1, 8, 2), (b"gAMA", struct.pack(">I", 45455)), + (b"tRNS", bytes(6)), zero_idat(1, 1, 8, 3))), + ("compressed zTXt (100 KB of text) and iTXt after IDAT", + assemble(ihdr(1, 1, 8, 2), zero_idat(1, 1, 8, 3), (b"zTXt", b"Comment\0\0" + zlib.compress(b"a" * 100_000)), + (b"iTXt", b"Title\0\1\0ru\0\0" + zlib.compress("Привет".encode())))), + ] + for name, data in good: + att = upload(alice, chat["id"], data, filename="good.png").json + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == data, "valid PNG changed") + print(f"{name} -> 201, downloaded byte for byte") + print("PASS: chunk contents are checked, not only CRCs; valid PNGs with metadata are still accepted") + + +def test_mime_substitution(): + alice, _, _, chat = setup() + cases = [ + ("PNG bytes sent as image/jpeg", images.sample_png(), "image/jpeg"), + ("JPEG bytes sent as image/png", images.sample_jpeg(), "image/png"), + ("GIF bytes sent as image/png", images.GIF, "image/png"), + ("PNG sent as image/gif", images.sample_png(), "image/gif"), + ("PNG sent as text/html", images.sample_png(), "text/html"), + ("PNG sent as application/octet-stream", images.sample_png(), "application/octet-stream"), + ("PNG sent as multipart/form-data", images.sample_png(), "multipart/form-data; boundary=x"), + ] + for name, data, ctype in cases: + check_error(upload(alice, chat["id"], data, ctype, expected=415), "unsupported_media_type") + print(f"{name} -> 415") + att = upload(alice, chat["id"], images.sample_png(), "IMAGE/PNG; charset=binary").json + check(att["content_type"] == "image/png", "content type must be normalised") + print("PNG sent as 'IMAGE/PNG; charset=binary' -> 201, stored as image/png") + print("PASS: the type is taken from the content; a declared type that does not match is refused") + + +def test_filenames(): + alice, _, _, chat = setup() + bad = ["../../etc/passwd", "a/b.png", "a\\b.png", "a\x00b.png", "line\nbreak.png", "tab\t.png", + "bell\x07.png", "del\x7f.png", "", " ", ".", "..", "x" * 256] + for name in bad: + check_error(upload(alice, chat["id"], images.sample_png(), filename=name, expected=400), "invalid_request") + print(f"{len(bad)} dangerous or invalid names (path, slashes, NUL, control characters, empty, 256 chars) -> 400") + check_error(upload(alice, chat["id"], images.sample_png(), filename=None, expected=400), "invalid_request") + print("no filename -> 400") + for name in ("фото отпуска 😀.png", "x" * 255, "..hidden.png", "a b%20c?.png"): + att = upload(alice, chat["id"], images.sample_png(), filename=name).json + check(att["filename"] == name, "display name changed") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + check("filename*=UTF-8''" in got.headers.get("content-disposition", ""), "Content-Disposition is not encoded") + print("Unicode, 255 chars, leading dots, %20 and ? are kept as display names only") + print("PASS: filename never becomes a path; dangerous names are refused") + + +def test_access_control(): + alice, bob, eve, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + for suffix in ("", "/content"): + reply = call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 404, token=eve["token"]) + check_error(reply, "not_found") + call("GET", f"/api/v1/attachments/{att['id']}{suffix}", 401) + call("GET", f"/api/v1/attachments/{uuid.uuid4()}{suffix}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/not-a-uuid{suffix}", 400, token=alice["token"]) + print("outsider -> 404 for metadata and bytes (same as a non-existent id); no token -> 401; bad UUID -> 400") + check_error(upload(eve, chat["id"], images.sample_png(), expected=404), "not_found") + check_error(upload(alice, str(uuid.uuid4()), images.sample_png(), expected=404), "not_found") + upload(None, chat["id"], images.sample_png(), expected=401) + print("upload into someone else's / unknown chat -> 404, without token -> 401") + call("GET", f"/api/v1/attachments/{att['id']}/content?variant=preview", 400, token=bob["token"]) + for variant in ("original", "thumbnail", "crop", "optimized"): + call("GET", f"/api/v1/attachments/{att['id']}/content?variant={variant}", 200, token=bob["token"]) + print("variant=preview -> 400 (not in the contract); original/thumbnail/crop/optimized -> 200 (lab5)") + print("PASS: only chat members can see an attachment") + + +def test_message_attachments(): + alice, bob, eve, chat = setup() + first = upload(alice, chat["id"], images.sample_png()).json + second = upload(bob, chat["id"], images.sample_jpeg(), "image/jpeg", "b.jpg").json + msg = post_message(alice, chat["id"], "", [second["id"], first["id"]]).json + check(msg["attachment_ids"] == [second["id"], first["id"]] and msg["text"] == "", "order or text changed") + print("text='' with 2 attachments -> 201, order kept; any member may attach a file uploaded by another member") + plain = post_message(bob, chat["id"], "без вложений").json + check(plain["attachment_ids"] == [], "default attachment_ids must be []") + reused = post_message(bob, chat["id"], "то же вложение ещё раз", [first["id"]]).json + history, _ = read_history(bob, chat["id"]) + check(history == [msg, plain, reused], "history differs from the confirmed messages") + print("history returns attachment_ids for every message, [] for a message without files; reuse is allowed") + + other_chat = new_chat(alice, eve) + foreign = upload(alice, other_chat["id"], images.sample_png()).json + for ids, what in (([foreign["id"]], "attachment of another chat (author is a member of both)"), + ([first["id"], foreign["id"]], "own + foreign attachment"), + ([str(uuid.uuid4())], "unknown attachment_id")): + check_error(post_message(alice, chat["id"], "x", ids, expected=404), "not_found") + print(f"{what} -> 404") + check_error(post_message(eve, chat["id"], "x", [first["id"]], expected=404), "not_found") + print("outsider referencing an attachment of the chat -> 404") + for ids, text, what in (([first["id"], first["id"]], "x", "duplicate ids"), + ([str(uuid.uuid4()) for _ in range(11)], "x", "11 ids"), + (["not-a-uuid"], "x", "bad UUID"), + ([], "", "empty text without attachments"), + ([], " ", "whitespace text without attachments")): + check_error(post_message(alice, chat["id"], text, ids, expected=400), "invalid_request") + print(f"{what} -> 400") + history_after, _ = read_history(bob, chat["id"]) + check(history_after == history, "a rejected message was stored") + print("PASS: only attachments of the same chat can be linked; rejected requests change nothing") + + +def test_idempotency_with_attachments(): + alice, bob, _, chat = setup() + att = upload(alice, chat["id"], images.sample_png()).json + other = upload(alice, chat["id"], images.sample_jpeg(), "image/jpeg").json + key = new_key() + first = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + again = post_message(alice, chat["id"], "фото", [att["id"]], key=key).json + check(again == first, "replay returned a different message") + check_error(post_message(alice, chat["id"], "фото", [other["id"]], key=key, expected=409), "conflict") + check_error(post_message(alice, chat["id"], "фото", [att["id"], other["id"]], key=key, expected=409), "conflict") + key2 = new_key() + plain = post_message(alice, chat["id"], "текст", None, key=key2).json + check(post_message(alice, chat["id"], "текст", [], key=key2).json == plain, "[] must equal a missing field") + history, _ = read_history(bob, chat["id"]) + check(len(history) == 2, "replays created extra messages") + print("same key + same attachments -> same 201; other attachments -> 409; missing field == []") + print("PASS: Idempotency-Key covers attachment_ids") + + +def test_aborted_upload(): + alice, _, _, chat = setup() + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: {URL.hostname}\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\nContent-Length: 5000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(1_000_000)) + sock.close() # клиент пропал на середине тела + print("sent 1.1 MB of a declared 5 MB body and closed the connection") + att = upload(alice, chat["id"], images.sample_png()).json + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=alice["token"]) + print("the API keeps serving uploads; nothing of the aborted request is visible " + "(DB/S3 side is checked by test_consistency.py)") + print("PASS: an interrupted upload is dropped") + + +if __name__ == "__main__": + run_tests([test_png_and_jpeg_roundtrip, test_size_limit_exact_boundary, test_size_limit_streaming, + test_pixel_limit, test_damaged_images, test_png_chunk_semantics, test_mime_substitution, test_filenames, test_access_control, + test_message_attachments, test_idempotency_with_attachments, test_aborted_upload]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 0000000..01f2251 --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,103 @@ +from collections import Counter + +import images +from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send, upload + + +def test_requests_spread_over_replicas(): + alice = new_user("alice") + replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)] + served = Counter(reply.instance for reply in replies) + print(f"20 x GET /users/me with one token -> replicas: {dict(served)}") + check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned") + check(len(served) >= 2, "all requests went to one replica: balancing does not work") + print("PASS: one session works on every replica, no second login (sessions are shared in Redis)") + + +def test_write_on_one_read_on_other(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = send(alice, chat["id"], "written on one replica") + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]) + check(sent.json in reply.json["items"], "confirmed message is not visible on another replica") + readers.add(reply.instance) + print(f"message written by {sent.instance}; read back by {sorted(readers)}") + check(readers - {sent.instance}, "no read went to another replica") + print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)") + + +def test_logout_seen_by_all_replicas(): + user = new_user("logout") + out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)] + print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}") + check(len(set(after)) >= 2, "only one replica answered") + print("PASS: logout on one replica revokes the session on all of them") + + +def test_idempotent_retry_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "retry me", key=key) + for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику + retry = send(alice, chat["id"], "retry me", key=key) + if retry.instance != first.instance: + break + print(f"first POST on {first.instance} -> 201 {first.json['id']}; " + f"same key on {retry.instance} -> {retry.status} {retry.json['id']}") + check(retry.instance != first.instance, "retry never reached another replica") + check(retry.json == first.json, "retry on another replica returned a different message") + items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"] + check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate") + print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message") + + +def test_attachment_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + sent = upload(alice, chat["id"], data) + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/attachments/{sent.json['id']}/content", 200, token=bob["token"]) + check(reply.data == data, "another replica returned different bytes") + readers.add(reply.instance) + print(f"file uploaded via {sent.instance}; downloaded byte for byte via {sorted(readers)}") + check(readers - {sent.instance}, "no download went to another replica") + print("PASS: a file uploaded through one replica is downloaded through another (shared S3, nothing local)") + + +def test_instance_header_everywhere(): + containers = api_containers() + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + replies = [ + call("GET", "/health/live", 200), + call("GET", "/health/ready", 200), + send(alice, chat["id"], "hello"), + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}), + call("GET", "/api/v1/users/me", 401), + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]), + call("POST", "/api/v1/users", 409, + body={"username": alice["username"], "display_name": "x", "password": alice["password"]}), + ] + for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер + check(any(container.startswith(reply.instance) for container in containers), + f"X-Instance-Id {reply.instance} is not a running api container") + print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container " + f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}") + print("PASS: X-Instance-Id is in every response and points to a real container") + + +if __name__ == "__main__": + run_tests([ + test_requests_spread_over_replicas, + test_write_on_one_read_on_other, + test_logout_seen_by_all_replicas, + test_idempotent_retry_on_other_replica, + test_attachment_on_other_replica, + test_instance_header_everywhere, + ]) diff --git a/student-tests/test_consistency.py b/student-tests/test_consistency.py new file mode 100644 index 0000000..ff3d4ab --- /dev/null +++ b/student-tests/test_consistency.py @@ -0,0 +1,276 @@ +"""Согласованность PostgreSQL и S3 при отказах; работа janitor (компенсация и очистка). + +Нужен запущенный стенд из compose.yaml. Тест останавливает/замораживает S3 (`docker compose stop|pause s3`), +выполняет код приложения внутри контейнера api (имитация падения между S3 PUT и COMMIT) и запускает +janitor вручную. Данные синтетические, volumes не удаляются. + + python student-tests/test_consistency.py [s3|crash|orphan|aborted|all] +""" +import json +import socket +import sys +import time +import urllib.parse + +import images +from common import (BASE_URL, app_python, call, check, compose, compose_output, docker, janitor, new_chat, + new_user, post_message, run_tests, sql_value, upload, wait_ready) + +URL = urllib.parse.urlsplit(BASE_URL) + +# Выполняется внутри контейнера api: настоящие attachments.store / storage / db, но enqueue_upload +# (lab5: UPDATE ... state='queued' + задание + outbox, COMMIT) падает — как если бы реплика умерла +# сразу после PUT в staging, до фиксации задания. +CRASH_AFTER_PUT = r''' +import json, struct, sys, tempfile, zlib +from uuid import UUID +import attachments, db, storage, uploads +chat_id, owner_id = UUID(sys.argv[1]), UUID(sys.argv[2]) +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +data = (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 0, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(b"\x00\x80\x80\x80\x80" * 4)) + chunk(b"IEND", b"")) +f = tempfile.SpooledTemporaryFile(); f.write(data) +import hashlib +upload = uploads.Upload(f, len(data), hashlib.sha256(data).hexdigest()) +image = uploads.inspect(upload, "image/png") +created = {} +real_intent = db.create_upload_intent +def remember(attachment_id, *args): + created["id"], created["key"] = str(attachment_id), args[-1] + return real_intent(attachment_id, *args) +def crash(attachment_id, *args): + raise RuntimeError("simulated crash between S3 PUT and DB COMMIT") +db.create_upload_intent = remember +db.enqueue_upload = crash +db.pool.open() +try: + attachments.store(chat_id, owner_id, "crash.png", upload, image) + outcome = "stored" +except RuntimeError as exc: + outcome = str(exc) +print(json.dumps({"outcome": outcome, "attachment_id": created["id"], "object_key": created["key"], + "object_exists": storage.exists(created["key"])})) +''' + +OBJECT_EXISTS = r''' +import json, sys, storage +print(json.dumps({"exists": storage.exists(sys.argv[1])})) +''' + +PUT_ORPHAN = r''' +import hashlib, io, json, uuid, storage +data = b"orphan object without a row in PostgreSQL" +key = storage.new_key(uuid.uuid4().hex) +storage.put(key, io.BytesIO(data), len(data), "image/png", hashlib.sha256(data).hexdigest()) +print(json.dumps({"key": key})) +''' + +DELETE_OBJECT = r''' +import json, sys, storage +storage.delete(sys.argv[1]); print(json.dumps({"deleted": sys.argv[1]})) +''' + +COUNT_OBJECTS = r''' +import json, storage +print(json.dumps({"objects": sum(1 for prefix in storage.ALL_PREFIXES for _ in storage.list_objects(prefix))})) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def object_exists(key): + return last_json(app_python(OBJECT_EXISTS, key))["exists"] + + +def backdate(where): + sql_value(f"UPDATE attachments SET created_at = created_at - interval '2 hours' WHERE {where}") + + +def uploading_rows(chat_id): + return int(sql_value(f"SELECT count(*) FROM attachments WHERE chat_id = '{chat_id}' AND state = 'uploading'")) + + +def wait_download(user, att, seconds=60): + deadline = time.monotonic() + seconds + while True: + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", (200, 503), token=user["token"]) + if reply.status == 200: + return reply + check(time.monotonic() < deadline, "download did not recover after S3 start") + time.sleep(1) + + +def s3_outage(stop_action, start_action, label): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + data = images.sample_png() + att = upload(alice, chat["id"], data).json + before = uploading_rows(chat["id"]) + print(f"--- S3 {label}: docker compose {stop_action} s3") + compose(stop_action, "s3") + try: + started = time.monotonic() + reply = upload(alice, chat["id"], data, expected=503) + ms = (time.monotonic() - started) * 1000 + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + body = reply.data.decode().lower() + check(not any(word in body for word in ("seaweed", "bucket", "s3:8333", "originals/", "traceback")), + "error leaks storage details") + print(f"upload -> 503 unavailable in {ms:.0f} ms, no storage details in the body") + reply = call("GET", f"/api/v1/attachments/{att['id']}/content", 503, token=bob["token"]) + print("download of an existing file -> 503") + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"]).json + check(meta["sha256"] == att["sha256"], "metadata changed") + post_message(alice, chat["id"], "текст без файла") + post_message(alice, chat["id"], "", [att["id"]]) + call("GET", "/health/ready", 200) + print("metadata, messages (also with an already stored attachment) and /health/ready keep working: " + "S3 is needed only for file bytes") + after = uploading_rows(chat["id"]) + print(f"uploading rows of this chat: before {before}, after the failed upload {after} " + "(kept as a pointer for janitor if the rollback could not reach S3); not visible through the API") + finally: + print(f"docker compose {start_action} s3") + compose(start_action, "s3") + got = wait_download(bob, att) + check(got.data == data, "file changed after S3 recovery") + print("after S3 recovery the old file downloads byte for byte") + upload(alice, chat["id"], data) + print("new upload -> 201") + leftover = uploading_rows(chat["id"]) + if leftover: + backdate(f"chat_id = '{chat['id']}' AND state = 'uploading'") + report = janitor() + print(f"janitor after grace period: {json.dumps(report)}") + check(uploading_rows(chat["id"]) == 0, "abandoned upload rows remain") + print(f"PASS: S3 {label} -> 503 for file operations only, no visible half-uploads, everything cleaned up") + + +def test_s3_stopped(): + s3_outage("stop", "start", "stopped (connection refused / name not resolved)") + + +def test_s3_frozen(): + s3_outage("pause", "unpause", "frozen (no answer, timeouts)") + + +def test_crash_between_s3_and_db(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + result = last_json(app_python(CRASH_AFTER_PUT, chat["id"], alice["id"])) + check(result["outcome"].startswith("simulated crash"), f"unexpected outcome {result['outcome']}") + att_id, key = result["attachment_id"], result["object_key"] + state = sql_value(f"SELECT state FROM attachments WHERE id = '{att_id}'") + check(state == "uploading" and result["object_exists"], "expected an uploading row and an object in S3") + print(f"crash after S3 PUT: row {att_id[:8]} state={state}, object {key} exists in S3") + call("GET", f"/api/v1/attachments/{att_id}", 404, token=alice["token"]) + call("GET", f"/api/v1/attachments/{att_id}/content", 404, token=alice["token"]) + post_message(alice, chat["id"], "x", [att_id], expected=404) + print("the half-written attachment is invisible: GET -> 404, attach to a message -> 404") + + report = janitor() + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1" and object_exists(key), + "fresh upload was deleted inside the grace period") + print(f"janitor with default grace ({report['grace_seconds']} s) right away: {json.dumps(report)} -> " + "fresh row and object are kept (the upload might still be running)") + backdate(f"id = '{att_id}'") + report = janitor("--dry-run") + check(report["stale_uploads"] >= 1 and report["stale_uploads_deleted"] == 0, "dry run must only report") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "1", "dry run deleted a row") + print(f"row made 2 h old; janitor --dry-run: {json.dumps(report)} -> nothing deleted") + report = janitor() + check(report["stale_uploads_deleted"] >= 1, "janitor did not reclaim the abandoned upload") + check(sql_value(f"SELECT count(*) FROM attachments WHERE id = '{att_id}'") == "0", "row still exists") + check(not object_exists(key), "object still exists") + print(f"janitor: {json.dumps(report)} -> row and object deleted") + report = janitor() + print(f"janitor again: {json.dumps(report)} -> idempotent") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment damaged by cleanup") + print("the ready attachment of the same chat is untouched") + print("PASS: S3 written but DB not committed -> invisible, reclaimed by janitor only after the grace period") + + +def test_orphan_objects(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + ready = upload(alice, chat["id"], images.sample_png()).json + key = last_json(app_python(PUT_ORPHAN))["key"] + print(f"object {key} put into the bucket without a row in PostgreSQL") + # Короткий grace только для теста: janitor требует grace >= 2 x UPLOAD_TIMEOUT_SECONDS, поэтому задаём 1 с. + short = {"UPLOAD_TIMEOUT_SECONDS": "1"} + report = janitor("--grace-seconds", "3", env=short) + check(object_exists(key), "fresh orphan deleted inside grace period") + print(f"janitor --grace-seconds 3 immediately: {json.dumps(report)} -> fresh object kept") + time.sleep(4) + report = janitor("--grace-seconds", "3", "--dry-run", env=short) + check(report["orphan_objects"] >= 1 and report["orphan_objects_deleted"] == 0 and object_exists(key), + "dry run must report and keep the orphan") + print(f"after 4 s, --dry-run: {json.dumps(report)} -> reported, kept") + report = janitor("--grace-seconds", "3", env=short) + check(report["orphan_objects_deleted"] >= 1 and not object_exists(key), "orphan not deleted") + print(f"janitor: {json.dumps(report)} -> orphan deleted") + got = call("GET", f"/api/v1/attachments/{ready['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "ready attachment was deleted with orphans") + print("ready attachments older than the grace period are untouched (they have rows)") + + victim = upload(alice, chat["id"], images.sample_png(), filename="victim.png").json + victim_key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{victim['id']}'") + app_python(DELETE_OBJECT, victim_key) + report = janitor("--grace-seconds", "3", env=short) + check(report["missing_objects"] >= 1, "missing object not reported") + check(sql_value(f"SELECT state FROM attachments WHERE id = '{victim['id']}'") == "ready", "row was deleted") + reply = call("GET", f"/api/v1/attachments/{victim['id']}/content", 503, token=bob["token"]) + print(f"object of a ready attachment deleted behind the API's back: janitor reports missing_objects=" + f"{report['missing_objects']} and keeps the row; download -> {reply.status} (not 500)") + print("PASS: orphans are found and deleted only after the grace period; missing objects are reported") + + +def api_tmp_files(): + counts = {} + for container in compose_output("ps", "-q", "api").split(): + counts[container[:12]] = int(docker("exec", container, "sh", "-c", "ls -A /tmp/uploads | wc -l") or 0) + return counts + + +def test_aborted_upload_leaves_nothing(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + objects = last_json(app_python(COUNT_OBJECTS))["objects"] + rows = int(sql_value("SELECT count(*) FROM attachments")) + for sent in (100_000, 3_000_000): + sock = socket.create_connection((URL.hostname, URL.port or 80), timeout=10) + head = (f"POST /api/v1/chats/{chat['id']}/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\n" + f"Authorization: Bearer {alice['token']}\r\nContent-Type: image/png\r\n" + f"Content-Length: 9000000\r\n\r\n") + sock.sendall(head.encode() + images.sample_png() + bytes(sent)) + sock.close() + print(f"declared 9 000 000 bytes, sent {sent + len(images.sample_png())}, closed the connection") + time.sleep(2) + check(int(sql_value("SELECT count(*) FROM attachments")) == rows, "an aborted upload left a row") + check(last_json(app_python(COUNT_OBJECTS))["objects"] == objects, "an aborted upload left an S3 object") + print(f"rows in attachments: {rows} -> {rows}; objects in bucket: {objects} -> {objects}") + tmp = api_tmp_files() + check(all(n == 0 for n in tmp.values()), f"temporary files left: {tmp}") + print(f"temporary upload files in api containers: {tmp} (removed when the request ends)") + for container in compose_output("ps", "-q", "api").split(): + mounts = docker("inspect", "-f", "{{json .Mounts}}", container) + # файловые secrets compose монтируются read-only в /run/secrets — это не хранилище данных + storage = [m for m in json.loads(mounts) + if not (m.get("Destination", "").startswith("/run/secrets/") and m.get("RW") is False)] + check(storage == [], f"api must not have volumes: originals live only in S3, got {storage}") + print("api containers have no volumes: the original is never stored locally") + print("PASS: an interrupted upload leaves no row, no object and no local file") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + groups = {"s3": [test_s3_stopped, test_s3_frozen], "crash": [test_crash_between_s3_and_db], + "orphan": [test_orphan_objects], "aborted": [test_aborted_upload_leaves_nothing]} + groups["all"] = groups["aborted"] + groups["crash"] + groups["orphan"] + groups["s3"] + check(target in groups, "usage: python student-tests/test_consistency.py [s3|crash|orphan|aborted|all]") + wait_ready() + run_tests(groups[target]) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 0000000..3d71fcc --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,101 @@ +import time +import uuid + +from common import call, check, compose_output, new_user, psql, run_tests + + +def sql_value(sql): + code, output, error = psql(sql) + check(code == 0, f"psql failed: {error}") + return output + + +def test_create_chat_rolls_back(): + alice = new_user("rollback") + title = "rollback-" + uuid.uuid4().hex[:8] + inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + call("POST", "/api/v1/chats", 400, token=alice["token"], + body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) + + left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") + orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") + inserted_after = inserted_before + deadline = time.monotonic() + 15 + while inserted_after == inserted_before and time.monotonic() < deadline: + time.sleep(0.5) + inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; " + f"chats insert counter: {inserted_before} -> {inserted_after}") + check(left == "0" and orphans == "0", "chat row survived the failed transaction") + check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown") + print("PASS: chat row was inserted and then rolled back together with members") + + +def test_message_needs_membership_in_db(): + code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") + print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") + check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") + print("PASS: the database itself rejects a message from a non-member") + + +def test_password_is_argon2id(): + user = new_user("hash") + stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'") + print(f"stored password: {stored[:31]}... (length {len(stored)})") + check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash") + print("PASS: password is stored as Argon2id hash") + + +def test_unique_username_constraint(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'users'::regclass AND contype = 'u'") + print(f"unique constraint on users: {constraint}") + check(constraint == "UNIQUE (username)", "no unique constraint on username") + print("PASS: username is unique in the database") + + +def test_migrations_are_not_repeated(): + versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + print(f"applied migrations: {versions}; running the migrate service once more") + output = compose_output("run", "--rm", "--no-deps", "-T", "migrate") + again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1] + print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}") + check(again == versions and "none" in last_log, "migrations were applied again") + print("PASS: migrations are applied once, a repeated run does nothing") + + +def test_replicas_do_not_migrate(): + """lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему.""" + logs = compose_output("logs", "api").splitlines() + checked = [line for line in logs if "Schema is up to date" in line] + migrated = [line for line in logs if "Migrations applied" in line] + migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': " + f"{len(migrated)}; migrate service: {migrate_exit}") + check(checked and not migrated, "api replicas must not apply migrations") + check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0") + print("PASS: only the migrate service applies migrations, replicas wait for the schema") + + +def test_idempotency_key_is_unique_in_db(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conname = 'messages_idempotency_key'") + print(f"unique constraint for Idempotency-Key: {constraint}") + check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys") + print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database") + + +if __name__ == "__main__": + run_tests([ + test_create_chat_rolls_back, + test_message_needs_membership_in_db, + test_password_is_argon2id, + test_unique_username_constraint, + test_migrations_are_not_repeated, + test_replicas_do_not_migrate, + test_idempotency_key_is_unique_in_db, + ]) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 0000000..e6017ce --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,57 @@ +import datetime as dt +import os +import time + +from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready + +SHORT_TTL = 2 + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env + else: + env["SESSION_TTL_SECONDS"] = str(ttl) + compose("up", "-d", "--no-deps", "--wait", "api", env=env) + wait_ready() + + +def seconds_left(expires_at): + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def test_session_expires(): + print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}") + restart_api(SHORT_TTL) + try: + user = new_user("ttl") + session = login(user["username"], user["password"]).json + token = session["token"] + left = seconds_left(session["expires_at"]) + print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: " + f"{redis_cli('pttl', session_key(token))} ms") + check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions") + + call("GET", "/api/v1/users/me", 200, token=token) + print("before expiry: GET /users/me -> 200") + + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0) + call("GET", "/api/v1/users/me", 401, token=token) + exists = redis_cli("exists", session_key(token)) + print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}") + check(exists == "0", "Redis must delete the expired session key") + finally: + print("recreating api with SESSION_TTL_SECONDS from .env") + restart_api() + + user = new_user("ttl_back") + left = seconds_left(login(user["username"], user["password"]).json["expires_at"]) + print(f"normal API again: new session expires in {left:.0f} s") + check(left > 300, "normal TTL was not restored") + print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401") + + +if __name__ == "__main__": + run_tests([test_session_expires]) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 0000000..8eaef38 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,110 @@ +import sys +import time +import uuid + +from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready + +FAKE_TOKEN = "A" * 43 + + +def timed(method, path, expected, **kwargs): + started = time.monotonic() + reply = call(method, path, expected, **kwargs) + return reply, (time.monotonic() - started) * 1000 + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + before = send(alice, chat["id"], "confirmed before outage").json + return alice, bob, chat, before + + +def recovered(alice, chat, before): + wait_ready() + history, _ = read_history(alice, chat["id"]) + check(before in history, "message confirmed before the outage is lost or changed") + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the old token works, confirmed messages are in place") + + +def test_redis_down(): + alice, bob, chat, before = prepare() + print("stopping redis") + compose("stop", "redis") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") + call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) + print("unknown token -> 503, not 200: nobody is trusted without a check") + call("GET", "/api/v1/chats", 401) + print("no token -> 401") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + print("login -> 503: there is nowhere to store the session") + call("POST", "/api/v1/users", 201, + body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("registration -> 201: it needs only PostgreSQL") + finally: + print("starting redis") + compose("start", "redis") + recovered(alice, chat, before) + print("PASS: Redis outage gives 503 and does not let anyone in without a session check") + + +def test_redis_crash(): + alice = new_user("crash") + print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") + compose("kill", "redis") + try: + # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. + status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status + print(f"right after the crash: GET /users/me -> {status}") + finally: + print("starting redis") + compose("start", "redis") + wait_ready() + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the session issued before the crash still works (AOF, appendfsync always)") + print("PASS: sessions survive a Redis crash, not only a clean restart") + + +def test_postgres_down(): + alice, bob, chat, before = prepare() + print("stopping postgres") + compose("stop", "postgres") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + for method, path, body in (("GET", "/api/v1/users/me", None), + ("GET", "/api/v1/chats", None), + ("GET", f"/api/v1/chats/{chat['id']}/messages", None), + ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): + reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + check("postgres" not in reply.data.decode().lower(), "error must not show database details") + print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") + call("GET", "/api/v1/chats", 401) + call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) + print("no token -> 401, unknown token -> 401: anonymous users are not let in") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + call("POST", "/api/v1/users", 503, + body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("login -> 503, registration -> 503") + finally: + print("starting postgres") + compose("start", "postgres") + recovered(alice, chat, before) + print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], + "all": [test_redis_down, test_redis_crash, test_postgres_down]} + check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") + run_tests(tests[target]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 0000000..407dea3 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,174 @@ +import json +import socket +import threading +import time +import urllib.parse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send + +PARALLEL = 20 + + +def post_raw(user, chat_id, raw, key): + """POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"], + raw=raw, headers={"Idempotency-Key": key}) + + +def texts_in(user, chat_id): + history, _ = read_history(user, chat_id) + return [message["text"] for message in history] + + +def test_replay_returns_same_message(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "Привет 👋", key=key) + # То же JSON-значение, записанное иначе: пробелы и \u-экранирование. + same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key) + again = send(alice, chat["id"], "Привет 👋", key=key) + print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}") + check(same.status == 201 and same.json == first.json and again.json == first.json, + "retry must return the original 201 and the same message") + check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate") + print("PASS: a retry with the same key returns the original 201 and message, no duplicate") + + +def test_same_key_parallel_through_replicas(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(_): + barrier.wait() # все запросы стартуют одновременно + return send(alice, chat["id"], "one message for 20 requests", key=key) + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + replies = list(pool.map(post, range(PARALLEL))) + bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies} + served = Counter(reply.instance for reply in replies) + count = texts_in(bob, chat["id"]).count("one message for 20 requests") + print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, " + f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}") + check(len(served) >= 2, "requests did not go through different replicas") + check(len(bodies) == 1 and count == 1, "parallel retries created more than one message") + print("PASS: parallel requests with one key through different replicas create exactly one message") + + +def test_same_key_different_bodies_parallel(): + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(index): + text = "body A" if index % 2 == 0 else "body B" + barrier.wait() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"], + body={"text": text}, headers={"Idempotency-Key": key}) + return text, reply + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(post, range(PARALLEL))) + texts = texts_in(alice, chat["id"]) + winner = [t for t in texts if t in ("body A", "body B")] + print(f"{PARALLEL} parallel POSTs, one key, two different bodies: " + f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}") + check(len(winner) == 1, "exactly one message must be created") + for text, reply in results: + expected = 201 if text == winner[0] else 409 + check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}") + print("PASS: the first body wins, the same body gets 201, the other body gets 409") + + +def test_key_scope(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + other = new_chat(alice, bob) + key = new_key() + a1 = send(alice, chat["id"], "scope", key=key).json + b1 = send(bob, chat["id"], "scope", key=key).json + a2 = send(alice, other["id"], "scope", key=key).json + print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages") + check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat") + call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"], + body={"text": "other text"}, headers={"Idempotency-Key": key}) + print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409") + + +def test_key_validation(): + alice = new_user("alice") + chat = new_chat(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": "no key"}) + for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"): + call("POST", path, 400, token=alice["token"], body={"text": "bad key"}, + headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8 + longest = ("Az09._:-" * 16)[:128] + call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest}) + print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201") + print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}") + + +def test_failed_request_does_not_burn_key(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key}) + call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key}) + created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key}) + print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}") + print("PASS: rejected requests (400, 404) do not store a result for the key") + + +def test_replay_checks_session_and_access(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + send(alice, chat["id"], "private", key=key) + stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}") + print("PASS: a retry checks the session and chat access again, the saved answer is not leaked") + + +def test_lost_response_then_retry(): + """Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен.""" + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + body = json.dumps({"text": "answer was lost"}).encode("utf-8") + url = urllib.parse.urlsplit(BASE_URL) + request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection: + connection.sendall(request + body) # и сразу закрываем, ответ не читаем + time.sleep(1) + retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"], + body={"text": "answer was lost"}, headers={"Idempotency-Key": key}) + count = texts_in(alice, chat["id"]).count("answer was lost") + print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}") + check(count == 1, "the retry after a lost answer created a duplicate") + print("PASS: an unconfirmed POST can be safely retried with the same key") + + +if __name__ == "__main__": + run_tests([ + test_replay_returns_same_message, + test_same_key_parallel_through_replicas, + test_same_key_different_bodies_parallel, + test_key_scope, + test_key_validation, + test_failed_request_does_not_burn_key, + test_replay_checks_session_and_access, + test_lost_response_then_retry, + ]) diff --git a/student-tests/test_jobs.py b/student-tests/test_jobs.py new file mode 100644 index 0000000..b382ee2 --- /dev/null +++ b/student-tests/test_jobs.py @@ -0,0 +1,567 @@ +"""Lab5: асинхронная обработка вложений — варианты, сохранность, отказы, повторы, DLQ, replay. + +Нужен запущенный стенд из compose.yaml с FAULT_INJECTION=1 (по умолчанию на учебном стенде). +Тест управляет только своими синтетическими данными и сервисами через `docker compose stop|start|restart`, +volumes не удаляет. Сбои внедряются файлами в volume faults (faults.py), SQL — только чтение. + + python student-tests/test_jobs.py [сценарий ...] # без аргументов — все по порядку + +Сценарии: variants limits worker-down broker-down outbox-crash crash-after-put crash-after-commit + lease retry exhausted-replay permanent concurrency dlq +""" +import base64 +import json +import sys +import time +import urllib.parse +from pathlib import Path + +from common import (call, check, compose, compose_output, new_chat, new_user, post_message, psql, run_tests, + send, sql_value, wait_ready) + +HERE = Path(__file__).resolve().parent +PROBE = (HERE / "worker_probe.py").read_text(encoding="utf-8") +FINAL_KEYS = ["v1/crop.jpg", "v1/optimized.webp", "v1/original", "v1/thumbnail.jpg"] +LEASE = 15 # JOB_LEASE_SECONDS стенда +TIMEOUT = 60 # JOB_TIMEOUT_SECONDS стенда +MAX_ATTEMPTS = 5 # JOB_MAX_ATTEMPTS стенда + +STATE = {} + + +# ---------- инфраструктура ---------- + +def in_container(service, code, *args): + """python - в контейнере service, код через stdin. Последняя строка stdout — JSON. + Если service остановлен (сценарии с выключенным worker) — в одноразовом контейнере того же сервиса.""" + import subprocess + from common import ROOT + running = subprocess.run(["docker", "compose", "ps", "--status", "running", "-q", service], cwd=ROOT, + capture_output=True, check=False).stdout.strip() + launch = (["exec", "-T", service, "python"] if running + else ["run", "--rm", "--no-deps", "-T", "--entrypoint", "python", service]) + result = subprocess.run(["docker", "compose", *launch, "-", *map(str, args)], cwd=ROOT, + input=code.encode("utf-8"), capture_output=True, check=False) + check(result.returncode == 0, f"{service}: {result.stderr.decode('utf-8', 'replace')[-500:]}") + return json.loads(result.stdout.decode("utf-8").strip().splitlines()[-1]) + + +def probe(*args): + return in_container("worker", PROBE, *args) + + +def ops(*args): + import subprocess + from common import ROOT + result = subprocess.run(["docker", "compose", "exec", "-T", "dispatcher", "python", "ops.py", *map(str, args)], + cwd=ROOT, capture_output=True, check=False) + check(result.returncode == 0, f"ops {args[0]}: {result.stderr.decode('utf-8', 'replace')[-400:]}") + return json.loads(result.stdout.decode("utf-8")) + + +def fault(point, times=1): + """Взвести точку сбоя: times файлов /faults/. (каждое срабатывание удаляет один файл).""" + code = ("import pathlib, sys, time\n" + "for n in range(int(sys.argv[2])):\n" + " pathlib.Path('/faults', f'{sys.argv[1]}.{time.time_ns()}_{n}').touch()\n" + "print('{}')\n") + in_container("worker", code, point, times) + + +def clear_faults(): + code = "import pathlib\nfor p in pathlib.Path('/faults').glob('*'):\n p.unlink()\nprint('{}')\n" + in_container("worker", code) + + +def image(kind): + return base64.b64decode(probe("make", kind)["b64"]) + + +def user_and_chat(): + if "alice" not in STATE: + STATE["alice"] = new_user("alice") + STATE["bob"] = new_user("bob") + STATE["chat"] = new_chat(STATE["alice"], STATE["bob"], title="Lab5 jobs") + return STATE["alice"], STATE["bob"], STATE["chat"] + + +def upload(data, content_type="image/png", filename="probe.png", expected=202): + alice, _bob, chat = user_and_chat() + query = "?filename=" + urllib.parse.quote(filename, safe="") + att = call("POST", f"/api/v1/chats/{chat['id']}/attachments{query}", expected, raw=data, token=alice["token"], + headers={"Content-Type": content_type}, timeout=60).json + if expected == 202: + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, + f"202 must be a queued snapshot, got {att['status']}") + return att + + +def get(att_id, user=None): + user = user or STATE["alice"] + return call("GET", f"/api/v1/attachments/{att_id}", 200, token=user["token"]).json + + +def wait_status(att_id, statuses, seconds=60): + statuses = (statuses,) if isinstance(statuses, str) else statuses + deadline = time.monotonic() + seconds + while True: + att = get(att_id) + if att["status"] in statuses: + return att + check(time.monotonic() < deadline, f"{att_id[:8]}: still {att['status']} after {seconds} s") + time.sleep(0.3) + + +def content(att_id, variant="original", expected=200): + return call("GET", f"/api/v1/attachments/{att_id}/content?variant={variant}", expected, + token=STATE["bob"]["token"]) + + +def job(att_id): + rows = sql_value(f"SELECT job_id || '|' || status || '|' || attempt || '|' || coalesce(error_code, '-') " + f"FROM attachment_jobs WHERE attachment_id = '{att_id}'") + check(rows and "\n" not in rows, f"expected exactly one job for {att_id}") + job_id, status, attempt, error = rows.split("|") + return {"job_id": job_id, "status": status, "attempt": int(attempt), "error_code": error} + + +def events(job_id): + out = sql_value(f"SELECT attempt || ':' || event || ':' || coalesce(detail, '') FROM job_events " + f"WHERE job_id = '{job_id}' ORDER BY id") + return out.splitlines() if out else [] + + +def names(evs): + return [e.split(":")[1] for e in evs] + + +def assert_one_result(att_id, label=""): + """Инвариант идемпотентности: ровно один набор вариантов и ровно 4 объекта под конечным prefix.""" + info = probe("inspect", att_id) + check(info["state"] == "ready", f"{label}: not ready in DB") + check(info["variant_rows"] == 3, f"{label}: {info['variant_rows']} variant rows, expected 3") + check(info["final_keys"] == FINAL_KEYS, f"{label}: final objects {info['final_keys']}") + check(info["original_sha_ok"] and info["original_size_ok"], f"{label}: original differs from upload") + check(all(v["sha_ok"] and v["meta_size_ok"] for v in info["variants"].values()), f"{label}: variant mismatch") + return info + + +def outbox_rows(job_id): + out = sql_value(f"SELECT attempt || ':' || (published_at IS NOT NULL) FROM outbox WHERE job_id = '{job_id}' " + f"ORDER BY id") + return out.splitlines() if out else [] + + +def queue_depth(): + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_queues", "-p", "messenger", + "name", "messages", "messages_unacknowledged") + depth = {} + for line in out.splitlines(): + parts = line.split() + if len(parts) == 3 and parts[1].isdigit(): + depth[parts[0]] = (int(parts[1]), int(parts[2])) + return depth + + +def log(text): + print(text, flush=True) + + +# ---------- сценарии ---------- + +def test_variants(): + """thumbnail/crop/optimized по контракту, EXIF orientation, alpha -> белый, без увеличения, без метаданных.""" + alice, bob, chat = user_and_chat() + cases = { + "thumb320": ("image/png", {"thumbnail": [256, 160], "crop": [128, 128], "optimized": [320, 200]}), + "exif6": ("image/jpeg", {"thumbnail": [192, 256], "crop": [128, 128], "optimized": [300, 400]}), + "alpha": ("image/png", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [300, 200]}), + "tiny": ("image/png", {"thumbnail": [40, 30], "crop": [128, 128], "optimized": [40, 30]}), + "large": ("image/jpeg", {"thumbnail": [256, 170], "crop": [128, 128], "optimized": [1280, 853]}), + "strip": ("image/png", {"thumbnail": [256, 1], "crop": [128, 128], "optimized": [1280, 1]}), + } + for kind, (ctype, sizes) in cases.items(): + data = image(kind) + att = upload(data, ctype, f"{kind}.{'jpg' if ctype == 'image/jpeg' else 'png'}") + ready = wait_status(att["id"], "ready") + meta = {v["name"]: v for v in ready["variants"]} + check(set(meta) == {"thumbnail", "crop", "optimized"}, f"{kind}: variants {sorted(meta)}") + for name, size in sizes.items(): + check([meta[name]["width"], meta[name]["height"]] == size, + f"{kind}/{name}: {meta[name]['width']}x{meta[name]['height']}, expected {size}") + check(meta["thumbnail"]["content_type"] == "image/jpeg" and meta["crop"]["content_type"] == "image/jpeg" + and meta["optimized"]["content_type"] == "image/webp", f"{kind}: wrong variant content types") + original = content(att["id"]) + check(original.data == data, f"{kind}: original is not byte-identical") + for name in ("thumbnail", "crop", "optimized"): + reply = content(att["id"], name) + check(reply.headers["content-type"] == meta[name]["content_type"], f"{kind}/{name}: Content-Type") + info = assert_one_result(att["id"], kind) + for name, v in info["variants"].items(): + check(v["exif_tags"] == 0 and not v["has_xmp"], f"{kind}/{name}: metadata not stripped") + if kind == "exif6": + top = info["variants"]["thumbnail"]["pixel_top_center"] + check(top[0] > 200 and top[1] < 60, f"exif6: orientation not applied, top pixel {top}") + if kind == "alpha": + left = info["variants"]["thumbnail"]["pixel_left_center"] + right = info["variants"]["thumbnail"]["pixel_right_center"] + check(left[:3] == [255, 255, 255] and right[2] > 200, f"alpha: not composited on white {left} {right}") + check(info["variants"]["optimized"]["mode"] == "RGBA", "alpha: optimized WebP lost transparency") + log(f"{kind}: " + ", ".join(f"{n} {s[0]}x{s[1]}" for n, s in sizes.items()) + + f"; original byte-identical; EXIF/XMP in variants: 0; S3 keys {info['final_keys']}") + # Отсутствующий вариант у ready -> 404 (нет такого имени в контракте -> 400), queued -> 409 проверяется ниже. + content(att["id"], "unknown", expected=400) + log("unknown variant name -> 400; variants carry their own size/sha256, original fields describe the upload") + + +def test_limits(): + """Лимиты: bytes и пиксели на API; память/пиксели/время процесса обработки на worker.""" + alice, _bob, chat = user_and_chat() + big = b"\x89PNG\r\n\x1a\n" + b"\0" * (10 * 1024 * 1024) + call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=big.png", 413, raw=big, token=alice["token"], + headers={"Content-Type": "image/png"}, timeout=60) + log("10 MiB + 8 bytes -> 413 before any job is created") + report = probe("imaging-limits") + at, over = report["at_limit_40mpx"], report["over_limit_max_10mpx"] + check(at["exit_code"] == 0, f"40 Mpx PNG at the limit must be processed, exit {at['exit_code']}") + check(over["exit_code"] == 4, f"worker-side pixel limit must reject (exit 4), got {over['exit_code']}") + check(at["peak_rss_mib_max_children"] < report["memory_limit_mib"], + f"peak RSS {at['peak_rss_mib_max_children']} MiB exceeds RLIMIT_AS {report['memory_limit_mib']} MiB") + log(f"imaging.py on 8000x5000 PNG: exit 0, peak RSS {at['peak_rss_mib_max_children']} MiB " + f"(RLIMIT_AS {report['memory_limit_mib']} MiB); same file with MAX_IMAGE_PIXELS=10M -> exit 4 (image_too_large)") + # Память: процесс обработки упирается в RLIMIT_AS -> MemoryError -> постоянная ошибка resource_limit. + fault("imaging_oom") + att = upload(image("thumb320"), filename="oom.png") + failed = wait_status(att["id"], ("failed", "ready"), 60) + check(failed["status"] == "failed" and failed["error_code"] == "resource_limit", + f"oom: expected failed/resource_limit, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == 1, "resource_limit is permanent: no retries expected") + log(f"memory bomb in the imaging process -> failed/resource_limit after attempt 1; worker itself survived") + STATE["oom_job"] = j["job_id"] + # Время: зависший процесс обработки убивается по JOB_TIMEOUT_SECONDS -> временная ошибка -> retry -> ready. + fault("imaging_hang") + att = upload(image("thumb320"), filename="hang.png") + started = time.monotonic() + ready = wait_status(att["id"], "ready", TIMEOUT + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check("lease_expired" not in names(evs), f"a long-running (alive) attempt must keep its lease: {evs}") + check(any("processing_timeout" in e for e in evs) and j["attempt"] == 2, f"timeout not handled as retry: {evs}") + log(f"imaging hang -> killed after {TIMEOUT} s (processing_timeout, temporary), lease renewed meanwhile " + f"(no lease_expired), attempt 2 ready after {time.monotonic() - started:.0f} s") + + +def test_worker_down(): + """Защита п.2-3: worker остановлен -> 202 -> сообщение с вложением -> рестарт API+broker -> worker -> ready.""" + alice, bob, chat = user_and_chat() + compose("stop", "worker") + log("worker stopped") + data = image("thumb320") + att = upload(data, filename="offline.png") + log(f"upload while worker is down -> 202 queued, attachment {att['id']}") + j = job(att["id"]) + source_key = sql_value(f"SELECT source_key FROM attachments WHERE id = '{att['id']}'") + check(source_key.startswith("staging/"), "source must be in the durable staging prefix") + check(probe("exists", source_key)["exists"], "staging object must exist before 202") + msg = post_message(alice, chat["id"], "photo while worker is down", [att["id"]]).json + check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment") + content(att["id"], expected=409) + content(att["id"], "thumbnail", expected=409) + send(bob, chat["id"], "other messages keep working") + deadline = time.monotonic() + 10 + while outbox_rows(j["job_id"]) != ["1:true"]: + check(time.monotonic() < deadline, "dispatcher did not publish the job") + time.sleep(0.2) + deadline = time.monotonic() + 15 # счётчики quorum-очереди в rabbitmqctl обновляются раз в несколько секунд + while (queue_depth().get("attachments.process.v1") or (0, 0))[0] < 1: + check(time.monotonic() < deadline, "published message is not visible in the queue") + time.sleep(0.5) + depth = queue_depth().get("attachments.process.v1") + log(f"message with attachment 201, content 409, other messages 201; job {j['job_id']} published, " + f"queue messages/unacked = {depth}") + compose("restart", "rabbitmq") + compose("restart", "api") + wait_ready() + depth_after = queue_depth().get("attachments.process.v1") + check(depth_after and depth_after[0] >= 1, f"message lost after broker restart: {depth_after}") + check(get(att["id"])["status"] == "queued", "job must still be queued after restarts") + log(f"rabbitmq and api restarted with volumes; queue still holds {depth_after[0]} message(s); status queued") + compose("start", "worker") + ready = wait_status(att["id"], "ready", 60) + j2 = job(att["id"]) + check(j2["job_id"] == j["job_id"] and j2["status"] == "ready", "same job_id must finish") + thumb = content(att["id"], "thumbnail") + check(len(thumb.data) == {v["name"]: v for v in ready["variants"]}["thumbnail"]["size_bytes"], "thumbnail size") + info = assert_one_result(att["id"], "worker-down") + check(not info["staging_present"] and not probe("exists", source_key)["exists"], + "staging must be cleaned after the safe commit") + log(f"worker started -> same job {j['job_id']} ready, thumbnail downloadable; staging object deleted") + + +def test_broker_down(): + """Брокер недоступен: API принимает (outbox), dispatcher копит и публикует после восстановления.""" + compose("stop", "rabbitmq") + log("rabbitmq stopped") + att = upload(image("thumb320"), filename="no-broker.png") + call("GET", "/health/ready", 200) + j = job(att["id"]) + time.sleep(3) + check(outbox_rows(j["job_id"]) == ["1:false"], "outbox row must wait while the broker is down") + log(f"upload -> 202 and /health/ready 200 without broker; outbox row pending for job {j['job_id']}") + compose("start", "rabbitmq") + wait_status(att["id"], "ready", 90) + check(outbox_rows(j["job_id"]) == ["1:true"], "outbox row must be published after recovery") + assert_one_result(att["id"], "broker-down") + log("rabbitmq started -> dispatcher published the pending row -> ready") + + +def test_outbox_crash(): + """Разрыв COMMIT/publish: dispatcher падает после COMMIT задания до publish и после confirm до отметки.""" + fault("dispatcher_crash_before_publish") + att = upload(image("thumb320"), filename="crash-before-publish.png") + ready = wait_status(att["id"], "ready", 60) + j = job(att["id"]) + logs = compose_output("logs", "--tail", "400", "dispatcher") + check("FAULT INJECTION: dispatcher_crash_before_publish" in logs, "fault did not fire") + check(outbox_rows(j["job_id"]) == ["1:true"] and j["attempt"] == 1, "outbox must publish exactly the same job") + assert_one_result(att["id"], "crash-before-publish") + log(f"dispatcher killed between COMMIT and publish; restarted by policy, outbox re-published -> job " + f"{j['job_id']} ready, attempt 1, no manual action") + fault("dispatcher_crash_after_confirm") + att = upload(image("thumb320"), filename="crash-after-confirm.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + deadline = time.monotonic() + 20 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"duplicate delivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + assert_one_result(att["id"], "crash-after-confirm") + log(f"dispatcher killed after publisher confirm, before marking outbox -> message published twice; " + f"worker: {names(events(j['job_id']))} -> one result set") + + +def test_crash_after_put(): + """Worker падает после записи объектов в S3, до COMMIT/ack: повтор завершает тот же результат.""" + fault("worker_crash_after_put") + att = upload(image("thumb320"), filename="crash-after-put.png") + wait_status(att["id"], "ready", LEASE + 40) + j = job(att["id"]) + evs = events(j["job_id"]) + check(names(evs).count("claimed") == 2 and "lease_expired" in names(evs), f"unexpected history {evs}") + assert_one_result(att["id"], "crash-after-put") + log(f"worker killed after S3 PUT (before COMMIT and ack): redelivery saw a live lease (no-op), lease expired " + f"after {LEASE} s, attempt {j['attempt']} rewrote the same keys -> one set; events {names(evs)}") + + +def test_crash_after_commit(): + """Worker падает после COMMIT результата, до ack: повторная доставка ready-задачи — no-op с ack.""" + fault("worker_crash_after_commit") + att = upload(image("thumb320"), filename="crash-after-commit.png") + wait_status(att["id"], "ready", 30) + j = job(att["id"]) + deadline = time.monotonic() + 30 + while "duplicate_ignored" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"redelivery not observed: {events(j['job_id'])}") + time.sleep(0.3) + evs = events(j["job_id"]) + check(names(evs).count("ready") == 1 and j["attempt"] == 1, f"ready must be recorded once: {evs}") + assert_one_result(att["id"], "crash-after-commit") + deadline = time.monotonic() + 20 # счётчики quorum-очереди обновляются с задержкой + while queue_depth().get("attachments.process.v1") != (0, 0): + check(time.monotonic() < deadline, f"message left in the queue: {queue_depth()}") + time.sleep(0.5) + log(f"worker killed after COMMIT, before ack: broker redelivered, worker acked as no-op ({evs[-1]}); " + f"queue empty (0 ready, 0 unacked)") + STATE["staging_check"] = att["id"] + + +def test_lease(): + """Зависший worker: аренда не продлевается -> dispatcher возвращает job -> другой поток завершает; + результат зависшего отклоняется fencing-токеном, ready не откатывается.""" + fault("worker_freeze") + att = upload(image("thumb320"), filename="frozen.png") + started = time.monotonic() + wait_status(att["id"], "ready", LEASE + 40) + took = time.monotonic() - started + j = job(att["id"]) + deadline = time.monotonic() + 3 * LEASE + 20 + while "stale_result_rejected" not in names(events(j["job_id"])): + check(time.monotonic() < deadline, f"stale result of the frozen attempt not seen: {events(j['job_id'])}") + time.sleep(0.5) + evs = events(j["job_id"]) + check(get(att["id"])["status"] == "ready", "late result must not roll ready back") + check(names(evs).count("ready") == 1, f"exactly one ready: {evs}") + assert_one_result(att["id"], "lease") + log(f"frozen attempt 1 stopped renewing; lease expired, recovered by dispatcher, attempt 2 ready after " + f"{took:.0f} s; frozen attempt woke up -> stale_result_rejected; events {names(evs)}") + + +def test_retry(): + """Временные ошибки S3: backoff с jitter, затем ready; задержки видны в журнале.""" + fault("worker_s3_error", 2) + att = upload(image("thumb320"), filename="retry.png") + wait_status(att["id"], "ready", 60) + j = job(att["id"]) + evs = [e for e in events(j["job_id"]) if ":retry_scheduled:" in e] + check(len(evs) == 2 and j["attempt"] == 3, f"expected 2 retries, got {evs}") + delays = [float(e.split("delay=")[1].rstrip("s")) for e in evs] + check(1.0 <= delays[0] <= 2.0 and 2.0 <= delays[1] <= 4.0, f"backoff out of range: {delays}") + assert_one_result(att["id"], "retry") + log(f"2 x storage_unavailable -> retries after {delays[0]:.2f} s and {delays[1]:.2f} s (equal jitter: " + f"1..2 s, 2..4 s), attempt 3 ready") + + +def test_exhausted_replay(): + """Исчерпание попыток -> failed/retry_exhausted, реестр failed, явный replay; replay ready — отказ.""" + alice, _bob, chat = user_and_chat() + fault("worker_s3_error", MAX_ATTEMPTS) + att = upload(image("thumb320"), filename="exhausted.png") + failed = wait_status(att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", + f"expected failed/retry_exhausted, got {failed['status']}/{failed['error_code']}") + j = job(att["id"]) + check(j["attempt"] == MAX_ATTEMPTS, f"attempts {j['attempt']}") + content(att["id"], expected=409) + post_message(alice, chat["id"], "", [att["id"]], expected=409) + registry = {row["job_id"]: row for row in ops("failed")} + check(registry.get(j["job_id"], {}).get("replayable") is True, "failed job must be in the registry") + log(f"{MAX_ATTEMPTS} temporary errors -> failed/retry_exhausted (public error_code), content 409, " + f"attach 409, listed by ops.py failed (replayable)") + replay = ops("replay", j["job_id"], "--reason", "s3 recovered") + check("replayed" in replay, f"replay refused: {replay}") + wait_status(att["id"], "ready", 60) + assert_one_result(att["id"], "replay") + j2 = job(att["id"]) + refused = ops("replay", j["job_id"]) + check("refused" in refused, "replay of a ready job must be refused") + again = job(att["id"]) + check(again == j2, "refused replay must not change the job") + check(names(events(j["job_id"])).count("replayed") == 1, "audit must contain the replay once") + log(f"ops replay -> same job_id {j['job_id']} attempt {j2['attempt']} ready; second replay refused: " + f"\"{refused['refused']}\"") + + +def test_permanent(): + """Постоянная ошибка (битое изображение в worker) -> failed сразу, без повторов.""" + fault("imaging_invalid") + att = upload(image("thumb320"), filename="invalid.png") + failed = wait_status(att["id"], ("failed", "ready"), 30) + check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", f"got {failed}") + j = job(att["id"]) + check(j["attempt"] == 1 and "retry_scheduled" not in names(events(j["job_id"])), "no retries for permanent") + log("invalid image in the worker -> failed/invalid_image at attempt 1, no retry (permanent error)") + + +def test_concurrency(): + """Ограниченная конкуренция: WORKER_CONCURRENCY потребителей с prefetch=1; одновременно processing <= 2.""" + out = compose_output("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "list_consumers", "-p", "messenger", + "queue_name", "prefetch_count", "ack_required") + lines = [line.split() for line in out.splitlines() if line.strip()] + header = lines[0] if lines and lines[0][0] == "queue_name" else ["queue_name", "prefetch_count", "ack_required"] + consumers = [dict(zip(header, line)) for line in lines if line[0] == "attachments.process.v1"] + check(len(consumers) == 2 and all(c["prefetch_count"] == "1" and c["ack_required"] == "true" + for c in consumers), f"consumers {out}") + # Копим очередь при остановленном worker, затем запускаем: работы больше, чем потоков. + compose("stop", "worker") + data = image("large") + ids = [upload(data, "image/jpeg", f"burst{n}.jpg")["id"] for n in range(8)] + id_list = ", ".join(f"'{i}'" for i in ids) + compose("start", "worker") + started = time.monotonic() + peak, samples = 0, 0 + while True: + busy, ready = map(int, sql_value( + f"SELECT count(*) FILTER (WHERE status = 'processing') || '|' || count(*) FILTER (WHERE status = 'ready') " + f"FROM attachment_jobs WHERE attachment_id IN ({id_list})").split("|")) + peak, samples = max(peak, busy), samples + 1 + if ready == len(ids): + break + check(time.monotonic() - started < 120, "burst not processed in 120 s") + time.sleep(0.05) + took = time.monotonic() - started + check(peak == 2, f"peak {peak} jobs in processing, expected exactly WORKER_CONCURRENCY=2") + log(f"2 consumers, prefetch_count 1, manual ack; 8 queued 3000x2000 JPEG drained in {took:.1f} s after " + f"worker start, peak processing {peak} of 8 ({samples} samples)") + + +def test_dlq(): + """Невалидные сообщения и сообщения о неизвестной job -> DLQ; dlq-replay их не исполняет.""" + publish = r''' +import json, sys, uuid +import broker, pipeline +p = broker.Publisher() +p.open().basic_publish(pipeline.EXCHANGE, pipeline.ROUTING_KEY, b'{"not":"a job"}', mandatory=True) +p.publish(pipeline.build_message(uuid.uuid4(), uuid.uuid4(), 1, 1, None)) +p.close() +print("{}") +''' + before = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + in_container("dispatcher", publish) + deadline = time.monotonic() + 15 + while queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] < before + 2: + check(time.monotonic() < deadline, "messages did not reach the DLQ") + time.sleep(0.3) + peek = ops("dlq", "--limit", "50") + reasons = {(m["invalid"] is not None, m["dead_letter_reason"]) for m in peek} + check((True, "rejected") in reasons, f"invalid message not in DLQ: {peek}") + replayed = ops("dlq-replay", "--limit", "50") + kept = [m for m in replayed if m["action"] == "keep"] + check(len(kept) >= 2, "invalid/unknown messages must be kept, not executed") + after = queue_depth().get("attachments.process.v1.dlq", (0, 0))[0] + check(after >= before + 2, "dlq-replay must not drop messages it cannot handle") + log(f"schema-invalid message and unknown job_id -> reject -> DLQ ({after} message(s)); " + f"dlq-replay kept them: {[m['note'] for m in kept][:2]}") + + +def test_staging_cleanup(): + """Janitor доделывает очистку staging, если worker упал до неё (после crash-after-commit).""" + att_id = STATE.get("staging_check") + if not att_id: + log("skip: run crash-after-commit first") + return + key = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + if key: + log(f"staging source still present after the crash (worker died before cleanup); waiting 60 s window") + time.sleep(62) + from common import janitor + report = janitor() + # фоновый janitor (проход раз в JANITOR_INTERVAL_SECONDS) мог успеть первым в окне 62 с — проверяем результат + left = sql_value(f"SELECT coalesce(source_key, '') FROM attachments WHERE id = '{att_id}'") + check(not left, f"staging source still referenced: {report}") + check(not probe("exists", key)["exists"], "staging object must be gone") + by = "this pass" if report["staging_ready_deleted"] else "the background janitor pass" + log(f"janitor removed the staging source of the ready attachment ({by}); source_key NULL, object gone") + else: + log("staging already cleaned by the worker") + + +SCENARIOS = { + "variants": test_variants, "limits": test_limits, "worker-down": test_worker_down, + "broker-down": test_broker_down, "outbox-crash": test_outbox_crash, "crash-after-put": test_crash_after_put, + "crash-after-commit": test_crash_after_commit, "staging": test_staging_cleanup, "lease": test_lease, + "retry": test_retry, "exhausted-replay": test_exhausted_replay, "permanent": test_permanent, + "concurrency": test_concurrency, "dlq": test_dlq, +} + + +def main(): + chosen = sys.argv[1:] or list(SCENARIOS) + wait_ready() + clear_faults() + tests = [] + for name in chosen: + def run(name=name): + log(f"--- {name}: {SCENARIOS[name].__doc__.splitlines()[0]}") + try: + SCENARIOS[name]() + finally: + compose("start", "worker") # идемпотентно: стенд остаётся рабочим даже после FAIL + compose("start", "rabbitmq") + run.__name__ = name + tests.append(run) + run_tests(tests) + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 0000000..4af44fa --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,57 @@ +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send + +CLIENTS = 10 +MESSAGES_PER_CLIENT = 10 +REGISTRATIONS = 20 + + +def test_concurrent_messages(): + users = [new_user(f"race{i}") for i in range(CLIENTS)] + chat = new_chat(*users, title="race") + barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно + + def client(index): + barrier.wait() + for n in range(MESSAGES_PER_CLIENT): + send(users[index], chat["id"], f"client-{index}-message-{n}") + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков + elapsed = time.monotonic() - started + + history, pages = read_history(users[0], chat["id"], limit=30) + ids = {message["id"] for message in history} + expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)} + print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; " + f"read {len(history)} in {pages} pages, unique ids: {len(ids)}") + check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages") + check(len(ids) == len(history), "duplicate message ids") + check({message["text"] for message in history} == expected, "lost or changed message texts") + print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates") + + +def test_concurrent_registration(): + username = "race_" + uuid.uuid4().hex[:10] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + barrier.wait() + body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD} + return call("POST", "/api/v1/users", (201, 409), body=body).status + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + statuses = list(pool.map(register, range(REGISTRATIONS))) + created, conflicts = statuses.count(201), statuses.count(409) + print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}") + check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win") + print("PASS: concurrent registration created exactly one user") + + +if __name__ == "__main__": + run_tests([test_concurrent_messages, test_concurrent_registration]) diff --git a/student-tests/test_s3_security.py b/student-tests/test_s3_security.py new file mode 100644 index 0000000..ece71a3 --- /dev/null +++ b/student-tests/test_s3_security.py @@ -0,0 +1,146 @@ +"""S3 вне API: bucket приватный, ключ API ограничен, порты и веб-интерфейсы SeaweedFS не открыты, bootstrap идемпотентен. + +Нужен запущенный стенд из compose.yaml. +""" +import json +import socket + +import images +from common import app_python, call, check, compose_output, new_chat, new_user, run_tests, sql_value, upload + +# Анонимные запросы прямо в S3 (SeaweedFS) из сети data (из контейнера api), без подписи и без API. +ANONYMOUS = r''' +import json, sys, urllib.request, urllib.error +import settings +base = settings.S3_ENDPOINT.rstrip("/") + "/" + settings.S3_BUCKET +def status(method, url, data=None): + try: + with urllib.request.urlopen(urllib.request.Request(url, data=data, method=method), timeout=5) as r: + return r.status, r.read(300).decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, e.read(300).decode("utf-8", "replace") +out = {} +for name, method, url, data in (("get_object", "GET", base + "/" + sys.argv[1], None), + ("head_object", "HEAD", base + "/" + sys.argv[1], None), + ("list_bucket", "GET", base + "?list-type=2", None), + ("put_object", "PUT", base + "/originals/anonymous-test", b"x"), + ("get_bucket_policy", "GET", base + "?policy", None)): + code, body = status(method, url, data) + out[name] = {"status": code, "s3_code": body.split("")[1].split("")[0] if "" in body else ""} +print(json.dumps(out)) +''' + +# Что разрешено и запрещено ключу API (политика infra/s3-api-policy.json). +API_KEY_RIGHTS = r''' +import io, json, sys, uuid +from botocore.exceptions import ClientError +import settings, storage +c, bucket = storage._client, settings.S3_BUCKET +def attempt(name, fn): + try: + fn(); return "allowed" + except ClientError as e: + return "denied: " + e.response.get("Error", {}).get("Code", "?") +public = json.dumps({"Version": "2012-10-17", "Statement": [{"Effect": "Allow", "Principal": {"AWS": ["*"]}, + "Action": ["s3:GetObject"], "Resource": [f"arn:aws:s3:::{bucket}/*"]}]}) +key = "originals/" + uuid.uuid4().hex +result = { + "get own object": attempt("get", lambda: c.get_object(Bucket=bucket, Key=sys.argv[1])["Body"].read()), + "put object": attempt("put", lambda: c.put_object(Bucket=bucket, Key=key, Body=b"x")), + "delete object": attempt("delete", lambda: c.delete_object(Bucket=bucket, Key=key)), + "list bucket": attempt("list", lambda: c.list_objects_v2(Bucket=bucket, MaxKeys=1)), + "create another bucket": attempt("create", lambda: c.create_bucket(Bucket="intruder-" + uuid.uuid4().hex[:8])), + "make bucket public (put policy)": attempt("policy", lambda: c.put_bucket_policy(Bucket=bucket, Policy=public)), + "read bucket policy": attempt("get_policy", lambda: c.get_bucket_policy(Bucket=bucket)), + "delete bucket": attempt("delete_bucket", lambda: c.delete_bucket(Bucket=bucket)), + "change versioning": attempt("versioning", lambda: c.put_bucket_versioning( + Bucket=bucket, VersioningConfiguration={"Status": "Enabled"})), +} +try: # хранилище может не отказать в ListBuckets, а показать только доступные ключу bucket + result["visible buckets"] = sorted(b["Name"] for b in c.list_buckets().get("Buckets", [])) +except ClientError as e: + result["visible buckets"] = "denied: " + e.response.get("Error", {}).get("Code", "?") +print(json.dumps(result)) +''' + + +def last_json(output): + return json.loads(output.splitlines()[-1]) + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + att = upload(alice, chat["id"], images.sample_png()).json + key = sql_value(f"SELECT object_key FROM attachments WHERE id = '{att['id']}'") + return alice, bob, att, key + + +def test_bucket_is_private(): + alice, bob, att, key = prepare() + result = last_json(app_python(ANONYMOUS, key)) + for name, reply in result.items(): + print(f"anonymous {name} directly to S3 (from the data network) -> {reply['status']} {reply['s3_code']}") + check(reply["status"] in (401, 403), f"anonymous {name} must be refused, got {reply['status']}") + call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + print("the same file through the API for a chat member -> 200") + print("PASS: the bucket is private: no anonymous read, list, write or policy read") + + +def test_api_key_is_limited(): + alice, bob, att, key = prepare() + result = last_json(app_python(API_KEY_RIGHTS, key)) + allowed = {"get own object", "put object", "delete object", "list bucket"} + visible = result.pop("visible buckets") + print(f"API key: {'list buckets':<32} -> {visible}") + check(isinstance(visible, str) or len(visible) <= 1, "API key must not see other buckets") + for name, outcome in result.items(): + print(f"API key: {name:<32} -> {outcome}") + if name in allowed: + check(outcome == "allowed", f"API key must be allowed to {name}") + else: + check(outcome.startswith("denied"), f"API key must NOT be allowed to {name}") + print("PASS: the API key can only get/put/delete/list objects of its own bucket; " + "bucket admin, policy and other buckets are denied") + + +def test_no_open_ports_or_console(): + config = json.loads(compose_output("config", "--format", "json")) + s3 = config["services"]["s3"] + check(not s3.get("ports"), "s3 must not publish ports") + check(sorted(s3["networks"]) == ["data"], "s3 must be only in the internal data network") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + print("compose: s3 has no published ports, only the internal network 'data' (S3 8333, master UI 9333, " + "filer UI 8888 are reachable only inside the stand)") + for port in (8333, 9333, 8888, 19333): + sock = socket.socket() + sock.settimeout(1) + refused = sock.connect_ex(("127.0.0.1", port)) != 0 + sock.close() + check(refused, f"127.0.0.1:{port} is reachable from the host") + print(f"127.0.0.1:{port} from the host -> connection refused") + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in config["services"].items()} + for name, items in sorted(secrets.items()): + print(f"secrets of {name}: {items or '-'}") + for name in ("api", "janitor"): + check(not set(secrets[name]) & {"s3_root_user", "s3_root_password", "s3_config"}, + f"{name} must not get S3 admin keys or the key config") + check(set(secrets["api"]) >= {"s3_access_key_id", "s3_secret_access_key"}, "api needs its own S3 key") + print("PASS: S3 is reachable only inside the stand; admin keys are only in s3 and s3-init") + + +def test_bootstrap_is_idempotent(): + alice, bob, att, key = prepare() + output = compose_output("run", "--rm", "--no-deps", "s3-init") + print("\n".join("s3-init: " + line for line in output.strip().splitlines() if line.strip())) + check("bucket is private" in output, "bootstrap did not confirm a private bucket") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + check(got.data == images.sample_png(), "file changed after repeated bootstrap") + upload(alice, att["chat_id"], images.sample_png()) + print("after the repeated bootstrap: old file downloads, new upload works") + print("PASS: bucket/user/policy bootstrap can be repeated safely") + + +if __name__ == "__main__": + run_tests([test_bucket_is_private, test_api_key_is_limited, test_no_open_ports_or_console, + test_bootstrap_is_idempotent]) diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 0000000..01f1ced --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,222 @@ +"""Эксперимент: >= 10 одновременных загрузок и пик памяти API-реплик. + +Гипотеза: память реплики почти не растёт от числа и размера одновременных загрузок, потому что +тело пишется во временный файл (в RAM не больше 256 KiB на загрузку), PNG проверяется потоково +(куски по 64 KiB), JPEG декодируется в масштабе 1/8, а одновременно проверяется не больше +DECODE_CONCURRENCY файлов на реплику. Наивный вариант (тело целиком в память + полное декодирование) +при тех же 10 загрузках не помещается в лимит 512 MiB. + +Методика: перед каждым сценарием `docker compose restart api` (сбросить пик), прогрев, затем N потоков +одновременно (barrier) загружают файл через балансировщик. Пик RSS процесса API — VmHWM из /proc/1/status +(high-water mark ядра, не пропускает короткие всплески); дополнительно память cgroup контейнера +(memory.peak, если ядро даёт; иначе максимум из опроса memory.current) и опрос VmRSS раз в ~0,1 с. + + python student-tests/upload_memory.py [--concurrency 10] [--output evidence/upload-memory.json] +""" +import argparse +import hashlib +import json +import platform +import statistics +import threading +import time + +import images +from common import (ROOT, app_python, call, check, compose, docker, new_chat, new_user, upload, wait_attachment, + wait_balanced, wait_ready) + + +def proc_status(container): + """VmRSS и VmHWM (КиБ) процесса API (PID 1) и память cgroup контейнера (байты).""" + script = ("cat /proc/1/status | grep -E '^Vm(RSS|HWM)'; " + "for f in /sys/fs/cgroup/memory.peak /sys/fs/cgroup/memory.current " + "/sys/fs/cgroup/memory/memory.max_usage_in_bytes /sys/fs/cgroup/memory/memory.usage_in_bytes; " + "do [ -r $f ] && echo \"$f $(cat $f)\"; done; true") + out = docker("exec", container, "sh", "-c", script) + values = {} + for line in out.splitlines(): + parts = line.replace(":", " ").split() + if parts[0] in ("VmRSS", "VmHWM"): + values[parts[0]] = int(parts[1]) + elif parts[0].endswith(("memory.peak", "max_usage_in_bytes")): + values["cgroup_peak"] = int(parts[1]) + elif parts[0].endswith(("memory.current", "usage_in_bytes")): + values["cgroup_current"] = int(parts[1]) + return values + + +class Sampler(threading.Thread): + def __init__(self, containers): + super().__init__(daemon=True) + self.containers = containers + self.stop = threading.Event() + self.rss = {c: 0 for c in containers} + self.cgroup = {c: 0 for c in containers} + self.samples = 0 + + def run(self): + while not self.stop.is_set(): + for c in self.containers: + try: + v = proc_status(c) + except Exception: # noqa: BLE001 — один пропущенный замер не важен + continue + self.rss[c] = max(self.rss[c], v.get("VmRSS", 0)) + self.cgroup[c] = max(self.cgroup[c], v.get("cgroup_current", 0)) + self.samples += 1 + + +def api_containers(): + from common import api_containers as running + return running() + + +def fresh_replicas(): + compose("restart", "api") + containers = wait_balanced(2, seconds=90) + wait_ready() + return containers + + +def warm_up(user, chat): + """По несколько загрузок на каждую реплику: в базовую линию входят импортированный Pillow, клиенты S3 и т.п.""" + for data, ctype in ((images.sample_png(), "image/png"), (images.sample_jpeg(), "image/jpeg")) * 4: + upload(user, chat["id"], data, ctype) + + +def scenario(name, payloads, concurrency): + user = new_user("mem") + chat = new_chat(user) + containers = fresh_replicas() + warm_up(user, chat) + before = {c: proc_status(c) for c in containers} + sampler = Sampler(containers) + sampler.start() + barrier = threading.Barrier(concurrency) + results = [None] * concurrency + + def worker(i): + data, ctype = payloads[i % len(payloads)] + barrier.wait() + started = time.monotonic() + try: + # lab5: замеряем только приём (202 queued); обработку worker'ом ждём ниже, вне замера + reply = upload(user, chat["id"], data, ctype, f"load-{i}.png", timeout=180, wait=False) + results[i] = (reply.status, time.monotonic() - started, reply.instance, reply.json, data) + except AssertionError as exc: + results[i] = (str(exc)[:120], time.monotonic() - started, None, None, data) + + threads = [threading.Thread(target=worker, args=(i,)) for i in range(concurrency)] + wall = time.monotonic() + for t in threads: + t.start() + for t in threads: + t.join() + wall = time.monotonic() - wall + time.sleep(0.5) + sampler.stop.set() + sampler.join() + after = {c: proc_status(c) for c in containers} + + ok = [r for r in results if r[0] == 202] + check(len(ok) == concurrency, f"{name}: not all uploads succeeded: {[r[0] for r in results if r[0] != 202]}") + for status, _, _, att, data in ok[:3]: + att = wait_attachment(user, att["id"], seconds=120) + check(att["status"] == "ready", f"{name}: attachment {att['id']} is {att['status']}, expected ready") + got = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=user["token"]) + check(got.data == data and att["sha256"] == hashlib.sha256(data).hexdigest(), "stored file differs") + latencies = sorted(r[1] for r in results) + per_replica = [] + for c in containers: + b, a = before[c], after[c] + per_replica.append({ + "replica": c[:12], + "uploads_served": sum(1 for r in ok if r[2] and c.startswith(r[2])), + "rss_before_mib": round(b["VmRSS"] / 1024, 1), + "rss_peak_mib": round(a["VmHWM"] / 1024, 1), + "rss_growth_mib": round((a["VmHWM"] - b["VmRSS"]) / 1024, 1), + "rss_peak_sampled_mib": round(sampler.rss[c] / 1024, 1), + "cgroup_peak_mib": round(a["cgroup_peak"] / 2**20, 1) if "cgroup_peak" in a else None, + "cgroup_peak_sampled_mib": round(sampler.cgroup[c] / 2**20, 1) if sampler.cgroup[c] else None, + }) + sizes = sorted({len(d) for d, _ in payloads}) + summary = { + "scenario": name, + "concurrency": concurrency, + "file_bytes": sizes, + "total_mib": round(sum(len(r[4]) for r in results) / 2**20, 1), + "all_202": True, + "wall_seconds": round(wall, 2), + "latency_s": {"p50": round(statistics.median(latencies), 2), "max": round(latencies[-1], 2)}, + "sampler_rounds": sampler.samples, + "replicas": per_replica, + } + print(json.dumps(summary, ensure_ascii=False)) + return summary + + +# Для сравнения: то же внутри контейнера api, но «наивно» — тело целиком в памяти и полное декодирование. +NAIVE_VS_STREAMING = r''' +import json, os, subprocess, sys +code = r""" +import io, os, resource, struct, sys, zlib +sys.path.insert(0, os.getcwd()) +os.environ.setdefault("S3_ACCESS_KEY_ID", "x") +from PIL import Image +import uploads # импорт до замера: сравниваем только саму проверку файла +Image.MAX_IMAGE_PIXELS = None +def chunk(k, d): return struct.pack(">I", len(d)) + k + d + struct.pack(">I", zlib.crc32(k + d)) +w, h = 8000, 5000 +comp = zlib.compressobj(6); row = b"\x00" + bytes(w * 3) +idat = b"".join(comp.compress(row) for _ in range(h)) + comp.flush() +data = b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", w, h, 8, 2, 0, 0, 0)) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") +del idat, comp, row +base = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss +if sys.argv[1] == "naive": + body = bytes(data) + bytes(9 * 1024 * 1024) # тело запроса целиком в памяти (9 MiB) + im = Image.open(io.BytesIO(data)); im.load() +else: + f = io.BytesIO(data); uploads.check_png(f) +print((resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - base) // 1024) +""" +out = {} +for mode in ("naive", "streaming"): + r = subprocess.run([sys.executable, "-c", code, mode], capture_output=True, text=True, cwd=os.getcwd()) + out[mode] = int(r.stdout.strip().splitlines()[-1]) if r.returncode == 0 else r.stderr[-200:] +print(json.dumps(out)) +''' + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--output", default="evidence/upload-memory.json") + args = parser.parse_args() + check(args.concurrency >= 10, "the experiment needs at least 10 concurrent uploads") + + print(f"client: Python {platform.python_version()} on {platform.system()} {platform.release()}") + print("preparing synthetic images (random-noise PNG ~9.5 MiB each, 40 Mpx PNG)...") + large = [(images.noise_png(int(9.5 * 2**20)), "image/png") for _ in range(4)] + pixels = [(images.png(8000, 5000), "image/png")] + jpeg = [(images.sample_jpeg(), "image/jpeg")] + print(f"large: {len(large[0][0])} bytes; 40 Mpx: {len(pixels[0][0])} bytes") + + runs = [ + scenario("10 x ~9.5 MiB PNG (noise)", large, args.concurrency), + scenario("10 x 40 Mpx PNG (8000x5000 RGB)", pixels, args.concurrency), + scenario(f"{2 * args.concurrency} mixed: large + 40 Mpx + JPEG", large + pixels + jpeg, 2 * args.concurrency), + ] + naive = json.loads(app_python(NAIVE_VS_STREAMING).splitlines()[-1]) + print(f"one 8000x5000 RGB PNG inside the api container, extra peak RSS: naive (body in RAM + full decode) " + f"{naive['naive']} MiB, streaming check {naive['streaming']} MiB") + limits = docker("inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", api_containers()[0]).split() + result = {"date": time.strftime("%Y-%m-%d %H:%M"), "api_limit_mib": int(limits[0]) // 2**20, + "api_limit_cpus": int(limits[1]) / 1e9, "runs": runs, "single_file_extra_rss_mib": naive} + (ROOT / args.output).write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8") + worst = max(r["rss_growth_mib"] for run in runs for r in run["replicas"]) + print(f"PASS: all uploads succeeded; largest RSS growth of one replica: {worst} MiB " + f"(limit {result['api_limit_mib']} MiB); results in {args.output}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/worker_probe.py b/student-tests/worker_probe.py new file mode 100644 index 0000000..7b75584 --- /dev/null +++ b/student-tests/worker_probe.py @@ -0,0 +1,148 @@ +"""Пробник, который выполняется ВНУТРИ контейнера worker (там Pillow, модули приложения, доступ к БД и S3). + +Тест передаёт этот файл через stdin: `docker compose exec -T worker python - <команда> [аргументы]`. +Секреты остаются в контейнере; наружу выходит одна строка JSON (без ключей доступа, только факты). + + make — тестовое изображение, base64 (kind: thumb320, exif6, alpha, tiny, large, strip, big40) + inspect — метаданные из БД + декодированные объекты из S3: размеры, формат, EXIF, + цвета контрольных точек, SHA-256, список ключей под attachments// + exists — есть ли объект в bucket + imaging-limits — imaging.py на 40-Мп PNG: пик RSS дочернего процесса и отказ по лимиту пикселей +""" +import base64 +import hashlib +import io +import json +import os +import resource +import subprocess +import sys +import tempfile +from uuid import UUID + +from PIL import Image + +import psycopg +from psycopg.rows import dict_row + +import settings +import storage + + +def make(kind: str) -> bytes: + out = io.BytesIO() + if kind == "thumb320": # 320x200 -> thumbnail 256x160 (контракт) + img = Image.new("RGB", (320, 200), (200, 30, 30)) + img.paste((30, 30, 200), (0, 0, 160, 200)) + img.save(out, "PNG") + elif kind == "exif6": # JPEG 400x300 с EXIF Orientation=6 и Make/Model/GPS + img = Image.new("RGB", (400, 300), (0, 160, 0)) + img.paste((255, 0, 0), (0, 0, 100, 300)) # левая полоса красная: после поворота на 90° по часовой — сверху + exif = Image.Exif() + exif[0x0112] = 6 + exif[0x010F] = "ProbeCamera" + exif[0x0110] = "Model-X" + exif[0x8825] = {1: "N", 2: (55.0, 45.0, 0.0)} + img.save(out, "JPEG", quality=95, exif=exif.tobytes()) + elif kind == "alpha": # RGBA 300x200: левая половина прозрачная + img = Image.new("RGBA", (300, 200), (0, 0, 255, 255)) + img.paste((0, 0, 0, 0), (0, 0, 150, 200)) + img.save(out, "PNG") + elif kind == "tiny": # 40x30: без увеличения для thumbnail/optimized + Image.new("RGB", (40, 30), (10, 120, 220)).save(out, "PNG") + elif kind == "large": # 3000x2000 JPEG: optimized 1280x853, thumbnail 256x170 + img = Image.radial_gradient("L").resize((3000, 2000)).convert("RGB") + img.save(out, "JPEG", quality=90) + elif kind == "strip": # 5000x2: thumbnail 256x1 (минимум 1 px) + Image.new("RGB", (5000, 2), (90, 90, 90)).save(out, "PNG") + elif kind == "big40": # 8000x5000 = 40 Мп ровно на лимите + Image.new("RGB", (8000, 5000), (120, 60, 30)).save(out, "PNG") + else: + raise SystemExit("unknown kind") + return out.getvalue() + + +def connect(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row) + + +def read(key: str) -> bytes: + stream = storage.open_read(key) + try: + return b"".join(storage.iter_chunks(stream)) + finally: + stream.close() + + +def inspect(attachment_id: str) -> dict: + with connect() as conn: + a = conn.execute("SELECT state, object_key, source_key, pipeline_version, sha256, size_bytes " + "FROM attachments WHERE id = %s", (UUID(attachment_id),)).fetchone() + variants = conn.execute("SELECT name, content_type, width, height, sha256, size_bytes, object_key " + "FROM attachment_variants WHERE attachment_id = %s ORDER BY name", + (UUID(attachment_id),)).fetchall() + result = {"state": a["state"], "staging_present": a["source_key"] is not None and storage.exists(a["source_key"]), + "variant_rows": len(variants), "variants": {}} + if a["object_key"]: + original = read(a["object_key"]) + result["original_sha_ok"] = hashlib.sha256(original).hexdigest() == a["sha256"] + result["original_size_ok"] = len(original) == a["size_bytes"] + for v in variants: + data = read(v["object_key"]) + img = Image.open(io.BytesIO(data)) + info = {"format": img.format, "size": list(img.size), "mode": img.mode, + "meta_size_ok": [v["width"], v["height"]] == list(img.size), + "sha_ok": hashlib.sha256(data).hexdigest() == v["sha256"] and len(data) == v["size_bytes"], + "exif_tags": len(img.getexif()), "has_xmp": "xmp" in img.info or b"http://ns.adobe.com/xap" in data, + "quality_hint": None} + rgb = img.convert("RGBA") + w, h = rgb.size + info["pixel_top_center"] = list(rgb.getpixel((w // 2, 0))) + info["pixel_left_center"] = list(rgb.getpixel((0, h // 2))) + info["pixel_right_center"] = list(rgb.getpixel((w - 1, h // 2))) + result["variants"][v["name"]] = info + prefix = f"attachments/{attachment_id}/" + result["final_keys"] = sorted(key[len(prefix):] for key, _m, _s in storage.list_objects(prefix)) + return result + + +def imaging_limits() -> dict: + """Пик памяти процесса обработки на худшем допустимом входе и отказ при превышении лимита пикселей.""" + here = os.path.dirname(os.path.abspath(storage.__file__)) + report = {} + with tempfile.TemporaryDirectory(dir=settings.WORKER_TMP_DIR) as tmp: + source = os.path.join(tmp, "big.png") + with open(source, "wb") as f: + f.write(make("big40")) + env = {"PATH": os.environ.get("PATH", ""), "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for label, max_pixels in (("at_limit_40mpx", 40_000_000), ("over_limit_max_10mpx", 10_000_000)): + out = os.path.join(tmp, label) + os.mkdir(out) + before = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + proc = subprocess.run([sys.executable, os.path.join(here, "imaging.py"), source, out, "image/png", + str(max_pixels)], env=env, capture_output=True, timeout=120) + peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss + report[label] = {"exit_code": proc.returncode, "peak_rss_mib_max_children": round(peak / 1024, 1), + "rss_grew": peak > before} + report["memory_limit_mib"] = settings.WORKER_MEMORY_LIMIT_MB + report["timeout_s"] = settings.JOB_TIMEOUT_SECONDS + return report + + +def main(): + command = sys.argv[1] + if command == "make": + print(json.dumps({"b64": base64.b64encode(make(sys.argv[2])).decode("ascii")})) + elif command == "inspect": + print(json.dumps(inspect(sys.argv[2]))) + elif command == "exists": + print(json.dumps({"exists": storage.exists(sys.argv[2])})) + elif command == "imaging-limits": + print(json.dumps(imaging_limits())) + else: + raise SystemExit("unknown command") + + +if __name__ == "__main__": + main() diff --git a/uploads.py b/uploads.py new file mode 100644 index 0000000..c01bfc1 --- /dev/null +++ b/uploads.py @@ -0,0 +1,519 @@ +"""Приём и проверка загружаемого изображения с ограниченной памятью. + +1. receive(): тело запроса читается кусками, сразу считается SHA-256 и размер. Больше + MAX_UPLOAD_BYTES -> 413 на первом лишнем байте, дальше не читаем. Данные лежат во временном + файле (до 256 KiB в памяти, остальное на диске контейнера) только на время запроса. +2. inspect(): тип определяется по сигнатуре, а не по Content-Type. Размеры в пикселях + читаются из заголовка и проверяются ДО распаковки. PNG проверяется потоково + (CRC каждого чанка + распаковка zlib кусками по 64 KiB с подсчётом точного объёма), + JPEG декодируется в масштабе 1/8. Ни один шаг не держит в памяти всё изображение. +""" +import asyncio +import hashlib +import logging +import struct +import tempfile +import threading +import time +import unicodedata +import zlib +from dataclasses import dataclass +from typing import BinaryIO + +from PIL import Image +from starlette.requests import ClientDisconnect, Request + +import settings +from errors import bad_request, payload_too_large, unsupported_media_type + +log = logging.getLogger("messenger.uploads") + +PNG = "image/png" +JPEG = "image/jpeg" +ALLOWED_TYPES = (PNG, JPEG) +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" + +SPOOL_IN_MEMORY = 256 * 1024 +INFLATE_CHUNK = 64 * 1024 +CRC_CHUNK = 64 * 1024 + +# Pillow сам откажется открывать картинки больше этого (защита на случай ошибки в нашей проверке). +Image.MAX_IMAGE_PIXELS = settings.MAX_IMAGE_PIXELS + +# Проверка изображения нагружает CPU; одновременно в одной реплике не больше DECODE_CONCURRENCY. +_decode_slots = threading.BoundedSemaphore(settings.DECODE_CONCURRENCY) + + +@dataclass +class Upload: + file: BinaryIO + size: int + sha256: str + + +@dataclass +class ImageInfo: + content_type: str + width: int + height: int + + +class Damaged(Exception): + pass + + +class TooManyPixels(Exception): + pass + + +# ---------- имя файла и заголовки ---------- + +def validate_filename(value: str | None) -> str: + """filename только для отображения: 1-255 символов, без / \\ NUL и управляющих символов.""" + if value is None: + raise bad_request("Query parameter filename is required") + if not 1 <= len(value) <= 255 or not value.strip(): + raise bad_request("filename must be 1-255 characters") + if value in (".", ".."): + raise bad_request("filename is not allowed") + for char in value: + if char in "/\\" or unicodedata.category(char) in ("Cc", "Cs"): + raise bad_request("filename contains forbidden characters") + return value + + +def declared_type(header: str | None) -> str: + base = (header or "").split(";", 1)[0].strip().lower() + if base not in ALLOWED_TYPES: + raise unsupported_media_type("Content-Type must be image/png or image/jpeg") + return base + + +def declared_length(header: str | None) -> int | None: + if header is None: + return None + if not header.isdigit(): + raise bad_request("Invalid Content-Length") + length = int(header) + if length > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + return length + + +# ---------- приём тела ---------- + +async def receive(request: Request) -> Upload: + """Читает тело с лимитом размера и таймаутами. При любой ошибке временный файл удаляется.""" + spool = tempfile.SpooledTemporaryFile(max_size=SPOOL_IN_MEMORY, dir=settings.UPLOAD_TMP_DIR) + digest = hashlib.sha256() + size = 0 + deadline = time.monotonic() + settings.UPLOAD_TIMEOUT_SECONDS + stream = request.stream().__aiter__() + try: + while True: + wait = min(settings.UPLOAD_IDLE_TIMEOUT_SECONDS, deadline - time.monotonic()) + if wait <= 0: + raise bad_request("Upload took too long") + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except StopAsyncIteration: + break + except asyncio.TimeoutError: + raise bad_request("Upload stalled or took too long") from None + if not chunk: + continue + size += len(chunk) + if size > settings.MAX_UPLOAD_BYTES: + raise payload_too_large(f"File is larger than {settings.MAX_UPLOAD_BYTES} bytes") + digest.update(chunk) + spool.write(chunk) + except ClientDisconnect: + spool.close() + log.info("Upload aborted by the client after %s bytes, nothing stored", size) + raise bad_request("Upload was interrupted") from None + except BaseException: + spool.close() + raise + if size == 0: + spool.close() + raise bad_request("Empty body") + return Upload(spool, size, digest.hexdigest()) + + +async def drain(request: Request, limit: int = 2 * settings.MAX_UPLOAD_BYTES, seconds: float = 10) -> None: + """Дочитать и выбросить тело перед ответом с ошибкой, чтобы клиент получил ответ, а не обрыв. + + Ничего не сохраняет. Ограничено объёмом и временем: дальше соединение просто закрывается. + """ + read = 0 + deadline = time.monotonic() + seconds + try: + stream = request.stream().__aiter__() + while read <= limit: + wait = deadline - time.monotonic() + if wait <= 0: + return + try: + chunk = await asyncio.wait_for(stream.__anext__(), timeout=wait) + except (StopAsyncIteration, asyncio.TimeoutError): + return + read += len(chunk) + except (ClientDisconnect, RuntimeError): + return + + +# ---------- проверка содержимого ---------- + +def sniff(head: bytes) -> str | None: + if head.startswith(PNG_SIGNATURE): + return PNG + if head.startswith(JPEG_SIGNATURE): + return JPEG + return None + + +def inspect(upload: Upload, declared: str) -> ImageInfo: + upload.file.seek(0) + detected = sniff(upload.file.read(16)) + if detected is None: + raise unsupported_media_type("File is not a PNG or JPEG image") + if detected != declared: + raise unsupported_media_type("Content-Type does not match the file content") + upload.file.seek(0) + try: + with _decode_slots: + if detected == PNG: + width, height = check_png(upload.file) + upload.file.seek(0) + check_png_metadata(upload.file) + else: + width, height = check_jpeg(upload.file) + except TooManyPixels: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except Damaged as exc: + log.info("Rejected damaged %s: %s", detected, exc) + raise unsupported_media_type("Image is damaged or not supported") from None + except Image.DecompressionBombError: + raise payload_too_large(f"Image has more than {settings.MAX_IMAGE_PIXELS} pixels") from None + except (OSError, SyntaxError, ValueError, EOFError, struct.error, zlib.error, MemoryError) as exc: + log.info("Rejected damaged %s: %s", detected, type(exc).__name__) + raise unsupported_media_type("Image is damaged or not supported") from None + finally: + upload.file.seek(0) + return ImageInfo(detected, width, height) + + +def check_pixels(width: int, height: int) -> None: + if width < 1 or height < 1: + raise Damaged("zero dimension") + if width * height > settings.MAX_IMAGE_PIXELS: + raise TooManyPixels() + + +def check_jpeg(file: BinaryIO) -> tuple[int, int]: + with Image.open(file, formats=["JPEG"]) as image: # читает только маркеры до SOS + width, height = image.size + check_pixels(width, height) + # Декодируем весь поток (обрезанный/битый файл даст ошибку), но в масштабе 1/8: + # 40 млн пикселей -> ~0,6 млн, около 2 MiB вместо 120 MiB. + image.draft("RGB", (max(1, width // 8), max(1, height // 8))) + image.load() + return width, height + + +# PNG: допустимые сочетания типа цвета и глубины (RFC 2083 / PNG spec, IHDR). +PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +PNG_DEPTHS = {0: {1, 2, 4, 8, 16}, 2: {8, 16}, 3: {1, 2, 4, 8}, 4: {8, 16}, 6: {8, 16}} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +FILTER_BYTES = bytes(range(5)) + + +class PngRows: + """Проверяет распакованный поток IDAT, не храня его: точный объём и тип фильтра каждой строки.""" + + def __init__(self, width, height, depth, color, interlace): + bits = depth * PNG_CHANNELS[color] + + def row_bytes(w): + return (w * bits + 7) // 8 + + passes = [(width, height)] if interlace == 0 else [ + ((width - x + dx - 1) // dx, (height - y + dy - 1) // dy) for x, y, dx, dy in ADAM7] + # (длина строки с байтом фильтра, байт в проходе) + self.passes = [(row_bytes(w) + 1, (row_bytes(w) + 1) * h) for w, h in passes if w > 0 and h > 0] + self.expected = sum(total for _, total in self.passes) + self.index, self.pos = 0, 0 + self.total = 0 + + def feed(self, data: bytes) -> None: + view = memoryview(data) + self.total += len(data) + if self.total > self.expected: + raise Damaged("more pixel data than the header says") + offset = 0 + while offset < len(view): + stride, size = self.passes[self.index] + take = min(len(view) - offset, size - self.pos) + first = (-self.pos) % stride # байты фильтра стоят на позициях, кратных stride + if first < take and bytes(view[offset + first:offset + take:stride]).translate(None, FILTER_BYTES): + raise Damaged("invalid filter type") + self.pos += take + offset += take + if self.pos == size: + self.index, self.pos = self.index + 1, 0 + + def done(self) -> bool: + return self.total == self.expected + + +# Чанки, содержимое которых проверяется (держим в памяти; размер ограничен спецификацией): +# IHDR — 13 байт, PLTE — до 256 x 3 = 768, tRNS — до 256 (палитра) / 6 (RGB). +SMALL_CHUNK_LIMIT = {b"IHDR": 13, b"PLTE": 768, b"tRNS": 256} +CRITICAL = (b"IHDR", b"PLTE", b"IDAT", b"IEND") +# После данных изображения допустимы только время и текст (строгий список). +AFTER_IDAT = {b"IEND", b"tIME", b"tEXt", b"zTXt", b"iTXt"} +TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt") +# Анимированные PNG (APNG) не поддерживаются: кадры — отдельная сущность, превью строится по одному кадру. +APNG_CHUNKS = (b"acTL", b"fcTL", b"fdAT") +# Как PngImagePlugin.MAX_TEXT_CHUNK в Pillow: больше 1 MiB распакованного текста в одном чанке — отказ. +TEXT_LIMIT = 1024 * 1024 +# Чанки метаданных до IDAT Pillow (Image.open) читает целиком: ограничиваем, чтобы память не росла. +# ICC-профили и EXIF реальных картинок — единицы-сотни KiB. +PRE_IDAT_CHUNK_LIMIT = 2 * 1024 * 1024 +# Чанки, которые по спецификации обязаны стоять ДО первого IDAT (PNG 3rd ed., 5.6 «Chunk ordering»). +# После IDAT допустимы только tIME, tEXt, zTXt, iTXt, кадры APNG и частные чанки. +BEFORE_IDAT = {b"PLTE", b"tRNS", b"cHRM", b"cICP", b"gAMA", b"iCCP", b"mDCV", b"cLLI", b"sBIT", b"sRGB", + b"bKGD", b"hIST", b"pHYs", b"sPLT", b"eXIf"} + + +def check_png(file: BinaryIO) -> tuple[int, int]: + """Структура PNG по спецификации (PNG 3rd ed., гл. 5 и 11) без декодирования пикселей в память. + + Кроме CRC каждого чанка проверяется СМЫСЛ критичных чанков и tRNS: CRC защищает только от + случайной порчи, а намеренно собранный файл с верными CRC может быть некорректным. + """ + if file.read(8) != PNG_SIGNATURE: + raise Damaged("signature") + rows = inflater = None + width = height = depth = 0 + color = None + palette_entries = 0 + seen = set() # какие чанки уже встречались + idat_started = idat_finished = False + first = True + while True: + header = file.read(8) + if len(header) < 8: + raise Damaged("truncated: no IEND") + length, kind = struct.unpack(">I4s", header) + if length > 0x7FFFFFFF: + raise Damaged("chunk length") + if not kind.isalpha() or not kind.isascii(): + raise Damaged("chunk type") + if first and kind != b"IHDR": + raise Damaged("IHDR must be first") + if kind[:1].isupper() and kind not in CRITICAL: + raise Damaged(f"unknown critical chunk {kind!r}") # такой чанк декодер обязан отвергнуть + if kind in SMALL_CHUNK_LIMIT and length > SMALL_CHUNK_LIMIT[kind]: + raise Damaged(f"{kind!r} is too long") + if kind == b"IDAT": + if rows is None or idat_finished: + raise Damaged("IDAT chunks must be consecutive") + idat_started = True + elif idat_started: + idat_finished = True + if kind in APNG_CHUNKS: + raise Damaged("animated PNG is not supported") + if kind in BEFORE_IDAT and idat_started: + raise Damaged(f"{kind!r} after IDAT") + if idat_finished and kind not in AFTER_IDAT: + raise Damaged(f"{kind!r} after IDAT") + if not idat_started and kind != b"IDAT" and length > PRE_IDAT_CHUNK_LIMIT: + raise Damaged(f"{kind!r} metadata chunk is too large") + if kind == b"tIME" and length != 7: + raise Damaged("tIME size") + if kind in (b"IHDR", b"PLTE", b"tRNS", b"IEND") and kind in seen: + raise Damaged(f"duplicate {kind!r}") + seen.add(kind) + + crc = zlib.crc32(kind) + kept = b"" + text = TextChunk(kind) if kind in TEXT_CHUNKS else None + left = length + while left: + piece = file.read(min(CRC_CHUNK, left)) + if not piece: + raise Damaged("truncated chunk") + left -= len(piece) + crc = zlib.crc32(piece, crc) + if kind in SMALL_CHUNK_LIMIT: + kept += piece + elif kind == b"IDAT": + feed_inflate(inflater, rows, piece) + elif text is not None: + text.feed(piece) + if text is not None: + text.finish() + stored = file.read(4) + if len(stored) < 4 or struct.unpack(">I", stored)[0] != crc & 0xFFFFFFFF: + raise Damaged(f"CRC mismatch in {kind!r}") + + if kind == b"IHDR": + if length != 13: + raise Damaged("IHDR") + width, height, depth, color, compression, filter_method, interlace = struct.unpack(">IIBBBBB", kept) + if width > 0x7FFFFFFF or height > 0x7FFFFFFF: + raise Damaged("dimensions") + check_pixels(width, height) # до распаковки + if color not in PNG_DEPTHS or depth not in PNG_DEPTHS[color]: + raise Damaged("color type / bit depth") + if compression != 0 or filter_method != 0 or interlace not in (0, 1): + raise Damaged("IHDR methods") + rows = PngRows(width, height, depth, color, interlace) + inflater = zlib.decompressobj() + elif kind == b"PLTE": + # 1-256 записей по 3 байта; для палитровых — не больше 2^глубина; у серых палитры быть не может. + if color in (0, 4): + raise Damaged("PLTE in a grayscale image") + if length == 0 or length % 3: + raise Damaged("palette size") + palette_entries = length // 3 + if color == 3 and palette_entries > 1 << depth: + raise Damaged("palette larger than the bit depth allows") + elif kind == b"tRNS": + # Прозрачность: серое — 2 байта, RGB — 6, палитра — не больше записей PLTE; при альфа-канале запрещён. + if color in (4, 6): + raise Damaged("tRNS with an alpha channel") + if color == 3 and not palette_entries: + raise Damaged("tRNS before PLTE") + expected_ok = {0: length == 2, 2: length == 6, 3: 1 <= length <= palette_entries}[color] + if not expected_ok: + raise Damaged("tRNS size") + elif kind == b"IEND": + if length != 0: + raise Damaged("IEND with data") + break + first = False + + if file.read(1): + raise Damaged("data after IEND") + if color == 3 and not palette_entries: + raise Damaged("palette image without PLTE") + if not idat_started: + raise Damaged("no IDAT") + tail = inflater.flush(INFLATE_CHUNK) + if tail: + rows.feed(tail) + if not inflater.eof or inflater.unused_data or not rows.done(): + raise Damaged("pixel data is incomplete") + return width, height + + +def check_png_metadata(file: BinaryIO) -> None: + """Второе мнение: стандартный парсер Pillow разбирает метаданные до IDAT (gAMA, iCCP, pHYs, tEXt…), + НЕ читая пиксели. Только open, без verify(): verify читает каждый чанк, включая IDAT до 10 MiB, + целиком в память, а CRC и данные после IDAT уже потоково проверил check_png. + Нужен для вспомогательных чанков: по спецификации их можно игнорировать, но если Pillow + не может их разобрать, файл нельзя будет обработать (превью в lab5) — лучше отклонить сразу. + """ + with Image.open(file, formats=["PNG"]): + pass # open разбирает все чанки до первого IDAT; после IDAT check_png пускает только время и текст + + +class TextChunk: + """Потоковая проверка tEXt / zTXt / iTXt: ключ 1-79 байт, метод сжатия 0, сжатый текст — + корректный zlib не больше TEXT_LIMIT после распаковки (распаковка кусками, текст не хранится).""" + + def __init__(self, kind: bytes): + self.kind = kind + self.head = b"" + self.done_header = False + self.inflater = None + self.size = 0 + + def feed(self, piece: bytes) -> None: + if not self.done_header: + self.head += piece + rest = self._header() + if rest is None: + if len(self.head) > 4096: + raise Damaged("text chunk header") + return + piece = rest + if self.inflater is not None and piece: + self._inflate(piece) + + def _keyword_end(self) -> int | None: + end = self.head.find(b"\x00") + if end == -1: + return None + if not 1 <= end <= 79: + raise Damaged("text keyword") + return end + + def _header(self) -> bytes | None: + end = self._keyword_end() + if end is None: + return None + h = self.head + if self.kind == b"tEXt": + self.done_header = True + return b"" + if self.kind == b"zTXt": + if len(h) < end + 2: + return None + if h[end + 1] != 0: + raise Damaged("zTXt compression method") + self.done_header, self.inflater = True, zlib.decompressobj() + return h[end + 2:] + # iTXt: keyword \0 flag method language \0 translated-keyword \0 text + if len(h) < end + 3: + return None + flag, method = h[end + 1], h[end + 2] + if flag not in (0, 1) or (flag and method != 0): + raise Damaged("iTXt compression") + lang_end = h.find(b"\x00", end + 3) + tkey_end = h.find(b"\x00", lang_end + 1) if lang_end != -1 else -1 + if tkey_end == -1: + return None + self.done_header = True + if flag: + self.inflater = zlib.decompressobj() + return h[tkey_end + 1:] + + def _inflate(self, data: bytes) -> None: + if self.inflater.eof: + raise Damaged("data after compressed text") + out = self.inflater.decompress(data, INFLATE_CHUNK) + self.size += len(out) + while self.inflater.unconsumed_tail and not self.inflater.eof and self.size <= TEXT_LIMIT: + out = self.inflater.decompress(self.inflater.unconsumed_tail, INFLATE_CHUNK) + self.size += len(out) + if self.size > TEXT_LIMIT: + raise Damaged("compressed text is too large") + if self.inflater.eof and self.inflater.unused_data: + raise Damaged("data after compressed text") + + def finish(self) -> None: + if not self.done_header: + raise Damaged("text chunk header") + if self.inflater is not None: + self.size += len(self.inflater.flush(INFLATE_CHUNK)) + if not self.inflater.eof or self.size > TEXT_LIMIT: + raise Damaged("compressed text") + + +def feed_inflate(inflater, rows: PngRows, data: bytes) -> None: + """Распаковка кусками не больше 64 KiB: «zip-бомба» не раздует память.""" + if inflater.eof: + if data: + raise Damaged("data after the end of zlib stream") + return + out = inflater.decompress(data, INFLATE_CHUNK) + rows.feed(out) + while inflater.unconsumed_tail and not inflater.eof: + out = inflater.decompress(inflater.unconsumed_tail, INFLATE_CHUNK) + rows.feed(out) + if inflater.eof and inflater.unused_data: + raise Damaged("data after the end of zlib stream") diff --git a/worker.py b/worker.py new file mode 100644 index 0000000..cbab0ad --- /dev/null +++ b/worker.py @@ -0,0 +1,390 @@ +"""Worker: обработка вложений из RabbitMQ (отдельный процесс, отдельный сервис Compose). + +Каждый из WORKER_CONCURRENCY потоков держит своё соединение с брокером и prefetch=WORKER_PREFETCH (1): +одновременно в работе не больше WORKER_CONCURRENCY заданий на процесс, остальное ждёт в очереди. + +Обработка одного сообщения: + 1. Проверка формата (jobs.schema.json). Не подходит -> reject без requeue -> DLQ. + 2. claim в PostgreSQL: queued -> processing, аренда lease_until = now() + JOB_LEASE_SECONDS и новый + lease_token. Если job уже ready/failed, ещё не пора или её держит живая аренда — ack как no-op. + 3. Пока идёт работа, отдельный поток продлевает аренду каждые lease/3 с. Жив, но долго работает -> + аренда продлевается; умер или завис -> аренда истекает, dispatcher вернёт job в queued. + Долгую работу ограничивает ещё и жёсткий JOB_TIMEOUT_SECONDS на обработку изображения. + 4. Исходник из staging -> временный файл; размер и SHA-256 сверяются с метаданными в БД. + 5. imaging.py в дочернем процессе (сам ставит RLIMIT_AS / RLIMIT_CPU / RLIMIT_FSIZE; таймаут здесь). + 6. PUT в конечный prefix: attachments//v<версия>/original | thumbnail.jpg | crop.jpg | optimized.webp. + Ключ определяется (attachment_id, pipeline_version, variant) — повтор пишет те же ключи. + 7. COMMIT результата с fencing (lease_token должен совпасть): варианты + ready одним COMMIT. + 8. ack. Только после устойчивого эффекта: объекты в S3 и COMMIT в БД. + 9. Удаление исходника из staging (если не вышло — доделает janitor). +Ошибки: постоянные (битое изображение, исходник испорчен) -> failed сразу; временные (S3/БД, +таймаут, неожиданное исключение) -> queued с backoff + jitter, после JOB_MAX_ATTEMPTS -> failed. +""" +import hashlib +import json +import logging +import os +import shutil +import signal +import subprocess +import sys +import tempfile +import threading +import time +from dataclasses import dataclass +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row + +import broker +import faults +import imaging +import pipeline +import settings +import storage + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger.worker") + +STOP = threading.Event() +HERE = Path(__file__).resolve().parent +VARIANT_KEYS = {"thumbnail": "thumbnail.jpg", "crop": "crop.jpg", "optimized": "optimized.webp"} +IMAGING_EXIT = {imaging.EXIT_INVALID: "invalid_image", imaging.EXIT_TOO_LARGE: "image_too_large", + imaging.EXIT_MEMORY: "resource_limit"} + + +class JobError(Exception): + def __init__(self, code: str, permanent: bool): + super().__init__(code) + self.code = code + self.permanent = permanent + + +class LeaseLost(Exception): + pass + + +def connect_db(): + return psycopg.connect(settings.DATABASE_URL, autocommit=True, row_factory=dict_row, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS, + options=f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS} -c TimeZone=UTC") + + +def final_prefix(claim: pipeline.Claim) -> str: + return f"attachments/{claim.attachment_id}/v{claim.pipeline_version}/" + + +class LeaseKeeper(threading.Thread): + """Продлевает аренду, пока идёт работа. Своё соединение с БД: не мешает основному потоку.""" + + def __init__(self, claim: pipeline.Claim): + super().__init__(daemon=True) + self.claim = claim + self.done = threading.Event() + self.lost = threading.Event() + self.frozen = threading.Event() # fault worker_freeze: перестать продлевать + + def run(self): + interval = max(1.0, settings.JOB_LEASE_SECONDS / 3) + conn = None + while not self.done.wait(interval): + if self.frozen.is_set(): + continue + try: + if conn is None: + conn = connect_db() + if not pipeline.renew_lease(conn, self.claim): + log.warning("job %s: lease lost, abandoning the attempt", self.claim.job_id) + self.lost.set() + return + except psycopg.Error as exc: + # Не смогли продлить — аренда может истечь; это безопасно (fencing на COMMIT). + log.warning("job %s: lease renewal failed: %s", self.claim.job_id, type(exc).__name__) + if conn is not None: + conn.close() + conn = None + if conn is not None: + conn.close() + + def check(self): + if self.lost.is_set(): + raise LeaseLost() + + +def run_imaging(source: Path, out_dir: Path, claim: pipeline.Claim, keeper: LeaseKeeper) -> dict: + # Дочернему процессу — только лимиты, никаких секретов окружения. + env = {"PATH": os.environ.get("PATH", "/usr/bin:/bin"), "PYTHONDONTWRITEBYTECODE": "1", + "IMAGING_MEMORY_MB": str(settings.WORKER_MEMORY_LIMIT_MB), + "IMAGING_CPU_SECONDS": str(settings.JOB_TIMEOUT_SECONDS)} + for fault in ("hang", "invalid", "oom"): + if faults.hit("imaging_" + fault): + env["FAULT_IMAGING"] = fault + process = subprocess.Popen( + [sys.executable, str(HERE / "imaging.py"), str(source), str(out_dir), claim.content_type, + str(settings.MAX_IMAGE_PIXELS)], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, env=env, start_new_session=True, cwd=str(out_dir), + ) + deadline = time.monotonic() + settings.JOB_TIMEOUT_SECONDS + try: + while True: + try: + output, _ = process.communicate(timeout=0.5) + break + except subprocess.TimeoutExpired: + if time.monotonic() > deadline: + raise JobError("processing_timeout", permanent=False) from None + keeper.check() + finally: + if process.poll() is None: + process.kill() # и таймаут, и потеря аренды: процесс обработки не переживает попытку + process.wait() + code = process.returncode + if code in IMAGING_EXIT: + raise JobError(IMAGING_EXIT[code], permanent=True) + if code != 0: + # Убит по RLIMIT_CPU (SIGXCPU/SIGKILL) или упал неожиданно: временная ошибка, попытки ограничены. + raise JobError("processing_failed" if code > 0 else "processing_killed", permanent=False) + return json.loads(output.decode("utf-8").strip().splitlines()[-1]) + + +def sha256_file(path: Path) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + with open(path, "rb") as file: + while chunk := file.read(1024 * 1024): + size += len(chunk) + digest.update(chunk) + return size, digest.hexdigest() + + +def put_file(key: str, path: Path, content_type: str, sha256: str, size: int) -> None: + if faults.hit("worker_s3_error"): + raise JobError("storage_unavailable", permanent=False) + with open(path, "rb") as file: + try: + storage.put(key, file, size, content_type, sha256) + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + + +def process_job(claim: pipeline.Claim, keeper: LeaseKeeper, workdir: Path) -> tuple[str, list[dict]]: + """Шаги 4-6. Возвращает ключ опубликованного оригинала и метаданные вариантов.""" + source = workdir / "source" + try: + storage.download(claim.source_key, source, max_bytes=settings.MAX_UPLOAD_BYTES) + except storage.ObjectMissing: + raise JobError("source_missing", permanent=True) from None + except storage.StorageUnavailable: + raise JobError("storage_unavailable", permanent=False) from None + size, digest = sha256_file(source) + if size != claim.size_bytes or digest != claim.sha256: + # Staging не совпадает с тем, что принял API: публиковать нельзя. + raise JobError("source_corrupted", permanent=True) + keeper.check() + + out_dir = workdir / "out" + out_dir.mkdir() + result = run_imaging(source, out_dir, claim, keeper) + keeper.check() + + prefix = final_prefix(claim) + original_key = prefix + "original" + put_file(original_key, source, claim.content_type, claim.sha256, claim.size_bytes) + variants = [] + for item in result["variants"]: + path = out_dir / item["file"] + v_size, v_sha = sha256_file(path) + key = prefix + VARIANT_KEYS[item["name"]] + put_file(key, path, item["content_type"], v_sha, v_size) + variants.append({"name": item["name"], "content_type": item["content_type"], "size_bytes": v_size, + "width": item["width"], "height": item["height"], "sha256": v_sha, "object_key": key}) + keeper.check() + return original_key, variants + + +@dataclass +class Outcome: + ack: bool = True # False -> nack с requeue (не смогли даже записать claim) + reject: bool = False # True -> reject без requeue -> DLQ + cleanup_key: str | None = None + attachment_id: object = None + + +class Consumer(threading.Thread): + def __init__(self, number: int): + super().__init__(name=f"consumer-{number}", daemon=True) + self.owner = f"{settings.INSTANCE_ID}/{os.getpid()}/{number}" + self.db = None + + def database(self): + if self.db is None or self.db.closed: + self.db = connect_db() + return self.db + + def handle(self, body: bytes, redelivered: bool) -> Outcome: + try: + message = pipeline.parse_message(body) + except pipeline.InvalidMessage as exc: + log.warning("invalid message -> DLQ (%s)", exc) + return Outcome(reject=True) + try: + result = pipeline.claim(self.database(), message, self.owner, redelivered) + except psycopg.Error as exc: + log.warning("job %s: cannot claim, PostgreSQL error %s; requeue", message["job_id"], type(exc).__name__) + self.db = None + STOP.wait(2) + return Outcome(ack=False) + if isinstance(result, pipeline.Reject): + log.warning("job %s: %s -> DLQ", message["job_id"], result.reason) + return Outcome(reject=True) + if isinstance(result, pipeline.Skip): + log.info("job %s: no-op (%s), ack", message["job_id"], result.reason) + return Outcome() + return self.execute(result) + + def execute(self, claim: pipeline.Claim) -> Outcome: + started = time.monotonic() + log.info("job %s attempt %s/%s: processing attachment %s", claim.job_id, claim.attempt, + claim.max_attempts, claim.attachment_id) + keeper = LeaseKeeper(claim) + keeper.start() + workdir = Path(tempfile.mkdtemp(dir=settings.WORKER_TMP_DIR)) + try: + try: + original_key, variants = process_job(claim, keeper, workdir) + faults.crash("worker_crash_after_put") + if faults.hit("worker_freeze"): + keeper.frozen.set() # «завис»: аренда не продлевается, её перехватят + time.sleep(settings.JOB_LEASE_SECONDS * 3) + committed = pipeline.finish_ready(self.database(), claim, original_key, variants, self.owner) + except JobError as exc: + outcome = pipeline.finish_error(self.database(), claim, exc.code, exc.permanent, self.owner) + log.warning("job %s attempt %s: %s (%s) -> %s", claim.job_id, claim.attempt, exc.code, + "permanent" if exc.permanent else "temporary", outcome) + return Outcome() + except LeaseLost: + log.warning("job %s attempt %s: lease lost, result discarded", claim.job_id, claim.attempt) + return Outcome() + except psycopg.Error: + raise + except Exception: # noqa: BLE001 — неожиданная ошибка кода: временная, попытки ограничены + log.exception("job %s attempt %s: unexpected error", claim.job_id, claim.attempt) + pipeline.finish_error(self.database(), claim, "processing_failed", False, self.owner) + return Outcome() + except psycopg.Error as exc: + # Не знаем, записан ли результат. Ack безопасен: если COMMIT не прошёл, аренда истечёт и + # dispatcher вернёт job в очередь; если прошёл — повтор увидит ready. + log.warning("job %s: PostgreSQL error %s while recording the result", claim.job_id, type(exc).__name__) + self.db = None + return Outcome() + finally: + keeper.done.set() + shutil.rmtree(workdir, ignore_errors=True) + if not committed: + log.warning("job %s attempt %s: lease was taken over, result rejected by fencing", claim.job_id, + claim.attempt) + return Outcome() + faults.crash("worker_crash_after_commit") + log.info("job %s attempt %s: ready in %.0f ms", claim.job_id, claim.attempt, + (time.monotonic() - started) * 1000) + return Outcome(cleanup_key=claim.source_key, attachment_id=claim.attachment_id) + + def cleanup_staging(self, outcome: Outcome) -> None: + """После COMMIT ready и ack: исходник в staging больше не нужен.""" + try: + storage.delete(outcome.cleanup_key) + self.database().execute("UPDATE attachments SET source_key = NULL WHERE id = %s AND state = 'ready' " + "AND source_key = %s", (outcome.attachment_id, outcome.cleanup_key)) + except (storage.StorageUnavailable, psycopg.Error) as exc: + log.info("staging cleanup postponed to janitor: %s", type(exc).__name__) + self.db = None + + def consume_once(self) -> None: + connection = broker.connect() + try: + channel = connection.channel() + channel.basic_qos(prefetch_count=settings.WORKER_PREFETCH) + broker.check_topology(channel) + log.info("%s: consuming %s (prefetch %s)", self.name, pipeline.QUEUE, settings.WORKER_PREFETCH) + for method, _properties, body in channel.consume(pipeline.QUEUE, inactivity_timeout=1): + if STOP.is_set(): + break + if method is None: + continue + box = {} + runner = threading.Thread(target=lambda: box.setdefault("outcome", self.safe_handle( + body, method.redelivered)), daemon=True) + runner.start() + while runner.is_alive(): + connection.process_data_events(time_limit=0.5) # heartbeat, пока идёт работа + outcome = box.get("outcome", Outcome(ack=False)) + if outcome.reject: + channel.basic_reject(method.delivery_tag, requeue=False) + elif outcome.ack: + channel.basic_ack(method.delivery_tag) + else: + channel.basic_nack(method.delivery_tag, requeue=True) + if outcome.cleanup_key: + self.cleanup_staging(outcome) + try: + channel.cancel() + except broker.BrokerError: + pass + finally: + try: + connection.close() + except broker.BrokerError: + pass + + def safe_handle(self, body, redelivered) -> Outcome: + try: + return self.handle(body, redelivered) + except Exception: # noqa: BLE001 + log.exception("unexpected error while handling a message; requeue") + self.db = None + STOP.wait(2) + return Outcome(ack=False) + + def run(self): + pause = 1.0 + while not STOP.is_set(): + try: + self.consume_once() + pause = 1.0 + except broker.BrokerError as exc: + log.warning("%s: broker connection problem (%s), reconnect in %.0f s", self.name, + type(exc).__name__, pause) + STOP.wait(pause) + pause = min(pause * 2, 15) + + +def main() -> None: + if not settings.DATABASE_URL or not settings.BROKER_URL: + raise SystemExit("DATABASE_URL_FILE and BROKER_URL_FILE are required") + Path(settings.WORKER_TMP_DIR).mkdir(parents=True, exist_ok=True) + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, lambda *_: STOP.set()) + consumers = [Consumer(n) for n in range(settings.WORKER_CONCURRENCY)] + for consumer in consumers: + consumer.start() + log.info("worker %s started: %s consumer(s), prefetch %s, lease %s s, timeout %s s, memory %s MiB, " + "fault injection %s", settings.INSTANCE_ID, len(consumers), settings.WORKER_PREFETCH, + settings.JOB_LEASE_SECONDS, settings.JOB_TIMEOUT_SECONDS, settings.WORKER_MEMORY_LIMIT_MB, + "ON" if settings.FAULT_INJECTION else "off") + while not STOP.wait(2): + if not all(consumer.is_alive() for consumer in consumers): + log.error("a consumer thread died, exiting so that Docker restarts the worker") + os._exit(1) + try: + Path(settings.ALIVE_FILE).touch() + except OSError: + pass + # SIGTERM: потоки доделывают текущее задание (ограничено JOB_TIMEOUT_SECONDS) и выходят. + for consumer in consumers: + consumer.join(timeout=settings.JOB_TIMEOUT_SECONDS + 10) + log.info("stopped") + + +if __name__ == "__main__": + main()