diff --git a/.dockerignore b/.dockerignore index bc16e556..07810edc 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,10 @@ .git .github +.dockercli +.dsh-tools +.tmp +.venv +.venv-libs .env .env.* !.env.example diff --git a/.env.example b/.env.example index 119384b5..de65a0c0 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,42 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# These are logical interface names; stack.yaml and src/messenger/config.py map +# them to the real processes. Secret values live in files under secrets/. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 + +# Private S3-compatible store (MinIO). Staging and final keys live in this bucket. +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments +S3_KEY_PREFIX=attachments/ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_CONNECT_TIMEOUT_SECONDS=1.5 +S3_READ_TIMEOUT_SECONDS=30 +S3_MAX_ATTEMPTS=2 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 + +# Durable broker: RabbitMQ quorum queue + DLX/DLQ. API only writes outbox; +# relay publishes with publisher confirms; worker acks after the DB commit. BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=15 +JOB_BACKOFF_BASE_SECONDS=1 +JOB_BACKOFF_MAX_SECONDS=30 +PIPELINE_VERSION=2 STAGING_RETENTION_SECONDS=86400 +RELAY_POLL_SECONDS=0.25 +REAPER_INTERVAL_SECONDS=1 +# Only the lab stand enables DB-backed one-shot fault injection. +ALLOW_FAULT_INJECTION=true diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7d1ff4a4..119a55bd 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,31 @@ ## Лабораторная 5 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Кокорин Денис, БВТ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 5.2 — доказанная идемпотентность версионированного pipeline -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: перенесены API/PostgreSQL/Valkey/MinIO/ +Traefik и контракт lab1–lab4; upload переведён на 202 с S3 staging и +транзакционной записью attachment+job+outbox. Добавлены RabbitMQ quorum queue +и DLQ, отдельные relay и worker, версионированный конвейер (thumbnail 256×256, +crop 128×128, optimized WebP), lease/reaper, retry с backoff и jitter, +DLQ/failed-реестр и явный replay, очистка staging, fault-injection и track 2. +Команды запуска из чистого clone: см. REPORT.md, раздел «Запуск из чистого +clone»; bootstrap secrets → deploy.sh/hooks.ps1 → make test → +make test-persistence/test-failover → student-tests. +Результаты проверок и ссылка на REPORT.md: public smoke PASS, 12/12 student +tests, persistence/failover PASS, track2 6/6 PASS; [REPORT.md](REPORT.md), +[evidence/README.md](evidence/README.md). +Ограничения / известные ошибки: один Docker host и single-node RabbitMQ; +final-объекты старых версий не удаляются автоматически; отдельный decode- +timeout не измерялся; Windows-хост запускает Python-тесты в контейнере. +Источники и использование ИИ: skeleton курса, документация RabbitMQ/ +PostgreSQL/MinIO/FastAPI/Pillow, собственная lab4; ИИ помогал с кодом, +ревью и текстом, все измерения получены на стенде. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.gitignore b/.gitignore index b5a71896..0e9967d1 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,13 @@ certs/private/ volumes/ data/ backups/ +# Local harness/Docker workaround directories (not part of the lab) +.dockercli/ +.dsh-tools/ +.tmp/ +.venv-libs/ +# Generated by student-tests/upload_experiment.py +student-tests/fixtures/large.png # Runtime/build artifacts .venv/ __pycache__/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..497d111e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,55 @@ +# Сборка и рантайм разделены: в итоговый образ не попадают кэш pip и инструменты +# сборки, а зависимости ставятся по зафиксированному requirements.txt. +# Базовый образ закреплён по неизменяемому digest, а не только по тегу. +FROM python:3.11.9-slim-bookworm@sha256:8fb099199b9f2d70342674bd9dbccd3ed03a258f26bbd1d556822c6dfc60c317 AS builder + +ENV PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PYTHONDONTWRITEBYTECODE=1 + +RUN python -m venv /opt/venv +COPY requirements.txt ./ +RUN /opt/venv/bin/pip install --no-cache-dir --timeout 60 -r requirements.txt + + +FROM python:3.11.9-slim-bookworm@sha256:8fb099199b9f2d70342674bd9dbccd3ed03a258f26bbd1d556822c6dfc60c317 AS runtime + +# Позволяет собрать новую ревизию того же исходника для проверки rolling update: +# метка меняет config образа, поэтому Swarm видит новую image ID и заменяет задачи. +ARG APP_REVISION=local + +LABEL org.opencontainers.image.title="mtusi-messenger-lab5" \ + org.opencontainers.image.description="API и worker мессенджера на PostgreSQL, Valkey, S3 и RabbitMQ, лабораторная 5 (MTUSI HighLoad)" \ + org.opencontainers.image.version="5.0.0" \ + org.opencontainers.image.revision="${APP_REVISION:-local}" + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PATH="/opt/venv/bin:$PATH" \ + PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin app + +COPY --from=builder /opt/venv /opt/venv + +# Миграции лежат в образе: роль контейнера задаётся командой (api или migrate). +COPY --chown=app:app src/ ./src/ +COPY --chown=app:app alembic.ini ./ +COPY --chown=app:app migrations/ ./migrations/ + +# Приложение работает без прав root: в образе нет ни компилятора, ни secrets. +USER 10001:10001 + +EXPOSE 8080 + +# SIGTERM доходит до PID 1 напрямую: uvicorn завершает запросы в своём бюджете. +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-m", "messenger.healthcheck"] + +CMD ["python", "-m", "messenger"] \ No newline at end of file diff --git a/Makefile b/Makefile index 5d86d171..ca31b7af 100644 --- a/Makefile +++ b/Makefile @@ -3,11 +3,17 @@ BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) +STACK ?= lab5 -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover \ + secrets deploy stack-down student-test reconcile track2 \ + stop-worker start-worker restart-broker help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "deploy: build image + deploy stack.yaml; secrets: create local files" + @echo "student-test: own lab5 checks; track2: fault-injection track; reconcile: staging/orphan sweep (dry-run)" + @echo "stop-worker/start-worker/restart-broker: controllable failure probes" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +34,30 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +secrets: + sh scripts/bootstrap-secrets.sh + +deploy: + STACK_NAME=$(STACK) sh scripts/deploy.sh + +stack-down: + docker stack rm $(STACK) + +stop-worker: + STACK_NAME=$(STACK) sh scripts/stop-worker.sh + +start-worker: + STACK_NAME=$(STACK) sh scripts/start-worker.sh + +restart-broker: + STACK_NAME=$(STACK) sh scripts/restart-broker.sh + +student-test: + $(PYTHON) student-tests/run_all.py --base-url "$(BASE_URL)" + +track2: + BASE_URL="$(BASE_URL)" $(PYTHON) student-tests/idempotency_track.py + +reconcile: + docker exec $$(docker ps -q -f name=$(STACK)_api | head -1) python -m messenger.reconcile --dry-run diff --git a/REPORT.md b/REPORT.md index e98f75e9..34ebdfa8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,384 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кокорин Денис, БВТ2301 +- Целевая оценка: **5** — весь уровень 3, весь уровень 4 и трек **5.2 + «Доказанная идемпотентность pipeline»** +- Commit/PR: коммиты в этой сессии намеренно не создавались; решение лежит в + рабочем дереве. Ссылка на PR появится после публикации автором. +- Источники: skeleton лабораторной 5 (README, `contracts/`, `tests/`), + собственная реализация lab4 из `mtusi2026lab4`, документация RabbitMQ + (reliability, confirms), PostgreSQL, MinIO, FastAPI/SQLAlchemy/Pillow. + Помощь ИИ: генерация и ревью кода, текста отчёта и сценариев проверки; все + команды и числа в [evidence](evidence/README.md) получены на описанном стенде. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +### Версии и предпосылки + +| Компонент | Версия | Примечание | +| --- | --- | --- | +| Python | 3.11.9 | приложение, миграции, worker; образ `python:3.11.9-slim-bookworm` с digest | +| Docker Engine | 28.0.1 | Docker Desktop, Linux containers | +| Docker Swarm | single manager/worker | `docker swarm init` при необходимости | +| PostgreSQL | `postgres:16.15-bookworm` @ `sha256:0ea6700a…` | метаданные, jobs/outbox/variants | +| Valkey | `valkey/valkey:8.0.11-trixie` @ `sha256:4436c94f…` | сессии, AOF | +| RabbitMQ | `rabbitmq:4.1.4-alpine` @ `sha256:b736d649…` | quorum queue + DLX/DLQ | +| MinIO | `minio/minio` @ `sha256:14cea493…` | приватный bucket, staging/final | +| Traefik | `traefik:v3.5.4` | единственная публичная точка входа | +| Приложение | `mtusi-messenger-lab5:5.0.0` | собирается `Dockerfile` | + +Стенд измерен на Docker Desktop Linux VM: **6 CPU / 4 GiB** +(`cpus=6 mem=4103159808`, [04-infrastructure.txt](evidence/04-infrastructure.txt)). +Лимиты контейнеров: API 1.0 CPU / 512 MiB, worker 1.0 / 512 MiB, relay 0.5 / +256 MiB, RabbitMQ 1.0 / 512 MiB, PostgreSQL 2.0 / 512 MiB, MinIO 1.0 / 512 MiB, +Valkey 1.0 / 256 MiB, Traefik 0.5 / 128 MiB. Абсолютные числа не переносятся +на другой хост; сравнивать нужно одинаковые профили. + +### Одна последовательность запуска (Swarm) + +Все зависимости поднимаются из этого репозитория. Linux/macOS: + +```sh +git clone && cd mtusi2026lab5 +chmod +x scripts/*.sh +sh scripts/bootstrap-secrets.sh # создаёт secrets/*, повторно не перезаписывает +sh scripts/deploy.sh # swarm init, secret create, build, stack deploy +# deploy.sh ждёт /health/ready до 180 с +curl -fsS http://127.0.0.1:8080/health/ready +``` + +Windows PowerShell (именно так получен evidence на этом стенде): + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\hooks.ps1 -Mode deploy +``` + +Проверки: + +```sh +make check +make validate PYTHON=.venv/bin/python # requirements-dev.txt +make test BASE_URL=http://127.0.0.1:8080 +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make student-test BASE_URL=http://127.0.0.1:8080 +make track2 BASE_URL=http://127.0.0.1:8080 +``` + +На этом Windows-хосте **нет рабочего системного Python** (WindowsApps alias +возвращает 9009), поэтому публичные и собственные Python-проверки запускались +в контейнере с приложением, смонтированным репозиторием; Docker-пробы — в +тестовом образе `lab5-probe:1.0.0` (Python + Docker CLI + curl, файл +`student-tests/Dockerfile.probe`). Для Linux команды выше выполняются штатно. +Это особенность окружения, а не приложения; все проверки идут по HTTP через +Traefik и по Docker API, как на обычном стенде. + +### Остановка и удаление учебных данных + +```sh +docker stack rm lab5 # убрать сервисы; named volumes остаются +docker volume rm lab5_pgdata lab5_valkeydata lab5_miniodata lab5_rabbitmqdata +docker secret rm lab5_database_url lab5_cache_url lab5_broker_url \ + lab5_postgres_password lab5_cursor_secret \ + lab5_rabbitmq_user lab5_rabbitmq_password \ + lab5_s3_root_user lab5_s3_root_password \ + lab5_s3_access_key_id lab5_s3_secret_access_key +docker config rm lab5_lab5_minio_init +``` + +`scripts/restart.sh|cmd` и `scripts/stop-one.sh|cmd` никогда не удаляют тома, +не пересоздают БД и не делают seed. `stop-one` останавливает ровно одну задачу +по `X-Instance-Id` (hostname контейнера); Swarm создаёт новую задачу с новым +hostname — это разрешено условиями лабораторной. `scripts/restore.sh|cmd` +возвращает 2 реплики и ждёт готовности. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +Клиент -> Traefik :8080 (единственная публикация, readiness /health/ready) + +--> api.1, api.2 (1 CPU / 512 MiB) --+--> PostgreSQL (истина) + +--> Valkey (сессии) + +--> MinIO (staging + final) + +--> relay (outbox -> RabbitMQ, reaper lease) + +--> RabbitMQ (quorum queue attachment.jobs + DLQ attachment.dlq) + +--> worker (1 процесс, обработка изображений, 1 CPU / 512 MiB) + minio-init --> private bucket + restricted S3 user + migrate --> alembic upgrade head (0004_async_pipeline) +``` + +- **Источник истины** — PostgreSQL: `attachments`, `jobs`, `outbox`, + `variants`, `job_events`, `job_replays`. Байты — только в приватном S3: + staging-ключ `attachments/staging/`, финальные ключи + `attachments/final//v/{original,thumbnail,crop,optimized}`. +- **Приём**: API проверяет доступ к чату (404 для чужого), filename, MIME по + сигнатуре, bytes/pixels; потоково читает тело во временный файл (64 KiB + чанки), считает SHA-256/MD5, кладёт объект в staging и затем **одной + транзакцией** пишет `attachment(queued)` + `job(queued)` + `outbox` + + событие аудита. Только после COMMIT возвращается **202** со снимком + `status=queued`. Если COMMIT не прошёл, staging удаляется компенсацией. +- **Публикация**: отдельный `relay` читает due-строки outbox, строит сообщение + ровно по `contracts/jobs.schema.json`, публикует в RabbitMQ с publisher + confirm и отмечает `published_at`. Сообщение содержит только `job_id`, + `attachment_id`, `pipeline_version`, `attempt`, `created_at`; bytes, + URL и credentials в брокер не попадают. +- **Обработка**: worker захватывает job под lease (UUID token), проверяет + версию, скачивает staging, сверяет size/SHA-256, декодирует изображение, + создаёт варианты, копирует оригинал в final, публикует варианты. Затем + **одной транзакцией** upsert-ит `variants` по ключу + `(attachment_id,pipeline_version,name)`, ставит attachment `ready` и job + `ready`; только после COMMIT отправляет consumer ack. При crash между + объектами и COMMIT сообщение переотправляется, повтор видит незавершённую + попытку и завершает тот же эффект. +- **Идемпотентность эффекта**: детерминированные ключи объектов и PK + `variants`; повторная доставка не создаёт второй набор. `jobs.lease_token` + защищает commit от старого attempt; `attachments.pipeline_version` + запрещает старой версии перезаписать более новую. Это not exactly-once + delivery, а effectively-once effect. +- **Версии конвейера**: v1 — original + thumbnail; v2 (текущая) — original + + thumbnail + crop + optimized WebP. Версия хранится в job и в attachment; + worker выбирает набор вариантов по версии job, а не по глобальной переменной. +- **Повторы**: временные ошибки (S3/DB/timeout) → `queued` + outbox с + экспоненциальным backoff и jitter ±20%; постоянные (`invalid_image`, + `image_too_large`, `original_missing`, `corrupt_original`) → `failed`; + после 5 попыток — `retry_exhausted`. Постоянный/исчерпанный отказ уходит в + DLQ через `x-dead-letter-exchange`. Явный replay разрешён только для + failed-задачи, создаёт новый `job_id` со ссылкой `replay_of` и аудитом; + replay already-ready отклоняется. +- **Зависший worker**: lease 15 с продлевается каждые ~5 с. Relay-reaper + возвращает processing с истёкшим lease в очередь; другой worker завершает + job; старый attempt при commit получает `lease_lost` и не меняет результат. +- **Staging cleanup**: удаляется после ready-COMMIT, но только если у + вложения нет другой активной job (например, более новой версии). Иначе + staging нужен новой версии; очистку позже делает reconciler/retention. +- **Готовность**: `/health/live` проверяет только процесс. `/health/ready` + проверяет PostgreSQL, ревизию `0004_async_pipeline`, Valkey и S3. Broker не + входит в readiness API: 202 обязан приниматься и при недоступном broker, + иначе outbox не выполнил бы свою роль; очередь разбирается после + восстановления. Worker/relay имеют heartbeat-healthcheck. +- **S3**: bucket приватный, anonymous GET 403; API использует ограниченного + пользователя только на `attachments/*`, root — только в `minio-init`. + Наружу опубликован только Traefik :8080; API/PostgreSQL/Valkey/MinIO/RabbitMQ + портов не публикуют. +- **Миграции**: отдельный сервис `migrate` с одной задачей; 0004 добавляет + staging/final-поля, `jobs`, `outbox`, `variants`, `job_events`, + `job_replays`, `fault_flags`; readiness не пропускает трафик до head-ревизии. + +### Что перенесено из lab4 и что изменено + +Перенесены: FastAPI-приложение и контракт lab1–lab4, PostgreSQL-репозиторий, +Valkey-сессии, Alembic, argon2id, конверт ошибок, `X-Request-Id`, +`X-Instance-Id`, JSON-логи, healthchecks, Dockerfile, Swarm stack, S3/MinIO, +restricted S3 policy, action-скрипты, тесты и evidence lab3/lab4. + +Добавлены: S3 staging, `jobs/outbox/variants/job_events/job_replays`, +RabbitMQ quorum queue и DLQ, relay + reaper, worker, версионированный конвейер +(crop/WebP), lease/heartbeat, retry/backoff/jitter, operational replay, +fault-injection, staging cleanup, новые student-tests и track 2. + +Обязательные текстовые POST лабораторных 1–3 остаются валидными; +`attachment_ids` по умолчанию `[]`. Upload меняет только статус-код +201 → 202 и становится асинхронным, как объявлено в COURSE.md. + +### Соответствие переменных `.env.example` + +| Переменная | Где используется | Значение/примечание | +| --- | --- | --- | +| `APP_PORT` | публикация Traefik | 8080; API слушает 8080 внутри | +| `API_IMAGE` | `stack.yaml` | `mtusi-messenger-lab5:5.0.0` | +| `API_REPLICAS` | `stack.yaml` | 2 | +| `SESSION_TTL_SECONDS` | API | 3600; TTL в Valkey | +| `DATABASE_URL_FILE` / `CACHE_URL_FILE` / `BROKER_URL_FILE` | secrets | файлы Docker secrets | +| `S3_ENDPOINT` / `S3_BUCKET` / `S3_KEY_PREFIX` | boto3 | MinIO, `attachments/` | +| `S3_ACCESS_KEY_ID_FILE` / `S3_SECRET_ACCESS_KEY_FILE` | restricted user | отдельно от root | +| `MAX_UPLOAD_BYTES` | Config | 10485760; больше контракта запрещено | +| `MAX_IMAGE_PIXELS` | Config/Pillow | 40000000; больше контракта запрещено | +| `PIPELINE_VERSION` | job/upload | 2 | +| `WORKER_CONCURRENCY` / `WORKER_PREFETCH` | worker | 2 / 2 | +| `JOB_MAX_ATTEMPTS` | worker | 5 | +| `JOB_TIMEOUT_SECONDS` | worker | 60 | +| `JOB_LEASE_SECONDS` | worker/relay | 15 | +| `JOB_BACKOFF_BASE_SECONDS` / `JOB_BACKOFF_MAX_SECONDS` | worker | 1 / 30 | +| `STAGING_RETENTION_SECONDS` | reconciler | 86400 | +| `RELAY_POLL_SECONDS` / `REAPER_INTERVAL_SECONDS` | relay | 0.25 / 1 | +| `ALLOW_FAULT_INJECTION` | worker/admin | true только в lab | + +Дополнительно поддерживаются `INSTANCE_ID`, `WORKER_ID`, `LOG_LEVEL`, +`DB_*`, `S3_*`, `GRACEFUL_SHUTDOWN_SECONDS`, `SHUTDOWN_DRAIN_SECONDS`. +Некорректное значение — отказ старта с кодом 2, а не тихая подстановка. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт и OpenAPI не сломаны | `scripts/check_contract.py`, `scripts/validate_openapi.py` | PASS, PASS | [01-contract-and-openapi.txt](evidence/01-contract-and-openapi.txt) | +| Публичный smoke lab5 | `tests/smoke.py` | PASS; upload 202, polling до ready, настоящий JPEG 256×160 | [02-public-smoke.txt](evidence/02-public-smoke.txt) | +| lab3/lab4-возможности сохранены | `student-tests/run_all.py` | 12/12 ok; 2 реплики, идемпотентность, PNG/JPEG, права | [03-student-tests.txt](evidence/03-student-tests.txt) | +| 202 только после устойчивого staging + задания; остановка API не теряет работу | worker=0 → upload → API restart → worker=1 | 202, status queued до и после restart, тот же SHA, затем ready | [worker-restart.json](evidence/worker-restart.json) | +| Отдельные broker и worker; at-least-once; ack после эффекта; worker down не включает sync resize | `docker service ls`, worker=0 сценарий | api 2/2, rabbitmq 1/1, worker 1/1, relay 1/1; upload при worker=0 — 202, обработка позже | [04-infrastructure.txt](evidence/04-infrastructure.txt), [worker-restart.json](evidence/worker-restart.json) | +| Состояния queued/processing/ready/failed в API | GET метаданных в probes | queued → processing → ready; failed + error_code | [worker-restart.json](evidence/worker-restart.json), [replay-dlq.json](evidence/replay-dlq.json), [track2.json](evidence/track2.json) | +| Worker сохраняет оригинал и thumbnail по контракту; original неизменен | smoke + student test | thumbnail JPEG 256×160, SHA/bytes оригинала и варианта совпадают | [02-public-smoke.txt](evidence/02-public-smoke.txt), [03-student-tests.txt](evidence/03-student-tests.txt) | +| Ограниченный retry (5), финальный failed с безопасным error_code, повтор не создаёт второй набор | transient_always ×5, затем replay | 4 backoff-retry 1.16/1.62/4.54/8.90 s, attempts=5, failed/retry_exhausted, DLQ, replay → ready | [retry.json](evidence/retry.json) | +| Worker остановлен → upload 202 → job создан → API restart → worker запущен → ready; broker restart с volume | `worker_restart.py` | тот же job id, attempts=1, 3 варианта; queue messages 1 до/после restart RabbitMQ | [worker-restart.json](evidence/worker-restart.json) | +| Crop 128×128 и optimized WebP по контракту; качество/удаление metadata описаны | student test + `image_contract_probe.py` | crop JPEG 128×128; optimized WebP 320×200; JPEG q85, WebP q80 method 6; EXIF/ICC удалены | [03-student-tests.txt](evidence/03-student-tests.txt), [image-contract.json](evidence/image-contract.json) | +| Невалидные/огромные изображения ограничены bytes/pixels/time/memory | `test_05`; config/stack | 413 на 10 MiB+1, 48 Мп, 100 Мп; 415 на повреждённый/MIME; 400 filename; timeout 60 s, worker 512 MiB | [03-student-tests.txt](evidence/03-student-tests.txt), [10-limits-config.txt](evidence/10-limits-config.txt) | +| Outbox закрывает разрыв COMMIT/publish; после восстановления та же задача доходит до ready | relay=0 → upload → relay=1 | pending outbox 1, published после старта, тот же job id → ready | [outbox-gap.json](evidence/outbox-gap.json) | +| Backoff с jitter; постоянные/временные ошибки; DLQ/failed; безопасный replay | `retry_probe.py`, `replay_dlq.py` | jitter-задержки; permanent → failed/invalid_image + DLQ; replay нового job; replay ready rc=3 | [retry.json](evidence/retry.json), [replay-dlq.json](evidence/replay-dlq.json) | +| Crash после записи объектов, но до ack; повтор завершает тот же результат | fault `crash_after_objects` | worker упал, attempts=2, ready, ровно 3 variant rows и 4 объекта | [track2.json](evidence/track2.json) | +| Timeout/lease зависшего processing; bounded concurrency/prefetch; staging cleanup | fault `stuck_before_commit`; cleanup probe | lease 15 s, attempts=2, ready; staging 1 → 0, final 0 → 4 | [track2.json](evidence/track2.json), [staging-cleanup.json](evidence/staging-cleanup.json) | +| Restart API и всех stores сохраняет данные и сессию | `tests/resilience.py --mode persistence` | PASS; тот же bearer, сообщение, файл | [06-persistence.txt](evidence/06-persistence.txt), [06-persistence-scope.txt](evidence/06-persistence-scope.txt) | +| Остановка одной API-реплики; восстановление без потерь и дублей | `tests/resilience.py --mode failover` | 113 успешных / 8 переходных, recovery 3.055 s, дублей нет | [07-failover.txt](evidence/07-failover.txt), [07-failover-scope.txt](evidence/07-failover-scope.txt) | +| Bucket private; API user ограничен префиксом | `check-bucket-private.sh` | private, anonymous 403, create bucket denied, put other bucket denied | [11-private-bucket.txt](evidence/11-private-bucket.txt) | +| Миграция 0004 и схема совпадают | `alembic current`, `alembic check` | head=0004_async_pipeline, «No new upgrade operations detected» | [12-alembic-check.txt](evidence/12-alembic-check.txt) | +| Трек 5.2: версии, fault-injection, инварианты, отсутствие потери/перезаписи | `student-tests/idempotency_track.py` | 6/6 сценариев PASS; duplicate 1 claim + 3 noop; stale v1 superseded; lease attempts=2 | [track2.json](evidence/track2.json), [08-track2-summary.txt](evidence/08-track2-summary.txt) | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Трек 5.2. Доказанная идемпотентность pipeline + +### Гипотеза + +At-least-once доставка RabbitMQ + детерминированный ключ эффекта +`(attachment_id, pipeline_version, variant)` + commit под lease/версионным +guard дают **ровно один эффект** для каждой принятой (202) задачи при дубле, +reorder, crash до/после side effect и устаревшей job после новой версии. +Доставка при этом остаётся at-least-once: дубликаты наблюдаются, но не +размножают результат. + +### Методика + +`student-tests/idempotency_track.py` автоматически управляет стендом через +Docker API и admin CLI внутри API-контейнера; сценарии идемпотентны и +перезапускаемы. Fault-injection — одноразовые флаги в таблице +`fault_flags`, которые переживают restart контейнера и потребляются ровно +один раз: + +| Fault | Что моделирует | +| --- | --- | +| `crash_before_objects` | процесс умирает после claim, до side effect | +| `crash_after_objects` | объекты записаны, процесс умирает до COMMIT и ack | +| `transient_once` / `transient_always` | временная ошибка; retry/backoff | +| `permanent_once` | постоянная ошибка; failed + DLQ | +| `stuck_before_commit` | worker «завис», не продлевает lease | + +Публичный API не имеет доступа к этим операциям; admin CLI запускается только +внутри контейнера с секретами. + +### Результаты + +Все 6 сценариев PASS; сырые значения — [track2.json](evidence/track2.json), +читаемая таблица — [08-track2-summary.txt](evidence/08-track2-summary.txt). + +| Сценарий | Наблюдение | Инвариант | +| --- | --- | --- | +| Дубликат: 1 original + 3 copies | `job_claimed=1`, `delivery_noop=3`, attempts=1, 3 variant rows, 4 объекта | один эффект, ready | +| Reorder v1/v2 | финальная версия 2, v2 ready, v1 ready (успел раньше), API отдаёт только v2-варианты | побеждает более новая версия | +| Crash до side effect | attempts=2, ready, 3 варианта, 4 объекта | потерянной работы нет | +| Crash после side effect до ack | attempts=2, ready, 3 варианта, 4 объекта | дубля эффекта нет | +| Stale v1 после ready v2 | v1 `superseded`, версия осталась 2, варианты и объекты не изменились | старая job не перезаписывает новую | +| Зависший worker (lease 15 s) | processing наблюдался, attempts=2, ready, 3 варианта, 4 объекта | lease восстанавливает работу | + +### Почему это не exactly-once доставка + +RabbitMQ и PostgreSQL не имеют общей атомарной транзакции. Возможны окна: +publish → crash до ack; consumer ack → потеря подтверждения; crash после +COMMIT, но до ack. В этих окнах сообщение доставляется повторно. Мы делаем +эффект идемпотентным (ключ, upsert, версионный guard, lease token), а не +доставку — «ровно один раз». Поэтому в evidence есть `delivery_noop=3`: это +нормальное поведение at-least-once, а не ошибка. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидали | Что наблюдали | Восстановление / потери | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| worker=0, upload, API restart, worker=1 | 202, работа не потеряна | queued до/после restart, тот же SHA, затем ready attempts=1 | данные не потеряны; [worker-restart.json](evidence/worker-restart.json) | +| RabbitMQ restart с volume при worker=0 | сообщение переживёт restart | `attachment.jobs` messages=1 до и после; job ready attempts=1 | очередь durable; [worker-restart.json](evidence/worker-restart.json) | +| relay=0 при upload | COMMIT есть, publish нет | pending outbox=1, 202; после relay=1 published и ready | намерение восстановлено; [outbox-gap.json](evidence/outbox-gap.json) | +| transient ×5 | 4 retry, затем failed | delays 1.16, 1.62, 4.54, 8.90 s; attempts=5; retry_exhausted; DLQ | replay → ready; [retry.json](evidence/retry.json) | +| permanent fault | failed + DLQ | invalid_image, DLQ; replay new job; replay ready rc=3 | нет дубля успешной job; [replay-dlq.json](evidence/replay-dlq.json) | +| crash до/после side effect | retry завершит тот же эффект | attempts=2, 3 варианта, 4 объекта в обоих | без потерь; [track2.json](evidence/track2.json) | +| зависший worker | lease истечёт, другой worker завершит | attempts=2, ready, 3 варианта, 4 объекта | старый commit отклонён; [track2.json](evidence/track2.json) | +| stale v1 после v2 | v1 не меняет результат | v1 superseded, версия 2, варианты/объекты те же | без перезаписи; [track2.json](evidence/track2.json) | +| API replica stop | вторая продолжает, запись восстанавливается | 113 успешных, 8 переходных, recovery 3.055 s | дублей нет; [07-failover.txt](evidence/07-failover.txt) | +| restart API+PostgreSQL+Valkey+MinIO+RabbitMQ+worker+relay | данные и сессия живы | PASS, тот же bearer, сообщение и файл | volumes сохранены; [06-persistence.txt](evidence/06-persistence.txt) | +| 15 s closed-loop load | baseline без ошибок | 1961 успешных, 0 ошибок, 129.873 RPS, p50/p95/p99 25/71/109 ms | client-bound, coordinated omission; [13-load.txt](evidence/13-load.txt) | -## Ограничения и следующие шаги +### Ограничения измерений -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Один Docker host**, Swarm из одного manager/worker. PostgreSQL, RabbitMQ, + MinIO, Valkey и Traefik — единственные экземпляры; потеря узла не + покрывается. Quorum queue на одном узле не даёт кворумной устойчивости. +- **RabbitMQ restart** проверен force update сервиса с сохранённым named + volume, а не потерей диска/узла. +- **Время обработки**: `JOB_TIMEOUT_SECONDS=60` и `asyncio.wait_for` есть в + коде, но отдельный «медленный decode» не измерялся; зависание проверено + fault `stuck_before_commit` на уровне lease. Пиковый RSS worker под + нагрузкой не измерялся; заявлены лимиты контейнера и pixel cap. +- **Старые версии**: после reorder v1 объекты остаются в final-prefix + (аудит/отладка); API их не показывает, staging чистится. Автоматической + очистки final-объектов старых версий нет — это следующий шаг. +- **Broker outage**: API продолжает принимать 202, потому что намерение + durable в PostgreSQL/outbox; при этом очередь outbox растёт, явного + admission control/лимита принятой работы нет (это тема трека 5.1, не 5.2). +- **DLQ** — очередь для оператора; автоматического массового replay нет, + есть явный CLI на одну failed-job. `attachment.dlq` в evidence содержит + несколько накопленных за прогоны сообщений; это ожидаемо и не влияет на + основные очереди. +- **Replay** разрешён только failed и только если у вложения нет ready + результата; это осознанно отказывает в «повторе» успешной job. +- **Секреты**: `secrets/`, `.env`, токены не входят в Git/отчёт; значения + не печатаются. В evidence нет credentials. +- **Windows-обвязка**: публичные Python-тесты запускались в контейнере, + потому что на хосте нет рабочего Python; это описано выше. Docker-хуки + имеют `.cmd`-двойники, потому что `Popen([script])` на Windows не + запускает `.sh`. +- **Абсолютные RPS/RAM** не переносимы на другую машину; Docker Desktop + виртуализирован. + +## Ответы на вопросы защиты + +- **Чем publisher confirm отличается от consumer ack?** Publisher confirm — + подтверждение брокера, что сообщение принято/сохранено (durable queue); + consumer ack — подтверждение worker-а, что эффект устойчиво зафиксирован. + Relay ждёт confirm перед `published_at`; worker делает ack только после + COMMIT результата. Между этими границами возможна повторная доставка. +- **Что именно сохранено в момент 202?** Полный проверенный оригинал в + приватном staging, строка `attachments(status=queued)`, `jobs(queued)` и + `outbox`-строка, закоммиченные одной транзакцией; плюс аудит-событие. + Обработка ещё не выполнена, но намерение и байты уже переживут остановку + API/worker/broker. +- **Почему повторная доставка нормальна и где ключ идемпотентности job?** + RabbitMQ at-least-once: crash после side effect до ack или потеря ack + приводит к redelivery. Ключ эффекта — `(attachment_id, pipeline_version, + variant)`; для оригинала — детерминированный final-ключ. `job_id` + идентифицирует логическую работу и не меняется при redelivery; `attempt` + увеличивается только при управляемом claim, а не при каждой доставке. +- **Как отличить зависший worker от долго работающего и не запустить два + неконтролируемых дубля?** Worker продлевает lease каждые ~5 с. Пока lease + жив, повторный claim возвращает `busy` и делает no-op ack (reaper позже + вернёт job, если процесс умер). Если lease истёк — reaper переводит job в + `queued`, новый attempt получает новый `lease_token`; старый commit + отклоняется по токену, поэтому два worker-а не могут одновременно + опубликовать результат. +- **Почему это не exactly-once?** Нет атомарной транзакции между RabbitMQ и + PostgreSQL. Дубликаты доставки неизбежны; exactly-once достигается только + для эффекта, а не для доставки, за счёт deterministic keys, upsert, + version guard и lease token. + +## Приложение: структура изменений + +``` +src/messenger/worker.py -> claim/lease, pipeline, retry, DLQ, staging cleanup +src/messenger/relay.py -> outbox publish + reaper expired leases +src/messenger/broker.py -> RabbitMQ quorum queue, confirm, DLQ, strict schema +src/messenger/jobs.py -> jobs/outbox/variants/lease/replay/fault SQL +src/messenger/pipeline.py -> версионированный thumbnail/crop/WebP +src/messenger/admin.py -> операторский show/stats/replay/enqueue/fault/duplicate +src/messenger/routers/attachments.py -> 202, staging, variants, 409 для неготового +src/messenger/repository.py -> queued attachment+job+outbox, variants, права +src/messenger/reconcile.py -> staging/failed cleanup, missing ready +migrations/versions/0004_async_pipeline.py +student-tests/test_lab5.py, worker_restart.py, outbox_gap.py, retry_probe.py, +student-tests/replay_dlq.py, staging_cleanup.py, image_contract_probe.py, +student-tests/idempotency_track.py, job_control.py, Dockerfile.probe +scripts/stop-worker.sh|cmd, start-worker.sh|cmd, restart-broker.sh|cmd, +scripts/check-bucket-private.sh, restart.sh|cmd, stop-one.sh|cmd +stack.yaml, Dockerfile, requirements.txt, .env.example, Makefile +``` diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 00000000..e299f789 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,38 @@ +# Конфигурация Alembic. URL базы здесь намеренно не задан: он приходит из +# окружения (DATABASE_URL или DATABASE_URL_FILE) и подставляется migrations/env.py. +[alembic] +script_location = migrations +prepend_sys_path = src + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = plain + +[logger_root] +level = WARNING +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARNING +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = plain + +[formatter_plain] +format = %(levelname)s [%(name)s] %(message)s diff --git a/evidence/01-contract-and-openapi.txt b/evidence/01-contract-and-openapi.txt new file mode 100644 index 00000000..0b39ec50 --- /dev/null +++ b/evidence/01-contract-and-openapi.txt @@ -0,0 +1,2 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/02-public-smoke.txt b/evidence/02-public-smoke.txt new file mode 100644 index 00000000..6f19bcb4 Binary files /dev/null and b/evidence/02-public-smoke.txt differ diff --git a/evidence/03-student-tests.txt b/evidence/03-student-tests.txt new file mode 100644 index 00000000..c68f3315 Binary files /dev/null and b/evidence/03-student-tests.txt differ diff --git a/evidence/04-infrastructure.txt b/evidence/04-infrastructure.txt new file mode 100644 index 00000000..c48c62b2 --- /dev/null +++ b/evidence/04-infrastructure.txt @@ -0,0 +1,98 @@ +== collected (UTC) == +2026-11-20T16:56:28Z +== stack services == +ID NAME MODE REPLICAS IMAGE PORTS +3u5otjjtqvcw lab5_api replicated 2/2 mtusi-messenger-lab5:5.0.0 +hyc9e4cyyxtw lab5_migrate replicated 0/1 mtusi-messenger-lab5:5.0.0 +7p4xb0oei8su lab5_minio replicated 1/1 minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e +zf0ud6h80v7k lab5_minio-init replicated 0/1 minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e +z11iinueuymd lab5_postgres replicated 1/1 postgres:16.15-bookworm +7l515sucqvjc lab5_rabbitmq replicated 1/1 rabbitmq:4.1.4-alpine +h2bmxcpffnsa lab5_relay replicated 1/1 mtusi-messenger-lab5:5.0.0 +9uwnrp3jndx0 lab5_traefik replicated 1/1 traefik:v3.5.4 *:8080->8080/tcp +wf8jaxfkx502 lab5_valkey replicated 1/1 valkey/valkey:8.0.11-trixie +qlxb6s57g8qn lab5_worker replicated 1/1 mtusi-messenger-lab5:5.0.0 +== task containers == +lab5_worker.1.lliu6swbh1jtdy4hpq79wt087 | Up 7 minutes (healthy) +lab5_relay.1.q2xk8k1s0ijrk4bfwh6xya40j | Up 7 minutes (healthy) +lab5_api.2.bsdeyp1jswve2ldivflico6kc | Up 8 minutes (healthy) +lab5_api.1.qrwkwm6iedaurlou447fd070n | Up 8 minutes (healthy) +lab5_traefik.1.kk1a33te5yqce1tb312ydvly7 | Up 8 minutes +lab5_postgres.1.ib0t3j4chi2ffmpy9tbtd8aq0 | Up 10 minutes (healthy) +lab5_valkey.1.bbcz5jyempwne6s3ovjb7xf8e | Up 10 minutes (healthy) +lab5_rabbitmq.1.taeu14whtvwi2uzvqg8gov3nf | Up 10 minutes (healthy) +lab5_minio.1.1qeuyr2km6n2hjnwihs7iar8k | Up 10 minutes (healthy) +== named volumes == +lab5_miniodata local +lab5_pgdata local +lab5_rabbitmqdata local +lab5_valkeydata local +== secret names only == +lab5_broker_url +lab5_cache_url +lab5_cursor_secret +lab5_database_url +lab5_postgres_password +lab5_rabbitmq_password +lab5_rabbitmq_user +lab5_s3_access_key_id +lab5_s3_root_password +lab5_s3_root_user +lab5_s3_secret_access_key +== published ports (only traefik expected) == +api: null +postgres: null +minio: null +rabbitmq: null +traefik: [{"Protocol":"tcp","TargetPort":8080,"PublishedPort":8080,"PublishMode":"ingress"}] +== image references == +api/worker/relay: mtusi-messenger-lab5:5.0.0 +postgres: postgres:16.15-bookworm@sha256:0ea6700a3b4f0ae6ce746519073558aed4d88a79d8d07622a9a644946c7319c4 +valkey: valkey/valkey:8.0.11-trixie@sha256:4436c94fc34ce4af0354b9379d433a1f258998fb955f995c89822f98c14a8cab +minio: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e +rabbitmq: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2 +traefik: traefik:v3.5.4 +== resource limits == +api: cpus=1000000000 mem=536870912 +worker: cpus=1000000000 mem=536870912 +relay: cpus=500000000 mem=268435456 +rabbitmq: cpus=1000000000 mem=536870912 +== API rolling update == +order=start-first parallelism=1 delay=5s failure=rollback +== schema revision and tables == +0004_async_pipeline +alembic_version +attachments +chat_members +chats +fault_flags +job_events +job_replays +jobs +message_attachments +message_idempotency +messages +outbox +users +variants +== jobs state counts / outbox == +{ + "jobs_by_state": { + "failed": 8, + "ready": 69, + "superseded": 3 + }, + "oldest_queued_at": null, + "outbox_pending": 0 +} +== rabbitmq queues == +Timeout: 60.0 seconds ... +Listing queues for vhost / ... +name type messages messages_ready messages_unacknowledged +attachment.jobs quorum 0 0 0 +attachment.dlq quorum 7 7 0 +== rabbitmq exchanges == +attachment.jobs direct +attachment.dlx fanout +== rabbitmq durable volume path == +/var/lib/docker/volumes/lab5_rabbitmqdata/_data diff --git a/evidence/06-persistence-scope.txt b/evidence/06-persistence-scope.txt new file mode 100644 index 00000000..450ff4a3 --- /dev/null +++ b/evidence/06-persistence-scope.txt @@ -0,0 +1,20 @@ +== persistence scope: restart.sh force-updated all services == +2026-11-20T16:50:31Z +== recent postgres task history (shutdown -> running) == +lab5_postgres.1 | Running 4 minutes ago | +lab5_postgres.1 | Shutdown 4 minutes ago | +lab5_postgres.1 | Shutdown 9 minutes ago | +lab5_postgres.1 | Shutdown 11 minutes ago | +== recent api task history == +lab5_api.1 | Running 2 minutes ago | +lab5_api.1 | Shutdown 2 minutes ago | +lab5_api.1 | Complete 3 minutes ago | +lab5_api.1 | Shutdown 3 minutes ago | +lab5_api.1 | Shutdown 9 minutes ago | +lab5_api.2 | Running 2 minutes ago | +== volumes before/after are the same named volumes == +lab5_miniodata +lab5_pgdata +lab5_rabbitmqdata +lab5_valkeydata +== session/data check is in 06-persistence.txt == diff --git a/evidence/06-persistence.txt b/evidence/06-persistence.txt new file mode 100644 index 00000000..c25cebac Binary files /dev/null and b/evidence/06-persistence.txt differ diff --git a/evidence/07-failover-scope.txt b/evidence/07-failover-scope.txt new file mode 100644 index 00000000..abeef428 --- /dev/null +++ b/evidence/07-failover-scope.txt @@ -0,0 +1,12 @@ +== failover scope: one API task stopped by X-Instance-Id == +2026-11-20T16:50:31Z +lab5_api.1 | Running 2 minutes ago | +lab5_api.1 | Shutdown 2 minutes ago | +lab5_api.1 | Complete 3 minutes ago | +lab5_api.1 | Shutdown 3 minutes ago | +lab5_api.1 | Shutdown 9 minutes ago | +lab5_api.2 | Running 2 minutes ago | +lab5_api.2 | Shutdown 2 minutes ago | +lab5_api.2 | Shutdown 4 minutes ago | +== current replicas == +lab5_api 2/2 diff --git a/evidence/07-failover.txt b/evidence/07-failover.txt new file mode 100644 index 00000000..8f46a0f6 Binary files /dev/null and b/evidence/07-failover.txt differ diff --git a/evidence/08-track2-summary.txt b/evidence/08-track2-summary.txt new file mode 100644 index 00000000..b4404595 --- /dev/null +++ b/evidence/08-track2-summary.txt @@ -0,0 +1,47 @@ +Track 5.2 — доказанная идемпотентность версионированного pipeline +Stand: lab5, BASE_URL=http://traefik:8080, pipeline v2 +Started: 2026-11-20T16:34:10Z +Finished: 2026-11-20T16:37:14Z + +Гипотеза +At-least-once delivery plus deterministic effect keys and lease-guarded versioned commits yields exactly-once effects for accepted jobs; it does not yield exactly-once delivery. + +Сценарии + +[PASS] duplicate_delivery + Повторная доставка не создаёт второй набор вариантов/объектов. + job=d17b570e-c435-4647-a2e2-42c61af7a24a attempts=1 job_claimed=1 delivery_noop=3 + variants=3 final_objects=4 status=ready + invariants: duplicates_were_delivered=true one_effect=true status_ready=true + +[PASS] reorder_versions + Порядок доставки версий не влияет на результат: побеждает более новая версия. + final_pipeline_version=2 jobs=v2:ready, v1:ready + visible_variants=crop,optimized,thumbnail objects=6 + invariants: attachment_ready=true final_version_is_2=true v2_objects_present=true visible_variants_are_v2=true + +[PASS] crash_before_side_effect + Crash в окне отказа не теряет принятую работу и не размножает эффект. + fault=crash_before_objects attempts=2 job=ready variants=3 objects=4 + invariants: one_effect=true recovered_to_ready=true retried=true + +[PASS] crash_after_side_effect_before_ack + Crash в окне отказа не теряет принятую работу и не размножает эффект. + fault=crash_after_objects attempts=2 job=ready variants=3 objects=4 + invariants: one_effect=true recovered_to_ready=true retried=true + +[PASS] stale_after_newer + Устаревшая job не перезаписывает более новую опубликованную версию. + stale_job=ce183646-ec82-4347-839a-f9e21d9e2df9 state=superseded attachment_version=2 + variants_unchanged=true objects_unchanged=true + invariants: objects_unchanged=true stale_job_superseded=true still_ready=true variants_unchanged=true version_not_downgraded=true + +[PASS] stuck_worker_lease + Зависший worker не удерживает job навсегда: lease истекает, работу завершает другой worker, старый commit отклоняется. + fault=stuck_before_commit lease=15s attempts=2 state=ready variants=3 objects=4 + invariants: attempts_ge_2=true one_effect=true recovered_to_ready=true stale_commit_was_rejected_or_not_observed=true + +All scenarios passed: true + +Почему это не exactly-once доставка +Broker redelivery and crash windows were observed; duplicates are normal. Exactly-once delivery would require an atomic broker-ack + DB-commit transaction, which is not available across RabbitMQ and PostgreSQL. diff --git a/evidence/09-failure-matrix.txt b/evidence/09-failure-matrix.txt new file mode 100644 index 00000000..4ff893ff --- /dev/null +++ b/evidence/09-failure-matrix.txt @@ -0,0 +1,19 @@ +Сводка окон отказа и уровня 4 +Stand: Docker Swarm lab5; generated from evidence/*.json + +[PASS] worker_stopped_api_restart attachment=f1e76ea6-ac6a-4e01-9876-60a931923c36 job=431db2e3-791a-431b-a786-cd1ee1738cc4 + 202 upload; status before restart=queued; content while queued=409; status after API restart=queued; same sha=true; final=ready; attempts=1; variants=crop,optimized,thumbnail +[PASS] broker_restart_with_volume attachment=74966990-7382-4707-b4a5-83128c8df987 job=2bb67c0c-81d9-412a-b249-5b45cc7f722e + queue messages before broker restart={"messages":1,"ready":1,"unacknowledged":0}; after={"messages":1,"ready":1,"unacknowledged":0}; final=ready; attempts=1; variants=crop,optimized,thumbnail + +[PASS] outbox gap: relay replicas=0; pending outbox rows before=1; status queued -> ready; attempts=1; variants=crop,optimized,thumbnail + +[PASS] bounded retry: max_attempts=5; attempts=5; failed=failed/retry_exhausted; retry delays=1.16s, 1.62s, 4.54s, 8.9s; DLQ={"messages":7,"ready":7,"unacknowledged":0}; replay -> ready; variants=crop,optimized,thumbnail + +[PASS] permanent failure + replay: failed=failed/invalid_image; original job=1addb168-76f0-4b23-bff7-10bf3fd7a44c; replay job=25870dc2-6c98-414e-a03e-7cddbd675d90 (replay_of=1addb168-76f0-4b23-bff7-10bf3fd7a44c); replay->ready; replay ready refused rc=3 + +[PASS] staging cleanup: before ready staging=1, final=0; after ready staging=0, final=4 + +[PASS] image contract: alpha corners RGB=[[255,255,255],[255,255,255]]; EXIF source 320,200 -> oriented 200,320, thumbnail=160,256, optimized=200,320, exif entries=0/0 + +All listed probes passed: true diff --git a/evidence/10-limits-config.txt b/evidence/10-limits-config.txt new file mode 100644 index 00000000..fb1405bf --- /dev/null +++ b/evidence/10-limits-config.txt @@ -0,0 +1,26 @@ +Ограничения и параметры конвейера (lab5) +======================================== +Источник значений: stack.yaml, src/messenger/config.py, src/messenger/pipeline.py. + +Приём: +- MAX_UPLOAD_BYTES = 10 485 760 (10 MiB), проверяется по Content-Length и потоковым счётчиком. +- MAX_IMAGE_PIXELS = 40 000 000; размеры читаются из заголовка до полной распаковки. +- MIME определяется по сигнатуре; filename не используется как путь. +- 413 на 10 MiB+1, 48 Мп и 100 Мп; 415 на повреждённый/подменённый MIME; 400 на опасный filename. + Наблюдение: evidence/03-student-tests.txt (test_05). + +Worker: +- контейнер: 1.0 CPU / 512 MiB; relay 0.5 CPU / 256 MiB; broker 1.0 / 512 MiB (evidence/04-infrastructure.txt). +- JOB_TIMEOUT_SECONDS=60; обработка обёрнута asyncio.wait_for. +- JOB_LEASE_SECONDS=15; lease продлевается каждые ~5 s; reaper возвращает просроченный processing в очередь. +- JOB_MAX_ATTEMPTS=5; backoff base=1 s, cap=30 s, jitter ±20%. + Наблюдение: evidence/retry.json — 1.16 s, 1.62 s, 4.54 s, 8.90 s; затем failed/retry_exhausted и DLQ. +- WORKER_CONCURRENCY=2, WORKER_PREFETCH=2; ограниченная конкуренция и prefetch. + +Pipeline: +- v1: original + thumbnail (256×256 fit, JPEG q85, optimize, progressive). +- v2 (текущая): v1 + crop (центрированный 128×128 JPEG q85) + optimized (WebP q80, method=6, fit 1280×1280 без увеличения). +- EXIF orientation применяется до масштабирования; alpha компонуется на белый; EXIF/ICC не переносятся. + Наблюдение: evidence/image-contract.json (200×320 после orientation, thumbnail 160×256, 0 EXIF entries, alpha=255,255,255). +- staging: STAGING_RETENTION_SECONDS=86400; удаляется после COMMIT, но только если нет другой активной версии. + Наблюдение: evidence/staging-cleanup.json. diff --git a/evidence/11-private-bucket.txt b/evidence/11-private-bucket.txt new file mode 100644 index 00000000..b746cad2 Binary files /dev/null and b/evidence/11-private-bucket.txt differ diff --git a/evidence/12-alembic-check.txt b/evidence/12-alembic-check.txt new file mode 100644 index 00000000..826a8b1d --- /dev/null +++ b/evidence/12-alembic-check.txt @@ -0,0 +1,8 @@ +INFO [alembic.runtime.migration] Context impl PostgresqlImpl. +INFO [alembic.runtime.migration] Will assume transactional DDL. +0004_async_pipeline (head) +INFO [alembic.runtime.migration] Context impl PostgresqlImpl. +INFO [alembic.runtime.migration] Will assume transactional DDL. +INFO [alembic.ddl.postgresql] Detected sequence named 'job_events_id_seq' as owned by integer column 'job_events(id)', assuming SERIAL and omitting +INFO [alembic.ddl.postgresql] Detected sequence named 'outbox_id_seq' as owned by integer column 'outbox(id)', assuming SERIAL and omitting +No new upgrade operations detected. diff --git a/evidence/13-load.txt b/evidence/13-load.txt new file mode 100644 index 00000000..51c7992a --- /dev/null +++ b/evidence/13-load.txt @@ -0,0 +1,22 @@ +HTTP load baseline (closed-loop, lab5 API) +========================================= +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15, + "elapsed_seconds": 15.099, + "concurrency": 4, + "successful_requests": 1961, + "failed_requests": 0, + "error_ratio": 0, + "successful_rps": 129.873, + "successful_latency_ms": { + "p50": 25.062, + "p95": 71.479, + "p99": 109.014 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} + +Примечание: это throughput HTTP-подтверждений POST messages, не обработка изображений. +Генератор closed-loop страдает coordinated omission; абсолютные RPS не переносимы на другой хост. diff --git a/evidence/README.md b/evidence/README.md index 8df70015..f1bf0fd4 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,33 @@ -# Доказательства +# Доказательства лабораторной 5 -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Все данные синтетические, токены и credential-значения в файлы не попадают. +Стенд: Docker Desktop, Linux containers, 6 CPU / 4 GiB; Swarm single manager; +версии закреплены в stack.yaml. Дата сбора: 2026-11-20. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +| Файл | Что доказывает | Как получен | +| --- | --- | --- | +| [01-contract-and-openapi.txt](01-contract-and-openapi.txt) | локальные ссылки, синтаксис, OpenAPI 3.1 и JSON Schema валидны | `scripts/check_contract.py`, `scripts/validate_openapi.py` в контейнере | +| [02-public-smoke.txt](02-public-smoke.txt) | публичный lab5 smoke (202→ready, thumbnail 256×160, checksums) | `tests/smoke.py` | +| [03-student-tests.txt](03-student-tests.txt) | 12 собственных тестов: lab3-инварианты, PNG/JPEG, crop/WebP, лимиты, права | `student-tests/run_all.py` | +| [04-infrastructure.txt](04-infrastructure.txt) | отдельные api/worker/relay/broker, replicas, volumes, limits, ports, schema 0004, quorum queue/DLQ | `docker service/stack inspect`, `psql`, `rabbitmqctl` | +| [06-persistence.txt](06-persistence.txt) | persistence hook: тот же токен, сообщение и файл после restart | `tests/resilience.py --mode persistence` | +| [06-persistence-scope.txt](06-persistence-scope.txt) | restart затронул API и все state stores, named volumes сохранены | `docker service ps/volume ls` до/после | +| [07-failover.txt](07-failover.txt) | отказ одной API-реплики: 113 успешных записей, recovery 3.055 s | `tests/resilience.py --mode failover` | +| [07-failover-scope.txt](07-failover-scope.txt) | остановлена именно задача с target X-Instance-Id, реплики восстановлены | `docker service ps` | +| [worker-restart.json](worker-restart.json) | worker=0 → 202 queued → content 409 → API restart → queued → worker=1 → ready; broker restart с volume и job в queue | `student-tests/worker_restart.py` | +| [outbox-gap.json](outbox-gap.json) | relay=0: attachment+job+outbox закоммичены, 202; после relay=1 та же job стала ready | `student-tests/outbox_gap.py` | +| [retry.json](retry.json) | 5 transient ошибок: 4 backoff-retry с jitter, затем failed/retry_exhausted + DLQ; replay → ready | `student-tests/retry_probe.py` | +| [replay-dlq.json](replay-dlq.json) | permanent failure → failed + DLQ; явный replay нового job; replay ready отклонён | `student-tests/replay_dlq.py` | +| [staging-cleanup.json](staging-cleanup.json) | staging-объект есть до commit, удалён после ready; final = original + 3 варианта | `student-tests/staging_cleanup.py` | +| [image-contract.json](image-contract.json) | EXIF orientation, alpha→white, удаление EXIF, реальные размеры JPEG/WebP | `student-tests/image_contract_probe.py` | +| [track2.json](track2.json) | трек 5.2: duplicate, reorder версий, crash до/после side effect, stale v1, lease/зависший worker; инварианты | `student-tests/idempotency_track.py` | +| [08-track2-summary.txt](08-track2-summary.txt) | читаемая сводка track2.json | сгенерирована из JSON | +| [09-failure-matrix.txt](09-failure-matrix.txt) | сводка всех окон отказа | сгенерирована из JSON | +| [10-limits-config.txt](10-limits-config.txt) | bytes/pixels/time/lease/retry/pipeline параметры | stack.yaml + config.py | +| [11-private-bucket.txt](11-private-bucket.txt) | bucket private, anonymous GET 403, API user не создаёт bucket и не пишет вне префикса | `check-bucket-private.sh` в MinIO-образе | +| [12-alembic-check.txt](12-alembic-check.txt) | head=0004_async_pipeline, `alembic check`: no new upgrade operations | `alembic current/check` в API-контейнере | +| [load.json](load.json), [13-load.txt](13-load.txt) | closed-loop baseline POST messages | `tests/load.py` | + +Команды запускались в контейнере с Python, потому что на этом Windows-хосте +нет рабочего системного Python; контейнер получает репозиторий ro/rw и (для +Docker-проб) Docker socket. Это не меняет наблюдаемое поведение стенда. diff --git a/evidence/image-contract.json b/evidence/image-contract.json new file mode 100644 index 00000000..2cb7074c --- /dev/null +++ b/evidence/image-contract.json @@ -0,0 +1,53 @@ +{ + "alpha": { + "alpha_composited_on_white": true, + "alpha_region_rgb": [ + [ + 255, + 255, + 255 + ], + [ + 255, + 255, + 255 + ] + ], + "attachment_id": "0b4e26d3-192e-48d3-8529-e8780cdd8b5b", + "status": "ready", + "thumbnail_size": [ + 100, + 50 + ] + }, + "exif_orientation": { + "attachment_id": "11adb7d4-57a3-4520-9ba5-ef2eaa541137", + "metadata_removed": true, + "optimized_exif_entries": 0, + "optimized_size": [ + 200, + 320 + ], + "orientation_applied": true, + "oriented_size": [ + 200, + 320 + ], + "source_size": [ + 320, + 200 + ], + "thumbnail_exif_entries": 0, + "thumbnail_size": [ + 160, + 256 + ] + }, + "passed": true, + "scenario": "image_contract", + "variants": { + "crop": "image/jpeg centered 128x128", + "optimized": "image/webp fit 1280x1280 without upscale", + "thumbnail": "image/jpeg 160x256 for rotated 200x320" + } +} \ No newline at end of file diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..44205813 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 5, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 15.0, + "elapsed_seconds": 15.099, + "concurrency": 4, + "successful_requests": 1961, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 129.873, + "successful_latency_ms": { + "p50": 25.062, + "p95": 71.479, + "p99": 109.014 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/outbox-gap.json b/evidence/outbox-gap.json new file mode 100644 index 00000000..3869f57c --- /dev/null +++ b/evidence/outbox-gap.json @@ -0,0 +1,37 @@ +{ + "attachment_id": "4cab0c1a-d03c-4fa9-a64b-5da756210327", + "job_attempts": 1, + "job_id": "47c8e86c-9347-4eb4-8767-ac9abb1d9cfb", + "note": "API committed attachment+job+outbox and returned 202 while relay was stopped. Starting relay published the exact pending outbox row and the same job id completed; no second upload was needed.", + "outbox_after": [ + { + "attempts": 1, + "available_at": "2026-11-20T16:59:32.484163Z", + "id": 95, + "job_id": "47c8e86c-9347-4eb4-8767-ac9abb1d9cfb", + "last_error": null, + "published_at": "2026-11-20T16:59:43.650236Z" + } + ], + "outbox_pending_before_relay": [ + { + "attempts": 0, + "available_at": "2026-11-20T16:59:32.484163Z", + "id": 95, + "job_id": "47c8e86c-9347-4eb4-8767-ac9abb1d9cfb", + "last_error": null, + "published_at": null + } + ], + "passed": true, + "relay_replicas_during_accept": 0, + "scenario": "crash_between_commit_and_publish", + "started_at": null, + "status_after_relay": "ready", + "status_before_relay": "queued", + "variants": [ + "crop", + "optimized", + "thumbnail" + ] +} \ No newline at end of file diff --git a/evidence/replay-dlq.json b/evidence/replay-dlq.json new file mode 100644 index 00000000..b1523ae9 --- /dev/null +++ b/evidence/replay-dlq.json @@ -0,0 +1,62 @@ +{ + "attachment_id": "d85cabe4-eb15-47ae-8d54-aece0bc6e26f", + "failed_attachment_status": "failed", + "failed_error_code": "invalid_image", + "failed_job": { + "attempts": 1, + "created_at": "2026-11-20T16:19:50.473300Z", + "finished_at": "2026-11-20T16:19:58.307023Z", + "id": "1addb168-76f0-4b23-bff7-10bf3fd7a44c", + "last_error_code": "invalid_image", + "lease_token": null, + "max_attempts": 5, + "pipeline_version": 2, + "replay_of": null, + "state": "failed", + "updated_at": "2026-11-20T16:19:58.307023Z", + "worker_id": "cf0cdb427054" + }, + "final_variants": [ + "crop", + "optimized", + "thumbnail" + ], + "note": "The permanent fault produced failed+error_code and a DLQ message. Explicit replay created a new job_id linked by replay_of; the effect key stayed the same. Replaying an already ready job was refused.", + "original_job_id": "1addb168-76f0-4b23-bff7-10bf3fd7a44c", + "passed": true, + "queues_after_failure": { + "attachment.dlq": { + "messages": 5, + "ready": 5, + "unacknowledged": 0 + }, + "attachment.jobs": { + "messages": 0, + "ready": 0, + "unacknowledged": 0 + } + }, + "replay_job": { + "attempts": 1, + "created_at": "2026-11-20T16:20:13.115972Z", + "finished_at": "2026-11-20T16:20:13.471779Z", + "id": "25870dc2-6c98-414e-a03e-7cddbd675d90", + "last_error_code": null, + "lease_token": null, + "max_attempts": 5, + "pipeline_version": 2, + "replay_of": "1addb168-76f0-4b23-bff7-10bf3fd7a44c", + "state": "ready", + "updated_at": "2026-11-20T16:20:13.471779Z", + "worker_id": "cf0cdb427054" + }, + "replay_job_id": "25870dc2-6c98-414e-a03e-7cddbd675d90", + "replay_of_ready_attempt": { + "return_code": 3, + "stderr": "replay rejected: only failed jobs can be replayed", + "stdout": "" + }, + "scenario": "permanent_failure_dlq_and_replay", + "started_at": null, + "status_after_replay": "ready" +} \ No newline at end of file diff --git a/evidence/retry.json b/evidence/retry.json new file mode 100644 index 00000000..43a31dd6 --- /dev/null +++ b/evidence/retry.json @@ -0,0 +1,79 @@ +{ + "attachment_id": "3a8999c9-76d6-4c12-906d-7ac24a0330cb", + "attempts": 5, + "failed_error_code": "retry_exhausted", + "failed_status": "failed", + "final_variants": [ + "crop", + "optimized", + "thumbnail" + ], + "job_id": "4692c5fb-0c93-4d74-942b-87f9bf41aed5", + "job_state": "failed", + "max_attempts": 5, + "note": "Transient failures produced four backoff retries with jitter, then the fifth attempt exhausted the configured limit and produced failed+DLQ. An explicit replay after clearing the fault completed the same effect key.", + "passed": true, + "queues_after_failure": { + "attachment.dlq": { + "messages": 7, + "ready": 7, + "unacknowledged": 0 + }, + "attachment.jobs": { + "messages": 0, + "ready": 0, + "unacknowledged": 0 + } + }, + "replay_job_id": "3f5bebb1-5ab5-4a1c-a777-dd80ed1729fa", + "retry_delays_seconds": [ + 1.16, + 1.62, + 4.54, + 8.9 + ], + "retry_events": [ + { + "attempt": 1, + "created_at": "2026-11-20T16:49:25.904347Z", + "detail": "worker_reported_failure; retry_in=1.16s", + "error_code": "storage_unavailable", + "from_state": "processing", + "job_id": "4692c5fb-0c93-4d74-942b-87f9bf41aed5", + "to_state": "queued", + "worker_id": "264df4defd71" + }, + { + "attempt": 2, + "created_at": "2026-11-20T16:49:27.196679Z", + "detail": "worker_reported_failure; retry_in=1.62s", + "error_code": "storage_unavailable", + "from_state": "processing", + "job_id": "4692c5fb-0c93-4d74-942b-87f9bf41aed5", + "to_state": "queued", + "worker_id": "264df4defd71" + }, + { + "attempt": 3, + "created_at": "2026-11-20T16:49:29.063348Z", + "detail": "worker_reported_failure; retry_in=4.54s", + "error_code": "storage_unavailable", + "from_state": "processing", + "job_id": "4692c5fb-0c93-4d74-942b-87f9bf41aed5", + "to_state": "queued", + "worker_id": "264df4defd71" + }, + { + "attempt": 4, + "created_at": "2026-11-20T16:49:33.715791Z", + "detail": "worker_reported_failure; retry_in=8.90s", + "error_code": "storage_unavailable", + "from_state": "processing", + "job_id": "4692c5fb-0c93-4d74-942b-87f9bf41aed5", + "to_state": "queued", + "worker_id": "264df4defd71" + } + ], + "scenario": "bounded_retry_then_failed_and_replay", + "status_after_replay": "ready" +} \ No newline at end of file diff --git a/evidence/staging-cleanup.json b/evidence/staging-cleanup.json new file mode 100644 index 00000000..080a6363 --- /dev/null +++ b/evidence/staging-cleanup.json @@ -0,0 +1,25 @@ +{ + "attachment_id": "6b8066ae-8492-4bca-a701-07bd8d63475b", + "attachment_status": "ready", + "final_after_ready": [ + "attachments/final/6b8066ae-8492-4bca-a701-07bd8d63475b/v2/crop", + "attachments/final/6b8066ae-8492-4bca-a701-07bd8d63475b/v2/optimized", + "attachments/final/6b8066ae-8492-4bca-a701-07bd8d63475b/v2/original", + "attachments/final/6b8066ae-8492-4bca-a701-07bd8d63475b/v2/thumbnail" + ], + "final_before_ready": [], + "note": "The original was visible under the staging prefix before commit. After the ready commit, worker deleted it; final prefix contains the original plus exactly three variants.", + "original_key_in_db": "attachments/final/6b8066ae-8492-4bca-a701-07bd8d63475b/v2/original", + "passed": true, + "scenario": "staging_cleanup_after_commit", + "staging_after_ready": [], + "staging_before_ready": [ + "attachments/staging/6b8066ae-8492-4bca-a701-07bd8d63475b" + ], + "staging_key_in_db": "attachments/staging/6b8066ae-8492-4bca-a701-07bd8d63475b", + "variants": [ + "crop", + "optimized", + "thumbnail" + ] +} \ No newline at end of file diff --git a/evidence/track2.json b/evidence/track2.json new file mode 100644 index 00000000..e0c07199 --- /dev/null +++ b/evidence/track2.json @@ -0,0 +1,244 @@ +{ + "all_passed": true, + "finished_at": "2026-11-20T16:37:14Z", + "hypothesis": "At-least-once delivery plus deterministic effect keys and lease-guarded versioned commits yields exactly-once effects for accepted jobs; it does not yield exactly-once delivery.", + "not_exactly_once": "Broker redelivery and crash windows were observed; duplicates are normal. Exactly-once delivery would require an atomic broker-ack + DB-commit transaction, which is not available across RabbitMQ and PostgreSQL.", + "scenarios": [ + { + "hypothesis": "Повторная доставка не создаёт второй набор вариантов/объектов.", + "invariants": { + "duplicates_were_delivered": true, + "one_effect": true, + "status_ready": true + }, + "observed": { + "attachment_id": "0c8438df-fe2f-49b8-9077-d0dbf1400405", + "job_attempts": 1, + "job_id": "d17b570e-c435-4647-a2e2-42c61af7a24a", + "objects_final_prefix": [ + "attachments/final/0c8438df-fe2f-49b8-9077-d0dbf1400405/v2/crop", + "attachments/final/0c8438df-fe2f-49b8-9077-d0dbf1400405/v2/optimized", + "attachments/final/0c8438df-fe2f-49b8-9077-d0dbf1400405/v2/original", + "attachments/final/0c8438df-fe2f-49b8-9077-d0dbf1400405/v2/thumbnail" + ], + "outbox_published_before_duplicate": [ + true + ], + "status": "ready", + "variant_rows": 3, + "worker_events": { + "delivery_noop": 3, + "job_claimed": 1 + } + }, + "passed": true, + "scenario": "duplicate_delivery" + }, + { + "hypothesis": "Порядок доставки версий не влияет на результат: побеждает более новая версия.", + "invariants": { + "attachment_ready": true, + "final_version_is_2": true, + "v2_objects_present": true, + "visible_variants_are_v2": true + }, + "observed": { + "attachment_id": "34f0e096-cb07-4939-8cbb-682a356eb5a9", + "attachment_pipeline_version": 2, + "jobs": [ + { + "attempts": 1, + "created_at": "2026-11-20T16:34:44.077196Z", + "finished_at": "2026-11-20T16:34:51.864594Z", + "id": "060b80c0-4fbf-4fbb-b819-c1d6f6becc11", + "last_error_code": null, + "lease_token": null, + "max_attempts": 5, + "pipeline_version": 2, + "replay_of": null, + "state": "ready", + "updated_at": "2026-11-20T16:34:51.864594Z", + "worker_id": "dbe6b432fc80" + }, + { + "attempts": 1, + "created_at": "2026-11-20T16:34:47.549202Z", + "finished_at": "2026-11-20T16:34:51.781556Z", + "id": "c655035d-fe56-44c8-b579-f2c97d4e0778", + "last_error_code": null, + "lease_token": null, + "max_attempts": 5, + "pipeline_version": 1, + "replay_of": null, + "state": "ready", + "updated_at": "2026-11-20T16:34:51.781556Z", + "worker_id": "dbe6b432fc80" + } + ], + "objects_final_prefix": [ + "attachments/final/34f0e096-cb07-4939-8cbb-682a356eb5a9/v1/original", + "attachments/final/34f0e096-cb07-4939-8cbb-682a356eb5a9/v1/thumbnail", + "attachments/final/34f0e096-cb07-4939-8cbb-682a356eb5a9/v2/crop", + "attachments/final/34f0e096-cb07-4939-8cbb-682a356eb5a9/v2/optimized", + "attachments/final/34f0e096-cb07-4939-8cbb-682a356eb5a9/v2/original", + "attachments/final/34f0e096-cb07-4939-8cbb-682a356eb5a9/v2/thumbnail" + ], + "v1_state": "ready", + "variant_names": [ + "crop", + "optimized", + "thumbnail" + ] + }, + "passed": true, + "scenario": "reorder_versions" + }, + { + "hypothesis": "Crash в окне отказа не теряет принятую работу и не размножает эффект.", + "invariants": { + "one_effect": true, + "recovered_to_ready": true, + "retried": true + }, + "observed": { + "attachment_id": "19cf7faf-1971-41ba-b836-6cfaadd441a1", + "claims_observed": 2, + "fault": "crash_before_objects", + "job_attempts": 2, + "job_id": "e525bc4e-4f94-4424-a34c-9bba41c3ffd8", + "job_state": "ready", + "objects_final_prefix": [ + "attachments/final/19cf7faf-1971-41ba-b836-6cfaadd441a1/v2/crop", + "attachments/final/19cf7faf-1971-41ba-b836-6cfaadd441a1/v2/optimized", + "attachments/final/19cf7faf-1971-41ba-b836-6cfaadd441a1/v2/original", + "attachments/final/19cf7faf-1971-41ba-b836-6cfaadd441a1/v2/thumbnail" + ], + "status": "ready", + "variant_rows": 3 + }, + "passed": true, + "scenario": "crash_before_side_effect" + }, + { + "hypothesis": "Crash в окне отказа не теряет принятую работу и не размножает эффект.", + "invariants": { + "one_effect": true, + "recovered_to_ready": true, + "retried": true + }, + "observed": { + "attachment_id": "2f3dfba0-7742-4113-8806-3f6694aeb711", + "claims_observed": 2, + "fault": "crash_after_objects", + "job_attempts": 2, + "job_id": "29a4a3cb-00a4-4fa2-ad6b-31ab27013304", + "job_state": "ready", + "objects_final_prefix": [ + "attachments/final/2f3dfba0-7742-4113-8806-3f6694aeb711/v2/crop", + "attachments/final/2f3dfba0-7742-4113-8806-3f6694aeb711/v2/optimized", + "attachments/final/2f3dfba0-7742-4113-8806-3f6694aeb711/v2/original", + "attachments/final/2f3dfba0-7742-4113-8806-3f6694aeb711/v2/thumbnail" + ], + "status": "ready", + "variant_rows": 3 + }, + "passed": true, + "scenario": "crash_after_side_effect_before_ack" + }, + { + "hypothesis": "Устаревшая job не перезаписывает более новую опубликованную версию.", + "invariants": { + "objects_unchanged": true, + "stale_job_superseded": true, + "still_ready": true, + "variants_unchanged": true, + "version_not_downgraded": true + }, + "observed": { + "attachment_id": "6268ce53-dd7a-482e-9ea6-91fb7c50317c", + "attachment_pipeline_version": 2, + "objects_after": [ + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/crop", + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/optimized", + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/original", + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/thumbnail" + ], + "objects_before": [ + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/crop", + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/optimized", + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/original", + "attachments/final/6268ce53-dd7a-482e-9ea6-91fb7c50317c/v2/thumbnail" + ], + "stale_job_id": "ce183646-ec82-4347-839a-f9e21d9e2df9", + "stale_job_state": "superseded", + "variants_after": [ + [ + "crop", + "3f8183ef5e7f2035834124a2221e1fd75894f144d5ccd392d1b2b6b28d7163f8" + ], + [ + "optimized", + "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24" + ], + [ + "thumbnail", + "e16c6a0ec37e9b01a5a3d013c0a2b4d1af5bf8b633df121e4073a7b3a58d5349" + ] + ], + "variants_before": [ + [ + "crop", + "3f8183ef5e7f2035834124a2221e1fd75894f144d5ccd392d1b2b6b28d7163f8" + ], + [ + "optimized", + "45c5694bda8dda06f9025ae80065ebc59dd0e98ab37b7ce0f94b1d628f02ba24" + ], + [ + "thumbnail", + "e16c6a0ec37e9b01a5a3d013c0a2b4d1af5bf8b633df121e4073a7b3a58d5349" + ] + ] + }, + "passed": true, + "scenario": "stale_after_newer" + }, + { + "hypothesis": "Зависший worker не удерживает job навсегда: lease истекает, работу завершает другой worker, старый commit отклоняется.", + "invariants": { + "attempts_ge_2": true, + "one_effect": true, + "recovered_to_ready": true, + "stale_commit_was_rejected_or_not_observed": true + }, + "observed": { + "attachment_id": "fe6ae439-ed1d-4915-adc2-f88c7ff2831b", + "fault": "stuck_before_commit", + "job_attempts": 2, + "job_id": "a22bf5a3-a692-4d9c-b768-5e89c10ad8d3", + "job_state": "ready", + "lease_seconds": 15.0, + "objects_final_prefix": [ + "attachments/final/fe6ae439-ed1d-4915-adc2-f88c7ff2831b/v2/crop", + "attachments/final/fe6ae439-ed1d-4915-adc2-f88c7ff2831b/v2/optimized", + "attachments/final/fe6ae439-ed1d-4915-adc2-f88c7ff2831b/v2/original", + "attachments/final/fe6ae439-ed1d-4915-adc2-f88c7ff2831b/v2/thumbnail" + ], + "observed_processing": true, + "stale_commit_rejections_in_logs": 0, + "variant_rows": 3 + }, + "passed": true, + "scenario": "stuck_worker_lease" + } + ], + "stand": { + "base_url": "http://traefik:8080", + "pipeline_v1": "original+thumbnail", + "pipeline_v2": "original+thumbnail+crop+optimized WebP", + "pipeline_version": 2, + "stack": "lab5" + }, + "started_at": "2026-11-20T16:34:10Z", + "track": "5.2 proven idempotent pipeline" +} \ No newline at end of file diff --git a/evidence/worker-restart.json b/evidence/worker-restart.json new file mode 100644 index 00000000..cc068504 --- /dev/null +++ b/evidence/worker-restart.json @@ -0,0 +1,74 @@ +{ + "all_passed": true, + "finished_at": "2026-11-20T16:59:15Z", + "note": "RabbitMQ quorum queue is durable; message was confirmed before restart. Named volume lab5_rabbitmqdata was preserved. The same 202 attachment and job id are used before and after restart.", + "scenarios": [ + { + "attachment_id": "f1e76ea6-ac6a-4e01-9876-60a931923c36", + "content_while_queued": 409, + "final_attempts": 1, + "final_status": "ready", + "final_variants": [ + "crop", + "optimized", + "thumbnail" + ], + "job_id": "431db2e3-791a-431b-a786-cd1ee1738cc4", + "name": "worker_stopped_api_restart", + "outbox_after_upload": [ + { + "attempts": 1, + "available_at": "2026-11-20T16:57:24.788142Z", + "id": 93, + "job_id": "431db2e3-791a-431b-a786-cd1ee1738cc4", + "last_error": null, + "published_at": "2026-11-20T16:57:25.029378Z" + } + ], + "passed": true, + "same_sha256": true, + "status_after_api_restart": "queued", + "status_before_restart": "queued", + "upload_response_status": 202 + }, + { + "attachment_id": "74966990-7382-4707-b4a5-83128c8df987", + "final_attempts": 1, + "final_status": "ready", + "final_variants": [ + "crop", + "optimized", + "thumbnail" + ], + "job_id": "2bb67c0c-81d9-412a-b249-5b45cc7f722e", + "name": "broker_restart_with_volume", + "passed": true, + "queue_after_restart": { + "attachment.dlq": { + "messages": 7, + "ready": 7, + "unacknowledged": 0 + }, + "attachment.jobs": { + "messages": 1, + "ready": 1, + "unacknowledged": 0 + } + }, + "queue_before_restart": { + "attachment.dlq": { + "messages": 7, + "ready": 7, + "unacknowledged": 0 + }, + "attachment.jobs": { + "messages": 1, + "ready": 1, + "unacknowledged": 0 + } + }, + "status_before_restart": "queued" + } + ], + "started_at": "2026-11-20T16:57:19Z" +} \ No newline at end of file diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 00000000..8478d568 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,58 @@ +"""Окружение Alembic: URL базы берётся из тех же переменных, что и у приложения.""" +from __future__ import annotations + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import async_engine_from_config + +from messenger.config import database_url_from_env +from messenger.schema import METADATA + +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Сравнение идёт ровно с теми объектами, по которым работает приложение: +# расхождение между схемой и миграциями ловится на `alembic check`. +target_metadata = METADATA + + +def run_migrations_offline() -> None: + """Режим без подключения: печатает SQL миграции, удобен для ревью.""" + context.configure( + url=database_url_from_env(), + target_metadata=target_metadata, + literal_binds=True, + compare_type=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def _run_migrations(connection) -> None: + context.configure(connection=connection, target_metadata=target_metadata, compare_type=True) + with context.begin_transaction(): + context.run_migrations() + + +async def run_migrations_online() -> None: + """Драйвер асинхронный (asyncpg), поэтому миграции идут через run_sync.""" + section = config.get_section(config.config_ini_section, {}) + section["sqlalchemy.url"] = database_url_from_env() + engine = async_engine_from_config(section, prefix="sqlalchemy.", poolclass=pool.NullPool) + try: + async with engine.connect() as connection: + await connection.run_sync(_run_migrations) + finally: + await engine.dispose() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + asyncio.run(run_migrations_online()) diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 00000000..c6284cb9 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from __future__ import annotations + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/0001_initial_schema.py b/migrations/versions/0001_initial_schema.py new file mode 100644 index 00000000..4cdd21cf --- /dev/null +++ b/migrations/versions/0001_initial_schema.py @@ -0,0 +1,103 @@ +"""Начальная схема: пользователи, чаты, участники и сообщения + +Revision ID: 0001_initial_schema +Revises: +Create Date: 2026-10-11 + +DDL написан вручную и повторяет src/messenger/schema.py: имена ограничений +и индексов совпадают, поэтому `alembic check` не находит расхождений между +схемой в базе и описанием, по которому работают запросы. +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "0001_initial_schema" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "users", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("username", sa.Text(), nullable=False), + sa.Column("display_name", sa.Text(), nullable=False), + sa.Column("password_hash", sa.Text(), nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "char_length(display_name) BETWEEN 1 AND 100", name="ck_users_display_name_length" + ), + sa.CheckConstraint(r"username ~ '^[a-z0-9_]{3,32}$'", name="ck_users_username_format"), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("username", name="uq_users_username"), + ) + + op.create_table( + "chats", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("title", sa.Text(), nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="ck_chats_title_length"), + sa.PrimaryKeyConstraint("id"), + ) + + op.create_table( + "chat_members", + sa.Column("chat_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name="fk_chat_members_chat", ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.id"], name="fk_chat_members_user", ondelete="RESTRICT" + ), + sa.PrimaryKeyConstraint("chat_id", "user_id"), + ) + op.create_index("ix_chat_members_user_id_chat_id", "chat_members", ["user_id", "chat_id"]) + + op.create_table( + "messages", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("chat_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("sender_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "text <> '' AND char_length(text) <= 4000", name="ck_messages_text_length" + ), + sa.ForeignKeyConstraint(["chat_id"], ["chats.id"], name="fk_messages_chat", ondelete="CASCADE"), + sa.ForeignKeyConstraint( + ["sender_id"], ["users.id"], name="fk_messages_sender", ondelete="RESTRICT" + ), + sa.PrimaryKeyConstraint("id"), + ) + # Порядок истории (created_at, id) и keyset-предикат обслуживает один индекс. + op.create_index("ix_messages_chat_id_created_at_id", "messages", ["chat_id", "created_at", "id"]) + + +def downgrade() -> None: + op.drop_index("ix_messages_chat_id_created_at_id", table_name="messages") + op.drop_table("messages") + op.drop_index("ix_chat_members_user_id_chat_id", table_name="chat_members") + op.drop_table("chat_members") + op.drop_table("chats") + op.drop_table("users") diff --git a/migrations/versions/0002_message_idempotency.py b/migrations/versions/0002_message_idempotency.py new file mode 100644 index 00000000..56f714ed --- /dev/null +++ b/migrations/versions/0002_message_idempotency.py @@ -0,0 +1,61 @@ +"""Общая идемпотентность отправки сообщения + +Revision ID: 0002_message_idempotency +Revises: 0001_initial_schema +Create Date: 2026-10-30 + +Таблица описана в src/messenger/schema.py; DDL повторяет имена ограничений и +индексов, чтобы alembic check не находил расхождений. +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "0002_message_idempotency" +down_revision = "0001_initial_schema" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "message_idempotency", + sa.Column("user_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("chat_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("idempotency_key", sa.Text(), nullable=False), + sa.Column("request_hash", sa.Text(), nullable=False), + sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "char_length(idempotency_key) BETWEEN 1 AND 128", + name="ck_message_idempotency_key_length", + ), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name="fk_message_idempotency_chat", ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["message_id"], + ["messages.id"], + name="fk_message_idempotency_message", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["user_id"], ["users.id"], name="fk_message_idempotency_user", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("user_id", "chat_id", "idempotency_key"), + ) + op.create_index( + "ix_message_idempotency_created_at", "message_idempotency", ["created_at"] + ) + + +def downgrade() -> None: + op.drop_index("ix_message_idempotency_created_at", table_name="message_idempotency") + op.drop_table("message_idempotency") diff --git a/migrations/versions/0003_attachments.py b/migrations/versions/0003_attachments.py new file mode 100644 index 00000000..d1387085 --- /dev/null +++ b/migrations/versions/0003_attachments.py @@ -0,0 +1,128 @@ +"""Вложения и связь сообщений с вложениями + +Revision ID: 0003_attachments +Revises: 0002_message_idempotency +Create Date: 2026-11-06 + +DDL повторяет src/messenger/schema.py, включая имена ограничений и индексов, +поэтому `alembic check` не находит расхождений. Отдельно ослабляется CHECK +текста сообщения: пустая строка теперь допустима только вместе с вложением, +а это правило проверяет API (CHECK не может ссылаться на другую таблицу). +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "0003_attachments" +down_revision = "0002_message_idempotency" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "attachments", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("chat_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("owner_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("object_key", sa.Text(), nullable=False), + sa.Column("filename", sa.Text(), nullable=False), + sa.Column("content_type", sa.Text(), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("sha256", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), server_default=sa.text("'pending'"), nullable=False), + sa.Column("error_code", sa.Text(), nullable=True), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "char_length(filename) BETWEEN 1 AND 255", name="ck_attachments_filename_length" + ), + sa.CheckConstraint( + "content_type IN ('image/png','image/jpeg')", name="ck_attachments_content_type" + ), + sa.CheckConstraint( + "size_bytes > 0 AND size_bytes <= 10485760", name="ck_attachments_size_bytes" + ), + sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="ck_attachments_sha256"), + sa.CheckConstraint( + "status IN ('pending','ready','failed')", name="ck_attachments_status" + ), + sa.CheckConstraint( + "error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64", + name="ck_attachments_error_code", + ), + sa.ForeignKeyConstraint( + ["chat_id"], ["chats.id"], name="fk_attachments_chat", ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["owner_id"], ["users.id"], name="fk_attachments_owner", ondelete="RESTRICT" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("object_key", name="uq_attachments_object_key"), + ) + op.create_index( + "ix_attachments_chat_id_created_at_id", + "attachments", + ["chat_id", "created_at", "id"], + ) + op.create_index("ix_attachments_status_updated_at", "attachments", ["status", "updated_at"]) + + op.create_table( + "message_attachments", + sa.Column("message_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("attachment_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("position", sa.SmallInteger(), nullable=False), + sa.CheckConstraint( + "position >= 0 AND position < 10", name="ck_message_attachments_position" + ), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name="fk_message_attachments_attachment", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["message_id"], + ["messages.id"], + name="fk_message_attachments_message", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("message_id", "attachment_id"), + sa.UniqueConstraint( + "message_id", "position", name="uq_message_attachments_position" + ), + ) + op.create_index( + "ix_message_attachments_attachment_id", + "message_attachments", + ["attachment_id"], + ) + + op.drop_constraint("ck_messages_text_length", "messages", type_="check") + op.create_check_constraint( + "ck_messages_text_length", "messages", "char_length(text) <= 4000" + ) + + +def downgrade() -> None: + op.drop_constraint("ck_messages_text_length", "messages", type_="check") + op.create_check_constraint( + "ck_messages_text_length", "messages", "text <> '' AND char_length(text) <= 4000" + ) + op.drop_index("ix_message_attachments_attachment_id", table_name="message_attachments") + op.drop_table("message_attachments") + op.drop_index("ix_attachments_status_updated_at", table_name="attachments") + op.drop_index("ix_attachments_chat_id_created_at_id", table_name="attachments") + op.drop_table("attachments") diff --git a/migrations/versions/0004_async_pipeline.py b/migrations/versions/0004_async_pipeline.py new file mode 100644 index 00000000..a82baac2 --- /dev/null +++ b/migrations/versions/0004_async_pipeline.py @@ -0,0 +1,292 @@ +"""Асинхронный pipeline: staging, jobs, outbox, варианты и аудит + +Revision ID: 0004_async_pipeline +Revises: 0003_attachments +Create Date: 2026-11-20 + +DDL повторяет src/messenger/schema.py, включая имена ограничений и индексов. +Существующие lab4-вложения трактуются консервативно: staging_key сбрасывается +в прежний object_key, завершённые ready-строки сохраняются, а незавершённые +pending-строки помечаются failed с безопасным error_code. +""" +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "0004_async_pipeline" +down_revision = "0003_attachments" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column("attachments", sa.Column("staging_key", sa.Text(), nullable=True)) + op.add_column( + "attachments", + sa.Column("pipeline_version", sa.Integer(), server_default=sa.text("0"), nullable=False), + ) + op.add_column( + "attachments", + sa.Column("ready_at", postgresql.TIMESTAMP(timezone=True), nullable=True), + ) + op.execute("UPDATE attachments SET staging_key = object_key WHERE staging_key IS NULL") + op.alter_column("attachments", "staging_key", nullable=False) + op.create_unique_constraint("uq_attachments_staging_key", "attachments", ["staging_key"]) + + op.drop_constraint("ck_attachments_status", "attachments", type_="check") + op.execute( + "UPDATE attachments SET status = 'failed', error_code = 'legacy_pending' " + "WHERE status = 'pending'" + ) + op.alter_column( + "attachments", "status", server_default=sa.text("'queued'"), existing_type=sa.Text() + ) + op.create_check_constraint( + "ck_attachments_status", + "attachments", + "status IN ('queued','processing','ready','failed')", + ) + op.create_check_constraint( + "ck_attachments_pipeline_version", "attachments", "pipeline_version >= 0" + ) + + op.create_table( + "jobs", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("attachment_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("pipeline_version", sa.Integer(), nullable=False), + sa.Column( + "type", + sa.Text(), + server_default=sa.text("'attachment.process.v1'"), + nullable=False, + ), + sa.Column("state", sa.Text(), server_default=sa.text("'queued'"), nullable=False), + sa.Column("attempts", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("max_attempts", sa.Integer(), nullable=False), + sa.Column("lease_token", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column("worker_id", sa.Text(), nullable=True), + sa.Column( + "lease_expires_at", postgresql.TIMESTAMP(timezone=True), nullable=True + ), + sa.Column("next_attempt_at", postgresql.TIMESTAMP(timezone=True), nullable=True), + sa.Column("last_error_code", sa.Text(), nullable=True), + sa.Column("replay_of", postgresql.UUID(as_uuid=True), nullable=True), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("finished_at", postgresql.TIMESTAMP(timezone=True), nullable=True), + sa.CheckConstraint("pipeline_version >= 1", name="ck_jobs_pipeline_version"), + sa.CheckConstraint( + "state IN ('queued','processing','ready','failed','superseded')", + name="ck_jobs_state", + ), + sa.CheckConstraint("attempts >= 0", name="ck_jobs_attempts"), + sa.CheckConstraint("max_attempts >= 1", name="ck_jobs_max_attempts"), + sa.CheckConstraint( + "last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64", + name="ck_jobs_last_error_code", + ), + sa.ForeignKeyConstraint( + ["attachment_id"], ["attachments.id"], name="fk_jobs_attachment", ondelete="CASCADE" + ), + sa.ForeignKeyConstraint( + ["replay_of"], ["jobs.id"], name="fk_jobs_replay_of", ondelete="SET NULL" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_jobs_attachment_id", "jobs", ["attachment_id"]) + op.create_index("ix_jobs_state_next_attempt_at", "jobs", ["state", "next_attempt_at"]) + op.create_index("ix_jobs_lease_expires_at", "jobs", ["lease_expires_at"]) + op.create_index( + "uq_jobs_initial_attachment_version", + "jobs", + ["attachment_id", "pipeline_version"], + unique=True, + postgresql_where=sa.text("replay_of IS NULL"), + ) + + op.create_table( + "outbox", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("job_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column( + "available_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column("published_at", postgresql.TIMESTAMP(timezone=True), nullable=True), + sa.Column("attempts", sa.Integer(), server_default=sa.text("0"), nullable=False), + sa.Column("last_error", sa.Text(), nullable=True), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["job_id"], ["jobs.id"], name="fk_outbox_job", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_outbox_due", "outbox", ["published_at", "available_at"]) + op.create_index( + "uq_outbox_pending_job", + "outbox", + ["job_id"], + unique=True, + postgresql_where=sa.text("published_at IS NULL"), + ) + + op.create_table( + "variants", + sa.Column("attachment_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("pipeline_version", sa.Integer(), nullable=False), + sa.Column("name", sa.Text(), nullable=False), + sa.Column("content_type", sa.Text(), nullable=False), + sa.Column("object_key", sa.Text(), nullable=False), + sa.Column("size_bytes", sa.BigInteger(), nullable=False), + sa.Column("sha256", sa.Text(), nullable=False), + sa.Column("width", sa.Integer(), nullable=False), + sa.Column("height", sa.Integer(), nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint("pipeline_version >= 1", name="ck_variants_pipeline_version"), + sa.CheckConstraint("name IN ('thumbnail','crop','optimized')", name="ck_variants_name"), + sa.CheckConstraint( + "content_type IN ('image/jpeg','image/png','image/webp')", + name="ck_variants_content_type", + ), + sa.CheckConstraint("size_bytes > 0", name="ck_variants_size_bytes"), + sa.CheckConstraint("width > 0 AND height > 0", name="ck_variants_dimensions"), + sa.CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="ck_variants_sha256"), + sa.ForeignKeyConstraint( + ["attachment_id"], + ["attachments.id"], + name="fk_variants_attachment", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("attachment_id", "pipeline_version", "name"), + sa.UniqueConstraint("object_key", name="uq_variants_object_key"), + ) + op.create_index("ix_variants_attachment_id", "variants", ["attachment_id"]) + + op.create_table( + "job_events", + sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False), + sa.Column("job_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("from_state", sa.Text(), nullable=True), + sa.Column("to_state", sa.Text(), nullable=False), + sa.Column("attempt", sa.Integer(), nullable=True), + sa.Column("error_code", sa.Text(), nullable=True), + sa.Column("worker_id", sa.Text(), nullable=True), + sa.Column("detail", sa.Text(), nullable=True), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["job_id"], ["jobs.id"], name="fk_job_events_job", ondelete="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index("ix_job_events_job_id_created_at", "job_events", ["job_id", "created_at"]) + + op.create_table( + "job_replays", + sa.Column("id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("source_job_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("replay_job_id", postgresql.UUID(as_uuid=True), nullable=False), + sa.Column("requested_by", sa.Text(), nullable=False), + sa.Column("reason", sa.Text(), nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "char_length(requested_by) BETWEEN 1 AND 128", + name="ck_job_replays_requested_by", + ), + sa.CheckConstraint("char_length(reason) BETWEEN 1 AND 512", name="ck_job_replays_reason"), + sa.ForeignKeyConstraint( + ["replay_job_id"], + ["jobs.id"], + name="fk_job_replays_replay", + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["source_job_id"], + ["jobs.id"], + name="fk_job_replays_source", + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("replay_job_id", name="uq_job_replays_replay_job"), + ) + + op.create_table( + "fault_flags", + sa.Column("kind", sa.Text(), nullable=False), + sa.Column("remaining", sa.Integer(), nullable=False), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint("remaining >= 0", name="ck_fault_flags_remaining"), + sa.PrimaryKeyConstraint("kind"), + ) + + +def downgrade() -> None: + op.drop_table("fault_flags") + op.drop_table("job_replays") + op.drop_index("ix_job_events_job_id_created_at", table_name="job_events") + op.drop_table("job_events") + op.drop_index("ix_variants_attachment_id", table_name="variants") + op.drop_table("variants") + op.drop_index("uq_outbox_pending_job", table_name="outbox") + op.drop_index("ix_outbox_due", table_name="outbox") + op.drop_table("outbox") + op.drop_index("uq_jobs_initial_attachment_version", table_name="jobs") + op.drop_index("ix_jobs_lease_expires_at", table_name="jobs") + op.drop_index("ix_jobs_state_next_attempt_at", table_name="jobs") + op.drop_index("ix_jobs_attachment_id", table_name="jobs") + op.drop_table("jobs") + + op.drop_constraint("ck_attachments_pipeline_version", "attachments", type_="check") + op.drop_constraint("ck_attachments_status", "attachments", type_="check") + op.execute("UPDATE attachments SET status = 'pending' WHERE status = 'failed'") + op.create_check_constraint( + "ck_attachments_status", + "attachments", + "status IN ('pending','ready','failed')", + ) + op.alter_column( + "attachments", "status", server_default=sa.text("'pending'"), existing_type=sa.Text() + ) + op.drop_constraint("uq_attachments_staging_key", "attachments", type_="unique") + op.drop_column("attachments", "ready_at") + op.drop_column("attachments", "pipeline_version") + op.drop_column("attachments", "staging_key") diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..9c99eb9e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,44 @@ +# Рантайм-зависимости лабораторной 5: точное замыкание (top-level и транзитивные), +# получено `pip freeze` после установки на Python 3.11.9: +# pip install fastapi==0.141.1 uvicorn==0.53.0 "sqlalchemy[asyncio]==2.0.44" \ +# asyncpg==0.30.0 alembic==1.16.5 redis==6.4.0 argon2-cffi==25.1.0 \ +# boto3==1.43.125 Pillow==13.0.0 aio-pika==9.5.5 +alembic==1.16.5 +aio-pika==9.5.5 +aiormq==6.8.1 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +asyncpg==0.30.0 +boto3==1.43.125 +botocore==1.43.125 +cffi==2.1.1 +click==8.5.0 +exceptiongroup==1.3.2 +fastapi==0.141.1 +greenlet==3.5.6 +h11==0.16.0 +idna==3.20 +jmespath==1.1.0 +Mako==1.4.3 +MarkupSafe==3.0.4 +multidict==7.2.1 +pamqp==3.3.0 +pycparser==3.11 +pydantic==2.14.0 +Pillow==13.0.0 +propcache==0.5.4 +pydantic_core==2.50.0 +python-dateutil==2.9.0.post0 +redis==6.4.0 +SQLAlchemy==2.0.44 +s3transfer==0.19.3 +six==1.17.0 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.5.0 +uvicorn==0.53.0 +yarl==1.25.1 \ No newline at end of file diff --git a/scripts/bootstrap-secrets.sh b/scripts/bootstrap-secrets.sh new file mode 100644 index 00000000..855cc823 --- /dev/null +++ b/scripts/bootstrap-secrets.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# Create local Swarm secret files under secrets/ (gitignored, never printed). +# Existing files are never overwritten, so rerunning the script is safe. +# Usage: sh scripts/bootstrap-secrets.sh +set -eu + +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +dir="$root/secrets" + +# Files are created with mode 600: group and other have no access. +umask 077 +mkdir -p "$dir" + +random_hex() { + head -c 32 /dev/urandom | od -An -tx1 | tr -d ' \n' +} + +# 10 bytes = 20 hex chars: MinIO access keys are limited to 20 characters. +random_access_key() { + head -c 10 /dev/urandom | od -An -tx1 | tr -d ' \n' +} + +# write : keeps the current file if it already has content. +write() { + name=$1 + value=$2 + if [ -s "$dir/$name" ]; then + printf 'keep secrets/%s\n' "$name" + return 0 + fi + printf '%s\n' "$value" > "$dir/$name" + printf 'create secrets/%s\n' "$name" +} + +write postgres_password "$(random_hex)" +write cursor_secret "$(random_hex)" +write cache_url 'redis://valkey:6379/0' +# The password is hex, so the URL needs no percent-encoding. +write database_url "postgresql+asyncpg://messenger:$(cat "$dir/postgres_password")@postgres:5432/messenger" + +# MinIO: root credentials only for minio-init; API uses a separate restricted +# pair. The access key is 20 hex chars (MinIO limit), the secret is 64 hex chars. +write s3_root_user 'messenger-root' +write s3_root_password "$(random_hex)" +write s3_access_key_id "$(random_access_key)" +write s3_secret_access_key "$(random_hex)" + +# RabbitMQ: default user создаётся при первом старте из этих файлов; URL +# использует URL-safe hex-пароль (percent-encoding не нужен). +write rabbitmq_user 'messenger' +write rabbitmq_password "$(random_hex)" +write broker_url "amqp://messenger:$(cat "$dir/rabbitmq_password")@rabbitmq:5672/%2F" + +printf 'done: %s\n' "$dir" +printf 'next: sh scripts/deploy.sh\n' diff --git a/scripts/check-bucket-private.sh b/scripts/check-bucket-private.sh new file mode 100644 index 00000000..e9d2d75a --- /dev/null +++ b/scripts/check-bucket-private.sh @@ -0,0 +1,48 @@ +#!/bin/sh +# Проверка приватности bucket и ограниченности API-пользователя. +# +# Запускается в образе MinIO (в нём есть mc и curl) на backend-сети стенда: +# docker run --rm --network lab5_backend --user 0 \ +# -v "$PWD/scripts:/scripts:ro" -v "$PWD/secrets:/run/secrets:ro" \ +# -e S3_ENDPOINT=http://minio:9000 -e S3_BUCKET=messenger-attachments \ +# --entrypoint /bin/sh minio/minio:... /scripts/check-bucket-private.sh +# +# Root нужен только для чтения local secrets в этом проверочном контейнере; +# API-пользователь проверяется отдельным alias и не имеет прав на bucket. +set -eu + +endpoint=${S3_ENDPOINT:-http://minio:9000} +bucket=${S3_BUCKET:-messenger-attachments} +export MC_CONFIG_DIR=${MC_CONFIG_DIR:-/tmp/.mc} +mkdir -p "$MC_CONFIG_DIR" + +root_user=$(cat "${MINIO_ROOT_USER_FILE:?}") +root_password=$(cat "${MINIO_ROOT_PASSWORD_FILE:?}") +access_key=$(cat "${S3_ACCESS_KEY_ID_FILE:?}") +secret_key=$(cat "${S3_SECRET_ACCESS_KEY_FILE:?}") + +mc alias set root "$endpoint" "$root_user" "$root_password" >/dev/null +mc alias set api "$endpoint" "$access_key" "$secret_key" >/dev/null + +printf 'anonymous-policy: ' +mc anonymous get "root/$bucket" 2>&1 || true +printf 'anonymous-http-get: ' +curl -s -o /dev/null -w '%{http_code}\n' "$endpoint/$bucket/attachments/does-not-exist" +printf 'api-list-attachments: ' +mc ls "api/$bucket/attachments/" >/tmp/list.txt 2>&1 && head -1 /tmp/list.txt || cat /tmp/list.txt +printf 'api-create-bucket: ' +if mc mb "api/should-not-exist" >/dev/null 2>&1; then + printf 'ALLOWED (unexpected)\n' + mc rb "api/should-not-exist" >/dev/null 2>&1 || true +else + printf 'denied (expected)\n' +fi +printf 'api-put-other-bucket: ' +echo test >/tmp/probe +if mc cp /tmp/probe "api/other-bucket/" >/dev/null 2>&1; then + printf 'ALLOWED (unexpected)\n' +else + printf 'denied (expected)\n' +fi +printf 'root-bucket-list: ' +mc ls "root/$bucket/" >/dev/null 2>&1 && printf 'ok\n' || printf 'failed\n' diff --git a/scripts/deploy.cmd b/scripts/deploy.cmd new file mode 100644 index 00000000..77170029 --- /dev/null +++ b/scripts/deploy.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (deploy). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode deploy %* +exit /b %ERRORLEVEL% diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100644 index 00000000..3c7f446a --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Build the application image, create local Swarm secrets once, deploy the stack. +# +# Prerequisites: Docker Engine with Swarm mode (docker swarm init is done here if +# needed) and the files created by scripts/bootstrap-secrets.sh. No secret value +# is printed. This is the one documented bootstrap sequence for stack.yaml. +set -eu +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" + +# The public failover test invokes the action script without a shell; it must be +# executable on POSIX. Git may not preserve the mode on a Windows checkout. +chmod +x scripts/stop-one.sh scripts/restore.sh scripts/restart.sh scripts/rolling-update.sh scripts/deploy.sh 2>/dev/null || true + +stack=${STACK_NAME:-lab5} +image=${API_IMAGE:-mtusi-messenger-lab5:5.0.0} + +state=$(docker info --format '{{.Swarm.LocalNodeState}}') +if [ "$state" != "active" ]; then + docker swarm init >/dev/null +fi + +sh scripts/bootstrap-secrets.sh +for name in database_url cache_url broker_url postgres_password cursor_secret \ + rabbitmq_user rabbitmq_password \ + s3_root_user s3_root_password s3_access_key_id s3_secret_access_key; do + if ! docker secret inspect "${stack}_${name}" >/dev/null 2>&1; then + docker secret create "${stack}_${name}" "secrets/${name}" >/dev/null + fi +done + +docker build -t "$image" . + +# Swarm configs неизменяемы. Перед повторным deploy убираем завершившийся +# minio-init и его config, иначе stack deploy попытается обновить config и +# получит "only updates to Labels are allowed". На первом deploy это no-op. +docker service rm "${stack}_minio-init" >/dev/null 2>&1 || true +docker config rm "${stack}_lab5_minio_init" >/dev/null 2>&1 || true + +API_IMAGE="$image" docker stack deploy --resolve-image never -c stack.yaml "$stack" + +i=0 +while [ "$i" -lt 90 ]; do + if curl -fsS http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then + printf 'deployed: http://127.0.0.1:8080 (image %s)\n' "$image" + docker stack services "$stack" + exit 0 + fi + i=$((i + 1)) + sleep 2 +done +printf 'stack did not become ready within 180s\n' >&2 +docker stack services "$stack" >&2 || true +exit 1 \ No newline at end of file diff --git a/scripts/hooks.ps1 b/scripts/hooks.ps1 new file mode 100644 index 00000000..b5c93766 --- /dev/null +++ b/scripts/hooks.ps1 @@ -0,0 +1,188 @@ +# Windows twin of scripts/deploy.sh, stop-one.sh, restore.sh, restart.sh and +# rolling-update.sh. The public resilience test launches an action script with +# Popen([script]) without a shell; on native Windows a .cmd wrapper is required. +# +# X-Instance-Id is the Swarm task container hostname, so a target is resolved by +# inspecting running task containers, not by trusting the header alone. +param( + [Parameter(Mandatory = $true)] + [ValidateSet('deploy', 'stop-one', 'restore', 'restart', 'rollout', 'stop-worker', 'start-worker', 'restart-broker')] + [string]$Mode, + [string]$Image = '' +) + +$ErrorActionPreference = 'Stop' +$root = Split-Path -Parent (Split-Path -Parent $MyInvocation.MyCommand.Path) +Set-Location $root +$stack = if ($env:STACK_NAME) { $env:STACK_NAME } else { 'lab5' } +$service = "$stack" + '_api' +if (-not $Image) { + $Image = if ($env:API_IMAGE) { $env:API_IMAGE } else { 'mtusi-messenger-lab5:5.0.0' } +} + +function Invoke-Docker { + param([Parameter(ValueFromRemainingArguments = $true)][string[]]$Arguments) + & docker @Arguments + if ($LASTEXITCODE -ne 0) { + throw "docker $($Arguments -join ' ') failed with exit code $LASTEXITCODE" + } +} + +function Test-Ready { + try { + $response = Invoke-WebRequest -UseBasicParsing -Uri 'http://127.0.0.1:8080/health/ready' -TimeoutSec 3 + return $response.StatusCode -eq 200 + } catch { + return $false + } +} + +function Wait-Ready { + param([int]$Seconds = 180) + $deadline = (Get-Date).AddSeconds($Seconds) + while ((Get-Date) -lt $deadline) { + if (Test-Ready) { return } + Start-Sleep -Seconds 2 + } + throw "stack did not become ready within $Seconds seconds" +} + +function Get-RunningTasks { + return @(& docker service ps -q $service --filter desired-state=running) +} + +function Find-TargetContainer { + param([string]$Target) + foreach ($task in (Get-RunningTasks)) { + $container = (& docker inspect -f '{{.Status.ContainerStatus.ContainerID}}' $task 2>$null) + if (-not $container) { continue } + $hostname = (& docker inspect -f '{{.Config.Hostname}}' $container 2>$null) + if ($hostname -eq $Target) { return $container } + } + return $null +} + +function New-HexSecret { + param([int]$Bytes = 32) + $buffer = New-Object byte[] $Bytes + [System.Security.Cryptography.RandomNumberGenerator]::Create().GetBytes($buffer) + return (-join ($buffer | ForEach-Object { $_.ToString('x2') })) +} + +function Ensure-SecretFiles { + New-Item -ItemType Directory -Force -Path 'secrets' | Out-Null + $passwordPath = Join-Path 'secrets' 'postgres_password' + $password = if ((Test-Path $passwordPath) -and (Get-Item $passwordPath).Length -gt 0) { + (Get-Content $passwordPath -Raw).Trim() + } else { + New-HexSecret + } + $rabbitPassword = if ((Test-Path (Join-Path 'secrets' 'rabbitmq_password')) -and (Get-Item (Join-Path 'secrets' 'rabbitmq_password')).Length -gt 0) { + (Get-Content (Join-Path 'secrets' 'rabbitmq_password') -Raw).Trim() + } else { + New-HexSecret + } + $defaults = [ordered]@{ + postgres_password = $password + cursor_secret = (New-HexSecret) + cache_url = 'redis://valkey:6379/0' + database_url = "postgresql+asyncpg://messenger:$password@postgres:5432/messenger" + rabbitmq_user = 'messenger' + rabbitmq_password = $rabbitPassword + broker_url = "amqp://messenger:$rabbitPassword@rabbitmq:5672/%2F" + s3_root_user = 'messenger-root' + s3_root_password = (New-HexSecret) + s3_access_key_id = (New-HexSecret -Bytes 10) + s3_secret_access_key = (New-HexSecret) + } + foreach ($name in $defaults.Keys) { + $path = Join-Path 'secrets' $name + if (-not (Test-Path $path) -or (Get-Item $path).Length -eq 0) { + Set-Content -NoNewline -Encoding ascii -Path $path -Value $defaults[$name] + } + } +} + +function Ensure-Secrets { + $names = @('database_url', 'cache_url', 'broker_url', 'postgres_password', 'cursor_secret', + 'rabbitmq_user', 'rabbitmq_password', + 's3_root_user', 's3_root_password', 's3_access_key_id', 's3_secret_access_key') + $existing = @(& docker secret ls --format '{{.Name}}') + foreach ($name in $names) { + $secret = "$stack" + '_' + $name + if ($existing -notcontains $secret) { + Invoke-Docker secret create $secret (Join-Path 'secrets' $name) | Out-Null + } + } +} + +switch ($Mode) { + 'deploy' { + $state = (& docker info --format '{{.Swarm.LocalNodeState}}').Trim() + if ($state -ne 'active') { Invoke-Docker swarm init | Out-Null } + Ensure-SecretFiles + Ensure-Secrets + Invoke-Docker build -t $Image . | Out-Null + $env:API_IMAGE = $Image + # Swarm configs неизменяемы: перед повторным deploy убираем завершившийся + # minio-init и его config; на первом deploy это no-op. + & cmd /c "docker service rm ${stack}_minio-init >nul 2>&1" + & cmd /c "docker config rm ${stack}_lab5_minio_init >nul 2>&1" + Invoke-Docker stack deploy --resolve-image never -c stack.yaml $stack | Out-Null + Wait-Ready + & docker stack services $stack + } + 'stop-one' { + $target = $env:TARGET_INSTANCE_ID + if (-not $target) { throw 'TARGET_INSTANCE_ID is required' } + $container = Find-TargetContainer $target + if (-not $container) { throw "target instance $target not found among running tasks" } + Write-Output "stopping instance $target container $container" + Invoke-Docker stop --time 3 $container | Out-Null + # Swarm may already be creating a replacement task; the target container + # itself was stopped by id and its X-Instance-Id will not be served again. + Write-Output "instance $target stopped" + } + 'restore' { + Invoke-Docker service update --restart-condition any --replicas 2 $service | Out-Null + $running = @(Get-RunningTasks).Count + if ($running -lt 2) { Invoke-Docker service update --force $service | Out-Null } + $deadline = (Get-Date).AddSeconds(120) + while ((Get-Date) -lt $deadline) { + $running = @(Get-RunningTasks).Count + if ($running -ge 2 -and (Test-Ready)) { + Write-Output "api restored: $running running tasks and readiness OK" + exit 0 + } + Start-Sleep -Seconds 2 + } + throw 'api did not recover within 120s' + } + 'restart' { + foreach ($name in @('postgres', 'valkey', 'minio', 'rabbitmq', 'relay', 'worker', 'api', 'traefik')) { + $svc = $stack + '_' + $name + Write-Output "restarting $svc" + Invoke-Docker service update --detach=true --force $svc | Out-Null + } + Wait-Ready + Write-Output 'stack ready after restart' + } + 'stop-worker' { + Invoke-Docker service scale "${stack}_worker=0" | Out-Null + Write-Output 'worker scaled to 0' + } + 'start-worker' { + Invoke-Docker service scale "${stack}_worker=1" | Out-Null + Write-Output 'worker scaled to 1' + } + 'restart-broker' { + Invoke-Docker service update --force "${stack}_rabbitmq" | Out-Null + Start-Sleep -Seconds 10 + Write-Output 'rabbitmq restart requested' + } + 'rollout' { + Invoke-Docker build --build-arg APP_REVISION=$Image -t $Image . | Out-Null + Invoke-Docker service update --image $Image --update-order start-first --update-parallelism 1 --update-delay 5s --update-failure-action rollback $service | Out-Null + & docker service ps $service --no-trunc + } +} \ No newline at end of file diff --git a/scripts/minio-init.sh b/scripts/minio-init.sh new file mode 100644 index 00000000..33402ee7 --- /dev/null +++ b/scripts/minio-init.sh @@ -0,0 +1,86 @@ +#!/bin/sh +# Идемпотентный bootstrap приватного bucket и ограниченного S3-пользователя. +# +# Запускается один раз сервисом minio-init (образ MinIO уже содержит mc). +# Повторный запуск не удаляет данные: bucket создаётся с --ignore-existing, +# пользователь переиспользуется, политика пересоздаётся и привязывается заново. +# Root-credentials используются только здесь; API получает отдельную пару. +set -eu + +endpoint=${S3_ENDPOINT:-http://minio:9000} +bucket=${S3_BUCKET:-messenger-attachments} +policy_name=${S3_POLICY_NAME:-messenger-attachments-rw} + +read_secret() { + # $1 — путь к Docker secret; секрет не печатается. + cat "$1" +} + +root_user=$(read_secret "${MINIO_ROOT_USER_FILE:?MINIO_ROOT_USER_FILE is required}") +root_password=$(read_secret "${MINIO_ROOT_PASSWORD_FILE:?MINIO_ROOT_PASSWORD_FILE is required}") +access_key=$(read_secret "${S3_ACCESS_KEY_ID_FILE:?S3_ACCESS_KEY_ID_FILE is required}") +secret_key=$(read_secret "${S3_SECRET_ACCESS_KEY_FILE:?S3_SECRET_ACCESS_KEY_FILE is required}") + +export MC_CONFIG_DIR=${MC_CONFIG_DIR:-/tmp/.mc} +mkdir -p "$MC_CONFIG_DIR" + +i=0 +until mc alias set local "$endpoint" "$root_user" "$root_password" --api S3v4 >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -ge 60 ]; then + printf 'minio is not reachable at %s\n' "$endpoint" >&2 + exit 1 + fi + sleep 2 +done + +until mc ready local >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -ge 120 ]; then + printf 'minio did not become ready\n' >&2 + exit 1 + fi + sleep 1 +done + +mc mb --ignore-existing "local/$bucket" >/dev/null +mc anonymous set none "local/$bucket" >/dev/null + +# Пользователь API: при повторном запуске оставляем существующего, чтобы не +# разорвать уже работающий API. Секрет при первом создании берётся из secrets. +mc admin user add local "$access_key" "$secret_key" >/dev/null 2>&1 || true +if ! mc admin user info local "$access_key" >/dev/null 2>&1; then + printf 'failed to create API user\n' >&2 + exit 1 +fi + +cat >/tmp/messenger-attachments-policy.json </dev/null 2>&1 || true +mc admin policy remove local "$policy_name" >/dev/null 2>&1 || true +mc admin policy create local "$policy_name" /tmp/messenger-attachments-policy.json >/dev/null +mc admin policy attach local "$policy_name" --user "$access_key" >/dev/null + +printf 'minio bootstrap complete: bucket=%s private=yes policy=%s\n' "$bucket" "$policy_name" diff --git a/scripts/reconcile.sh b/scripts/reconcile.sh new file mode 100644 index 00000000..6b46bba1 --- /dev/null +++ b/scripts/reconcile.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Запуск сверки PostgreSQL/S3 внутри работающего API-контейнера. +# +# Использует те же secrets и переменные окружения, что и API, поэтому не +# требует отдельного набора credentials. По умолчанию — dry-run; для удаления +# мусора добавьте --apply. +set -eu +stack=${STACK_NAME:-lab5} +container=$(docker ps -q -f "name=${stack}_api" | head -n 1) +if [ -z "$container" ]; then + printf 'no running %s_api container\n' "$stack" >&2 + exit 2 +fi +docker exec "$container" python -m messenger.reconcile "$@" diff --git a/scripts/restart-broker.cmd b/scripts/restart-broker.cmd new file mode 100644 index 00000000..7f2c2280 --- /dev/null +++ b/scripts/restart-broker.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (restart-broker). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode restart-broker %* +exit /b %ERRORLEVEL% diff --git a/scripts/restart-broker.sh b/scripts/restart-broker.sh new file mode 100644 index 00000000..d238382a --- /dev/null +++ b/scripts/restart-broker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Restart RabbitMQ with its named volume; active durable messages must survive. +set -eu +stack=${STACK_NAME:-lab5} +docker service update --force "${stack}_rabbitmq" >/dev/null +printf 'rabbitmq restart requested\n' diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..bd5a8b23 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (restart). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode restart %* +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..e418eaf3 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Restart API and every state store without deleting volumes. +# +# Used by make test-persistence. Sessions live in Valkey with AOF and users, +# chats and messages in PostgreSQL, so the same bearer token and data survive. +set -eu +stack=${STACK_NAME:-lab5} +ready_url=${READY_URL:-http://127.0.0.1:8080/health/ready} +for service in "${stack}_postgres" "${stack}_valkey" "${stack}_minio" "${stack}_rabbitmq" \ + "${stack}_relay" "${stack}_worker" "${stack}_api" "${stack}_traefik"; do + printf 'restarting %s\n' "$service" + # detach: не ждём сходимости каждого сервиса; готовность проверяется один раз ниже. + docker service update --detach=true --force "$service" >/dev/null +done + +i=0 +while [ "$i" -lt 60 ]; do + if curl -fsS "$ready_url" >/dev/null 2>&1; then + printf 'stack ready after restart\n' + exit 0 + fi + i=$((i + 1)) + sleep 2 +done +printf 'stack did not become ready within 120s\n' >&2 +exit 1 diff --git a/scripts/restore.cmd b/scripts/restore.cmd new file mode 100644 index 00000000..0d72b653 --- /dev/null +++ b/scripts/restore.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (restore). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode restore %* +exit /b %ERRORLEVEL% diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100644 index 00000000..146c7ca8 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Restore the API service to 2 replicas after scripts/stop-one.sh. +set -eu +stack=${STACK_NAME:-lab5} +service="${stack}_api" + +# Swarm normally creates the replacement itself after stopping one task. This hook +# restores the declared replica count and waits for two healthy tasks; a forced +# rolling update is used only when the service is still under-replicated. +docker service update --restart-condition any --replicas 2 "$service" >/dev/null +running=$(docker service ps -q "$service" --filter desired-state=running | wc -l | tr -d ' ') +if [ "$running" -lt 2 ]; then + docker service update --force "$service" >/dev/null +fi + +i=0 +while [ "$i" -lt 60 ]; do + running=$(docker service ps -q "$service" --filter desired-state=running | wc -l | tr -d ' ') + if [ "$running" -ge 2 ] && curl -fsS http://127.0.0.1:8080/health/ready >/dev/null 2>&1; then + printf 'api restored: %s running tasks and readiness OK\n' "$running" + exit 0 + fi + i=$((i + 1)) + sleep 2 +done +printf 'api did not recover within 120s\n' >&2 +exit 1 \ No newline at end of file diff --git a/scripts/rolling-update.cmd b/scripts/rolling-update.cmd new file mode 100644 index 00000000..62f46a43 --- /dev/null +++ b/scripts/rolling-update.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (rollout). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode rollout %* +exit /b %ERRORLEVEL% diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..c9efaa66 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Start-first rolling update of the API service under load. +# +# Swarm replaces one task at a time (parallelism 1), waiting for the new task's +# health check before removing the old one. Traefik keeps routing to ready tasks. +set -eu +root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$root" +stack=${STACK_NAME:-lab5} +image=${API_IMAGE:-mtusi-messenger-lab5:5.0.0} + +docker build --build-arg APP_REVISION="$image" -t "$image" . +docker service update \ + --image "$image" \ + --update-order start-first \ + --update-parallelism 1 \ + --update-delay 5s \ + --update-failure-action rollback \ + "${stack}_api" +docker service ps "${stack}_api" --no-trunc \ No newline at end of file diff --git a/scripts/start-worker.cmd b/scripts/start-worker.cmd new file mode 100644 index 00000000..379c0621 --- /dev/null +++ b/scripts/start-worker.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (start-worker). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode start-worker %* +exit /b %ERRORLEVEL% diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100644 index 00000000..0306b76f --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Restore one worker replica after scripts/stop-worker.sh. +set -eu +stack=${STACK_NAME:-lab5} +docker service scale "${stack}_worker=1" >/dev/null +printf 'worker scaled to 1\n' diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..a7838d62 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (stop-one). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode stop-one %* +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..3e5b2f98 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,50 @@ +#!/bin/sh +# Stop exactly one API replica selected by TARGET_INSTANCE_ID. +# +# The public failover test sets TARGET_INSTANCE_ID to the X-Instance-Id that served +# the test request. In this stack X-Instance-Id is the container hostname, so the +# target task is found by inspecting every running task's container and stopped by +# container id. Swarm then creates a *new* task (new hostname) after its restart +# delay: the laboratory explicitly allows that, and the test checks that the old +# X-Instance-Id disappears from responses. Restore with scripts/restore.sh. +set -eu + +stack=${STACK_NAME:-lab5} +service="${stack}_api" +target=${TARGET_INSTANCE_ID:-} +if [ -z "$target" ]; then + printf 'TARGET_INSTANCE_ID is required\n' >&2 + exit 2 +fi + +if ! docker service inspect "$service" >/dev/null 2>&1; then + printf 'service %s not found\n' "$service" >&2 + exit 3 +fi + +find_container() { + # Prints the container id whose hostname equals $target; empty if not found. + for task in $(docker service ps -q "$service" --filter desired-state=running); do + container=$(docker inspect -f '{{.Status.ContainerStatus.ContainerID}}' "$task" 2>/dev/null || true) + [ -n "$container" ] || continue + hostname=$(docker inspect -f '{{.Config.Hostname}}' "$container" 2>/dev/null || true) + if [ "$hostname" = "$target" ]; then + printf '%s\n' "$container" + return 0 + fi + done + return 0 +} + +container=$(find_container) +if [ -z "$container" ]; then + printf 'target instance %s not found among running tasks\n' "$target" >&2 + exit 4 +fi + +printf 'stopping instance %s container %s\n' "$target" "$container" +docker stop --time 3 "$container" >/dev/null + +# Swarm may already be creating a replacement task with a different hostname; +# the target container itself was stopped by id and is no longer served. +printf 'instance %s stopped\n' "$target" \ No newline at end of file diff --git a/scripts/stop-worker.cmd b/scripts/stop-worker.cmd new file mode 100644 index 00000000..e48afc90 --- /dev/null +++ b/scripts/stop-worker.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper for scripts/hooks.ps1 (stop-worker). +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0hooks.ps1" -Mode stop-worker %* +exit /b %ERRORLEVEL% diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100644 index 00000000..e1c30751 --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Scale the worker to zero; accepted jobs stay durable in the outbox/queue. +set -eu +stack=${STACK_NAME:-lab5} +docker service scale "${stack}_worker=0" >/dev/null +printf 'worker scaled to 0\n' diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..a9faa31b --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,5 @@ +"""Мессенджер на PostgreSQL, Valkey, S3 и RabbitMQ: лабораторная 5 (MTUSI).""" + +__all__ = ["__version__"] + +__version__ = "5.0.0" \ No newline at end of file diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..df116f9f --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,38 @@ +"""Точка входа: `python -m messenger` (в контейнере — PID 1).""" +from __future__ import annotations + +import sys + +import uvicorn + +from .app import create_app +from .config import Config, ConfigError + + +def main() -> int: + try: + config = Config.from_env() + except ConfigError as exc: + print(f"config error: {exc}", file=sys.stderr) + return 2 + + uvicorn.run( + create_app(config), + host=config.host, + port=config.port, + # Свой формат логов: uvicorn не пишет access-лог с query-строкой. + log_config=None, + access_log=False, + # Верхняя граница ожидания активных запросов при остановке. + timeout_graceful_shutdown=config.graceful_shutdown_seconds, + # Один процесс: лимиты пула соединений заданы на процесс, а не на весь стенд. + workers=1, + server_header=False, + date_header=False, + proxy_headers=False, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/messenger/access.py b/src/messenger/access.py new file mode 100644 index 00000000..a03d541e --- /dev/null +++ b/src/messenger/access.py @@ -0,0 +1,16 @@ +"""Правила доступа: чужой объект неотличим от отсутствующего (404, не 403).""" +from __future__ import annotations + +from uuid import UUID + +from .domain import ChatRecord, UserRecord +from .errors import not_found +from .repository import Repository + + +async def visible_chat(repository: Repository, chat_id: UUID, user: UserRecord) -> ChatRecord: + """Возвращает чат, если пользователь его участник, иначе 404.""" + chat = await repository.chat_for_member(chat_id, user.id) + if chat is None: + raise not_found("Чат не найден") + return chat diff --git a/src/messenger/admin.py b/src/messenger/admin.py new file mode 100644 index 00000000..35af4fbd --- /dev/null +++ b/src/messenger/admin.py @@ -0,0 +1,175 @@ +"""Операторский CLI: снимок состояния, replay, fault-injection и duplicate. + +Запускается внутри контейнера с теми же секретами, что API: + python -m messenger.admin show --attachment + python -m messenger.admin stats + python -m messenger.admin replay --job --operator denis --reason "..." + +Публичного HTTP-доступа к этим операциям нет: это не часть обязательного API. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import os +import sys +from uuid import UUID + +from .broker import Broker, build_payload +from .config import Config, ConfigError +from .db import Database +from .jobs import FAULT_KINDS, JobRepository, ReplayRejected +from .pipeline import UnsupportedPipeline, variant_specs +from .storage import S3Storage + + +def _json(value) -> None: + print(json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True)) + + +async def _run(args: argparse.Namespace) -> int: + config = Config.from_env() + database = Database(config) + jobs = JobRepository(database, config) + broker: Broker | None = None + try: + if args.command == "show": + _json(await jobs.report(args.attachment)) + return 0 + if args.command == "stats": + _json(await jobs.queue_stats()) + return 0 + if args.command == "objects": + storage = S3Storage(config) + try: + objects = await storage.list_objects(config.s3_key_prefix + args.prefix) + finally: + await storage.close() + _json( + [ + { + "key": item.key, + "size_bytes": item.size_bytes, + "last_modified": item.last_modified.isoformat().replace("+00:00", "Z"), + } + for item in objects + ] + ) + return 0 + if args.command == "jobs": + _json( + [ + { + "id": str(job.id), + "pipeline_version": job.pipeline_version, + "state": job.state, + "attempts": job.attempts, + "max_attempts": job.max_attempts, + "last_error_code": job.last_error_code, + "replay_of": str(job.replay_of) if job.replay_of else None, + } + for job in await jobs.jobs_for_attachment(args.attachment) + ] + ) + return 0 + if args.command == "fault": + if not config.allow_fault_injection: + print("fault injection is disabled (ALLOW_FAULT_INJECTION=false)", file=sys.stderr) + return 2 + await jobs.set_fault(args.kind, args.count) + _json({"kind": args.kind, "remaining": args.count}) + return 0 + if args.command == "replay": + if not args.job and not args.attachment: + print("replay requires --job or --attachment", file=sys.stderr) + return 2 + try: + new_id = await jobs.replay( + job_id=args.job, + attachment_id=args.attachment, + requested_by=args.operator, + reason=args.reason, + ) + except ReplayRejected as exc: + print(f"replay rejected: {exc}", file=sys.stderr) + return 3 + _json({"replay_job_id": str(new_id)}) + return 0 + if args.command == "enqueue": + try: + variant_specs(args.pipeline_version) + except UnsupportedPipeline as exc: + print(f"enqueue rejected: {exc}", file=sys.stderr) + return 2 + try: + new_id = await jobs.force_enqueue(args.attachment, args.pipeline_version) + except ReplayRejected as exc: + print(f"enqueue rejected: {exc}", file=sys.stderr) + return 3 + _json({"job_id": str(new_id), "pipeline_version": args.pipeline_version}) + return 0 + if args.command == "duplicate": + job = await jobs.get_job(args.job) + if job is None: + print("job not found", file=sys.stderr) + return 2 + broker = Broker(config) + await broker.connect() + for index in range(args.count): + payload = build_payload(job, attempt=job.attempts + 1) + await broker.publish_job(payload, message_id=str(job.id)) + _json({"job_id": str(job.id), "published": args.count, "attempt": job.attempts + 1}) + return 0 + print("unknown command", file=sys.stderr) + return 2 + finally: + if broker is not None: + await broker.close() + await database.dispose() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Операторский CLI lab5") + sub = parser.add_subparsers(dest="command", required=True) + + show = sub.add_parser("show", help="снимок attachment/jobs/outbox/events") + show.add_argument("--attachment", type=UUID, required=True) + + stats = sub.add_parser("stats", help="счётчики job/outbox") + stats.set_defaults() + + jobs_parser = sub.add_parser("jobs", help="задания вложения") + jobs_parser.add_argument("--attachment", type=UUID, required=True) + + objects_parser = sub.add_parser("objects", help="объекты S3 под префиксом") + objects_parser.add_argument("--prefix", required=True) + + fault = sub.add_parser("fault", help="одноразовый fault flag (только lab)") + fault.add_argument("--kind", choices=FAULT_KINDS, required=True) + fault.add_argument("--count", type=int, default=1) + + replay = sub.add_parser("replay", help="явный replay failed-задачи") + replay.add_argument("--job", type=UUID, default=None) + replay.add_argument("--attachment", type=UUID, default=None) + replay.add_argument("--operator", default=os.environ.get("OPERATOR", "operator")) + replay.add_argument("--reason", default="manual replay") + + enqueue = sub.add_parser("enqueue", help="поставить job конкретной версии (тест/оператор)") + enqueue.add_argument("--attachment", type=UUID, required=True) + enqueue.add_argument("--pipeline-version", type=int, required=True) + + duplicate = sub.add_parser("duplicate", help="опубликовать то же сообщение N раз") + duplicate.add_argument("--job", type=UUID, required=True) + duplicate.add_argument("--count", type=int, default=2) + + args = parser.parse_args(argv) + try: + return asyncio.run(_run(args)) + except ConfigError as exc: + print(f"config error: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..25e10af3 --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,142 @@ +"""Сборка ASGI-приложения: жизненный цикл, обработчики ошибок, маршруты.""" +from __future__ import annotations + +import asyncio +import logging +from contextlib import asynccontextmanager +from typing import AsyncIterator, Awaitable, Callable + +from fastapi import FastAPI + +from . import __version__ +from .config import Config +from .cursor import CursorCodec +from .db import Database +from .errorhandlers import register_error_handlers +from .logging_setup import configure_logging, log +from .middleware import RequestContextMiddleware +from .routers import attachments, auth, chats, health, messages, users +from .sessions import RedisSessionStore +from .storage import S3Storage + +ROUTERS = ( + health.router, + auth.router, + users.router, + chats.router, + messages.router, + attachments.router, +) + + +async def _release( + logger: logging.Logger, resource: str, closer: Callable[[], Awaitable[None]] +) -> None: + """Освобождает ресурс: отказ зависимости при остановке не должен мешать выходу.""" + try: + await closer() + except Exception as exc: + log( + logger, + logging.WARNING, + "release_failed", + exc_info=exc, + resource=resource, + error=type(exc).__name__, + ) + + +def build_fastapi(config: Config, logger: logging.Logger) -> FastAPI: + """Создаёт приложение FastAPI с обработчиками ошибок и маршрутами.""" + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + # Клиенты создаются здесь, но к зависимостям не подключаются: старт не + # должен падать, если PostgreSQL или Valkey поднимаются одновременно с API. + # За трафик отвечает готовность, а не перезапуск контейнера. + database = Database(config) + sessions = RedisSessionStore( + config.cache_url, config.session_ttl_seconds, config.cache_timeout_seconds + ) + storage = S3Storage(config) + app.state.database = database + app.state.sessions = sessions + app.state.storage = storage + app.state.config = config + app.state.cursors = CursorCodec(config.cursor_secret) + app.state.draining = False + log( + logger, + logging.INFO, + "startup", + version=__version__, + host=config.host, + port=config.port, + shutdown_budget_seconds=config.shutdown_budget_seconds, + storage="postgresql+s3", + s3_bucket=config.s3_bucket, + max_upload_bytes=config.max_upload_bytes, + max_image_pixels=config.max_image_pixels, + instance_id=config.instance_id, + session_store=config.session_store, + session_ttl_seconds=config.session_ttl_seconds, + db_pool_size=config.db_pool_size, + db_max_overflow=config.db_max_overflow, + db_pool_timeout_seconds=config.db_pool_timeout_seconds, + db_connect_timeout_seconds=config.db_connect_timeout_seconds, + db_command_timeout_seconds=config.db_command_timeout_seconds, + db_statement_timeout_ms=config.db_statement_timeout_ms, + cache_timeout_seconds=config.cache_timeout_seconds, + ) + if config.binds_to_loopback: + # Bind на loopback внутри контейнера не виден через published port. + log( + logger, + logging.WARNING, + "loopback_bind", + detail="bind на loopback недоступен через published port контейнера", + ) + try: + yield + finally: + app.state.draining = True + log( + logger, + logging.INFO, + "shutdown_begin", + drain_seconds=config.shutdown_drain_seconds, + note="слушающий сокет уже закрыт: новые соединения не принимаются", + ) + if config.shutdown_drain_seconds > 0: + # Окно управляемого снятия с балансировки: процесс ещё жив, + # но трафик на него уже не идёт. + await asyncio.sleep(config.shutdown_drain_seconds) + await _release(logger, "storage", storage.close) + await _release(logger, "sessions", sessions.close) + await _release(logger, "database", database.dispose) + app.state.draining = False + log(logger, logging.INFO, "shutdown_complete") + + app = FastAPI( + title="MTUSI Messenger — лабораторная 5", + version=__version__, + lifespan=lifespan, + # Источник истины — contracts/openapi.json: свои /docs и /openapi.json не публикуются. + docs_url=None, + redoc_url=None, + openapi_url=None, + ) + register_error_handlers(app, logger) + for router in ROUTERS: + app.include_router(router) + return app + + +def create_app( + config: Config | None = None, + logger: logging.Logger | None = None, +) -> RequestContextMiddleware: + """Создаёт приложение; разбор окружения идёт здесь, а не при импорте модуля.""" + config = Config.from_env() if config is None else config + logger = configure_logging(config.log_level, config.env) if logger is None else logger + return RequestContextMiddleware(build_fastapi(config, logger), logger, config.instance_id) \ No newline at end of file diff --git a/src/messenger/broker.py b/src/messenger/broker.py new file mode 100644 index 00000000..93afee58 --- /dev/null +++ b/src/messenger/broker.py @@ -0,0 +1,182 @@ +"""RabbitMQ: durable quorum queue, publisher confirms и ручной consumer ack. + +Брокер переносит только ссылки/служебные поля задания по jobs.schema.json. +Байты изображения и credentials в сообщение не попадают. Publisher confirm +подтверждает приём брокером, consumer ack — устойчивую обработку; это разные +границы, и между ними возможна повторная доставка. +""" +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Awaitable, Callable +from uuid import UUID + +from aio_pika import DeliveryMode, ExchangeType, Message, connect_robust +from aio_pika.abc import AbstractChannel, AbstractIncomingMessage, AbstractQueue + +from .config import ( + DEFAULT_DLQ_EXCHANGE, + DEFAULT_DLQ_QUEUE, + DEFAULT_JOB_QUEUE, + DEFAULT_JOB_ROUTING_KEY, + Config, +) +from .domain import rfc3339_z +from .jobs import JOB_TYPE, JobRecord + +logger = logging.getLogger("messenger.broker") + +REQUIRED_FIELDS = ( + "schema_version", + "job_id", + "type", + "attachment_id", + "pipeline_version", + "attempt", + "created_at", +) +OPTIONAL_FIELDS = ("traceparent", "tracestate") + + +class InvalidJobMessage(ValueError): + """Сообщение не соответствует jobs.schema.json; оно уходит в DLQ.""" + + +def build_payload(job: JobRecord, *, attempt: int) -> dict: + """Точное представление задания: только ID и служебные поля.""" + return { + "schema_version": 1, + "job_id": str(job.id), + "type": JOB_TYPE, + "attachment_id": str(job.attachment_id), + "pipeline_version": job.pipeline_version, + "attempt": attempt, + "created_at": rfc3339_z(job.created_at), + } + + +def parse_payload(body: bytes) -> dict: + """Строгая проверка сообщения до обращения к БД.""" + try: + payload = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + raise InvalidJobMessage("message is not valid UTF-8 JSON") from exc + if not isinstance(payload, dict): + raise InvalidJobMessage("message must be a JSON object") + extra = set(payload).difference(REQUIRED_FIELDS + OPTIONAL_FIELDS) + if extra: + raise InvalidJobMessage("unexpected message fields") + missing = [field for field in REQUIRED_FIELDS if field not in payload] + if missing: + raise InvalidJobMessage("missing message fields") + if payload["schema_version"] != 1 or payload["type"] != JOB_TYPE: + raise InvalidJobMessage("unsupported schema_version/type") + try: + UUID(str(payload["job_id"])) + UUID(str(payload["attachment_id"])) + except (ValueError, TypeError) as exc: + raise InvalidJobMessage("invalid UUID") from exc + if not isinstance(payload["pipeline_version"], int) or payload["pipeline_version"] < 1: + raise InvalidJobMessage("invalid pipeline_version") + if not isinstance(payload["attempt"], int) or payload["attempt"] < 1: + raise InvalidJobMessage("invalid attempt") + if not isinstance(payload["created_at"], str) or not payload["created_at"].endswith("Z"): + raise InvalidJobMessage("invalid created_at") + return payload + + +class Broker: + """Одно соединение на процесс; каналы и topology объявляются один раз.""" + + def __init__(self, config: Config) -> None: + self._url = config.require_broker() + self._queue_name = DEFAULT_JOB_QUEUE + self._routing_key = DEFAULT_JOB_ROUTING_KEY + self._dlx_name = DEFAULT_DLQ_EXCHANGE + self._dlq_name = DEFAULT_DLQ_QUEUE + self._connection = None + self._channel: AbstractChannel | None = None + self._exchange = None + self._queue: AbstractQueue | None = None + + async def connect(self) -> None: + self._connection = await connect_robust( + self._url, + heartbeat=30, + timeout=10, + connection_attempts=10, + retry_interval=1, + ) + self._channel = await self._connection.channel( + publisher_confirms=True, + on_return_raises=True, + ) + assert self._channel is not None + self._exchange = await self._channel.declare_exchange( + self._queue_name, ExchangeType.DIRECT, durable=True + ) + self._queue = await self._channel.declare_queue( + self._queue_name, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-dead-letter-exchange": self._dlx_name, + }, + ) + await self._queue.bind(self._exchange, routing_key=self._routing_key) + dead_letter_exchange = await self._channel.declare_exchange( + self._dlx_name, ExchangeType.FANOUT, durable=True + ) + dead_letter_queue = await self._channel.declare_queue( + self._dlq_name, + durable=True, + arguments={"x-queue-type": "quorum"}, + ) + await dead_letter_queue.bind(dead_letter_exchange) + logger.info( + "broker_connected", + extra={"fields": {"queue": self._queue_name, "dlq": self._dlq_name}}, + ) + + async def publish_job(self, payload: dict, *, message_id: str) -> None: + """Публикация с publisher confirm; исключение означает отсутствие приёма.""" + if self._exchange is None: + raise RuntimeError("broker is not connected") + message = Message( + body=json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"), + content_type="application/json", + content_encoding="utf-8", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=message_id, + type=JOB_TYPE, + timestamp=datetime.now(timezone.utc), + headers={"schema_version": 1, "pipeline_version": payload["pipeline_version"]}, + ) + await self._exchange.publish( + message, routing_key=self._routing_key, mandatory=True + ) + + async def consume( + self, + callback: Callable[[AbstractIncomingMessage], Awaitable[None]], + *, + prefetch: int, + ) -> AbstractQueue: + if self._channel is None or self._queue is None: + raise RuntimeError("broker is not connected") + await self._channel.set_qos(prefetch_count=prefetch) + await self._queue.consume(callback) + return self._queue + + async def close(self) -> None: + if self._connection is not None: + await self._connection.close() + self._connection = None + self._channel = None + self._queue = None + + @property + def queue_name(self) -> str: + return self._queue_name diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..06b383c1 --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,393 @@ +"""Конфигурация процесса: значения приходят только из переменных окружения. + +Секреты читаются либо напрямую (NAME), либо из файла (NAME_FILE) — второй +вариант используется стендом Swarm и Docker secrets. Значения секретов +никогда не попадают в текст ошибки. Один и тот же разбор используют API, +worker, relay и операторский CLI. +""" +from __future__ import annotations + +import hashlib +import os +import re +import socket +import uuid +from dataclasses import dataclass +from pathlib import Path +from typing import Mapping + +DEFAULT_HOST = "0.0.0.0" +DEFAULT_PORT = 8080 +DEFAULT_ENV = "lab" +DEFAULT_LOG_LEVEL = "info" +DEFAULT_GRACEFUL_SHUTDOWN_SECONDS = 5 +DEFAULT_SHUTDOWN_DRAIN_SECONDS = 0.0 + +MAX_SHUTDOWN_BUDGET_SECONDS = 10 + +DEFAULT_SESSION_TTL_SECONDS = 3600 +MIN_SESSION_TTL_SECONDS = 1 +MAX_SESSION_TTL_SECONDS = 86400 + +DEFAULT_DB_POOL_SIZE = 5 +MAX_DB_POOL_SIZE = 32 +DEFAULT_DB_MAX_OVERFLOW = 5 +MAX_DB_MAX_OVERFLOW = 32 +DEFAULT_DB_POOL_TIMEOUT_SECONDS = 2.0 +MAX_DB_POOL_TIMEOUT_SECONDS = 30.0 +DEFAULT_DB_CONNECT_TIMEOUT_SECONDS = 1.5 +MAX_DB_CONNECT_TIMEOUT_SECONDS = 30.0 +DEFAULT_DB_COMMAND_TIMEOUT_SECONDS = 2.5 +MAX_DB_COMMAND_TIMEOUT_SECONDS = 60.0 +DEFAULT_DB_STATEMENT_TIMEOUT_MS = 2000 +MAX_DB_STATEMENT_TIMEOUT_MS = 60000 + +DEFAULT_CACHE_TIMEOUT_SECONDS = 1.0 +MAX_CACHE_TIMEOUT_SECONDS = 10.0 + +# Контракт лабораторной 5 фиксирует эти два предела; переменные окружения +# позволяют выбрать меньший предел для эксперимента, но не больший. +MAX_UPLOAD_BYTES_HARD_LIMIT = 10_485_760 +MAX_IMAGE_PIXELS_HARD_LIMIT = 40_000_000 +DEFAULT_MAX_UPLOAD_BYTES = MAX_UPLOAD_BYTES_HARD_LIMIT +DEFAULT_MAX_IMAGE_PIXELS = MAX_IMAGE_PIXELS_HARD_LIMIT + +DEFAULT_S3_REGION = "us-east-1" +DEFAULT_S3_BUCKET = "messenger-attachments" +DEFAULT_S3_KEY_PREFIX = "attachments/" +DEFAULT_S3_CONNECT_TIMEOUT_SECONDS = 1.5 +DEFAULT_S3_READ_TIMEOUT_SECONDS = 30.0 +DEFAULT_S3_MAX_ATTEMPTS = 2 +MAX_S3_READ_TIMEOUT_SECONDS = 120.0 +DEFAULT_ATTACHMENT_ORPHAN_GRACE_SECONDS = 3600 +MAX_ATTACHMENT_ORPHAN_GRACE_SECONDS = 7 * 24 * 3600 + +# Конвейер версионирован. Версия 1 — original+thumbnail (минимум лабораторной), +# версия 2 — original+thumbnail+crop+optimized (уровень 4). Текущая версия +# задаётся PIPELINE_VERSION и попадает в новое job, а не в глобальную переменную +# worker-а: уже поставленная работа продолжает выполняться своей версией. +DEFAULT_PIPELINE_VERSION = 2 +MAX_SUPPORTED_PIPELINE_VERSION = 2 + +DEFAULT_WORKER_CONCURRENCY = 2 +MAX_WORKER_CONCURRENCY = 16 +DEFAULT_WORKER_PREFETCH = 2 +MAX_WORKER_PREFETCH = 64 +DEFAULT_JOB_MAX_ATTEMPTS = 5 +MAX_JOB_MAX_ATTEMPTS = 20 +DEFAULT_JOB_TIMEOUT_SECONDS = 60.0 +MAX_JOB_TIMEOUT_SECONDS = 600.0 +DEFAULT_JOB_LEASE_SECONDS = 15.0 +MAX_JOB_LEASE_SECONDS = 600.0 +DEFAULT_JOB_BACKOFF_BASE_SECONDS = 1.0 +MAX_JOB_BACKOFF_SECONDS = 600.0 +DEFAULT_JOB_BACKOFF_MAX_SECONDS = 30.0 +DEFAULT_STAGING_RETENTION_SECONDS = 86400 +MAX_STAGING_RETENTION_SECONDS = 7 * 24 * 3600 +DEFAULT_RELAY_POLL_SECONDS = 0.25 +MAX_RELAY_POLL_SECONDS = 10.0 +DEFAULT_REAPER_INTERVAL_SECONDS = 1.0 +MAX_REAPER_INTERVAL_SECONDS = 60.0 +DEFAULT_WORKER_HEARTBEAT_SECONDS = 5.0 + +# Имена очередей и DLQ зафиксированы контрактом задания (jobs.schema.json). +DEFAULT_JOB_QUEUE = "attachment.jobs" +DEFAULT_JOB_ROUTING_KEY = "attachment.process.v1" +DEFAULT_DLQ_EXCHANGE = "attachment.dlx" +DEFAULT_DLQ_QUEUE = "attachment.dlq" + +ENVIRONMENTS = ("lab", "dev", "prod") +LOG_LEVELS = ("debug", "info", "warning", "error") + +SESSION_STORES = ("redis",) +DEFAULT_SESSION_STORE = "redis" + +DATABASE_URL_SCHEME = "postgresql+asyncpg://" +CACHE_URL_SCHEMES = ("redis://", "rediss://", "unix://") +BROKER_URL_SCHEMES = ("amqp://", "amqps://") + +LOOPBACK_HOSTS = ("127.0.0.1", "localhost", "::1") + +INSTANCE_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +BUCKET_PATTERN = re.compile(r"^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$") +KEY_PREFIX_PATTERN = re.compile(r"^[A-Za-z0-9._/-]{0,128}$") + +FAULT_MODES = ("none", "after_s3_put") + + +class ConfigError(RuntimeError): + """Некорректная конфигурация. Текст не раскрывает значений переменных.""" + + +def _choice(environ: Mapping[str, str], name: str, default: str, allowed: tuple[str, ...]) -> str: + value = environ.get(name, default).strip().lower() + if value not in allowed: + raise ConfigError(f"{name} must be one of: {', '.join(allowed)}") + return value + + +def _integer(environ: Mapping[str, str], name: str, default: int, minimum: int, maximum: int) -> int: + raw = environ.get(name, "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError: + raise ConfigError(f"{name} must be an integer") from None + if not minimum <= value <= maximum: + raise ConfigError(f"{name} must be within {minimum}..{maximum}") + return value + + +def _seconds(environ: Mapping[str, str], name: str, default: float, maximum: float) -> float: + raw = environ.get(name, "").strip() + if not raw: + return default + try: + value = float(raw) + except ValueError: + raise ConfigError(f"{name} must be a number of seconds") from None + if not 0.0 <= value <= maximum: + raise ConfigError(f"{name} must be within 0..{maximum}") + return value + + +def _boolean(environ: Mapping[str, str], name: str, default: bool) -> bool: + raw = environ.get(name, "").strip().lower() + if not raw: + return default + if raw in ("1", "true", "yes", "on"): + return True + if raw in ("0", "false", "no", "off"): + return False + raise ConfigError(f"{name} must be a boolean") + + +def _secret(environ: Mapping[str, str], name: str) -> str | None: + direct = environ.get(name, "").strip() + path = environ.get(f"{name}_FILE", "").strip() + if direct and path: + raise ConfigError(f"{name} and {name}_FILE must not be set together") + if direct: + return direct + if not path: + return None + try: + value = Path(path).read_text(encoding="utf-8").strip() + except OSError: + raise ConfigError(f"{name}_FILE must point to a readable file") from None + if not value: + raise ConfigError(f"{name}_FILE must not be empty") + return value + + +def _required_secret(environ: Mapping[str, str], name: str) -> str: + value = _secret(environ, name) + if value is None: + raise ConfigError(f"{name} or {name}_FILE is required") + return value + + +def _checked_url(value: str, *, name: str, schemes: tuple[str, ...]) -> str: + if not value.startswith(schemes): + raise ConfigError(f"{name} must start with one of: {', '.join(schemes)}") + return value + + +def database_url_from_env(environ: Mapping[str, str] | None = None) -> str: + source: Mapping[str, str] = os.environ if environ is None else environ + return _checked_url( + _required_secret(source, "DATABASE_URL"), + name="DATABASE_URL", + schemes=(DATABASE_URL_SCHEME,), + ) + + +def instance_id_from_env(environ: Mapping[str, str]) -> str: + raw = environ.get("INSTANCE_ID", "").strip() or socket.gethostname().strip() + if not raw: + raw = "api-" + uuid.uuid4().hex[:12] + if INSTANCE_ID_PATTERN.fullmatch(raw): + return raw + return "api-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] + + +def process_identity_from_env(environ: Mapping[str, str], prefix: str) -> str: + """Стабильный идентификатор процесса для логов, lease и аудита.""" + raw = environ.get("WORKER_ID", "").strip() or socket.gethostname().strip() + if not raw: + raw = prefix + "-" + uuid.uuid4().hex[:12] + if INSTANCE_ID_PATTERN.fullmatch(raw): + return raw + return prefix + "-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:16] + + +@dataclass(frozen=True) +class Config: + host: str + port: int + env: str + instance_id: str + log_level: str + graceful_shutdown_seconds: int + shutdown_drain_seconds: float + database_url: str + cache_url: str + session_store: str + session_ttl_seconds: int + db_pool_size: int + db_max_overflow: int + db_pool_timeout_seconds: float + db_connect_timeout_seconds: float + db_command_timeout_seconds: float + db_statement_timeout_ms: int + cache_timeout_seconds: float + cursor_secret: bytes | None + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_key_prefix: str + s3_access_key_id: str + s3_secret_access_key: str + s3_connect_timeout_seconds: float + s3_read_timeout_seconds: float + s3_max_attempts: int + max_upload_bytes: int + max_image_pixels: int + attachment_orphan_grace_seconds: int + attachment_fault_mode: str + broker_url: str | None + pipeline_version: int + worker_id: str + worker_concurrency: int + worker_prefetch: int + job_max_attempts: int + job_timeout_seconds: float + job_lease_seconds: float + job_backoff_base_seconds: float + job_backoff_max_seconds: float + staging_retention_seconds: int + allow_fault_injection: bool + relay_poll_seconds: float + reaper_interval_seconds: float + + def __post_init__(self) -> None: + if self.shutdown_budget_seconds > MAX_SHUTDOWN_BUDGET_SECONDS: + raise ConfigError( + "GRACEFUL_SHUTDOWN_SECONDS + SHUTDOWN_DRAIN_SECONDS must be " + f"at most {MAX_SHUTDOWN_BUDGET_SECONDS} seconds" + ) + if not self.s3_endpoint.startswith(("http://", "https://")): + raise ConfigError("S3_ENDPOINT must start with http:// or https://") + if not BUCKET_PATTERN.fullmatch(self.s3_bucket): + raise ConfigError("S3_BUCKET must be a valid S3 bucket name") + if not KEY_PREFIX_PATTERN.fullmatch(self.s3_key_prefix) or ".." in self.s3_key_prefix: + raise ConfigError("S3_KEY_PREFIX is not a safe object prefix") + if self.broker_url is not None and not self.broker_url.startswith(BROKER_URL_SCHEMES): + raise ConfigError("BROKER_URL must start with amqp:// or amqps://") + if self.job_backoff_max_seconds < self.job_backoff_base_seconds: + raise ConfigError("JOB_BACKOFF_MAX_SECONDS must be >= JOB_BACKOFF_BASE_SECONDS") + if self.job_timeout_seconds <= 0: + raise ConfigError("JOB_TIMEOUT_SECONDS must be > 0") + if self.job_lease_seconds <= 0: + raise ConfigError("JOB_LEASE_SECONDS must be > 0") + if self.pipeline_version < 1 or self.pipeline_version > MAX_SUPPORTED_PIPELINE_VERSION: + raise ConfigError( + f"PIPELINE_VERSION must be within 1..{MAX_SUPPORTED_PIPELINE_VERSION}" + ) + + @property + def shutdown_budget_seconds(self) -> float: + return self.graceful_shutdown_seconds + self.shutdown_drain_seconds + + @property + def binds_to_loopback(self) -> bool: + return self.host in LOOPBACK_HOSTS + + def require_broker(self) -> str: + """Worker/relay не могут работать без брокера; API — может (outbox).""" + if not self.broker_url: + raise ConfigError("BROKER_URL or BROKER_URL_FILE is required for worker/relay") + return self.broker_url + + @classmethod + def from_env(cls, environ: Mapping[str, str] | None = None) -> "Config": + source: Mapping[str, str] = os.environ if environ is None else environ + env = _choice(source, "APP_ENV", DEFAULT_ENV, ENVIRONMENTS) + host = source.get("APP_HOST", DEFAULT_HOST).strip() + if not host: + raise ConfigError("APP_HOST must not be empty") + default_log_level = "debug" if env == "dev" else DEFAULT_LOG_LEVEL + database_url = database_url_from_env(source) + cache_url = _checked_url( + _required_secret(source, "CACHE_URL"), + name="CACHE_URL", + schemes=CACHE_URL_SCHEMES, + ) + cursor_secret = _secret(source, "CURSOR_SECRET") + fault_mode = _choice(source, "ATTACHMENT_FAULT_MODE", "none", FAULT_MODES) + if env == "prod" and fault_mode != "none": + raise ConfigError("ATTACHMENT_FAULT_MODE must be none in prod") + allow_faults = _boolean(source, "ALLOW_FAULT_INJECTION", env != "prod") + if env == "prod" and allow_faults: + raise ConfigError("ALLOW_FAULT_INJECTION must be false in prod") + key_prefix = source.get("S3_KEY_PREFIX", DEFAULT_S3_KEY_PREFIX).strip() or DEFAULT_S3_KEY_PREFIX + broker_url = _secret(source, "BROKER_URL") + job_max_attempts = _integer( + source, "JOB_MAX_ATTEMPTS", DEFAULT_JOB_MAX_ATTEMPTS, 1, MAX_JOB_MAX_ATTEMPTS + ) + return cls( + host=host, + port=_integer(source, "APP_PORT", DEFAULT_PORT, 1, 65535), + env=env, + instance_id=instance_id_from_env(source), + log_level=_choice(source, "LOG_LEVEL", default_log_level, LOG_LEVELS), + graceful_shutdown_seconds=_integer( + source, "GRACEFUL_SHUTDOWN_SECONDS", DEFAULT_GRACEFUL_SHUTDOWN_SECONDS, 1, 10 + ), + shutdown_drain_seconds=_seconds( + source, "SHUTDOWN_DRAIN_SECONDS", DEFAULT_SHUTDOWN_DRAIN_SECONDS, 5.0 + ), + database_url=database_url, + cache_url=cache_url, + session_store=_choice(source, "SESSION_STORE", DEFAULT_SESSION_STORE, SESSION_STORES), + session_ttl_seconds=_integer( + source, "SESSION_TTL_SECONDS", DEFAULT_SESSION_TTL_SECONDS, + MIN_SESSION_TTL_SECONDS, MAX_SESSION_TTL_SECONDS, + ), + db_pool_size=_integer(source, "DB_POOL_SIZE", DEFAULT_DB_POOL_SIZE, 1, MAX_DB_POOL_SIZE), + db_max_overflow=_integer(source, "DB_MAX_OVERFLOW", DEFAULT_DB_MAX_OVERFLOW, 0, MAX_DB_MAX_OVERFLOW), + db_pool_timeout_seconds=_seconds(source, "DB_POOL_TIMEOUT_SECONDS", DEFAULT_DB_POOL_TIMEOUT_SECONDS, MAX_DB_POOL_TIMEOUT_SECONDS), + db_connect_timeout_seconds=_seconds(source, "DB_CONNECT_TIMEOUT_SECONDS", DEFAULT_DB_CONNECT_TIMEOUT_SECONDS, MAX_DB_CONNECT_TIMEOUT_SECONDS), + db_command_timeout_seconds=_seconds(source, "DB_COMMAND_TIMEOUT_SECONDS", DEFAULT_DB_COMMAND_TIMEOUT_SECONDS, MAX_DB_COMMAND_TIMEOUT_SECONDS), + db_statement_timeout_ms=_integer(source, "DB_STATEMENT_TIMEOUT_MS", DEFAULT_DB_STATEMENT_TIMEOUT_MS, 1, MAX_DB_STATEMENT_TIMEOUT_MS), + cache_timeout_seconds=_seconds(source, "CACHE_TIMEOUT_SECONDS", DEFAULT_CACHE_TIMEOUT_SECONDS, MAX_CACHE_TIMEOUT_SECONDS), + cursor_secret=None if cursor_secret is None else cursor_secret.encode("utf-8"), + s3_endpoint=_checked_url(_required_secret(source, "S3_ENDPOINT"), name="S3_ENDPOINT", schemes=("http://", "https://")), + s3_region=source.get("S3_REGION", DEFAULT_S3_REGION).strip() or DEFAULT_S3_REGION, + s3_bucket=source.get("S3_BUCKET", DEFAULT_S3_BUCKET).strip() or DEFAULT_S3_BUCKET, + s3_key_prefix=key_prefix, + s3_access_key_id=_required_secret(source, "S3_ACCESS_KEY_ID"), + s3_secret_access_key=_required_secret(source, "S3_SECRET_ACCESS_KEY"), + s3_connect_timeout_seconds=_seconds(source, "S3_CONNECT_TIMEOUT_SECONDS", DEFAULT_S3_CONNECT_TIMEOUT_SECONDS, MAX_S3_READ_TIMEOUT_SECONDS), + s3_read_timeout_seconds=_seconds(source, "S3_READ_TIMEOUT_SECONDS", DEFAULT_S3_READ_TIMEOUT_SECONDS, MAX_S3_READ_TIMEOUT_SECONDS), + s3_max_attempts=_integer(source, "S3_MAX_ATTEMPTS", DEFAULT_S3_MAX_ATTEMPTS, 1, 5), + max_upload_bytes=_integer(source, "MAX_UPLOAD_BYTES", DEFAULT_MAX_UPLOAD_BYTES, 1, MAX_UPLOAD_BYTES_HARD_LIMIT), + max_image_pixels=_integer(source, "MAX_IMAGE_PIXELS", DEFAULT_MAX_IMAGE_PIXELS, 1, MAX_IMAGE_PIXELS_HARD_LIMIT), + attachment_orphan_grace_seconds=_integer(source, "ATTACHMENT_ORPHAN_GRACE_SECONDS", DEFAULT_ATTACHMENT_ORPHAN_GRACE_SECONDS, 0, MAX_ATTACHMENT_ORPHAN_GRACE_SECONDS), + attachment_fault_mode=fault_mode, + broker_url=broker_url, + pipeline_version=_integer(source, "PIPELINE_VERSION", DEFAULT_PIPELINE_VERSION, 1, MAX_SUPPORTED_PIPELINE_VERSION), + worker_id=process_identity_from_env(source, "worker"), + worker_concurrency=_integer(source, "WORKER_CONCURRENCY", DEFAULT_WORKER_CONCURRENCY, 1, MAX_WORKER_CONCURRENCY), + worker_prefetch=_integer(source, "WORKER_PREFETCH", DEFAULT_WORKER_PREFETCH, 1, MAX_WORKER_PREFETCH), + job_max_attempts=job_max_attempts, + job_timeout_seconds=_seconds(source, "JOB_TIMEOUT_SECONDS", DEFAULT_JOB_TIMEOUT_SECONDS, MAX_JOB_TIMEOUT_SECONDS), + job_lease_seconds=_seconds(source, "JOB_LEASE_SECONDS", DEFAULT_JOB_LEASE_SECONDS, MAX_JOB_LEASE_SECONDS), + job_backoff_base_seconds=_seconds(source, "JOB_BACKOFF_BASE_SECONDS", DEFAULT_JOB_BACKOFF_BASE_SECONDS, MAX_JOB_BACKOFF_SECONDS), + job_backoff_max_seconds=_seconds(source, "JOB_BACKOFF_MAX_SECONDS", DEFAULT_JOB_BACKOFF_MAX_SECONDS, MAX_JOB_BACKOFF_SECONDS), + staging_retention_seconds=_integer(source, "STAGING_RETENTION_SECONDS", DEFAULT_STAGING_RETENTION_SECONDS, 0, MAX_STAGING_RETENTION_SECONDS), + allow_fault_injection=allow_faults, + relay_poll_seconds=_seconds(source, "RELAY_POLL_SECONDS", DEFAULT_RELAY_POLL_SECONDS, MAX_RELAY_POLL_SECONDS), + reaper_interval_seconds=_seconds(source, "REAPER_INTERVAL_SECONDS", DEFAULT_REAPER_INTERVAL_SECONDS, MAX_REAPER_INTERVAL_SECONDS), + ) diff --git a/src/messenger/context.py b/src/messenger/context.py new file mode 100644 index 00000000..78f326f4 --- /dev/null +++ b/src/messenger/context.py @@ -0,0 +1,23 @@ +"""Контекст одного HTTP-запроса: данные живут в ASGI-scope.""" +from __future__ import annotations + +from typing import Any, MutableMapping + +CONTEXT_KEY = "messenger.request_context" + +REQUEST_ID = "request_id" +USER_ID = "user_id" + + +def context_of(scope: MutableMapping[str, Any]) -> dict[str, Any]: + """Возвращает (при необходимости создавая) словарь контекста запроса.""" + context = scope.get(CONTEXT_KEY) + if context is None: + context = {} + scope[CONTEXT_KEY] = context + return context + + +def request_context(request: Any) -> dict[str, Any]: + """Контекст по объекту Request/WebSocket.""" + return context_of(request.scope) diff --git a/src/messenger/cursor.py b/src/messenger/cursor.py new file mode 100644 index 00000000..7f55bc15 --- /dev/null +++ b/src/messenger/cursor.py @@ -0,0 +1,119 @@ +"""Keyset-курсор: подписан ключом процесса и привязан к ресурсу (kind + scope).""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import json +import secrets +from dataclasses import dataclass +from uuid import UUID + +from .domain import Position, parse_rfc3339_z + +CURSOR_VERSION = 1 +SIGNATURE_BYTES = 16 +SEPARATOR = "." + +KIND_CHATS = "chats" +KIND_MESSAGES = "messages" + + +class InvalidCursor(ValueError): + """Курсор повреждён, подделан или выпущен для другого ресурса.""" + + +def _b64encode(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _b64decode(value: str) -> bytes: + return base64.urlsafe_b64decode(value + "=" * (-len(value) % 4)) + + +def _parse_timestamp(value: object) -> str: + """Проверяет, что метка времени — RFC 3339 UTC с суффиксом Z.""" + if not isinstance(value, str): + raise InvalidCursor("cursor timestamp is not a string") + try: + parse_rfc3339_z(value) + except ValueError: + raise InvalidCursor("cursor timestamp is not RFC 3339 UTC with Z suffix") from None + return value + + +@dataclass(frozen=True) +class Cursor: + """Позиция последнего отданного элемента выборки.""" + + kind: str + scope: str + created_at: str + item_id: str + + @property + def position(self) -> Position: + """Место в порядке (created_at, id), с которого продолжается выборка.""" + return (parse_rfc3339_z(self.created_at), UUID(self.item_id)) + + +class CursorCodec: + """Кодирует и проверяет курсоры ключом, известным только процессу.""" + + def __init__(self, key: bytes | None = None) -> None: + self._key = secrets.token_bytes(32) if key is None else key + + def encode(self, kind: str, scope: str, created_at: str, item_id: str) -> str: + payload = json.dumps( + { + "v": CURSOR_VERSION, + "kind": kind, + "scope": scope, + "created_at": created_at, + "id": item_id, + }, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + signature = hmac.new(self._key, payload, hashlib.sha256).digest()[:SIGNATURE_BYTES] + return _b64encode(payload) + SEPARATOR + _b64encode(signature) + + def decode(self, token: str, *, kind: str, scope: str) -> Cursor: + """Разбирает курсор или поднимает InvalidCursor — наружу это всегда 400.""" + if not isinstance(token, str) or not token: + raise InvalidCursor("cursor is empty") + encoded_payload, separator, encoded_signature = token.partition(SEPARATOR) + if not separator: + raise InvalidCursor("cursor has no signature part") + try: + payload = _b64decode(encoded_payload) + signature = _b64decode(encoded_signature) + except (binascii.Error, ValueError): + raise InvalidCursor("cursor is not valid base64url") from None + + expected = hmac.new(self._key, payload, hashlib.sha256).digest()[:SIGNATURE_BYTES] + if not hmac.compare_digest(signature, expected): + raise InvalidCursor("cursor signature mismatch") + + try: + data = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + raise InvalidCursor("cursor payload is not valid JSON") from None + if not isinstance(data, dict) or data.get("v") != CURSOR_VERSION: + raise InvalidCursor("unsupported cursor version") + + cursor = Cursor( + kind=str(data.get("kind", "")), + scope=str(data.get("scope", "")), + created_at=_parse_timestamp(data.get("created_at")), + item_id=str(data.get("id", "")), + ) + try: + UUID(cursor.item_id) + except ValueError: + raise InvalidCursor("cursor id is not a UUID") from None + if cursor.kind != kind or cursor.scope != scope: + raise InvalidCursor("cursor belongs to another resource") + return cursor diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..a03d7d57 --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,74 @@ +"""Асинхронный доступ к PostgreSQL: движок, ограниченный пул и таймауты.""" +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import AsyncIterator + +from sqlalchemy import text +from sqlalchemy.exc import ProgrammingError +from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine + +from .config import Config + +APPLICATION_NAME = "messenger" + +# Незакоммиченная транзакция не должна держать соединение пула бесконечно. +IDLE_IN_TRANSACTION_TIMEOUT_MS = 10000 + + +class Database: + """Владеет пулом соединений; сессия живёт ровно один HTTP-запрос.""" + + def __init__(self, config: Config) -> None: + self._engine: AsyncEngine = create_async_engine( + config.database_url, + # Пул ограничен: лишние запросы ждут освобождения соединения. + pool_size=config.db_pool_size, + max_overflow=config.db_max_overflow, + pool_timeout=config.db_pool_timeout_seconds, + pool_pre_ping=True, + pool_recycle=1800, + connect_args={ + # Границы на круг asyncpg, на установление соединения (включая + # разрешение имени) и на стороне сервера. + "timeout": config.db_connect_timeout_seconds, + "command_timeout": config.db_command_timeout_seconds, + "server_settings": { + "application_name": APPLICATION_NAME, + "statement_timeout": str(config.db_statement_timeout_ms), + "idle_in_transaction_session_timeout": str(IDLE_IN_TRANSACTION_TIMEOUT_MS), + }, + }, + ) + self._sessions = async_sessionmaker(self._engine, expire_on_commit=False) + + @asynccontextmanager + async def session(self) -> AsyncIterator[AsyncSession]: + """Сессия на один запрос: закрывается всегда, откат делает SQLAlchemy.""" + session = self._sessions() + try: + yield session + finally: + await session.close() + + async def ping(self) -> None: + """Проверка готовности: одно дешёвое обращение к серверу.""" + async with self.session() as session: + await session.execute(text("SELECT 1")) + + async def schema_revision(self) -> str | None: + """Текущая ревизия Alembic; None, если таблицы версий ещё нет. + + До завершения однократной миграции API обязан быть неготовым, иначе + Traefik отправит трафик на процесс, который не может писать в схему. + Отсутствие таблицы — это не отказ соединения, а нормальное состояние + холодного старта, поэтому исключение гасится здесь. + """ + async with self.session() as session: + try: + return await session.scalar(text("SELECT version_num FROM alembic_version")) + except ProgrammingError: + return None + + async def dispose(self) -> None: + await self._engine.dispose() \ No newline at end of file diff --git a/src/messenger/deps.py b/src/messenger/deps.py new file mode 100644 index 00000000..ccb101f4 --- /dev/null +++ b/src/messenger/deps.py @@ -0,0 +1,133 @@ +"""Зависимости FastAPI: PostgreSQL, хранилище сессий и текущий пользователь. + +Личность в этой работе определяется только сессией: заголовок X-User-Id из ЛР1 +не читается нигде и не влияет на ответ. +""" +from __future__ import annotations + +from collections.abc import AsyncIterator +from dataclasses import dataclass +from typing import Annotated + +from fastapi import Depends, Header, Request +from sqlalchemy.ext.asyncio import AsyncSession + +from .config import Config +from .context import USER_ID, request_context +from .cursor import CursorCodec +from .db import Database +from .domain import UserRecord +from .errors import BASIC_CHALLENGE, unauthorized +from .repository import Repository +from .security import parse_basic, parse_bearer +from .sessions import SessionStore +from .storage import S3Storage + +AUTHORIZATION_HEADER = "Authorization" + + +@dataclass(frozen=True) +class BasicCredentials: + """Разобранные учётные данные входа по Basic.""" + + username: str + password: str + + +def get_database(request: Request) -> Database: + """Пул соединений живёт в состоянии приложения: один на процесс.""" + return request.app.state.database + + +def get_sessions(request: Request) -> SessionStore: + """Хранилище сессий живёт в состоянии приложения.""" + return request.app.state.sessions + + +def get_cursors(request: Request) -> CursorCodec: + """Кодек курсоров привязан к процессу: ключ подписи не покидает память.""" + return request.app.state.cursors + + +def get_config(request: Request) -> Config: + """Разобранная конфигурация процесса живёт в состоянии приложения.""" + return request.app.state.config + + +def get_storage(request: Request) -> S3Storage: + """S3-клиент один на процесс; байты не проходят через PostgreSQL.""" + return request.app.state.storage + + +DatabaseDep = Annotated[Database, Depends(get_database)] +SessionsDep = Annotated[SessionStore, Depends(get_sessions)] +CursorsDep = Annotated[CursorCodec, Depends(get_cursors)] +ConfigDep = Annotated[Config, Depends(get_config)] +StorageDep = Annotated[S3Storage, Depends(get_storage)] + + +async def get_db_session(database: DatabaseDep) -> AsyncIterator[AsyncSession]: + """Сессия БД на один запрос: закрывается вместе с ответом.""" + async with database.session() as session: + yield session + + +DbSession = Annotated[AsyncSession, Depends(get_db_session)] + + +def get_repository(session: DbSession) -> Repository: + """Репозиторий поверх сессии запроса: всё SQL живёт в нём.""" + return Repository(session) + + +RepositoryDep = Annotated[Repository, Depends(get_repository)] + + +def basic_credentials( + authorization: Annotated[str | None, Header(alias=AUTHORIZATION_HEADER)] = None, +) -> BasicCredentials: + """Учётные данные входа: единственное место, где принимается схема Basic.""" + parsed = parse_basic(authorization) + if parsed is None: + raise unauthorized("Требуется Basic-аутентификация", challenge=BASIC_CHALLENGE) + return BasicCredentials(username=parsed[0], password=parsed[1]) + + +BasicCredentialsDep = Annotated[BasicCredentials, Depends(basic_credentials)] + + +def bearer_token( + authorization: Annotated[str | None, Header(alias=AUTHORIZATION_HEADER)] = None, +) -> str: + """Токен сессии из заголовка Authorization; иначе 401 с Bearer-челленджем.""" + token = parse_bearer(authorization) + if token is None: + raise unauthorized("Требуется Bearer-токен сессии") + return token + + +BearerToken = Annotated[str, Depends(bearer_token)] + + +async def get_current_user( + request: Request, + sessions: SessionsDep, + repository: RepositoryDep, + token: BearerToken, +) -> UserRecord: + """Владелец действующей сессии. + + Порядок важен: сначала сессия в Valkey, потом пользователь в PostgreSQL. + Неизвестный или истёкший токен — это 401, а не «анонимный доступ». + """ + state = await sessions.resolve(token) + if state is None: + raise unauthorized("Сессия не найдена или истекла") + user = await repository.get_user(state.user_id) + if user is None: + raise unauthorized("Владелец сессии больше не существует") + request_context(request)[USER_ID] = str(user.id) + return user + + +CurrentUser = Annotated[UserRecord, Depends(get_current_user)] diff --git a/src/messenger/domain.py b/src/messenger/domain.py new file mode 100644 index 00000000..056320dc --- /dev/null +++ b/src/messenger/domain.py @@ -0,0 +1,137 @@ +"""Сущности предметной области, порядок сортировки и формат метки времени.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Protocol, Sequence +from uuid import UUID + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def rfc3339_z(moment: datetime) -> str: + """RFC 3339 в UTC с суффиксом Z и микросекундной точностью.""" + return moment.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def parse_rfc3339_z(value: str) -> datetime: + """Разбирает метку в формате `rfc3339_z`; иначе ValueError.""" + if not value.endswith("Z"): + raise ValueError("timestamp is not UTC with Z suffix") + return datetime.fromisoformat(value[:-1] + "+00:00") + + +class Sortable(Protocol): + """Общая часть записей: пара, по которой строится порядок и keyset-курсор.""" + + created_at: datetime + id: UUID + + +Position = tuple[datetime, UUID] + +ATTACHMENT_STATUS_QUEUED = "queued" +ATTACHMENT_STATUS_PROCESSING = "processing" +ATTACHMENT_STATUS_READY = "ready" +ATTACHMENT_STATUS_FAILED = "failed" +# Историческое имя lab4: строка до публикации результата. В lab5 это queued. +ATTACHMENT_STATUS_PENDING = ATTACHMENT_STATUS_QUEUED + + +class UsernameTaken(Exception): + """username уже занят другим пользователем.""" + + +class UnknownMembers(Exception): + """В member_ids есть идентификаторы несуществующих пользователей.""" + + def __init__(self, member_ids: Sequence[UUID]) -> None: + super().__init__("unknown members") + self.member_ids = tuple(member_ids) + + +class IdempotencyConflict(Exception): + """Тот же ключ идемпотентности использован с другим телом запроса.""" + + +class AttachmentUnknown(Exception): + """Вложение не существует, чужое или принадлежит другому чату.""" + + +class AttachmentNotReady(Exception): + """Вложение существует, но ещё не готово (pending/failed).""" + + +@dataclass(frozen=True) +class UserRecord: + id: UUID + username: str + display_name: str + created_at: datetime + + +@dataclass(frozen=True) +class ChatRecord: + id: UUID + title: str + member_ids: tuple[UUID, ...] + created_at: datetime + + def has_member(self, user_id: UUID) -> bool: + """Состав участников задаётся при создании и не меняется.""" + return user_id in self.member_ids + + +@dataclass(frozen=True) +class MessageRecord: + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + attachment_ids: tuple[UUID, ...] = field(default_factory=tuple) + + +@dataclass(frozen=True) +class VariantRecord: + """Опубликованный вариант изображения: метаданные и ключ объекта S3.""" + + attachment_id: UUID + pipeline_version: int + name: str + content_type: str + object_key: str + size_bytes: int + sha256: str + width: int + height: int + created_at: datetime + + +@dataclass(frozen=True) +class AttachmentRecord: + """Метаданные вложения. Байты в этой записи отсутствуют принципиально. + + object_key — конечный ключ оригинала; staging_key — приватный staging-ключ. + pipeline_version — версия конвейера, результат которой опубликован (0 — нет). + variants — только варианты этой версии, в порядке thumbnail/crop/optimized. + """ + + id: UUID + chat_id: UUID + owner_id: UUID + object_key: str + staging_key: str + pipeline_version: int + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + error_code: str | None + ready_at: datetime | None + created_at: datetime + updated_at: datetime + variants: tuple[VariantRecord, ...] = field(default_factory=tuple) diff --git a/src/messenger/errorhandlers.py b/src/messenger/errorhandlers.py new file mode 100644 index 00000000..61e923ab --- /dev/null +++ b/src/messenger/errorhandlers.py @@ -0,0 +1,184 @@ +"""Обработчики ошибок: единый конверт контракта и заголовки к нему.""" +from __future__ import annotations + +import logging +import uuid +from typing import Mapping + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from redis.exceptions import RedisError +from sqlalchemy.exc import DataError, DisconnectionError, InterfaceError, OperationalError +from sqlalchemy.exc import TimeoutError as PoolTimeoutError +from starlette.exceptions import HTTPException as StarletteHTTPException + +from .context import REQUEST_ID, request_context +from .errors import INTERNAL_ERROR, INVALID_REQUEST, NOT_FOUND, UNAVAILABLE, ApiError, code_for_status +from .logging_setup import log +from .storage import StorageObjectMissing, StorageUnavailable + +FALLBACK_MESSAGES: Mapping[int, str] = { + 404: "Маршрут не найден", + 405: "Метод не поддерживается для этого маршрута", +} + +# Отказ критичной зависимости (PostgreSQL или Valkey): наружу 503 без деталей, +# подробности остаются в логе. Для клиента «сессии недоступны» и «база недоступна» +# выглядят одинаково — и это правильно, различать их он не должен. +# +# Здесь только отказы соединения и исчерпание пула. Более широкий SQLAlchemyError +# сюда не годится: он включает DataError, и тогда значение, которое PostgreSQL +# не может принять, выглядело бы как недоступность базы. +DEPENDENCY_ERRORS = ( + OperationalError, + InterfaceError, + DisconnectionError, + PoolTimeoutError, + RedisError, + OSError, +) +DEPENDENCY_MESSAGE = "Сервис временно недоступен: критичная зависимость не отвечает" +DATA_ERROR_MESSAGE = "Значение поля не может быть сохранено" + + +def request_id_of(request: Request) -> str: + """Идентификатор запроса; запасной вариант — если middleware не подключён.""" + context = request_context(request) + identifier = context.get(REQUEST_ID) + if not identifier: + identifier = uuid.uuid4().hex + context[REQUEST_ID] = identifier + return identifier + + +def error_response( + request: Request, + status: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, +) -> JSONResponse: + """Собирает конверт ошибки контракта; request_id совпадает с заголовком.""" + identifier = request_id_of(request) + response_headers = {"X-Request-Id": identifier} + response_headers.update(headers or {}) + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": identifier}}, + headers=response_headers, + ) + + +def describe_validation_error(exc: RequestValidationError) -> str: + """Краткое описание без значений полей: наружу уходит только расположение ошибки. + + Значения сюда не попадают намеренно: среди них может быть пароль. + """ + errors = exc.errors() + if not errors: + return "Тело запроса не соответствует контракту" + first = errors[0] + if str(first.get("type", "")) == "json_invalid": + return "Тело запроса не является корректным JSON" + location = ".".join( + str(part) + for part in first.get("loc", ()) + if part not in ("body", "query", "path", "header") + ) + return f"Некорректный запрос: {location or 'body'}" + + +def register_error_handlers(app: FastAPI, logger: logging.Logger) -> None: + """Ставит обработчики так, чтобы наружу не ушёл ответ без конверта.""" + + @app.exception_handler(ApiError) + async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse: + return error_response(request, exc.status, exc.code, exc.message, exc.headers) + + @app.exception_handler(RequestValidationError) + async def handle_validation_error( + request: Request, exc: RequestValidationError + ) -> JSONResponse: + return error_response(request, 400, INVALID_REQUEST, describe_validation_error(exc)) + + @app.exception_handler(StarletteHTTPException) + async def handle_http_exception( + request: Request, exc: StarletteHTTPException + ) -> JSONResponse: + status = exc.status_code + return error_response( + request, + status, + code_for_status(status), + FALLBACK_MESSAGES.get(status, "Запрос не может быть обработан"), + getattr(exc, "headers", None), + ) + + async def handle_dependency_failure(request: Request, exc: Exception) -> JSONResponse: + log( + logger, + logging.ERROR, + "dependency_unavailable", + exc_info=exc, + request_id=request_id_of(request), + error=type(exc).__name__, + ) + return error_response(request, 503, UNAVAILABLE, DEPENDENCY_MESSAGE) + + @app.exception_handler(StorageObjectMissing) + async def handle_storage_missing(request: Request, exc: StorageObjectMissing) -> JSONResponse: + """Объект, на который ссылаются метаданные, отсутствует — 404.""" + log( + logger, + logging.WARNING, + "storage_object_missing", + request_id=request_id_of(request), + error=type(exc).__name__, + ) + return error_response(request, 404, NOT_FOUND, "Объект вложения отсутствует") + + @app.exception_handler(StorageUnavailable) + async def handle_storage_unavailable( + request: Request, exc: StorageUnavailable + ) -> JSONResponse: + """S3 недоступен: 503, детали остаются в логе.""" + log( + logger, + logging.ERROR, + "storage_unavailable", + request_id=request_id_of(request), + error=type(exc).__name__, + ) + return error_response(request, 503, UNAVAILABLE, "Хранилище вложений недоступно") + + @app.exception_handler(DataError) + async def handle_data_error(request: Request, exc: DataError) -> JSONResponse: + """Данные клиента, которые СУБД не в состоянии сохранить, — это 400. + + На границе такие значения уже отсекаются; обработчик оставлен страховкой, + чтобы ошибка данных никогда не выдавалась за отказ зависимости. + """ + log( + logger, + logging.WARNING, + "invalid_stored_value", + request_id=request_id_of(request), + error=type(exc).__name__, + ) + return error_response(request, 400, INVALID_REQUEST, DATA_ERROR_MESSAGE) + + for error_type in DEPENDENCY_ERRORS: + app.add_exception_handler(error_type, handle_dependency_failure) + + @app.exception_handler(Exception) + async def handle_unexpected(request: Request, exc: Exception) -> JSONResponse: + log( + logger, + logging.ERROR, + "unhandled_exception", + exc_info=exc, + request_id=request_id_of(request), + error=type(exc).__name__, + ) + return error_response(request, 500, INTERNAL_ERROR, "Внутренняя ошибка сервера") diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..6541a70c --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,88 @@ +"""Ошибки API и единый конверт ответа контракта.""" +from __future__ import annotations + +from typing import Mapping + +INVALID_REQUEST = "invalid_request" +UNAUTHORIZED = "unauthorized" +NOT_FOUND = "not_found" +CONFLICT = "conflict" +PAYLOAD_TOO_LARGE = "payload_too_large" +UNSUPPORTED_MEDIA_TYPE = "unsupported_media_type" +RATE_LIMITED = "rate_limited" +UNAVAILABLE = "unavailable" +INTERNAL_ERROR = "internal_error" + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + +DEFAULT_CODE_BY_STATUS: Mapping[int, str] = { + 400: INVALID_REQUEST, + 401: UNAUTHORIZED, + 403: UNAUTHORIZED, + 404: NOT_FOUND, + 405: INVALID_REQUEST, + 409: CONFLICT, + 413: PAYLOAD_TOO_LARGE, + 415: UNSUPPORTED_MEDIA_TYPE, + 429: RATE_LIMITED, + 500: INTERNAL_ERROR, + 503: UNAVAILABLE, +} + + +class ApiError(Exception): + """Ошибка, которую можно безопасно показать клиенту в конверте контракта.""" + + def __init__( + self, + status: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, + ) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = dict(headers or {}) + + +def code_for_status(status: int) -> str: + """Подбирает код конверта для статуса, пришедшего извне контракта.""" + if status in DEFAULT_CODE_BY_STATUS: + return DEFAULT_CODE_BY_STATUS[status] + return INTERNAL_ERROR if status >= 500 else INVALID_REQUEST + + +def invalid_request(message: str) -> ApiError: + return ApiError(400, INVALID_REQUEST, message) + + +def unauthorized(message: str, challenge: str = BEARER_CHALLENGE) -> ApiError: + """401 с challenge схемы: Basic — только на создании сессии, иначе Bearer.""" + return ApiError(401, UNAUTHORIZED, message, headers={"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + """Чужой объект неотличим от отсутствующего: 403 не используется.""" + return ApiError(404, NOT_FOUND, message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, CONFLICT, message) + + +def unavailable(message: str) -> ApiError: + """503: зависимость недоступна, отказ ограничен и не раскрывает внутренностей.""" + return ApiError(503, UNAVAILABLE, message) + + +def payload_too_large(message: str) -> ApiError: + """413: тело или декодированное изображение превышает контрактный предел.""" + return ApiError(413, PAYLOAD_TOO_LARGE, message) + + +def unsupported_media_type(message: str) -> ApiError: + """415: тип не PNG/JPEG или содержимое не совпало с объявленным типом.""" + return ApiError(415, UNSUPPORTED_MEDIA_TYPE, message) diff --git a/src/messenger/healthcheck.py b/src/messenger/healthcheck.py new file mode 100644 index 00000000..d0585bd0 --- /dev/null +++ b/src/messenger/healthcheck.py @@ -0,0 +1,26 @@ +"""Проверка живости для Docker HEALTHCHECK: python -m messenger.healthcheck.""" +from __future__ import annotations + +import os +import urllib.error +import urllib.request + +DEFAULT_PORT = "8080" +TIMEOUT_SECONDS = 2 + + +def liveness_url() -> str: + port = os.environ.get("APP_PORT", DEFAULT_PORT).strip() or DEFAULT_PORT + return "http://127.0.0.1:%s/health/live" % port + + +def main() -> int: + try: + with urllib.request.urlopen(liveness_url(), timeout=TIMEOUT_SECONDS) as response: + return 0 if response.status == 200 else 1 + except (urllib.error.URLError, OSError, ValueError): + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/messenger/idempotency.py b/src/messenger/idempotency.py new file mode 100644 index 00000000..f56cd9c4 --- /dev/null +++ b/src/messenger/idempotency.py @@ -0,0 +1,42 @@ +"""Идемпотентность POST сообщения: проверка ключа и канонический хэш тела.""" +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Sequence +from uuid import UUID + +from .errors import invalid_request + +# Контракт: ASCII [A-Za-z0-9._:-]{1,128}. Полное совпадение, а не поиск. +IDEMPOTENCY_KEY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +def require_idempotency_key(value: str | None) -> str: + """Возвращает валидный ключ или 400; отсутствие ключа — тоже 400.""" + if value is None or not IDEMPOTENCY_KEY_PATTERN.fullmatch(value): + raise invalid_request( + "Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}" + ) + return value + + +def message_request_hash(text: str, attachment_ids: Sequence[UUID] = ()) -> str: + """Хэш семантически нормализованного тела запроса. + + Порядок ключей JSON и отсутствие необязательных полей не должны менять + результат. Список вложений канонизируется как множество: порядок ссылок в + сообщении не является частью семантики запроса (порядок хранения при этом + сохраняется отдельно, в message_attachments.position). + """ + canonical = json.dumps( + { + "attachment_ids": sorted(str(item) for item in attachment_ids), + "text": text, + }, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() diff --git a/src/messenger/images.py b/src/messenger/images.py new file mode 100644 index 00000000..fc7fb2a5 --- /dev/null +++ b/src/messenger/images.py @@ -0,0 +1,88 @@ +"""Проверка изображения по содержимому, без доверия к Content-Type клиента.""" +from __future__ import annotations + +import struct +import warnings +from dataclasses import dataclass + +from PIL import Image, UnidentifiedImageError + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SIGNATURE = b"\xff\xd8\xff" + + +class UnsupportedImage(ValueError): + """Не PNG/JPEG или файл повреждён.""" + + +class ImageTooLarge(ValueError): + """Пиксельный предел контракта превышен.""" + + +@dataclass(frozen=True) +class ImageInfo: + content_type: str + width: int + height: int + + +def detect_content_type(head: bytes) -> str | None: + """MIME по первым байтам, а не по заголовку запроса.""" + if head.startswith(PNG_SIGNATURE): + return "image/png" + if head.startswith(JPEG_SIGNATURE): + return "image/jpeg" + return None + + +def inspect_image(path: str, *, expected_content_type: str, max_pixels: int) -> ImageInfo: + """Проверяет сигнатуру, формат, целостность и число пикселей. + + Размеры читаются из заголовка и проверяются до полной распаковки; для + отсечения бомб Pillow получает те же пределы, а предупреждение о бомбе + превращается в `ImageTooLarge`. Полное декодирование в этой лабораторной + не требуется: вариантов изображения нет. + """ + with open(path, "rb") as probe: + head = probe.read(len(PNG_SIGNATURE)) + detected = detect_content_type(head) + if detected is None: + raise UnsupportedImage("Содержимое не является PNG/JPEG") + if detected != expected_content_type: + raise UnsupportedImage("Content-Type не совпадает с содержимым") + + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(path) as image: + width, height = image.size + image_format = image.format + if width <= 0 or height <= 0: + raise UnsupportedImage("Некорректные размеры изображения") + if width * height > max_pixels: + raise ImageTooLarge("Изображение превышает предел пикселей") + if image_format not in ("PNG", "JPEG"): + raise UnsupportedImage("Формат изображения не PNG/JPEG") + image.verify() + except (UnsupportedImage, ImageTooLarge): + raise + except Image.DecompressionBombWarning as exc: + raise ImageTooLarge("Изображение превышает предел пикселей") from exc + except Image.DecompressionBombError as exc: + raise ImageTooLarge("Изображение превышает предел пикселей") from exc + except ( + UnidentifiedImageError, + OSError, + SyntaxError, + ValueError, + struct.error, + IndexError, + OverflowError, + ) as exc: + raise UnsupportedImage("Изображение повреждено") from exc + + if detected == "image/png" and image_format != "PNG": + raise UnsupportedImage("Расширение и формат PNG не совпали") + if detected == "image/jpeg" and image_format != "JPEG": + raise UnsupportedImage("Расширение и формат JPEG не совпали") + return ImageInfo(content_type=detected, width=width, height=height) diff --git a/src/messenger/jobs.py b/src/messenger/jobs.py new file mode 100644 index 00000000..4296806a --- /dev/null +++ b/src/messenger/jobs.py @@ -0,0 +1,1144 @@ +"""Задания обработки, lease, outbox, варианты и операторский replay. + +Модуль содержит SQL/транзакционные границы асинхронного pipeline. API использует +`insert_upload_work` в своей транзакции, а worker/relay/admin — `JobRepository` +с отдельной сессией на операцию. Метаданные PostgreSQL — источник истины; +брокер лишь доставляет ссылки на job. +""" +from __future__ import annotations + +import random +from dataclasses import dataclass +from datetime import datetime +from typing import Sequence +from uuid import UUID, uuid4 + +from sqlalchemy import func, insert, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from .config import Config +from .db import Database +from .domain import AttachmentRecord, VariantRecord, utc_now +from .pipeline import GeneratedVariant +from .schema import ( + ATTACHMENTS, + JOBS, + JOB_EVENTS, + JOB_REPLAYS, + OUTBOX, + VARIANTS, + FAULT_FLAGS, +) + +TERMINAL_JOB_STATES = ("ready", "failed", "superseded") +JOB_TYPE = "attachment.process.v1" +FAULT_KINDS = ( + "crash_before_objects", + "crash_after_objects", + "transient_once", + "permanent_once", + "stuck_before_commit", + "transient_always", +) + + +class ReplayRejected(RuntimeError): + """Явный replay невозможен: задача не failed или результат уже новее.""" + + +@dataclass(frozen=True) +class JobRecord: + id: UUID + attachment_id: UUID + pipeline_version: int + type: str + state: str + attempts: int + max_attempts: int + lease_token: UUID | None + worker_id: str | None + lease_expires_at: datetime | None + next_attempt_at: datetime | None + last_error_code: str | None + replay_of: UUID | None + created_at: datetime + updated_at: datetime + finished_at: datetime | None + + +@dataclass(frozen=True) +class Claim: + granted: bool + reason: str + job: JobRecord | None = None + attachment: AttachmentRecord | None = None + lease_token: UUID | None = None + + +@dataclass(frozen=True) +class OutboxItem: + outbox_id: int + job: JobRecord + attachment: AttachmentRecord + + +def _interval(seconds: float): + """Интервал PostgreSQL от текущего времени; значение — число из конфигурации.""" + return text(f"INTERVAL '{float(seconds)} seconds'") + + +def _job_record(row) -> JobRecord: + return JobRecord( + id=row.id, + attachment_id=row.attachment_id, + pipeline_version=row.pipeline_version, + type=row.type, + state=row.state, + attempts=row.attempts, + max_attempts=row.max_attempts, + lease_token=row.lease_token, + worker_id=row.worker_id, + lease_expires_at=row.lease_expires_at, + next_attempt_at=row.next_attempt_at, + last_error_code=row.last_error_code, + replay_of=row.replay_of, + created_at=row.created_at, + updated_at=row.updated_at, + finished_at=row.finished_at, + ) + + +def _variant_record(row) -> VariantRecord: + return VariantRecord( + attachment_id=row.attachment_id, + pipeline_version=row.pipeline_version, + name=row.name, + content_type=row.content_type, + object_key=row.object_key, + size_bytes=row.size_bytes, + sha256=row.sha256, + width=row.width, + height=row.height, + created_at=row.created_at, + ) + + +def _attachment_record(row, variants: Sequence[VariantRecord] = ()) -> AttachmentRecord: + return AttachmentRecord( + id=row.id, + chat_id=row.chat_id, + owner_id=row.owner_id, + object_key=row.object_key, + staging_key=row.staging_key, + pipeline_version=row.pipeline_version, + filename=row.filename, + content_type=row.content_type, + size_bytes=row.size_bytes, + sha256=row.sha256, + status=row.status, + error_code=row.error_code, + ready_at=row.ready_at, + created_at=row.created_at, + updated_at=row.updated_at, + variants=tuple(variants), + ) + + +async def _variants_of( + session: AsyncSession, attachment_id: UUID, pipeline_version: int +) -> list[VariantRecord]: + if pipeline_version <= 0: + return [] + rows = ( + await session.execute( + select(VARIANTS) + .where( + VARIANTS.c.attachment_id == attachment_id, + VARIANTS.c.pipeline_version == pipeline_version, + ) + .order_by(VARIANTS.c.name) + ) + ).all() + return [_variant_record(row) for row in rows] + + +async def insert_upload_work( + session: AsyncSession, + *, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + staging_key: str, + object_key: str, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + pipeline_version: int, + max_attempts: int, +) -> None: + """Одна транзакция: queued-вложение + job + outbox-строка. + + Вызывается из upload-запроса до COMMIT. После успешного COMMIT API имеет + право ответить 202: и байты (staging), и намерение обработки устойчивы. + """ + await session.execute( + ATTACHMENTS.insert().values( + id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + object_key=object_key, + staging_key=staging_key, + pipeline_version=0, + filename=filename, + content_type=content_type, + size_bytes=size_bytes, + sha256=sha256, + status="queued", + ) + ) + job_id = uuid4() + await session.execute( + JOBS.insert().values( + id=job_id, + attachment_id=attachment_id, + pipeline_version=pipeline_version, + type=JOB_TYPE, + state="queued", + attempts=0, + max_attempts=max_attempts, + ) + ) + await session.execute(OUTBOX.insert().values(job_id=job_id, available_at=func.now())) + await session.execute( + JOB_EVENTS.insert().values( + job_id=job_id, + from_state=None, + to_state="queued", + attempt=0, + detail="upload_accepted", + ) + ) + + +class JobRepository: + """Транзакционные операции над jobs/outbox/variants.""" + + def __init__(self, database: Database, config: Config) -> None: + self._db = database + self._config = config + + # --- чтение ----------------------------------------------------------- + + async def get_job(self, job_id: UUID) -> JobRecord | None: + async with self._db.session() as session: + row = ( + await session.execute(select(JOBS).where(JOBS.c.id == job_id).limit(1)) + ).first() + return None if row is None else _job_record(row) + + async def get_attachment(self, attachment_id: UUID) -> AttachmentRecord | None: + async with self._db.session() as session: + row = ( + await session.execute( + select(ATTACHMENTS).where(ATTACHMENTS.c.id == attachment_id).limit(1) + ) + ).first() + if row is None: + return None + return _attachment_record( + row, await _variants_of(session, attachment_id, row.pipeline_version) + ) + + async def has_other_active_jobs(self, attachment_id: UUID, exclude_job_id: UUID) -> bool: + """Есть ли ещё queued/processing job того же вложения (другая версия/attempt).""" + async with self._db.session() as session: + found = await session.scalar( + select(JOBS.c.id) + .where( + JOBS.c.attachment_id == attachment_id, + JOBS.c.id != exclude_job_id, + JOBS.c.state.in_(("queued", "processing")), + ) + .limit(1) + ) + return found is not None + + async def jobs_for_attachment(self, attachment_id: UUID) -> list[JobRecord]: + async with self._db.session() as session: + rows = ( + await session.execute( + select(JOBS) + .where(JOBS.c.attachment_id == attachment_id) + .order_by(JOBS.c.created_at, JOBS.c.id) + ) + ).all() + return [_job_record(row) for row in rows] + + # --- worker ----------------------------------------------------------- + + async def claim( + self, job_id: UUID, *, worker_id: str, lease_seconds: float + ) -> Claim: + """Захватывает job под lease. Повторная доставка не создаёт второй эффект.""" + async with self._db.session() as session: + async with session.begin(): + row = ( + await session.execute( + select(JOBS).where(JOBS.c.id == job_id).with_for_update() + ) + ).first() + if row is None: + return Claim(False, "missing") + job = _job_record(row) + db_now = await session.scalar(select(func.now())) + if job.state in TERMINAL_JOB_STATES: + return Claim(False, job.state, job=job) + attachment_row = ( + await session.execute( + select(ATTACHMENTS) + .where(ATTACHMENTS.c.id == job.attachment_id) + .with_for_update() + ) + ).first() + if attachment_row is None: + return Claim(False, "missing_attachment", job=job) + attachment = _attachment_record(attachment_row) + if attachment.pipeline_version > job.pipeline_version: + await self._supersede(session, job, "newer_pipeline_ready") + return Claim(False, "superseded", job=job) + if ( + job.state == "processing" + and job.lease_expires_at is not None + and job.lease_expires_at >= db_now + ): + return Claim(False, "busy", job=job) + if job.attempts >= job.max_attempts: + await self._fail_locked( + session, + job, + attachment, + error_code="retry_exhausted", + permanent=True, + detail="attempts_exhausted_on_claim", + ) + return Claim(False, "exhausted", job=job) + attempt = job.attempts + 1 + token = uuid4() + await session.execute( + update(JOBS) + .where(JOBS.c.id == job.id) + .values( + state="processing", + attempts=attempt, + lease_token=token, + worker_id=worker_id, + lease_expires_at=func.now() + _interval(lease_seconds), + next_attempt_at=None, + updated_at=func.now(), + ) + ) + if ( + attachment.pipeline_version <= job.pipeline_version + and attachment.status in ("queued", "processing") + ): + await session.execute( + update(ATTACHMENTS) + .where(ATTACHMENTS.c.id == attachment.id) + .values(status="processing", error_code=None, updated_at=func.now()) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=job.id, + from_state=job.state, + to_state="processing", + attempt=attempt, + worker_id=worker_id, + detail="lease_acquired", + ) + ) + claimed = _job_record( + ( + await session.execute( + select(JOBS).where(JOBS.c.id == job.id) + ) + ).first() + ) + return Claim( + True, + "claimed", + job=claimed, + attachment=attachment, + lease_token=token, + ) + + async def extend_lease(self, job_id: UUID, lease_token: UUID, lease_seconds: float) -> bool: + async with self._db.session() as session: + result = await session.execute( + update(JOBS) + .where( + JOBS.c.id == job_id, + JOBS.c.lease_token == lease_token, + JOBS.c.state == "processing", + ) + .values(lease_expires_at=func.now() + _interval(lease_seconds)) + ) + await session.commit() + return bool(result.rowcount) + + async def complete( + self, + *, + job_id: UUID, + lease_token: UUID, + worker_id: str, + original_key: str, + variants: Sequence[GeneratedVariant], + variant_keys: dict[str, str], + ) -> str: + """Атомарно публикует результат. Гард версии не даёт старой job перезаписать новую.""" + async with self._db.session() as session: + async with session.begin(): + row = ( + await session.execute( + select(JOBS).where(JOBS.c.id == job_id).with_for_update() + ) + ).first() + if row is None: + return "missing" + job = _job_record(row) + if job.state != "processing" or job.lease_token != lease_token: + return "lease_lost" + attachment_row = ( + await session.execute( + select(ATTACHMENTS) + .where(ATTACHMENTS.c.id == job.attachment_id) + .with_for_update() + ) + ).first() + if attachment_row is None: + return "missing_attachment" + attachment = _attachment_record(attachment_row) + if attachment.pipeline_version > job.pipeline_version: + await self._supersede(session, job, "newer_pipeline_ready") + return "superseded" + for variant in variants: + object_key = variant_keys.get(variant.name) + if not object_key: + raise ValueError("variant object key is required") + await session.execute( + pg_insert(VARIANTS) + .values( + attachment_id=job.attachment_id, + pipeline_version=job.pipeline_version, + name=variant.name, + content_type=variant.content_type, + object_key=object_key, + size_bytes=variant.size_bytes, + sha256=variant.sha256, + width=variant.width, + height=variant.height, + ) + .on_conflict_do_update( + index_elements=[ + VARIANTS.c.attachment_id, + VARIANTS.c.pipeline_version, + VARIANTS.c.name, + ], + set_={ + "content_type": variant.content_type, + "object_key": object_key, + "size_bytes": variant.size_bytes, + "sha256": variant.sha256, + "width": variant.width, + "height": variant.height, + }, + ) + ) + await session.execute( + update(ATTACHMENTS) + .where(ATTACHMENTS.c.id == job.attachment_id) + .values( + status="ready", + pipeline_version=job.pipeline_version, + object_key=original_key, + ready_at=func.now(), + error_code=None, + updated_at=func.now(), + ) + ) + await session.execute( + update(JOBS) + .where(JOBS.c.id == job.id) + .values( + state="ready", + lease_token=None, + lease_expires_at=None, + next_attempt_at=None, + updated_at=func.now(), + finished_at=func.now(), + ) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=job.id, + from_state="processing", + to_state="ready", + attempt=job.attempts, + worker_id=worker_id, + detail="effect_committed", + ) + ) + return "ready" + + async def fail( + self, + *, + job_id: UUID, + lease_token: UUID, + worker_id: str, + error_code: str, + permanent: bool, + ) -> str: + async with self._db.session() as session: + async with session.begin(): + row = ( + await session.execute( + select(JOBS).where(JOBS.c.id == job_id).with_for_update() + ) + ).first() + if row is None: + return "missing" + job = _job_record(row) + if job.state != "processing" or job.lease_token != lease_token: + return "lease_lost" + attachment_row = ( + await session.execute( + select(ATTACHMENTS) + .where(ATTACHMENTS.c.id == job.attachment_id) + .with_for_update() + ) + ).first() + if attachment_row is None: + return "missing_attachment" + attachment = _attachment_record(attachment_row) + if attachment.pipeline_version > job.pipeline_version: + await self._supersede(session, job, "newer_pipeline_ready") + return "superseded" + return await self._fail_locked( + session, + job, + attachment, + error_code=error_code, + permanent=permanent, + worker_id=worker_id, + detail="worker_reported_failure", + ) + + async def _fail_locked( + self, + session: AsyncSession, + job: JobRecord, + attachment: AttachmentRecord, + *, + error_code: str, + permanent: bool, + worker_id: str | None = None, + detail: str, + ) -> str: + terminal = permanent or job.attempts >= job.max_attempts + code = "retry_exhausted" if (not permanent and terminal) else error_code + if terminal: + await session.execute( + update(JOBS) + .where(JOBS.c.id == job.id) + .values( + state="failed", + lease_token=None, + lease_expires_at=None, + last_error_code=code, + updated_at=func.now(), + finished_at=func.now(), + ) + ) + # Готовый результат более новой/этой версии не откатываем. + if attachment.pipeline_version <= job.pipeline_version and attachment.status != "ready": + await session.execute( + update(ATTACHMENTS) + .where(ATTACHMENTS.c.id == attachment.id) + .values(status="failed", error_code=code, updated_at=func.now()) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=job.id, + from_state=job.state, + to_state="failed", + attempt=job.attempts, + error_code=code, + worker_id=worker_id, + detail=detail, + ) + ) + return "failed" + + backoff = self._backoff_seconds(job.attempts) + next_at = func.now() + _interval(backoff) + await session.execute( + update(JOBS) + .where(JOBS.c.id == job.id) + .values( + state="queued", + lease_token=None, + lease_expires_at=None, + next_attempt_at=next_at, + last_error_code=code, + updated_at=func.now(), + ) + ) + await session.execute( + pg_insert(OUTBOX) + .values(job_id=job.id, available_at=next_at) + .on_conflict_do_nothing() + ) + if attachment.pipeline_version <= job.pipeline_version: + await session.execute( + update(ATTACHMENTS) + .where(ATTACHMENTS.c.id == attachment.id) + .values(status="queued", error_code=None, updated_at=func.now()) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=job.id, + from_state=job.state, + to_state="queued", + attempt=job.attempts, + error_code=code, + worker_id=worker_id, + detail=f"{detail}; retry_in={backoff:.2f}s", + ) + ) + return "retry" + + def _backoff_seconds(self, attempts: int) -> float: + base = self._config.job_backoff_base_seconds + cap = self._config.job_backoff_max_seconds + raw = min(cap, base * (2 ** max(0, attempts - 1))) + # Jitter ±20%: несколько worker-ов не просыпаются одновременно. + return max(0.05, raw * random.uniform(0.8, 1.2)) + + async def _supersede(self, session: AsyncSession, job: JobRecord, detail: str) -> None: + await session.execute( + update(JOBS) + .where(JOBS.c.id == job.id) + .values( + state="superseded", + lease_token=None, + lease_expires_at=None, + updated_at=func.now(), + finished_at=func.now(), + ) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=job.id, + from_state=job.state, + to_state="superseded", + attempt=job.attempts, + detail=detail, + ) + ) + + async def reap_expired(self, limit: int = 50) -> int: + """Возвращает в очередь processing с истёкшим lease; считает такие job.""" + async with self._db.session() as session: + ids = ( + await session.execute( + select(JOBS.c.id) + .where( + JOBS.c.state == "processing", + JOBS.c.lease_expires_at.is_not(None), + JOBS.c.lease_expires_at < func.now(), + ) + .order_by(JOBS.c.lease_expires_at) + .limit(limit) + ) + ).scalars().all() + recovered = 0 + for job_id in ids: + async with self._db.session() as session: + async with session.begin(): + row = ( + await session.execute( + select(JOBS).where(JOBS.c.id == job_id).with_for_update() + ) + ).first() + if row is None: + continue + job = _job_record(row) + db_now = await session.scalar(select(func.now())) + if ( + job.state != "processing" + or job.lease_expires_at is None + or job.lease_expires_at >= db_now + ): + continue + attachment_row = ( + await session.execute( + select(ATTACHMENTS) + .where(ATTACHMENTS.c.id == job.attachment_id) + .with_for_update() + ) + ).first() + if attachment_row is None: + continue + attachment = _attachment_record(attachment_row) + if attachment.pipeline_version > job.pipeline_version: + await self._supersede(session, job, "newer_pipeline_ready") + continue + if job.attempts >= job.max_attempts: + await self._fail_locked( + session, + job, + attachment, + error_code="retry_exhausted", + permanent=True, + detail="lease_expired_after_last_attempt", + ) + else: + await session.execute( + update(JOBS) + .where(JOBS.c.id == job.id) + .values( + state="queued", + lease_token=None, + lease_expires_at=None, + next_attempt_at=func.now(), + last_error_code="lease_expired", + updated_at=func.now(), + ) + ) + await session.execute( + pg_insert(OUTBOX) + .values(job_id=job.id, available_at=func.now()) + .on_conflict_do_nothing() + ) + await session.execute( + update(ATTACHMENTS) + .where(ATTACHMENTS.c.id == attachment.id) + .values(status="queued", updated_at=func.now()) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=job.id, + from_state="processing", + to_state="queued", + attempt=job.attempts, + error_code="lease_expired", + detail="reaper_requeued", + ) + ) + recovered += 1 + return recovered + + # --- outbox relay ----------------------------------------------------- + + async def outbox_due(self, limit: int = 20) -> list[OutboxItem]: + async with self._db.session() as session: + rows = ( + await session.execute( + select( + OUTBOX.c.id.label("outbox_id"), + JOBS.c.id.label("job_id"), + JOBS.c.attachment_id, + JOBS.c.pipeline_version, + JOBS.c.type, + JOBS.c.state, + JOBS.c.attempts, + JOBS.c.max_attempts, + JOBS.c.lease_token, + JOBS.c.worker_id, + JOBS.c.lease_expires_at, + JOBS.c.next_attempt_at, + JOBS.c.last_error_code, + JOBS.c.replay_of, + JOBS.c.created_at, + JOBS.c.updated_at, + JOBS.c.finished_at, + ) + .select_from(OUTBOX.join(JOBS, JOBS.c.id == OUTBOX.c.job_id)) + .where( + OUTBOX.c.published_at.is_(None), + OUTBOX.c.available_at <= func.now(), + ) + .order_by(OUTBOX.c.available_at, OUTBOX.c.id) + .limit(limit) + ) + ).all() + items: list[OutboxItem] = [] + for row in rows: + mapping = row._mapping + job = JobRecord( + id=mapping["job_id"], + attachment_id=mapping["attachment_id"], + pipeline_version=mapping["pipeline_version"], + type=mapping["type"], + state=mapping["state"], + attempts=mapping["attempts"], + max_attempts=mapping["max_attempts"], + lease_token=mapping["lease_token"], + worker_id=mapping["worker_id"], + lease_expires_at=mapping["lease_expires_at"], + next_attempt_at=mapping["next_attempt_at"], + last_error_code=mapping["last_error_code"], + replay_of=mapping["replay_of"], + created_at=mapping["created_at"], + updated_at=mapping["updated_at"], + finished_at=mapping["finished_at"], + ) + attachment_row = ( + await session.execute( + select(ATTACHMENTS).where(ATTACHMENTS.c.id == job.attachment_id) + ) + ).first() + if attachment_row is None: + continue + attachment = _attachment_record(attachment_row) + items.append( + OutboxItem( + outbox_id=int(mapping["outbox_id"]), + job=job, + attachment=attachment, + ) + ) + return items + + async def mark_published(self, outbox_id: int) -> None: + async with self._db.session() as session: + await session.execute( + update(OUTBOX) + .where(OUTBOX.c.id == outbox_id, OUTBOX.c.published_at.is_(None)) + .values( + published_at=func.now(), + attempts=OUTBOX.c.attempts + 1, + last_error=None, + ) + ) + await session.commit() + + async def mark_publish_failed(self, outbox_id: int, error: str) -> None: + async with self._db.session() as session: + await session.execute( + update(OUTBOX) + .where(OUTBOX.c.id == outbox_id, OUTBOX.c.published_at.is_(None)) + .values(attempts=OUTBOX.c.attempts + 1, last_error=error[:200]) + ) + await session.commit() + + # --- operator --------------------------------------------------------- + + async def replay( + self, + *, + job_id: UUID | None, + attachment_id: UUID | None, + requested_by: str, + reason: str, + ) -> UUID: + """Явный replay только failed-задачи; создаёт новый job_id и outbox-строку.""" + if not requested_by.strip() or len(requested_by) > 128: + raise ReplayRejected("requested_by must contain 1..128 characters") + if not reason.strip() or len(reason) > 512: + raise ReplayRejected("reason must contain 1..512 characters") + async with self._db.session() as session: + async with session.begin(): + if job_id is not None: + row = ( + await session.execute( + select(JOBS).where(JOBS.c.id == job_id).with_for_update() + ) + ).first() + else: + row = ( + await session.execute( + select(JOBS) + .where( + JOBS.c.attachment_id == attachment_id, + JOBS.c.state == "failed", + ) + .order_by(JOBS.c.created_at.desc()) + .limit(1) + .with_for_update() + ) + ).first() + if row is None: + raise ReplayRejected("failed job not found") + source = _job_record(row) + if source.state != "failed": + raise ReplayRejected("only failed jobs can be replayed") + attachment_row = ( + await session.execute( + select(ATTACHMENTS) + .where(ATTACHMENTS.c.id == source.attachment_id) + .with_for_update() + ) + ).first() + if attachment_row is None: + raise ReplayRejected("attachment not found") + if attachment_row.status == "ready": + # Успешный эффект уже существует: replay не должен создавать дубль. + raise ReplayRejected("attachment already has a ready result") + if attachment_row.pipeline_version > source.pipeline_version: + raise ReplayRejected("a newer pipeline result already exists") + inflight = ( + await session.execute( + select(JOBS.c.id) + .where( + JOBS.c.attachment_id == source.attachment_id, + JOBS.c.state.in_(("queued", "processing")), + ) + .limit(1) + ) + ).first() + if inflight is not None: + raise ReplayRejected("another job is already queued or processing") + new_id = uuid4() + await session.execute( + insert(JOBS).values( + id=new_id, + attachment_id=source.attachment_id, + pipeline_version=source.pipeline_version, + type=JOB_TYPE, + state="queued", + attempts=0, + max_attempts=source.max_attempts, + replay_of=source.id, + ) + ) + await session.execute( + update(ATTACHMENTS) + .where(ATTACHMENTS.c.id == source.attachment_id) + .values(status="queued", error_code=None, updated_at=func.now()) + ) + await session.execute( + insert(OUTBOX).values(job_id=new_id, available_at=func.now()) + ) + await session.execute( + insert(JOB_REPLAYS).values( + id=uuid4(), + source_job_id=source.id, + replay_job_id=new_id, + requested_by=requested_by.strip(), + reason=reason.strip(), + ) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=new_id, + from_state=None, + to_state="queued", + attempt=0, + detail=f"operator_replay_of={source.id}", + ) + ) + return new_id + + async def force_enqueue(self, attachment_id: UUID, pipeline_version: int) -> UUID: + """Тестовый/операторский enqueue конкретной версии (для fault-трека).""" + async with self._db.session() as session: + async with session.begin(): + attachment_row = ( + await session.execute( + select(ATTACHMENTS) + .where(ATTACHMENTS.c.id == attachment_id) + .with_for_update() + ) + ).first() + if attachment_row is None: + raise ReplayRejected("attachment not found") + job_id = uuid4() + try: + await session.execute( + insert(JOBS).values( + id=job_id, + attachment_id=attachment_id, + pipeline_version=pipeline_version, + type=JOB_TYPE, + state="queued", + attempts=0, + max_attempts=self._config.job_max_attempts, + ) + ) + except IntegrityError: + raise ReplayRejected("initial job for this version already exists") from None + await session.execute( + insert(OUTBOX).values(job_id=job_id, available_at=func.now()) + ) + await session.execute( + insert(JOB_EVENTS).values( + job_id=job_id, + from_state=None, + to_state="queued", + attempt=0, + detail="operator_force_enqueue", + ) + ) + return job_id + + async def set_fault(self, kind: str, count: int) -> None: + if kind not in FAULT_KINDS: + raise ValueError("unknown fault kind") + if count < 0: + raise ValueError("count must be >= 0") + async with self._db.session() as session: + await session.execute( + pg_insert(FAULT_FLAGS) + .values(kind=kind, remaining=count, updated_at=func.now()) + .on_conflict_do_update( + index_elements=[FAULT_FLAGS.c.kind], + set_={"remaining": count, "updated_at": func.now()}, + ) + ) + await session.commit() + + async def consume_fault(self) -> str | None: + if not self._config.allow_fault_injection: + return None + async with self._db.session() as session: + async with session.begin(): + row = ( + await session.execute( + select(FAULT_FLAGS) + .where( + FAULT_FLAGS.c.kind.in_(FAULT_KINDS), + FAULT_FLAGS.c.remaining > 0, + ) + .order_by(FAULT_FLAGS.c.kind) + .limit(1) + .with_for_update(skip_locked=True) + ) + ).first() + if row is None: + return None + await session.execute( + update(FAULT_FLAGS) + .where(FAULT_FLAGS.c.kind == row.kind) + .values(remaining=FAULT_FLAGS.c.remaining - 1, updated_at=func.now()) + ) + return row.kind + + async def report(self, attachment_id: UUID) -> dict: + """Снимок состояния для evidence и оператора (без секретов).""" + attachment = await self.get_attachment(attachment_id) + jobs = await self.jobs_for_attachment(attachment_id) + async with self._db.session() as session: + outbox_rows = ( + await session.execute( + select(OUTBOX) + .where(OUTBOX.c.job_id.in_([job.id for job in jobs])) + .order_by(OUTBOX.c.id) + ) + ).all() + event_rows = ( + await session.execute( + select(JOB_EVENTS) + .where(JOB_EVENTS.c.job_id.in_([job.id for job in jobs])) + .order_by(JOB_EVENTS.c.id) + ) + ).all() + + def moment(value): + return None if value is None else value.isoformat().replace("+00:00", "Z") + + return { + "attachment": None + if attachment is None + else { + "id": str(attachment.id), + "status": attachment.status, + "error_code": attachment.error_code, + "pipeline_version": attachment.pipeline_version, + "object_key": attachment.object_key, + "staging_key": attachment.staging_key, + "size_bytes": attachment.size_bytes, + "sha256": attachment.sha256, + "ready_at": moment(attachment.ready_at), + "updated_at": moment(attachment.updated_at), + "variants": [ + { + "name": item.name, + "pipeline_version": item.pipeline_version, + "content_type": item.content_type, + "object_key": item.object_key, + "size_bytes": item.size_bytes, + "sha256": item.sha256, + "width": item.width, + "height": item.height, + } + for item in attachment.variants + ], + }, + "jobs": [ + { + "id": str(job.id), + "pipeline_version": job.pipeline_version, + "state": job.state, + "attempts": job.attempts, + "max_attempts": job.max_attempts, + "lease_token": str(job.lease_token) if job.lease_token else None, + "worker_id": job.worker_id, + "last_error_code": job.last_error_code, + "replay_of": str(job.replay_of) if job.replay_of else None, + "created_at": moment(job.created_at), + "updated_at": moment(job.updated_at), + "finished_at": moment(job.finished_at), + } + for job in jobs + ], + "outbox": [ + { + "id": row.id, + "job_id": str(row.job_id), + "available_at": moment(row.available_at), + "published_at": moment(row.published_at), + "attempts": row.attempts, + "last_error": row.last_error, + } + for row in outbox_rows + ], + "events": [ + { + "job_id": str(row.job_id), + "from_state": row.from_state, + "to_state": row.to_state, + "attempt": row.attempt, + "error_code": row.error_code, + "worker_id": row.worker_id, + "detail": row.detail, + "created_at": moment(row.created_at), + } + for row in event_rows + ], + } + + async def queue_stats(self) -> dict: + async with self._db.session() as session: + job_rows = ( + await session.execute( + select(JOBS.c.state, func.count()).group_by(JOBS.c.state) + ) + ).all() + outbox_pending = await session.scalar( + select(func.count()).select_from(OUTBOX).where(OUTBOX.c.published_at.is_(None)) + ) + oldest = await session.scalar( + select(func.min(JOBS.c.created_at)).where(JOBS.c.state == "queued") + ) + return { + "jobs_by_state": {row[0]: int(row[1]) for row in job_rows}, + "outbox_pending": int(outbox_pending or 0), + "oldest_queued_at": None + if oldest is None + else oldest.isoformat().replace("+00:00", "Z"), + } diff --git a/src/messenger/logging_setup.py b/src/messenger/logging_setup.py new file mode 100644 index 00000000..88ee2cd1 --- /dev/null +++ b/src/messenger/logging_setup.py @@ -0,0 +1,76 @@ +"""Структурные логи в stdout: одна JSON-строка на событие, без тел и query-строки.""" +from __future__ import annotations + +import json +import logging +import sys +from datetime import datetime, timezone + + +class JsonFormatter(logging.Formatter): + """Сериализует запись лога в компактный JSON без потери структуры.""" + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, object] = { + "ts": datetime.fromtimestamp(record.created, timezone.utc) + .isoformat(timespec="milliseconds") + .replace("+00:00", "Z"), + "level": record.levelname.lower(), + "logger": record.name, + "event": record.getMessage(), + } + payload.update(getattr(record, "fields", {}) or {}) + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + + +class _EnvironmentFilter(logging.Filter): + """Добавляет метку окружения в каждую запись, включая логи uvicorn.""" + + def __init__(self, env: str) -> None: + super().__init__() + self._env = env + + def filter(self, record: logging.LogRecord) -> bool: + fields = dict(getattr(record, "fields", {}) or {}) + fields.setdefault("env", self._env) + record.fields = fields + return True + + +def configure_logging(level: str, env: str) -> logging.Logger: + """Перенаправляет логи процесса в stdout в структурированном виде.""" + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + handler.addFilter(_EnvironmentFilter(env)) + + root = logging.getLogger() + root.handlers[:] = [handler] + root.setLevel(level.upper()) + + for name in ("uvicorn", "uvicorn.error", "uvicorn.access"): + logger = logging.getLogger(name) + logger.handlers[:] = [] + logger.propagate = True + logging.getLogger("uvicorn.access").setLevel(logging.WARNING) + + return logging.getLogger("messenger") + + +def log( + logger: logging.Logger, + level: int, + event: str, + exc_info: BaseException | None = None, + **fields: object, +) -> None: + """Пишет событие с произвольными структурированными полями.""" + if not logger.isEnabledFor(level): + return + logger.log( + level, + event, + extra={"fields": {k: v for k, v in fields.items() if v is not None}}, + exc_info=exc_info, + ) diff --git a/src/messenger/middleware.py b/src/messenger/middleware.py new file mode 100644 index 00000000..63b4365d --- /dev/null +++ b/src/messenger/middleware.py @@ -0,0 +1,99 @@ +"""Внешний ASGI-слой: X-Request-Id, X-Instance-Id (в том числе на 500) и access-лог.""" +from __future__ import annotations + +import logging +import re +import time +import uuid +from typing import Any, Awaitable, Callable, MutableMapping + +from .context import REQUEST_ID, USER_ID, context_of +from .logging_setup import log + +SAFE_REQUEST_ID = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +Scope = MutableMapping[str, Any] +Message = MutableMapping[str, Any] +Receive = Callable[[], Awaitable[Message]] +Send = Callable[[Message], Awaitable[None]] +ASGIApp = Callable[[Scope, Receive, Send], Awaitable[None]] + + +def incoming_request_id(scope: Scope) -> str | None: + """Возвращает заголовок клиента, если он безопасен для логирования и ответа.""" + for name, value in scope.get("headers") or (): + if name.lower() != b"x-request-id": + continue + try: + candidate = value.decode("ascii") + except UnicodeDecodeError: + return None + return candidate if SAFE_REQUEST_ID.match(candidate) else None + return None + + +def set_response_header(message: Message, name: bytes, value: bytes) -> None: + """Идемпотентно выставляет заголовок в ASGI-сообщении ответа.""" + headers = message.setdefault("headers", []) + lowered = name.lower() + for index, (key, _) in enumerate(headers): + if key.lower() == lowered: + headers[index] = (name, value) + return + headers.append((name, value)) + + +class RequestContextMiddleware: + """Присваивает запросу идентификатор, гарантирует заголовок и пишет access-лог.""" + + def __init__(self, app: ASGIApp, logger: logging.Logger, instance_id: str) -> None: + if not instance_id: + raise ValueError("instance_id must not be empty") + self._app = app + self._logger = logger + self._instance_id = instance_id + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self._app(scope, receive, send) + return + + request_id = incoming_request_id(scope) or uuid.uuid4().hex + context = context_of(scope) + context[REQUEST_ID] = request_id + started = time.perf_counter() + status_holder: dict[str, int] = {} + instance_id = self._instance_id.encode("ascii") + + async def send_with_request_id(message: Message) -> None: + if message["type"] == "http.response.start": + status_holder["status"] = int(message["status"]) + set_response_header(message, b"x-request-id", request_id.encode("ascii")) + # Значение стабильно на жизнь процесса; LB не переписывает его. + set_response_header(message, b"x-instance-id", instance_id) + await send(message) + + try: + await self._app(scope, receive, send_with_request_id) + finally: + self._write_access_log(scope, status_holder.get("status"), started) + + def _write_access_log(self, scope: Scope, status: int | None, started: float) -> None: + context = context_of(scope) + # Логируется шаблон маршрута, а не сырой путь: query-строка в лог не попадает. + route = getattr(scope.get("route"), "path", None) + endpoint = scope.get("endpoint") + log( + self._logger, + logging.INFO, + "request", + request_id=context.get(REQUEST_ID), + method=scope.get("method"), + route=route, + operation=None if endpoint is None else getattr(endpoint, "__name__", None), + matched=route is not None, + status=status, + duration_ms=round((time.perf_counter() - started) * 1000, 3), + user_id=context.get(USER_ID), + instance_id=self._instance_id, + ) \ No newline at end of file diff --git a/src/messenger/models.py b/src/messenger/models.py new file mode 100644 index 00000000..36ba3c5e --- /dev/null +++ b/src/messenger/models.py @@ -0,0 +1,283 @@ +"""Схемы запросов и ответов ровно по contracts/openapi.json.""" +from __future__ import annotations + +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator + +from .domain import AttachmentRecord, ChatRecord, MessageRecord, UserRecord, VariantRecord, rfc3339_z +from .sessions import IssuedSession + +USERNAME_PATTERN = r"^[a-z0-9_]{3,32}$" +MIN_PASSWORD = 12 +MAX_PASSWORD = 128 +MAX_DISPLAY_NAME = 100 +MAX_TITLE = 100 +MAX_TEXT = 4000 +MAX_MEMBERS = 100 +MAX_ATTACHMENTS = 10 +MAX_LIMIT = 100 +DEFAULT_LIMIT = 50 + +ATTACHMENT_CONTENT_TYPES = ("image/png", "image/jpeg") +SHA256_PATTERN = r"^[a-f0-9]{64}$" + + +def _require_content(value: str, *, field: str) -> str: + """Обязательная строка не может быть пустой или состоять из пробельных символов.""" + if not value.strip(): + raise ValueError(f"{field} must not be empty or contain only whitespace") + return value + + +def _visible(value: str, *, field: str, maximum: int) -> str: + """Проверяет строку, которую предстоит сохранить в PostgreSQL.""" + _require_content(value, field=field) + if "\u0000" in value: + raise ValueError(f"{field} must not contain U+0000") + if len(value) > maximum: + raise ValueError(f"{field} must be at most {maximum} Unicode code points") + return value + + +class Health(BaseModel): + """Тело ответа /health/live и /health/ready.""" + + model_config = ConfigDict(extra="forbid") + + status: Literal["ok"] = "ok" + + +class User(BaseModel): + """Публичное представление пользователя: ни пароля, ни сессий здесь нет.""" + + model_config = ConfigDict(extra="forbid") + + id: UUID + username: str + display_name: str + created_at: str + + @classmethod + def from_record(cls, record: UserRecord) -> "User": + return cls( + id=record.id, + username=record.username, + display_name=record.display_name, + created_at=rfc3339_z(record.created_at), + ) + + +class CreateUser(BaseModel): + """Регистрация: пароль приходит открытым текстом и дальше живёт только как хэш.""" + + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=USERNAME_PATTERN) + display_name: str + password: str = Field(min_length=MIN_PASSWORD, max_length=MAX_PASSWORD) + + @field_validator("display_name") + @classmethod + def _check_display_name(cls, value: str) -> str: + return _visible(value, field="display_name", maximum=MAX_DISPLAY_NAME) + + @field_validator("password") + @classmethod + def _check_password(cls, value: str) -> str: + return _require_content(value, field="password") + + +class Session(BaseModel): + """Выданная сессия: токен показывается клиенту один раз и в логи не попадает.""" + + model_config = ConfigDict(extra="forbid") + + token: str + token_type: Literal["Bearer"] = "Bearer" + expires_at: str + user: User + + @classmethod + def from_issued(cls, issued: IssuedSession, user: UserRecord) -> "Session": + return cls( + token=issued.token, + expires_at=rfc3339_z(issued.state.expires_at), + user=User.from_record(user), + ) + + +class Chat(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + title: str + member_ids: list[UUID] + created_at: str + + @classmethod + def from_record(cls, record: ChatRecord) -> "Chat": + return cls( + id=record.id, + title=record.title, + member_ids=list(record.member_ids), + created_at=rfc3339_z(record.created_at), + ) + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str + member_ids: list[UUID] + + @field_validator("title") + @classmethod + def _check_title(cls, value: str) -> str: + return _visible(value, field="title", maximum=MAX_TITLE) + + @field_validator("member_ids") + @classmethod + def _check_member_ids(cls, value: list[UUID]) -> list[UUID]: + if not 1 <= len(value) <= MAX_MEMBERS: + raise ValueError(f"member_ids must contain 1..{MAX_MEMBERS} items") + if len(set(value)) != len(value): + raise ValueError("member_ids must not contain duplicates") + return value + + +class Message(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: str + attachment_ids: list[UUID] + + @classmethod + def from_record(cls, record: MessageRecord) -> "Message": + return cls( + id=record.id, + chat_id=record.chat_id, + sender_id=record.sender_id, + text=record.text, + created_at=rfc3339_z(record.created_at), + attachment_ids=list(record.attachment_ids), + ) + + +class CreateMessage(BaseModel): + """Поле sender_id не принимается: отправитель выводится из идентификации. + + Текст может быть пустым только при наличии хотя бы одного вложения. + """ + + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=MAX_TEXT) + attachment_ids: list[UUID] = Field(default_factory=list, max_length=MAX_ATTACHMENTS) + + @field_validator("text") + @classmethod + def _check_text(cls, value: str) -> str: + if "\u0000" in value: + raise ValueError("text must not contain U+0000") + if len(value) > MAX_TEXT: + raise ValueError(f"text must be at most {MAX_TEXT} Unicode code points") + return value + + @field_validator("attachment_ids") + @classmethod + def _check_attachment_ids(cls, value: list[UUID]) -> list[UUID]: + if len(set(value)) != len(value): + raise ValueError("attachment_ids must not contain duplicates") + return value + + @model_validator(mode="after") + def _check_content(self) -> "CreateMessage": + if not self.text.strip() and not self.attachment_ids: + raise ValueError("text must not be empty or contain only whitespace without attachments") + return self + + +class ChatPage(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[Chat] + next_cursor: str | None + + @classmethod + def from_records(cls, records: list[ChatRecord], next_cursor: str | None) -> "ChatPage": + return cls(items=[Chat.from_record(record) for record in records], next_cursor=next_cursor) + + +class MessagePage(BaseModel): + model_config = ConfigDict(extra="forbid") + + items: list[Message] + next_cursor: str | None + + @classmethod + def from_records(cls, records: list[MessageRecord], next_cursor: str | None) -> "MessagePage": + return cls(items=[Message.from_record(record) for record in records], next_cursor=next_cursor) + + +class Variant(BaseModel): + """Опубликованный вариант изображения ровно по contracts/openapi.json.""" + + model_config = ConfigDict(extra="forbid") + + name: Literal["thumbnail", "crop", "optimized"] + content_type: Literal["image/jpeg", "image/png", "image/webp"] + size_bytes: int = Field(gt=0) + width: int = Field(gt=0) + height: int = Field(gt=0) + sha256: str = Field(pattern=SHA256_PATTERN) + + @classmethod + def from_record(cls, record: VariantRecord) -> "Variant": + return cls( + name=record.name, + content_type=record.content_type, + size_bytes=record.size_bytes, + width=record.width, + height=record.height, + sha256=record.sha256, + ) + + +class Attachment(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: UUID + chat_id: UUID + owner_id: UUID + filename: str = Field(min_length=1, max_length=255) + content_type: Literal["image/png", "image/jpeg"] + size_bytes: int = Field(gt=0, le=10485760) + sha256: str = Field(pattern=SHA256_PATTERN) + status: Literal["queued", "processing", "ready", "failed"] + variants: list[Variant] = Field(default_factory=list) + error_code: str | None = None + created_at: str + updated_at: str + + @classmethod + def from_record(cls, record: AttachmentRecord) -> "Attachment": + return cls( + id=record.id, + chat_id=record.chat_id, + owner_id=record.owner_id, + filename=record.filename, + content_type=record.content_type, + size_bytes=record.size_bytes, + sha256=record.sha256, + status=record.status, + variants=[Variant.from_record(item) for item in record.variants], + error_code=record.error_code, + created_at=rfc3339_z(record.created_at), + updated_at=rfc3339_z(record.updated_at), + ) diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 00000000..4a342dc2 --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,38 @@ +"""Keyset-пагинация: разбор курсора клиента и сборка следующего.""" +from __future__ import annotations + +from typing import Sequence + +from .cursor import CursorCodec, InvalidCursor +from .domain import Position, Sortable, rfc3339_z +from .errors import invalid_request + + +def decoded_cursor( + cursors: CursorCodec, + token: str | None, + *, + kind: str, + scope: str, +) -> Position | None: + """Позиция из курсора клиента; None означает «начать с начала выборки».""" + if token is None: + return None + try: + return cursors.decode(token, kind=kind, scope=scope).position + except InvalidCursor: + raise invalid_request("cursor недействителен для этого запроса") from None + + +def build_next_cursor( + cursors: CursorCodec, + kind: str, + scope: str, + records: Sequence[Sortable], + has_more: bool, +) -> str | None: + """next_cursor равен null, если на момент чтения следующей записи нет.""" + if not has_more or not records: + return None + last = records[-1] + return cursors.encode(kind, scope, rfc3339_z(last.created_at), str(last.id)) diff --git a/src/messenger/pipeline.py b/src/messenger/pipeline.py new file mode 100644 index 00000000..8007786c --- /dev/null +++ b/src/messenger/pipeline.py @@ -0,0 +1,223 @@ +"""Версионированный конвейер изображений: thumbnail, crop и optimized WebP. + +Версия 1 — обязательный минимум лабораторной (original + thumbnail 256×256). +Версия 2 — уровень 4 (additional crop 128×128 и optimized WebP до 1280×1280). + +Все варианты детерминированно вписываются в размеры с сохранением пропорций, +EXIF orientation применяется до масштабирования, alpha компонуется на белый, +маленькие исходники не увеличиваются (кроме явно разрешённого crop). Метаданные +(EXIF/ICC) не переносятся: выходные файлы создаются из пиксельных данных. +""" +from __future__ import annotations + +import hashlib +import os +from dataclasses import dataclass + +from PIL import Image, ImageOps, UnidentifiedImageError + +THUMBNAIL_SIZE = (256, 256) +CROP_SIZE = (128, 128) +OPTIMIZED_SIZE = (1280, 1280) +JPEG_QUALITY = 85 +WEBP_QUALITY = 80 +WEBP_METHOD = 6 + +SUPPORTED_PIPELINE_VERSIONS = (1, 2) + + +class PipelineError(Exception): + """Ошибка конвейера с безопасным машинным кодом и признаком постоянства.""" + + permanent = False + code = "processing_failed" + + +class PermanentPipelineError(PipelineError): + permanent = True + + +class InvalidImage(PermanentPipelineError): + code = "invalid_image" + + +class ImageTooLarge(PermanentPipelineError): + code = "image_too_large" + + +class CorruptOriginal(PermanentPipelineError): + code = "corrupt_original" + + +class UnsupportedPipeline(PermanentPipelineError): + code = "unsupported_pipeline" + + +@dataclass(frozen=True) +class GeneratedVariant: + """Готовый вариант на локальном диске worker-а до публикации в S3.""" + + name: str + content_type: str + path: str + size_bytes: int + sha256: str + width: int + height: int + + def cleanup(self) -> None: + try: + os.unlink(self.path) + except OSError: + pass + + +def variant_specs(pipeline_version: int) -> tuple[tuple[str, str], ...]: + """(name, content_type) в порядке публикации для версии конвейера.""" + if pipeline_version == 1: + return (("thumbnail", "image/jpeg"),) + if pipeline_version == 2: + return ( + ("thumbnail", "image/jpeg"), + ("crop", "image/jpeg"), + ("optimized", "image/webp"), + ) + raise UnsupportedPipeline("Неизвестная версия конвейера") + + +def _suffix(name: str) -> str: + return ".webp" if name == "optimized" else ".jpg" + + +def _to_rgb(image: Image.Image) -> Image.Image: + """Alpha компонуется на белый: JPEG/WebP не должны содержать прозрачность.""" + if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info): + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.getchannel("A")) + return background + return image.convert("RGB") + + +def _save(image: Image.Image, path: str, name: str) -> None: + if name == "optimized": + image.save( + path, + format="WEBP", + quality=WEBP_QUALITY, + method=WEBP_METHOD, + exif=b"", + icc_profile=None, + ) + else: + image.save( + path, + format="JPEG", + quality=JPEG_QUALITY, + optimize=True, + progressive=True, + exif=b"", + icc_profile=None, + ) + + +def _measure(path: str) -> tuple[int, str]: + digest = hashlib.sha256() + size = 0 + with open(path, "rb") as handle: + while True: + chunk = handle.read(64 * 1024) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + return size, digest.hexdigest() + + +def render_variants( + source_path: str, + *, + pipeline_version: int, + workdir: str, + max_pixels: int, +) -> list[GeneratedVariant]: + """Декодирует оригинал и создаёт ровно варианты заявленной версии. + + Вызывается в рабочем потоке. Ошибки Pillow преобразуются в типизированные + постоянные ошибки, чтобы worker не делал бессмысленных повторов. + """ + specs = variant_specs(pipeline_version) + Image.MAX_IMAGE_PIXELS = max_pixels + try: + with Image.open(source_path) as opened: + opened.load() + oriented = ImageOps.exif_transpose(opened) + width, height = oriented.size + if width <= 0 or height <= 0: + raise InvalidImage("Некорректные размеры изображения") + if width * height > max_pixels: + raise ImageTooLarge("Изображение превышает предел пикселей") + base = _to_rgb(oriented) + except (InvalidImage, ImageTooLarge): + raise + except Image.DecompressionBombError as exc: + raise ImageTooLarge("Изображение превышает предел пикселей") from exc + except ( + UnidentifiedImageError, + OSError, + SyntaxError, + ValueError, + EOFError, + IndexError, + OverflowError, + ) as exc: + raise InvalidImage("Изображение повреждено") from exc + + generated: list[GeneratedVariant] = [] + try: + for name, content_type in specs: + if name == "thumbnail": + rendered = base.copy() + rendered.thumbnail(THUMBNAIL_SIZE, Image.Resampling.LANCZOS) + elif name == "crop": + # Crop — центрированный квадрат; маленькие исходники можно увеличивать. + rendered = ImageOps.fit( + base.copy(), + CROP_SIZE, + method=Image.Resampling.LANCZOS, + centering=(0.5, 0.5), + ) + elif name == "optimized": + rendered = base.copy() + rendered.thumbnail(OPTIMIZED_SIZE, Image.Resampling.LANCZOS) + else: # pragma: no cover - защищено variant_specs + raise UnsupportedPipeline("Неизвестный вариант") + + path = os.path.join(workdir, name + _suffix(name)) + _save(rendered, path, name) + rendered.close() + size, digest = _measure(path) + if size <= 0: + raise InvalidImage("Пустой вариант изображения") + with Image.open(path) as check: + out_width, out_height = check.size + generated.append( + GeneratedVariant( + name=name, + content_type=content_type, + path=path, + size_bytes=size, + sha256=digest, + width=out_width, + height=out_height, + ) + ) + return generated + except PipelineError: + for item in generated: + item.cleanup() + raise + except (OSError, ValueError, Image.DecompressionBombError) as exc: + for item in generated: + item.cleanup() + raise InvalidImage("Не удалось построить вариант изображения") from exc diff --git a/src/messenger/reconcile.py b/src/messenger/reconcile.py new file mode 100644 index 00000000..2f15c1c7 --- /dev/null +++ b/src/messenger/reconcile.py @@ -0,0 +1,204 @@ +"""Сверка PostgreSQL и S3: staging, orphans и missing ready. + +Запуск идемпотентен и по умолчанию только показывает план (`--dry-run`). +Правила: +- staging текущей (queued/processing) работы не удаляется никогда; +- staging ready/failed-вложения удаляется только после retention; +- variant-объекты ready-вложения считаются используемыми; +- ready-вложение с пропавшим оригиналом только репортится, metadata не теряется. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta +from uuid import UUID + +from .config import Config +from .db import Database +from .domain import AttachmentRecord, utc_now +from .repository import Repository +from .storage import ObjectInfo, S3Storage, StorageUnavailable + +logger = logging.getLogger("messenger.reconcile") + + +@dataclass(frozen=True) +class PlanItem: + kind: str + identifier: str + object_key: str | None + age_seconds: int | None + action: str + + +def build_plan( + rows: list[AttachmentRecord], + objects: list[ObjectInfo], + *, + variant_keys: set[str], + now: datetime, + grace_seconds: int, +) -> tuple[list[PlanItem], list[PlanItem]]: + objects_by_key = {item.key: item for item in objects} + cutoff = now - timedelta(seconds=grace_seconds) + items: list[PlanItem] = [] + protected: set[str] = set(variant_keys) + failed_rows: dict[str, AttachmentRecord] = {} + + for row in rows: + if row.status == "ready": + protected.add(row.object_key) + elif row.status in ("queued", "processing"): + protected.add(row.staging_key) + elif row.status == "failed": + failed_rows[row.staging_key] = row + + for item in objects: + if item.key in protected: + continue + row = failed_rows.get(item.key) + if row is not None: + age = max(0, int((now - item.last_modified).total_seconds())) + stale = row.updated_at < cutoff and item.last_modified < cutoff + items.append( + PlanItem( + kind="stale_failed", + identifier=str(row.id), + object_key=item.key, + age_seconds=age, + action="delete_object_and_row" if stale else "keep_fresh", + ) + ) + continue + age = max(0, int((now - item.last_modified).total_seconds())) + items.append( + PlanItem( + kind="orphan_object", + identifier=item.key, + object_key=item.key, + age_seconds=age, + action="keep_fresh" if item.last_modified >= cutoff else "delete", + ) + ) + + # ready-объект отсутствует: это авария хранилища, metadata не удаляем. + missing = [ + PlanItem( + kind="missing_ready", + identifier=str(row.id), + object_key=row.object_key, + age_seconds=max(0, int((now - row.updated_at).total_seconds())), + action="report_only", + ) + for row in rows + if row.status == "ready" and row.object_key not in objects_by_key + ] + # failed-строка без staging: удалять нечего, но metadata старше retention + # остаётся доступной оператору (replay/аудит). + return items, missing + + +async def run(args: argparse.Namespace) -> int: + config = Config.from_env() + database = Database(config) + storage = S3Storage(config) + grace = config.staging_retention_seconds if args.grace_seconds is None else args.grace_seconds + if grace < 0: + raise SystemExit("--grace-seconds must be >= 0") + try: + async with database.session() as session: + repository = Repository(session) + rows = await repository.list_attachment_states() + variant_keys = await repository.list_variant_keys() + objects = await storage.list_objects() + now = utc_now() + items, missing = build_plan( + rows, + objects, + variant_keys=variant_keys, + now=now, + grace_seconds=grace, + ) + deletable = [ + item + for item in items + if item.action in ("delete", "delete_object_and_row") + ] + summary = { + "mode": "apply" if args.apply else "dry-run", + "grace_seconds": grace, + "prefix": config.s3_key_prefix, + "metadata_rows": len(rows), + "variant_keys": len(variant_keys), + "objects": len(objects), + "plan": [asdict(item) for item in items], + "missing_ready": [asdict(item) for item in missing], + "would_delete_objects": len( + [item for item in items if item.action in ("delete", "delete_object_and_row")] + ), + "would_delete_rows": len( + [item for item in items if item.action == "delete_object_and_row"] + ), + } + if not args.apply: + print(json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True)) + return 1 if (missing and args.fail_on_missing) else 0 + + deleted_objects = 0 + deleted_rows = 0 + async with database.session() as session: + repository = Repository(session) + for item in deletable: + if item.object_key: + try: + await storage.delete(item.object_key) + deleted_objects += 1 + except StorageUnavailable: + logger.warning( + "reconcile_object_delete_failed", + extra={"fields": {"key": item.object_key}}, + ) + continue + if item.action == "delete_object_and_row": + try: + if await repository.delete_attachment_if_stale( + UUID(item.identifier), now - timedelta(seconds=grace) + ): + deleted_rows += 1 + except Exception: + logger.warning( + "reconcile_row_delete_failed", + extra={"fields": {"id": item.identifier}}, + ) + summary["deleted_objects"] = deleted_objects + summary["deleted_rows"] = deleted_rows + print(json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True)) + return 1 if (missing and args.fail_on_missing) else 0 + finally: + await storage.close() + await database.dispose() + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Сверка вложений PostgreSQL и S3") + parser.add_argument("--apply", action="store_true", help="действительно удалять мусор") + parser.add_argument("--dry-run", action="store_true", help="только показать план (по умолчанию)") + parser.add_argument( + "--grace-seconds", + type=int, + default=None, + help="возраст, после которого staging failed/ready удаляется", + ) + parser.add_argument("--fail-on-missing", action="store_true", help="код 1, если ready-объект отсутствует") + args = parser.parse_args(argv) + if args.dry_run: + args.apply = False + return asyncio.run(run(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/messenger/relay.py b/src/messenger/relay.py new file mode 100644 index 00000000..8e953294 --- /dev/null +++ b/src/messenger/relay.py @@ -0,0 +1,208 @@ +"""Relay: transactional outbox → RabbitMQ + reaper истёкших lease. + +Relay отделён от API: COMMIT задания и публикация не могут быть одной +транзакцией, поэтому строка outbox коммитится вместе с job и публикуется +повторно до первого successful publisher confirm. Падение между COMMIT и +publish не теряет принятую работу; повторная публикация допустима, потому что +worker идемпотентен. +""" +from __future__ import annotations + +import asyncio +import logging +import os +import signal +import time +from pathlib import Path + +from .broker import Broker, build_payload +from .config import Config, ConfigError +from .db import Database +from .jobs import JobRepository +from .logging_setup import configure_logging, log + +logger = logging.getLogger("messenger.relay") + +HEARTBEAT_FILE = os.environ.get("HEARTBEAT_FILE", "/tmp/messenger-relay-heartbeat") +HEARTBEAT_SECONDS = 5.0 + + +def _heartbeat() -> None: + try: + Path(HEARTBEAT_FILE).write_text(str(time.time()), encoding="ascii") + except OSError: + pass + + +class Relay: + def __init__(self, config: Config) -> None: + self._config = config + self._database = Database(config) + self._jobs = JobRepository(self._database, config) + self._broker = Broker(config) + self._stop = asyncio.Event() + self._published = 0 + self._reaped = 0 + + def request_stop(self) -> None: + self._stop.set() + + async def _heartbeat_loop(self) -> None: + while not self._stop.is_set(): + _heartbeat() + try: + await asyncio.wait_for(self._stop.wait(), timeout=HEARTBEAT_SECONDS) + except asyncio.TimeoutError: + continue + + async def _reaper_loop(self) -> None: + while not self._stop.is_set(): + try: + recovered = await self._jobs.reap_expired() + if recovered: + self._reaped += recovered + log(logger, logging.WARNING, "lease_reaped", jobs=recovered) + except Exception as exc: + log(logger, logging.ERROR, "reaper_failed", exc_info=exc) + try: + await asyncio.wait_for( + self._stop.wait(), timeout=self._config.reaper_interval_seconds + ) + except asyncio.TimeoutError: + continue + + async def run(self) -> None: + log( + logger, + logging.INFO, + "relay_startup", + queue=self._broker.queue_name, + poll_seconds=self._config.relay_poll_seconds, + ) + await self._broker.connect() + _heartbeat() + heartbeat_task = asyncio.create_task(self._heartbeat_loop()) + reaper_task = asyncio.create_task(self._reaper_loop()) + try: + while not self._stop.is_set(): + try: + items = await self._jobs.outbox_due(20) + except Exception as exc: + log(logger, logging.ERROR, "outbox_read_failed", exc_info=exc) + await asyncio.sleep(self._config.relay_poll_seconds) + continue + if not items: + try: + await asyncio.wait_for( + self._stop.wait(), timeout=self._config.relay_poll_seconds + ) + except asyncio.TimeoutError: + pass + continue + for item in items: + if self._stop.is_set(): + break + if item.job.state != "queued": + # Job уже завершён/захвачен; outbox-строка больше не нужна. + await self._jobs.mark_published(item.outbox_id) + log( + logger, + logging.INFO, + "outbox_retired", + job_id=str(item.job.id), + state=item.job.state, + ) + continue + payload = build_payload(item.job, attempt=item.job.attempts + 1) + try: + await self._broker.publish_job( + payload, message_id=str(item.job.id) + ) + except Exception as exc: + await self._jobs.mark_publish_failed( + item.outbox_id, type(exc).__name__ + ) + log( + logger, + logging.WARNING, + "outbox_publish_failed", + exc_info=exc, + job_id=str(item.job.id), + ) + # Брокер мог перезапускаться: пересоздаём соединение. + try: + await self._broker.close() + except Exception: + pass + await asyncio.sleep(min(2.0, 0.2 * (item.job.attempts + 1))) + try: + await self._broker.connect() + except Exception as reconnect_exc: + log( + logger, + logging.ERROR, + "broker_reconnect_failed", + exc_info=reconnect_exc, + ) + await asyncio.sleep(2.0) + break + await self._jobs.mark_published(item.outbox_id) + self._published += 1 + log( + logger, + logging.INFO, + "outbox_published", + job_id=str(item.job.id), + attachment_id=str(item.job.attachment_id), + pipeline_version=item.job.pipeline_version, + attempt=payload["attempt"], + ) + await asyncio.sleep(0) + finally: + self._stop.set() + heartbeat_task.cancel() + reaper_task.cancel() + await asyncio.gather(heartbeat_task, reaper_task, return_exceptions=True) + try: + await self._broker.close() + except Exception: + pass + await self._database.dispose() + log( + logger, + logging.INFO, + "relay_stopped", + published=self._published, + reaped=self._reaped, + ) + + +async def _run() -> int: + try: + config = Config.from_env() + config.require_broker() + except ConfigError as exc: + print(f"config error: {exc}", file=os.sys.stderr) + return 2 + root = configure_logging(config.log_level, config.env) + relay = Relay(config) + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, relay.request_stop) + except (NotImplementedError, RuntimeError): + signal.signal(sig, lambda *_: relay.request_stop()) + try: + await relay.run() + except Exception as exc: + log(root, logging.ERROR, "relay_fatal", exc_info=exc) + return 1 + return 0 + + +def main() -> int: + return asyncio.run(_run()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/messenger/repository.py b/src/messenger/repository.py new file mode 100644 index 00000000..c87c392b --- /dev/null +++ b/src/messenger/repository.py @@ -0,0 +1,609 @@ +"""Запросы к PostgreSQL: единственное место, где выполняется SQL. + +Все значения передаются bind-параметрами — строки запросов не собираются из +данных пользователя. Транзакционную границу задаёт репозиторий: операция, +меняющая несколько таблиц, выполняется целиком или не выполняется вовсе. + +Единица работы — неявная транзакция сессии запроса: пишущие методы завершают +её `commit`, а незавершённую откатывает закрытие сессии. +""" +from __future__ import annotations + +from typing import Sequence +from uuid import UUID, uuid4 + +from sqlalchemy import Row, Select, cast, delete, func, insert, literal, select, text, tuple_ +from sqlalchemy.dialects.postgresql import TIMESTAMP, UUID as PGUUID +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncSession + +from .domain import ( + ATTACHMENT_STATUS_FAILED, + AttachmentNotReady, + AttachmentRecord, + AttachmentUnknown, + ChatRecord, + IdempotencyConflict, + MessageRecord, + Position, + UnknownMembers, + UserRecord, + VariantRecord, +) +from .jobs import insert_upload_work +from .schema import ( + ATTACHMENTS, + CHAT_MEMBERS, + CHATS, + MESSAGE_ATTACHMENTS, + MESSAGE_IDEMPOTENCY, + MESSAGES, + USERS, + VARIANTS, +) + +FOREIGN_KEY_VIOLATION = "23503" + +# Ключ идемпотентности живёт 24 часа; срок считает PostgreSQL, а не часы +# приложения, чтобы реплики с разными часами принимали одинаковое решение. +IDEMPOTENCY_FRESH_PREDICATE = MESSAGE_IDEMPOTENCY.c.created_at > ( + func.now() - text("INTERVAL '24 hours'") +) + + +def _sqlstate(error: IntegrityError) -> str | None: + """Код ошибки PostgreSQL: драйвер кладёт его в исходное исключение.""" + return getattr(getattr(error, "orig", None), "sqlstate", None) + + +def _after_predicate(created_at_column, id_column, after: Position): + """Keyset-условие «строго после пары (created_at, id)» одним сравнением строк.""" + moment, item_id = after + return tuple_(created_at_column, id_column) > tuple_( + cast(literal(moment), TIMESTAMP(timezone=True)), + cast(literal(item_id), PGUUID(as_uuid=True)), + ) + + +def _user_record(row: Row) -> UserRecord: + return UserRecord( + id=row.id, username=row.username, display_name=row.display_name, created_at=row.created_at + ) + + +def _chat_columns() -> tuple: + return (CHATS.c.id, CHATS.c.title, CHATS.c.created_at) + + +def _message_columns() -> tuple: + return ( + MESSAGES.c.id, + MESSAGES.c.chat_id, + MESSAGES.c.sender_id, + MESSAGES.c.text, + MESSAGES.c.created_at, + ) + + +def _attachment_columns() -> tuple: + return ( + ATTACHMENTS.c.id, + ATTACHMENTS.c.chat_id, + ATTACHMENTS.c.owner_id, + ATTACHMENTS.c.object_key, + ATTACHMENTS.c.staging_key, + ATTACHMENTS.c.pipeline_version, + ATTACHMENTS.c.filename, + ATTACHMENTS.c.content_type, + ATTACHMENTS.c.size_bytes, + ATTACHMENTS.c.sha256, + ATTACHMENTS.c.status, + ATTACHMENTS.c.error_code, + ATTACHMENTS.c.ready_at, + ATTACHMENTS.c.created_at, + ATTACHMENTS.c.updated_at, + ) + + +def _variant_record(row: Row) -> VariantRecord: + return VariantRecord( + attachment_id=row.attachment_id, + pipeline_version=row.pipeline_version, + name=row.name, + content_type=row.content_type, + object_key=row.object_key, + size_bytes=row.size_bytes, + sha256=row.sha256, + width=row.width, + height=row.height, + created_at=row.created_at, + ) + + +def _attachment_record( + row: Row, variants: Sequence[VariantRecord] = () +) -> AttachmentRecord: + return AttachmentRecord( + id=row.id, + chat_id=row.chat_id, + owner_id=row.owner_id, + object_key=row.object_key, + staging_key=row.staging_key, + pipeline_version=row.pipeline_version, + filename=row.filename, + content_type=row.content_type, + size_bytes=row.size_bytes, + sha256=row.sha256, + status=row.status, + error_code=row.error_code, + ready_at=row.ready_at, + created_at=row.created_at, + updated_at=row.updated_at, + variants=tuple(variants), + ) + + +def _message_record(row: Row, attachment_ids: Sequence[UUID] = ()) -> MessageRecord: + return MessageRecord( + id=row.id, + chat_id=row.chat_id, + sender_id=row.sender_id, + text=row.text, + created_at=row.created_at, + attachment_ids=tuple(attachment_ids), + ) + + +class Repository: + """Набор операций над PostgreSQL в рамках одной сессии (одного запроса).""" + + def __init__(self, session: AsyncSession) -> None: + self._session = session + + async def ping(self) -> None: + """Дешёвая проверка живости соединения для /health/ready.""" + await self._session.execute(select(literal(1))) + + async def username_taken(self, username: str) -> bool: + found = await self._session.scalar( + select(USERS.c.id).where(USERS.c.username == username).limit(1) + ) + return found is not None + + async def create_user( + self, username: str, display_name: str, password_hash: str + ) -> UserRecord | None: + row = ( + await self._session.execute( + pg_insert(USERS) + .values( + id=uuid4(), + username=username, + display_name=display_name, + password_hash=password_hash, + ) + .on_conflict_do_nothing(index_elements=[USERS.c.username]) + .returning(*USERS.c) + ) + ).first() + await self._session.commit() + return None if row is None else _user_record(row) + + async def credentials_for(self, username: str) -> tuple[UserRecord, str] | None: + row = ( + await self._session.execute( + select(USERS.c.id, USERS.c.username, USERS.c.display_name, USERS.c.created_at, + USERS.c.password_hash) + .where(USERS.c.username == username) + .limit(1) + ) + ).first() + if row is None: + return None + return _user_record(row), row.password_hash + + async def get_user(self, user_id: UUID) -> UserRecord | None: + row = ( + await self._session.execute( + select(USERS.c.id, USERS.c.username, USERS.c.display_name, USERS.c.created_at) + .where(USERS.c.id == user_id) + .limit(1) + ) + ).first() + return None if row is None else _user_record(row) + + async def create_chat(self, title: str, member_ids: Sequence[UUID]) -> ChatRecord: + chat_id = uuid4() + row = ( + await self._session.execute( + insert(CHATS).values(id=chat_id, title=title).returning(*_chat_columns()) + ) + ).one() + try: + await self._session.execute( + insert(CHAT_MEMBERS), + [{"chat_id": chat_id, "user_id": member_id} for member_id in member_ids], + ) + except IntegrityError as error: + if _sqlstate(error) != FOREIGN_KEY_VIOLATION: + raise + await self._session.rollback() + raise UnknownMembers(member_ids) from None + await self._session.commit() + return ChatRecord( + id=row.id, + title=row.title, + member_ids=tuple(sorted(member_ids, key=lambda item: item.int)), + created_at=row.created_at, + ) + + async def chat_for_member(self, chat_id: UUID, user_id: UUID) -> ChatRecord | None: + row = ( + await self._session.execute( + select(*_chat_columns()) + .select_from(CHATS.join(CHAT_MEMBERS, CHAT_MEMBERS.c.chat_id == CHATS.c.id)) + .where(CHATS.c.id == chat_id, CHAT_MEMBERS.c.user_id == user_id) + .limit(1) + ) + ).first() + if row is None: + return None + members = await self._members_of([chat_id]) + return ChatRecord( + id=row.id, title=row.title, member_ids=members[chat_id], created_at=row.created_at + ) + + async def list_chats( + self, user_id: UUID, after: Position | None, limit: int + ) -> tuple[list[ChatRecord], bool]: + statement = ( + select(*_chat_columns()) + .select_from(CHATS.join(CHAT_MEMBERS, CHAT_MEMBERS.c.chat_id == CHATS.c.id)) + .where(CHAT_MEMBERS.c.user_id == user_id) + ) + rows = await self._page(statement, CHATS, after, limit) + more = len(rows) > limit + rows = rows[:limit] + members = await self._members_of([row.id for row in rows]) + chats = [ + ChatRecord( + id=row.id, title=row.title, member_ids=members[row.id], created_at=row.created_at + ) + for row in rows + ] + return chats, more + + async def append_message( + self, chat_id: UUID, sender_id: UUID, text: str, attachment_ids: Sequence[UUID] = () + ) -> MessageRecord: + message_id = uuid4() + row = ( + await self._session.execute( + insert(MESSAGES) + .values(id=message_id, chat_id=chat_id, sender_id=sender_id, text=text) + .returning(*_message_columns()) + ) + ).one() + await self._link_attachments(message_id, attachment_ids) + await self._session.commit() + return _message_record(row, attachment_ids) + + async def append_message_idempotent( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + attachment_ids: Sequence[UUID], + idempotency_key: str, + request_hash: str, + ) -> MessageRecord: + """Создаёт сообщение ровно один раз для ключа (пользователь, чат, ключ). + + Побочный эффект (messages + message_attachments) и запись результата + (message_idempotency) фиксируются одной транзакцией. + """ + lock_scope = f'{sender_id}:{chat_id}:{idempotency_key}' + await self._session.execute( + select(func.pg_advisory_xact_lock(func.hashtextextended(lock_scope, 0))) + ) + fresh = IDEMPOTENCY_FRESH_PREDICATE + existing = ( + await self._session.execute( + select(*_message_columns(), MESSAGE_IDEMPOTENCY.c.request_hash, fresh.label('fresh')) + .select_from( + MESSAGE_IDEMPOTENCY.join( + MESSAGES, MESSAGES.c.id == MESSAGE_IDEMPOTENCY.c.message_id + ) + ) + .where( + MESSAGE_IDEMPOTENCY.c.user_id == sender_id, + MESSAGE_IDEMPOTENCY.c.chat_id == chat_id, + MESSAGE_IDEMPOTENCY.c.idempotency_key == idempotency_key, + ) + .limit(1) + ) + ).first() + if existing is not None: + if existing.fresh and existing.request_hash == request_hash: + linked = await self._attachment_ids_of([existing.id]) + record = _message_record(existing, linked.get(existing.id, ())) + await self._session.commit() + return record + if existing.fresh: + await self._session.rollback() + raise IdempotencyConflict() + # Срок хранения ключа истёк: старый ключ можно использовать снова. + await self._session.execute( + delete(MESSAGE_IDEMPOTENCY).where( + MESSAGE_IDEMPOTENCY.c.user_id == sender_id, + MESSAGE_IDEMPOTENCY.c.chat_id == chat_id, + MESSAGE_IDEMPOTENCY.c.idempotency_key == idempotency_key, + ) + ) + message_id = uuid4() + row = ( + await self._session.execute( + insert(MESSAGES) + .values(id=message_id, chat_id=chat_id, sender_id=sender_id, text=text) + .returning(*_message_columns()) + ) + ).one() + await self._link_attachments(message_id, attachment_ids) + await self._session.execute( + insert(MESSAGE_IDEMPOTENCY).values( + user_id=sender_id, + chat_id=chat_id, + idempotency_key=idempotency_key, + request_hash=request_hash, + message_id=message_id, + ) + ) + await self._session.commit() + return _message_record(row, attachment_ids) + + async def list_messages( + self, chat_id: UUID, after: Position | None, limit: int + ) -> tuple[list[MessageRecord], bool]: + statement = select(*_message_columns()).where(MESSAGES.c.chat_id == chat_id) + rows = await self._page(statement, MESSAGES, after, limit) + more = len(rows) > limit + page = rows[:limit] + linked = await self._attachment_ids_of([row.id for row in page]) + messages = [ + _message_record(row, linked.get(row.id, ())) + for row in page + ] + return messages, more + + async def _link_attachments(self, message_id: UUID, attachment_ids: Sequence[UUID]) -> None: + if not attachment_ids: + return + await self._session.execute( + insert(MESSAGE_ATTACHMENTS), + [ + {"message_id": message_id, "attachment_id": attachment_id, "position": position} + for position, attachment_id in enumerate(attachment_ids) + ], + ) + + async def _attachment_ids_of(self, message_ids: Sequence[UUID]) -> dict[UUID, tuple[UUID, ...]]: + if not message_ids: + return {} + rows = await self._session.execute( + select(MESSAGE_ATTACHMENTS.c.message_id, MESSAGE_ATTACHMENTS.c.attachment_id) + .where(MESSAGE_ATTACHMENTS.c.message_id.in_(message_ids)) + .order_by(MESSAGE_ATTACHMENTS.c.message_id, MESSAGE_ATTACHMENTS.c.position) + ) + grouped: dict[UUID, list[UUID]] = {} + for message_id, attachment_id in rows: + grouped.setdefault(message_id, []).append(attachment_id) + return {message_id: tuple(items) for message_id, items in grouped.items()} + + async def _page( + self, statement: Select, table, after: Position | None, limit: int + ) -> list[Row]: + if after is not None: + statement = statement.where(_after_predicate(table.c.created_at, table.c.id, after)) + result = await self._session.execute( + statement.order_by(table.c.created_at, table.c.id).limit(limit + 1) + ) + return list(result) + + async def _members_of(self, chat_ids: Sequence[UUID]) -> dict[UUID, tuple[UUID, ...]]: + if not chat_ids: + return {} + rows = await self._session.execute( + select(CHAT_MEMBERS.c.chat_id, CHAT_MEMBERS.c.user_id) + .where(CHAT_MEMBERS.c.chat_id.in_(chat_ids)) + .order_by(CHAT_MEMBERS.c.chat_id, CHAT_MEMBERS.c.user_id) + ) + grouped: dict[UUID, list[UUID]] = {} + for chat_id, user_id in rows: + grouped.setdefault(chat_id, []).append(user_id) + return {chat_id: tuple(members) for chat_id, members in grouped.items()} + + # --- Вложения --------------------------------------------------------- + + async def create_queued_attachment( + self, + attachment_id: UUID, + chat_id: UUID, + owner_id: UUID, + staging_key: str, + object_key: str, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + pipeline_version: int, + max_attempts: int, + ) -> AttachmentRecord: + """Одна транзакция: queued-вложение + job + outbox + аудит. + + Вызывается только после успешного S3 PUT тела в staging. COMMIT этой + транзакции — граница, после которой API отвечает 202. Незавершённая + транзакция не оставляет ни строки, ни намерения. + """ + await insert_upload_work( + self._session, + attachment_id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + staging_key=staging_key, + object_key=object_key, + filename=filename, + content_type=content_type, + size_bytes=size_bytes, + sha256=sha256, + pipeline_version=pipeline_version, + max_attempts=max_attempts, + ) + row = ( + await self._session.execute( + select(*_attachment_columns()).where(ATTACHMENTS.c.id == attachment_id) + ) + ).one() + await self._session.commit() + return _attachment_record(row) + + async def delete_attachment(self, attachment_id: UUID) -> bool: + """Удаляет метаданные ещё не опубликованного вложения при компенсации. + + Готовые вложения удалять нельзя: на них могут ссылаться сообщения, + поэтому FK `RESTRICT` не даст сделать это случайно. + """ + try: + row = ( + await self._session.execute( + delete(ATTACHMENTS) + .where( + ATTACHMENTS.c.id == attachment_id, + ATTACHMENTS.c.status.in_(("queued", "processing")), + ) + .returning(ATTACHMENTS.c.id) + ) + ).first() + await self._session.commit() + return row is not None + except IntegrityError: + await self._session.rollback() + return False + + async def _variants_of( + self, attachment_id: UUID, pipeline_version: int + ) -> tuple[VariantRecord, ...]: + if pipeline_version <= 0: + return () + rows = ( + await self._session.execute( + select(VARIANTS) + .where( + VARIANTS.c.attachment_id == attachment_id, + VARIANTS.c.pipeline_version == pipeline_version, + ) + .order_by(VARIANTS.c.name) + ) + ).all() + return tuple(_variant_record(row) for row in rows) + + async def attachment_for_member( + self, attachment_id: UUID, user_id: UUID + ) -> AttachmentRecord | None: + """Вложение глазами участника чата; чужое неотличимо от отсутствующего.""" + row = ( + await self._session.execute( + select(*_attachment_columns()) + .select_from( + ATTACHMENTS.join( + CHAT_MEMBERS, CHAT_MEMBERS.c.chat_id == ATTACHMENTS.c.chat_id + ) + ) + .where(ATTACHMENTS.c.id == attachment_id, CHAT_MEMBERS.c.user_id == user_id) + .limit(1) + ) + ).first() + if row is None: + return None + return _attachment_record( + row, await self._variants_of(attachment_id, row.pipeline_version) + ) + + async def attachment_by_id(self, attachment_id: UUID) -> AttachmentRecord | None: + row = ( + await self._session.execute( + select(*_attachment_columns()).where(ATTACHMENTS.c.id == attachment_id).limit(1) + ) + ).first() + if row is None: + return None + return _attachment_record( + row, await self._variants_of(attachment_id, row.pipeline_version) + ) + + async def validate_message_attachments( + self, chat_id: UUID, attachment_ids: Sequence[UUID] + ) -> list[AttachmentRecord]: + """Проверяет, что все вложения существуют и принадлежат чату. + + Неизвестное или чужое — 404; failed — 409. queued/processing допустимы + с lab5: сообщение можно отправить до завершения фоновой обработки. + Порядок возвращаемых записей совпадает с порядком `attachment_ids`. + """ + if not attachment_ids: + return [] + rows = ( + await self._session.execute( + select(*_attachment_columns()).where(ATTACHMENTS.c.id.in_(attachment_ids)) + ) + ).all() + found = {row.id: _attachment_record(row) for row in rows} + if any(attachment_id not in found for attachment_id in attachment_ids): + raise AttachmentUnknown() + if any(found[attachment_id].chat_id != chat_id for attachment_id in attachment_ids): + raise AttachmentUnknown() + if any( + found[attachment_id].status == ATTACHMENT_STATUS_FAILED + for attachment_id in attachment_ids + ): + raise AttachmentNotReady() + return [found[attachment_id] for attachment_id in attachment_ids] + + async def list_variant_keys(self) -> set[str]: + """Ключи всех опубликованных вариантов: reconciler не считает их мусором.""" + rows = await self._session.execute(select(VARIANTS.c.object_key)) + return {str(value) for value in rows.scalars().all()} + + async def list_attachment_states(self) -> list[AttachmentRecord]: + """Все строки метаданных для reconciler (без байтов и вариантов).""" + rows = ( + await self._session.execute( + select(*_attachment_columns()).order_by(ATTACHMENTS.c.created_at, ATTACHMENTS.c.id) + ) + ).all() + return [_attachment_record(row) for row in rows] + + async def delete_attachment_if_stale(self, attachment_id: UUID, before) -> bool: + """Удаляет только failed-строку старше `before`. + + queued/processing — текущая работа, ready — данные пользователя. + Автоматическое удаление этих строк было бы потерей. + """ + try: + row = ( + await self._session.execute( + delete(ATTACHMENTS) + .where( + ATTACHMENTS.c.id == attachment_id, + ATTACHMENTS.c.status == ATTACHMENT_STATUS_FAILED, + ATTACHMENTS.c.updated_at < before, + ) + .returning(ATTACHMENTS.c.id) + ) + ).first() + await self._session.commit() + return row is not None + except IntegrityError: + await self._session.rollback() + return False + diff --git a/src/messenger/routers/__init__.py b/src/messenger/routers/__init__.py new file mode 100644 index 00000000..acd99363 --- /dev/null +++ b/src/messenger/routers/__init__.py @@ -0,0 +1 @@ +"""Маршруты HTTP API: только разбор запроса, вызов зависимостей и сборка ответа.""" diff --git a/src/messenger/routers/attachments.py b/src/messenger/routers/attachments.py new file mode 100644 index 00000000..6a9f9b41 --- /dev/null +++ b/src/messenger/routers/attachments.py @@ -0,0 +1,214 @@ +"""Загрузка, метаданные и авторизованная выдача вложений. + +Lab5: upload принимает байты в устойчивый S3 staging, одной транзакцией пишет +attachment(queued) + job + outbox и отвечает 202. Тяжёлая обработка выполняется +отдельным worker-ом; API никогда не делает resize синхронно. +""" +from __future__ import annotations + +import logging +from typing import Annotated, AsyncIterator +from uuid import UUID, uuid4 + +import anyio +from fastapi import APIRouter, Path, Query, Request, status +from fastapi.responses import StreamingResponse + +from ..access import visible_chat +from ..deps import ConfigDep, CurrentUser, RepositoryDep, StorageDep +from ..domain import VariantRecord +from ..errors import conflict, invalid_request, not_found, payload_too_large, unavailable, unsupported_media_type +from ..models import Attachment +from ..storage import StorageObjectMissing, StorageUnavailable +from ..upload import receive_image + +router = APIRouter(prefix="/api/v1", tags=["attachments"]) +logger = logging.getLogger("messenger") + +ALLOWED_CONTENT_TYPES = ("image/png", "image/jpeg") +ALLOWED_VARIANTS = ("thumbnail", "crop", "optimized") +FORBIDDEN_FILENAME_CHARS = ("/", "\\", "\x00") +NOT_READY_STATUSES = ("queued", "processing", "failed") + + +def declared_content_type(value: str | None) -> str | None: + """Медиа-тип без параметров; None — если он не PNG/JPEG.""" + if not value: + return None + media_type = value.split(";", 1)[0].strip().lower() + return media_type if media_type in ALLOWED_CONTENT_TYPES else None + + +def validate_filename(value: str) -> str: + """filename — только отображаемое имя, но опасные символы отклоняются.""" + if not value or len(value) > 255: + raise invalid_request("filename должен содержать 1..255 code points") + if any(char in value for char in FORBIDDEN_FILENAME_CHARS): + raise invalid_request("filename не должен содержать /, \\ или NUL") + if any(ord(char) < 0x20 or ord(char) == 0x7F for char in value): + raise invalid_request("filename не должен содержать управляющие символы") + return value + + +async def _stream_body(body) -> AsyncIterator[bytes]: + """Читает S3-поток в рабочем потоке, не блокируя event loop.""" + try: + while True: + chunk = await anyio.to_thread.run_sync(body.read, 64 * 1024) + if not chunk: + break + yield chunk + finally: + close = getattr(body, "close", None) + if close is not None: + close() + + +@router.post( + "/chats/{chat_id}/attachments", + status_code=status.HTTP_202_ACCEPTED, + summary="Принять PNG/JPEG в staging и поставить задание в очередь", +) +async def upload_attachment( + chat_id: Annotated[UUID, Path()], + request: Request, + repository: RepositoryDep, + user: CurrentUser, + config: ConfigDep, + storage: StorageDep, + filename: Annotated[str, Query(min_length=1, max_length=255)], +) -> Attachment: + """Сначала доступ к чату, затем лимиты и MIME; 202 — после staging + COMMIT. + + 202 означает: полный проверенный оригинал лежит в приватном staging, а + намерение обработать его (job + outbox) закоммичено. Оно переживёт остановку + API, worker и broker; клиент узнаёт готовность через GET метаданных. + """ + await visible_chat(repository, chat_id, user) + validate_filename(filename) + expected_type = declared_content_type(request.headers.get("content-type")) + if expected_type is None: + raise unsupported_media_type("Content-Type должен быть image/png или image/jpeg") + + raw_length = request.headers.get("content-length") + if raw_length is not None: + try: + length = int(raw_length) + except ValueError: + raise invalid_request("Content-Length должен быть целым числом") from None + if length < 0: + raise invalid_request("Content-Length не может быть отрицательным") + if length > config.max_upload_bytes: + raise payload_too_large(f"Файл больше {config.max_upload_bytes} байт") + + received = await receive_image( + request, expected_content_type=expected_type, config=config + ) + try: + attachment_id = uuid4() + staging_key = storage.staging_key(str(attachment_id)) + object_key = storage.original_key(str(attachment_id), config.pipeline_version) + with open(received.path, "rb") as file: + await storage.put( + staging_key, + file, + content_type=received.info.content_type, + size_bytes=received.size_bytes, + sha256=received.sha256, + md5_hex=received.md5, + ) + if config.attachment_fault_mode == "after_s3_put": + # Учебное внедрение отказа между staging PUT и COMMIT задания. + # Строки job нет, staging-объект уберёт reconciler после retention. + raise StorageUnavailable("Смоделирован отказ после staging PUT") + try: + queued = await repository.create_queued_attachment( + attachment_id=attachment_id, + chat_id=chat_id, + owner_id=user.id, + staging_key=staging_key, + object_key=object_key, + filename=filename, + content_type=received.info.content_type, + size_bytes=received.size_bytes, + sha256=received.sha256, + pipeline_version=config.pipeline_version, + max_attempts=config.job_max_attempts, + ) + except BaseException: + # COMMIT не прошёл: принятая работа не подтверждена, staging не + # должен пережить запрос. Ошибку удаления доберёт reconciler. + try: + await storage.delete(staging_key) + except Exception: + logger.warning( + "staging_compensation_failed", + extra={"fields": {"attachment_id": str(attachment_id)}}, + ) + raise + return Attachment.from_record(queued) + finally: + received.cleanup() + + +@router.get("/attachments/{attachment_id}", summary="Текущее состояние вложения") +async def get_attachment( + attachment_id: Annotated[UUID, Path()], + repository: RepositoryDep, + user: CurrentUser, +) -> Attachment: + """Возвращает queued/processing/ready/failed; готовность — отдельный вопрос. + + Метаданные доступны на любом состоянии: клиент опрашивает этот endpoint. + Скачивание content для неготового вложения — 409. + """ + record = await repository.attachment_for_member(attachment_id, user.id) + if record is None: + raise not_found("Вложение не найдено") + return Attachment.from_record(record) + + +@router.get("/attachments/{attachment_id}/content", summary="Скачать байты через API") +async def download_attachment( + attachment_id: Annotated[UUID, Path()], + repository: RepositoryDep, + user: CurrentUser, + storage: StorageDep, + variant: Annotated[str | None, Query()] = None, +) -> StreamingResponse: + """Оригинал или готовый вариант отдаётся потоком, без redirect/presigned URL.""" + name = "original" if variant in (None, "", "original") else variant + if name not in ("original",) + ALLOWED_VARIANTS: + raise invalid_request("variant должен быть original, thumbnail, crop или optimized") + record = await repository.attachment_for_member(attachment_id, user.id) + if record is None: + raise not_found("Вложение не найдено") + if record.status in NOT_READY_STATUSES: + raise conflict("Вложение ещё не готово") + if name == "original": + object_key = record.object_key + content_type = record.content_type + size_bytes = record.size_bytes + else: + chosen = next((item for item in record.variants if item.name == name), None) + if chosen is None: + raise not_found("Вариант вложения не найден") + object_key = chosen.object_key + content_type = chosen.content_type + size_bytes = chosen.size_bytes + try: + stored = await storage.open(object_key) + except StorageObjectMissing: + raise not_found("Объект вложения отсутствует") from None + except StorageUnavailable: + raise unavailable("Хранилище вложений недоступно") from None + if stored.size_bytes != size_bytes: + close = getattr(stored.body, "close", None) + if close is not None: + close() + raise unavailable("Размер объекта в хранилище не совпадает с метаданными") + return StreamingResponse( + _stream_body(stored.body), + media_type=content_type, + headers={"Content-Length": str(size_bytes)}, + ) diff --git a/src/messenger/routers/auth.py b/src/messenger/routers/auth.py new file mode 100644 index 00000000..c9ccb566 --- /dev/null +++ b/src/messenger/routers/auth.py @@ -0,0 +1,51 @@ +"""Вход по Basic и отзыв текущей сессии.""" +from __future__ import annotations + +from fastapi import APIRouter, Response, status + +from ..deps import BasicCredentialsDep, BearerToken, RepositoryDep, SessionsDep +from ..errors import BASIC_CHALLENGE, unauthorized +from ..models import Session +from ..security import spend_verification_time, verify_password + +router = APIRouter(prefix="/api/v1/auth/sessions", tags=["auth"]) + +INVALID_CREDENTIALS = "Неверный username или пароль" + + +@router.post("", status_code=status.HTTP_201_CREATED, summary="Вход: Basic username:password") +async def create_session( + credentials: BasicCredentialsDep, + repository: RepositoryDep, + sessions: SessionsDep, +) -> Session: + """Проверяет пароль и выдаёт opaque-токен сессии; тело запроса не читается. + + Неизвестный пользователь и неверный пароль неразличимы: одинаковый 401 с + Basic-челленджем и одинаковая затрата времени на KDF. + """ + found = await repository.credentials_for(credentials.username) + if found is None: + await spend_verification_time() + raise unauthorized(INVALID_CREDENTIALS, challenge=BASIC_CHALLENGE) + user, password_hash = found + if not await verify_password(credentials.password, password_hash): + raise unauthorized(INVALID_CREDENTIALS, challenge=BASIC_CHALLENGE) + issued = await sessions.create(user.id) + return Session.from_issued(issued, user) + + +@router.delete( + "/current", + status_code=status.HTTP_204_NO_CONTENT, + summary="Отозвать текущую сессию", +) +async def revoke_session(token: BearerToken, sessions: SessionsDep) -> Response: + """Отзыв действует сразу: тот же токен после него даёт 401. + + Ответ пустой (204), поэтому клиент не получает ни тела, ни нового состояния + сессии. Отзыв не трогает PostgreSQL: удаление ключа в Valkey атомарно. + """ + if not await sessions.revoke(token): + raise unauthorized("Сессия не найдена или истекла") + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/src/messenger/routers/chats.py b/src/messenger/routers/chats.py new file mode 100644 index 00000000..dee42e57 --- /dev/null +++ b/src/messenger/routers/chats.py @@ -0,0 +1,55 @@ +"""Создание, список и чтение чатов.""" +from __future__ import annotations + +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Path, Query, status + +from ..access import visible_chat +from ..cursor import KIND_CHATS +from ..deps import CursorsDep, CurrentUser, RepositoryDep +from ..domain import UnknownMembers +from ..errors import invalid_request +from ..models import DEFAULT_LIMIT, MAX_LIMIT, Chat, ChatPage, CreateChat +from ..pagination import build_next_cursor, decoded_cursor + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +@router.post("", status_code=status.HTTP_201_CREATED, summary="Создать чат") +async def create_chat(payload: CreateChat, repository: RepositoryDep, user: CurrentUser) -> Chat: + """Создаёт чат; состав участников задаётся при создании и не меняется.""" + if user.id not in payload.member_ids: + raise invalid_request("Создатель чата обязан входить в member_ids") + try: + record = await repository.create_chat(payload.title, payload.member_ids) + except UnknownMembers: + raise invalid_request("member_ids содержит неизвестных пользователей") from None + return Chat.from_record(record) + + +@router.get("", summary="Чаты текущего пользователя") +async def list_chats( + repository: RepositoryDep, + cursors: CursorsDep, + user: CurrentUser, + limit: Annotated[int, Query(ge=1, le=MAX_LIMIT)] = DEFAULT_LIMIT, + cursor: Annotated[str | None, Query(min_length=1)] = None, +) -> ChatPage: + """Список видимых чатов по (created_at, id) по возрастанию.""" + scope = str(user.id) + after = decoded_cursor(cursors, cursor, kind=KIND_CHATS, scope=scope) + records, has_more = await repository.list_chats(user.id, after, limit) + return ChatPage.from_records( + records, build_next_cursor(cursors, KIND_CHATS, scope, records, has_more) + ) + + +@router.get("/{chat_id}", summary="Получить доступный чат") +async def read_chat( + chat_id: Annotated[UUID, Path()], + repository: RepositoryDep, + user: CurrentUser, +) -> Chat: + return Chat.from_record(await visible_chat(repository, chat_id, user)) diff --git a/src/messenger/routers/health.py b/src/messenger/routers/health.py new file mode 100644 index 00000000..7ac6910f --- /dev/null +++ b/src/messenger/routers/health.py @@ -0,0 +1,46 @@ +"""Проверки живости и готовности.""" +from __future__ import annotations + +from fastapi import APIRouter, Request + +from ..deps import DatabaseDep, SessionsDep, StorageDep +from ..errors import unavailable +from ..models import Health +from ..schema import SCHEMA_REVISION + +router = APIRouter(prefix="/health", tags=["health"]) + + +@router.get("/live", summary="Процесс жив") +async def liveness() -> Health: + """Проверяет только то, что процесс обслуживает запросы. + + Зависимости здесь намеренно не проверяются: их отказ — не повод убивать + контейнер и устраивать цикл перезапусков. + """ + return Health() + + +@router.get("/ready", summary="Готов принимать трафик") +async def readiness( + request: Request, + database: DatabaseDep, + sessions: SessionsDep, + storage: StorageDep, +) -> Health: + """Готовность = процесс не на остановке и критические зависимости отвечают. + + Отказ зависимости превращается в 503 общим обработчиком: у готовности и у + бизнес-операций один и тот же признак неработоспособности. + """ + if getattr(request.app.state, "draining", False): + raise unavailable("Сервис завершает работу и не принимает новые запросы") + await database.ping() + revision = await database.schema_revision() + if revision != SCHEMA_REVISION: + raise unavailable("Схема базы данных ещё не готова") + await sessions.ping() + # S3 — критичная зависимость обязательного синхронного upload/download. + # Без него API не должен получать трафик от балансировщика. + await storage.ping() + return Health() \ No newline at end of file diff --git a/src/messenger/routers/messages.py b/src/messenger/routers/messages.py new file mode 100644 index 00000000..9d614570 --- /dev/null +++ b/src/messenger/routers/messages.py @@ -0,0 +1,74 @@ +"""Отправка и чтение истории сообщений.""" +from __future__ import annotations + +from typing import Annotated +from uuid import UUID + +from fastapi import APIRouter, Header, Path, Query, status + +from ..access import visible_chat +from ..cursor import KIND_MESSAGES +from ..deps import CursorsDep, CurrentUser, RepositoryDep +from ..domain import AttachmentNotReady, AttachmentUnknown, IdempotencyConflict +from ..errors import conflict, not_found +from ..idempotency import message_request_hash, require_idempotency_key +from ..models import DEFAULT_LIMIT, MAX_LIMIT, CreateMessage, Message, MessagePage +from ..pagination import build_next_cursor, decoded_cursor + +router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"]) + + +@router.post("", status_code=status.HTTP_201_CREATED, summary="Отправить сообщение") +async def create_message( + chat_id: Annotated[UUID, Path()], + payload: CreateMessage, + repository: RepositoryDep, + user: CurrentUser, + idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key")] = None, +) -> Message: + """Отправитель выводится из идентификации; sender_id в теле не принимается. + + Порядок проверок контракта: аутентификация (зависимость user), доступ к + чату (404 для чужого), затем ключ идемпотентности. Повтор с тем же ключом + и телом возвращает то же сообщение и тот же код 201. + """ + await visible_chat(repository, chat_id, user) + try: + await repository.validate_message_attachments(chat_id, payload.attachment_ids) + except AttachmentUnknown: + # Чужое и несуществующее вложение неотличимы от отсутствующего (404). + raise not_found("Вложение не найдено") from None + except AttachmentNotReady: + raise conflict("Вложение ещё не готово") from None + key = require_idempotency_key(idempotency_key) + try: + record = await repository.append_message_idempotent( + chat_id, + user.id, + payload.text, + payload.attachment_ids, + key, + message_request_hash(payload.text, payload.attachment_ids), + ) + except IdempotencyConflict: + raise conflict("Idempotency-Key уже использован с другим телом запроса") from None + return Message.from_record(record) + + +@router.get("", summary="Сообщения по (created_at, id) по возрастанию") +async def list_messages( + chat_id: Annotated[UUID, Path()], + repository: RepositoryDep, + cursors: CursorsDep, + user: CurrentUser, + limit: Annotated[int, Query(ge=1, le=MAX_LIMIT)] = DEFAULT_LIMIT, + cursor: Annotated[str | None, Query(min_length=1)] = None, +) -> MessagePage: + """Keyset-пагинация: страница читается по индексу, предыдущие не повторяются.""" + await visible_chat(repository, chat_id, user) + scope = str(chat_id) + after = decoded_cursor(cursors, cursor, kind=KIND_MESSAGES, scope=scope) + records, has_more = await repository.list_messages(chat_id, after, limit) + return MessagePage.from_records( + records, build_next_cursor(cursors, KIND_MESSAGES, scope, records, has_more) + ) \ No newline at end of file diff --git a/src/messenger/routers/users.py b/src/messenger/routers/users.py new file mode 100644 index 00000000..86dfb6bc --- /dev/null +++ b/src/messenger/routers/users.py @@ -0,0 +1,34 @@ +"""Регистрация и текущий пользователь.""" +from __future__ import annotations + +from fastapi import APIRouter, status + +from ..deps import CurrentUser, RepositoryDep +from ..errors import conflict +from ..models import CreateUser, User +from ..security import hash_password + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + +USERNAME_TAKEN = "username уже занят" + + +@router.post("", status_code=status.HTTP_201_CREATED, summary="Создать пользователя") +async def register_user(payload: CreateUser, repository: RepositoryDep) -> User: + """Регистрация публична; username уникален, повтор даёт 409.""" + if await repository.username_taken(payload.username): + # Предпроверка экономит KDF на заведомо занятом username; гонку закрывает + # уникальный индекс в create_user, поэтому оба пути дают один и тот же 409. + raise conflict(USERNAME_TAKEN) + record = await repository.create_user( + payload.username, payload.display_name, await hash_password(payload.password) + ) + if record is None: + raise conflict(USERNAME_TAKEN) + return User.from_record(record) + + +@router.get("/me", summary="Текущий пользователь") +async def read_current_user(user: CurrentUser) -> User: + """Владелец Bearer-сессии; пароль и хэш пароля наружу не выходят.""" + return User.from_record(user) diff --git a/src/messenger/schema.py b/src/messenger/schema.py new file mode 100644 index 00000000..5897d1c2 --- /dev/null +++ b/src/messenger/schema.py @@ -0,0 +1,376 @@ +"""Описание таблиц PostgreSQL: один источник истины для запросов и миграций.""" +from __future__ import annotations + +from sqlalchemy import ( + BigInteger, + CheckConstraint, + Column, + ForeignKey, + Index, + Integer, + MetaData, + SmallInteger, + Table, + Text, + UniqueConstraint, + text, +) +from sqlalchemy.dialects.postgresql import TIMESTAMP, UUID + +METADATA = MetaData() + +# Версия схемы, которую процесс считает совместимой с собой. Готовность +# проверяет её в alembic_version, поэтому API не начинает принимать трафик +# раньше однократной миграции. +SCHEMA_REVISION = "0004_async_pipeline" + +USERS = Table( + "users", + METADATA, + Column("id", UUID(as_uuid=True), primary_key=True), + Column("username", Text, nullable=False), + Column("display_name", Text, nullable=False), + Column("password_hash", Text, nullable=False), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + UniqueConstraint("username", name="uq_users_username"), + CheckConstraint(r"username ~ '^[a-z0-9_]{3,32}$'", name="ck_users_username_format"), + CheckConstraint( + "char_length(display_name) BETWEEN 1 AND 100", name="ck_users_display_name_length" + ), +) + +CHATS = Table( + "chats", + METADATA, + Column("id", UUID(as_uuid=True), primary_key=True), + Column("title", Text, nullable=False), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="ck_chats_title_length"), +) + +CHAT_MEMBERS = Table( + "chat_members", + METADATA, + Column( + "chat_id", + UUID(as_uuid=True), + ForeignKey("chats.id", name="fk_chat_members_chat", ondelete="CASCADE"), + primary_key=True, + ), + Column( + "user_id", + UUID(as_uuid=True), + ForeignKey("users.id", name="fk_chat_members_user", ondelete="RESTRICT"), + primary_key=True, + ), + Index("ix_chat_members_user_id_chat_id", "user_id", "chat_id"), +) + +MESSAGES = Table( + "messages", + METADATA, + Column("id", UUID(as_uuid=True), primary_key=True), + Column( + "chat_id", + UUID(as_uuid=True), + ForeignKey("chats.id", name="fk_messages_chat", ondelete="CASCADE"), + nullable=False, + ), + Column( + "sender_id", + UUID(as_uuid=True), + ForeignKey("users.id", name="fk_messages_sender", ondelete="RESTRICT"), + nullable=False, + ), + Column("text", Text, nullable=False), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + # Пустой текст допустим только вместе с вложениями; это правило проверяет + # API, потому что CHECK не может ссылаться на другую таблицу. + CheckConstraint("char_length(text) <= 4000", name="ck_messages_text_length"), + # Порядок истории (created_at, id) и keyset-предикат обслуживает один индекс. + Index("ix_messages_chat_id_created_at_id", "chat_id", "created_at", "id"), +) + +# Общая идемпотентность отправки сообщения. Область уникальности — +# (пользователь, чат, ключ); тело запроса сравнивается по каноническому хэшу. +# Строка и само сообщение пишутся одной транзакцией, поэтому подтверждённый +# ответ и побочный эффект не могут разойтись. +MESSAGE_IDEMPOTENCY = Table( + "message_idempotency", + METADATA, + Column( + "user_id", + UUID(as_uuid=True), + ForeignKey("users.id", name="fk_message_idempotency_user", ondelete="CASCADE"), + primary_key=True, + ), + Column( + "chat_id", + UUID(as_uuid=True), + ForeignKey("chats.id", name="fk_message_idempotency_chat", ondelete="CASCADE"), + primary_key=True, + ), + Column("idempotency_key", Text, primary_key=True), + Column("request_hash", Text, nullable=False), + Column( + "message_id", + UUID(as_uuid=True), + ForeignKey("messages.id", name="fk_message_idempotency_message", ondelete="CASCADE"), + nullable=False, + ), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + CheckConstraint( + "char_length(idempotency_key) BETWEEN 1 AND 128", + name="ck_message_idempotency_key_length", + ), + Index("ix_message_idempotency_created_at", "created_at"), +) + +# Вложение: метаданные в PostgreSQL, байты — в приватном S3. +# object_key — конечный ключ оригинала (final prefix), staging_key — приватный +# staging-ключ принятого тела. pipeline_version — версия конвейера, результат +# которой сейчас опубликован (0 — результата ещё нет). +ATTACHMENTS = Table( + "attachments", + METADATA, + Column("id", UUID(as_uuid=True), primary_key=True), + Column( + "chat_id", + UUID(as_uuid=True), + ForeignKey("chats.id", name="fk_attachments_chat", ondelete="CASCADE"), + nullable=False, + ), + Column( + "owner_id", + UUID(as_uuid=True), + ForeignKey("users.id", name="fk_attachments_owner", ondelete="RESTRICT"), + nullable=False, + ), + Column("object_key", Text, nullable=False), + Column("staging_key", Text, nullable=False), + Column("pipeline_version", Integer, nullable=False, server_default=text("0")), + Column("filename", Text, nullable=False), + Column("content_type", Text, nullable=False), + Column("size_bytes", BigInteger, nullable=False), + Column("sha256", Text, nullable=False), + Column("status", Text, nullable=False, server_default=text("'queued'")), + Column("error_code", Text, nullable=True), + Column("ready_at", TIMESTAMP(timezone=True), nullable=True), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + Column("updated_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + UniqueConstraint("object_key", name="uq_attachments_object_key"), + UniqueConstraint("staging_key", name="uq_attachments_staging_key"), + CheckConstraint("char_length(filename) BETWEEN 1 AND 255", name="ck_attachments_filename_length"), + CheckConstraint("content_type IN ('image/png','image/jpeg')", name="ck_attachments_content_type"), + CheckConstraint( + "size_bytes > 0 AND size_bytes <= 10485760", name="ck_attachments_size_bytes" + ), + CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="ck_attachments_sha256"), + CheckConstraint( + "status IN ('queued','processing','ready','failed')", name="ck_attachments_status" + ), + CheckConstraint("pipeline_version >= 0", name="ck_attachments_pipeline_version"), + CheckConstraint( + "error_code IS NULL OR char_length(error_code) BETWEEN 1 AND 64", + name="ck_attachments_error_code", + ), + Index("ix_attachments_chat_id_created_at_id", "chat_id", "created_at", "id"), + Index("ix_attachments_status_updated_at", "status", "updated_at"), +) + +# Одна логическая работа на (attachment, pipeline_version). job_id не меняется +# при повторной доставке; attempts увеличивается только при управляемом +# захвате lease. replay_of ссылается на исходную failed-работу при явном replay: +# replay создаёт новый job_id, но ключ эффекта остаётся прежним. +JOBS = Table( + "jobs", + METADATA, + Column("id", UUID(as_uuid=True), primary_key=True), + Column( + "attachment_id", + UUID(as_uuid=True), + ForeignKey("attachments.id", name="fk_jobs_attachment", ondelete="CASCADE"), + nullable=False, + ), + Column("pipeline_version", Integer, nullable=False), + Column("type", Text, nullable=False, server_default=text("'attachment.process.v1'")), + Column("state", Text, nullable=False, server_default=text("'queued'")), + Column("attempts", Integer, nullable=False, server_default=text("0")), + Column("max_attempts", Integer, nullable=False), + Column("lease_token", UUID(as_uuid=True), nullable=True), + Column("worker_id", Text, nullable=True), + Column("lease_expires_at", TIMESTAMP(timezone=True), nullable=True), + Column("next_attempt_at", TIMESTAMP(timezone=True), nullable=True), + Column("last_error_code", Text, nullable=True), + Column( + "replay_of", + UUID(as_uuid=True), + ForeignKey("jobs.id", name="fk_jobs_replay_of", ondelete="SET NULL"), + nullable=True, + ), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + Column("updated_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + Column("finished_at", TIMESTAMP(timezone=True), nullable=True), + CheckConstraint("pipeline_version >= 1", name="ck_jobs_pipeline_version"), + CheckConstraint( + "state IN ('queued','processing','ready','failed','superseded')", + name="ck_jobs_state", + ), + CheckConstraint("attempts >= 0", name="ck_jobs_attempts"), + CheckConstraint("max_attempts >= 1", name="ck_jobs_max_attempts"), + CheckConstraint( + "last_error_code IS NULL OR char_length(last_error_code) BETWEEN 1 AND 64", + name="ck_jobs_last_error_code", + ), + Index("ix_jobs_attachment_id", "attachment_id"), + Index("ix_jobs_state_next_attempt_at", "state", "next_attempt_at"), + Index("ix_jobs_lease_expires_at", "lease_expires_at"), + # Один первичный job на версию; replay-задачи имеют replay_of и не конфликтуют. + Index( + "uq_jobs_initial_attachment_version", + "attachment_id", + "pipeline_version", + unique=True, + postgresql_where=text("replay_of IS NULL"), + ), +) + +# Transactional outbox: строка коммитится вместе с заданием и публикуется +# отдельным relay-процессом. Именно она закрывает разрыв COMMIT/publish. +OUTBOX = Table( + "outbox", + METADATA, + Column("id", BigInteger, primary_key=True, autoincrement=True), + Column( + "job_id", + UUID(as_uuid=True), + ForeignKey("jobs.id", name="fk_outbox_job", ondelete="CASCADE"), + nullable=False, + ), + Column("available_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + Column("published_at", TIMESTAMP(timezone=True), nullable=True), + Column("attempts", Integer, nullable=False, server_default=text("0")), + Column("last_error", Text, nullable=True), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + Index("ix_outbox_due", "published_at", "available_at"), + Index( + "uq_outbox_pending_job", + "job_id", + unique=True, + postgresql_where=text("published_at IS NULL"), + ), +) + +# Варианты изображения. Ключ эффекта — (attachment_id, pipeline_version, name): +# повторная доставка и повторный запуск версии не создают вторую строку. +# Оригинал хранится в attachments.object_key и в этой таблице не дублируется. +VARIANTS = Table( + "variants", + METADATA, + Column( + "attachment_id", + UUID(as_uuid=True), + ForeignKey("attachments.id", name="fk_variants_attachment", ondelete="CASCADE"), + primary_key=True, + ), + Column("pipeline_version", Integer, primary_key=True), + Column("name", Text, primary_key=True), + Column("content_type", Text, nullable=False), + Column("object_key", Text, nullable=False), + Column("size_bytes", BigInteger, nullable=False), + Column("sha256", Text, nullable=False), + Column("width", Integer, nullable=False), + Column("height", Integer, nullable=False), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + CheckConstraint("pipeline_version >= 1", name="ck_variants_pipeline_version"), + CheckConstraint("name IN ('thumbnail','crop','optimized')", name="ck_variants_name"), + CheckConstraint( + "content_type IN ('image/jpeg','image/png','image/webp')", + name="ck_variants_content_type", + ), + CheckConstraint("size_bytes > 0", name="ck_variants_size_bytes"), + CheckConstraint("width > 0 AND height > 0", name="ck_variants_dimensions"), + CheckConstraint("sha256 ~ '^[a-f0-9]{64}$'", name="ck_variants_sha256"), + UniqueConstraint("object_key", name="uq_variants_object_key"), + Index("ix_variants_attachment_id", "attachment_id"), +) + +# Аудит переходов job: используется оператором и как evidence. Секретов здесь нет. +JOB_EVENTS = Table( + "job_events", + METADATA, + Column("id", BigInteger, primary_key=True, autoincrement=True), + Column( + "job_id", + UUID(as_uuid=True), + ForeignKey("jobs.id", name="fk_job_events_job", ondelete="CASCADE"), + nullable=False, + ), + Column("from_state", Text, nullable=True), + Column("to_state", Text, nullable=False), + Column("attempt", Integer, nullable=True), + Column("error_code", Text, nullable=True), + Column("worker_id", Text, nullable=True), + Column("detail", Text, nullable=True), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + Index("ix_job_events_job_id_created_at", "job_id", "created_at"), +) + +# Реестр явных операторских replay: кто, когда и какую failed-задачу повторил. +JOB_REPLAYS = Table( + "job_replays", + METADATA, + Column("id", UUID(as_uuid=True), primary_key=True), + Column( + "source_job_id", + UUID(as_uuid=True), + ForeignKey("jobs.id", name="fk_job_replays_source", ondelete="CASCADE"), + nullable=False, + ), + Column( + "replay_job_id", + UUID(as_uuid=True), + ForeignKey("jobs.id", name="fk_job_replays_replay", ondelete="CASCADE"), + nullable=False, + ), + Column("requested_by", Text, nullable=False), + Column("reason", Text, nullable=False), + Column("created_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + UniqueConstraint("replay_job_id", name="uq_job_replays_replay_job"), + CheckConstraint("char_length(requested_by) BETWEEN 1 AND 128", name="ck_job_replays_requested_by"), + CheckConstraint("char_length(reason) BETWEEN 1 AND 512", name="ck_job_replays_reason"), +) + +# Тестовый fault-injection: одноразовый флаг в БД переживает restart контейнера +# и потребляется ровно один раз. В prod-окружении флаги запрещены конфигурацией. +FAULT_FLAGS = Table( + "fault_flags", + METADATA, + Column("kind", Text, primary_key=True), + Column("remaining", Integer, nullable=False), + Column("updated_at", TIMESTAMP(timezone=True), nullable=False, server_default=text("now()")), + CheckConstraint("remaining >= 0", name="ck_fault_flags_remaining"), +) + +# Связь сообщения с вложениями. position сохраняет порядок, в котором клиент +# передал attachment_ids; FK не даёт сослаться на несуществующее вложение. +MESSAGE_ATTACHMENTS = Table( + "message_attachments", + METADATA, + Column( + "message_id", + UUID(as_uuid=True), + ForeignKey("messages.id", name="fk_message_attachments_message", ondelete="CASCADE"), + primary_key=True, + ), + Column( + "attachment_id", + UUID(as_uuid=True), + ForeignKey("attachments.id", name="fk_message_attachments_attachment", ondelete="RESTRICT"), + primary_key=True, + ), + Column("position", SmallInteger, nullable=False), + UniqueConstraint("message_id", "position", name="uq_message_attachments_position"), + CheckConstraint("position >= 0 AND position < 10", name="ck_message_attachments_position"), + Index("ix_message_attachments_attachment_id", "attachment_id"), +) diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 00000000..ea0e343a --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,119 @@ +"""Пароли и токены: argon2id для пароля, случайный opaque-токен для доступа. + +Пароль не хранится и не передаётся дальше: в базе лежит только строка argon2id. +KDF считает десятки миллисекунд, поэтому вызывается в рабочем потоке — цикл +событий не должен ждать хэширование. +""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import secrets +from functools import lru_cache + +import anyio +from argon2 import PasswordHasher +from argon2.exceptions import Argon2Error, InvalidHashError + +# Минимальные параметры Argon2id из OWASP Password Storage Cheat Sheet: +# m=19456 КиБ (19 МиБ), t=2, p=1. Память на проверку ограничена осознанно. +TIME_COST = 2 +MEMORY_COST_KIB = 19456 +PARALLELISM = 1 +HASH_BYTES = 32 +SALT_BYTES = 16 + +TOKEN_BYTES = 32 +TOKEN_MIN_LENGTH = 32 + +BASIC_SCHEME = "basic" +BEARER_SCHEME = "bearer" + +_DUMMY_PASSWORD = "messenger-dummy-password-never-matches" + +_hasher = PasswordHasher( + time_cost=TIME_COST, + memory_cost=MEMORY_COST_KIB, + parallelism=PARALLELISM, + hash_len=HASH_BYTES, + salt_len=SALT_BYTES, +) + + +def _hash(password: str) -> str: + return _hasher.hash(password) + + +def _verify(password: str, stored_hash: str) -> bool: + """True только при совпадении; неподходящий или испорченный хэш — тоже «нет».""" + try: + return _hasher.verify(stored_hash, password) + except (Argon2Error, InvalidHashError): + return False + + +@lru_cache(maxsize=1) +def _dummy_hash() -> str: + """Хэш-заглушка: нужен, чтобы ответ про неизвестного пользователя стоил столько же.""" + return _hash(_DUMMY_PASSWORD) + + +async def hash_password(password: str) -> str: + """Хэширует пароль; длина пароля ограничена контрактом, усечения нет.""" + return await anyio.to_thread.run_sync(_hash, password) + + +async def verify_password(password: str, stored_hash: str) -> bool: + return await anyio.to_thread.run_sync(_verify, password, stored_hash) + + +async def spend_verification_time() -> None: + """Выполняет KDF впустую: время ответа не выдаёт существование пользователя.""" + await anyio.to_thread.run_sync(_verify, _DUMMY_PASSWORD, _dummy_hash()) + + +def new_token() -> str: + """Opaque-токен сессии: 256 бит энтропии, длина не меньше требуемых 32 символов.""" + return secrets.token_urlsafe(TOKEN_BYTES) + + +def token_fingerprint(token: str) -> str: + """Ключ, под которым сессия лежит в хранилище: сам токен там не появляется.""" + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _credentials(header: str | None, scheme: str) -> str | None: + """Хвост заголовка Authorization, если схема совпала; иначе None.""" + if not header: + return None + name, separator, value = header.strip().partition(" ") + if not separator or name.lower() != scheme: + return None + value = value.strip() + return value or None + + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """Разбирает `Authorization: Basic ...` в пару (username, password). + + None означает «заголовка нет, схема не та или значение испорчено»; наружу все + эти случаи дают один и тот же 401, поэтому различать их незачем. + """ + encoded = _credentials(header, BASIC_SCHEME) + if encoded is None: + return None + try: + decoded = base64.b64decode(encoded + "=" * (-len(encoded) % 4), validate=True) + text = decoded.decode("utf-8") + except (binascii.Error, ValueError): + return None + username, separator, password = text.partition(":") + if not separator or not username: + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + """Токен из `Authorization: Bearer ...`; None — если заголовок не подходит.""" + return _credentials(header, BEARER_SCHEME) diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 00000000..e9e5700a --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,122 @@ +"""Хранилище сессий в Valkey: opaque-токен → состояние с абсолютным временем жизни. + +Ключ строится из sha256 токена, поэтому в дампе и в `MONITOR` нет действующих +токенов — есть только их отпечатки. TTL выставляет сам сервер (`SET ... EX`), +а время истечения хранится абсолютным: перезапуск процесса или сервера срок +жизни сессии не продлевает. +""" +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import Protocol +from uuid import UUID + +from redis.asyncio import Redis + +from .domain import parse_rfc3339_z, rfc3339_z, utc_now +from .security import new_token, token_fingerprint + +SESSION_KEY_PREFIX = "messenger:session:" + + +@dataclass(frozen=True) +class SessionState: + """Состояние сессии: владелец и границы жизни.""" + + user_id: UUID + created_at: datetime + expires_at: datetime + + +@dataclass(frozen=True) +class IssuedSession: + """Выданная сессия: сам токен клиент видит ровно один раз.""" + + token: str + state: SessionState + + +class SessionStore(Protocol): + """Контракт хранилища сессий: его выполняет Valkey, подменяется в тестах.""" + + async def create(self, user_id: UUID) -> IssuedSession: ... + + async def resolve(self, token: str) -> SessionState | None: ... + + async def revoke(self, token: str) -> bool: ... + + async def ping(self) -> None: ... + + async def close(self) -> None: ... + + +def _key(token: str) -> str: + return SESSION_KEY_PREFIX + token_fingerprint(token) + + +def _state_of(payload: str) -> SessionState | None: + """Разбирает значение сессии; испорченное значение равносильно её отсутствию.""" + try: + data = json.loads(payload) + return SessionState( + user_id=UUID(data["user_id"]), + created_at=parse_rfc3339_z(data["created_at"]), + expires_at=parse_rfc3339_z(data["expires_at"]), + ) + except (ValueError, KeyError, TypeError): + return None + + +class RedisSessionStore: + """Сессии в Valkey/Redis с постоянным хранением (AOF). + + Таймауты на сокет ограничены: недоступная зависимость обязана отвечать + отказом быстро, а не держать запрос до таймаута клиента. + """ + + def __init__(self, url: str, ttl_seconds: int, timeout_seconds: float) -> None: + self._ttl_seconds = ttl_seconds + self._client = Redis.from_url( + url, + decode_responses=True, + socket_timeout=timeout_seconds, + socket_connect_timeout=timeout_seconds, + ) + + async def create(self, user_id: UUID) -> IssuedSession: + token = new_token() + created_at = utc_now() + expires_at = created_at + timedelta(seconds=self._ttl_seconds) + payload = json.dumps( + { + "user_id": str(user_id), + "created_at": rfc3339_z(created_at), + "expires_at": rfc3339_z(expires_at), + }, + separators=(",", ":"), + sort_keys=True, + ) + await self._client.set(_key(token), payload, ex=self._ttl_seconds) + return IssuedSession(token=token, state=SessionState(user_id, created_at, expires_at)) + + async def resolve(self, token: str) -> SessionState | None: + """Состояние действующей сессии; None — токен неизвестен или срок истёк.""" + payload = await self._client.get(_key(token)) + if payload is None: + return None + state = _state_of(payload) + if state is None or state.expires_at <= utc_now(): + return None + return state + + async def revoke(self, token: str) -> bool: + """Отзыв: GETDEL проверяет и удаляет одной атомарной операцией.""" + return await self._client.getdel(_key(token)) is not None + + async def ping(self) -> None: + await self._client.ping() + + async def close(self) -> None: + await self._client.aclose() diff --git a/src/messenger/storage.py b/src/messenger/storage.py new file mode 100644 index 00000000..65210c0d --- /dev/null +++ b/src/messenger/storage.py @@ -0,0 +1,308 @@ +"""Приватное S3-хранилище: только байты и служебные операции над объектами. + +Метаданные вложений живут в PostgreSQL; этот модуль не знает о чатах и правах. +Ключ объекта всегда назначает сервер. Ошибки S3 преобразуются в два типа: +`StorageObjectMissing` (404) и `StorageUnavailable` (503), чтобы наружу не +уходили детали boto3/MinIO. +""" +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, BinaryIO + +import anyio +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError + +from .config import Config + +CHUNK_SIZE = 64 * 1024 +MISSING_CODES = {"404", "NoSuchKey", "NotFound", "NoSuchBucket"} + + +class StorageUnavailable(RuntimeError): + """S3/MinIO не ответил или вернул ошибку, которую нельзя исправить запросом.""" + + +class StorageObjectMissing(RuntimeError): + """Объект отсутствует в bucket (оригинал потерян или ключ неверен).""" + + +@dataclass(frozen=True) +class S3Object: + """Открытый поток объекта и его наблюдаемый размер.""" + + body: Any + size_bytes: int + content_type: str | None + + +@dataclass(frozen=True) +class ObjectInfo: + """Элемент листинга bucket для сверки.""" + + key: str + size_bytes: int + last_modified: datetime + + +@dataclass(frozen=True) +class ObjectHead: + """Наблюдаемые метаданные одного объекта.""" + + key: str + size_bytes: int + content_type: str | None + etag: str + sha256: str | None + + +def _is_missing(error: ClientError) -> bool: + response = getattr(error, "response", {}) or {} + code = str((response.get("Error") or {}).get("Code", "")) + status = int((response.get("ResponseMetadata") or {}).get("HTTPStatusCode", 0) or 0) + return code in MISSING_CODES or status == 404 + + +class S3Storage: + """Один boto3-клиент на процесс; вызовы уходят в рабочий поток.""" + + def __init__(self, config: Config) -> None: + self._bucket = config.s3_bucket + self._prefix = config.s3_key_prefix + boto_config = BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=config.s3_connect_timeout_seconds, + read_timeout=config.s3_read_timeout_seconds, + retries={"max_attempts": config.s3_max_attempts, "mode": "standard"}, + max_pool_connections=32, + ) + import boto3 + + self._client = boto3.session.Session().client( + "s3", + endpoint_url=config.s3_endpoint, + region_name=config.s3_region, + aws_access_key_id=config.s3_access_key_id, + aws_secret_access_key=config.s3_secret_access_key, + config=boto_config, + ) + + @property + def bucket(self) -> str: + return self._bucket + + @property + def key_prefix(self) -> str: + return self._prefix + + def _key(self, relative: str) -> str: + return f"{self._prefix}{relative}" + + def staging_key(self, attachment_id: str) -> str: + """Приватный staging-ключ принятого тела (до публикации результата).""" + return self._key(f"staging/{attachment_id}") + + def original_key(self, attachment_id: str, pipeline_version: int) -> str: + """Детерминированный конечный ключ оригинала для версии конвейера.""" + return self._key(f"final/{attachment_id}/v{int(pipeline_version)}/original") + + def variant_key(self, attachment_id: str, pipeline_version: int, name: str) -> str: + """Детерминированный ключ варианта: ключ эффекта (id, version, name).""" + if name not in ("thumbnail", "crop", "optimized"): + raise ValueError("unknown variant name") + return self._key(f"final/{attachment_id}/v{int(pipeline_version)}/{name}") + + async def ping(self) -> None: + """Готовность: листинг одного объекта под служебным префиксом. + + Это проверяет и доступность S3, и наличие bucket, и право ListBucket, + которое выдано API-пользователю только на префикс вложений. + """ + try: + await anyio.to_thread.run_sync(self._list_one) + except (BotoCoreError, ClientError, OSError) as exc: + raise StorageUnavailable("S3 не отвечает") from exc + + def _list_one(self) -> None: + self._client.list_objects_v2(Bucket=self._bucket, Prefix=self._prefix, MaxKeys=1) + + async def put( + self, + key: str, + file: BinaryIO, + *, + content_type: str, + size_bytes: int, + sha256: str, + md5_hex: str, + ) -> None: + """Загружает один объект и проверяет, что S3 сохранил ровно эти байты.""" + try: + response = await anyio.to_thread.run_sync( + lambda: self._client.put_object( + Bucket=self._bucket, + Key=key, + Body=file, + ContentLength=size_bytes, + ContentType=content_type, + Metadata={"sha256": sha256}, + ) + ) + except (BotoCoreError, ClientError, OSError) as exc: + raise StorageUnavailable("S3 не принял объект") from exc + etag = str(response.get("ETag", "")).strip('"').lower() + # Для одиночного PUT ETag — MD5 тела; это дешёвая проверка целостности + # передачи. Если S3 не вернул ETag, полагаемся на SHA-256 в метаданных. + if etag and etag != md5_hex.lower(): + try: + await self.delete(key) + except StorageUnavailable: + pass + raise StorageUnavailable("S3 сохранил объект с другим ETag") + + async def open(self, key: str) -> S3Object: + """Открывает объект для потоковой отдачи без redirect.""" + try: + response = await anyio.to_thread.run_sync( + lambda: self._client.get_object(Bucket=self._bucket, Key=key) + ) + except ClientError as exc: + if _is_missing(exc): + raise StorageObjectMissing("Объект отсутствует в S3") from exc + raise StorageUnavailable("S3 не отдал объект") from exc + except (BotoCoreError, OSError) as exc: + raise StorageUnavailable("S3 не отдал объект") from exc + return S3Object( + body=response["Body"], + size_bytes=int(response.get("ContentLength", 0)), + content_type=response.get("ContentType"), + ) + + async def delete(self, key: str) -> None: + try: + await anyio.to_thread.run_sync( + lambda: self._client.delete_object(Bucket=self._bucket, Key=key) + ) + except (BotoCoreError, ClientError, OSError) as exc: + raise StorageUnavailable("S3 не удалил объект") from exc + + async def head(self, key: str) -> ObjectHead: + """Метаданные объекта без скачивания тела.""" + try: + response = await anyio.to_thread.run_sync( + lambda: self._client.head_object(Bucket=self._bucket, Key=key) + ) + except ClientError as exc: + if _is_missing(exc): + raise StorageObjectMissing("Объект отсутствует в S3") from exc + raise StorageUnavailable("S3 не отдал метаданные объекта") from exc + except (BotoCoreError, OSError) as exc: + raise StorageUnavailable("S3 не отдал метаданные объекта") from exc + metadata = response.get("Metadata") or {} + return ObjectHead( + key=key, + size_bytes=int(response.get("ContentLength", 0)), + content_type=response.get("ContentType"), + etag=str(response.get("ETag", "")).strip('"').lower(), + sha256=metadata.get("sha256"), + ) + + async def download_to(self, key: str, path: str) -> int: + """Скачивает объект в файл чанками; возвращает наблюдаемый размер.""" + stored = await self.open(key) + size = 0 + try: + with open(path, "wb") as target: + while True: + chunk = await anyio.to_thread.run_sync(stored.body.read, CHUNK_SIZE) + if not chunk: + break + target.write(chunk) + size += len(chunk) + finally: + close = getattr(stored.body, "close", None) + if close is not None: + close() + if stored.size_bytes and size != stored.size_bytes: + raise StorageUnavailable("Размер скачанного объекта не совпал с ContentLength") + return size + + async def copy(self, source_key: str, target_key: str, *, expected_size: int) -> ObjectHead: + """Серверная копия staging → финальный ключ без буферизации в API/worker.""" + try: + await anyio.to_thread.run_sync( + lambda: self._client.copy_object( + Bucket=self._bucket, + Key=target_key, + CopySource={"Bucket": self._bucket, "Key": source_key}, + MetadataDirective="COPY", + ) + ) + except ClientError as exc: + if _is_missing(exc): + raise StorageObjectMissing("Исходный объект отсутствует в S3") from exc + raise StorageUnavailable("S3 не скопировал объект") from exc + except (BotoCoreError, OSError) as exc: + raise StorageUnavailable("S3 не скопировал объект") from exc + head = await self.head(target_key) + if head.size_bytes != expected_size: + raise StorageUnavailable("Размер скопированного оригинала не совпал") + return head + + async def put_bytes( + self, + key: str, + data: bytes, + *, + content_type: str, + sha256: str, + ) -> ObjectHead: + """Публикует детерминированно сгенерированный вариант из памяти.""" + import hashlib + import io + + md5_hex = hashlib.md5(data).hexdigest() + await self.put( + key, + io.BytesIO(data), + content_type=content_type, + size_bytes=len(data), + sha256=sha256, + md5_hex=md5_hex, + ) + head = await self.head(key) + if head.size_bytes != len(data) or (head.sha256 and head.sha256 != sha256): + raise StorageUnavailable("S3 сохранил вариант с другими метаданными") + return head + + async def list_objects(self, prefix: str | None = None) -> list[ObjectInfo]: + """Объекты под префиксом (по умолчанию весь префикс вложений).""" + try: + return await anyio.to_thread.run_sync(self._list_all, prefix or self._prefix) + except (BotoCoreError, ClientError, OSError) as exc: + raise StorageUnavailable("S3 не отдал список объектов") from exc + + def _list_all(self, prefix: str) -> list[ObjectInfo]: + paginator = self._client.get_paginator("list_objects_v2") + result: list[ObjectInfo] = [] + for page in paginator.paginate(Bucket=self._bucket, Prefix=prefix): + for item in page.get("Contents", []) or []: + last_modified = item["LastModified"] + if last_modified.tzinfo is None: + last_modified = last_modified.replace(tzinfo=timezone.utc) + result.append( + ObjectInfo( + key=str(item["Key"]), + size_bytes=int(item.get("Size", 0)), + last_modified=last_modified, + ) + ) + return result + + async def close(self) -> None: + close = getattr(self._client, "close", None) + if close is not None: + await anyio.to_thread.run_sync(close) diff --git a/src/messenger/upload.py b/src/messenger/upload.py new file mode 100644 index 00000000..896ea5fb --- /dev/null +++ b/src/messenger/upload.py @@ -0,0 +1,100 @@ +"""Потоковый приём тела загрузки: диск вместо RAM и ранние лимиты.""" +from __future__ import annotations + +import hashlib +import os +import tempfile +from dataclasses import dataclass + +import anyio +from starlette.requests import ClientDisconnect, Request + +from .config import Config +from .errors import payload_too_large, unsupported_media_type +from .images import ImageInfo, ImageTooLarge, UnsupportedImage, inspect_image + +READ_CHUNK = 64 * 1024 + + +@dataclass +class SpooledImage: + """Принятое тело на временном диске и его измеренные характеристики.""" + + path: str + size_bytes: int + sha256: str + md5: str + info: ImageInfo + + def cleanup(self) -> None: + try: + os.unlink(self.path) + except OSError: + pass + + +async def receive_image(request: Request, *, expected_content_type: str, config: Config) -> SpooledImage: + """Читает ровно один файл, не поднимая всё тело в память. + + `request.stream()` отдаёт чанки по мере прихода; счётчик байт обрывает + передачу сразу после превышения предела. Временный файл нужен только как + staging до S3; после запроса он удаляется (или ОС уберёт его при падении + процесса). В PostgreSQL и локальном каталоге API оригинал не остаётся. + """ + handle = tempfile.NamedTemporaryFile(prefix="upload-", suffix=".bin", delete=False) + path = handle.name + size = 0 + sha = hashlib.sha256() + md5 = hashlib.md5() + try: + try: + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > config.max_upload_bytes: + raise payload_too_large( + f"Файл больше {config.max_upload_bytes} байт" + ) + handle.write(chunk) + sha.update(chunk) + md5.update(chunk) + except ClientDisconnect as exc: + raise unsupported_media_type("Загрузка прервана клиентом") from exc + finally: + handle.close() + if size == 0: + raise unsupported_media_type("Пустое тело загрузки") + try: + info = await anyio.to_thread.run_sync( + lambda: inspect_image( + path, + expected_content_type=expected_content_type, + max_pixels=config.max_image_pixels, + ) + ) + except ImageTooLarge: + raise payload_too_large( + f"Изображение превышает {config.max_image_pixels} пикселей" + ) from None + except UnsupportedImage: + raise unsupported_media_type( + "Содержимое не является корректным PNG/JPEG" + ) from None + return SpooledImage( + path=path, + size_bytes=size, + sha256=sha.hexdigest(), + md5=md5.hexdigest(), + info=info, + ) + except BaseException: + try: + handle.close() + except Exception: + pass + try: + os.unlink(path) + except OSError: + pass + raise diff --git a/src/messenger/worker.py b/src/messenger/worker.py new file mode 100644 index 00000000..31438bab --- /dev/null +++ b/src/messenger/worker.py @@ -0,0 +1,530 @@ +"""Worker: отдельный процесс, который выполняет задания из RabbitMQ. + +Границы: +- claim в PostgreSQL под lease — единственный способ получить право на эффект; +- S3-объекты публикуются под детерминированными ключами (id, version, variant); +- результат (variants + attachment ready + job ready) фиксируется одной + транзакцией, и только после неё отправляется consumer ack; +- ошибки делятся на постоянные и временные; временные уходят в outbox-retry с + backoff и jitter, постоянные/исчерпанные — в failed и DLQ. +""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import signal +import tempfile +import time +from pathlib import Path +from uuid import UUID + +import anyio +from aio_pika.abc import AbstractIncomingMessage + +from .broker import Broker, InvalidJobMessage, build_payload, parse_payload +from .config import Config, ConfigError +from .db import Database +from .jobs import Claim, JobRecord, JobRepository +from .logging_setup import configure_logging, log +from .pipeline import ( + CorruptOriginal, + GeneratedVariant, + PermanentPipelineError, + PipelineError, + render_variants, +) +from .storage import S3Storage, StorageObjectMissing, StorageUnavailable + +logger = logging.getLogger("messenger.worker") + +HEARTBEAT_FILE = os.environ.get("HEARTBEAT_FILE", "/tmp/messenger-worker-heartbeat") +HEARTBEAT_SECONDS = 5.0 +EXIT_CRASH_BEFORE_OBJECTS = 42 +EXIT_CRASH_AFTER_OBJECTS = 43 + + +class TransientJobError(Exception): + """Временная ошибка: job вернётся в очередь с backoff.""" + + def __init__(self, code: str = "processing_failed", message: str = "") -> None: + super().__init__(message or code) + self.code = code + + +def _md5(path: str) -> str: + digest = hashlib.md5() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(64 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _sha256(path: str) -> str: + digest = hashlib.sha256() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(64 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _heartbeat() -> None: + try: + Path(HEARTBEAT_FILE).write_text(str(time.time()), encoding="ascii") + except OSError: + pass + + +class Worker: + def __init__(self, config: Config) -> None: + self._config = config + self._database = Database(config) + self._storage = S3Storage(config) + self._jobs = JobRepository(self._database, config) + self._broker = Broker(config) + self._semaphore = asyncio.Semaphore(config.worker_concurrency) + self._active: set[asyncio.Task] = set() + self._stop = asyncio.Event() + self._queue = None + self._deliveries = 0 + + def request_stop(self) -> None: + self._stop.set() + + async def _heartbeat_loop(self) -> None: + while not self._stop.is_set(): + _heartbeat() + try: + await asyncio.wait_for(self._stop.wait(), timeout=HEARTBEAT_SECONDS) + except asyncio.TimeoutError: + continue + + async def run(self) -> None: + log( + logger, + logging.INFO, + "worker_startup", + worker_id=self._config.worker_id, + pipeline_version=self._config.pipeline_version, + concurrency=self._config.worker_concurrency, + prefetch=self._config.worker_prefetch, + lease_seconds=self._config.job_lease_seconds, + max_attempts=self._config.job_max_attempts, + queue=self._broker.queue_name, + ) + await self._broker.connect() + _heartbeat() + heartbeat_task = asyncio.create_task(self._heartbeat_loop()) + self._queue = await self._broker.consume( + self._on_message, prefetch=self._config.worker_prefetch + ) + try: + await self._stop.wait() + finally: + self._stop.set() + if self._queue is not None: + try: + await self._queue.cancel() + except Exception: + pass + pending = [task for task in self._active if not task.done()] + if pending: + log( + logger, + logging.INFO, + "worker_drain", + active=len(pending), + timeout_seconds=self._config.graceful_shutdown_seconds, + ) + done, still = await asyncio.wait( + pending, timeout=self._config.graceful_shutdown_seconds + ) + for task in still: + task.cancel() + if still: + await asyncio.gather(*still, return_exceptions=True) + heartbeat_task.cancel() + await asyncio.gather(heartbeat_task, return_exceptions=True) + await self._broker.close() + await self._storage.close() + await self._database.dispose() + log(logger, logging.INFO, "worker_stopped", deliveries=self._deliveries) + + async def _on_message(self, message: AbstractIncomingMessage) -> None: + task = asyncio.current_task() + if task is not None: + self._active.add(task) + try: + async with self._semaphore: + await self._handle(message) + except asyncio.CancelledError: + raise + except Exception as exc: # последняя линия: не терять сообщение молча + log( + logger, + logging.ERROR, + "worker_handle_failed", + exc_info=exc, + error=type(exc).__name__, + ) + try: + await message.nack(requeue=True) + except Exception: + pass + finally: + if task is not None: + self._active.discard(task) + + async def _handle(self, message: AbstractIncomingMessage) -> None: + self._deliveries += 1 + delivery_count = (message.headers or {}).get("x-delivery-count") + try: + payload = parse_payload(message.body) + except InvalidJobMessage as exc: + log( + logger, + logging.WARNING, + "invalid_job_message", + error=str(exc), + delivery_count=delivery_count, + ) + await message.nack(requeue=False) + return + job_id = UUID(payload["job_id"]) + claim = await self._jobs.claim( + job_id, worker_id=self._config.worker_id, lease_seconds=self._config.job_lease_seconds + ) + if not claim.granted: + log( + logger, + logging.INFO, + "delivery_noop", + job_id=str(job_id), + reason=claim.reason, + message_attempt=payload["attempt"], + delivery_count=delivery_count, + ) + await message.ack() + return + + assert claim.job is not None and claim.attachment is not None and claim.lease_token is not None + job = claim.job + log( + logger, + logging.INFO, + "job_claimed", + job_id=str(job.id), + attachment_id=str(job.attachment_id), + pipeline_version=job.pipeline_version, + attempt=job.attempts, + worker_id=self._config.worker_id, + message_attempt=payload["attempt"], + delivery_count=delivery_count, + ) + fault = await self._jobs.consume_fault() + if fault == "crash_before_objects": + log(logger, logging.WARNING, "fault_crash_before_objects", job_id=str(job.id)) + os._exit(EXIT_CRASH_BEFORE_OBJECTS) + if fault in ("transient_once", "transient_always"): + log( + logger, + logging.WARNING, + "fault_transient", + job_id=str(job.id), + kind=fault, + ) + await self._jobs.fail( + job_id=job.id, + lease_token=claim.lease_token, + worker_id=self._config.worker_id, + error_code="storage_unavailable", + permanent=False, + ) + await message.ack() + return + if fault == "permanent_once": + log(logger, logging.WARNING, "fault_permanent_once", job_id=str(job.id)) + await self._jobs.fail( + job_id=job.id, + lease_token=claim.lease_token, + worker_id=self._config.worker_id, + error_code="invalid_image", + permanent=True, + ) + await message.nack(requeue=False) + return + + lease_stop = asyncio.Event() + if fault == "stuck_before_commit": + # Учебный зависший worker: lease не продлевается, reaper вернёт job + # в очередь, а более поздний commit старого attempt будет отклонён. + stuck_seconds = self._config.job_lease_seconds + 5.0 + log( + logger, + logging.WARNING, + "fault_stuck_before_commit", + job_id=str(job.id), + seconds=stuck_seconds, + ) + await asyncio.sleep(stuck_seconds) + lease_task = asyncio.create_task(self._lease_loop(job, claim.lease_token, lease_stop)) + tempdir = tempfile.mkdtemp(prefix=f"job-{job.id}-") + generated: list[GeneratedVariant] = [] + try: + generated = await asyncio.wait_for( + self._process(job, claim, tempdir, fault), + timeout=self._config.job_timeout_seconds, + ) + await message.ack() + except PermanentPipelineError as exc: + await self._fail(job, claim, message, exc.code, permanent=True, requeue=False) + except StorageObjectMissing as exc: + await self._fail( + job, claim, message, "original_missing", permanent=True, requeue=False + ) + except asyncio.TimeoutError: + log( + logger, + logging.WARNING, + "job_timeout", + job_id=str(job.id), + timeout_seconds=self._config.job_timeout_seconds, + ) + await self._fail(job, claim, message, "processing_timeout", permanent=False, requeue=True) + except (StorageUnavailable, TransientJobError) as exc: + code = getattr(exc, "code", "storage_unavailable") + await self._fail(job, claim, message, code, permanent=False, requeue=True) + except PipelineError as exc: + await self._fail( + job, claim, message, exc.code, permanent=exc.permanent, requeue=not exc.permanent + ) + except Exception as exc: + log( + logger, + logging.ERROR, + "job_processing_exception", + exc_info=exc, + job_id=str(job.id), + error=type(exc).__name__, + ) + await self._fail( + job, claim, message, "processing_failed", permanent=False, requeue=True + ) + finally: + lease_stop.set() + lease_task.cancel() + await asyncio.gather(lease_task, return_exceptions=True) + for item in generated: + item.cleanup() + try: + for name in os.listdir(tempdir): + try: + os.unlink(os.path.join(tempdir, name)) + except OSError: + pass + os.rmdir(tempdir) + except OSError: + pass + + async def _fail( + self, + job: JobRecord, + claim: Claim, + message: AbstractIncomingMessage, + code: str, + *, + permanent: bool, + requeue: bool, + ) -> None: + assert claim.lease_token is not None + try: + outcome = await self._jobs.fail( + job_id=job.id, + lease_token=claim.lease_token, + worker_id=self._config.worker_id, + error_code=code, + permanent=permanent, + ) + except Exception as exc: + log( + logger, + logging.ERROR, + "job_fail_persist_failed", + exc_info=exc, + job_id=str(job.id), + ) + # Не подтверждаем сообщение, если не смогли сохранить причину. + await message.nack(requeue=True) + return + log( + logger, + logging.INFO, + "job_failed" if outcome == "failed" else "job_retry_scheduled", + job_id=str(job.id), + outcome=outcome, + error_code=code, + attempts=job.attempts, + permanent=permanent, + ) + if outcome == "failed": + await message.nack(requeue=False) + elif outcome == "retry": + await message.ack() + else: + await message.ack() + + async def _lease_loop(self, job: JobRecord, token: UUID, stop: asyncio.Event) -> None: + interval = max(1.0, self._config.job_lease_seconds / 3.0) + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=interval) + return + except asyncio.TimeoutError: + pass + try: + extended = await self._jobs.extend_lease( + job.id, token, self._config.job_lease_seconds + ) + except Exception as exc: + log( + logger, + logging.WARNING, + "lease_extend_failed", + exc_info=exc, + job_id=str(job.id), + ) + continue + if not extended: + log(logger, logging.WARNING, "lease_lost", job_id=str(job.id)) + return + + async def _process( + self, + job: JobRecord, + claim: Claim, + tempdir: str, + fault: str | None, + ) -> list[GeneratedVariant]: + attachment = claim.attachment + assert attachment is not None and claim.lease_token is not None + original_path = os.path.join(tempdir, "original") + downloaded = await self._storage.download_to(attachment.staging_key, original_path) + if downloaded != attachment.size_bytes or _sha256(original_path) != attachment.sha256: + raise CorruptOriginal("Оригинал в staging повреждён") + generated = await anyio.to_thread.run_sync( + lambda: render_variants( + original_path, + pipeline_version=job.pipeline_version, + workdir=tempdir, + max_pixels=self._config.max_image_pixels, + ) + ) + original_key = self._storage.original_key(str(attachment.id), job.pipeline_version) + await self._storage.copy( + attachment.staging_key, original_key, expected_size=attachment.size_bytes + ) + variant_keys: dict[str, str] = {} + for variant in generated: + key = self._storage.variant_key( + str(attachment.id), job.pipeline_version, variant.name + ) + with open(variant.path, "rb") as handle: + await self._storage.put( + key, + handle, + content_type=variant.content_type, + size_bytes=variant.size_bytes, + sha256=variant.sha256, + md5_hex=_md5(variant.path), + ) + variant_keys[variant.name] = key + if fault == "crash_after_objects": + log( + logger, + logging.WARNING, + "fault_crash_after_objects_before_commit", + job_id=str(job.id), + objects=len(variant_keys) + 1, + ) + os._exit(EXIT_CRASH_AFTER_OBJECTS) + outcome = await self._jobs.complete( + job_id=job.id, + lease_token=claim.lease_token, + worker_id=self._config.worker_id, + original_key=original_key, + variants=generated, + variant_keys=variant_keys, + ) + if outcome != "ready": + log( + logger, + logging.WARNING, + "job_complete_not_applied", + job_id=str(job.id), + outcome=outcome, + ) + return generated + # Staging — общий вход для всех версий. Удаляем его только когда нет + # другой активной job этого вложения (например, более новой версии). + try: + if await self._jobs.has_other_active_jobs(attachment.id, job.id): + log( + logger, + logging.INFO, + "staging_cleanup_deferred_active_job", + job_id=str(job.id), + attachment_id=str(attachment.id), + ) + else: + await self._storage.delete(attachment.staging_key) + except Exception as exc: + log( + logger, + logging.WARNING, + "staging_cleanup_deferred", + exc_info=exc, + job_id=str(job.id), + ) + log( + logger, + logging.INFO, + "job_ready", + job_id=str(job.id), + attachment_id=str(attachment.id), + pipeline_version=job.pipeline_version, + variants=[variant.name for variant in generated], + attempt=job.attempts, + ) + return generated + + +async def _run() -> int: + try: + config = Config.from_env() + config.require_broker() + except ConfigError as exc: + print(f"config error: {exc}", file=os.sys.stderr) + return 2 + logger_root = configure_logging(config.log_level, config.env) + worker = Worker(config) + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + try: + loop.add_signal_handler(sig, worker.request_stop) + except (NotImplementedError, RuntimeError): + signal.signal(sig, lambda *_: worker.request_stop()) + try: + await worker.run() + except Exception as exc: + log(logger_root, logging.ERROR, "worker_fatal", exc_info=exc) + return 1 + return 0 + + +def main() -> int: + return asyncio.run(_run()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/messenger/worker_health.py b/src/messenger/worker_health.py new file mode 100644 index 00000000..ff682d82 --- /dev/null +++ b/src/messenger/worker_health.py @@ -0,0 +1,22 @@ +"""Проверка живости worker/relay по heartbeat-файлу для Docker HEALTHCHECK.""" +from __future__ import annotations + +import os +import time +from pathlib import Path + +DEFAULT_TIMEOUT_SECONDS = 30.0 + + +def main() -> int: + path = Path(os.environ.get("HEARTBEAT_FILE", "/tmp/messenger-worker-heartbeat")) + timeout = float(os.environ.get("HEARTBEAT_TIMEOUT_SECONDS", DEFAULT_TIMEOUT_SECONDS)) + try: + age = time.time() - path.stat().st_mtime + except OSError: + return 1 + return 0 if age <= timeout else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/stack.yaml b/stack.yaml new file mode 100644 index 00000000..1a1b0b2b --- /dev/null +++ b/stack.yaml @@ -0,0 +1,491 @@ +# Docker Swarm stack лабораторной 5. +# +# Роли: Traefik (единственная публичная точка входа), api (2 взаимозаменяемые +# реплики), PostgreSQL (метаданные, jobs/outbox/variants), Valkey (сессии), +# MinIO (приватные байты и staging), RabbitMQ (durable quorum queue + DLQ), +# relay (transactional outbox + reaper lease), worker (обработка изображений), +# minio-init (bootstrap bucket/прав), migrate (однократная схема). +# Наружу опубликован только Traefik :8080; API, PostgreSQL, Valkey, MinIO и +# RabbitMQ портов не публикуют. +# +# Версии образов закреплены тегом и digest-ом; для приложения тег задаёт +# scripts/deploy.sh. Compose-вариант не заявляется: выбран Swarm. + +x-api-environment: &api-environment + APP_ENV: lab + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + LOG_LEVEL: info + SESSION_STORE: redis + SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}" + DB_POOL_SIZE: "5" + DB_MAX_OVERFLOW: "5" + DB_POOL_TIMEOUT_SECONDS: "2" + DB_CONNECT_TIMEOUT_SECONDS: "1.5" + DB_COMMAND_TIMEOUT_SECONDS: "2.5" + DB_STATEMENT_TIMEOUT_MS: "2000" + CACHE_TIMEOUT_SECONDS: "1" + GRACEFUL_SHUTDOWN_SECONDS: "5" + SHUTDOWN_DRAIN_SECONDS: "2" + S3_ENDPOINT: "http://minio:9000" + S3_REGION: "us-east-1" + S3_BUCKET: "messenger-attachments" + S3_KEY_PREFIX: "attachments/" + S3_CONNECT_TIMEOUT_SECONDS: "1.5" + S3_READ_TIMEOUT_SECONDS: "30" + S3_MAX_ATTEMPTS: "2" + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + ATTACHMENT_ORPHAN_GRACE_SECONDS: "3600" + STAGING_RETENTION_SECONDS: "86400" + ATTACHMENT_FAULT_MODE: "none" + PIPELINE_VERSION: "2" + JOB_MAX_ATTEMPTS: "5" + ALLOW_FAULT_INJECTION: "true" + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + CURSOR_SECRET_FILE: /run/secrets/cursor_secret + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +x-worker-environment: &worker-environment + APP_ENV: lab + LOG_LEVEL: info + DB_POOL_SIZE: "3" + DB_MAX_OVERFLOW: "3" + DB_POOL_TIMEOUT_SECONDS: "2" + DB_CONNECT_TIMEOUT_SECONDS: "1.5" + DB_COMMAND_TIMEOUT_SECONDS: "5" + DB_STATEMENT_TIMEOUT_MS: "10000" + CACHE_TIMEOUT_SECONDS: "1" + GRACEFUL_SHUTDOWN_SECONDS: "10" + S3_ENDPOINT: "http://minio:9000" + S3_REGION: "us-east-1" + S3_BUCKET: "messenger-attachments" + S3_KEY_PREFIX: "attachments/" + S3_CONNECT_TIMEOUT_SECONDS: "1.5" + S3_READ_TIMEOUT_SECONDS: "60" + S3_MAX_ATTEMPTS: "3" + MAX_UPLOAD_BYTES: "10485760" + MAX_IMAGE_PIXELS: "40000000" + PIPELINE_VERSION: "2" + WORKER_CONCURRENCY: "${WORKER_CONCURRENCY:-2}" + WORKER_PREFETCH: "${WORKER_PREFETCH:-2}" + JOB_MAX_ATTEMPTS: "${JOB_MAX_ATTEMPTS:-5}" + JOB_TIMEOUT_SECONDS: "${JOB_TIMEOUT_SECONDS:-60}" + JOB_LEASE_SECONDS: "${JOB_LEASE_SECONDS:-15}" + JOB_BACKOFF_BASE_SECONDS: "1" + JOB_BACKOFF_MAX_SECONDS: "30" + STAGING_RETENTION_SECONDS: "86400" + ALLOW_FAULT_INJECTION: "true" + RELAY_POLL_SECONDS: "0.25" + REAPER_INTERVAL_SECONDS: "1" + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + BROKER_URL_FILE: /run/secrets/broker_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + +services: + traefik: + image: traefik:v3.5.4 + command: + - --providers.swarm=true + - --providers.swarm.endpoint=unix:///var/run/docker.sock + - --providers.swarm.exposedByDefault=false + - --providers.swarm.network=backend + - --providers.swarm.watch=true + - --entryPoints.web.address=:8080 + - --log.level=INFO + - --accesslog=false + ports: + - target: 8080 + published: 8080 + protocol: tcp + mode: ingress + volumes: + - /var/run/docker.sock:/var/run/docker.sock:ro + networks: + - backend + deploy: + replicas: 1 + placement: + constraints: + - node.role == manager + resources: + limits: + cpus: "0.50" + memory: 128M + restart_policy: + condition: any + delay: 5s + + postgres: + image: postgres:16.15-bookworm@sha256:0ea6700a3b4f0ae6ce746519073558aed4d88a79d8d07622a9a644946c7319c4 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256" + secrets: + - source: lab5_postgres_password + target: postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + networks: + - backend + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + replicas: 1 + resources: + limits: + cpus: "2.0" + memory: 512M + restart_policy: + condition: any + delay: 5s + + valkey: + image: valkey/valkey:8.0.11-trixie@sha256:4436c94fc34ce4af0354b9379d433a1f258998fb955f995c89822f98c14a8cab + command: + - valkey-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --save + - "" + volumes: + - valkeydata:/data + networks: + - backend + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + deploy: + replicas: 1 + resources: + limits: + cpus: "1.0" + memory: 256M + restart_policy: + condition: any + delay: 5s + + rabbitmq: + # Версия фиксируется digest-ом (см. evidence/04-infrastructure.txt). + image: rabbitmq:4.1.4-alpine@sha256:b736d649308e1b3e1a116c3f36986b605ee3d03e88f10166be2900083d2e63f2 + entrypoint: ["/bin/sh", "-c"] + command: + - >- + export RABBITMQ_DEFAULT_USER=$$(cat /run/secrets/rabbitmq_user); + export RABBITMQ_DEFAULT_PASS=$$(cat /run/secrets/rabbitmq_password); + exec docker-entrypoint.sh rabbitmq-server + environment: + # Стабильный node name: имя контейнера в Swarm меняется, а mnesia-каталог + # должен переживать restart вместе с volume. + RABBITMQ_NODENAME: "rabbit@localhost" + RABBITMQ_USE_LONGNAME: "false" + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "+S 2:2" + secrets: + - source: lab5_rabbitmq_user + target: rabbitmq_user + - source: lab5_rabbitmq_password + target: rabbitmq_password + volumes: + - rabbitmqdata:/var/lib/rabbitmq + networks: + - backend + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 10s + timeout: 5s + retries: 18 + start_period: 30s + deploy: + replicas: 1 + resources: + limits: + cpus: "1.0" + memory: 512M + restart_policy: + condition: any + delay: 5s + + minio: + # RELEASE.2025-09-07T16-13-09Z; digest фиксирует именно проверенный образ. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + entrypoint: ["/bin/sh", "-c"] + command: + - >- + export MINIO_ROOT_USER=$$(cat /run/secrets/s3_root_user); + export MINIO_ROOT_PASSWORD=$$(cat /run/secrets/s3_root_password); + exec /usr/bin/minio server /data --console-address :9001 + environment: + MINIO_BROWSER: "off" + secrets: + - source: lab5_s3_root_user + target: s3_root_user + - source: lab5_s3_root_password + target: s3_root_password + volumes: + - miniodata:/data + networks: + - backend + healthcheck: + test: ["CMD", "curl", "-f", "http://127.0.0.1:9000/minio/health/live"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + deploy: + replicas: 1 + resources: + limits: + cpus: "1.0" + memory: 512M + restart_policy: + condition: any + delay: 5s + + minio-init: + # Тот же проверенный MinIO-образ: в нём уже есть /bin/sh и mc. + image: minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + entrypoint: ["/bin/sh", "/etc/minio-init.sh"] + environment: + S3_ENDPOINT: "http://minio:9000" + S3_BUCKET: "messenger-attachments" + S3_POLICY_NAME: "messenger-attachments-rw" + MC_CONFIG_DIR: "/tmp/.mc" + MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - source: lab5_s3_root_user + target: s3_root_user + - source: lab5_s3_root_password + target: s3_root_password + - source: lab5_s3_access_key_id + target: s3_access_key_id + - source: lab5_s3_secret_access_key + target: s3_secret_access_key + configs: + - source: lab5_minio_init + target: /etc/minio-init.sh + mode: 0555 + networks: + - backend + deploy: + replicas: 1 + restart_policy: + condition: none + resources: + limits: + cpus: "0.5" + memory: 128M + + migrate: + image: ${API_IMAGE:-mtusi-messenger-lab5:5.0.0} + command: + - /bin/sh + - -c + - "until alembic upgrade head; do echo migration waiting for postgres; sleep 2; done" + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - source: lab5_database_url + target: database_url + networks: + - backend + deploy: + replicas: 1 + resources: + limits: + cpus: "1.0" + memory: 256M + restart_policy: + condition: none + + api: + image: ${API_IMAGE:-mtusi-messenger-lab5:5.0.0} + environment: + <<: *api-environment + secrets: + - source: lab5_database_url + target: database_url + - source: lab5_cache_url + target: cache_url + - source: lab5_broker_url + target: broker_url + - source: lab5_cursor_secret + target: cursor_secret + - source: lab5_s3_access_key_id + target: s3_access_key_id + - source: lab5_s3_secret_access_key + target: s3_secret_access_key + networks: + - backend + stop_grace_period: 15s + healthcheck: + test: ["CMD", "python", "-m", "messenger.healthcheck"] + interval: 5s + timeout: 3s + start_period: 15s + retries: 3 + deploy: + replicas: 2 + labels: + - traefik.enable=true + - traefik.swarm.network=backend + - traefik.http.routers.messenger.rule=PathPrefix(`/`) + - traefik.http.routers.messenger.entrypoints=web + - traefik.http.routers.messenger.service=messenger + - traefik.http.services.messenger.loadbalancer.server.port=8080 + - traefik.http.services.messenger.loadbalancer.server.scheme=http + - traefik.http.services.messenger.loadbalancer.healthcheck.path=/health/ready + - traefik.http.services.messenger.loadbalancer.healthcheck.interval=2s + - traefik.http.services.messenger.loadbalancer.healthcheck.unhealthyinterval=1s + - traefik.http.services.messenger.loadbalancer.healthcheck.timeout=1s + - traefik.http.services.messenger.loadbalancer.healthcheck.scheme=http + update_config: + parallelism: 1 + order: start-first + delay: 5s + monitor: 10s + failure_action: rollback + rollback_config: + parallelism: 1 + order: start-first + delay: 5s + restart_policy: + condition: any + delay: 1s + max_attempts: 0 + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + + relay: + image: ${API_IMAGE:-mtusi-messenger-lab5:5.0.0} + command: ["python", "-m", "messenger.relay"] + environment: + <<: *worker-environment + HEARTBEAT_FILE: /tmp/messenger-relay-heartbeat + secrets: + - source: lab5_database_url + target: database_url + - source: lab5_cache_url + target: cache_url + - source: lab5_broker_url + target: broker_url + - source: lab5_s3_access_key_id + target: s3_access_key_id + - source: lab5_s3_secret_access_key + target: s3_secret_access_key + networks: + - backend + stop_grace_period: 15s + healthcheck: + test: ["CMD", "python", "-m", "messenger.worker_health"] + interval: 10s + timeout: 3s + start_period: 10s + retries: 3 + deploy: + replicas: 1 + restart_policy: + condition: any + delay: 1s + resources: + limits: + cpus: "0.5" + memory: 256M + + worker: + image: ${API_IMAGE:-mtusi-messenger-lab5:5.0.0} + command: ["python", "-m", "messenger.worker"] + environment: + <<: *worker-environment + HEARTBEAT_FILE: /tmp/messenger-worker-heartbeat + secrets: + - source: lab5_database_url + target: database_url + - source: lab5_cache_url + target: cache_url + - source: lab5_broker_url + target: broker_url + - source: lab5_s3_access_key_id + target: s3_access_key_id + - source: lab5_s3_secret_access_key + target: s3_secret_access_key + networks: + - backend + stop_grace_period: 20s + healthcheck: + test: ["CMD", "python", "-m", "messenger.worker_health"] + interval: 10s + timeout: 3s + start_period: 10s + retries: 3 + deploy: + replicas: 1 + restart_policy: + condition: any + delay: 1s + resources: + limits: + cpus: "1.0" + memory: 512M + +configs: + lab5_minio_init: + file: ./scripts/minio-init.sh + +secrets: + lab5_database_url: + external: true + lab5_cache_url: + external: true + lab5_broker_url: + external: true + lab5_postgres_password: + external: true + lab5_cursor_secret: + external: true + lab5_rabbitmq_user: + external: true + lab5_rabbitmq_password: + external: true + lab5_s3_root_user: + external: true + lab5_s3_root_password: + external: true + lab5_s3_access_key_id: + external: true + lab5_s3_secret_access_key: + external: true + +volumes: + pgdata: + valkeydata: + miniodata: + rabbitmqdata: + +networks: + backend: + driver: overlay + attachable: true diff --git a/student-tests/Dockerfile.probe b/student-tests/Dockerfile.probe new file mode 100644 index 00000000..dacdb7d6 --- /dev/null +++ b/student-tests/Dockerfile.probe @@ -0,0 +1,10 @@ +# Test-only runner: Python + Docker CLI + curl for failure probes that control +# the Swarm stack while talking to its internal network. +FROM docker:27-cli AS cli +FROM python:3.11.9-slim-bookworm +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* +COPY --from=cli /usr/local/bin/docker /usr/local/bin/docker +WORKDIR /repo +ENTRYPOINT ["python"] diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..1fbe0a02 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,31 @@ +# Собственные проверки лабораторной 5 + +Публичный клиент из `tests/` переиспользуется, поэтому проверки идут через +контракт и LB, а не через внутренние функции приложения. + +```sh +make student-test BASE_URL=http://127.0.0.1:8080 +make track2 BASE_URL=http://127.0.0.1:8080 +python3 student-tests/worker_restart.py +python3 student-tests/outbox_gap.py +python3 student-tests/replay_dlq.py +python3 student-tests/staging_cleanup.py +``` + +| Файл | Что проверяет | +| --- | --- | +| `harness.py` | клиент публичного `tests/client.py`, сбор `X-Instance-Id`, история, ожидание готовности | +| `test_lab3.py` | lab3-инварианты: ≥2 реплики, конкурентная идемпотентность, 409 на другое тело | +| `test_lab5.py` | upload 202 queued, ready через polling, thumbnail 256×160, crop 128×128, optimized WebP, checksums, 413/415/400/404, чужие вложения, queued attachment в сообщении | +| `job_control.py` | Docker/admin-обвязка: scale, restart, `messenger.admin`, листинг S3, RabbitMQ queue stats | +| `worker_restart.py` | worker остановлен → 202 → content 409 → API restart → worker старт → ready; broker restart с volume | +| `outbox_gap.py` | relay остановлен: COMMIT+202 без publish; запуск relay завершает ту же job | +| `replay_dlq.py` | постоянная ошибка → failed + DLQ; явный replay создаёт новый job; replay ready отклоняется | +| `staging_cleanup.py` | staging существует до commit и удаляется после ready; final prefix содержит original + 3 варианта | +| `retry_probe.py` | 5 transient ошибок → backoff/jitter → failed/retry_exhausted + DLQ; replay → ready | +| `image_contract_probe.py` | EXIF orientation, alpha→white, удаление EXIF, реальные размеры JPEG/WebP | +| `idempotency_track.py` | трек 5.2: duplicate, reorder версий, crash до/после side effect, stale v1 после v2, lease/STOP worker; пишет `evidence/track2.json` | + +Docker-пробы требуют запущенного Swarm-стенда и доступа к Docker socket. +Они меняют только сервисы `${STACK_NAME}_*` (по умолчанию `lab5`) и не удаляют +volumes. В конце track2 возвращает worker к 1 реплике. Токены не печатаются. diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 00000000..5968d308 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/harness.py b/student-tests/harness.py new file mode 100644 index 00000000..11395040 --- /dev/null +++ b/student-tests/harness.py @@ -0,0 +1,74 @@ +"""Общая обвязка собственных проверок лабораторной 4. + +Публичный клиент из tests/ переиспользуется, а не копируется: он уже проверяет +схему ответа, X-Request-Id, X-Instance-Id, конверт ошибки и challenge. Здесь +живут только помощники, специфичные для балансировки и идемпотентности. +""" +from __future__ import annotations + +import os +import sys +import time +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TESTS = ROOT / "tests" +HERE = Path(__file__).resolve().parent +for candidate in (str(TESTS), str(HERE)): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +from client import Client, Failure # noqa: E402 + +DEFAULT_BASE_URL = "http://127.0.0.1:8080" +DEFAULT_TIMEOUT = 10.0 + + +def client(timeout: float = DEFAULT_TIMEOUT) -> Client: + return Client(os.environ.get("BASE_URL", DEFAULT_BASE_URL), None, timeout) + + +def instance_ids(api: Client, actor: dict, count: int = 40) -> set[str]: + """Собирает X-Instance-Id разных ответов через LB, не доверяя одному запросу.""" + seen: set[str] = set() + for _ in range(count): + seen.add( + api.call("GET", "/api/v1/users/me", actor=actor, schema="User")["headers"][ + "X-Instance-Id" + ] + ) + return seen + + +def wait_ready(timeout: float = 120.0) -> None: + api = client(timeout=5) + deadline = time.monotonic() + timeout + last = "no attempts" + while time.monotonic() < deadline: + try: + api.call("GET", "/health/ready", schema="Health") + return + except (Failure, OSError) as exc: + last = str(exc) + time.sleep(0.5) + raise Failure("stand did not become ready within %.0fs: %s" % (timeout, last)) + + +def history(api: Client, actor: dict, chat_id: str) -> list[dict]: + items: list[dict] = [] + cursor = None + for _ in range(1000): + path = "/api/v1/chats/%s/messages?limit=100" % chat_id + if cursor is not None: + path += "&cursor=" + cursor + page = api.call("GET", path, actor=actor, schema="MessagePage")["json"] + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items + raise Failure("history pagination exceeded safety bound") + + +def unique_key() -> str: + return uuid.uuid4().hex diff --git a/student-tests/idempotency_probe.py b/student-tests/idempotency_probe.py new file mode 100644 index 00000000..42dcdc56 --- /dev/null +++ b/student-tests/idempotency_probe.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Печатает обезличенную сводку конкурентного повтора одного Idempotency-Key.""" +from __future__ import annotations + +import json +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import harness + + +def main() -> int: + api = harness.client() + alice, bob, _, chat = api.bootstrap() + key = uuid4().hex + text = "probe-" + uuid4().hex + workers = 32 + barrier = threading.Barrier(workers) + + def worker(_: int) -> dict: + local = harness.client(10) + barrier.wait() + return local.message(alice, chat, text, key=key) + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=workers) as pool: + responses = list(pool.map(worker, range(workers))) + elapsed = time.monotonic() - started + bodies = [r["json"] for r in responses] + instances = sorted({r["headers"]["X-Instance-Id"] for r in responses}) + messages = [m for m in harness.history(api, bob, chat["id"]) if m["text"] == text] + summary = { + "concurrent_requests": workers, + "elapsed_seconds": round(elapsed, 3), + "statuses": sorted({r["status"] for r in responses}), + "distinct_instances": instances, + "distinct_message_ids": len({b["id"] for b in bodies}), + "history_matches": len(messages), + "all_responses_identical": all(b == bodies[0] for b in bodies), + } + print(json.dumps(summary, ensure_ascii=False, indent=2)) + return 0 if ( + summary["statuses"] == [201] + and summary["distinct_message_ids"] == 1 + and summary["history_matches"] == 1 + and len(instances) >= 2 + ) else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/student-tests/idempotency_track.py b/student-tests/idempotency_track.py new file mode 100644 index 00000000..8f724471 --- /dev/null +++ b/student-tests/idempotency_track.py @@ -0,0 +1,330 @@ +#!/usr/bin/env python3 +"""Track 5.2: доказанная идемпотентность версионированного pipeline. + +Гипотеза: at-least-once доставка + детерминированные ключи эффекта +(attachment_id, pipeline_version, variant) + lease-guarded commit дают +эффект «ровно один раз» для каждой принятой (202) задачи при дубле, reorder, +crash до/после side effect и устаревшей job после новой версии, хотя доставка +остаётся at-least-once (не exactly-once). + +Сценарии выполняются автоматически на живом Swarm-стенде; результаты и +инварианты пишутся в evidence/track2.json. Скрипт не печатает токены. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +for candidate in (str(HERE), str(ROOT / "tests")): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +import job_control as jc # noqa: E402 +from client import Client # noqa: E402 + +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +EXPECTED_VARIANTS = 3 +EXPECTED_OBJECTS = 4 + + +def now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def job_for(report: dict, pipeline_version: int) -> dict | None: + for job in report.get("jobs", []): + if job["pipeline_version"] == pipeline_version: + return job + return None + + +def wait_all_terminal(attachment_id: str, timeout: float = 120.0) -> dict: + deadline = time.monotonic() + timeout + report = jc.admin_json("show", "--attachment", attachment_id) + while time.monotonic() < deadline: + report = jc.admin_json("show", "--attachment", attachment_id) + states = [job["state"] for job in report.get("jobs", [])] + attachment_state = (report.get("attachment") or {}).get("status") + if all(state in ("ready", "failed", "superseded") for state in states) and attachment_state in ( + "ready", + "failed", + ): + return report + time.sleep(0.25) + raise jc.ProbeError("jobs did not settle: %s" % report) + + +def accepted_upload(api: Client, actor: dict, chat: dict) -> tuple[dict, dict]: + attachment = jc.upload(api, actor, chat, PNG) + report = jc.wait_job(api, actor, attachment["id"], "queued", timeout=30) + job = report["jobs"][0] + return attachment, job + + +def final_objects(attachment_id: str) -> list[str]: + return jc.list_objects("final/%s/" % attachment_id) + + +def result_record(name: str, hypothesis: str, observed: dict, invariants: dict) -> dict: + return { + "scenario": name, + "hypothesis": hypothesis, + "observed": observed, + "invariants": invariants, + "passed": all(invariants.values()), + } + + +def scenario_duplicate(api: Client, actor: dict, chat: dict, started: str) -> dict: + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment, job = accepted_upload(api, actor, chat) + # Original already published by relay; добавляем три копии того же сообщения. + time.sleep(1.0) + outbox_before = jc.admin_json("show", "--attachment", attachment["id"]) + jc.admin_json("duplicate", "--job", job["id"], "--count", "3") + jc.scale("worker", 1) + jc.wait_service("worker", 1) + ready = jc.wait_attachment(api, actor, attachment["id"], "ready", timeout=120) + report = jc.admin_json("show", "--attachment", attachment["id"]) + time.sleep(1.0) + events = [ + event + for event in jc.service_logs("worker", since="10m") + if event.get("job_id") == job["id"] and event.get("ts", "") >= started + ] + claimed = sum(1 for event in events if event.get("event") == "job_claimed") + noops = sum(1 for event in events if event.get("event") == "delivery_noop") + objects = final_objects(attachment["id"]) + variants = report["attachment"]["variants"] + return result_record( + "duplicate_delivery", + "Повторная доставка не создаёт второй набор вариантов/объектов.", + { + "attachment_id": attachment["id"], + "job_id": job["id"], + "outbox_published_before_duplicate": [ + row["published_at"] is not None for row in outbox_before["outbox"] + ], + "worker_events": {"job_claimed": claimed, "delivery_noop": noops}, + "job_attempts": report["jobs"][0]["attempts"], + "variant_rows": len(variants), + "objects_final_prefix": objects, + "status": ready["status"], + }, + { + "one_effect": len(variants) == EXPECTED_VARIANTS and len(objects) == EXPECTED_OBJECTS, + "duplicates_were_delivered": claimed + noops >= 1, + "status_ready": ready["status"] == "ready" and ready["error_code"] is None, + }, + ) + + +def scenario_reorder(api: Client, actor: dict, chat: dict, started: str) -> dict: + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment, v2_job = accepted_upload(api, actor, chat) + v1_job = jc.admin_json("enqueue", "--attachment", attachment["id"], "--pipeline-version", "1") + jc.scale("worker", 1) + jc.wait_service("worker", 1) + report = wait_all_terminal(attachment["id"], timeout=150) + objects = final_objects(attachment["id"]) + variants = report["attachment"]["variants"] + v1_state = job_for(report, 1)["state"] + return result_record( + "reorder_versions", + "Порядок доставки версий не влияет на результат: побеждает более новая версия.", + { + "attachment_id": attachment["id"], + "jobs": report["jobs"], + "attachment_pipeline_version": report["attachment"]["pipeline_version"], + "variant_names": [item["name"] for item in variants], + "v1_state": v1_state, + "objects_final_prefix": objects, + }, + { + "final_version_is_2": report["attachment"]["pipeline_version"] == 2, + "visible_variants_are_v2": len(variants) == EXPECTED_VARIANTS, + "v2_objects_present": any("/v2/" in key for key in objects), + "attachment_ready": report["attachment"]["status"] == "ready", + }, + ) + + +def scenario_crash(api: Client, actor: dict, chat: dict, kind: str, label: str) -> dict: + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment, job = accepted_upload(api, actor, chat) + jc.admin_json("fault", "--kind", kind, "--count", "1") + jc.scale("worker", 1) + jc.wait_service("worker", 1) + ready = jc.wait_attachment(api, actor, attachment["id"], "ready", timeout=180) + report = jc.admin_json("show", "--attachment", attachment["id"]) + variants = report["attachment"]["variants"] + objects = final_objects(attachment["id"]) + events = [ + event + for event in jc.service_logs("worker", since="20m") + if event.get("job_id") == job["id"] and event.get("event") == "job_claimed" + ] + return result_record( + label, + "Crash в окне отказа не теряет принятую работу и не размножает эффект.", + { + "attachment_id": attachment["id"], + "job_id": job["id"], + "fault": kind, + "job_attempts": report["jobs"][0]["attempts"], + "claims_observed": len(events), + "job_state": report["jobs"][0]["state"], + "variant_rows": len(variants), + "objects_final_prefix": objects, + "status": ready["status"], + }, + { + "recovered_to_ready": ready["status"] == "ready" and ready["error_code"] is None, + "retried": report["jobs"][0]["attempts"] == 2, + "one_effect": len(variants) == EXPECTED_VARIANTS and len(objects) == EXPECTED_OBJECTS, + }, + ) + + +def scenario_stale(api: Client, actor: dict, chat: dict, started: str) -> dict: + attachment = jc.upload(api, actor, chat, PNG) + ready = jc.wait_attachment(api, actor, attachment["id"], "ready", timeout=120) + before = jc.admin_json("show", "--attachment", attachment["id"]) + before_variants = [(item["name"], item["sha256"]) for item in before["attachment"]["variants"]] + before_objects = final_objects(attachment["id"]) + v1_job = jc.admin_json("enqueue", "--attachment", attachment["id"], "--pipeline-version", "1") + report = wait_all_terminal(attachment["id"], timeout=90) + after_variants = [(item["name"], item["sha256"]) for item in report["attachment"]["variants"]] + after_objects = final_objects(attachment["id"]) + v1_state = job_for(report, 1)["state"] + return result_record( + "stale_after_newer", + "Устаревшая job не перезаписывает более новую опубликованную версию.", + { + "attachment_id": attachment["id"], + "stale_job_id": v1_job["job_id"], + "stale_job_state": v1_state, + "attachment_pipeline_version": report["attachment"]["pipeline_version"], + "variants_before": before_variants, + "variants_after": after_variants, + "objects_before": before_objects, + "objects_after": after_objects, + }, + { + "stale_job_superseded": v1_state == "superseded", + "version_not_downgraded": report["attachment"]["pipeline_version"] == 2, + "variants_unchanged": before_variants == after_variants, + "objects_unchanged": before_objects == after_objects, + "still_ready": ready["status"] == "ready" and report["attachment"]["status"] == "ready", + }, + ) + + +def scenario_lease(api: Client, actor: dict, chat: dict, started: str) -> dict: + # Fault «зависший worker»: lease не продлевается на время сна. Второй + # worker дождётся истечения lease, заберёт job и завершит её; старый attempt + # проснётся и получит lease_lost при commit. + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment = jc.upload(api, actor, chat, PNG) + job = jc.wait_job(api, actor, attachment["id"], "queued", timeout=30)["jobs"][0] + jc.admin_json("fault", "--kind", "stuck_before_commit", "--count", "1") + jc.scale("worker", 2) + jc.wait_service("worker", 2) + processing = jc.wait_job(api, actor, attachment["id"], "processing", timeout=30) + lease_seconds = 15.0 + ready = jc.wait_attachment(api, actor, attachment["id"], "ready", timeout=180) + time.sleep(5.0) + final = jc.admin_json("show", "--attachment", attachment["id"]) + variants = final["attachment"]["variants"] + objects = final_objects(attachment["id"]) + events = [ + event + for event in jc.service_logs("worker", since="15m") + if event.get("job_id") == job["id"] and event.get("ts", "") >= started + ] + stale_commits = sum( + 1 for event in events if event.get("event") == "job_complete_not_applied" + ) + return result_record( + "stuck_worker_lease", + "Зависший worker не удерживает job навсегда: lease истекает, работу " + "завершает другой worker, старый commit отклоняется.", + { + "attachment_id": attachment["id"], + "job_id": job["id"], + "fault": "stuck_before_commit", + "lease_seconds": lease_seconds, + "observed_processing": bool(processing), + "job_attempts": final["jobs"][0]["attempts"], + "job_state": final["jobs"][0]["state"], + "variant_rows": len(variants), + "objects_final_prefix": objects, + "stale_commit_rejections_in_logs": stale_commits, + }, + { + "recovered_to_ready": ready["status"] == "ready", + "one_effect": len(variants) == EXPECTED_VARIANTS and len(objects) == EXPECTED_OBJECTS, + "attempts_ge_2": final["jobs"][0]["attempts"] >= 2, + "stale_commit_was_rejected_or_not_observed": stale_commits <= 1, + }, + ) + + +def main() -> int: + api = jc.client(timeout=20) + api.call("GET", "/health/ready", schema="Health") + alice, bob, eve, chat = api.bootstrap() + started = now() + results = [] + results.append(scenario_duplicate(api, alice, chat, started)) + results.append(scenario_reorder(api, alice, chat, started)) + results.append(scenario_crash(api, alice, chat, "crash_before_objects", "crash_before_side_effect")) + results.append(scenario_crash(api, alice, chat, "crash_after_objects", "crash_after_side_effect_before_ack")) + results.append(scenario_stale(api, alice, chat, started)) + results.append(scenario_lease(api, alice, chat, started)) + jc.scale("worker", 1) + jc.wait_service("worker", 1) + summary = { + "track": "5.2 proven idempotent pipeline", + "started_at": started, + "finished_at": now(), + "hypothesis": ( + "At-least-once delivery plus deterministic effect keys and lease-guarded " + "versioned commits yields exactly-once effects for accepted jobs; it does " + "not yield exactly-once delivery." + ), + "stand": { + "base_url": jc.BASE_URL, + "stack": jc.STACK, + "pipeline_version": 2, + "pipeline_v1": "original+thumbnail", + "pipeline_v2": "original+thumbnail+crop+optimized WebP", + }, + "scenarios": results, + "all_passed": all(item["passed"] for item in results), + "not_exactly_once": ( + "Broker redelivery and crash windows were observed; duplicates are normal. " + "Exactly-once delivery would require an atomic broker-ack + DB-commit " + "transaction, which is not available across RabbitMQ and PostgreSQL." + ), + } + output = ROOT / "evidence" / "track2.json" + output.write_text(json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(summary, ensure_ascii=False, indent=2, sort_keys=True)) + print("evidence:", output) + return 0 if summary["all_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/image_contract_probe.py b/student-tests/image_contract_probe.py new file mode 100644 index 00000000..d0957bae --- /dev/null +++ b/student-tests/image_contract_probe.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""Контракт изображений: EXIF orientation, alpha→white, метаданные удалены.""" +from __future__ import annotations + +import io +import json +import sys +from pathlib import Path + +from PIL import Image + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +for candidate in (str(HERE), str(ROOT / "tests")): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +from client import Client, arguments # noqa: E402 + + +def png_rgba() -> bytes: + image = Image.new("RGBA", (100, 50), (255, 0, 0, 255)) + for x in range(50, 100): + for y in range(50): + image.putpixel((x, y), (0, 0, 255, 0)) + buffer = io.BytesIO() + image.save(buffer, format="PNG") + return buffer.getvalue() + + +def jpeg_with_exif() -> bytes: + image = Image.new("RGB", (320, 200), (10, 120, 220)) + exif = Image.Exif() + exif[274] = 6 # Orientation: rotate 90 CW + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=90, exif=exif) + return buffer.getvalue() + + +def upload(api: Client, actor: dict, chat: dict, content: bytes, filename: str, content_type: str) -> dict: + from urllib.parse import quote + + return api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe="")), + 202, + actor=actor, + raw=content, + headers={"Content-Type": content_type}, + schema="Attachment", + )["json"] + + +def download(api: Client, actor: dict, attachment_id: str, variant: str) -> bytes: + return api.call( + "GET", + "/api/v1/attachments/" + attachment_id + "/content?variant=" + variant, + actor=actor, + )["bytes"] + + +def main() -> int: + args = arguments("Image contract probe") + api = Client(args.base_url, args.ca_file, 30) + alice, bob, eve, chat = api.bootstrap() + + rgba = upload(api, alice, chat, png_rgba(), "alpha.png", "image/png") + rgba_ready = api.wait_ready(bob, rgba) + thumb = Image.open(io.BytesIO(download(api, bob, rgba["id"], "thumbnail"))).convert("RGB") + thumb_pixels = [thumb.getpixel((90, 10)), thumb.getpixel((90, 40))] + + exif = upload(api, alice, chat, jpeg_with_exif(), "oriented.jpg", "image/jpeg") + exif_ready = api.wait_ready(bob, exif) + optimized = Image.open(io.BytesIO(download(api, bob, exif["id"], "optimized"))) + thumbnail = Image.open(io.BytesIO(download(api, bob, exif["id"], "thumbnail"))) + + evidence = { + "scenario": "image_contract", + "alpha": { + "attachment_id": rgba["id"], + "status": rgba_ready["status"], + "thumbnail_size": thumb.size, + "alpha_region_rgb": thumb_pixels, + "alpha_composited_on_white": all(min(pixel) >= 245 for pixel in thumb_pixels), + }, + "exif_orientation": { + "attachment_id": exif["id"], + "source_size": [320, 200], + "oriented_size": [200, 320], + "thumbnail_size": list(thumbnail.size), + "optimized_size": list(optimized.size), + "thumbnail_exif_entries": len(getattr(thumbnail, "getexif", lambda: {})()), + "optimized_exif_entries": len(getattr(optimized, "getexif", lambda: {})()), + "orientation_applied": thumbnail.size == (160, 256), + "metadata_removed": len(getattr(thumbnail, "getexif", lambda: {})()) == 0 + and len(getattr(optimized, "getexif", lambda: {})()) == 0, + }, + "variants": { + "thumbnail": "image/jpeg 160x256 for rotated 200x320", + "crop": "image/jpeg centered 128x128", + "optimized": "image/webp fit 1280x1280 without upscale", + }, + } + evidence["passed"] = ( + evidence["alpha"]["alpha_composited_on_white"] + and evidence["exif_orientation"]["orientation_applied"] + and evidence["exif_orientation"]["metadata_removed"] + ) + output = ROOT / "evidence" / "image-contract.json" + output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if evidence["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/job_control.py b/student-tests/job_control.py new file mode 100644 index 00000000..d26b354a --- /dev/null +++ b/student-tests/job_control.py @@ -0,0 +1,245 @@ +"""Docker/admin helpers for lab5 failure probes (host-side, no secrets printed). + +Requires the deployed Swarm stack and a Docker socket. All commands operate only +on services named ${STACK_NAME}_*; object listings use MinIO's own mc with the +read-only root credentials already mounted in the container, and never print them. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TESTS = ROOT / "tests" +for candidate in (str(TESTS), str(Path(__file__).resolve().parent)): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +from client import Client, Failure # noqa: E402 + +STACK = os.environ.get("STACK_NAME", "lab5") +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") + + +class ProbeError(RuntimeError): + pass + + +def run(args: list[str], *, check: bool = True, timeout: float = 120.0) -> subprocess.CompletedProcess: + result = subprocess.run(args, capture_output=True, text=True, encoding="utf-8", errors="replace", timeout=timeout) + if check and result.returncode != 0: + raise ProbeError("%s failed: %s" % (" ".join(args[:4]), (result.stderr or result.stdout).strip()[:500])) + return result + + +def docker(*args: str, check: bool = True, timeout: float = 120.0) -> str: + return run(["docker", *args], check=check, timeout=timeout).stdout.strip() + + +def api_container() -> str: + value = docker("ps", "-q", "-f", "name=%s_api" % STACK, "-f", "status=running") + containers = [line for line in value.splitlines() if line.strip()] + if not containers: + raise ProbeError("no running %s_api container" % STACK) + return containers[0] + + +def admin_json(*args: str) -> dict: + output = docker("exec", api_container(), "python", "-m", "messenger.admin", *args, timeout=120) + return json.loads(output) + + +def upload(api: Client, actor: dict, chat: dict, content: bytes, filename: str = "probe.png") -> dict: + from urllib.parse import quote + + return api.call( + "POST", + "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe="")), + 202, + actor=actor, + raw=content, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + + +def wait_attachment(api: Client, actor: dict, attachment_id: str, status: str, timeout: float = 90.0) -> dict: + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = api.call("GET", "/api/v1/attachments/" + attachment_id, actor=actor, schema="Attachment")["json"] + if last["status"] == status: + return last + if last["status"] == "failed" and status != "failed": + raise ProbeError("attachment failed with %s while waiting for %s" % (last["error_code"], status)) + time.sleep(0.25) + raise ProbeError("attachment did not reach %s: %s" % (status, last)) + + +def wait_job(api: Client, actor: dict, attachment_id: str, state: str, timeout: float = 90.0) -> dict: + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + report = admin_json("show", "--attachment", attachment_id) + states = [job["state"] for job in report.get("jobs", [])] + if state in states: + return report + last = states + time.sleep(0.25) + raise ProbeError("job did not reach %s: %s" % (state, last)) + + +def scale(service: str, replicas: int) -> None: + docker("service", "scale", "%s_%s=%d" % (STACK, service, replicas), timeout=180) + + +def wait_service(service: str, replicas: int, timeout: float = 180.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + tasks = docker( + "service", "ps", "-q", "%s_%s" % (STACK, service), "--filter", "desired-state=running" + ) + if len([line for line in tasks.splitlines() if line.strip()]) >= replicas: + return + time.sleep(1.0) + raise ProbeError("%s did not reach %d running tasks" % (service, replicas)) + + +def restart(service: str, timeout: float = 240.0) -> None: + docker("service", "update", "--force", "%s_%s" % (STACK, service), timeout=timeout) + + +def minio_container() -> str: + value = docker("ps", "-q", "-f", "name=%s_minio" % STACK, "-f", "status=running") + containers = [line for line in value.splitlines() if line.strip()] + if not containers: + raise ProbeError("minio container not found") + return containers[0] + + +def rabbitmq_container() -> str: + value = docker("ps", "-q", "-f", "name=%s_rabbitmq" % STACK, "-f", "status=running") + containers = [line for line in value.splitlines() if line.strip()] + if not containers: + raise ProbeError("rabbitmq container not found") + return containers[0] + + +def list_objects(prefix: str) -> list[str]: + """Ключи S3 через admin CLI внутри API (без печати credentials).""" + items = admin_json("objects", "--prefix", prefix) + return [item["key"] for item in items] + + +def wait_queue_messages(queue: str, minimum: int, timeout: float = 60.0) -> dict: + deadline = time.monotonic() + timeout + stats = {} + while time.monotonic() < deadline: + stats = queue_stats() + if stats.get(queue, {}).get("messages", 0) >= minimum: + return stats + time.sleep(0.5) + raise ProbeError("queue %s did not reach %d messages: %s" % (queue, minimum, stats)) + + +def queue_stats() -> dict: + output = docker( + "exec", rabbitmq_container(), "rabbitmqctl", "list_queues", "name", "messages", "messages_ready", "messages_unacknowledged" + ) + stats = {} + for line in output.splitlines(): + parts = line.split() + if len(parts) >= 4 and parts[0] not in ("name", "Timeout:"): + try: + stats[parts[0]] = { + "messages": int(parts[1]), + "ready": int(parts[2]), + "unacknowledged": int(parts[3]), + } + except ValueError: + continue + return stats + + +def worker_container_for(hostname: str) -> str | None: + containers = [ + line for line in docker("ps", "-q", "-f", "name=%s_worker" % STACK).splitlines() if line.strip() + ] + for container in containers: + value = docker("inspect", "-f", "{{.Config.Hostname}}", container, check=False) + if value.strip() == hostname: + return container + return None + + +def pause_container(container: str) -> None: + run(["docker", "kill", "--signal=STOP", container], check=False) + + +def resume_container(container: str) -> None: + # Swarm мог заменить task, пока контейнер был остановлен; это не ошибка пробы. + run(["docker", "kill", "--signal=CONT", container], check=False) + + +def wait_ready(timeout: float = 180.0) -> None: + api = Client(BASE_URL, None, 5) + deadline = time.monotonic() + timeout + last = "not attempted" + while time.monotonic() < deadline: + try: + api.call("GET", "/health/ready", schema="Health") + return + except (Failure, OSError) as exc: + last = str(exc) + time.sleep(0.5) + raise ProbeError("stand did not become ready: %s" % last) + + +def wait_broker(timeout: float = 180.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + result = run( + ["docker", "exec", rabbitmq_container(), "rabbitmqctl", "status"], + check=False, + timeout=20, + ) + if result.returncode == 0: + return + time.sleep(1.0) + raise ProbeError("rabbitmq did not become ready") + + +def service_logs(service: str, since: str = "20m") -> list[dict]: + """JSON-логи всех task-контейнеров сервиса (включая завершившиеся crash-и).""" + containers = [ + line + for line in docker( + "ps", "-a", "-q", "-f", "name=%s_%s" % (STACK, service), check=False + ).splitlines() + if line.strip() + ] + events = [] + for container in containers: + output = run( + ["docker", "logs", "--since", since, container], + check=False, + timeout=120, + ) + for line in (output.stdout + "\n" + output.stderr).splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + try: + events.append(json.loads(line)) + except ValueError: + continue + return events + + +def client(timeout: float = 15.0) -> Client: + return Client(BASE_URL, None, timeout) diff --git a/student-tests/outbox_gap.py b/student-tests/outbox_gap.py new file mode 100644 index 00000000..6a9befca --- /dev/null +++ b/student-tests/outbox_gap.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""COMMIT задания без publish: relay восстанавливает намерение из outbox.""" +from __future__ import annotations + +import json +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +for candidate in (str(HERE), str(ROOT / "tests")): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +import job_control as jc # noqa: E402 + +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def main() -> int: + api = jc.client(timeout=20) + api.call("GET", "/health/ready", schema="Health") + alice, bob, eve, chat = api.bootstrap() + jc.scale("relay", 0) + jc.wait_service("relay", 0) + attachment = jc.upload(api, alice, chat, PNG) + report = jc.wait_job(api, alice, attachment["id"], "queued", timeout=30) + time.sleep(2.0) + before = jc.admin_json("show", "--attachment", attachment["id"]) + pending = [row for row in before["outbox"] if row["published_at"] is None] + jc.scale("relay", 1) + jc.wait_service("relay", 1) + ready = jc.wait_attachment(api, alice, attachment["id"], "ready", timeout=120) + after = jc.admin_json("show", "--attachment", attachment["id"]) + evidence = { + "started_at": None, + "scenario": "crash_between_commit_and_publish", + "attachment_id": attachment["id"], + "job_id": report["jobs"][0]["id"], + "relay_replicas_during_accept": 0, + "outbox_pending_before_relay": pending, + "status_before_relay": "queued", + "status_after_relay": ready["status"], + "job_attempts": after["jobs"][0]["attempts"], + "variants": [item["name"] for item in after["attachment"]["variants"]], + "outbox_after": after["outbox"], + "passed": ( + len(pending) == 1 + and ready["status"] == "ready" + and len(after["attachment"]["variants"]) == 3 + ), + "note": ( + "API committed attachment+job+outbox and returned 202 while relay was " + "stopped. Starting relay published the exact pending outbox row and the " + "same job id completed; no second upload was needed." + ), + } + output = ROOT / "evidence" / "outbox-gap.json" + output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if evidence["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/replay_dlq.py b/student-tests/replay_dlq.py new file mode 100644 index 00000000..d282e67a --- /dev/null +++ b/student-tests/replay_dlq.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Постоянная ошибка → failed + DLQ; явный replay → новый job и ready.""" +from __future__ import annotations + +import json +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +for candidate in (str(HERE), str(ROOT / "tests")): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +import job_control as jc # noqa: E402 + +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def main() -> int: + api = jc.client(timeout=20) + api.call("GET", "/health/ready", schema="Health") + alice, bob, eve, chat = api.bootstrap() + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment = jc.upload(api, alice, chat, PNG) + report = jc.wait_job(api, alice, attachment["id"], "queued", timeout=30) + original_job = report["jobs"][0]["id"] + jc.admin_json("fault", "--kind", "permanent_once", "--count", "1") + jc.scale("worker", 1) + jc.wait_service("worker", 1) + failed = jc.wait_attachment(api, alice, attachment["id"], "failed", timeout=120) + failed_report = jc.admin_json("show", "--attachment", attachment["id"]) + time.sleep(2.0) + queues = jc.queue_stats() + replay = jc.admin_json( + "replay", + "--job", + failed_report["jobs"][0]["id"], + "--operator", + "student", + "--reason", + "retry after transient storage fault was fixed", + ) + # Replay может завершиться быстрее, чем клиент успевает увидеть queued; + # поэтому ждём терминальное ready, а не промежуточное состояние. + ready = jc.wait_attachment(api, bob, attachment["id"], "ready", timeout=120) + final = jc.admin_json("show", "--attachment", attachment["id"]) + ready_job_id = replay["replay_job_id"] + refused = subprocess.run( + [ + "docker", + "exec", + jc.api_container(), + "python", + "-m", + "messenger.admin", + "replay", + "--job", + ready_job_id, + "--operator", + "student", + "--reason", + "must be refused", + ], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + ) + evidence = { + "started_at": None, + "scenario": "permanent_failure_dlq_and_replay", + "attachment_id": attachment["id"], + "original_job_id": original_job, + "failed_attachment_status": failed["status"], + "failed_error_code": failed["error_code"], + "failed_job": failed_report["jobs"][0], + "queues_after_failure": queues, + "replay_job_id": replay["replay_job_id"], + "replay_job": next( + (job for job in final["jobs"] if job["id"] == replay["replay_job_id"]), None + ), + "status_after_replay": ready["status"], + "final_variants": [item["name"] for item in final["attachment"]["variants"]], + "replay_of_ready_attempt": { + "return_code": refused.returncode, + "stderr": (refused.stderr or "").strip(), + "stdout": (refused.stdout or "").strip(), + }, + "passed": ( + failed["status"] == "failed" + and failed["error_code"] == "invalid_image" + and queues.get("attachment.dlq", {}).get("messages", 0) >= 1 + and ready["status"] == "ready" + and len(final["attachment"]["variants"]) == 3 + and refused.returncode != 0 + ), + "note": ( + "The permanent fault produced failed+error_code and a DLQ message. " + "Explicit replay created a new job_id linked by replay_of; the effect " + "key stayed the same. Replaying an already ready job was refused." + ), + } + output = ROOT / "evidence" / "replay-dlq.json" + output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if evidence["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/retry_probe.py b/student-tests/retry_probe.py new file mode 100644 index 00000000..ce8eb39b --- /dev/null +++ b/student-tests/retry_probe.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Backoff/retry и исчерпание попыток: transient → retry, затем failed + DLQ, replay.""" +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +for candidate in (str(HERE), str(ROOT / "tests")): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +import job_control as jc # noqa: E402 + +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def main() -> int: + api = jc.client(timeout=20) + api.call("GET", "/health/ready", schema="Health") + alice, bob, eve, chat = api.bootstrap() + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment = jc.upload(api, alice, chat, PNG) + job = jc.wait_job(api, alice, attachment["id"], "queued", timeout=30)["jobs"][0] + # Пять временных отказов: ровно столько попыток разрешено JOB_MAX_ATTEMPTS. + jc.admin_json("fault", "--kind", "transient_always", "--count", "5") + jc.scale("worker", 1) + jc.wait_service("worker", 1) + failed = jc.wait_attachment(api, alice, attachment["id"], "failed", timeout=180) + report = jc.admin_json("show", "--attachment", attachment["id"]) + time.sleep(1.0) + queues = jc.queue_stats() + replay = jc.admin_json( + "replay", + "--job", + job["id"], + "--operator", + "student", + "--reason", + "bound retries demonstrated; transient fault cleared", + ) + ready = jc.wait_attachment(api, bob, attachment["id"], "ready", timeout=120) + final = jc.admin_json("show", "--attachment", attachment["id"]) + retries = [ + event + for event in report["events"] + if event["to_state"] == "queued" and event["detail"] and "retry_in=" in (event["detail"] or "") + ] + evidence = { + "scenario": "bounded_retry_then_failed_and_replay", + "attachment_id": attachment["id"], + "job_id": job["id"], + "max_attempts": report["jobs"][0]["max_attempts"], + "attempts": report["jobs"][0]["attempts"], + "failed_status": failed["status"], + "failed_error_code": failed["error_code"], + "job_state": report["jobs"][0]["state"], + "retry_events": retries, + "retry_delays_seconds": [ + float(event["detail"].split("retry_in=")[1].rstrip("s")) for event in retries + ], + "queues_after_failure": queues, + "replay_job_id": replay["replay_job_id"], + "status_after_replay": ready["status"], + "final_variants": [item["name"] for item in final["attachment"]["variants"]], + "passed": ( + failed["status"] == "failed" + and failed["error_code"] == "retry_exhausted" + and report["jobs"][0]["attempts"] == 5 + and len(retries) == 4 + and queues.get("attachment.dlq", {}).get("messages", 0) >= 1 + and ready["status"] == "ready" + and len(final["attachment"]["variants"]) == 3 + ), + "note": ( + "Transient failures produced four backoff retries with jitter, then the " + "fifth attempt exhausted the configured limit and produced failed+DLQ. " + "An explicit replay after clearing the fault completed the same effect key." + ), + } + output = ROOT / "evidence" / "retry.json" + output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if evidence["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/run_all.py b/student-tests/run_all.py new file mode 100644 index 00000000..3854979f --- /dev/null +++ b/student-tests/run_all.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Запуск собственных проверок лабораторной 5.""" +from __future__ import annotations + +import argparse +import os +import sys +import unittest +from pathlib import Path + +HERE = Path(__file__).resolve().parent + + +def main() -> int: + parser = argparse.ArgumentParser(description="Собственные проверки лабораторной 5") + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8080")) + arguments = parser.parse_args() + os.environ["BASE_URL"] = arguments.base_url + if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(line_buffering=True) + print("BASE_URL =", arguments.base_url, flush=True) + suite = unittest.defaultTestLoader.discover( + str(HERE), pattern="test_*.py", top_level_dir=str(HERE) + ) + result = unittest.TextTestRunner(verbosity=2).run(suite) + return 0 if result.wasSuccessful() else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/staging_cleanup.py b/student-tests/staging_cleanup.py new file mode 100644 index 00000000..ef95d62c --- /dev/null +++ b/student-tests/staging_cleanup.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Staging удаляется только после успешной фиксации результата.""" +from __future__ import annotations + +import json +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +for candidate in (str(HERE), str(ROOT / "tests")): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +import job_control as jc # noqa: E402 + +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def main() -> int: + api = jc.client(timeout=20) + api.call("GET", "/health/ready", schema="Health") + alice, bob, eve, chat = api.bootstrap() + # Worker остановлен: видно staging-объект до публикации результата. + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment = jc.upload(api, alice, chat, PNG) + staging_before = jc.list_objects("staging/%s" % attachment["id"]) + final_before = jc.list_objects("final/%s/" % attachment["id"]) + jc.scale("worker", 1) + jc.wait_service("worker", 1) + ready = jc.wait_attachment(api, alice, attachment["id"], "ready", timeout=120) + time.sleep(1.0) + staging_after = jc.list_objects("staging/%s" % attachment["id"]) + final_after = jc.list_objects("final/%s/" % attachment["id"]) + report = jc.admin_json("show", "--attachment", attachment["id"]) + evidence = { + "scenario": "staging_cleanup_after_commit", + "attachment_id": attachment["id"], + "staging_before_ready": staging_before, + "final_before_ready": final_before, + "staging_after_ready": staging_after, + "final_after_ready": final_after, + "attachment_status": ready["status"], + "staging_key_in_db": report["attachment"]["staging_key"], + "original_key_in_db": report["attachment"]["object_key"], + "variants": [item["name"] for item in report["attachment"]["variants"]], + "passed": ( + len(staging_before) == 1 + and len(staging_after) == 0 + and len(final_after) == 4 + and ready["status"] == "ready" + ), + "note": ( + "The original was visible under the staging prefix before commit. " + "After the ready commit, worker deleted it; final prefix contains the " + "original plus exactly three variants." + ), + } + output = ROOT / "evidence" / "staging-cleanup.json" + output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if evidence["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/student-tests/test_lab3.py b/student-tests/test_lab3.py new file mode 100644 index 00000000..4519bb70 --- /dev/null +++ b/student-tests/test_lab3.py @@ -0,0 +1,108 @@ +"""Собственные проверки лабораторной 3: реплики, LB и общая идемпотентность.""" +from __future__ import annotations + +import threading +import unittest +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +import harness +from client import Client + + +class Lab3Tests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + harness.wait_ready() + cls.api = harness.client() + cls.alice, cls.bob, cls.eve, cls.chat = cls.api.bootstrap() + + def test_01_at_least_two_instances_and_stable_header(self) -> None: + """LB действительно распределяет один и тот же токен между разными процессами.""" + observed = harness.instance_ids(self.api, self.alice, 40) + self.assertGreaterEqual(len(observed), 2, "less than two X-Instance-Id values through LB") + for _ in range(10): + me = self.api.call("GET", "/api/v1/users/me", actor=self.alice, schema="User")["json"] + self.assertEqual(me["id"], self.alice["id"]) + + def test_02_concurrent_same_key_through_lb_creates_one_message(self) -> None: + """Один ключ + одно тело одновременно через разные реплики = одно сообщение.""" + key = uuid4().hex + text = "concurrent-idempotency-" + uuid4().hex + workers = 24 + barrier = threading.Barrier(workers) + + def worker(_: int) -> dict: + local = harness.client(10) + barrier.wait() + return local.message(self.alice, self.chat, text, key=key) + + with ThreadPoolExecutor(max_workers=workers) as pool: + responses = list(pool.map(worker, range(workers))) + + bodies = [response["json"] for response in responses] + instances = {response["headers"]["X-Instance-Id"] for response in responses} + self.assertGreaterEqual(len(instances), 2, "concurrent requests did not reach both replicas") + self.assertEqual(bodies, [bodies[0]] * workers, "replay returned a different Message") + messages = [m for m in harness.history(self.api, self.bob, self.chat["id"]) if m["text"] == text] + self.assertEqual(len(messages), 1, "idempotency key created %d messages" % len(messages)) + self.assertEqual(messages[0], bodies[0]) + + def test_03_same_key_other_body_conflicts(self) -> None: + key = uuid4().hex + first = self.api.message(self.alice, self.chat, "first-" + uuid4().hex, key=key)["json"] + replay = self.api.message(self.alice, self.chat, first["text"], key=key)["json"] + self.assertEqual(replay, first) + self.api.call( + "POST", + "/api/v1/chats/" + self.chat["id"] + "/messages", + 409, + actor=self.alice, + body={"text": "different body"}, + headers={"Idempotency-Key": key}, + ) + + def test_04_key_scope_user_and_chat(self) -> None: + key = uuid4().hex + text_alice = "scope-a-" + uuid4().hex + text_bob = "scope-b-" + uuid4().hex + alice_message = self.api.message(self.alice, self.chat, text_alice, key=key)["json"] + bob_message = self.api.message(self.bob, self.chat, text_bob, key=key)["json"] + self.assertNotEqual(alice_message["id"], bob_message["id"]) + other_chat = self.api.call( + "POST", + "/api/v1/chats", + 201, + actor=self.alice, + body={"title": "Другой чат", "member_ids": [self.alice["id"]]}, + schema="Chat", + )["json"] + other = self.api.message(self.alice, other_chat, text_alice, key=key)["json"] + self.assertNotEqual(other["id"], alice_message["id"]) + + def test_05_key_validation(self) -> None: + path = "/api/v1/chats/" + self.chat["id"] + "/messages" + for bad in ("", "has space", "bad/slash", "x" * 129): + self.api.call( + "POST", + path, + 400, + actor=self.alice, + body={"text": "bad-key"}, + headers={"Idempotency-Key": bad}, + ) + self.api.call("POST", path, 400, actor=self.alice, body={"text": "no key"}) + for good in ("a", "z" * 128): + self.api.message(self.alice, self.chat, "boundary-" + uuid4().hex, key=good) + + def test_06_uncertain_retry_is_safe(self) -> None: + """Повтор с тем же ключом после потери ответа не создаёт второй объект.""" + key = uuid4().hex + text = "retry-after-conflict-" + uuid4().hex + first = self.api.message(self.alice, self.chat, text, key=key)["json"] + again = self.api.message(self.alice, self.chat, text, key=key)["json"] + self.assertEqual(first, again) + + +if __name__ == "__main__": + unittest.main(verbosity=2) \ No newline at end of file diff --git a/student-tests/test_lab5.py b/student-tests/test_lab5.py new file mode 100644 index 00000000..e2f20791 --- /dev/null +++ b/student-tests/test_lab5.py @@ -0,0 +1,235 @@ +"""Собственные проверки лабораторной 5: асинхронный pipeline и варианты. + +Проверки идут через публичный контракт и LB; Docker-сценарии (остановка worker, +брокера, fault injection) лежат в отдельных probes и в evidence. +""" +from __future__ import annotations + +import hashlib +import struct +import unittest +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote + +import harness +from client import Client +from image_probe import jpeg_size + +ROOT = Path(__file__).resolve().parents[1] +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +JPEG = (Path(__file__).resolve().parent / "fixtures" / "sample.jpg").read_bytes() +MAX_BYTES = 10_485_760 +EXPECTED_VARIANTS = {"thumbnail", "crop", "optimized"} + + +def webp_size(data: bytes) -> tuple[int, int]: + """Размеры WebP без внешних библиотек: VP8X, VP8L или lossy VP8.""" + assert data[:4] == b"RIFF" and data[8:12] == b"WEBP", "not WebP" + offset = 12 + while offset + 8 <= len(data): + fourcc = data[offset : offset + 4] + length = int.from_bytes(data[offset + 4 : offset + 8], "little") + payload = data[offset + 8 : offset + 8 + length] + if fourcc == b"VP8X" and len(payload) >= 10: + width = 1 + int.from_bytes(payload[4:7], "little") + height = 1 + int.from_bytes(payload[7:10], "little") + return width, height + if fourcc == b"VP8L" and len(payload) >= 5: + bits = int.from_bytes(payload[1:5], "little") + return (bits & 0x3FFF) + 1, ((bits >> 14) & 0x3FFF) + 1 + if fourcc == b"VP8 " and len(payload) >= 10: + assert payload[3:6] == b"\x9d\x01\x2a", "not a VP8 keyframe" + width = int.from_bytes(payload[6:8], "little") & 0x3FFF + height = int.from_bytes(payload[8:10], "little") & 0x3FFF + return width, height + offset += 8 + length + (length & 1) + raise AssertionError("WebP dimensions not found") + + +def upload( + api: Client, + actor: dict, + chat: dict, + content: bytes, + *, + filename: str = "photo.png", + content_type: str = "image/png", + expected: int = 202, +): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe="")) + return api.call( + "POST", + path, + expected, + actor=actor, + raw=content, + headers={"Content-Type": content_type}, + schema="Attachment" if expected == 202 else None, + ) + + +def png_with_dimensions(width: int, height: int) -> bytes: + """Меняет IHDR у реального PNG и пересчитывает CRC: файл остаётся PNG.""" + data = bytearray(PNG) + struct.pack_into(">I", data, 16, width) + struct.pack_into(">I", data, 20, height) + struct.pack_into(">I", data, 29, zlib.crc32(bytes(data[12:29])) & 0xFFFFFFFF) + return bytes(data) + + +class Lab5AttachmentTests(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + harness.wait_ready() + cls.api = harness.client() + cls.alice, cls.bob, cls.eve, cls.chat = cls.api.bootstrap() + cls.other_chat = cls.api.call( + "POST", + "/api/v1/chats", + 201, + actor=cls.alice, + body={"title": "Другой чат lab5", "member_ids": [cls.alice["id"], cls.bob["id"]]}, + schema="Chat", + )["json"] + + def test_01_png_and_jpeg_pipeline(self) -> None: + for content, content_type, filename in ( + (PNG, "image/png", "sample.png"), + (JPEG, "image/jpeg", "sample.jpg"), + ): + queued = upload( + self.api, self.alice, self.chat, content, filename=filename, content_type=content_type + )["json"] + self.assertEqual(queued["status"], "queued") + self.assertEqual(queued["variants"], []) + self.assertIsNone(queued["error_code"]) + self.assertEqual(queued["sha256"], hashlib.sha256(content).hexdigest()) + self.assertEqual(queued["size_bytes"], len(content)) + ready = self.api.wait_ready(self.bob, queued) + self.assertEqual(ready["status"], "ready") + self.assertIsNone(ready["error_code"]) + names = {item["name"] for item in ready["variants"]} + self.assertEqual(names, EXPECTED_VARIANTS) + variants = {item["name"]: item for item in ready["variants"]} + self.assertEqual( + (variants["thumbnail"]["content_type"], variants["thumbnail"]["width"], variants["thumbnail"]["height"]), + ("image/jpeg", 256, 160), + ) + self.assertEqual( + (variants["crop"]["content_type"], variants["crop"]["width"], variants["crop"]["height"]), + ("image/jpeg", 128, 128), + ) + self.assertEqual(variants["optimized"]["content_type"], "image/webp") + self.assertLessEqual(max(variants["optimized"]["width"], variants["optimized"]["height"]), 1280) + original = self.api.call("GET", "/api/v1/attachments/" + queued["id"] + "/content", actor=self.bob) + self.assertEqual(original["bytes"], content) + self.assertEqual(original["headers"].get_content_type(), content_type) + self.assertEqual(original["headers"].get("Content-Length"), str(len(content))) + for name, expected_size in (("thumbnail", (256, 160)), ("crop", (128, 128))): + response = self.api.call( + "GET", + "/api/v1/attachments/" + queued["id"] + "/content?variant=" + name, + actor=self.bob, + ) + self.assertEqual(response["headers"].get_content_type(), "image/jpeg") + self.assertEqual(hashlib.sha256(response["bytes"]).hexdigest(), variants[name]["sha256"]) + self.assertEqual(len(response["bytes"]), variants[name]["size_bytes"]) + self.assertEqual(jpeg_size(response["bytes"]), expected_size) + optimized = self.api.call( + "GET", + "/api/v1/attachments/" + queued["id"] + "/content?variant=optimized", + actor=self.bob, + ) + self.assertEqual(optimized["headers"].get_content_type(), "image/webp") + self.assertEqual(hashlib.sha256(optimized["bytes"]).hexdigest(), variants["optimized"]["sha256"]) + self.assertEqual( + webp_size(optimized["bytes"]), + (variants["optimized"]["width"], variants["optimized"]["height"]), + ) + + def test_02_stranger_and_nonmember(self) -> None: + attachment = upload(self.api, self.alice, self.chat, PNG)["json"] + for suffix in ("", "/content"): + self.api.call("GET", "/api/v1/attachments/" + attachment["id"] + suffix, 404, actor=self.eve) + upload(self.api, self.eve, self.chat, PNG, expected=404) + self.api.call( + "GET", + "/api/v1/attachments/" + attachment["id"] + "/content?variant=thumbnail", + 404, + actor=self.eve, + ) + + def test_03_attachment_from_another_chat_is_404(self) -> None: + foreign = upload(self.api, self.alice, self.other_chat, PNG)["json"] + self.api.call( + "POST", + "/api/v1/chats/" + self.chat["id"] + "/messages", + 404, + actor=self.alice, + body={"text": "", "attachment_ids": [foreign["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + own = self.api.message(self.alice, self.other_chat, "", attachments=[foreign["id"]])["json"] + self.assertEqual(own["attachment_ids"], [foreign["id"]]) + + def test_04_queued_attachment_can_be_message_and_idempotency(self) -> None: + attachment = upload(self.api, self.alice, self.chat, PNG)["json"] + path = "/api/v1/chats/" + self.chat["id"] + "/messages" + self.api.call("POST", path, 400, actor=self.alice, body={"text": ""}, headers={"Idempotency-Key": uuid.uuid4().hex}) + self.api.call( + "POST", + path, + 400, + actor=self.alice, + body={"text": "too many", "attachment_ids": [str(uuid.uuid4()) for _ in range(11)]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + self.api.call( + "POST", + path, + 400, + actor=self.alice, + body={"text": "duplicate", "attachment_ids": [attachment["id"], attachment["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + key = uuid.uuid4().hex + first = self.api.message(self.alice, self.chat, "", key=key, attachments=[attachment["id"]])["json"] + self.assertEqual(first["text"], "") + self.assertEqual(first["attachment_ids"], [attachment["id"]]) + replay = self.api.message(self.alice, self.chat, "", key=key, attachments=[attachment["id"]])["json"] + self.assertEqual(replay, first) + self.api.call( + "POST", + path, + 409, + actor=self.alice, + body={"text": "different", "attachment_ids": []}, + headers={"Idempotency-Key": key}, + ) + + def test_05_rejects_bad_images_and_filenames(self) -> None: + upload(self.api, self.alice, self.chat, b"\x00" * (MAX_BYTES + 1), expected=413) + upload(self.api, self.alice, self.chat, PNG[:100], expected=415) + upload(self.api, self.alice, self.chat, PNG, content_type="image/jpeg", expected=415) + upload(self.api, self.alice, self.chat, b"not an image", expected=415) + upload(self.api, self.alice, self.chat, b"", expected=415) + upload(self.api, self.alice, self.chat, png_with_dimensions(8000, 6000), expected=413) + upload(self.api, self.alice, self.chat, png_with_dimensions(10000, 10000), expected=413) + for bad in ("../secret.png", "a/b.png", "a\\b.png", "a\x00b.png", "a\nb.png", "x" * 256): + upload(self.api, self.alice, self.chat, PNG, filename=bad, expected=400) + + def test_06_unknown_variant_is_400(self) -> None: + attachment = upload(self.api, self.alice, self.chat, PNG)["json"] + ready = self.api.wait_ready(self.bob, attachment) + self.api.call( + "GET", + "/api/v1/attachments/" + ready["id"] + "/content?variant=bogus", + 400, + actor=self.bob, + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/student-tests/worker_restart.py b/student-tests/worker_restart.py new file mode 100644 index 00000000..df123846 --- /dev/null +++ b/student-tests/worker_restart.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Демонстрация уровня 3: принятая работа переживает worker/API/broker restart. + +Сценарии: +A. worker остановлен → upload 202 queued → content 409 → API restart → + тот же attachment остаётся queued → worker запущен → ready, варианты на месте. +B. worker остановлен → upload → durable сообщение в RabbitMQ → broker restart + с volume → worker запущен → ready. +""" +from __future__ import annotations + +import json +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent +for candidate in (str(HERE), str(ROOT / "tests")): + if candidate not in sys.path: + sys.path.insert(0, candidate) + +import job_control as jc # noqa: E402 +from client import Client # noqa: E402 + +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def now() -> str: + return datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + + +def wait_published(attachment_id: str, timeout: float = 60.0) -> dict: + deadline = time.monotonic() + timeout + report = jc.admin_json("show", "--attachment", attachment_id) + while time.monotonic() < deadline: + report = jc.admin_json("show", "--attachment", attachment_id) + if report.get("outbox") and all(row["published_at"] for row in report["outbox"]): + return report + time.sleep(0.25) + raise jc.ProbeError("outbox was not published: %s" % report) + + +def main() -> int: + api = jc.client(timeout=20) + api.call("GET", "/health/ready", schema="Health") + alice, bob, eve, chat = api.bootstrap() + started = now() + evidence: dict = {"started_at": started, "scenarios": []} + + # A: worker остановлен; 202 и 409; API restart не теряет работу. + jc.scale("worker", 0) + jc.wait_service("worker", 0) + attachment = jc.upload(api, alice, chat, PNG) + queued = jc.wait_attachment(api, alice, attachment["id"], "queued", timeout=30) + api.call("GET", "/api/v1/attachments/" + attachment["id"] + "/content", 409, actor=bob) + content_status = 409 + report_before = jc.admin_json("show", "--attachment", attachment["id"]) + jc.restart("api") + jc.wait_ready(timeout=240) + api = jc.client(timeout=20) + after_restart = api.call( + "GET", "/api/v1/attachments/" + attachment["id"], actor=bob, schema="Attachment" + )["json"] + jc.scale("worker", 1) + jc.wait_service("worker", 1) + ready = jc.wait_attachment(api, bob, attachment["id"], "ready", timeout=120) + report_after = jc.admin_json("show", "--attachment", attachment["id"]) + evidence["scenarios"].append( + { + "name": "worker_stopped_api_restart", + "attachment_id": attachment["id"], + "job_id": report_before["jobs"][0]["id"], + "upload_response_status": 202, + "status_before_restart": queued["status"], + "content_while_queued": content_status, + "outbox_after_upload": report_before["outbox"], + "status_after_api_restart": after_restart["status"], + "same_sha256": after_restart["sha256"] == attachment["sha256"], + "final_status": ready["status"], + "final_attempts": report_after["jobs"][0]["attempts"], + "final_variants": [item["name"] for item in report_after["attachment"]["variants"]], + "passed": ( + queued["status"] == "queued" + and content_status == 409 + and after_restart["status"] == "queued" + and ready["status"] == "ready" + and len(report_after["attachment"]["variants"]) == 3 + ), + } + ) + + # B: durable сообщение в broker; restart RabbitMQ с volume; затем worker. + jc.scale("worker", 0) + jc.wait_service("worker", 0) + time.sleep(5.0) + attachment_b = jc.upload(api, alice, chat, PNG) + published = wait_published(attachment_b["id"]) + # Сообщение должно лежать именно в durable queue, а не быть уже забранным. + queue_before = jc.wait_queue_messages("attachment.jobs", 1, timeout=60) + jc.restart("rabbitmq") + jc.wait_broker(timeout=240) + queue_after = jc.queue_stats() + jc.scale("worker", 1) + jc.wait_service("worker", 1) + ready_b = jc.wait_attachment(api, bob, attachment_b["id"], "ready", timeout=150) + report_b = jc.admin_json("show", "--attachment", attachment_b["id"]) + evidence["scenarios"].append( + { + "name": "broker_restart_with_volume", + "attachment_id": attachment_b["id"], + "job_id": published["jobs"][0]["id"], + "queue_before_restart": queue_before, + "queue_after_restart": queue_after, + "status_before_restart": "queued", + "final_status": ready_b["status"], + "final_attempts": report_b["jobs"][0]["attempts"], + "final_variants": [item["name"] for item in report_b["attachment"]["variants"]], + "passed": ( + ready_b["status"] == "ready" + and len(report_b["attachment"]["variants"]) == 3 + and any( + key == "attachment.jobs" and value["messages"] >= 1 + for key, value in queue_before.items() + ) + ), + } + ) + + evidence["finished_at"] = now() + evidence["all_passed"] = all(item["passed"] for item in evidence["scenarios"]) + evidence["note"] = ( + "RabbitMQ quorum queue is durable; message was confirmed before restart. " + "Named volume lab5_rabbitmqdata was preserved. The same 202 attachment " + "and job id are used before and after restart." + ) + output = ROOT / "evidence" / "worker-restart.json" + output.write_text(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(evidence, ensure_ascii=False, indent=2, sort_keys=True)) + return 0 if evidence["all_passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main())