From 633fb96920f04dc5bdff258406f338aa46d8843f Mon Sep 17 00:00:00 2001 From: alenchha Date: Fri, 6 Nov 2026 12:10:16 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=205=20=E2=80=94=20=D0=91=D0=92?= =?UTF-8?q?=D0=A22301=20=E2=80=94=20=D0=9A=D1=83=D1=80=D0=BB=D0=BE=D0=B2?= =?UTF-8?q?=D0=B0=20=E2=80=94=20=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit RabbitMQ + transactional outbox, отдельный worker (thumbnail/crop/WebP), lease и heartbeat, retry с backoff+jitter, DLQ и replay, admission control; тесты отказов, трек 1 и отчёт. --- .env.example | 42 +- .gitignore | 2 + Dockerfile | 35 + REPORT.md | 212 +++++- app/__init__.py | 0 app/broker.py | 78 +++ app/config.py | 98 +++ app/cursor.py | 43 ++ app/db.py | 99 +++ app/errors.py | 37 ++ app/faults.py | 38 ++ app/imagecheck.py | 213 ++++++ app/imageproc.py | 115 ++++ app/jobs.py | 318 +++++++++ app/main.py | 693 ++++++++++++++++++++ app/outbox.py | 86 +++ app/passwords.py | 16 + app/reconcile.py | 155 +++++ app/repo.py | 338 ++++++++++ app/s3.py | 122 ++++ app/sessions.py | 154 +++++ app/worker.py | 329 ++++++++++ compose.yaml | 312 +++++++++ evidence/failover.txt | 13 + evidence/failure_windows.txt | 82 +++ evidence/persistence.txt | 36 + evidence/smoke.txt | 6 + evidence/track1/results.jsonl | 28 + evidence/track1/summary.md | 21 + evidence/variants_limits.txt | 20 + evidence/worker_down.txt | 16 + infra/README.md | 10 +- infra/minio/app-policy.json | 15 + infra/minio/init.sh | 19 + infra/nginx-health.sh | 63 ++ infra/nginx.conf | 75 +++ infra/rabbitmq/entrypoint.sh | 14 + infra/rabbitmq/rabbitmq.conf | 10 + migrations/0001_init.sql | 46 ++ migrations/0002_idempotency.sql | 12 + migrations/0003_attachments.sql | 28 + migrations/0004_allow_empty_text.sql | 2 + migrations/0005_attachment_upload_state.sql | 5 + migrations/0006_async_processing.sql | 75 +++ requirements.txt | 37 ++ scripts/jobs.py | 88 +++ scripts/migrate.py | 60 ++ scripts/reconcile.py | 32 + scripts/restart.sh | 6 + scripts/stop-one.sh | 5 + student-tests/queue_load.py | 250 +++++++ student-tests/stand.py | 132 ++++ student-tests/test_failure_windows.py | 194 ++++++ student-tests/test_worker_down.py | 84 +++ student-tests/track1_matrix.sh | 23 + student-tests/track1_summary.py | 48 ++ student-tests/variants_check.py | 126 ++++ 57 files changed, 5194 insertions(+), 22 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/broker.py create mode 100644 app/config.py create mode 100644 app/cursor.py create mode 100644 app/db.py create mode 100644 app/errors.py create mode 100644 app/faults.py create mode 100644 app/imagecheck.py create mode 100644 app/imageproc.py create mode 100644 app/jobs.py create mode 100644 app/main.py create mode 100644 app/outbox.py create mode 100644 app/passwords.py create mode 100644 app/reconcile.py create mode 100644 app/repo.py create mode 100644 app/s3.py create mode 100644 app/sessions.py create mode 100644 app/worker.py create mode 100644 compose.yaml create mode 100644 evidence/failover.txt create mode 100644 evidence/failure_windows.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/track1/results.jsonl create mode 100644 evidence/track1/summary.md create mode 100644 evidence/variants_limits.txt create mode 100644 evidence/worker_down.txt create mode 100644 infra/minio/app-policy.json create mode 100755 infra/minio/init.sh create mode 100644 infra/nginx-health.sh create mode 100644 infra/nginx.conf create mode 100644 infra/rabbitmq/entrypoint.sh create mode 100644 infra/rabbitmq/rabbitmq.conf create mode 100644 migrations/0001_init.sql create mode 100644 migrations/0002_idempotency.sql create mode 100644 migrations/0003_attachments.sql create mode 100644 migrations/0004_allow_empty_text.sql create mode 100644 migrations/0005_attachment_upload_state.sql create mode 100644 migrations/0006_async_processing.sql create mode 100644 requirements.txt create mode 100755 scripts/jobs.py create mode 100644 scripts/migrate.py create mode 100644 scripts/reconcile.py create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 student-tests/queue_load.py create mode 100755 student-tests/stand.py create mode 100755 student-tests/test_failure_windows.py create mode 100755 student-tests/test_worker_down.py create mode 100755 student-tests/track1_matrix.sh create mode 100755 student-tests/track1_summary.py create mode 100644 student-tests/variants_check.py diff --git a/.env.example b/.env.example index 119384b5..8dcf61da 100644 --- a/.env.example +++ b/.env.example @@ -1,25 +1,57 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Values below are the ones used for evidence/ (see REPORT.md for the mapping). APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +GRACEFUL_TIMEOUT=8 +MAX_INFLIGHT_REQUESTS=64 +MAX_BODY_BYTES=65536 +MAX_RSS_MB=0 +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT=3 +DB_CONNECT_TIMEOUT=3 +DB_STATEMENT_TIMEOUT_MS=5000 +DB_OP_TIMEOUT=6 +REQUEST_TIMEOUT=30 +DB_READY_TIMEOUT=1.5 +READY_CHECK_SECONDS=1 +READY_FAILURES=2 +SESSION_CACHE_TTL_SECONDS=30 +SESSION_SWEEP_SECONDS=60 +CACHE_TIMEOUT=0.2 +SESSION_REVOKED_TTL_SECONDS=60 +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +UPLOAD_CONCURRENCY=2 +RECONCILE_INTERVAL_SECONDS=300 +RECONCILE_GRACE_SECONDS=900 +# --- lab5: broker, jobs, worker BROKER_URL_FILE=/run/secrets/broker_url WORKER_CONCURRENCY=2 +WORKER_PREFETCH=2 JOB_MAX_ATTEMPTS=5 JOB_TIMEOUT_SECONDS=60 +JOB_LEASE_SECONDS=20 +JOB_BACKOFF_BASE_SECONDS=2 +JOB_BACKOFF_MAX_SECONDS=30 +JOB_SWEEP_SECONDS=5 +WORKER_CHILD_MEMORY_MB=1024 STAGING_RETENTION_SECONDS=86400 +# Admission control (track 1): uploads get 503 + Retry-After while this many jobs are queued/processing. +MAX_PENDING_JOBS=200 +JPEG_QUALITY=85 +WEBP_QUALITY=80 +# Lab stand only: enables filename-selected fault injection (app/faults.py). Keep 0 anywhere else. +FAULT_INJECTION=1 diff --git a/.gitignore b/.gitignore index b5a71896..bed20757 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +# Track 1 images are regenerated by student-tests/queue_load.py +evidence/track1/fixture-* diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a0c7a86d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-deps -r requirements.txt + +COPY app/ app/ +COPY migrations/ migrations/ +COPY scripts/migrate.py scripts/reconcile.py scripts/jobs.py scripts/ + +USER 10001:10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + +STOPSIGNAL SIGTERM + +CMD ["sh", "-c", "exec uvicorn app.main:app --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""] diff --git a/REPORT.md b/REPORT.md index e98f75e9..b9c6f85e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,221 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Алёна Курлова, БВТ2301 +- Целевая оценка; выбранный трек 5: **5, трек 1 «Очередь под перегрузкой»** +- Commit/PR: TODO после публикации (ветка `solution/bvt2301-kurlova`) +- Источники, библиотеки, помощь ИИ: FastAPI, psycopg 3, boto3, Pillow 11, aio-pika 9.5 (RabbitMQ 4.0), MinIO; документация RabbitMQ [reliability](https://www.rabbitmq.com/docs/reliability), [confirms](https://www.rabbitmq.com/docs/confirms), [quorum queues](https://www.rabbitmq.com/docs/quorum-queues). Код lab4 перенесён и расширен. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора; все результаты ниже получены на локальном стенде и сохранены в `evidence/`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Desktop (Compose v2.20+, нужен `--wait`), Python 3.9+ (только stdlib для тестов). Проверено на macOS arm64, Docker 29.7, 10 CPU / 8 GiB у Docker VM. + +```sh +cp .env.example .env # те же значения, что использовались для evidence/ +mkdir -p secrets +printf 'messenger' > secrets/postgres_password +printf 'postgresql://messenger:messenger@postgres:5432/messenger' > secrets/database_url +printf 'valkey://valkey:6379/0' > secrets/cache_url +printf 'messenger_root' > secrets/minio_root_user +python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/minio_root_password +printf 'messenger_app' > secrets/s3_access_key_id +python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/s3_secret_access_key +python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/rabbitmq_password +printf 'amqp://messenger:%s@rabbitmq:5672/messenger' "$(cat secrets/rabbitmq_password)" > secrets/broker_url +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait # postgres, valkey, minio(+init), rabbitmq, migrate, api-1/2, nginx, worker x2 +make check && make test +``` + +Bootstrap выполняется автоматически при каждом `up` и безопасен при повторе: `migrate` (версионированные SQL, checksum, advisory lock; lab5 добавляет `0006_async_processing.sql`), `minio-init` (bucket, приватность, пользователь приложения), топология RabbitMQ объявляется кодом идемпотентно. Действий оператора не требуется. + +Безопасная остановка с сохранением данных: `docker compose stop` (worker получает SIGTERM, перестаёт брать сообщения и ждёт до 25 с текущие задачи; неподтверждённые сообщения брокер вернёт в очередь). Намеренное удаление учебных данных: `docker compose down -v`. `secrets/` и `.env` в git не попадают. + +Собственные проверки (стенд должен быть запущен; `FAULT_INJECTION=1` только на учебном стенде): + +```sh +python3 student-tests/test_worker_down.py # уровень 3: демонстрация из README + повторная доставка +python3 student-tests/test_failure_windows.py # уровень 4: сценарии A–I (окна отказа, retry, DLQ, replay, lease) +docker compose exec -T --index 1 worker python - < student-tests/variants_check.py # правила вариантов и лимиты worker +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # затем docker compose start api-1 api-2 и повтор +./student-tests/track1_matrix.sh && python3 student-tests/track1_summary.py # трек 1, ~25 минут +``` + +Операторский CLI: `docker compose exec api-1 python scripts/jobs.py failed | stats | show | replay --actor <имя>`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client ──HTTP──▶ nginx ──▶ api-1 / api-2 ──(1) PUT staging/──────────────▶ MinIO (приватный bucket) + │ (2) COMMIT: attachment=queued + job + outbox ▲ ▲ + ▼ │ │ + PostgreSQL ◀── relay (в каждой API, SKIP LOCKED) ──(3) publish+confirm──▶ RabbitMQ + ▲ │ quorum queue + │ claim/lease/heartbeat, (6) COMMIT результата ▼ + └──────────────────────────── worker x2 ◀──(4) deliver, prefetch─┘ + (5) child process: resize; copy original, PUT variants ──▶ MinIO + (7) delete staging, ack +``` + +| Роль | Сервис | Версия | Health | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | nginx | 1.27.3-alpine | `/health/live`; скрипт опрашивает `/health/ready` реплик | — | `127.0.0.1:8080` | — | +| API ×2 | api-1, api-2 | из исходников, python 3.12.8-slim@sha256 | `/health/ready` | нет | внутренняя | Docker secrets | +| Worker ×2 | worker (replicas) | тот же образ, `python -m app.worker` | mtime heartbeat-файла < 15 с | нет | внутренняя, портов нет | secrets: БД, S3, broker | +| Брокер | rabbitmq | 4.0.5-alpine@sha256:e045b516… | `rabbitmq-diagnostics check_port_connectivity` | `rabbitmqdata` | внутренняя, порты не опубликованы | пароль из secret → conf.d при старте | +| БД | postgres | 17.7-alpine@sha256 | `pg_isready` | `pgdata` | внутренняя | secret | +| Кэш сессий | valkey | 8.1-alpine@sha256 | `valkey-cli ping` | — | внутренняя | — | +| S3 | minio (+minio-init) | `madlabcg/minio@sha256:ad134b2d…` | `mc ready local` | `miniodata` | внутренняя | root только у minio/minio-init | + +**Источник истины.** PostgreSQL: `attachments`, `attachment_jobs` (состояние задания, попытки, lease), `attachment_variants` (результат), `outbox`, `job_audit`. Брокер — только транспорт ссылки на задание; если сообщение потеряно или продублировано, правильное состояние восстанавливается из БД. Байты — только в S3. + +**Что сохранено в момент 202** (ответ на вопрос защиты): (а) все байты оригинала, проверенные по размеру/MIME/целостности/пикселям (lab4-проверки сохранены), лежат в `staging/` в MinIO с persistent volume; (б) одним COMMIT записаны `attachments.status='queued'`, строка `attachment_jobs` и строка `outbox` с сообщением. Публикация в брокер происходит **после** 202 асинхронно; её надёжность обеспечивает outbox, а не HTTP-запрос. Граница честно такая: staging уже в S3, worker делает server-side copy в `attachments//original` и создаёт варианты в `attachments//v1/`. + +**Состояния** (contracts/attachment-states.md): `uploading` (внутреннее, клиенту не видно, как в lab4) → `queued` → `processing` → `ready` | `failed`; `processing → queued` при временной ошибке или истёкшем lease; `failed → queued` только явным replay. Повторная доставка ready-задания — no-op с ack (дочищает staging, если worker упал до этого). + +**Сообщение** соответствует `contracts/jobs.schema.json` (поля `schema_version, job_id, type, attachment_id, pipeline_version, attempt, created_at`); `message_id` = `job_id`, header `attempt` дублирует поле. Никаких байтов, URL, ключей S3, filename и credentials. Worker берёт `staging_key`/`object_key`/`sha256` из БД и сверяет SHA-256 и размер скачанного staging перед обработкой. Невалидное сообщение → `reject(requeue=false)` → DLX `attachments.dlx` → `attachments.dlq`. + +**Брокер.** RabbitMQ 4.0, quorum-очередь `attachments.process` (Raft-лог на диске, durable), persistent-сообщения, publisher confirms + `mandatory` (relay помечает outbox-строку опубликованной только после confirm), ручной ack, `prefetch = WORKER_PREFETCH`, `x-delivery-limit=20` (защита от сообщения, которое валит consumer), `consumer_timeout=10 мин` как последняя страховка брокера. **Publisher confirm** означает «брокер принял ответственность за сообщение»; **consumer ack** — «эффект обработки устойчиво зафиксирован в БД». Это разные границы, и мы ждём обе. + +**Transactional outbox** ([app/outbox.py](app/outbox.py)). Relay работает в каждой API-реплике: `SELECT … FOR UPDATE SKIP LOCKED` → publish с confirm → `published_at` в той же транзакции. Будится сразу после upload и каждые 1 с. Crash после confirm, но до COMMIT — повторная публикация (дубль безвреден). Недоступный брокер не влияет на `/health/ready` и приём файлов: задания копятся в outbox (сценарий I). Опубликованные строки удаляются через 24 ч. + +**Идемпотентность и ключ эффекта.** Ключ эффекта — `(attachment_id, pipeline_version, variant)`: PK `attachment_variants`, детерминированные S3-ключи `attachments//v1/{thumbnail.jpg,crop.jpg,optimized.webp}`. Повтор перезаписывает те же ключи теми же байтами и вставляет варианты `ON CONFLICT DO NOTHING`, второго набора не возникает. `job_id` один на `(attachment_id, pipeline_version)` и не меняется при redelivery/retry; `attempt` растёт только при управляемой новой попытке (retry, истёкший lease, replay). Сообщение со старым `attempt` — no-op (`stale`). COMMIT результата фенсится условием `state='processing' AND lease_owner=<этот worker>`; `attachments.status` меняется на ready только если ещё не ready и `pipeline_version` не новее. + +**Lease, heartbeat, зависший worker.** Claim ставит `lease_until = now()+20 с`; пока задача жива, heartbeat продлевает lease каждые ~7 с. Зависший worker (заблокирован event loop, сеть до БД) перестаёт продлевать lease. Sweeper любого worker раз в 5 с переводит такие задания в `queued` с новой попыткой (или в `failed/retry_exhausted`). Если «зависший» потом проснётся, его heartbeat/COMMIT не пройдут фенсинг, и он сделает ack без записи (сценарий G). Долго работающий worker отличается от зависшего именно heartbeat'ом. Жёсткий потолок одной попытки — `JOB_TIMEOUT_SECONDS=60` (wall clock дочернего процесса) и `2×` на весь job. + +**Retry.** `JOB_MAX_ATTEMPTS=5`. Backoff экспоненциальный с equal jitter: `d = min(30, 2·2^(n-1))`, задержка `d/2 + U(0, d/2)`. Отложенная попытка — строка outbox с `available_at` в будущем, без плагинов брокера. Классификация ошибок: + +| Код | Тип | Причина | +| --- | --- | --- | +| `invalid_image` | постоянная | дочерний процесс не смог декодировать (exit 3) | +| `image_too_large` | постоянная | > 40 Мпикс по заголовку, до декодирования (exit 4) | +| `image_too_complex` | постоянная | MemoryError при RLIMIT_AS (exit 5) или SIGXCPU при RLIMIT_CPU | +| `staging_missing`, `checksum_mismatch` | постоянная | исходник пропал или не совпадает с принятым | +| `processing_timeout` | временная | превышен wall-clock 60 с (вероятна перегрузка) | +| `worker_resource_exhausted` | временная | дочерний процесс убит SIGKILL (OOM контейнера) | +| `dependency_unavailable` | временная | ошибки S3/БД/сети | +| `lease_expired` | временная | worker умер/завис | +| `retry_exhausted` | финальная | временные ошибки на всех 5 попытках | + +**DLQ / реестр failed / replay.** Бизнес-отказы — это `attachment_jobs.state='failed'` (`scripts/jobs.py failed`) с `error_code`, `last_error` и полным `job_audit`. Ядовитые сообщения попадают в `attachments.dlq`. Replay (`scripts/jobs.py replay --actor <кто>`) в одной транзакции под `FOR UPDATE` проверяет, что задание и вложение именно `failed`, а staging ещё существует. Затем ставит `queued`, `attempt+1`, новый бюджет из 5 попыток, пишет outbox и audit `replayed`. Для `ready/queued/processing` replay отказывает (audit `replay_refused`, код выхода 3), поэтому «повторить» никогда не исполняет дубль успешной job. + +**Варианты** ([app/imageproc.py](app/imageproc.py)), выполняются в дочернем процессе с `RLIMIT_AS=1024 MiB`, `RLIMIT_CPU=60 с`, `nice 5`, wall-timeout 60 с: +- thumbnail — JPEG q=85, вписан в 256×256, без увеличения, floor, минимум 1 px; 320×200 → 256×160; +- crop — JPEG q=85, центральный квадрат → 128×128 (маленькие увеличиваются); +- optimized — WebP q=80, method=4, вписан в 1280×1280 без увеличения, alpha сохраняется. + +EXIF orientation применяется до расчёта размеров. Alpha композитится на белый для JPEG. 16-бит PNG сводится к 8 бит, CMYK → RGB. EXIF/XMP/ICC в варианты не копируются (пиксели трактуются как sRGB). Уменьшение каждого маленького файла не обещаем. Для JPEG используется DCT-scaling (`draft`) до ≥ нужного размера: декодирование 40-Мпикс JPEG занимает 0,3 с, а не секунды. Лимит пикселей проверяется по заголовку до декодирования (и в API, и в worker). + +**Ограничения работы, принятой системой.** Bytes: 10 MiB (nginx и API). Pixels: 40 Мпикс (API и worker). Время и память: rlimits дочернего процесса, `WORKER_CONCURRENCY`, `prefetch`. Количество: admission control `MAX_PENDING_JOBS=200` — ранняя проверка до чтения тела и строгая в транзакции постановки в очередь, превышение → `503` + `Retry-After: 5` (трек 1). + +**Staging cleanup.** Worker удаляет `staging/` сразу после COMMIT результата (и при повторной доставке завершённой задачи). Reconciler из lab4 ([app/reconcile.py](app/reconcile.py), раз в 5 мин, advisory lock) расширен: удаляет staging у `ready`, staging `failed`-вложений старше `STAGING_RETENTION_SECONDS=86400` (после этого replay отказывает), orphan-объекты в `staging/` и `attachments/` старше grace 15 мин, stale `uploading`. Текущую работу (queued/processing) он не трогает. + +**Что перенесено из lab4 и что изменено.** Весь API lab1–4 сохранён. Upload теперь `202 queued` вместо `201 ready`. GET метаданных возвращает варианты только у ready. `content?variant=` поддерживает `thumbnail|crop|optimized`: неизвестное имя — 400, queued/processing/failed — 409, отсутствующий вариант ready-объекта — 404. Прикрепить queued/processing к сообщению можно, failed — 409. Ключи S3: `staging/`, `attachments//original`, `attachments//v1/*`; политика пользователя приложения расширена на `staging/*`. + +**Критичные зависимости `/health/ready`:** PostgreSQL и S3 (как в lab4). RabbitMQ **не** критичен для API: благодаря outbox приём файлов продолжается и при недоступном брокере. + +**Переменные .env.example → реальная конфигурация.** Все ключи `.env.example` читаются [app/config.py](app/config.py) с теми же именами. `BROKER_URL_FILE` → secret `broker_url`. `WORKER_CONCURRENCY` — семафор параллельных задач на процесс. `WORKER_PREFETCH` — `basic.qos`. `JOB_MAX_ATTEMPTS`, `JOB_TIMEOUT_SECONDS` (wall/CPU лимит попытки), `JOB_LEASE_SECONDS`, `JOB_BACKOFF_*`, `STAGING_RETENTION_SECONDS` — как описано выше. `API_REPLICAS` информационный (реплики заданы сервисами api-1/api-2). `FAULT_INJECTION=1` только для учебного стенда. ## Доказательства критериев +Дата прогона: 2026-11-06, macOS arm64, Docker 29.7 (10 CPU / 8 GiB VM), рабочее дерево поверх skeleton-коммита `276dbda`. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `make check && make test` | `PASS: lab5 public API scenario`: 202 queued → ready, original побайтно, JPEG thumbnail 256×160 по реальному SOF | [evidence/smoke.txt](evidence/smoke.txt) | +| 3: минимум lab4 сохранён, 201 → 202 | `make test`, `make test-persistence`, `make test-failover` ×2 | все PASS; failover: 0 неудачных записей, восстановление 0,8–1,0 с | [smoke](evidence/smoke.txt), [persistence](evidence/persistence.txt), [failover](evidence/failover.txt) | +| 3: 202 только после staging + job; рестарт API не теряет работу | `test_worker_down.py`, сценарий A | upload при остановленных worker → 202; после рестарта api-1/api-2/rabbitmq (StartedAt изменился) запись queued, сообщение в очереди; API, убитый сразу после COMMIT, не теряет задание | [worker_down](evidence/worker_down.txt), [failure_windows](evidence/failure_windows.txt) | +| 3: отдельные broker и worker, durable, at-least-once, ack после эффекта | compose.yaml; `test_worker_down.py`; сценарии B, C | quorum-очередь пережила рестарт брокера (`messages=1` до и после); crash до COMMIT → повтор; crash после COMMIT до ack → redelivery `decision=done` | [worker_down](evidence/worker_down.txt), [failure_windows](evidence/failure_windows.txt) | +| 3: worker недоступен → API не делает resize синхронно | `test_worker_down.py` | при остановленных worker статус остаётся queued, content → 409, другие сообщения работают | [worker_down](evidence/worker_down.txt) | +| 3: queued/processing/ready/failed, thumbnail по контракту, original неизменен | сценарии B/E, `variants_check.py` | наблюдались `queued → processing → queued → processing → ready` и `failed`; 320×200 → 256×160; SHA-256 original совпадает | [failure_windows](evidence/failure_windows.txt), [variants_limits](evidence/variants_limits.txt) | +| 3: ограниченный retry, failed с безопасным error_code, повтор не создаёт второго набора | сценарии D, E, F; повторная доставка | ровно 5 попыток → `retry_exhausted`; `invalid_image` без повторов; после дубля по-прежнему 3 строки вариантов, 1 success, 4 объекта | [failure_windows](evidence/failure_windows.txt), [worker_down](evidence/worker_down.txt) | +| 3: демонстрация из README | `test_worker_down.py` | worker stop → 202 → сообщение с вложением → restart API+broker → worker start → ready + thumbnail | [worker_down](evidence/worker_down.txt) | +| 4: crop 128×128 и optimized WebP, параметры | `variants_check.py` | crop 128×128 (с увеличением маленьких); WebP q=80 вписан в 1280 без увеличения; EXIF удалён; alpha на белый / сохранена в WebP; 16-bit, CMYK, palette | [variants_limits](evidence/variants_limits.txt) | +| 4: лимиты bytes/pixels/время/память worker | `variants_check.py` | pixels → exit 4 до декодирования; битый PNG → exit 3; RLIMIT_AS → exit 5 (`image_too_complex`); RLIMIT_CPU → SIGXCPU; 39,5 Мпикс PNG за 0,9 с, ≈322 MiB RSS дочернего процесса | [variants_limits](evidence/variants_limits.txt) | +| 4: outbox, crash между COMMIT и publish | сценарии A и I | API `os._exit` после COMMIT → relay другой реплики публикует ровно один раз → ready; брокер остановлен → upload 202, задание ждёт в outbox, после старта брокера → ready | [failure_windows](evidence/failure_windows.txt) | +| 4: backoff с jitter, постоянные/временные ошибки, DLQ/реестр, безопасный replay | сценарии D, E, F, H | задержки 1,7 / 3,1 / 7,8 с в границах [1–2], [2–4], [4–8]; `invalid_image` без повторов; `jobs.py failed` содержит job; replay → ready; повторный replay готовой job отклонён; poison → `attachments.dlq` | [failure_windows](evidence/failure_windows.txt) | +| 4: остановка worker после записи объекта до ack | сценарий B | crash после PUT → lease истёк (sweeper) → попытка 2 записала те же ключи; 1 набор вариантов, staging удалён | [failure_windows](evidence/failure_windows.txt) | +| 4: timeout/lease зависшего processing, prefetch, очистка staging | сценарий G; `WORKER_PREFETCH`; сценарии B/C | зависший worker без heartbeat → другой worker забрал задание, проснувшийся отфенсен (`lease lost`); prefetch = 2; staging удаляется после COMMIT и при повторной доставке | [failure_windows](evidence/failure_windows.txt) | +| 5, трек 1 | `student-tests/track1_matrix.sh`, `track1_summary.py` | см. «Эксперименты»; строгий admission limit держит очередь ровно на 30 | [summary.md](evidence/track1/summary.md), [results.jsonl](evidence/track1/results.jsonl), [matrix.log](evidence/track1/matrix.log) | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Проверка hook: [evidence/persistence.txt](evidence/persistence.txt) содержит `State.StartedAt` всех 9 контейнеров до и после `scripts/restart.sh`. Перезапущены postgres, valkey, minio, rabbitmq, оба worker, обе API и nginx; volumes `pgdata`, `miniodata`, `rabbitmqdata` сохранены. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Трек 1. Очередь под перегрузкой + +**Гипотеза.** Обработка CPU-bound, поэтому пропускная способность растёт с concurrency до числа доступных CPU и дальше не растёт. Каждый лишний слот concurrency только увеличивает пиковую память, примерно на размер декодированного изображения. Prefetch больше concurrency не ускоряет, а только переносит ожидание из брокера в буфер одного worker. Без ограничения приёма очередь и время до ready растут неограниченно, пока нагрузка выше ёмкости. Со строгим лимитом очередь ограничена, лишняя работа получает 503 + Retry-After, а восстановление после нагрузки занимает время разбора ограниченной очереди. + +**Методика** ([queue_load.py](student-tests/queue_load.py), [track1_matrix.sh](student-tests/track1_matrix.sh)): +- Один worker-контейнер с лимитом 2 CPU / 1,5 GiB; API ×2 по 0,5 CPU. +- Open-loop нагрузка: запрос i отправляется в момент `t0 + i/rate` независимо от ответов, без повторов. 30 с на прогон, затем ожидание полного разбора очереди. +- Размеры: small — JPEG 640×400, 80 KiB, 10/с; medium — JPEG 4000×3000 (12 Мпикс), 3,9 MiB, 5/с; large — PNG 6000×4000 (24 Мпикс, без DCT-scaling), 0,4 MiB, 3/с. +- Настройки worker `concurrency/prefetch`: 1/1, 2/2, 4/4, 2/16. По 2 повтора каждой. +- Метрики из `attachment_jobs`: queue lag = `first_started_at − created_at`; time to ready (ttr) = `finished_at − attachment.created_at`; throughput = готовые / (последний finish − первое поступление). Раз в секунду: `docker stats` worker (память без page cache, CPU) и число незавершённых заданий. +- Калибровка при c=1 и низкой нагрузке, время обработки одной задачи: small 0,2 с, medium 0,34 с, large 0,53 с. + +**Результаты** (среднее по 2 повторам, в скобках min–max; полный вывод — [summary.md](evidence/track1/summary.md)): + +| size | rate/s | c/p | throughput jobs/s | lag p95 s | ttr p50 s | ttr p95 s | pending max | worker mem peak MiB | cpu peak % | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| small | 10 | 1/1 | 5.53 (4.98–6.08) | 23.70 (18.39–29.00) | 12.11 | 23.86 (18.55–29.18) | 131 (117–145) | 90 | 96 | +| small | 10 | 2/2 | 8.28 (6.58–9.98) | 7.35 (0.03–14.68) | 4.13 | 7.59 (0.22–14.96) | 50 (3–96) | 99 | 192 | +| small | 10 | 2/16 | 8.40 (6.83–9.98) | 6.72 (0.01–13.43) | 4.02 | 6.96 (0.19–13.72) | 50 (2–97) | 101 | 179 | +| small | 10 | 4/4 | 7.78 (5.58–9.98) | 11.58 (0.01–23.15) | 5.76 | 11.92 (0.19–23.65) | 68 (2–134) | 108 | 192 | +| medium | 5 | 1/1 | 2.80 (2.48–3.12) | 23.42 (17.32–29.51) | 14.22 | 23.73 (17.62–29.83) | 70 (58–83) | 110 | 97 | +| medium | 5 | 2/2 | 4.58 (4.27–4.90) | 2.64 (0.58–4.70) | 1.65 | 3.05 (0.97–5.13) | 14 (5–23) | 142 | 190 | +| medium | 5 | 2/16 | 4.30 (3.70–4.90) | 4.79 (0.60–8.97) | 2.34 | 5.26 (1.00–9.52) | 20 (5–35) | 164 | 190 | +| medium | 5 | 4/4 | 4.50 (4.04–4.96) | 2.74 (0.08–5.39) | 1.71 | 3.53 (0.49–6.57) | 13 (2–24) | 166 (126–205) | 200 | +| large | 3 | 1/1 | 1.73 (1.59–1.87) | 21.01 (17.21–24.82) | 10.85 | 21.59 (17.73–25.45) | 38 (34–41) | 197 | 100 | +| large | 3 | 2/2 | 2.44 (1.96–2.91) | 6.77 (0.24–13.31) | 3.94 | 7.65 (0.85–14.45) | 14 (3–25) | 294 (267–322) | 197 | +| large | 3 | 2/16 | 2.32 (2.31–2.33) | 7.19 (6.33–8.04) | 2.00 | 8.11 (7.38–8.84) | 16 (13–20) | 280 | 197 | +| large | 3 | 4/4 | 2.44 (2.05–2.84) | 5.65 (0.89–10.40) | 3.92 | 7.50 (2.29–12.71) | 15 (7–23) | 517 (485–549) | 203 | + +Перегрузка: large, 8/с (≈3× ёмкости), c=2/p=2, `MAX_PENDING_JOBS=30`, 240 запросов, после разбора очереди — проба одной маленькой загрузки: + +| вариант admission | принято | 503 | прочие ошибки | pending max (лимит) | throughput | ttr p95 s | разбор после нагрузки s | проба ready s | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| мягкий, r1 | 128 | 112 | 0 | 74 (30) | 2.24 | 32.7 | 28.4 | 0.23 | +| мягкий, r2 | 143 | 97 | 0 | 81 (30) | 2.38 | 34.1 | 30.7 | 0.51 | +| строгий, r1 | 108 | 132 | 0 | **30 (30)** | 2.38 | 18.9 | 12.5 | 0.23 | +| строгий, r2 | 100 | 140 | 0 | **30 (30)** | 2.21 | 16.4 | 15.2 | 0.24 | + +«Мягкий» — первая версия, где admission проверялся только до чтения тела по счётчику, обновляемому раз в секунду: загрузки, уже прошедшие проверку и ещё передающие/валидирующие байты, не учитывались. «Строгий» — исправленная версия: тот же лимит проверяется ещё и в транзакции перевода в `queued` под `pg_advisory_xact_lock` ([app/repo.py](app/repo.py) `queue_attachment`). Ранняя дешёвая проверка оставлена, чтобы не принимать тело зря. + +**Выводы.** +- **Ёмкость ограничена CPU, а не concurrency.** При c=1 worker держит ~1 CPU (96–100%) и не успевает ни на одном размере: очередь растёт до 38–131 задания, ttr p95 — 18–30 с. При c=2 используются оба CPU (~190%) и нагрузка в основном выдерживается. При c=4 CPU тот же (~200%), throughput не растёт (medium 4,50 против 4,58/с; large 2,44 против 2,44/с), а пиковая память для 24 Мпикс PNG растёт с 294 до 517 MiB. Это и есть компромисс latency/ресурсы: concurrency выше числа CPU почти не уменьшает задержку, но требует почти вдвое больше памяти. Рабочая настройка — concurrency = число CPU worker-контейнера. +- **Prefetch 16 при concurrency 2 не помогает и ухудшает хвост:** medium ttr p95 5,3 против 3,1 с, large 8,1 против 7,7 с, при этом памяти больше. Сообщения, взятые одним worker «про запас», не может забрать другой освободившийся worker. Поэтому prefetch = concurrency. +- **Backpressure работает только со строгой проверкой.** Мягкая проверка при лимите 30 пропустила 74–81 задание в очередь, ttr p95 — 33–34 с. Со строгой проверкой pending max ровно 30 в обоих повторах; 132–140 из 240 запросов получили 503 с `Retry-After: 5`; ttr p95 принятых снизился до 16–19 с, разбор после нагрузки — с 28–31 до 12–15 с. Принятая работа не терялась: все принятые задания дошли до ready, `other_errors = 0`. Время до ready для принятого ограничено сверху ≈ лимит / ёмкость (30 / 2,3 ≈ 13 с + обработка). +- **Восстановление:** после разбора очереди новая загрузка становится ready за 0,23–0,51 с, то есть как без нагрузки. + +**Ограничения измерения.** +- Между повторами большой разброс, во втором повторе всё медленнее (small c=2: 9,98 против 6,58/с при том же ~190% CPU). Значит, на хосте выросла стоимость CPU-секунды: ноутбук, троттлинг, общие ядра Docker VM. Сравнивать конфигурации корректно внутри одного повтора, абсолютные числа — не SLA. +- Два повтора — мало для доверительных интервалов. +- Один worker-контейнер; генератор нагрузки и стенд на одной машине. +- Память по `docker stats` снимается раз в секунду, короткий пик дочернего процесса может не попасть в выборку (отдельно измерено ≈322–335 MiB на один 40-Мпикс файл, [variants_limits](evidence/variants_limits.txt)). +- Queue lag и ttr используют часы PostgreSQL (одни для всех событий). +- Сырые данные: [results.jsonl](evidence/track1/results.jsonl), лог прогона: [matrix.log](evidence/track1/matrix.log). Тестовые изображения генерируются заново (`evidence/track1/fixture-*` не в git, шум JPEG случайный). + +### Отказы | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановлены все worker, upload, рестарт API и RabbitMQ | 202, задание в durable-очереди, после старта worker → ready | `messages=1` до и после рестарта брокера, затем `queued → ready` | ~4 с после старта worker; потерь нет | +| API падает сразу после COMMIT, до publish (A) | relay любой реплики опубликует из outbox | клиент получил 502 от LB; запись есть; опубликована 1 раз; ready | ~1 с; потерь нет | +| RabbitMQ остановлен (I) | upload всё равно 202, задание ждёт в outbox | 202, outbox: 1 неопубликованная строка; после старта брокера → ready | ~10 с (старт брокера + reconnect); потерь нет | +| Worker падает после PUT, до COMMIT (B) | lease истекает, попытка 2 перезаписывает те же ключи | `lease_expired` → attempt 2 succeeded; 3 варианта, 4 объекта | ~25 с (lease 20 с + sweep 5 с); дублей нет | +| Worker падает после COMMIT, до ack (C) | redelivery → no-op, staging дочищен | `decision=done`, 1 success, staging удалён | мгновенно; дублей нет | +| Worker завис (G) | другой worker забирает задание, зависший отфенсен | `lease_expired` → другой owner → ready; проснувшийся: `lease lost` | ~25 с; дублей нет | +| Временные ошибки ×3 (D) / ×6 (F) | backoff с jitter; при исчерпании failed, replay → ready | D: ready с 4-й попытки; F: `retry_exhausted` после 5, replay → ready, повторный replay отклонён | D ~14 с, F ~52 с | +| Ядовитое сообщение (H) | в DLQ, без бесконечных повторов | `attachments.dlq` 0 → 1 | — | +| Отказ одной API-реплики (failover ×2) | LB уводит трафик | 0 неудачных записей, первая успешная запись через 0,8–1,0 с | без потерь и дублей | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Брокер — один узел RabbitMQ. Quorum-очередь даёт durability на диске, но не HA: при недоступном брокере приём продолжается через outbox, а обработка ждёт. +- Задание, у которого outbox-строка опубликована, но сообщение потом потеряно вне контракта брокера (например, удалён volume RabbitMQ), само не восстановится. Нужен «reaper», переиздающий давно queued задания. Не сделан, потому что при штатной работе он плодит дубли в длинной очереди. Обходной путь — повторный INSERT в outbox, как в тесте redelivery. +- Fault injection включается по имени файла и только при `FAULT_INJECTION=1`; в production-конфигурации флаг должен быть 0. +- Строгий admission limit сериализует COMMIT загрузок одним advisory lock: при наших скоростях (≤10 upload/с) это незаметно, но при сотнях загрузок в секунду станет узким местом. Тогда нужен счётчик-токен вместо `count(*)`. Загрузка, отклонённая на этом шаге, уже передала все байты (пропускная способность потрачена), поэтому дешёвая ранняя проверка до чтения тела оставлена. +- Exactly-once нет: at-least-once доставка + идемпотентный эффект. Дубли сообщений штатны и видны в логах как `noop decision=done|stale|busy`. +- ICC-профиль удаляется: изображения в Adobe RGB/Display P3 в вариантах будут чуть менее насыщенными. +- Один пользователь S3 для API и worker. Следующий шаг — раздельные политики (API: put staging / get final; worker: get staging / put final / delete staging). diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/broker.py b/app/broker.py new file mode 100644 index 00000000..12e5b274 --- /dev/null +++ b/app/broker.py @@ -0,0 +1,78 @@ +"""RabbitMQ topology and confirmed publishing (aio-pika). + +Topology (declared idempotently by every publisher and consumer): + exchange attachments direct, durable + queue attachments.process quorum, durable, dead-letter -> attachments.dlx, x-delivery-limit + exchange attachments.dlx fanout, durable + queue attachments.dlq quorum, durable (poison / undeliverable messages for the operator) +Messages are persistent; the channel uses publisher confirms, so `publish` returns only after the broker +has taken responsibility for the message (written to the quorum log). +""" +import json +import logging + +import aio_pika +from aio_pika import DeliveryMode, ExchangeType, Message + +from app import config + +log = logging.getLogger("messenger.broker") +ROUTING_KEY = "attachment.process.v1" + + +async def connect(): + return await aio_pika.connect_robust(config.BROKER_URL, timeout=5, client_properties={"connection_name": config.INSTANCE_ID}) + + +async def declare(channel): + dlx = await channel.declare_exchange(config.BROKER_DLX, ExchangeType.FANOUT, durable=True) + dlq = await channel.declare_queue(config.BROKER_DLQ, durable=True, arguments={"x-queue-type": "quorum"}) + await dlq.bind(dlx) + exchange = await channel.declare_exchange(config.BROKER_EXCHANGE, ExchangeType.DIRECT, durable=True) + queue = await channel.declare_queue( + config.BROKER_QUEUE, + durable=True, + arguments={ + "x-queue-type": "quorum", + "x-dead-letter-exchange": config.BROKER_DLX, + "x-delivery-limit": config.BROKER_DELIVERY_LIMIT, + }, + ) + await queue.bind(exchange, ROUTING_KEY) + return exchange, queue + + +class Publisher: + def __init__(self) -> None: + self.connection = None + self.channel = None + self.exchange = None + + async def ensure(self) -> None: + if self.channel is not None and not self.channel.is_closed: + return + if self.connection is None or self.connection.is_closed: + self.connection = await connect() + self.channel = await self.connection.channel(publisher_confirms=True) + self.exchange, _ = await declare(self.channel) + + async def publish(self, body: dict) -> None: + await self.ensure() + message = Message( + json.dumps(body, separators=(",", ":")).encode(), + content_type="application/json", + delivery_mode=DeliveryMode.PERSISTENT, + message_id=body["job_id"], + type=body["type"], + headers={"attempt": body["attempt"]}, + ) + # mandatory: an unroutable message is an error, not a silent drop; awaited confirm = broker ack. + await self.exchange.publish(message, routing_key=ROUTING_KEY, mandatory=True, timeout=5) + + async def close(self) -> None: + try: + if self.connection is not None: + await self.connection.close() + except Exception: + pass + self.connection = self.channel = self.exchange = None diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..fc6a68d4 --- /dev/null +++ b/app/config.py @@ -0,0 +1,98 @@ +import os +import socket + + +def _int(name, default): + value = os.environ.get(name) + return int(value) if value not in (None, "") else default + + +def _float(name, default): + value = os.environ.get(name) + return float(value) if value not in (None, "") else default + + +def _secret(name, default=""): + path = os.environ.get(name + "_FILE") + if path: + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + return os.environ.get(name, default) + + +DATABASE_URL = _secret("DATABASE_URL") +CACHE_URL = _secret("CACHE_URL") +SESSION_STORE = os.environ.get("SESSION_STORE", "postgres") +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30) +SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60) +DB_POOL_MIN = _int("DB_POOL_MIN", 1) +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0) +DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3) +DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000) +DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0) +REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0) +DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5) +READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0) +READY_FAILURES = _int("READY_FAILURES", 2) +CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2) +MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64) +MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536) +MAX_RSS_MB = _int("MAX_RSS_MB", 0) +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 +SESSION_REVOKED_TTL_SECONDS = max( + _int("SESSION_REVOKED_TTL_SECONDS", 60), + SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5, +) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY") +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760) +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000) +S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0) +S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0) +UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576) +RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300) +RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900) +UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2) + +# --- lab5: broker, jobs, worker --- +BROKER_URL = _secret("BROKER_URL") +BROKER_EXCHANGE = os.environ.get("BROKER_EXCHANGE", "attachments") +BROKER_QUEUE = os.environ.get("BROKER_QUEUE", "attachments.process") +BROKER_DLX = os.environ.get("BROKER_DLX", "attachments.dlx") +BROKER_DLQ = os.environ.get("BROKER_DLQ", "attachments.dlq") +BROKER_DELIVERY_LIMIT = _int("BROKER_DELIVERY_LIMIT", 20) +PIPELINE_VERSION = _int("PIPELINE_VERSION", 1) +WORKER_CONCURRENCY = _int("WORKER_CONCURRENCY", 2) +WORKER_PREFETCH = _int("WORKER_PREFETCH", 0) or WORKER_CONCURRENCY +JOB_MAX_ATTEMPTS = _int("JOB_MAX_ATTEMPTS", 5) +JOB_TIMEOUT_SECONDS = _int("JOB_TIMEOUT_SECONDS", 60) +JOB_LEASE_SECONDS = _int("JOB_LEASE_SECONDS", 20) +JOB_BACKOFF_BASE_SECONDS = _float("JOB_BACKOFF_BASE_SECONDS", 2.0) +JOB_BACKOFF_MAX_SECONDS = _float("JOB_BACKOFF_MAX_SECONDS", 30.0) +JOB_SWEEP_SECONDS = _float("JOB_SWEEP_SECONDS", 5.0) +WORKER_CHILD_MEMORY_MB = _int("WORKER_CHILD_MEMORY_MB", 768) +WORKER_GRACE_SECONDS = _int("WORKER_GRACE_SECONDS", 25) +WORKER_HEARTBEAT_FILE = os.environ.get("WORKER_HEARTBEAT_FILE", "/tmp/worker-heartbeat") +OUTBOX_POLL_SECONDS = _float("OUTBOX_POLL_SECONDS", 1.0) +OUTBOX_BATCH = _int("OUTBOX_BATCH", 50) +OUTBOX_RETENTION_SECONDS = _int("OUTBOX_RETENTION_SECONDS", 86400) +STAGING_RETENTION_SECONDS = _int("STAGING_RETENTION_SECONDS", 86400) +# Admission control: uploads get 503 while this many jobs are queued/processing (0 disables). +MAX_PENDING_JOBS = _int("MAX_PENDING_JOBS", 200) +PENDING_CHECK_SECONDS = _float("PENDING_CHECK_SECONDS", 1.0) +THUMBNAIL_SIZE = 256 +CROP_SIZE = 128 +OPTIMIZED_SIZE = 1280 +JPEG_QUALITY = _int("JPEG_QUALITY", 85) +WEBP_QUALITY = _int("WEBP_QUALITY", 80) +# Test stand only: filename markers like "fault-worker_crash_after_put" trigger crashes (see app/faults.py). +FAULT_INJECTION = os.environ.get("FAULT_INJECTION", "0") == "1" diff --git a/app/cursor.py b/app/cursor.py new file mode 100644 index 00000000..113aabc8 --- /dev/null +++ b/app/cursor.py @@ -0,0 +1,43 @@ +import base64 +import json +import re +import uuid +from datetime import datetime, timezone + +TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z") +TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class BadCursor(Exception): + pass + + +def fmt(ts: datetime) -> str: + return ts.astimezone(timezone.utc).strftime(TS_FORMAT) + + +def encode_cursor(scope: str, created_at: str, item_id: str) -> str: + raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def decode_cursor(scope: str, cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)) + data = json.loads(raw) + except Exception: + raise BadCursor() from None + if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope: + raise BadCursor() + ts, item_id = data["t"], data["i"] + if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str): + raise BadCursor() + try: + parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc) + if str(uuid.UUID(item_id)) != item_id: + raise BadCursor() + except ValueError: + raise BadCursor() from None + if encode_cursor(scope, ts, item_id) != cursor: + raise BadCursor() + return parsed, item_id diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..0bd63089 --- /dev/null +++ b/app/db.py @@ -0,0 +1,99 @@ +import asyncio +from contextlib import asynccontextmanager + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from app import config +from app.errors import unavailable + +BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout) + + +async def discard(conn) -> None: + try: + await conn.close() + except Exception: + pass + + +class Guarded: + def __init__(self, conn, timeout: float): + self.conn = conn + self.timeout = timeout + + async def execute(self, *args, **kwargs): + try: + return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout) + except asyncio.TimeoutError: + await discard(self.conn) + raise unavailable("Хранилище не ответило за отведённое время") from None + + def transaction(self): + return self.conn.transaction() + + +class Database: + def __init__(self) -> None: + options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % ( + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS * 2, + ) + self.pool = AsyncConnectionPool( + conninfo=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + timeout=config.DB_POOL_TIMEOUT, + max_lifetime=600.0, + open=False, + kwargs={ + "autocommit": True, + "row_factory": dict_row, + "connect_timeout": config.DB_CONNECT_TIMEOUT, + "options": options, + }, + ) + + async def open(self) -> None: + await self.pool.open(wait=False) + + async def close(self) -> None: + await self.pool.close() + + @asynccontextmanager + async def connection(self): + try: + async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn: + yield Guarded(conn, config.DB_OP_TIMEOUT) + except BROKEN: + raise unavailable("Хранилище недоступно") from None + + @asynccontextmanager + async def transaction(self): + async with self.connection() as conn: + async with conn.transaction(): + yield conn + + async def in_transaction(self, action): + async def run(): + async with self.transaction() as conn: + return await action(conn) + + try: + return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT) + except asyncio.TimeoutError: + raise unavailable("Хранилище не ответило за отведённое время") from None + + async def ready(self) -> bool: + try: + async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn: + try: + await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT) + except asyncio.TimeoutError: + await discard(conn) + return False + return True + except Exception: + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..8110d273 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,37 @@ +from typing import Optional + +from fastapi.responses import JSONResponse + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None): + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def bad(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(scheme: str = "Bearer") -> ApiError: + challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE + message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия" + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Объект не найден") + + +def unavailable(message: str) -> ApiError: + return ApiError(503, "unavailable", message, {"Retry-After": "1"}) + + +def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse: + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})}, + ) diff --git a/app/faults.py b/app/faults.py new file mode 100644 index 00000000..d9a2dfc0 --- /dev/null +++ b/app/faults.py @@ -0,0 +1,38 @@ +"""Fault injection for the lab stand (FAULT_INJECTION=1 only; the default is off). + +A fault is selected by the client-chosen display filename, e.g. `fault-worker_crash_after_put.png`. +Filenames never reach S3 keys or broker messages; the worker reads them from the DB like any metadata. +Crash faults fire only on the first attempt and first delivery, so recovery can be observed. + api_crash_after_commit API process exits right after COMMIT of attachment+job+outbox, before publish + worker_crash_after_put worker exits after writing final objects to S3, before the result COMMIT + worker_crash_after_commit worker exits after the result COMMIT, before broker ack + worker_hang worker blocks its event loop (no heartbeats) for 2.5 x lease on attempt 1 + transientN attempts 1..N fail with a transient error (N = 1..9) + permanent permanent error on the first attempt -> failed without retries +""" +import logging +import os +import re +from typing import Optional + +from app import config + +log = logging.getLogger("messenger.faults") +TRANSIENT_RE = re.compile(r"fault-transient([1-9])") + + +def active(filename: Optional[str], name: str) -> bool: + return config.FAULT_INJECTION and bool(filename) and ("fault-" + name) in filename + + +def transient_until(filename: Optional[str]) -> int: + if not config.FAULT_INJECTION or not filename: + return 0 + match = TRANSIENT_RE.search(filename) + return int(match.group(1)) if match else 0 + + +def crash(point: str) -> None: + log.error("FAULT INJECTION: process exits at %s", point) + logging.shutdown() + os._exit(70) diff --git a/app/imagecheck.py b/app/imagecheck.py new file mode 100644 index 00000000..fe698c55 --- /dev/null +++ b/app/imagecheck.py @@ -0,0 +1,213 @@ +"""Integrity checks for uploaded PNG/JPEG with bounded memory. + +PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in +0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy +stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is +only a warning. Both finish before anything is stored. +""" +import logging +import os +import struct +import subprocess +import threading +import time +import zlib + +from PIL import Image + +from app import config +from app.errors import ApiError, unavailable + +log = logging.getLogger("messenger.imagecheck") + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +Image.MAX_IMAGE_PIXELS = None +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +FULL_DECODE_MAX_PIXELS = 4000000 +SCAN_SECONDS = 10.0 +OUT_STEP = 1 << 20 +# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit. +DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null") +jpeg_slot = threading.Lock() + + +class Invalid(Exception): + pass + + +def invalid() -> ApiError: + return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG") + + +class RowPlan: + def __init__(self, segments): + self.segments = [s for s in segments if s[0] > 1 and s[1] > 0] + self.seg = 0 + self.done = 0 + self.pos = 0 + + def feed(self, data: bytes) -> None: + i, n = 0, len(data) + while i < n: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + if self.seg >= len(self.segments): + raise Invalid("data after the last row") + length, rows = self.segments[self.seg] + if self.pos > 0: + take = min(length - self.pos, n - i) + i += take + self.pos += take + if self.pos == length: + self.pos = 0 + self.done += 1 + continue + full = min((n - i) // length, rows - self.done) + if full > 0: + if max(data[i:i + full * length:length]) > 4: + raise Invalid("bad filter type") + i += full * length + self.done += full + else: + if data[i] > 4: + raise Invalid("bad filter type") + self.pos = n - i + i = n + + def complete(self) -> bool: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + return self.seg >= len(self.segments) and self.pos == 0 + + +def png_plan(width, height, depth, color, interlace): + bits = CHANNELS[color] * depth + if not interlace: + return RowPlan([(1 + (width * bits + 7) // 8, height)]) + segments = [] + for x0, y0, dx, dy in ADAM7: + w = (width - x0 + dx - 1) // dx if width > x0 else 0 + h = (height - y0 + dy - 1) // dy if height > y0 else 0 + if w and h: + segments.append((1 + (w * bits + 7) // 8, h)) + return RowPlan(segments) + + +def scan_png(fileobj) -> None: + deadline = time.monotonic() + SCAN_SECONDS + fileobj.seek(0) + if fileobj.read(8) != PNG_SIGNATURE: + raise Invalid("signature") + plan = decomp = None + palette = seen_idat = idat_closed = False + color = None + first = True + while True: + if time.monotonic() > deadline: + raise Invalid("scan too slow") + head = fileobj.read(8) + if len(head) < 8: + raise Invalid("truncated before IEND") + length, kind = struct.unpack(">I4s", head) + if length > 0x7FFFFFFF: + raise Invalid("chunk length") + if first and kind != b"IHDR": + raise Invalid("IHDR must be first") + crc = zlib.crc32(kind) + if kind == b"IHDR": + body = fileobj.read(length) + if not first or length != 13 or len(body) != 13: + raise Invalid("IHDR") + width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body) + if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1): + raise Invalid("IHDR fields") + if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]: + raise Invalid("color type / depth") + plan = png_plan(width, height, depth, color, interlace) + decomp = zlib.decompressobj() + crc = zlib.crc32(body, crc) + else: + remaining = length + while remaining: + buf = fileobj.read(min(remaining, 65536)) + if not buf: + raise Invalid("truncated chunk") + remaining -= len(buf) + crc = zlib.crc32(buf, crc) + if kind == b"IDAT": + if idat_closed or (color == 3 and not palette): + raise Invalid("IDAT order") + seen_idat = True + out = decomp.decompress(buf, OUT_STEP) + while True: + plan.feed(out) + if not decomp.unconsumed_tail: + break + if time.monotonic() > deadline: + raise Invalid("scan too slow") + out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP) + if kind == b"PLTE": + palette = True + if color in (0, 4) or length % 3 or not 3 <= length <= 768: + raise Invalid("PLTE") + elif kind != b"IDAT" and seen_idat: + idat_closed = True + tail = fileobj.read(4) + if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF: + raise Invalid("crc") + first = False + if kind == b"IEND": + if length or not seen_idat: + raise Invalid("IEND") + break + if not decomp.eof or decomp.unused_data or not plan.complete(): + raise Invalid("pixel data does not match IHDR") + + +def check_jpeg(fileobj) -> None: + fileobj.flush() + descriptor = fileobj.fileno() + os.lseek(descriptor, 0, os.SEEK_SET) + try: + with jpeg_slot: + done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=SCAN_SECONDS, check=False) + except subprocess.TimeoutExpired: + log.error("jpeg_check_timeout") + raise unavailable("Проверка изображения не успела завершиться") from None + # A signal means the checker died (e.g. OOM killer), which says nothing about the file. + if done.returncode < 0: + log.error("jpeg_check_killed signal=%d", -done.returncode) + raise unavailable("Проверка изображения прервана") + if done.returncode != 0: + raise invalid() + + +def inspect_image(fileobj, size: int, content_type: str) -> None: + """Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type).""" + try: + fileobj.seek(0) + img = Image.open(fileobj) + width, height = img.size + if img.format != FORMATS[content_type]: + raise invalid() + if width * height > config.MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей") + if content_type == "image/png": + scan_png(fileobj) + if width * height <= FULL_DECODE_MAX_PIXELS: + fileobj.seek(0) + Image.open(fileobj).load() + else: + check_jpeg(fileobj) + except ApiError: + raise + except Exception: + raise invalid() from None + finally: + fileobj.seek(0) diff --git a/app/imageproc.py b/app/imageproc.py new file mode 100644 index 00000000..13842a66 --- /dev/null +++ b/app/imageproc.py @@ -0,0 +1,115 @@ +"""Variant generation, run by the worker as a CHILD PROCESS with rlimits (address space, CPU seconds) and a +wall-clock timeout, so a hostile or huge image can only kill this process, never the worker. + +usage: python -m app.imageproc +stdout: JSON list of produced files. Exit codes: 0 ok, 3 invalid_image, 4 image_too_large, 5 image_too_complex. + +Rules (contracts/README.md "Фоновая обработка"): + thumbnail JPEG, fit into 256x256 keeping aspect ratio, never upscale, floor rounding, min 1 px + crop JPEG, centred square scaled to 128x128 (small sources are upscaled) + optimized WebP, fit into 1280x1280, never upscale, quality WEBP_QUALITY, method 4 +EXIF orientation is applied first; alpha is composited on white for JPEG (WebP keeps alpha). +EXIF/XMP/ICC are not copied to any variant (pixels are treated as sRGB). +""" +import json +import math +import os +import sys + +from PIL import Image, ImageFile, ImageOps + +Image.MAX_IMAGE_PIXELS = None # we enforce our own limit before decoding +ImageFile.LOAD_TRUNCATED_IMAGES = False + +THUMB, CROP, OPTIMIZED = 256, 128, 1280 +JPEG_QUALITY = int(os.environ.get("JPEG_QUALITY", "85")) +WEBP_QUALITY = int(os.environ.get("WEBP_QUALITY", "80")) +ORIENTATION_TAG = 0x0112 +EXIT_INVALID, EXIT_TOO_LARGE, EXIT_TOO_COMPLEX = 3, 4, 5 + + +def fit(width: int, height: int, bound: int): + """Largest size inside bound x bound with the same aspect ratio; floor rounding; never upscales.""" + if width <= bound and height <= bound: + return width, height + if width >= height: + return bound, max(1, height * bound // width) + return max(1, width * bound // height), bound + + +def has_alpha(img) -> bool: + return img.mode in ("RGBA", "LA", "PA") or (img.mode == "P" and "transparency" in img.info) + + +def to_8bit(img): + if img.mode in ("I;16", "I;16B", "I;16L", "I"): + return img.convert("I").point(lambda v: v * (1 / 257)).convert("L") + return img + + +def on_white(img): + if has_alpha(img): + rgba = img.convert("RGBA") + flat = Image.new("RGB", rgba.size, (255, 255, 255)) + flat.paste(rgba, mask=rgba.getchannel("A")) + return flat, rgba + return img.convert("RGB"), None + + +def process(source: str, out_dir: str, max_pixels: int) -> list: + img = Image.open(source) + if img.format not in ("PNG", "JPEG"): + raise ValueError("format") + width, height = img.size + if width * height > max_pixels: + sys.exit(EXIT_TOO_LARGE) + orientation = img.getexif().get(ORIENTATION_TAG, 1) + upright = (height, width) if orientation in (5, 6, 7, 8) else (width, height) + thumb_size = fit(*upright, THUMB) + opt_size = fit(*upright, OPTIMIZED) + if img.format == "JPEG": + # DCT scaling: decode directly at >= the largest size we need (memory and time, exact output sizes kept). + need = opt_size if orientation not in (5, 6, 7, 8) else (opt_size[1], opt_size[0]) + img.draft(img.mode, (math.ceil(need[0]), math.ceil(need[1]))) + img.load() + img = ImageOps.exif_transpose(img) + img = to_8bit(img) + flat, rgba = on_white(img) + del img + + results = [] + + def save(name, image, ext, fmt, content_type, **params): + path = os.path.join(out_dir, name + "." + ext) + image.save(path, fmt, **params) + results.append({"name": name, "file": path, "content_type": content_type, + "width": image.size[0], "height": image.size[1]}) + + jpeg = {"quality": JPEG_QUALITY, "optimize": True, "progressive": False} + save("thumbnail", flat.resize(thumb_size, Image.Resampling.LANCZOS, reducing_gap=3.0), + "jpg", "JPEG", "image/jpeg", **jpeg) + w, h = flat.size + side = min(w, h) + left, top = (w - side) // 2, (h - side) // 2 + save("crop", flat.resize((CROP, CROP), Image.Resampling.LANCZOS, box=(left, top, left + side, top + side)), + "jpg", "JPEG", "image/jpeg", **jpeg) + source_for_webp = rgba if rgba is not None else flat + optimized = source_for_webp if source_for_webp.size == opt_size else source_for_webp.resize( + opt_size, Image.Resampling.LANCZOS, reducing_gap=3.0) + save("optimized", optimized, "webp", "WEBP", "image/webp", quality=WEBP_QUALITY, method=4) + return results + + +def main() -> None: + source, out_dir, max_pixels = sys.argv[1], sys.argv[2], int(sys.argv[3]) + try: + results = process(source, out_dir, max_pixels) + except MemoryError: + sys.exit(EXIT_TOO_COMPLEX) + except (OSError, ValueError, SyntaxError, Image.DecompressionBombError): + sys.exit(EXIT_INVALID) + json.dump(results, sys.stdout) + + +if __name__ == "__main__": + main() diff --git a/app/jobs.py b/app/jobs.py new file mode 100644 index 00000000..8d57aa93 --- /dev/null +++ b/app/jobs.py @@ -0,0 +1,318 @@ +"""Durable attachment jobs: state machine in PostgreSQL, delivery through the outbox and the broker. + +State lives in `attachment_jobs` (source of truth); the broker only carries a reference to it +(contracts/jobs.schema.json). Every transition below is one SQL transaction that also writes +`job_audit`, and, when the job must run again, an `outbox` row in the same COMMIT. + + enqueue nothing -> queued (API, same txn as attachment 'queued') + claim queued(due) -> processing, lease (worker; also takes over an expired lease) + heartbeat processing: lease_until += lease (only while owner is still alive) + complete processing -> succeeded + variants (fenced by lease_owner) + fail_attempt processing -> queued(+backoff) | failed (transient vs permanent, bounded attempts) + sweep_leases processing(expired lease) -> queued | failed + replay failed -> queued (explicit operator action, audited) +""" +import json +import random +import uuid +from datetime import datetime, timedelta, timezone +from typing import List, Optional + +from app import config +from app.cursor import fmt + +JOB_TYPE = "attachment.process.v1" +SCHEMA_VERSION = 1 +VARIANT_ORDER = ("thumbnail", "crop", "optimized") +REQUIRED_VARIANTS = VARIANT_ORDER + + +def backoff_seconds(attempt: int) -> float: + """Equal jitter: half of the exponential delay is fixed, the other half random (never zero, never herd).""" + ceiling = min(config.JOB_BACKOFF_MAX_SECONDS, config.JOB_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))) + return ceiling / 2 + random.uniform(0, ceiling / 2) + + +def payload(job_id: str, attachment_id: str, pipeline_version: int, attempt: int, created_at: datetime) -> dict: + return { + "schema_version": SCHEMA_VERSION, + "job_id": str(job_id), + "type": JOB_TYPE, + "attachment_id": str(attachment_id), + "pipeline_version": pipeline_version, + "attempt": attempt, + "created_at": fmt(created_at), + } + + +def parse_message(body: bytes) -> Optional[dict]: + """Strict check against jobs.schema.json; anything else goes to the DLQ untouched.""" + try: + data = json.loads(body.decode("utf-8")) + except (UnicodeDecodeError, ValueError): + return None + required = {"schema_version", "job_id", "type", "attachment_id", "pipeline_version", "attempt", "created_at"} + if not isinstance(data, dict) or not required <= set(data) or set(data) - required - {"traceparent", "tracestate"}: + return None + if data["schema_version"] != SCHEMA_VERSION or data["type"] != JOB_TYPE: + return None + for key in ("pipeline_version", "attempt"): + if not isinstance(data[key], int) or isinstance(data[key], bool) or data[key] < 1: + return None + try: + data["job_id"] = str(uuid.UUID(data["job_id"])) + data["attachment_id"] = str(uuid.UUID(data["attachment_id"])) + except (ValueError, AttributeError, TypeError): + return None + return data + + +async def audit(conn, job_id, attachment_id, action: str, actor: str, attempt: int, detail: Optional[dict] = None): + await conn.execute( + "INSERT INTO job_audit (job_id, attachment_id, action, actor, attempt, detail, created_at)" + " VALUES (%s, %s, %s, %s, %s, %s::jsonb, clock_timestamp())", + (job_id, attachment_id, action, actor, attempt, json.dumps(detail or {})), + ) + + +async def _outbox(conn, job: dict, attempt: int, available_at_sql: str = "clock_timestamp()", delay: float = 0.0): + body = payload(job["id"], job["attachment_id"], job["pipeline_version"], attempt, job["created_at"]) + await conn.execute( + "INSERT INTO outbox (job_id, payload, available_at, created_at)" + " VALUES (%s, %s::jsonb, " + available_at_sql + " + make_interval(secs => %s), clock_timestamp())", + (job["id"], json.dumps(body), delay), + ) + + +async def enqueue(conn, attachment_id: str, actor: str) -> dict: + cur = await conn.execute( + "INSERT INTO attachment_jobs (id, attachment_id, pipeline_version, state, attempt, max_attempts," + " next_attempt_at, created_at, updated_at)" + " VALUES (gen_random_uuid(), %s, %s, 'queued', 1, %s, clock_timestamp(), clock_timestamp(), clock_timestamp())" + " RETURNING id, attachment_id, pipeline_version, attempt, created_at", + (attachment_id, config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS), + ) + job = await cur.fetchone() + await _outbox(conn, job, 1) + await audit(conn, job["id"], attachment_id, "enqueued", actor, 1) + return job + + +async def pending_count(conn) -> int: + cur = await conn.execute("SELECT count(*) AS n FROM attachment_jobs WHERE state IN ('queued', 'processing')") + return (await cur.fetchone())["n"] + + +# ---------------------------------------------------------------- worker side + +JOB_COLUMNS = ( + "j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.lease_owner, j.lease_until," + " j.next_attempt_at, j.created_at, (j.lease_until < clock_timestamp()) AS lease_expired," + " (j.next_attempt_at <= clock_timestamp()) AS due" +) + + +async def claim(conn, msg: dict, owner: str): + """Returns (decision, job). decision: claimed | done | busy | stale | early | missing | newer_version.""" + cur = await conn.execute( + "SELECT " + JOB_COLUMNS + ", a.pipeline_version AS published_version, a.status AS attachment_status" + " FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.id = %s FOR UPDATE OF j", + (msg["job_id"],), + ) + job = await cur.fetchone() + if job is None or str(job["attachment_id"]) != msg["attachment_id"]: + return "missing", None + if job["state"] in ("succeeded", "failed"): + return "done", job + if msg["pipeline_version"] != job["pipeline_version"] or job["published_version"] > job["pipeline_version"]: + return "newer_version", job + if msg["attempt"] != job["attempt"]: + return "stale", job + if job["state"] == "processing" and not job["lease_expired"]: + return "busy", job + if job["state"] == "queued" and not job["due"]: + return "early", job + takeover = job["state"] == "processing" + await conn.execute( + "UPDATE attachment_jobs SET state = 'processing', lease_owner = %s," + " lease_until = clock_timestamp() + make_interval(secs => %s)," + " started_at = clock_timestamp(), first_started_at = coalesce(first_started_at, clock_timestamp())," + " updated_at = clock_timestamp() WHERE id = %s", + (owner, config.JOB_LEASE_SECONDS, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'processing', updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (job["attachment_id"],), + ) + await audit(conn, job["id"], job["attachment_id"], "lease_takeover" if takeover else "claimed", owner, job["attempt"]) + return "claimed", job + + +async def heartbeat(conn, job_id, owner: str) -> bool: + cur = await conn.execute( + "UPDATE attachment_jobs SET lease_until = clock_timestamp() + make_interval(secs => %s), updated_at = clock_timestamp()" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s", + (config.JOB_LEASE_SECONDS, job_id, owner), + ) + return cur.rowcount == 1 + + +async def attachment_source(conn, attachment_id) -> Optional[dict]: + cur = await conn.execute( + "SELECT id, filename, content_type, size_bytes, sha256, staging_key, object_key, status" + " FROM attachments WHERE id = %s", + (attachment_id,), + ) + return await cur.fetchone() + + +async def complete(conn, job: dict, owner: str, variants: List[dict]) -> bool: + """Atomically publish the result. False when the lease was lost: someone else owns the effect now.""" + cur = await conn.execute( + "UPDATE attachment_jobs SET state = 'succeeded', finished_at = clock_timestamp(), lease_owner = NULL," + " lease_until = NULL, error_code = NULL, updated_at = clock_timestamp()" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s", + (job["id"], owner), + ) + if cur.rowcount != 1: + return False + for v in variants: + await conn.execute( + "INSERT INTO attachment_variants (attachment_id, pipeline_version, name, object_key, content_type," + " size_bytes, width, height, sha256, created_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, clock_timestamp())" + " ON CONFLICT (attachment_id, pipeline_version, name) DO NOTHING", + (job["attachment_id"], job["pipeline_version"], v["name"], v["object_key"], v["content_type"], + v["size_bytes"], v["width"], v["height"], v["sha256"]), + ) + # Never move a newer pipeline_version back, never touch an attachment that is already ready. + await conn.execute( + "UPDATE attachments SET status = 'ready', error_code = NULL, pipeline_version = %s, updated_at = clock_timestamp()" + " WHERE id = %s AND status <> 'ready' AND pipeline_version <= %s", + (job["pipeline_version"], job["attachment_id"], job["pipeline_version"]), + ) + await audit(conn, job["id"], job["attachment_id"], "succeeded", owner, job["attempt"], + {"variants": [v["name"] for v in variants]}) + return True + + +async def _reschedule_or_fail(conn, job: dict, actor: str, error_code: str, detail: str, transient: bool) -> str: + attempt = job["attempt"] + if transient and attempt < job["max_attempts"]: + delay = backoff_seconds(attempt) + await conn.execute( + "UPDATE attachment_jobs SET state = 'queued', attempt = attempt + 1, lease_owner = NULL, lease_until = NULL," + " next_attempt_at = clock_timestamp() + make_interval(secs => %s), last_error = %s, error_code = %s," + " updated_at = clock_timestamp() WHERE id = %s", + (delay, detail[:500], error_code, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (job["attachment_id"],), + ) + await _outbox(conn, job, attempt + 1, delay=delay) + await audit(conn, job["id"], job["attachment_id"], "retry_scheduled", actor, attempt, + {"error_code": error_code, "detail": detail[:200], "delay_seconds": round(delay, 2)}) + return "retry" + final_code = error_code if not transient else "retry_exhausted" + await conn.execute( + "UPDATE attachment_jobs SET state = 'failed', lease_owner = NULL, lease_until = NULL, finished_at = clock_timestamp()," + " last_error = %s, error_code = %s, updated_at = clock_timestamp() WHERE id = %s", + (detail[:500], final_code, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = %s, updated_at = clock_timestamp()" + " WHERE id = %s AND status IN ('queued', 'processing')", + (final_code, job["attachment_id"]), + ) + await audit(conn, job["id"], job["attachment_id"], "failed", actor, attempt, + {"error_code": final_code, "cause": error_code, "detail": detail[:200]}) + return "failed" + + +async def fail_attempt(conn, job: dict, owner: str, error_code: str, detail: str, transient: bool) -> str: + """Returns retry | failed | lost (lease is not ours any more: the new owner decides).""" + cur = await conn.execute( + "SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at FROM attachment_jobs" + " WHERE id = %s AND state = 'processing' AND lease_owner = %s FOR UPDATE", + (job["id"], owner), + ) + row = await cur.fetchone() + if row is None: + return "lost" + return await _reschedule_or_fail(conn, row, owner, error_code, detail, transient) + + +async def sweep_leases(conn, actor: str) -> List[str]: + """Expired lease = the owner is dead or hung (no heartbeat). Counts as a used attempt.""" + cur = await conn.execute( + "SELECT id, attachment_id, pipeline_version, attempt, max_attempts, created_at, lease_owner" + " FROM attachment_jobs WHERE state = 'processing' AND lease_until < clock_timestamp()" + " ORDER BY lease_until LIMIT 20 FOR UPDATE SKIP LOCKED", + ) + outcomes = [] + for row in await cur.fetchall(): + outcome = await _reschedule_or_fail(conn, row, actor, "lease_expired", + "lease of %s expired" % (row["lease_owner"] or "?"), True) + outcomes.append("%s:%s" % (row["id"], outcome)) + return outcomes + + +async def replay(conn, attachment_id: str, actor: str, staging_exists) -> dict: + """failed -> queued with a fresh attempt budget. Refuses (no-op) for anything that is not failed.""" + cur = await conn.execute( + "SELECT j.id, j.attachment_id, j.pipeline_version, j.state, j.attempt, j.max_attempts, j.created_at," + " a.status AS attachment_status, a.staging_key" + " FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.attachment_id = %s ORDER BY j.pipeline_version DESC LIMIT 1 FOR UPDATE OF j, a", + (attachment_id,), + ) + job = await cur.fetchone() + if job is None: + return {"result": "refused", "reason": "no job for attachment"} + if job["state"] != "failed" or job["attachment_status"] != "failed": + await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], + {"state": job["state"], "attachment_status": job["attachment_status"]}) + return {"result": "refused", "reason": "job is %s, attachment is %s" % (job["state"], job["attachment_status"])} + if not job["staging_key"] or not await staging_exists(job["staging_key"]): + await audit(conn, job["id"], job["attachment_id"], "replay_refused", actor, job["attempt"], {"reason": "staging_missing"}) + return {"result": "refused", "reason": "staging object is gone (retention), re-upload is required"} + attempt = job["attempt"] + 1 + await conn.execute( + "UPDATE attachment_jobs SET state = 'queued', attempt = %s, max_attempts = %s, next_attempt_at = clock_timestamp()," + " finished_at = NULL, error_code = NULL, updated_at = clock_timestamp() WHERE id = %s", + (attempt, attempt + config.JOB_MAX_ATTEMPTS - 1, job["id"]), + ) + await conn.execute( + "UPDATE attachments SET status = 'queued', error_code = NULL, updated_at = clock_timestamp()" + " WHERE id = %s AND status = 'failed'", + (job["attachment_id"],), + ) + await _outbox(conn, job, attempt) + await audit(conn, job["id"], job["attachment_id"], "replayed", actor, attempt) + return {"result": "queued", "job_id": str(job["id"]), "attempt": attempt} + + +async def failed_jobs(conn, limit: int = 50) -> List[dict]: + cur = await conn.execute( + "SELECT j.id, j.attachment_id, j.attempt, j.max_attempts, j.error_code, j.last_error, j.updated_at" + " FROM attachment_jobs j WHERE j.state = 'failed' ORDER BY j.updated_at DESC LIMIT %s", + (limit,), + ) + return [ + {"job_id": str(r["id"]), "attachment_id": str(r["attachment_id"]), "attempt": r["attempt"], + "max_attempts": r["max_attempts"], "error_code": r["error_code"], "last_error": r["last_error"], + "failed_at": fmt(r["updated_at"])} + for r in await cur.fetchall() + ] + + +def utcnow() -> datetime: + return datetime.now(timezone.utc) + + +def older_than(seconds: int) -> datetime: + return utcnow() - timedelta(seconds=seconds) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..3b74bfd1 --- /dev/null +++ b/app/main.py @@ -0,0 +1,693 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import logging +import os +import re +import resource +import signal +import tempfile +import unicodedata +import sys +import uuid +from contextlib import asynccontextmanager +from typing import List, Optional, Tuple + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from starlette.concurrency import iterate_in_threadpool, run_in_threadpool +from starlette.responses import StreamingResponse + +from app import config, faults, jobs, passwords, reconcile, repo +from app.cursor import BadCursor, decode_cursor, encode_cursor +from app.db import Database +from app.imagecheck import inspect_image +from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable +from app.broker import Publisher +from app.outbox import Relay +from app.s3 import S3Storage +from app.sessions import SessionStore + +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger") + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"}) +EXEMPT_PATHS = ("/health/live", "/health/ready") +PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096 +ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"} +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" + +upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY) +background = set() +db = Database() +sessions = SessionStore(db) +s3 = S3Storage() +publisher = Publisher() +relay = Relay(db, publisher) +state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}, "pending_jobs": 0} +VARIANTS = ("original", "thumbnail", "crop", "optimized") + + +def rss_mb() -> float: + try: + with open("/proc/self/statm") as handle: + return int(handle.read().split()[1]) * PAGE_SIZE / 1048576 + except OSError: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak / 1048576 if sys.platform == "darwin" else peak / 1024 + + +def count_rejection(code: str) -> None: + state["rejected"][code] = state["rejected"].get(code, 0) + 1 + + +async def sweeper() -> None: + while True: + await asyncio.sleep(config.SESSION_SWEEP_SECONDS) + try: + removed = await sessions.sweep() + if removed: + log.info("sessions_expired removed=%d", removed) + except Exception: + log.warning("session sweep skipped: storage unavailable") + try: + async with db.connection() as conn: + removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS) + if removed: + log.info("idempotency_cleanup removed=%d", removed) + except Exception: + log.warning("idempotency cleanup skipped") + try: + removed = await relay.cleanup() + if removed: + log.info("outbox_cleanup removed=%d", removed) + except Exception: + log.warning("outbox cleanup skipped") + + +async def pending_watch() -> None: + """Admission control input: number of accepted-but-unfinished jobs, refreshed once per second.""" + while True: + try: + async with db.connection() as conn: + state["pending_jobs"] = await jobs.pending_count(conn) + except Exception: + pass + await asyncio.sleep(config.PENDING_CHECK_SECONDS) + + +async def reconciler_loop() -> None: + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + while True: + try: + result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False) + if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))): + log.info("reconcile %s", result) + except Exception: + log.warning("reconcile skipped: dependency unavailable") + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + + +async def readiness_watch() -> None: + failures = 0 + while True: + if await db.ready(): + failures = 0 + if not state["ready"]: + state["ready"] = True + log.info("ready again: database reachable") + else: + failures += 1 + if failures >= config.READY_FAILURES and state["ready"]: + state["ready"] = False + log.warning("not ready: database unreachable, business requests get 503") + s3_ok = await run_in_threadpool(s3.probe) + if s3_ok != state["s3_ok"]: + state["s3_ok"] = s3_ok + log.info("object storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503") + await asyncio.sleep(config.READY_CHECK_SECONDS) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + previous = {} + + def on_signal(signum, frame): + if not state["draining"]: + state["draining"] = True + log.info("signal %s: draining, new requests get 503", signal.Signals(signum).name) + handler = previous.get(signum) + if callable(handler): + handler(signum, frame) + + for sig in (signal.SIGTERM, signal.SIGINT): + previous[sig] = signal.getsignal(sig) + signal.signal(sig, on_signal) + await db.open() + await sessions.open() + await run_in_threadpool(s3.open) + state["s3_ok"] = await run_in_threadpool(s3.probe) + tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop()), + asyncio.create_task(relay.loop()), asyncio.create_task(pending_watch())] + log.info( + "started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s", + config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS, + config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES, + config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET, + ) + log.info("lab5: pipeline_version=%d max_attempts=%d max_pending_jobs=%d fault_injection=%s", + config.PIPELINE_VERSION, config.JOB_MAX_ATTEMPTS, config.MAX_PENDING_JOBS, config.FAULT_INJECTION) + yield + for task in tasks: + task.cancel() + await publisher.close() + await sessions.close() + await db.close() + log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"]) + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +@app.middleware("http") +async def guard(request: Request, call_next): + rid = uuid.uuid4().hex + request.state.rid = rid + exempt = request.url.path in EXEMPT_PATHS + if state["draining"] and request.url.path != "/health/live": + count_rejection("draining") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"}) + if not exempt and not state["ready"]: + count_rejection("not_ready") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"}) + if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS: + count_rejection("inflight") + return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"}) + state["inflight"] += 1 + try: + response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT) + except asyncio.TimeoutError: + count_rejection("request_timeout") + log.warning("request timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path) + response = error_body( + rid, config.INSTANCE_ID, 503, "unavailable", + "Запрос не удалось обработать за отведённое время", {"Retry-After": "1"}, + ) + except Exception: + log.exception("unhandled error request_id=%s", rid) + response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера") + finally: + state["inflight"] -= 1 + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + if exc.status in (413, 429, 503): + count_rejection(exc.code) + return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers) + + +def parse_uuid(value: object) -> Optional[str]: + if not isinstance(value, str): + return None + try: + return str(uuid.UUID(value)) + except ValueError: + return None + + +def non_blank(value: object, max_len: int) -> bool: + if not isinstance(value, str) or not 1 <= len(value) <= max_len: + return False + return value.translate(ZERO_WIDTH).strip() != "" + + +async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict: + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое") + declared = request.headers.get("content-length") + if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES: + raise too_large + raw = bytearray() + async for chunk in request.stream(): + raw += chunk + if len(raw) > config.MAX_BODY_BYTES: + raise too_large + try: + body = json.loads(bytes(raw).decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad("Некорректный JSON") from None + if not isinstance(body, dict): + raise bad("Ожидается JSON-объект") + allowed = set(required) | set(optional) + if set(body) - allowed: + raise bad("Лишние поля в запросе") + if set(required) - set(body): + raise bad("Отсутствуют обязательные поля") + return body + + +def ensure_capacity() -> None: + if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB: + raise unavailable("Достигнут лимит памяти процесса, запись недоступна") + + +def bearer_token(request: Request) -> str: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token: + raise unauthorized() + return token + + +async def actor(request: Request) -> dict: + user = await sessions.resolve(bearer_token(request)) + if user is None: + raise unauthorized() + return user + + +def basic_credentials(request: Request) -> Tuple[str, str]: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized("Basic") + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized("Basic") from None + username, separator, password = decoded.partition(":") + if not separator: + raise unauthorized("Basic") + return username, password + + +async def member_chat(user: dict, chat_id: str) -> dict: + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + async with db.connection() as conn: + chat = await repo.chat_for_member(conn, cid, user["id"]) + if chat is None: + raise not_found() + return chat + + +def page_params(request: Request) -> Tuple[int, Optional[str]]: + query = request.query_params + limit = 50 + if "limit" in query: + if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100: + raise bad("limit должен быть целым 1..100") + limit = int(query["limit"]) + cursor = query.get("cursor") + if cursor is not None and cursor == "": + raise bad("Некорректный cursor") + return limit, cursor + + +def keyset(scope: str, cursor: Optional[str]): + if cursor is None: + return None + try: + return decode_cursor(scope, cursor) + except BadCursor: + raise bad("Некорректный cursor") from None + + +def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse: + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None + return JSONResponse({"items": items, "next_cursor": next_cursor}) + + +def detect_content_type(head: bytes) -> Optional[str]: + if head[:8] == PNG_MAGIC: + return "image/png" + if head[:3] == JPEG_MAGIC: + return "image/jpeg" + return None + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."): + raise bad("filename: 1..255 символов, не только пробелы") + if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename): + raise bad("filename содержит недопустимые символы") + + +@app.get("/health/live") +async def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def ready(): + if not await db.ready(): + raise unavailable("База данных недоступна") + if not state["s3_ok"]: + raise unavailable("Хранилище файлов недоступно") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + body = await read_json(request, ("username", "display_name", "password")) + if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]): + raise bad("username должен соответствовать [a-z0-9_]{3,32}") + if not non_blank(body["display_name"], 100): + raise bad("display_name: 1..100 символов, не только пробелы") + password = body["password"] + if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX)) + ensure_capacity() + password_hash = await run_in_threadpool(passwords.hash_password, password) + async with db.connection() as conn: + user = await repo.create_user(conn, body["username"], body["display_name"], password_hash) + if user is None: + raise ApiError(409, "conflict", "username уже занят") + log.info("user_created id=%s", user["id"]) + return JSONResponse(user, status_code=201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + if await request.body(): + raise bad("POST /auth/sessions не принимает тело запроса") + username, password = basic_credentials(request) + row = None + if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + async with db.connection() as conn: + row = await repo.credentials(conn, username) + stored = row["password_hash"] if row else passwords.DUMMY_HASH + matches = await run_in_threadpool(passwords.verify_password, stored, password) + if row is None or not matches: + log.info("login_failed username_known=%s", row is not None) + raise unauthorized("Basic") + user = repo.public_user(row) + token, expires_at = await sessions.create(user) + log.info("session_created user=%s", user["id"]) + return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request): + token = bearer_token(request) + user = await sessions.resolve(token) + if user is None: + raise unauthorized() + await sessions.revoke(token) + log.info("session_revoked user=%s", user["id"]) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + return await actor(request) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await actor(request) + body = await read_json(request, ("title", "member_ids")) + if not non_blank(body["title"], 100): + raise bad("title: 1..100 символов, не только пробелы") + raw_ids = body["member_ids"] + if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100: + raise bad("member_ids: массив из 1..100 UUID") + ids: List[str] = [] + for value in raw_ids: + parsed = parse_uuid(value) + if parsed is None: + raise bad("member_ids содержит некорректный UUID") + ids.append(parsed) + if len(set(ids)) != len(ids): + raise bad("member_ids содержит дубликаты") + if user["id"] not in ids: + raise bad("Создатель должен входить в member_ids") + ensure_capacity() + + async def insert(conn): + return await repo.create_chat(conn, body["title"], ids) + + try: + chat = await db.in_transaction(insert) + except repo.UnknownMember: + raise bad("member_ids содержит неизвестного пользователя") from None + log.info("chat_created id=%s members=%d", chat["id"], len(ids)) + return JSONResponse(chat, status_code=201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await actor(request) + limit, cursor = page_params(request) + scope = "chats:" + user["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_chats(conn, user["id"], limit, after) + return page(scope, items, has_more) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + return await member_chat(await actor(request), chat_id) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + user = await actor(request) + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + + idem_key = request.headers.get("idempotency-key", "") + if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key): + raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}") + + body = await read_json(request, ("text",), ("attachment_ids",)) + + text = body["text"] + attachment_ids_raw = body.get("attachment_ids", []) + + if not isinstance(attachment_ids_raw, list): + raise bad("attachment_ids должен быть массивом UUID") + if len(attachment_ids_raw) > 10: + raise bad("attachment_ids: максимум 10 вложений") + + attachment_ids: List[str] = [] + for value in attachment_ids_raw: + parsed = parse_uuid(value) + if parsed is None: + raise bad("attachment_ids содержит некорректный UUID") + attachment_ids.append(parsed) + + if len(set(attachment_ids)) != len(attachment_ids): + raise bad("attachment_ids содержит дубликаты") + + has_text = isinstance(text, str) and non_blank(text, 4000) + has_attachments = len(attachment_ids) > 0 + + if not has_text and not has_attachments: + raise bad("text: 1..4000 символов, не только пробелы") + + if isinstance(text, str) and len(text) > 4000: + raise bad("text: 1..4000 символов, не только пробелы") + + if not isinstance(text, str): + raise bad("text должен быть строкой") + + ensure_capacity() + + normalized = {"text": text} + if attachment_ids: + normalized["attachment_ids"] = attachment_ids + body_hash = hashlib.sha256( + json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode() + ).hexdigest() + + async def do(conn): + return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash) + + status, message = await db.in_transaction(do) + + if status == "conflict": + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса") + if status == "not_member": + raise not_found() + if status == "attachment_not_found": + raise not_found() + if status == "attachment_failed": + raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить") + return JSONResponse(message, status_code=201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + chat = await member_chat(await actor(request), chat_id) + limit, cursor = page_params(request) + scope = "messages:" + chat["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_messages(conn, chat["id"], limit, after) + return page(scope, items, has_more) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment(chat_id: str, request: Request): + user = await actor(request) + chat = await member_chat(user, chat_id) + + filename = request.query_params.get("filename", "") + validate_filename(filename) + + declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if declared_ct not in ALLOWED_CONTENT_TYPES: + raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg") + + declared_length = request.headers.get("content-length") + too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB") + if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES: + raise too_large + ensure_capacity() + # Backpressure: do not accept more work than the pipeline can hold; the client retries later. + if config.MAX_PENDING_JOBS and state["pending_jobs"] >= config.MAX_PENDING_JOBS: + count_rejection("queue_full") + raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"}) + + sha = hashlib.sha256() + size = 0 + with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool: + head = b"" + async for chunk in request.stream(): + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + raise too_large + if len(head) < 8: + head += chunk[: 8 - len(head)] + if len(head) == 8 and detect_content_type(head) is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + sha.update(chunk) + spool.write(chunk) + if size == 0: + raise bad("Пустой файл") + actual_ct = detect_content_type(head) + if actual_ct is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + if actual_ct != declared_ct: + raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла") + async with upload_slots: + await run_in_threadpool(inspect_image, spool, size, actual_ct) + + attachment_id = str(uuid.uuid4()) + staging_key = "staging/" + attachment_id + object_key = "attachments/%s/original" % attachment_id + async with db.connection() as conn: + await repo.begin_attachment( + conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(), + staging_key, object_key, + ) + try: + async with upload_slots: + await run_in_threadpool(s3.put, staging_key, spool, size, actual_ct) + except BaseException: + defer(compensate(attachment_id, staging_key)) + raise + + # Durable handoff: staging object exists; one COMMIT makes the attachment 'queued' and writes job + outbox. + try: + att = await db.in_transaction(lambda conn: repo.queue_attachment( + conn, attachment_id, "api:" + config.INSTANCE_ID, config.MAX_PENDING_JOBS)) + except BaseException: + await compensate(attachment_id, staging_key) + raise + if att == repo.QUEUE_FULL: + defer(compensate(attachment_id, staging_key)) + count_rejection("queue_full") + raise ApiError(503, "unavailable", "Очередь обработки переполнена, повторите позже", {"Retry-After": "5"}) + if att is None: + raise unavailable("Загрузка отменена сверкой, повторите запрос") + if faults.active(filename, "api_crash_after_commit"): + faults.crash("API after COMMIT of job/outbox, before publish") + relay.notify() + log.info("attachment_queued id=%s chat=%s size=%d", att["id"], chat["id"], size) + return JSONResponse(att, status_code=202) + + +def defer(coro) -> None: + task = asyncio.ensure_future(coro) + background.add(task) + task.add_done_callback(background.discard) + + +async def compensate(attachment_id: str, object_key: str) -> None: + """Best effort; whatever is left behind is collected by the reconciler after the grace period.""" + try: + async with db.connection() as conn: + await repo.abort_attachment(conn, attachment_id) + await run_in_threadpool(s3.delete, object_key) + except BaseException: + log.warning("compensation incomplete attachment=%s, left for reconciler", attachment_id) + + +async def accessible_attachment(request: Request, attachment_id: str) -> dict: + user = await actor(request) + aid = parse_uuid(attachment_id) + if aid is None: + raise bad("attachment_id должен быть UUID") + async with db.connection() as conn: + row = await repo.get_attachment(conn, aid) + if row is None: + raise not_found() + chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"]) + if chat is None: + raise not_found() + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, request: Request): + row = await accessible_attachment(request, attachment_id) + variants = [] + if row["status"] == "ready": + async with db.connection() as conn: + variants = await repo.get_variants(conn, row["id"], row["pipeline_version"]) + return repo._attachment(row, variants) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, request: Request): + row = await accessible_attachment(request, attachment_id) + variant = request.query_params.get("variant", "original") + if variant not in VARIANTS: + raise bad("variant: original, thumbnail, crop или optimized") + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение ещё не готово" if row["status"] in ("queued", "processing") + else "Обработка вложения завершилась ошибкой") + key, content_type = row["object_key"], row["content_type"] + if variant != "original": + async with db.connection() as conn: + found = [v for v in await repo.get_variants(conn, row["id"], row["pipeline_version"]) if v["name"] == variant] + if not found: + raise not_found() + key, content_type = found[0]["object_key"], found[0]["content_type"] + opened = await run_in_threadpool(s3.open_stream, key) + if opened is None: + log.error("object_missing attachment=%s variant=%s", row["id"], variant) + raise unavailable("Файл временно недоступен") + chunks, length = opened + return StreamingResponse( + iterate_in_threadpool(chunks), media_type=content_type, headers={"Content-Length": str(length)}, + ) + + +@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def fallback(request: Request): + raise not_found() diff --git a/app/outbox.py b/app/outbox.py new file mode 100644 index 00000000..0f652942 --- /dev/null +++ b/app/outbox.py @@ -0,0 +1,86 @@ +"""Transactional outbox relay (runs inside every API replica; SKIP LOCKED keeps replicas from double work). + +Rows are locked while they are published; `published_at` is set in the same transaction only after the +broker confirmed the message. A crash between confirm and COMMIT republishes the row: duplicates are +expected and harmless (the worker's claim is idempotent). A crash between the upload COMMIT and publish +leaves an unpublished row that any replica picks up within OUTBOX_POLL_SECONDS. +""" +import asyncio +import logging + +from app import config + +log = logging.getLogger("messenger.outbox") + + +class Relay: + def __init__(self, db, publisher) -> None: + self.db = db + self.publisher = publisher + self.wake = asyncio.Event() + self.broker_ok = False + self.published = 0 + + def notify(self) -> None: + self.wake.set() + + async def run_once(self) -> int: + sent, failure = 0, None + async with self.db.connection() as conn: + async with conn.transaction(): + cur = await conn.execute( + "SELECT id, payload FROM outbox WHERE published_at IS NULL AND available_at <= clock_timestamp()" + " ORDER BY available_at, id LIMIT %s FOR UPDATE SKIP LOCKED", + (config.OUTBOX_BATCH,), + ) + for row in await cur.fetchall(): + try: + await self.publisher.publish(row["payload"]) + except Exception as exc: + # Keep what was already confirmed: commit those rows, record the error, stop the batch. + await conn.execute( + "UPDATE outbox SET publish_attempts = publish_attempts + 1, last_error = %s WHERE id = %s", + (type(exc).__name__, row["id"]), + ) + failure = exc + break + await conn.execute( + "UPDATE outbox SET published_at = clock_timestamp(), publish_attempts = publish_attempts + 1" + " WHERE id = %s", + (row["id"],), + ) + sent += 1 + self.published += sent + if failure is not None: + raise failure + return sent + + async def loop(self) -> None: + delay = config.OUTBOX_POLL_SECONDS + while True: + try: + await asyncio.wait_for(self.wake.wait(), delay) + except asyncio.TimeoutError: + pass + self.wake.clear() + try: + while await self.run_once() == config.OUTBOX_BATCH: + pass + if not self.broker_ok: + log.info("outbox relay: broker reachable, publishing") + self.broker_ok = True + delay = config.OUTBOX_POLL_SECONDS + except Exception as exc: + if self.broker_ok: + log.warning("outbox relay paused: %s; jobs stay in outbox", type(exc).__name__) + self.broker_ok = False + await self.publisher.close() + delay = min(10.0, max(delay * 2, 1.0)) + + async def cleanup(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute( + "DELETE FROM outbox WHERE published_at < clock_timestamp() - make_interval(secs => %s)", + (config.OUTBOX_RETENTION_SECONDS,), + ) + return cur.rowcount diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..b509e4a7 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,16 @@ +from argon2 import PasswordHasher +from argon2.exceptions import Argon2Error, VerifyMismatchError + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash("there-is-no-such-user-password") + + +def hash_password(password: str) -> str: + return HASHER.hash(password) + + +def verify_password(stored: str, password: str) -> bool: + try: + return HASHER.verify(stored, password) + except (VerifyMismatchError, Argon2Error, ValueError): + return False diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..60a138f9 --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,155 @@ +"""DB <-> S3 reconciliation and staging retention (lab4 reconciler extended for the lab5 key layout). + +Key layout (all keys are generated by the server): + staging/ written by the API before 202, read by the worker + attachments//original server-side copy made by the worker + attachments//v/.* variants of pipeline version N +Invariant: a row is inserted as 'uploading' BEFORE the staging object is written, so every object has a row. + +Cases (all gated by a grace period so in-flight work is never touched): + stale_uploading row 'uploading' older than grace AND its staging object (if any) older than grace -> delete both + orphan_object object without any row/variant row, older than grace -> delete object + staging_done staging object of a 'ready' attachment (worker crashed before its own cleanup) -> delete + staging_expired staging object of a 'failed' attachment older than STAGING_RETENTION_SECONDS -> delete + (after that a replay is refused; the attachment stays failed and visible) + missing_object 'ready' row whose original is gone (re-checked by HEAD) -> 'failed'/object_missing +Queued/processing work is never touched: its staging object is the input of a live job. +""" +import logging +import time +from datetime import datetime, timedelta, timezone + +from starlette.concurrency import run_in_threadpool + +from app import config + +log = logging.getLogger("messenger.reconcile") + +LOCK_ID = 2026_0104 +PREFIXES = ("staging/", "attachments/") +BATCH = 500 + + +async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict: + started = time.monotonic() + result = { + "dry_run": dry_run, "grace_seconds": grace_seconds, + "stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, "staging_done": 0, "staging_expired": 0, + "deleted_objects": 0, "removed_rows": 0, "marked_failed": 0, + "objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0, + } + async with db.connection() as conn: + cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,)) + if not (await cur.fetchone())["got"]: + return {"skipped": "another reconciler is running"} + try: + now = datetime.now(timezone.utc) + cutoff = now - timedelta(seconds=grace_seconds) + await _stale_uploading(conn, s3, cutoff, dry_run, result) + await _staging_cleanup(conn, s3, now - timedelta(seconds=config.STAGING_RETENTION_SECONDS), dry_run, result) + await _orphan_objects(conn, s3, cutoff, dry_run, result) + await _missing_objects(conn, s3, cutoff, dry_run, result) + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + result["seconds"] = round(time.monotonic() - started, 3) + return result + + +async def _stale_uploading(conn, s3, cutoff, dry_run, result): + cur = await conn.execute( + "SELECT id, coalesce(staging_key, object_key) AS key FROM attachments" + " WHERE status = 'uploading' AND created_at < %s", + (cutoff,), + ) + for row in await cur.fetchall(): + modified = await run_in_threadpool(s3.head, row["key"]) + if modified is not None and modified >= cutoff: + result["protected_fresh_objects"] += 1 + continue + result["stale_uploading"] += 1 + if dry_run: + continue + if modified is not None: + await run_in_threadpool(s3.delete, row["key"]) + result["deleted_objects"] += 1 + done = await conn.execute( + "DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff), + ) + result["removed_rows"] += done.rowcount + + +async def _staging_cleanup(conn, s3, retention_cutoff, dry_run, result): + cur = await conn.execute( + "SELECT id, staging_key, status FROM attachments WHERE staging_key IS NOT NULL" + " AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))", + (retention_cutoff,), + ) + for row in await cur.fetchall(): + if await run_in_threadpool(s3.head, row["staging_key"]) is None: + continue + result["staging_done" if row["status"] == "ready" else "staging_expired"] += 1 + if dry_run: + continue + again = await conn.execute( + "SELECT status FROM attachments WHERE id = %s AND (status = 'ready' OR (status = 'failed' AND updated_at < %s))", + (row["id"], retention_cutoff), + ) + if await again.fetchone() is not None: + await run_in_threadpool(s3.delete, row["staging_key"]) + result["deleted_objects"] += 1 + + +async def _known_keys(conn, keys): + cur = await conn.execute( + "SELECT object_key AS k FROM attachments WHERE object_key = ANY(%s)" + " UNION SELECT staging_key FROM attachments WHERE staging_key = ANY(%s)" + " UNION SELECT object_key FROM attachment_variants WHERE object_key = ANY(%s)", + (keys, keys, keys), + ) + return {row["k"] for row in await cur.fetchall()} + + +async def _orphan_objects(conn, s3, cutoff, dry_run, result): + for prefix in PREFIXES: + pages = await run_in_threadpool(lambda: iter(s3.list_pages(prefix))) + while True: + page = await run_in_threadpool(next, pages, None) + if page is None: + break + result["objects_scanned"] += len(page) + keys = [o["Key"] for o in page] + if not keys: + continue + known = await _known_keys(conn, keys) + for obj in page: + if obj["Key"] in known or obj["LastModified"] >= cutoff: + continue + result["orphan_objects"] += 1 + if not dry_run and not await _known_keys(conn, [obj["Key"]]): + await run_in_threadpool(s3.delete, obj["Key"]) + result["deleted_objects"] += 1 + + +async def _missing_objects(conn, s3, cutoff, dry_run, result): + last = None + while True: + cur = await conn.execute( + "SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s" + " AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s", + (cutoff, last, last, BATCH), + ) + rows = await cur.fetchall() + if not rows: + return + for row in rows: + result["rows_scanned"] += 1 + if await run_in_threadpool(s3.exists, row["object_key"]): + continue + result["missing_objects"] += 1 + if not dry_run: + done = await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'object_missing'," + " updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],), + ) + result["marked_failed"] += done.rowcount + last = str(rows[-1]["id"]) diff --git a/app/repo.py b/app/repo.py new file mode 100644 index 00000000..8eb215dc --- /dev/null +++ b/app/repo.py @@ -0,0 +1,338 @@ +import hashlib +import json +from typing import List, Optional, Tuple + +import psycopg + +from app import config, jobs +from app.cursor import fmt + +MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" +CHAT_QUERY = ( + "SELECT c.id, c.title, c.created_at," + " ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids" +) + + +class UnknownMember(Exception): + pass + + +def decoded(value) -> str: + return bytes(value).decode("utf-8") + + +def public_user(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": decoded(row["display_name"]), + "created_at": fmt(row["created_at"]), + } + + +def _chat(row) -> dict: + return { + "id": str(row["id"]), + "title": decoded(row["title"]), + "member_ids": [str(value) for value in row["member_ids"]], + "created_at": fmt(row["created_at"]), + } + + +def _message(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": decoded(row["text"]), + "created_at": fmt(row["created_at"]), + } + + +def _message_with_attachments(row, attachment_ids: List[str]) -> dict: + msg = _message(row) + msg["attachment_ids"] = attachment_ids + return msg + + +VARIANT_RANK = {"thumbnail": 0, "crop": 1, "optimized": 2} + + +def _variant(row) -> dict: + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "width": row["width"], + "height": row["height"], + "sha256": row["sha256"], + } + + +def _attachment(row, variants=None) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [_variant(v) for v in (variants or [])] if row["status"] == "ready" else [], + "error_code": row["error_code"], + "created_at": fmt(row["created_at"]), + "updated_at": fmt(row["updated_at"]), + } + + +async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]: + try: + cur = await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at)" + " VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())" + " RETURNING id, username, display_name, created_at", + (username, display_name.encode("utf-8"), password_hash), + ) + except psycopg.errors.UniqueViolation: + return None + return public_user(await cur.fetchone()) + + +async def credentials(conn, username: str): + cur = await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,), + ) + return await cur.fetchone() + + +async def create_chat(conn, title: str, member_ids: List[str]) -> dict: + cur = await conn.execute( + "INSERT INTO chats (id, title, created_at)" + " VALUES (gen_random_uuid(), %s, clock_timestamp())" + " RETURNING id, title, created_at", + (title.encode("utf-8"),), + ) + chat = await cur.fetchone() + cur = await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])", + (chat["id"], member_ids), + ) + if cur.rowcount != len(member_ids): + raise UnknownMember() + chat["member_ids"] = list(member_ids) + return _chat(chat) + + +async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]: + cur = await conn.execute( + CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s", + (user_id, chat_id), + ) + row = await cur.fetchone() + return _chat(row) if row else None + + +async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + CHAT_QUERY + + " FROM chats c JOIN chat_members m ON m.chat_id = c.id" + " WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))" + " ORDER BY c.created_at, c.id LIMIT %s", + (user_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + return [_chat(row) for row in rows[:limit]], len(rows) > limit + + +async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]: + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id), + ) + row = await cur.fetchone() + return _message(row) if row else None + + +async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]: + cur = await conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position", + (message_id,), + ) + rows = await cur.fetchall() + return [str(row["attachment_id"]) for row in rows] + + +async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None: + for pos, aid in enumerate(attachment_ids): + await conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message_id, aid, pos), + ) + + +async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + "SELECT " + MESSAGE_COLUMNS + " FROM messages" + " WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))" + " ORDER BY created_at, id LIMIT %s", + (chat_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + messages = [] + for row in rows[:limit]: + aids = await _fetch_message_attachment_ids(conn, row["id"]) + messages.append(_message_with_attachments(row, aids)) + return messages, len(rows) > limit + + +def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int: + raw = "%s:%s:%s" % (user_id, chat_id, key) + return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16) + + +async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str, + attachment_ids: List[str], idem_key: str, body_hash: str): + lock_id = _idem_lock_id(sender_id, chat_id, idem_key) + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,)) + + cur = await conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ) + if await cur.fetchone() is None: + return "not_member", None + + cur = await conn.execute( + "SELECT request_body_hash, response_json FROM idempotency_keys" + " WHERE user_id = %s AND chat_id = %s AND key = %s", + (sender_id, chat_id, idem_key), + ) + existing = await cur.fetchone() + + if existing: + if existing["request_body_hash"] != body_hash: + return "conflict", None + return "replay", json.loads(existing["response_json"]) + + if attachment_ids: + cur = await conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'", + (attachment_ids,), + ) + att_rows = await cur.fetchall() + found_ids = {str(row["id"]) for row in att_rows} + for aid in attachment_ids: + if aid not in found_ids: + return "attachment_not_found", None + for row in att_rows: + if str(row["chat_id"]) != chat_id: + return "attachment_not_found", None + if row["status"] == "failed": + return "attachment_failed", None + + text_bytes = text.encode("utf-8") if text else b"" + + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text_bytes, chat_id, sender_id), + ) + row = await cur.fetchone() + if row is None: + return "not_member", None + + if attachment_ids: + await _link_attachments(conn, row["id"], chat_id, attachment_ids) + + message = _message_with_attachments(row, list(attachment_ids)) + + await conn.execute( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)" + " VALUES (%s, %s, %s, %s, %s, clock_timestamp())", + (sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)), + ) + + return "created", message + + +async def cleanup_idempotency(conn, ttl_seconds: int) -> int: + cur = await conn.execute( + "DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)", + (ttl_seconds,), + ) + return cur.rowcount + + +async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str, + content_type: str, size_bytes: int, sha256: str, staging_key: str, object_key: str) -> None: + await conn.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key," + " staging_key, status, error_code, pipeline_version, created_at, updated_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, %s, clock_timestamp(), clock_timestamp())", + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, + config.PIPELINE_VERSION), + ) + + +ATTACHMENT_COLUMNS = ( + "id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, staging_key, status," + " error_code, pipeline_version, created_at, updated_at" +) + + +QUEUE_FULL = "queue_full" +ADMISSION_LOCK = 2026_0105 + + +async def queue_attachment(conn, attachment_id: str, actor: str, max_pending: int = 0): + """uploading -> queued plus job and outbox row, all in the caller's transaction (the durable handoff). + + With max_pending > 0 the admission limit is enforced here, serialized by a transaction-scoped advisory lock, + so accepted-but-unfinished jobs never exceed the limit regardless of how many uploads are in flight. + """ + if max_pending: + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (ADMISSION_LOCK,)) + if await jobs.pending_count(conn) >= max_pending: + return QUEUE_FULL + cur = await conn.execute( + "UPDATE attachments SET status = 'queued', updated_at = clock_timestamp()" + " WHERE id = %s AND status = 'uploading' RETURNING " + ATTACHMENT_COLUMNS, + (attachment_id,), + ) + row = await cur.fetchone() + if row is None: + return None + await jobs.enqueue(conn, attachment_id, actor) + return _attachment(row) + + +async def abort_attachment(conn, attachment_id: str) -> None: + await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,)) + + +async def get_attachment(conn, attachment_id: str): + cur = await conn.execute( + "SELECT " + ATTACHMENT_COLUMNS + " FROM attachments WHERE id = %s AND status <> 'uploading'", + (attachment_id,), + ) + return await cur.fetchone() + + +async def get_variants(conn, attachment_id, pipeline_version: int) -> List[dict]: + cur = await conn.execute( + "SELECT name, object_key, content_type, size_bytes, width, height, sha256 FROM attachment_variants" + " WHERE attachment_id = %s AND pipeline_version = %s", + (attachment_id, pipeline_version), + ) + rows = await cur.fetchall() + return sorted(rows, key=lambda r: VARIANT_RANK.get(r["name"], 99)) diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 00000000..642dd227 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,122 @@ +import logging +from typing import Iterator, Optional, Tuple + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError + +from app import config +from app.errors import unavailable + +log = logging.getLogger("messenger.s3") + +MISSING = ("NoSuchKey", "404", "NotFound") + + +class S3Storage: + def __init__(self) -> None: + self.client = None + self.probe_client = None + + def open(self) -> None: + self.probe_client = self._client(1.0, 1.5) + self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT) + + def _client(self, connect: float, read: float): + return boto3.client( + "s3", + endpoint_url=config.S3_ENDPOINT, + region_name=config.S3_REGION, + aws_access_key_id=config.S3_ACCESS_KEY_ID, + aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect, + read_timeout=read, + retries={"max_attempts": 0}, + ), + ) + + def probe(self) -> bool: + try: + self.probe_client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False + + def put(self, key: str, fileobj, size: int, content_type: str) -> None: + try: + fileobj.seek(0) + self.client.put_object( + Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type, + ) + except (BotoCoreError, ClientError) as exc: + log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + + def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]: + try: + response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + except BotoCoreError as exc: + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + body = response["Body"] + return body.iter_chunks(65536), int(response["ContentLength"]) + + def head(self, key: str): + """LastModified of the object, or None when it does not exist.""" + try: + return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"] + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + raise + + def exists(self, key: str) -> bool: + return self.head(key) is not None + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=config.S3_BUCKET, Key=key) + + def list_pages(self, prefix: str): + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}): + yield page.get("Contents", []) + + # ---- lab5 helpers used by the worker (exceptions propagate: the caller classifies them as transient) + + def put_file(self, key: str, path: str, content_type: str) -> None: + with open(path, "rb") as handle: + self.client.put_object(Bucket=config.S3_BUCKET, Key=key, Body=handle, ContentType=content_type) + + def download(self, key: str, path: str) -> bool: + """False when the object does not exist.""" + try: + response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return False + raise + with open(path, "wb") as handle: + for chunk in response["Body"].iter_chunks(1 << 20): + handle.write(chunk) + return True + + def copy(self, source_key: str, target_key: str) -> None: + """Server-side copy: the original bytes never pass through the worker again.""" + self.client.copy_object( + Bucket=config.S3_BUCKET, Key=target_key, CopySource={"Bucket": config.S3_BUCKET, "Key": source_key}, + ) + + def ready(self) -> bool: + try: + self.client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..31d472f4 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,154 @@ +import hashlib +import json +import logging +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional + +import valkey.asyncio as valkey + +from app import config, repo +from app.cursor import fmt +from app.errors import unavailable + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionStore: + def __init__(self, db) -> None: + self.db = db + self.client = None + self.stats = {"hits": 0, "misses": 0, "errors": 0} + + async def open(self) -> None: + if not config.CACHE_URL: + log.info("session cache disabled: CACHE_URL is empty") + return + self.client = valkey.Valkey.from_url( + config.CACHE_URL, + socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, + decode_responses=True, + ) + + async def close(self) -> None: + if self.client is not None: + await self.client.aclose() + self.client = None + + @staticmethod + def digest(token: str) -> bytes: + return hashlib.sha256(token.encode("utf-8")).digest() + + @staticmethod + def key(digest: bytes) -> str: + return "session:" + digest.hex() + + async def create(self, user: dict) -> tuple: + token = secrets.token_urlsafe(32) + digest = self.digest(token) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + async with self.db.connection() as conn: + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at)" + " VALUES (%s, %s, clock_timestamp(), %s)", + (digest, user["id"], expires_at), + ) + await self._cache_put(digest, user, expires_at) + return token, fmt(expires_at) + + async def resolve(self, token: str) -> Optional[dict]: + digest = self.digest(token) + cached = await self._cache_get(digest) + if cached == REVOKED: + self.stats["hits"] += 1 + return None + if cached is not None: + self.stats["hits"] += 1 + return cached + self.stats["misses"] += 1 + async with self.db.connection() as conn: + cur = await conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at" + " FROM sessions s JOIN users u ON u.id = s.user_id" + " WHERE s.token_hash = %s AND s.expires_at > now()", + (digest,), + ) + row = await cur.fetchone() + if row is None: + return None + user = repo.public_user(row) + await self._cache_put(digest, user, row["expires_at"]) + return user + + async def revoke(self, token: str) -> None: + digest = self.digest(token) + await self._cache_revoke(digest) + try: + async with self.db.connection() as conn: + await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,)) + except Exception: + await self._cache_release(digest) + raise + + async def sweep(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + return cur.rowcount + + def _cache_ttl(self, expires_at: datetime) -> int: + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + return min(config.SESSION_CACHE_TTL_SECONDS, remaining) + + async def _cache_get(self, digest: bytes) -> Optional[dict]: + if self.client is None: + return None + try: + raw = await self.client.get(self.key(digest)) + except Exception: + self._cache_failed("get") + return None + if not raw: + return None + if raw == REVOKED: + return REVOKED + try: + return json.loads(raw) + except ValueError: + return None + + async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None: + if self.client is None: + return + ttl = self._cache_ttl(expires_at) + if ttl <= 0: + return + payload = json.dumps(user) + try: + await self.client.set(self.key(digest), payload, ex=ttl, nx=True) + except Exception: + self._cache_failed("set") + + async def _cache_release(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.delete(self.key(digest)) + except Exception: + self._cache_failed("release") + + async def _cache_revoke(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS) + except Exception: + self._cache_failed("revoke") + raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None + + def _cache_failed(self, operation: str) -> None: + self.stats["errors"] += 1 + if self.stats["errors"] % 50 == 1: + log.warning("session cache %s failed, falling back to postgres (errors=%d)", operation, self.stats["errors"]) diff --git a/app/worker.py b/app/worker.py new file mode 100644 index 00000000..a6ff6cb0 --- /dev/null +++ b/app/worker.py @@ -0,0 +1,329 @@ +"""Attachment worker: a separate process consuming attachments.process (python -m app.worker). + +Per delivery: + 1. parse/validate the message (jobs.schema.json); garbage -> reject(requeue=False) -> DLQ + 2. claim the job in PostgreSQL (lease); duplicates, stale attempts and finished jobs are acked as no-op + 3. download staging object, verify size+sha256 against DB metadata + 4. child process (app.imageproc) with RLIMIT_AS / RLIMIT_CPU / wall timeout builds the variants + 5. server-side copy staging -> final original key, PUT variants to deterministic keys (overwrite = same bytes) + 6. one COMMIT: variants + attachment ready + job succeeded, fenced by lease_owner + 7. delete staging (best effort, reconciler finishes it), then ack +Ack happens only after the durable effect (6) or after the failure/retry decision is committed. +A heartbeat extends the lease while the job is alive; an expired lease is recovered by sweep_leases. +""" +import asyncio +import hashlib +import json +import logging +import os +import resource +import signal +import sys +import tempfile +import time +import uuid + +from app import config, faults, jobs +from app.broker import connect, declare +from app.db import Database +from app.s3 import S3Storage + +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger.worker") + +PERMANENT_EXIT = {3: "invalid_image", 4: "image_too_large", 5: "image_too_complex"} + + +class Permanent(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail or code + + +class Transient(Exception): + def __init__(self, code: str, detail: str = ""): + super().__init__(code) + self.code, self.detail = code, detail or code + + +class LeaseLost(Exception): + pass + + +def child_limits() -> None: + memory = config.WORKER_CHILD_MEMORY_MB * 1024 * 1024 + resource.setrlimit(resource.RLIMIT_AS, (memory, memory)) + cpu = config.JOB_TIMEOUT_SECONDS + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + os.nice(5) + + +def sha256_file(path: str): + digest, size = hashlib.sha256(), 0 + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1 << 20), b""): + digest.update(chunk) + size += len(chunk) + return digest.hexdigest(), size + + +class Worker: + def __init__(self) -> None: + self.owner = "%s:%s" % (config.INSTANCE_ID, uuid.uuid4().hex[:8]) + self.db = Database() + self.s3 = S3Storage() + self.slots = asyncio.Semaphore(config.WORKER_CONCURRENCY) + self.stopping = asyncio.Event() + self.inflight = 0 + self.stats = {"succeeded": 0, "retry": 0, "failed": 0, "noop": 0, "dlq": 0, "lost": 0} + + # ------------------------------------------------------------------ lifecycle + + async def run(self) -> None: + loop = asyncio.get_running_loop() + for sig in (signal.SIGTERM, signal.SIGINT): + loop.add_signal_handler(sig, self.stopping.set) + await self.db.open() + await asyncio.to_thread(self.s3.open) + connection = await connect() + channel = await connection.channel() + await channel.set_qos(prefetch_count=config.WORKER_PREFETCH) + _, queue = await declare(channel) + tag = await queue.consume(self.on_message, no_ack=False) + background = [asyncio.create_task(self.sweeper()), asyncio.create_task(self.alive())] + log.info("worker started owner=%s concurrency=%d prefetch=%d lease=%ds timeout=%ds max_attempts=%d child_mem=%dMB", + self.owner, config.WORKER_CONCURRENCY, config.WORKER_PREFETCH, config.JOB_LEASE_SECONDS, + config.JOB_TIMEOUT_SECONDS, config.JOB_MAX_ATTEMPTS, config.WORKER_CHILD_MEMORY_MB) + await self.stopping.wait() + log.info("stopping: cancel consumer, wait up to %ds for %d in-flight job(s)", config.WORKER_GRACE_SECONDS, self.inflight) + try: + await queue.cancel(tag) + except Exception: + pass + deadline = time.monotonic() + config.WORKER_GRACE_SECONDS + while self.inflight and time.monotonic() < deadline: + await asyncio.sleep(0.2) + for task in background: + task.cancel() + # Unacked deliveries go back to the queue when the connection closes; their leases expire or are re-claimed. + await connection.close() + await self.db.close() + log.info("stopped stats=%s", self.stats) + + async def alive(self) -> None: + while True: + try: + with open(config.WORKER_HEARTBEAT_FILE, "w") as handle: + handle.write(str(time.time())) + except OSError: + pass + await asyncio.sleep(2) + + async def sweeper(self) -> None: + while True: + await asyncio.sleep(config.JOB_SWEEP_SECONDS) + try: + swept = await self.db.in_transaction(lambda conn: jobs.sweep_leases(conn, "sweeper:" + self.owner)) + if swept: + log.warning("lease_expired recovered=%s", swept) + except Exception: + log.warning("lease sweep skipped: database unavailable") + + # ------------------------------------------------------------------ delivery + + async def on_message(self, message) -> None: + async with self.slots: + self.inflight += 1 + try: + await self.handle(message) + except Exception: + log.exception("unexpected error, message returned to queue") + await asyncio.sleep(1) + await self.settle(message.nack(requeue=True)) + finally: + self.inflight -= 1 + + @staticmethod + async def settle(awaitable) -> None: + try: + await awaitable + except Exception: + log.warning("could not settle delivery (connection lost); broker will redeliver") + + async def handle(self, message) -> None: + msg = jobs.parse_message(message.body) + if msg is None: + self.stats["dlq"] += 1 + log.error("invalid message -> DLQ message_id=%s", message.message_id) + await self.settle(message.reject(requeue=False)) + return + try: + decision, job = await self.db.in_transaction(lambda conn: jobs.claim(conn, msg, self.owner)) + except Exception: + log.warning("claim failed: database unavailable, requeue job=%s", msg["job_id"]) + await asyncio.sleep(2) + await self.settle(message.nack(requeue=True)) + return + if decision == "done" and job["state"] == "succeeded": + await self.cleanup_staging(job["attachment_id"]) + if decision != "claimed": + self.stats["noop"] += 1 + log.info("noop job=%s attempt=%d decision=%s redelivered=%s", msg["job_id"], msg["attempt"], decision, + message.redelivered) + await self.settle(message.ack()) + return + started = time.monotonic() + log.info("claimed job=%s attachment=%s attempt=%d/%d", job["id"], job["attachment_id"], job["attempt"], + job["max_attempts"]) + lease_lost = asyncio.Event() + keeper = asyncio.create_task(self.keep_lease(job["id"], lease_lost)) + outcome = None + work = asyncio.create_task(self.process(job, message)) + lost = asyncio.create_task(lease_lost.wait()) + try: + done, _ = await asyncio.wait({work, lost}, timeout=config.JOB_TIMEOUT_SECONDS * 2, + return_when=asyncio.FIRST_COMPLETED) + if work not in done: + work.cancel() + raise LeaseLost() if lost in done else Transient("processing_timeout", "job exceeded hard timeout") + outcome = "succeeded" if work.result() else "lost" + except LeaseLost: + outcome = "lost" + except Permanent as exc: + outcome = await self.record_failure(job, exc.code, exc.detail, transient=False) + except Transient as exc: + outcome = await self.record_failure(job, exc.code, exc.detail, transient=True) + except Exception as exc: # S3/DB/network errors + outcome = await self.record_failure(job, "dependency_unavailable", type(exc).__name__, transient=True) + finally: + keeper.cancel() + lost.cancel() + if outcome is None: + # Could not record the decision: keep the message; the lease expires and the sweeper takes over. + await asyncio.sleep(2) + await self.settle(message.nack(requeue=True)) + return + self.stats[outcome] = self.stats.get(outcome, 0) + 1 + log.info("job=%s attempt=%d outcome=%s seconds=%.2f", job["id"], job["attempt"], outcome, time.monotonic() - started) + await self.settle(message.ack()) + + async def cleanup_staging(self, attachment_id) -> None: + """Redelivery of a finished job (e.g. crash after COMMIT, before ack): finish the idempotent cleanup.""" + try: + async with self.db.connection() as conn: + src = await jobs.attachment_source(conn, attachment_id) + if src and src["status"] == "ready" and src["staging_key"]: + await asyncio.to_thread(self.s3.delete, src["staging_key"]) + except Exception: + log.warning("staging cleanup deferred to reconciler attachment=%s", attachment_id) + + async def keep_lease(self, job_id, lease_lost: asyncio.Event) -> None: + interval = max(1.0, config.JOB_LEASE_SECONDS / 3) + while True: + await asyncio.sleep(interval) + try: + alive = await self.db.in_transaction(lambda conn: jobs.heartbeat(conn, job_id, self.owner)) + except Exception: + continue # DB blip: keep trying until the lease really expires + if not alive: + log.warning("lease lost job=%s: another worker owns it now", job_id) + lease_lost.set() + return + + async def record_failure(self, job, code: str, detail: str, transient: bool): + log.warning("job=%s attempt=%d error=%s transient=%s", job["id"], job["attempt"], code, transient) + try: + return await self.db.in_transaction( + lambda conn: jobs.fail_attempt(conn, job, self.owner, code, detail, transient)) + except Exception: + log.warning("could not record failure job=%s: database unavailable", job["id"]) + return None + + # ------------------------------------------------------------------ the actual work + + async def process(self, job, message) -> bool: + async with self.db.connection() as conn: + src = await jobs.attachment_source(conn, job["attachment_id"]) + if src is None or not src["staging_key"]: + raise Permanent("staging_missing", "attachment has no staging object") + filename = src["filename"] + first_delivery = job["attempt"] == 1 and not message.redelivered + if faults.active(filename, "worker_hang") and job["attempt"] == 1: + log.error("FAULT INJECTION: blocking event loop for %ds", int(config.JOB_LEASE_SECONDS * 2.5)) + time.sleep(config.JOB_LEASE_SECONDS * 2.5) + if faults.active(filename, "permanent"): + raise Permanent("invalid_image", "injected permanent failure") + if job["attempt"] <= faults.transient_until(filename): + raise Transient("dependency_unavailable", "injected transient failure") + + with tempfile.TemporaryDirectory(prefix="job-") as tmp: + source = os.path.join(tmp, "source") + if not await asyncio.to_thread(self.s3.download, src["staging_key"], source): + raise Permanent("staging_missing", "staging object not found") + digest, size = await asyncio.to_thread(sha256_file, source) + if digest != src["sha256"] or size != src["size_bytes"]: + raise Permanent("checksum_mismatch", "staging bytes differ from accepted upload") + out_dir = os.path.join(tmp, "out") + os.mkdir(out_dir) + produced = await self.run_child(source, out_dir) + + variants = [] + prefix = "attachments/%s/v%d/" % (job["attachment_id"], job["pipeline_version"]) + await asyncio.to_thread(self.s3.copy, src["staging_key"], src["object_key"]) + for item in produced: + digest, size = await asyncio.to_thread(sha256_file, item["file"]) + key = prefix + os.path.basename(item["file"]) + await asyncio.to_thread(self.s3.put_file, key, item["file"], item["content_type"]) + variants.append({"name": item["name"], "object_key": key, "content_type": item["content_type"], + "size_bytes": size, "width": item["width"], "height": item["height"], "sha256": digest}) + if set(v["name"] for v in variants) != set(jobs.REQUIRED_VARIANTS): + raise Permanent("processing_failed", "not all variants were produced") + + if first_delivery and faults.active(filename, "worker_crash_after_put"): + faults.crash("after S3 put, before result COMMIT") + committed = await self.db.in_transaction(lambda conn: jobs.complete(conn, job, self.owner, variants)) + if not committed: + log.warning("result not committed job=%s: lease lost, effect belongs to the new owner", job["id"]) + return False + if first_delivery and faults.active(filename, "worker_crash_after_commit"): + faults.crash("after result COMMIT, before ack") + try: + await asyncio.to_thread(self.s3.delete, src["staging_key"]) + except Exception: + log.warning("staging cleanup deferred to reconciler attachment=%s", job["attachment_id"]) + return True + + async def run_child(self, source: str, out_dir: str) -> list: + env = {"PATH": os.environ.get("PATH", ""), "MALLOC_ARENA_MAX": "2", "PYTHONPATH": "/app", + "JPEG_QUALITY": str(config.JPEG_QUALITY), "WEBP_QUALITY": str(config.WEBP_QUALITY)} + proc = await asyncio.create_subprocess_exec( + sys.executable, "-m", "app.imageproc", source, out_dir, str(config.MAX_IMAGE_PIXELS), + stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.DEVNULL, env=env, preexec_fn=child_limits, + ) + try: + stdout, _ = await asyncio.wait_for(proc.communicate(), config.JOB_TIMEOUT_SECONDS) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + raise Transient("processing_timeout", "wall-clock limit %ds" % config.JOB_TIMEOUT_SECONDS) from None + except asyncio.CancelledError: + proc.kill() + raise + code = proc.returncode + if code == 0: + return json.loads(stdout) + if code in PERMANENT_EXIT: + raise Permanent(PERMANENT_EXIT[code]) + if code == -signal.SIGXCPU: + raise Permanent("image_too_complex", "CPU limit exceeded") + if code == -signal.SIGKILL: + raise Transient("worker_resource_exhausted", "child killed (OOM)") + raise Permanent("processing_failed", "child exit %s" % code) + + +def main() -> None: + asyncio.run(Worker().run()) + + +if __name__ == "__main__": + main() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..16a75fa9 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,312 @@ +name: mtusi-lab5 + +x-api: &api-defaults + build: + context: . + image: mtusi-messenger:lab5 + env_file: + - .env + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + rabbitmq: + condition: service_healthy + postgres: + condition: service_healthy + valkey: + condition: service_healthy + minio-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""] + interval: 5s + timeout: 3s + retries: 3 + start_period: 15s + restart: unless-stopped + +services: + nginx: + image: nginx:1.27.3-alpine + command: ["/bin/sh", "/opt/nginx-health.sh"] + environment: + API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2} + HEALTH_INTERVAL: "1" + HEALTH_FALL: "2" + HEALTH_RISE: "2" + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx-health.sh:/opt/nginx-health.sh:ro + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s + restart: unless-stopped + + api-1: + <<: *api-defaults + environment: + INSTANCE_ID: api-1 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://minio:9000 + BROKER_URL_FILE: /run/secrets/broker_url + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200} + + api-2: + <<: *api-defaults + environment: + INSTANCE_ID: api-2 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://minio:9000 + BROKER_URL_FILE: /run/secrets/broker_url + MAX_PENDING_JOBS: ${MAX_PENDING_JOBS:-200} + + worker: + build: + context: . + image: mtusi-messenger:lab5 + command: ["python", "-m", "app.worker"] + env_file: + - .env + environment: + DATABASE_URL_FILE: /run/secrets/database_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + BROKER_URL_FILE: /run/secrets/broker_url + S3_ENDPOINT: http://minio:9000 + CACHE_URL_FILE: "" + INSTANCE_ID: "" + DB_POOL_MAX: "6" + # Overridable from the shell for the track-1 experiment (shell env wins over .env interpolation). + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + WORKER_PREFETCH: ${WORKER_PREFETCH:-2} + secrets: + - database_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + depends_on: + rabbitmq: + condition: service_healthy + postgres: + condition: service_healthy + minio-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + stop_grace_period: 30s + deploy: + replicas: ${WORKER_REPLICAS:-2} + resources: + limits: + cpus: "${WORKER_CPUS:-2.00}" + memory: 1536M + healthcheck: + test: ["CMD-SHELL", "python -c \"import os,sys,time; sys.exit(0 if time.time()-os.path.getmtime('/tmp/worker-heartbeat') < 15 else 1)\""] + interval: 5s + timeout: 3s + retries: 3 + start_period: 15s + restart: unless-stopped + + rabbitmq: + image: rabbitmq:4.0.5-alpine@sha256:e045b5166b9188a38eee5c99411709aa1f0ffa2ac75664053e64cb69411f7c32 + hostname: rabbitmq + entrypoint: ["/bin/sh", "/opt/entrypoint.sh"] + secrets: + - rabbitmq_password + volumes: + - rabbitmqdata:/var/lib/rabbitmq + - ./infra/rabbitmq/rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf:ro + - ./infra/rabbitmq/entrypoint.sh:/opt/entrypoint.sh:ro + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "check_port_connectivity"] + interval: 5s + timeout: 10s + retries: 20 + start_period: 20s + restart: unless-stopped + + postgres: + image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: --data-checksums + command: + - postgres + - -c + - synchronous_commit=on + - -c + - shared_buffers=128MB + - -c + - work_mem=8MB + - -c + - max_connections=50 + - -c + - track_io_timing=on + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: + - valkey-server + - --save + - "" + - --appendonly + - "no" + - --maxmemory + - 64mb + - --maxmemory-policy + - allkeys-lru + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + restart: unless-stopped + + minio: + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + command: ["server", "/data"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + MINIO_BROWSER: "off" + secrets: + - minio_root_user + - minio_root_password + volumes: + - miniodata:/data + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "mc ready local || exit 1"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + restart: unless-stopped + + minio-init: + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + entrypoint: ["/bin/sh", "/opt/init.sh"] + environment: + S3_BUCKET: messenger-attachments + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio/init.sh:/opt/init.sh:ro + - ./infra/minio/app-policy.json:/etc/minio-policy/app-policy.json:ro + depends_on: + minio: + condition: service_healthy + restart: "no" + + migrate: + build: + context: . + image: mtusi-messenger:lab5 + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: "no" + +volumes: + pgdata: + miniodata: + rabbitmqdata: + +secrets: + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + broker_url: + file: ./secrets/broker_url + rabbitmq_password: + file: ./secrets/rabbitmq_password diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..a87917dc --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,13 @@ +# failover — 2026-11-06T09:01:21Z +## run 1 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 0.571} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api-1 running +## run 2 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 0.784} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api-2 running diff --git a/evidence/failure_windows.txt b/evidence/failure_windows.txt new file mode 100644 index 00000000..2822acf1 --- /dev/null +++ b/evidence/failure_windows.txt @@ -0,0 +1,82 @@ +# grade 4 failure windows — 2026-11-06T08:58:05Z, base commit 276dbda + working tree, FAULT_INJECTION=1 +$ python3 student-tests/test_failure_windows.py +11:58:05 scenario A: scenario_a +11:58:05 client saw: missing X-Request-Id +11:58:05 committed attachment b5b077f9-ac62-4fa2-8936-9c1b3ee7b3a3 survived the API crash +11:58:06 statuses: processing -> ready +11:58:06 audit attempt=1 enqueued api:api-2 +11:58:06 audit attempt=1 claimed 926c7492ded8:13fd47b0 +11:58:06 audit attempt=1 succeeded 926c7492ded8:13fd47b0 +11:58:07 scenario A PASS (1.7s) +11:58:07 scenario B: scenario_b +11:58:30 statuses: queued -> processing -> queued -> ready +11:58:31 audit attempt=1 enqueued api:api-1 +11:58:31 audit attempt=1 claimed 0a275053a46c:9a3ac0c4 +11:58:31 audit attempt=1 retry_scheduled sweeper:0a275053a46c:e765083 lease_expired delay=1.6s +11:58:31 audit attempt=2 claimed 926c7492ded8:13fd47b0 +11:58:31 audit attempt=2 succeeded 926c7492ded8:13fd47b0 +11:58:32 scenario B PASS (24.9s) +11:58:32 scenario C: scenario_c +11:58:32 statuses: queued -> ready +11:58:32 audit attempt=1 enqueued api:api-2 +11:58:32 audit attempt=1 claimed 0a275053a46c:e765083c +11:58:32 audit attempt=1 succeeded 0a275053a46c:e765083c +11:58:42 scenario C PASS (10.3s) +11:58:42 scenario D: scenario_d +11:58:55 statuses: processing -> queued -> ready +11:58:56 audit attempt=1 enqueued api:api-1 +11:58:56 audit attempt=1 claimed 926c7492ded8:13fd47b0 +11:58:56 audit attempt=1 retry_scheduled 926c7492ded8:13fd47b0 dependency_unavailable delay=1.7s +11:58:56 audit attempt=2 claimed 0a275053a46c:fd20ea5b +11:58:56 audit attempt=2 retry_scheduled 0a275053a46c:fd20ea5b dependency_unavailable delay=3.13s +11:58:56 audit attempt=3 claimed 926c7492ded8:13fd47b0 +11:58:56 audit attempt=3 retry_scheduled 926c7492ded8:13fd47b0 dependency_unavailable delay=7.83s +11:58:56 audit attempt=4 claimed 0a275053a46c:fd20ea5b +11:58:56 audit attempt=4 succeeded 0a275053a46c:fd20ea5b +11:58:56 backoff delays: [1.7, 3.13, 7.83] +11:58:56 scenario D PASS (14.3s) +11:58:56 scenario E: scenario_e +11:58:57 audit attempt=1 enqueued api:api-2 +11:58:57 audit attempt=1 claimed 926c7492ded8:13fd47b0 +11:58:57 audit attempt=1 failed 926c7492ded8:13fd47b0 invalid_image +11:58:57 scenario E PASS (0.8s) +11:58:57 scenario F: scenario_f +11:59:22 failed registry contains the job; replay +11:59:46 statuses: queued -> processing -> ready +11:59:46 audit attempt=1 enqueued api:api-1 +11:59:46 audit attempt=1 claimed 0a275053a46c:fd20ea5b +11:59:46 audit attempt=1 retry_scheduled 0a275053a46c:fd20ea5b dependency_unavailable delay=1.05s +11:59:46 audit attempt=2 claimed 926c7492ded8:13fd47b0 +11:59:46 audit attempt=2 retry_scheduled 926c7492ded8:13fd47b0 dependency_unavailable delay=2.84s +11:59:46 audit attempt=3 claimed 0a275053a46c:fd20ea5b +11:59:46 audit attempt=3 retry_scheduled 0a275053a46c:fd20ea5b dependency_unavailable delay=5.8s +11:59:46 audit attempt=4 claimed 926c7492ded8:13fd47b0 +11:59:46 audit attempt=4 retry_scheduled 926c7492ded8:13fd47b0 dependency_unavailable delay=12.89s +11:59:46 audit attempt=5 claimed 0a275053a46c:fd20ea5b +11:59:46 audit attempt=5 failed 0a275053a46c:fd20ea5b retry_exhausted +11:59:46 audit attempt=6 replayed operator:test-suite +11:59:46 audit attempt=6 claimed 926c7492ded8:13fd47b0 +11:59:46 audit attempt=6 retry_scheduled 926c7492ded8:13fd47b0 dependency_unavailable delay=22.22s +11:59:46 audit attempt=7 claimed 0a275053a46c:fd20ea5b +11:59:46 audit attempt=7 succeeded 0a275053a46c:fd20ea5b +11:59:49 second replay refused: job is succeeded, attachment is ready +11:59:50 scenario F PASS (52.5s) +11:59:50 scenario G: scenario_g +12:00:15 statuses: queued -> processing -> queued -> processing -> ready +12:00:16 audit attempt=1 enqueued api:api-2 +12:00:16 audit attempt=1 claimed 926c7492ded8:13fd47b0 +12:00:16 audit attempt=1 retry_scheduled sweeper:0a275053a46c:fd20ea5 lease_expired delay=1.8s +12:00:16 audit attempt=2 claimed 0a275053a46c:fd20ea5b +12:00:16 audit attempt=2 succeeded 0a275053a46c:fd20ea5b +12:00:47 scenario G PASS (57.8s) +12:00:47 scenario H: scenario_h +12:00:55 DLQ depth 1 -> 2 +12:00:55 scenario H PASS (7.1s) +12:00:55 scenario I: scenario_i +12:00:59 broker down: upload 202, outbox holds 1 unpublished job +12:01:05 statuses: queued -> processing -> ready +12:01:05 audit attempt=1 enqueued api:api-1 +12:01:05 audit attempt=1 claimed 0a275053a46c:fd20ea5b +12:01:05 audit attempt=1 succeeded 0a275053a46c:fd20ea5b +12:01:06 scenario I PASS (11.8s) +PASS: failure windows A B C D E F G H I diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..c9311e12 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,36 @@ +# persistence — 2026-11-06T09:01:14Z +$ cat scripts/restart.sh +#!/bin/sh +# Restart API, LB, worker and every store (postgres, valkey, minio, rabbitmq) keeping all volumes. +set -eu +cd "$(dirname "$0")/.." +docker compose restart -t 10 postgres valkey minio rabbitmq +docker compose restart -t 25 worker api-1 api-2 nginx +## containers StartedAt before +mtusi-lab5-api-1-1 2026-11-06T08:57:35.409218424Z +mtusi-lab5-api-2-1 2026-11-06T08:58:05.819373965Z +mtusi-lab5-minio-1 2026-11-06T08:23:49.358710597Z +mtusi-lab5-nginx-1 2026-11-06T08:23:50.667392333Z +mtusi-lab5-postgres-1 2026-11-06T08:23:49.384126305Z +mtusi-lab5-rabbitmq-1 2026-11-06T09:00:59.611106504Z +mtusi-lab5-valkey-1 2026-11-06T08:23:49.401115472Z +mtusi-lab5-worker-1 2026-11-06T08:57:50.89592275Z +mtusi-lab5-worker-2 2026-11-06T08:58:32.593535839Z +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +## containers StartedAt after +mtusi-lab5-api-1-1 2026-11-06T09:01:17.294250804Z +mtusi-lab5-api-2-1 2026-11-06T09:01:17.309623971Z +mtusi-lab5-minio-1 2026-11-06T09:01:15.638306762Z +mtusi-lab5-nginx-1 2026-11-06T09:01:16.932816596Z +mtusi-lab5-postgres-1 2026-11-06T09:01:15.667794262Z +mtusi-lab5-rabbitmq-1 2026-11-06T09:01:16.680692637Z +mtusi-lab5-valkey-1 2026-11-06T09:01:15.692763803Z +mtusi-lab5-worker-1 2026-11-06T09:01:20.784383042Z +mtusi-lab5-worker-2 2026-11-06T09:01:20.824222792Z +## volumes +mtusi-lab5_miniodata +mtusi-lab5_pgdata +mtusi-lab5_rabbitmqdata diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..f09526af --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# make check / make test — 2026-11-06T08:57:29Z, base commit 276dbda + working tree, macOS arm64, Docker 29.7, 10 CPU/8 GiB VM +$ make check && make test +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/track1/results.jsonl b/evidence/track1/results.jsonl new file mode 100644 index 00000000..970512ce --- /dev/null +++ b/evidence/track1/results.jsonl @@ -0,0 +1,28 @@ +{"label": "c1-p1-r1", "tag": "09c4ed86", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 1, "prefetch": 1, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.92, "offered": 300, "drained": true, "drain_after_load_s": 19.81, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 6.076, "lag_p50_s": 9.771, "lag_p95_s": 18.388, "lag_max_s": 19.342, "ttr_p50_s": 9.936, "ttr_p95_s": 18.551, "ttr_max_s": 19.497, "pending_max": 117, "worker_mem_peak_mib": 91.8, "worker_mem_median_mib": 83.3, "worker_cpu_peak_pct": 96.1, "at": "2026-11-06T11:32:11"} +{"label": "c2-p2-r1", "tag": "1e2fd008", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.92, "offered": 300, "drained": true, "drain_after_load_s": 1.2, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 9.976, "lag_p50_s": 0.007, "lag_p95_s": 0.03, "lag_max_s": 0.066, "ttr_p50_s": 0.172, "ttr_p95_s": 0.217, "ttr_max_s": 0.245, "pending_max": 3, "worker_mem_peak_mib": 94.9, "worker_mem_median_mib": 84.6, "worker_cpu_peak_pct": 190.1, "at": "2026-11-06T11:32:51"} +{"label": "c4-p4-r1", "tag": "8befd323", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 4, "prefetch": 4, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.91, "offered": 300, "drained": true, "drain_after_load_s": 1.22, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 9.979, "lag_p50_s": 0.007, "lag_p95_s": 0.01, "lag_max_s": 0.022, "ttr_p50_s": 0.17, "ttr_p95_s": 0.186, "ttr_max_s": 0.225, "pending_max": 2, "worker_mem_peak_mib": 95.8, "worker_mem_median_mib": 83.0, "worker_cpu_peak_pct": 172.3, "at": "2026-11-06T11:33:31"} +{"label": "c2-p16-r1", "tag": "50386d2d", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 16, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.91, "offered": 300, "drained": true, "drain_after_load_s": 1.18, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 9.978, "lag_p50_s": 0.007, "lag_p95_s": 0.013, "lag_max_s": 0.046, "ttr_p50_s": 0.17, "ttr_p95_s": 0.19, "ttr_max_s": 0.219, "pending_max": 2, "worker_mem_peak_mib": 96.4, "worker_mem_median_mib": 82.8, "worker_cpu_peak_pct": 164.1, "at": "2026-11-06T11:34:11"} +{"label": "c1-p1-r1", "tag": "2e5ee814", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 1, "prefetch": 1, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.93, "offered": 150, "drained": true, "drain_after_load_s": 18.68, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 3.123, "lag_p50_s": 9.361, "lag_p95_s": 17.324, "lag_max_s": 18.089, "ttr_p50_s": 9.669, "ttr_p95_s": 17.622, "ttr_max_s": 18.39, "pending_max": 58, "worker_mem_peak_mib": 108.8, "worker_mem_median_mib": 79.6, "worker_cpu_peak_pct": 95.9, "at": "2026-11-06T11:35:09"} +{"label": "c2-p2-r1", "tag": "f236141e", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.93, "offered": 150, "drained": true, "drain_after_load_s": 1.27, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 4.898, "lag_p50_s": 0.336, "lag_p95_s": 0.578, "lag_max_s": 0.635, "ttr_p50_s": 0.738, "ttr_p95_s": 0.971, "ttr_max_s": 1.023, "pending_max": 5, "worker_mem_peak_mib": 148.3, "worker_mem_median_mib": 109.7, "worker_cpu_peak_pct": 190.5, "at": "2026-11-06T11:35:50"} +{"label": "c4-p4-r1", "tag": "7dd1545c", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 4, "prefetch": 4, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.94, "offered": 150, "drained": true, "drain_after_load_s": 1.2, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 4.956, "lag_p50_s": 0.034, "lag_p95_s": 0.082, "lag_max_s": 0.172, "ttr_p50_s": 0.425, "ttr_p95_s": 0.492, "ttr_max_s": 0.588, "pending_max": 2, "worker_mem_peak_mib": 126.5, "worker_mem_median_mib": 105.6, "worker_cpu_peak_pct": 198.6, "at": "2026-11-06T11:36:30"} +{"label": "c2-p16-r1", "tag": "2e139fa3", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 16, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.92, "offered": 150, "drained": true, "drain_after_load_s": 1.24, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 4.903, "lag_p50_s": 0.301, "lag_p95_s": 0.602, "lag_max_s": 0.649, "ttr_p50_s": 0.699, "ttr_p95_s": 0.999, "ttr_max_s": 1.036, "pending_max": 5, "worker_mem_peak_mib": 168.3, "worker_mem_median_mib": 101.8, "worker_cpu_peak_pct": 189.7, "at": "2026-11-06T11:37:11"} +{"label": "c1-p1-r1", "tag": "f7c65b4d", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 1, "prefetch": 1, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.81, "offered": 90, "drained": true, "drain_after_load_s": 18.64, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 1.871, "lag_p50_s": 8.815, "lag_p95_s": 17.208, "lag_max_s": 18.01, "ttr_p50_s": 9.335, "ttr_p95_s": 17.735, "ttr_max_s": 18.533, "pending_max": 34, "worker_mem_peak_mib": 197.9, "worker_mem_median_mib": 174.5, "worker_cpu_peak_pct": 100.2, "at": "2026-11-06T11:38:08"} +{"label": "c2-p2-r1", "tag": "b593ef8c", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.96, "offered": 90, "drained": true, "drain_after_load_s": 1.23, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 2.914, "lag_p50_s": 0.012, "lag_p95_s": 0.239, "lag_max_s": 0.41, "ttr_p50_s": 0.626, "ttr_p95_s": 0.855, "ttr_max_s": 1.142, "pending_max": 3, "worker_mem_peak_mib": 322.1, "worker_mem_median_mib": 183.5, "worker_cpu_peak_pct": 196.4, "at": "2026-11-06T11:38:48"} +{"label": "c4-p4-r1", "tag": "1e1f1621", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 4, "prefetch": 4, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.89, "offered": 90, "drained": true, "drain_after_load_s": 2.38, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 2.837, "lag_p50_s": 0.142, "lag_p95_s": 0.89, "lag_max_s": 1.078, "ttr_p50_s": 1.481, "ttr_p95_s": 2.289, "ttr_max_s": 2.57, "pending_max": 7, "worker_mem_peak_mib": 485.2, "worker_mem_median_mib": 336.5, "worker_cpu_peak_pct": 203.2, "at": "2026-11-06T11:39:29"} +{"label": "c2-p16-r1", "tag": "6d4dbd63", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 16, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 31.36, "offered": 90, "drained": true, "drain_after_load_s": 8.22, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 2.335, "lag_p50_s": 0.887, "lag_p95_s": 6.334, "lag_max_s": 6.523, "ttr_p50_s": 1.857, "ttr_p95_s": 7.375, "ttr_max_s": 7.667, "pending_max": 13, "worker_mem_peak_mib": 275.3, "worker_mem_median_mib": 190.6, "worker_cpu_peak_pct": 196.2, "at": "2026-11-06T11:40:17"} +{"label": "c1-p1-r2", "tag": "73bf39d6", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 1, "prefetch": 1, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.91, "offered": 300, "drained": true, "drain_after_load_s": 31.13, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 4.978, "lag_p50_s": 14.078, "lag_p95_s": 29.005, "lag_max_s": 30.141, "ttr_p50_s": 14.279, "ttr_p95_s": 29.175, "ttr_max_s": 30.364, "pending_max": 145, "worker_mem_peak_mib": 87.6, "worker_mem_median_mib": 79.8, "worker_cpu_peak_pct": 96.4, "at": "2026-11-06T11:41:27"} +{"label": "c2-p2-r2", "tag": "b5f91857", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.92, "offered": 300, "drained": true, "drain_after_load_s": 16.12, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 6.579, "lag_p50_s": 7.808, "lag_p95_s": 14.677, "lag_max_s": 15.458, "ttr_p50_s": 8.089, "ttr_p95_s": 14.96, "ttr_max_s": 15.765, "pending_max": 96, "worker_mem_peak_mib": 104.0, "worker_mem_median_mib": 90.2, "worker_cpu_peak_pct": 193.0, "at": "2026-11-06T11:42:22"} +{"label": "c4-p4-r2", "tag": "4a06a2db", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 4, "prefetch": 4, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.92, "offered": 300, "drained": true, "drain_after_load_s": 24.98, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 5.578, "lag_p50_s": 10.647, "lag_p95_s": 23.151, "lag_max_s": 23.579, "ttr_p50_s": 11.356, "ttr_p95_s": 23.652, "ttr_max_s": 24.047, "pending_max": 134, "worker_mem_peak_mib": 120.3, "worker_mem_median_mib": 100.8, "worker_cpu_peak_pct": 210.9, "at": "2026-11-06T11:43:27"} +{"label": "c2-p16-r2", "tag": "fcd642ae", "size": "small", "file_kib": 80, "rate": 10.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 16, "max_pending": null, "accepted": 300, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.92, "offered": 300, "drained": true, "drain_after_load_s": 14.88, "succeeded": 300, "failed": 0, "retries": 0, "throughput_jobs_s": 6.827, "lag_p50_s": 7.553, "lag_p95_s": 13.428, "lag_max_s": 13.882, "ttr_p50_s": 7.87, "ttr_p95_s": 13.72, "ttr_max_s": 14.122, "pending_max": 97, "worker_mem_peak_mib": 106.1, "worker_mem_median_mib": 88.6, "worker_cpu_peak_pct": 193.4, "at": "2026-11-06T11:44:20"} +{"label": "c1-p1-r2", "tag": "31ff31ee", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 1, "prefetch": 1, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 30.11, "offered": 150, "drained": true, "drain_after_load_s": 31.34, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 2.477, "lag_p50_s": 18.359, "lag_p95_s": 29.508, "lag_max_s": 30.392, "ttr_p50_s": 18.781, "ttr_p95_s": 29.83, "ttr_max_s": 30.727, "pending_max": 83, "worker_mem_peak_mib": 110.4, "worker_mem_median_mib": 98.7, "worker_cpu_peak_pct": 98.8, "at": "2026-11-06T11:45:30"} +{"label": "c2-p2-r2", "tag": "629220cd", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.98, "offered": 150, "drained": true, "drain_after_load_s": 5.68, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 4.272, "lag_p50_s": 1.973, "lag_p95_s": 4.696, "lag_max_s": 4.871, "ttr_p50_s": 2.554, "ttr_p95_s": 5.134, "ttr_max_s": 5.304, "pending_max": 23, "worker_mem_peak_mib": 136.3, "worker_mem_median_mib": 108.4, "worker_cpu_peak_pct": 188.6, "at": "2026-11-06T11:46:13"} +{"label": "c4-p4-r2", "tag": "a553faab", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 4, "prefetch": 4, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 30.07, "offered": 150, "drained": true, "drain_after_load_s": 8.24, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 4.042, "lag_p50_s": 2.017, "lag_p95_s": 5.392, "lag_max_s": 6.512, "ttr_p50_s": 2.993, "ttr_p95_s": 6.574, "ttr_max_s": 7.505, "pending_max": 24, "worker_mem_peak_mib": 204.7, "worker_mem_median_mib": 136.8, "worker_cpu_peak_pct": 202.2, "at": "2026-11-06T11:47:00"} +{"label": "c2-p16-r2", "tag": "de9f2ad5", "size": "medium", "file_kib": 3908, "rate": 5.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 16, "max_pending": null, "accepted": 150, "rejected_503": 0, "other_errors": 0, "send_seconds": 30.03, "offered": 150, "drained": true, "drain_after_load_s": 11.59, "succeeded": 150, "failed": 0, "retries": 0, "throughput_jobs_s": 3.703, "lag_p50_s": 3.276, "lag_p95_s": 8.968, "lag_max_s": 9.955, "ttr_p50_s": 3.976, "ttr_p95_s": 9.518, "ttr_max_s": 10.622, "pending_max": 35, "worker_mem_peak_mib": 159.9, "worker_mem_median_mib": 109.1, "worker_cpu_peak_pct": 189.4, "at": "2026-11-06T11:47:50"} +{"label": "c1-p1-r2", "tag": "9e51fc5d", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 1, "prefetch": 1, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 29.88, "offered": 90, "drained": true, "drain_after_load_s": 27.83, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 1.592, "lag_p50_s": 11.683, "lag_p95_s": 24.82, "lag_max_s": 26.202, "ttr_p50_s": 12.359, "ttr_p95_s": 25.449, "ttr_max_s": 26.881, "pending_max": 41, "worker_mem_peak_mib": 196.8, "worker_mem_median_mib": 173.9, "worker_cpu_peak_pct": 100.8, "at": "2026-11-06T11:48:57"} +{"label": "c2-p2-r2", "tag": "3d93c160", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 31.54, "offered": 90, "drained": true, "drain_after_load_s": 15.14, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 1.956, "lag_p50_s": 6.214, "lag_p95_s": 13.308, "lag_max_s": 14.352, "ttr_p50_s": 7.253, "ttr_p95_s": 14.445, "ttr_max_s": 15.382, "pending_max": 25, "worker_mem_peak_mib": 266.7, "worker_mem_median_mib": 194.0, "worker_cpu_peak_pct": 198.1, "at": "2026-11-06T11:49:53"} +{"label": "c4-p4-r2", "tag": "2e50998d", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 4, "prefetch": 4, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 31.45, "offered": 90, "drained": true, "drain_after_load_s": 12.87, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 2.051, "lag_p50_s": 4.859, "lag_p95_s": 10.405, "lag_max_s": 11.567, "ttr_p50_s": 6.358, "ttr_p95_s": 12.708, "ttr_max_s": 13.84, "pending_max": 23, "worker_mem_peak_mib": 548.7, "worker_mem_median_mib": 329.9, "worker_cpu_peak_pct": 202.3, "at": "2026-11-06T11:50:46"} +{"label": "c2-p16-r2", "tag": "63cdb6c9", "size": "large", "file_kib": 400, "rate": 3.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 16, "max_pending": null, "accepted": 90, "rejected_503": 0, "other_errors": 0, "send_seconds": 30.01, "offered": 90, "drained": true, "drain_after_load_s": 9.24, "succeeded": 90, "failed": 0, "retries": 0, "throughput_jobs_s": 2.309, "lag_p50_s": 1.364, "lag_p95_s": 8.043, "lag_max_s": 8.52, "ttr_p50_s": 2.146, "ttr_p95_s": 8.84, "ttr_max_s": 9.233, "pending_max": 20, "worker_mem_peak_mib": 285.6, "worker_mem_median_mib": 193.4, "worker_cpu_peak_pct": 197.2, "at": "2026-11-06T11:51:35"} +{"label": "overload-soft-only-r1", "tag": "44ae8a0a", "size": "large", "file_kib": 400, "rate": 8.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": 30, "accepted": 128, "rejected_503": 112, "other_errors": 0, "send_seconds": 29.88, "offered": 240, "drained": true, "drain_after_load_s": 28.41, "succeeded": 128, "failed": 0, "retries": 0, "throughput_jobs_s": 2.239, "lag_p50_s": 18.807, "lag_p95_s": 31.84, "lag_max_s": 32.462, "ttr_p50_s": 19.553, "ttr_p95_s": 32.708, "ttr_max_s": 33.358, "pending_max": 74, "worker_mem_peak_mib": 354.1, "worker_mem_median_mib": 192.1, "worker_cpu_peak_pct": 200.8, "at": "2026-11-06T11:52:48", "recovery_probe_ready_s": 0.23} +{"label": "overload-soft-only-r2", "tag": "3ca187f1", "size": "large", "file_kib": 400, "rate": 8.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": 30, "accepted": 143, "rejected_503": 97, "other_errors": 0, "send_seconds": 29.88, "offered": 240, "drained": true, "drain_after_load_s": 30.71, "succeeded": 143, "failed": 0, "retries": 0, "throughput_jobs_s": 2.379, "lag_p50_s": 15.954, "lag_p95_s": 33.178, "lag_max_s": 34.043, "ttr_p50_s": 16.722, "ttr_p95_s": 34.072, "ttr_max_s": 34.912, "pending_max": 81, "worker_mem_peak_mib": 336.4, "worker_mem_median_mib": 210.1, "worker_cpu_peak_pct": 198.1, "at": "2026-11-06T11:54:05", "recovery_probe_ready_s": 0.51} +{"label": "overload-strict-r1", "tag": "c0336ae6", "size": "large", "file_kib": 400, "rate": 8.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": 30, "accepted": 108, "rejected_503": 132, "other_errors": 0, "send_seconds": 34.25, "offered": 240, "drained": true, "drain_after_load_s": 12.45, "succeeded": 108, "failed": 0, "retries": 0, "throughput_jobs_s": 2.376, "lag_p50_s": 12.922, "lag_p95_s": 18.092, "lag_max_s": 18.737, "ttr_p50_s": 13.77, "ttr_p95_s": 18.916, "ttr_max_s": 19.593, "pending_max": 30, "worker_mem_peak_mib": 315.6, "worker_mem_median_mib": 200.0, "worker_cpu_peak_pct": 196.8, "at": "2026-11-06T11:56:03", "recovery_probe_ready_s": 0.23} +{"label": "overload-strict-r2", "tag": "de4b3bbc", "size": "large", "file_kib": 400, "rate": 8.0, "duration": 30.0, "replicas": 1, "concurrency": 2, "prefetch": 2, "max_pending": 30, "accepted": 100, "rejected_503": 140, "other_errors": 0, "send_seconds": 30.69, "offered": 240, "drained": true, "drain_after_load_s": 15.17, "succeeded": 100, "failed": 0, "retries": 0, "throughput_jobs_s": 2.212, "lag_p50_s": 12.916, "lag_p95_s": 15.414, "lag_max_s": 16.136, "ttr_p50_s": 13.843, "ttr_p95_s": 16.417, "ttr_max_s": 17.059, "pending_max": 30, "worker_mem_peak_mib": 380.1, "worker_mem_median_mib": 207.9, "worker_cpu_peak_pct": 197.6, "at": "2026-11-06T11:57:07", "recovery_probe_ready_s": 0.24} diff --git a/evidence/track1/summary.md b/evidence/track1/summary.md new file mode 100644 index 00000000..48d97bc7 --- /dev/null +++ b/evidence/track1/summary.md @@ -0,0 +1,21 @@ +| size | rate/s | c/p | n | accepted | throughput jobs/s | lag p95 s | ttr p50 s | ttr p95 s | pending max | worker mem peak MiB | cpu peak % | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| small | 10.0 | 1/1 | 2 | 300 | 5.53 (4.98–6.08) | 23.70 (18.39–29.00) | 12.11 (9.94–14.28) | 23.86 (18.55–29.18) | 131 (117–145) | 90 (88–92) | 96 (96–96) | +| small | 10.0 | 2/2 | 2 | 300 | 8.28 (6.58–9.98) | 7.35 (0.03–14.68) | 4.13 (0.17–8.09) | 7.59 (0.22–14.96) | 50 (3–96) | 99 (95–104) | 192 (190–193) | +| small | 10.0 | 2/16 | 2 | 300 | 8.40 (6.83–9.98) | 6.72 (0.01–13.43) | 4.02 (0.17–7.87) | 6.96 (0.19–13.72) | 50 (2–97) | 101 (96–106) | 179 (164–193) | +| small | 10.0 | 4/4 | 2 | 300 | 7.78 (5.58–9.98) | 11.58 (0.01–23.15) | 5.76 (0.17–11.36) | 11.92 (0.19–23.65) | 68 (2–134) | 108 (96–120) | 192 (172–211) | +| medium | 5.0 | 1/1 | 2 | 150 | 2.80 (2.48–3.12) | 23.42 (17.32–29.51) | 14.22 (9.67–18.78) | 23.73 (17.62–29.83) | 70 (58–83) | 110 (109–110) | 97 (96–99) | +| medium | 5.0 | 2/2 | 2 | 150 | 4.58 (4.27–4.90) | 2.64 (0.58–4.70) | 1.65 (0.74–2.55) | 3.05 (0.97–5.13) | 14 (5–23) | 142 (136–148) | 190 (189–190) | +| medium | 5.0 | 2/16 | 2 | 150 | 4.30 (3.70–4.90) | 4.79 (0.60–8.97) | 2.34 (0.70–3.98) | 5.26 (1.00–9.52) | 20 (5–35) | 164 (160–168) | 190 (189–190) | +| medium | 5.0 | 4/4 | 2 | 150 | 4.50 (4.04–4.96) | 2.74 (0.08–5.39) | 1.71 (0.42–2.99) | 3.53 (0.49–6.57) | 13 (2–24) | 166 (126–205) | 200 (199–202) | +| large | 3.0 | 1/1 | 2 | 90 | 1.73 (1.59–1.87) | 21.01 (17.21–24.82) | 10.85 (9.34–12.36) | 21.59 (17.73–25.45) | 38 (34–41) | 197 (197–198) | 100 (100–101) | +| large | 3.0 | 2/2 | 2 | 90 | 2.44 (1.96–2.91) | 6.77 (0.24–13.31) | 3.94 (0.63–7.25) | 7.65 (0.85–14.45) | 14 (3–25) | 294 (267–322) | 197 (196–198) | +| large | 3.0 | 2/16 | 2 | 90 | 2.32 (2.31–2.33) | 7.19 (6.33–8.04) | 2.00 (1.86–2.15) | 8.11 (7.38–8.84) | 16 (13–20) | 280 (275–286) | 197 (196–197) | +| large | 3.0 | 4/4 | 2 | 90 | 2.44 (2.05–2.84) | 5.65 (0.89–10.40) | 3.92 (1.48–6.36) | 7.50 (2.29–12.71) | 15 (7–23) | 517 (485–549) | 203 (202–203) | + +| overload run | offered | accepted | rejected 503 | other errors | pending max (limit) | throughput | ttr p95 s | drain after load s | recovery probe ready s | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| overload-soft-only-r1 | 240 @ 8.0/s | 128 | 112 | 0 | 74 (30) | 2.24 | 32.708 | 28.41 | 0.23 | +| overload-soft-only-r2 | 240 @ 8.0/s | 143 | 97 | 0 | 81 (30) | 2.38 | 34.072 | 30.71 | 0.51 | +| overload-strict-r1 | 240 @ 8.0/s | 108 | 132 | 0 | 30 (30) | 2.38 | 18.916 | 12.45 | 0.23 | +| overload-strict-r2 | 240 @ 8.0/s | 100 | 140 | 0 | 30 (30) | 2.21 | 16.417 | 15.17 | 0.24 | diff --git a/evidence/variants_limits.txt b/evidence/variants_limits.txt new file mode 100644 index 00000000..578b469f --- /dev/null +++ b/evidence/variants_limits.txt @@ -0,0 +1,20 @@ +ok 320x200 -> thumbnail 256x160 +ok crop 128x128 +ok optimized WebP 320x200, no upscale +ok thumbnail/crop are JPEG +ok 40x10: thumbnail 40x10, crop 128x128 +ok 3000x2 -> 256x1 / 1280x1 (min 1px) +ok 1001x999 -> 256x255 (floor) +ok EXIF orientation applied: 400x200 (o=6) -> 128x256 +ok rotated content: red half on top after orientation 6 +ok EXIF removed from thumbnail/optimized +ok transparent PNG -> white JPEG thumbnail +ok WebP optimized keeps alpha +ok 16-bit PNG, CMYK JPEG, palette+transparency PNG are processed +ok pixel limit enforced before decode -> image_too_large (exit 4) +ok corrupted PNG -> invalid_image (exit 3) +ok 36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got 5 +ok 39.5 Mpx PNG (31 MB file) ok in 0.9s, peak child RSS 322 MiB +ok CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got -24 +ok 39.5 Mpx JPEG (7 MB file) ok in 0.3s, peak child RSS 335 MiB +RESULT: PASS diff --git a/evidence/worker_down.txt b/evidence/worker_down.txt new file mode 100644 index 00000000..bc967e72 --- /dev/null +++ b/evidence/worker_down.txt @@ -0,0 +1,16 @@ +# grade 3 demo — 2026-11-06T08:57:30Z, base commit 276dbda + working tree, macOS arm64, Docker 29.7, 10 CPU/8 GiB VM +$ python3 student-tests/test_worker_down.py +11:57:30 stop all workers +11:57:31 upload 202 queued id=4ebb6253-d722-4c83-b9fe-87f15342c14b +11:57:31 message 0fdac623-179d-40f3-8792-3332b63e2623 created with queued attachment +11:57:34 broker queue while worker is down: {'messages': 1, 'ready': 1, 'unacked': 0} +11:57:34 restart api-1 api-2 rabbitmq (volumes kept) +11:57:47 started before: mtusi-lab5-api-1-1 2026-11-06T08:57:08.369668592Z; mtusi-lab5-api-2-1 2026-11-06T08:57:08.47338055Z; mtusi-lab5-rabbitmq-1 2026-11-06T08:23:50.417899292Z +11:57:47 started after: mtusi-lab5-api-1-1 2026-11-06T08:57:35.409218424Z; mtusi-lab5-api-2-1 2026-11-06T08:57:35.46578734Z; mtusi-lab5-rabbitmq-1 2026-11-06T08:57:36.162050341Z +11:57:47 broker queue after broker restart: {'messages': 1, 'ready': 1, 'unacked': 0} +11:57:47 start workers +11:57:52 status sequence observed: queued -> ready +11:57:52 objects: ['attachments/4ebb6253-d722-4c83-b9fe-87f15342c14b/original', 'attachments/4ebb6253-d722-4c83-b9fe-87f15342c14b/v1/crop.jpg', 'attachments/4ebb6253-d722-4c83-b9fe-87f15342c14b/v1/optimized.webp', 'attachments/4ebb6253-d722-4c83-b9fe-87f15342c14b/v1/thumbnail.jpg'] +11:57:52 redelivery: republish the finished job through the outbox (duplicate message) +11:57:58 duplicate acked as no-op: variants=3 successes=1 +PASS: worker down / API+broker restart / redelivery no-op diff --git a/infra/README.md b/infra/README.md index 5a39e862..61d432df 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,11 @@ # Инфраструктура студента -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +Основная топология — корневой [compose.yaml](../compose.yaml) (Compose v2). `compose.example.yaml` — только worksheet ролей. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. +| Каталог | Что внутри | +| --- | --- | +| [nginx.conf](nginx.conf), [nginx-health.sh](nginx-health.sh) | LB перед двумя API-репликами, активная проверка `/health/ready` (из lab3/lab4) | +| [minio/](minio/) | идемпотентный bootstrap приватного bucket и пользователя приложения; политика `attachments/*` + `staging/*` | +| [rabbitmq/](rabbitmq/) | `rabbitmq.conf` (consumer_timeout, disk/memory alarms) и entrypoint, который берёт пароль из Docker secret | -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Топология брокера (exchange, quorum-очереди, DLX/DLQ) объявляется кодом идемпотентно при каждом подключении API и worker ([app/broker.py](../app/broker.py)), отдельного bootstrap-шага нет. Повторный запуск `docker compose up` безопасен: миграции пропускают применённые версии с проверкой checksum, `minio-init` использует `--ignore-existing`, объявление очередей с теми же аргументами — no-op. diff --git a/infra/minio/app-policy.json b/infra/minio/app-policy.json new file mode 100644 index 00000000..9195e447 --- /dev/null +++ b/infra/minio/app-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/attachments/*", "arn:aws:s3:::messenger-attachments/staging/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/minio/init.sh b/infra/minio/init.sh new file mode 100755 index 00000000..2b69c79f --- /dev/null +++ b/infra/minio/init.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Idempotent: safe to run on every `compose up`. Root credentials are used only here, never by the API. +set -eu +BUCKET="${S3_BUCKET:-messenger-attachments}" +ROOT_USER="$(cat /run/secrets/minio_root_user)" +ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +APP_USER="$(cat /run/secrets/s3_access_key_id)" +APP_PASSWORD="$(cat /run/secrets/s3_secret_access_key)" + +until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do + echo "waiting for minio"; sleep 2 +done +mc mb --ignore-existing "local/$BUCKET" +mc anonymous set none "local/$BUCKET" +mc admin policy create local app-attachments /etc/minio-policy/app-policy.json 2>/dev/null \ + || mc admin policy create local app-attachments /etc/minio-policy/app-policy.json --overwrite 2>/dev/null || true +mc admin user add local "$APP_USER" "$APP_PASSWORD" 2>/dev/null || true +mc admin policy attach local app-attachments --user "$APP_USER" 2>/dev/null || true +echo "minio bootstrap complete: bucket=$BUCKET private, app user limited to attachments/*" diff --git a/infra/nginx-health.sh b/infra/nginx-health.sh new file mode 100644 index 00000000..692cab75 --- /dev/null +++ b/infra/nginx-health.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -u + +SERVERS="${API_UPSTREAMS:-api-1 api-2}" +INTERVAL="${HEALTH_INTERVAL:-1}" +FALL="${HEALTH_FALL:-2}" +RISE="${HEALTH_RISE:-2}" +OUT=/etc/nginx/upstream/servers.conf + +mkdir -p "$(dirname "$OUT")" + +var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; } + +render() { + any_up=0 + for s in $SERVERS; do + eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1 + done + for s in $SERVERS; do + mark="" + if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then + mark=" down" + fi + echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;" + done +} + +for s in $SERVERS; do + v=$(var "$s") + eval "up_$v=1 ok_$v=0 bad_$v=0" +done +render > "$OUT" + +watch() { + pending=0 + while sleep "$INTERVAL"; do + changed=0 + for s in $SERVERS; do + v=$(var "$s") + if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then + eval "ok_$v=\$((ok_$v + 1)) bad_$v=0" + if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then + eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream" + fi + else + eval "bad_$v=\$((bad_$v + 1)) ok_$v=0" + if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then + eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down" + fi + fi + done + if [ "$changed" = 1 ]; then + render > "$OUT" + pending=1 + fi + if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then + pending=0 + fi + done +} + +watch & +exec nginx -g 'daemon off;' diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..5d791ed9 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,75 @@ +worker_processes 2; + +events { + worker_connections 256; +} + +http { + resolver 127.0.0.11 valid=5s ipv6=off; + + upstream api { + zone api 64k; + include /etc/nginx/upstream/servers.conf; + keepalive 32; + keepalive_timeout 4s; + } + + log_format main '$remote_addr - $upstream_addr [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$upstream_response_time" "$request_time"'; + + access_log /var/log/nginx/access.log main; + + server { + listen 8080; + client_max_body_size 64k; + error_page 413 @payload_too_large; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 1s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + location ~ ^/api/v1/chats/[^/]+/messages$ { + proxy_pass http://api; + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + } + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + client_max_body_size 10m; + proxy_request_buffering off; + proxy_pass http://api; + proxy_next_upstream off; + } + + location @payload_too_large { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id lb always; + add_header Connection close always; + return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}'; + } + + location /health/ { + proxy_pass http://api; + proxy_next_upstream error timeout; + } + + location ~ ^/api/v1/attachments/ { + proxy_max_temp_file_size 0; + proxy_pass http://api; + proxy_next_upstream error timeout; + } + + location / { + proxy_pass http://api; + proxy_next_upstream error timeout http_502 http_503; + } + } +} diff --git a/infra/rabbitmq/entrypoint.sh b/infra/rabbitmq/entrypoint.sh new file mode 100644 index 00000000..2e99ea01 --- /dev/null +++ b/infra/rabbitmq/entrypoint.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Credentials come from a Docker secret; RabbitMQ's config cannot read files, so render a conf.d fragment at start. +set -eu +PASS="$(cat /run/secrets/rabbitmq_password)" +mkdir -p /etc/rabbitmq/conf.d +cat > /etc/rabbitmq/conf.d/20-credentials.conf < staging -> job/outbox -> broker -> worker -> final prefix + variants. +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ADD CONSTRAINT attachments_status + CHECK (status IN ('uploading', 'queued', 'processing', 'ready', 'failed')); +-- staging_key: private S3 prefix written by the API before 202; object_key: final original written by the worker. +ALTER TABLE attachments ADD COLUMN staging_key text; +ALTER TABLE attachments ADD COLUMN pipeline_version integer NOT NULL DEFAULT 1; +CREATE UNIQUE INDEX attachments_staging_key_idx ON attachments (staging_key) WHERE staging_key IS NOT NULL; + +-- Effect key of the pipeline: (attachment_id, pipeline_version, name). A repeated job cannot add a second set. +CREATE TABLE attachment_variants ( + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + name text NOT NULL CONSTRAINT variants_name CHECK (name IN ('thumbnail', 'crop', 'optimized')), + object_key text NOT NULL UNIQUE, + content_type text NOT NULL CONSTRAINT variants_ct CHECK (content_type IN ('image/jpeg', 'image/png', 'image/webp')), + size_bytes integer NOT NULL CONSTRAINT variants_size CHECK (size_bytes > 0), + width integer NOT NULL CONSTRAINT variants_width CHECK (width > 0), + height integer NOT NULL CONSTRAINT variants_height CHECK (height > 0), + sha256 text NOT NULL CONSTRAINT variants_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + created_at timestamptz NOT NULL, + PRIMARY KEY (attachment_id, pipeline_version, name) +); + +-- One logical job per (attachment, pipeline_version); job_id never changes on redelivery or retry. +CREATE TABLE attachment_jobs ( + id uuid PRIMARY KEY, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE CASCADE, + pipeline_version integer NOT NULL, + state text NOT NULL + CONSTRAINT jobs_state CHECK (state IN ('queued', 'processing', 'succeeded', 'failed')), + attempt integer NOT NULL DEFAULT 1 CONSTRAINT jobs_attempt CHECK (attempt >= 1), + max_attempts integer NOT NULL CONSTRAINT jobs_max_attempts CHECK (max_attempts >= 1), + next_attempt_at timestamptz NOT NULL, + lease_owner text, + lease_until timestamptz, + error_code text, + last_error text, + created_at timestamptz NOT NULL, + first_started_at timestamptz, + started_at timestamptz, + finished_at timestamptz, + updated_at timestamptz NOT NULL, + UNIQUE (attachment_id, pipeline_version) +); +CREATE INDEX attachment_jobs_lease_idx ON attachment_jobs (lease_until) WHERE state = 'processing'; +CREATE INDEX attachment_jobs_failed_idx ON attachment_jobs (updated_at) WHERE state = 'failed'; +CREATE INDEX attachment_jobs_pending_idx ON attachment_jobs (state) WHERE state IN ('queued', 'processing'); + +-- Transactional outbox: written in the same transaction as the job; the relay publishes with confirms. +CREATE TABLE outbox ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + payload jsonb NOT NULL, + available_at timestamptz NOT NULL, + published_at timestamptz, + publish_attempts integer NOT NULL DEFAULT 0, + last_error text, + created_at timestamptz NOT NULL +); +CREATE INDEX outbox_pending_idx ON outbox (available_at, id) WHERE published_at IS NULL; +CREATE INDEX outbox_published_idx ON outbox (published_at) WHERE published_at IS NOT NULL; + +-- Operator-visible history of every state change of a job (including explicit replay). +CREATE TABLE job_audit ( + id bigserial PRIMARY KEY, + job_id uuid NOT NULL REFERENCES attachment_jobs (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL, + action text NOT NULL, + actor text NOT NULL, + attempt integer NOT NULL, + detail jsonb NOT NULL DEFAULT '{}'::jsonb, + created_at timestamptz NOT NULL +); +CREATE INDEX job_audit_job_idx ON job_audit (job_id, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..2035f130 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,37 @@ +aio-pika==9.5.5 +aiormq==6.8.1 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +argon2-cffi==25.1.0 +boto3==1.35.81 +botocore==1.35.81 +cffi==2.1.1 +click==8.5.0 +exceptiongroup==1.2.2 +fastapi==0.115.6 +h11==0.16.0 +httptools==0.9.0 +idna==3.20 +jmespath==1.0.1 +multidict==6.1.0 +pamqp==3.3.0 +pillow==11.0.0 +propcache==0.2.1 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +psycopg==3.3.6 +pycparser==3.11 +pydantic-core==2.50.1 +pydantic==2.14.1 +python-dateutil==2.9.0.post0 +s3transfer==0.10.4 +six==1.17.0 +starlette==0.41.3 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +urllib3==2.3.0 +uvicorn==0.34.0 +uvloop==0.23.0 +valkey==6.2.0 +yarl==1.18.3 diff --git a/scripts/jobs.py b/scripts/jobs.py new file mode 100755 index 00000000..91edc79b --- /dev/null +++ b/scripts/jobs.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Operator CLI for attachment jobs (run inside an API container: docker compose exec api-1 python scripts/jobs.py ...). + + failed registry of failed jobs (the "DLQ" of business failures) + show job state + audit trail + replay --actor NAME explicit failed -> queued; refused (no-op) for ready/queued/processing + stats counts by state and outbox backlog +Poison messages that are not valid jobs are in the RabbitMQ queue attachments.dlq (see REPORT.md). +""" +import argparse +import asyncio +import json +import sys + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import jobs # noqa: E402 +from app.cursor import fmt # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + + +def out(value) -> None: + print(json.dumps(value, ensure_ascii=False, indent=2, default=str)) + + +async def main() -> int: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="cmd", required=True) + sub.add_parser("failed") + sub.add_parser("stats") + show = sub.add_parser("show") + show.add_argument("attachment_id") + rp = sub.add_parser("replay") + rp.add_argument("attachment_id") + rp.add_argument("--actor", required=True, help="who requests the replay (written to job_audit)") + args = parser.parse_args() + + db = Database() + await db.open() + try: + if args.cmd == "failed": + async with db.connection() as conn: + out(await jobs.failed_jobs(conn)) + elif args.cmd == "stats": + async with db.connection() as conn: + cur = await conn.execute("SELECT state, count(*) AS n FROM attachment_jobs GROUP BY state ORDER BY state") + states = {r["state"]: r["n"] for r in await cur.fetchall()} + cur = await conn.execute("SELECT count(*) AS n FROM outbox WHERE published_at IS NULL") + backlog = (await cur.fetchone())["n"] + out({"jobs": states, "outbox_unpublished": backlog}) + elif args.cmd == "show": + async with db.connection() as conn: + cur = await conn.execute( + "SELECT j.id, j.state, j.attempt, j.max_attempts, j.error_code, j.lease_owner, j.lease_until," + " a.status AS attachment_status FROM attachment_jobs j JOIN attachments a ON a.id = j.attachment_id" + " WHERE j.attachment_id = %s", (args.attachment_id,)) + job = await cur.fetchone() + cur = await conn.execute( + "SELECT action, actor, attempt, detail, created_at FROM job_audit WHERE attachment_id = %s ORDER BY id", + (args.attachment_id,)) + trail = [{**r, "created_at": fmt(r["created_at"])} for r in await cur.fetchall()] + cur = await conn.execute( + "SELECT o.id, o.payload->>'attempt' AS attempt, o.available_at, o.published_at, o.publish_attempts" + " FROM outbox o JOIN attachment_jobs j ON j.id = o.job_id WHERE j.attachment_id = %s ORDER BY o.id", + (args.attachment_id,)) + outbox = await cur.fetchall() + out({"job": job, "audit": trail, "outbox": outbox}) + elif args.cmd == "replay": + s3 = S3Storage() + await run_in_threadpool(s3.open) + + async def exists(key): + return await run_in_threadpool(s3.exists, key) + + result = await db.in_transaction( + lambda conn: jobs.replay(conn, args.attachment_id, "operator:" + args.actor, exists)) + out(result) + return 0 if result["result"] == "queued" else 3 + finally: + await db.close() + return 0 + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..5c19e228 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run.""" +import hashlib +import os +import pathlib +import sys + +import psycopg + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MIGRATIONS = ROOT / "migrations" +LOCK_ID = 2026_0004 + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return pathlib.Path(path).read_text(encoding="utf-8").strip() + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + sys.exit("DATABASE_URL or DATABASE_URL_FILE is required") + return url + + +def main() -> None: + files = sorted(MIGRATIONS.glob("*.sql")) + if not files: + sys.exit("no migration files found in %s" % MIGRATIONS) + with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY," + " checksum text NOT NULL," + " applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall() + applied = {row[0]: row[1] for row in rows} + for path in files: + version = path.stem + body = path.read_text(encoding="utf-8") + checksum = hashlib.sha256(body.encode("utf-8")).hexdigest() + if version in applied: + if applied[version] != checksum: + sys.exit("migration %s changed after it was applied" % version) + print("skip %s" % version) + continue + with conn.transaction(): + conn.execute(body) + conn.execute( + "INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", + (version, checksum), + ) + print("apply %s" % version) + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + print("migrations complete: %d file(s)" % len(files)) + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 00000000..10cd7b68 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything.""" +import argparse +import asyncio +import json +import sys + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import config, reconcile # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports") + parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS) + args = parser.parse_args() + db, s3 = Database(), S3Storage() + await db.open() + await run_in_threadpool(s3.open) + try: + print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True)) + finally: + await db.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..0af8fb4a --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Restart API, LB, worker and every store (postgres, valkey, minio, rabbitmq) keeping all volumes. +set -eu +cd "$(dirname "$0")/.." +docker compose restart -t 10 postgres valkey minio rabbitmq +docker compose restart -t 25 worker api-1 api-2 nginx diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..23c94b55 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +docker compose stop -t 3 "$TARGET_INSTANCE_ID" diff --git a/student-tests/queue_load.py b/student-tests/queue_load.py new file mode 100644 index 00000000..65f03b3d --- /dev/null +++ b/student-tests/queue_load.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Track 1: the queue under load. Open-loop upload arrivals against the running stack, metrics from the DB. + +Each run: + 1. (re)creates the worker service with WORKER_REPLICAS / WORKER_CONCURRENCY / WORKER_PREFETCH + 2. sends uploads at a FIXED arrival rate (open loop: the schedule does not wait for responses) + 3. samples worker container memory (docker stats) and accepted-but-unfinished jobs once per second + 4. waits until every accepted upload is ready/failed and computes from attachment_jobs: + queue lag = first_started_at - created_at (time a job waited for a worker) + time to ready = finished_at - attachment.created_at + throughput = finished jobs / (last finish - first arrival) + Rejections (503 queue_full from admission control) are counted, not retried. + + python3 student-tests/queue_load.py run --size medium --rate 4 --duration 30 --concurrency 2 --prefetch 2 + python3 student-tests/queue_load.py overload --max-pending 30 --size medium --rate 12 --duration 30 +Results are appended as JSON lines to evidence/track1/results.jsonl. +""" +import argparse +import json +import os +import statistics +import subprocess +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +from stand import ROOT, Client, Failure, compose, log, sql, wait_http_ready + +OUT = ROOT / "evidence/track1" +SIZES = {"small": (640, 400, "JPEG"), "medium": (4000, 3000, "JPEG"), "large": (6000, 4000, "PNG")} + +GEN = r''' +import io, random, sys +from PIL import Image, ImageChops, ImageDraw +w, h, fmt = int(sys.argv[1]), int(sys.argv[2]), sys.argv[3] +if fmt == "JPEG": # photo-like noise: realistic JPEG sizes + chan = [Image.effect_noise((max(8, w // 24), max(8, h // 24)), 90).resize((w, h), Image.Resampling.BICUBIC) for _ in range(3)] + grain = Image.effect_noise((w, h), 12) + img = Image.merge("RGB", [ImageChops.add(c, grain, 1.0, -6) for c in chan]) +else: # graphics-like PNG: compresses well (stays under 10 MiB) but must be fully decoded (no DCT scaling) + img = Image.linear_gradient("L").resize((w, h)).convert("RGB") + draw, rnd = ImageDraw.Draw(img), random.Random(1) + for _ in range(400): + x, y, r = rnd.randrange(w), rnd.randrange(h), rnd.randrange(20, 300) + draw.ellipse((x, y, x + r, y + r), fill=(rnd.randrange(256), rnd.randrange(256), rnd.randrange(256))) +buf = io.BytesIO() +img.save(buf, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 6})) +sys.stdout.buffer.write(buf.getvalue()) +''' + + +def image(size): + w, h, fmt = SIZES[size] + cache = OUT / ("fixture-%s.%s" % (size, fmt.lower())) + if not cache.exists(): + OUT.mkdir(parents=True, exist_ok=True) + data = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "-c", GEN, str(w), str(h), fmt], + cwd=ROOT, capture_output=True, check=True).stdout + cache.write_bytes(data) + return cache.read_bytes(), "image/png" if fmt == "PNG" else "image/jpeg" + + +def set_worker(replicas, concurrency, prefetch, cpus="2.00"): + env = dict(os.environ, WORKER_REPLICAS=str(replicas), WORKER_CONCURRENCY=str(concurrency), + WORKER_PREFETCH=str(prefetch), WORKER_CPUS=cpus) + subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "worker"], + cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) + + +def set_admission(max_pending): + env = dict(os.environ, MAX_PENDING_JOBS=str(max_pending)) + subprocess.run(["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--wait", "api-1", "api-2"], + cwd=ROOT, env=env, check=True, capture_output=True, timeout=180) + + +def parse_mem(text): + value = text.split("/")[0].strip() + for unit, mult in (("GiB", 1024), ("MiB", 1), ("KiB", 1 / 1024), ("B", 1 / 1048576)): + if value.endswith(unit): + return float(value[: -len(unit)]) * mult + return 0.0 + + +class Sampler(threading.Thread): + def __init__(self, tag): + super().__init__(daemon=True) + self.tag = tag + self.stop = threading.Event() + self.rss, self.pending, self.cpu = [], [], [] + + def run(self): + while not self.stop.is_set(): + ids = compose("ps", "-q", "worker", check_rc=False).split() + if ids: + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}|{{.CPUPerc}}", *ids], + capture_output=True, text=True).stdout + rows = [line.split("|") for line in out.splitlines() if "|" in line] + self.rss.append(sum(parse_mem(r[0]) for r in rows)) + self.cpu.append(sum(float(r[1].rstrip("%") or 0) for r in rows)) + try: + self.pending.append(int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" + % self.tag)[0][0])) + except Exception: + pass + self.stop.wait(1.0) + + +def pct(values, q): + if not values: + return None + values = sorted(values) + return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3) + + +def drive(c, actor, chat, body, ctype, rate, duration, tag): + """Open-loop: request i is sent at t0 + i/rate regardless of earlier responses.""" + total = int(rate * duration) + results = {"accepted": 0, "rejected_503": 0, "other_errors": 0} + lock = threading.Lock() + + def one(i): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote("load-%s-%05d.img" % (tag, i))) + try: + r = c.call("POST", path, [202, 503], actor=actor, raw=body, headers={"Content-Type": ctype}) + key = "accepted" if r["status"] == 202 else "rejected_503" + except Exception: # connection errors, timeouts: counted, never retried + key = "other_errors" + with lock: + results[key] += 1 + + t0 = time.monotonic() + with ThreadPoolExecutor(max_workers=64) as pool: + for i in range(total): + delay = t0 + i / rate - time.monotonic() + if delay > 0: + time.sleep(delay) + pool.submit(one, i) + results["send_seconds"] = round(time.monotonic() - t0, 2) + results["offered"] = total + return results + + +def wait_drained(tag, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + left = int(sql("SELECT count(*) FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%' AND j.state IN ('queued','processing')" % tag)[0][0]) + if left == 0: + return True + time.sleep(1) + return False + + +def job_metrics(tag): + rows = sql( + "SELECT extract(epoch FROM a.created_at), extract(epoch FROM j.first_started_at), extract(epoch FROM j.finished_at)," + " j.state, j.attempt FROM attachment_jobs j JOIN attachments a ON a.id=j.attachment_id" + " WHERE a.filename LIKE 'load-%s-%%'" % tag) + done = [(float(a), float(s), float(f)) for a, s, f, st, _ in rows if st == "succeeded"] + lag = [s - a for a, s, f in done] + ttr = [f - a for a, s, f in done] + span = (max(f for _, _, f in done) - min(a for a, _, _ in done)) if done else 0 + return { + "succeeded": len(done), "failed": sum(1 for r in rows if r[3] == "failed"), + "retries": sum(int(r[4]) - 1 for r in rows), + "throughput_jobs_s": round(len(done) / span, 3) if span else None, + "lag_p50_s": pct(lag, 0.5), "lag_p95_s": pct(lag, 0.95), "lag_max_s": pct(lag, 1.0), + "ttr_p50_s": pct(ttr, 0.5), "ttr_p95_s": pct(ttr, 0.95), "ttr_max_s": pct(ttr, 1.0), + } + + +def record(summary): + OUT.mkdir(parents=True, exist_ok=True) + with open(OUT / "results.jsonl", "a") as handle: + handle.write(json.dumps(summary, ensure_ascii=False) + "\n") + log(json.dumps(summary, ensure_ascii=False)) + + +def experiment(args, label, max_pending=None): + c = Client(timeout=60) + wait_http_ready(c) + alice = c.user("load") + chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "load", "member_ids": [alice["id"]]}, + schema="Chat")["json"] + body, ctype = image(args.size) + set_worker(args.replicas, args.concurrency, args.prefetch) + time.sleep(2) + tag = uuid.uuid4().hex[:8] + sampler = Sampler(tag) + sampler.start() + log("run %s tag=%s size=%s (%d KiB) rate=%s/s duration=%ss workers=%dx(c=%d,p=%d)" % ( + label, tag, args.size, len(body) >> 10, args.rate, args.duration, args.replicas, args.concurrency, args.prefetch)) + sent = drive(c, alice, chat, body, ctype, args.rate, args.duration, tag) + load_end = time.monotonic() + drained = wait_drained(tag, args.drain_timeout) + drain_seconds = round(time.monotonic() - load_end, 2) + sampler.stop.set() + sampler.join() + summary = {"label": label, "tag": tag, "size": args.size, "file_kib": len(body) >> 10, "rate": args.rate, + "duration": args.duration, "replicas": args.replicas, "concurrency": args.concurrency, + "prefetch": args.prefetch, "max_pending": max_pending, **sent, "drained": drained, + "drain_after_load_s": drain_seconds, **job_metrics(tag), + "pending_max": max(sampler.pending or [0]), + "worker_mem_peak_mib": round(max(sampler.rss or [0]), 1), + "worker_mem_median_mib": round(statistics.median(sampler.rss), 1) if sampler.rss else None, + "worker_cpu_peak_pct": round(max(sampler.cpu or [0]), 1), + "at": time.strftime("%Y-%m-%dT%H:%M:%S")} + return summary, c, alice, chat + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=["run", "overload"]) + parser.add_argument("--size", choices=sorted(SIZES), default="medium") + parser.add_argument("--rate", type=float, default=4) + parser.add_argument("--duration", type=float, default=30) + parser.add_argument("--replicas", type=int, default=1) + parser.add_argument("--concurrency", type=int, default=2) + parser.add_argument("--prefetch", type=int, default=2) + parser.add_argument("--max-pending", type=int, default=30) + parser.add_argument("--drain-timeout", type=float, default=600) + parser.add_argument("--label", default="") + args = parser.parse_args() + if args.mode == "run": + summary, *_ = experiment(args, args.label or "c%d-p%d" % (args.concurrency, args.prefetch)) + record(summary) + return + # Overload: admission limit on, offered rate far above capacity, then recovery. + set_admission(args.max_pending) + try: + summary, c, alice, chat = experiment(args, args.label or "overload", args.max_pending) + # Recovery probe: after the backlog drained, a fresh upload must be ready quickly again. + body, ctype = image("small") + t = time.monotonic() + att = c.call("POST", "/api/v1/chats/%s/attachments?filename=probe.png" % chat["id"], 202, actor=alice, raw=body, + headers={"Content-Type": ctype}, schema="Attachment")["json"] + while c.call("GET", "/api/v1/attachments/" + att["id"], actor=alice, schema="Attachment")["json"]["status"] != "ready": + time.sleep(0.2) + summary["recovery_probe_ready_s"] = round(time.monotonic() - t, 2) + record(summary) + finally: + set_admission(200) + + +if __name__ == "__main__": + main() diff --git a/student-tests/stand.py b/student-tests/stand.py new file mode 100755 index 00000000..8ac5e7ed --- /dev/null +++ b/student-tests/stand.py @@ -0,0 +1,132 @@ +"""Helpers for student resilience tests: the public HTTP client plus docker compose / psql / mc access. + +Standard library only (Python 3.9+). Images that need Pillow are generated inside the API container. +Never prints tokens or secrets. +""" +import json +import subprocess +import sys +import time +import uuid +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check, run # noqa: E402,F401 + +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def log(*parts): + print(time.strftime("%H:%M:%S"), *parts, flush=True) + + +def compose(*args, check_rc=True, capture=True, timeout=180): + done = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=capture, text=True, timeout=timeout) + if check_rc and done.returncode != 0: + raise Failure("docker compose %s failed: %s" % (" ".join(args[:2]), (done.stderr or "")[-300:])) + return done.stdout if capture else "" + + +def started_at(*services): + """Container name -> State.StartedAt, proof that a restart really happened.""" + ids = compose("ps", "-q", *services).split() + out = subprocess.run(["docker", "inspect", "-f", "{{.Name}} {{.State.StartedAt}}", *ids], + capture_output=True, text=True).stdout + return "; ".join(line.strip("/")[:60] for line in out.splitlines()) + + +def sql(query): + """Run a read query in postgres; returns rows as lists of strings (tab separated).""" + out = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "\t", "-c", query) + return [line.split("\t") for line in out.splitlines() if line] + + +def s3_keys(prefix): + """Object keys starting with prefix, listed with the root credentials inside a throwaway minio-init container.""" + wanted, prefix = prefix, prefix[:prefix.rfind("/") + 1] + script = ('mc alias set l http://minio:9000 "$(cat /run/secrets/minio_root_user)" ' + '"$(cat /run/secrets/minio_root_password)" >/dev/null && ' + 'mc ls --recursive --json l/messenger-attachments/%s || true' % prefix) + out = compose("run", "--rm", "--no-deps", "-T", "--entrypoint", "sh", "minio-init", "-c", script) + keys = [] + for line in out.splitlines(): + try: + item = json.loads(line) + except ValueError: + continue + if item.get("key"): + keys.append(prefix + item["key"]) + return sorted(k for k in keys if k.startswith(wanted)) + + +def queue_depth(queue="attachments.process"): + out = compose("exec", "-T", "rabbitmq", "rabbitmqctl", "-q", "--vhost", "messenger", "list_queues", + "name", "messages", "messages_ready", "messages_unacknowledged") + for line in out.splitlines(): + parts = line.split() + if parts and parts[0] == queue: + return {"messages": int(parts[1]), "ready": int(parts[2]), "unacked": int(parts[3])} + return None + + +def wait_queue(min_ready=1, seconds=30): + """Quorum queue counters are refreshed by the stats emitter (~5 s), so poll.""" + deadline = time.monotonic() + seconds + depth = None + while time.monotonic() < deadline: + depth = queue_depth() + if depth and depth["ready"] >= min_ready: + return depth + time.sleep(1) + return depth + + +def jobs_cli(*args): + done = subprocess.run(["docker", "compose", "exec", "-T", "api-1", "python", "scripts/jobs.py", *args], + cwd=ROOT, capture_output=True, text=True, timeout=60) + return done.returncode, json.loads(done.stdout) if done.stdout.strip() else None + + +def wait_http_ready(c, seconds=120): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + c.call("GET", "/health/ready", 200) + return + except Failure: + time.sleep(1) + raise Failure("API not ready after %ds" % seconds) + + +def upload(c, actor, chat, body=SAMPLE, filename="sample.png", ctype="image/png", expected=202): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe="")) + schema = "Attachment" if expected == 202 else None + return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype}, schema=schema)["json"] + + +def get(c, actor, att_id): + return c.call("GET", "/api/v1/attachments/" + att_id, actor=actor, schema="Attachment")["json"] + + +def wait_status(c, actor, att_id, statuses, seconds=120): + deadline = time.monotonic() + seconds + seen = [] + while time.monotonic() < deadline: + att = get(c, actor, att_id) + if not seen or seen[-1] != att["status"]: + seen.append(att["status"]) + if att["status"] in statuses: + return att, seen + time.sleep(0.5) + raise Failure("attachment %s did not reach %s in %ds (seen %s)" % (att_id, statuses, seconds, seen)) + + +def objects_of(att_id): + return s3_keys("attachments/%s/" % att_id) + s3_keys("staging/%s" % att_id) + + +def unique_name(fault=None): + tag = uuid.uuid4().hex[:8] + return ("fault-%s-%s.png" % (fault, tag)) if fault else ("plain-%s.png" % tag) diff --git a/student-tests/test_failure_windows.py b/student-tests/test_failure_windows.py new file mode 100755 index 00000000..05f617f7 --- /dev/null +++ b/student-tests/test_failure_windows.py @@ -0,0 +1,194 @@ +#!/usr/bin/env python3 +"""Grade 4 fault-injection suite (needs FAULT_INJECTION=1 on the stand, see REPORT.md). + +Each scenario uploads a file whose display name selects an injected fault (app/faults.py) and checks the +recovery through the public API plus DB/S3/broker state: + A api_crash_after_commit crash between COMMIT(job+outbox) and publish -> another replica's relay publishes + B worker_crash_after_put crash after S3 writes, before result COMMIT -> lease expiry -> same keys, one result + C worker_crash_after_commit crash after COMMIT, before ack -> redelivery is a no-op + D transient3 3 transient errors -> retries with growing jittered backoff -> ready + E permanent permanent error -> failed at attempt 1, no retry, content/attach 409 + F transient6 attempts exhausted -> failed/retry_exhausted -> registry -> explicit replay -> ready; + replay of a ready job is refused + G worker_hang hung worker (no heartbeats) -> other worker takes over, hung one is fenced + H poison message invalid broker message -> DLQ, not retried forever + I broker down RabbitMQ stopped: upload still 202 (job waits in outbox) -> broker back -> ready +usage: python3 student-tests/test_failure_windows.py [A B C ...] +""" +import sys +import time + +from stand import (Client, Failure, check, compose, get, jobs_cli, log, objects_of, queue_depth, run, sql, upload, + unique_name, wait_http_ready, wait_status) + + +def audit(att_id): + return sql("SELECT action, actor, attempt, coalesce(detail->>'error_code',''), coalesce(detail->>'delay_seconds','')" + " FROM job_audit WHERE attachment_id='%s' ORDER BY id" % att_id) + + +def show_audit(att_id): + for action, actor, attempt, code, delay in audit(att_id): + log(" audit attempt=%s %-16s %-28s %s %s" % (attempt, action, actor[:28], code, ("delay=%ss" % delay) if delay else "")) + + +def one_result(att_id): + variants = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att_id)[0][0] + succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att_id)[0][0] + keys = objects_of(att_id) + check(variants == "3", "exactly one set of 3 variant rows, got %s" % variants) + check(succeeded == "1", "exactly one committed success, got %s" % succeeded) + check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), + "original + 3 variants, staging cleaned: %s" % keys) + + +def ready_and_single(c, actor, att_id, seconds=120): + att, seen = wait_status(c, actor, att_id, ("ready", "failed"), seconds) + check(att["status"] == "ready", "expected ready, got %s/%s" % (att["status"], att["error_code"])) + log(" statuses:", " -> ".join(seen)) + show_audit(att_id) + one_result(att_id) + return att + + +def scenario_a(c, alice, bob, chat): + name = unique_name("api_crash_after_commit") + try: + upload(c, alice, chat, filename=name, expected=[202, 500, 502, 503]) + except Failure as exc: + log(" client saw:", exc) + wait_http_ready(c) + rows = sql("SELECT id FROM attachments WHERE filename='%s'" % name) + check(len(rows) == 1, "committed attachment must exist after API crash") + att_id = rows[0][0] + log(" committed attachment %s survived the API crash" % att_id) + ready_and_single(c, bob, att_id) + pub = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NOT NULL" % att_id)[0][0] + check(pub == "1", "outbox row published exactly once by the relay") + + +def scenario_b(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_crash_after_put")) + ready_and_single(c, bob, att["id"]) + actions = [r[0] for r in audit(att["id"])] + check("retry_scheduled" in actions, "lease expiry must schedule a new attempt") + + +def scenario_c(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_crash_after_commit")) + ready_and_single(c, bob, att["id"]) + time.sleep(8) + logs = compose("logs", "--since", "60s", "worker") + check("decision=done" in logs, "redelivered message after crash must be a no-op") + one_result(att["id"]) + + +def scenario_d(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("transient3")) + ready_and_single(c, bob, att["id"]) + delays = [float(r[4]) for r in audit(att["id"]) if r[0] == "retry_scheduled"] + check(len(delays) == 3, "three retries expected, got %s" % delays) + bounds = [(1, 2), (2, 4), (4, 8)] + check(all(lo <= d <= hi for d, (lo, hi) in zip(delays, bounds)), "equal-jitter backoff bounds %s" % delays) + log(" backoff delays:", delays) + + +def scenario_e(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("permanent")) + failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 60) + check(failed["status"] == "failed" and failed["error_code"] == "invalid_image", "permanent -> failed/invalid_image") + check(failed["variants"] == [], "failed attachment has no variants") + show_audit(att["id"]) + check([r[0] for r in audit(att["id"])].count("claimed") == 1, "permanent errors are not retried") + c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 409, actor=alice, + body={"text": "", "attachment_ids": [att["id"]]}, headers={"Idempotency-Key": "k-" + att["id"]}) + + +def scenario_f(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("transient6")) + failed, _ = wait_status(c, bob, att["id"], ("failed", "ready"), 120) + check(failed["status"] == "failed" and failed["error_code"] == "retry_exhausted", "exhausted -> retry_exhausted") + attempts = sql("SELECT attempt, max_attempts FROM attachment_jobs WHERE attachment_id='%s'" % att["id"])[0] + check(attempts == ["5", "5"], "bounded: stopped at attempt 5 of 5, got %s" % attempts) + rc, registry = jobs_cli("failed") + check(any(r["attachment_id"] == att["id"] for r in registry), "failed job listed in operator registry") + log(" failed registry contains the job; replay") + rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite") + check(rc == 0 and result["result"] == "queued", "replay must queue: %s" % result) + ready_and_single(c, bob, att["id"], 120) + rc, result = jobs_cli("replay", att["id"], "--actor", "test-suite") + check(rc == 3 and result["result"] == "refused", "replay of a ready job must be refused: %s" % result) + log(" second replay refused:", result["reason"]) + one_result(att["id"]) + + +def scenario_g(c, alice, bob, chat): + att = upload(c, alice, chat, filename=unique_name("worker_hang")) + ready_and_single(c, bob, att["id"], 150) + rows = audit(att["id"]) + check(any(r[0] == "retry_scheduled" and r[3] == "lease_expired" for r in rows), "lease expiry detected") + time.sleep(30) # the hung worker wakes up and must be fenced + logs = compose("logs", "--since", "180s", "worker") + check("lease lost" in logs or "result not committed" in logs, "hung worker must be fenced by the lease") + one_result(att["id"]) + + +POISON = ("import asyncio, aio_pika\n" + "from app import config, broker\n" + "async def m():\n" + " conn = await broker.connect(); ch = await conn.channel(publisher_confirms=True)\n" + " ex, _ = await broker.declare(ch)\n" + " await ex.publish(aio_pika.Message(b'{not json', delivery_mode=2), routing_key=broker.ROUTING_KEY)\n" + " await conn.close()\n" + "asyncio.run(m())\n") + + +def scenario_h(c, alice, bob, chat): + before = (queue_depth("attachments.dlq") or {}).get("messages", 0) + compose("exec", "-T", "api-1", "python", "-c", POISON) + deadline = time.monotonic() + 30 + after = before + while time.monotonic() < deadline and after <= before: + time.sleep(1) + after = (queue_depth("attachments.dlq") or {}).get("messages", 0) + check(after == before + 1, "poison message must land in attachments.dlq (%s -> %s)" % (before, after)) + log(" DLQ depth %s -> %s" % (before, after)) + + +def scenario_i(c, alice, bob, chat): + compose("stop", "-t", "10", "rabbitmq") + try: + att = upload(c, alice, chat, filename=unique_name()) + check(att["status"] == "queued", "upload accepted while broker is down") + c.message(alice, chat, "", attachments=[att["id"]]) + time.sleep(3) + pending = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0] + check(pending == "1", "job must wait in the outbox, got %s" % pending) + check(get(c, bob, att["id"])["status"] == "queued", "still queued while broker is down") + log(" broker down: upload 202, outbox holds 1 unpublished job") + finally: + compose("start", "rabbitmq") + ready_and_single(c, bob, att["id"], 120) + + +SCENARIOS = {"A": scenario_a, "B": scenario_b, "C": scenario_c, "D": scenario_d, "E": scenario_e, + "F": scenario_f, "G": scenario_g, "H": scenario_h, "I": scenario_i} + + +def main(): + c = Client(timeout=30) + alice, bob, eve, chat = c.bootstrap() + picked = sys.argv[1:] or list(SCENARIOS) + for key in picked: + log("scenario %s: %s" % (key, SCENARIOS[key].__name__)) + started = time.monotonic() + SCENARIOS[key](c, alice, bob, chat) + log("scenario %s PASS (%.1fs)" % (key, time.monotonic() - started)) + print("PASS: failure windows %s" % " ".join(picked)) + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_worker_down.py b/student-tests/test_worker_down.py new file mode 100755 index 00000000..49fc71e9 --- /dev/null +++ b/student-tests/test_worker_down.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Grade 3 demo: worker stopped -> upload 202 -> message with attachment -> restart API + broker (volumes kept) +-> worker started -> ready + thumbnail. Then an explicit duplicate delivery of the finished job is a no-op. + +python3 student-tests/test_worker_down.py (stack from REPORT.md must be running) +""" +import hashlib +import time + +from stand import (Client, check, compose, started_at, get, log, objects_of, wait_queue, run, sql, upload, wait_http_ready, + wait_status, unique_name, SAMPLE) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + log("stop all workers") + compose("stop", "-t", "20", "worker") + try: + att = upload(c, alice, chat, filename=unique_name()) + check(att["status"] == "queued" and att["variants"] == [] and att["error_code"] is None, "upload must be 202 queued") + log("upload 202 queued id=%s" % att["id"]) + msg = c.message(alice, chat, "", attachments=[att["id"]])["json"] + check(msg["attachment_ids"] == [att["id"]], "message must reference the queued attachment") + log("message %s created with queued attachment" % msg["id"]) + c.call("GET", "/api/v1/attachments/%s/content" % att["id"], 409, actor=bob) + c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], 409, actor=bob) + c.message(bob, chat, "другие сообщения работают") + depth = wait_queue() + log("broker queue while worker is down:", depth) + check(depth and depth["ready"] >= 1, "job must be waiting in the broker") + unpublished = sql("SELECT count(*) FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id" + " WHERE j.attachment_id='%s' AND o.published_at IS NULL" % att["id"])[0][0] + check(unpublished == "0", "outbox row must be published with confirm") + + log("restart api-1 api-2 rabbitmq (volumes kept)") + before = started_at("api-1", "api-2", "rabbitmq") + compose("restart", "-t", "10", "rabbitmq", "api-1", "api-2") + wait_http_ready(c) + after = started_at("api-1", "api-2", "rabbitmq") + log("started before:", before) + log("started after: ", after) + check(before != after, "containers must really restart") + check(get(c, bob, att["id"])["status"] == "queued", "still queued after restart, nothing lost") + depth = wait_queue() + log("broker queue after broker restart:", depth) + check(depth and depth["ready"] >= 1, "durable message must survive broker restart") + finally: + log("start workers") + compose("start", "worker") + ready, seen = wait_status(c, bob, att["id"], ("ready", "failed"), 90) + check(ready["status"] == "ready", "expected ready, got %s" % ready["status"]) + log("status sequence observed:", " -> ".join(seen)) + names = sorted(v["name"] for v in ready["variants"]) + check(names == ["crop", "optimized", "thumbnail"], "expected thumbnail+crop+optimized, got %s" % names) + thumb = [v for v in ready["variants"] if v["name"] == "thumbnail"][0] + check((thumb["width"], thumb["height"]) == (256, 160), "thumbnail 256x160") + body = c.call("GET", "/api/v1/attachments/%s/content?variant=thumbnail" % att["id"], actor=bob)["bytes"] + check(hashlib.sha256(body).hexdigest() == thumb["sha256"], "thumbnail checksum") + orig = c.call("GET", "/api/v1/attachments/%s/content" % att["id"], actor=bob)["bytes"] + check(orig == SAMPLE, "original unchanged byte for byte") + keys = objects_of(att["id"]) + log("objects:", keys) + check(len(keys) == 4 and not any(k.startswith("staging/") for k in keys), "original + 3 variants, staging removed") + + log("redelivery: republish the finished job through the outbox (duplicate message)") + sql("INSERT INTO outbox (job_id, payload, available_at, created_at) SELECT o.job_id, o.payload, now(), now()" + " FROM outbox o JOIN attachment_jobs j ON j.id=o.job_id WHERE j.attachment_id='%s' ORDER BY o.id LIMIT 1" + % att["id"]) + time.sleep(5) + again = get(c, bob, att["id"]) + check(again["status"] == "ready" and again["variants"] == ready["variants"], "duplicate must not change the result") + rows = sql("SELECT count(*) FROM attachment_variants WHERE attachment_id='%s'" % att["id"])[0][0] + succeeded = sql("SELECT count(*) FROM job_audit WHERE attachment_id='%s' AND action='succeeded'" % att["id"])[0][0] + check(rows == "3" and succeeded == "1", "one set of variants and one success (rows=%s succeeded=%s)" % (rows, succeeded)) + check(len(objects_of(att["id"])) == 4, "no new objects after duplicate") + noop = compose("logs", "--since", "30s", "worker") + check("decision=done" in noop, "worker must log the duplicate as no-op") + log("duplicate acked as no-op: variants=%s successes=%s" % (rows, succeeded)) + print("PASS: worker down / API+broker restart / redelivery no-op") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/track1_matrix.sh b/student-tests/track1_matrix.sh new file mode 100755 index 00000000..d03a5e8f --- /dev/null +++ b/student-tests/track1_matrix.sh @@ -0,0 +1,23 @@ +#!/bin/sh +# Track 1 matrix: 3 sizes x 4 worker settings x REPEATS, then the overload/recovery scenario. +# One worker replica with a 2-CPU limit; offered rates are fixed per size (open loop, 30 s each). +set -eu +cd "$(dirname "$0")/.." +REPEATS="${REPEATS:-2}" +for rep in $(seq 1 "$REPEATS"); do + for spec in "small 10" "medium 5" "large 3"; do + set -- $spec + for cp in "1 1" "2 2" "4 4" "2 16"; do + c=${cp% *}; p=${cp#* } + python3 student-tests/queue_load.py run --size "$1" --rate "$2" --duration 30 \ + --concurrency "$c" --prefetch "$p" --label "c$c-p$p-r$rep" | tail -1 + done + done +done +for rep in $(seq 1 "$REPEATS"); do + python3 student-tests/queue_load.py overload --max-pending 30 --size large --rate 8 --duration 30 \ + --concurrency 2 --prefetch 2 --label "overload-r$rep" | tail -1 +done +# back to the default topology +WORKER_REPLICAS=2 docker compose --env-file .env up -d --no-deps --wait worker >/dev/null +echo MATRIX_DONE diff --git a/student-tests/track1_summary.py b/student-tests/track1_summary.py new file mode 100755 index 00000000..80bf17a1 --- /dev/null +++ b/student-tests/track1_summary.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Aggregate evidence/track1/results.jsonl into evidence/track1/summary.md (mean over repeats, min..max shown).""" +import json +from collections import defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +rows = [json.loads(line) for line in (ROOT / "evidence/track1/results.jsonl").read_text().splitlines() if line.strip()] +groups = defaultdict(list) +for r in rows: + if r["label"].startswith("calib"): + continue + key = ("overload" if r["label"].startswith("overload") else "run", r["size"], r["concurrency"], r["prefetch"]) + groups[key].append(r) + + +def agg(items, field, digits=2): + values = [i[field] for i in items if i.get(field) is not None] + if not values: + return "—" + mean = sum(values) / len(values) + if len(values) == 1 or max(values) == min(values): + return ("%." + str(digits) + "f") % mean + return ("%." + str(digits) + "f (%." + str(digits) + "f–%." + str(digits) + "f)") % (mean, min(values), max(values)) + + +order = {"small": 0, "medium": 1, "large": 2} +lines = ["| size | rate/s | c/p | n | accepted | throughput jobs/s | lag p95 s | ttr p50 s | ttr p95 s | pending max |" + " worker mem peak MiB | cpu peak % |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"] +for key in sorted((k for k in groups if k[0] == "run"), key=lambda k: (order[k[1]], k[2], k[3])): + items = groups[key] + lines.append("| %s | %s | %d/%d | %d | %s | %s | %s | %s | %s | %s | %s | %s |" % ( + key[1], items[0]["rate"], key[2], key[3], len(items), agg(items, "accepted", 0), agg(items, "throughput_jobs_s"), + agg(items, "lag_p95_s"), agg(items, "ttr_p50_s"), agg(items, "ttr_p95_s"), agg(items, "pending_max", 0), + agg(items, "worker_mem_peak_mib", 0), agg(items, "worker_cpu_peak_pct", 0))) +over = [r for k, v in groups.items() if k[0] == "overload" for r in v] +if over: + lines += ["", "| overload run | offered | accepted | rejected 503 | other errors | pending max (limit) | throughput |" + " ttr p95 s | drain after load s | recovery probe ready s |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |"] + for r in over: + lines.append("| %s | %d @ %s/s | %d | %d | %d | %d (%d) | %.2f | %s | %s | %s |" % ( + r["label"], r["offered"], r["rate"], r["accepted"], r["rejected_503"], r["other_errors"], r["pending_max"], + r["max_pending"], r["throughput_jobs_s"] or 0, r["ttr_p95_s"], r["drain_after_load_s"], + r.get("recovery_probe_ready_s"))) +(ROOT / "evidence/track1/summary.md").write_text("\n".join(lines) + "\n") +print("\n".join(lines)) diff --git a/student-tests/variants_check.py b/student-tests/variants_check.py new file mode 100644 index 00000000..6ab763ca --- /dev/null +++ b/student-tests/variants_check.py @@ -0,0 +1,126 @@ +"""Variant rules and worker resource limits. Runs INSIDE the worker container (Pillow + app code): + +docker compose exec -T --index 1 worker python - < student-tests/variants_check.py +""" +import io +import json +import os +import resource +import subprocess +import sys +import tempfile +import time + +from PIL import Image + +sys.path.insert(0, "/app") +from app import config # noqa: E402 + +FAILS = [] + + +def check(cond, msg): + print(("ok " if cond else "FAIL ") + msg, flush=True) + if not cond: + FAILS.append(msg) + + +def run_child(path, mem_mb=config.WORKER_CHILD_MEMORY_MB, cpu=config.JOB_TIMEOUT_SECONDS, max_pixels=config.MAX_IMAGE_PIXELS): + out = tempfile.mkdtemp() + + def limits(): + resource.setrlimit(resource.RLIMIT_AS, (mem_mb << 20, mem_mb << 20)) + resource.setrlimit(resource.RLIMIT_CPU, (cpu, cpu + 5)) + + start = time.monotonic() + done = subprocess.run([sys.executable, "-m", "app.imageproc", path, out, str(max_pixels)], capture_output=True, + preexec_fn=limits, timeout=120, env={"PATH": os.environ["PATH"], "MALLOC_ARENA_MAX": "2"}) + usage = resource.getrusage(resource.RUSAGE_CHILDREN) + produced = {v["name"]: v for v in json.loads(done.stdout)} if done.returncode == 0 else {} + return done.returncode, produced, time.monotonic() - start, usage.ru_maxrss / 1024 + + +def save(img, fmt, **kw): + path = tempfile.mktemp(suffix="." + fmt.lower()) + img.save(path, fmt, **kw) + return path + + +def dims(v): + return v["width"], v["height"] + + +def real(v): + with Image.open(v["file"]) as im: + return im.format, im.size, "exif" in im.info, im + + +# 1. fixture 320x200 -> 256x160, crop 128, optimized not upscaled +rc, p, _, _ = run_child("/app/tests/fixtures/sample.png" if os.path.exists("/app/tests/fixtures/sample.png") else + save(Image.new("RGB", (320, 200), "red"), "PNG")) +check(rc == 0 and dims(p["thumbnail"]) == (256, 160), "320x200 -> thumbnail 256x160") +check(dims(p["crop"]) == (128, 128), "crop 128x128") +check(dims(p["optimized"]) == (320, 200) and real(p["optimized"])[0] == "WEBP", "optimized WebP 320x200, no upscale") +check(real(p["thumbnail"])[0] == "JPEG" and real(p["crop"])[0] == "JPEG", "thumbnail/crop are JPEG") + +# 2. small source: no upscale for thumbnail, upscale allowed for crop +rc, p, _, _ = run_child(save(Image.new("RGB", (40, 10), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (40, 10) and dims(p["crop"]) == (128, 128), "40x10: thumbnail 40x10, crop 128x128") + +# 3. extreme aspect ratio: floor rounding, minimum 1 px +rc, p, _, _ = run_child(save(Image.new("RGB", (3000, 2), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (256, 1) and dims(p["optimized"]) == (1280, 1), "3000x2 -> 256x1 / 1280x1 (min 1px)") +rc, p, _, _ = run_child(save(Image.new("RGB", (1001, 999), "blue"), "PNG")) +check(dims(p["thumbnail"]) == (256, 255), "1001x999 -> 256x255 (floor)") + +# 4. EXIF orientation 6 (rotate 90): stored 400x200 -> upright 200x400 -> thumbnail 128x256; EXIF stripped +exif = Image.Exif() +exif[0x0112] = 6 +img = Image.new("RGB", (400, 200), "white") +img.paste((255, 0, 0), (0, 0, 200, 200)) # left half red +rc, p, _, _ = run_child(save(img, "JPEG", exif=exif.tobytes(), quality=95)) +fmt, size, has_exif, im = real(p["thumbnail"]) +check(size == (128, 256), "EXIF orientation applied: 400x200 (o=6) -> 128x256") +with Image.open(p["thumbnail"]["file"]) as t: + top = t.convert("RGB").getpixel((64, 20)) +check(top[0] > 200 and top[1] < 80, "rotated content: red half on top after orientation 6") +check(not has_exif and "exif" not in Image.open(p["optimized"]["file"]).info, "EXIF removed from thumbnail/optimized") + +# 5. alpha composited on white for JPEG, kept for WebP +rgba = Image.new("RGBA", (100, 100), (0, 0, 0, 0)) +rc, p, _, _ = run_child(save(rgba, "PNG")) +with Image.open(p["thumbnail"]["file"]) as t: + px = t.convert("RGB").getpixel((50, 50)) +check(min(px) >= 250, "transparent PNG -> white JPEG thumbnail") +check(Image.open(p["optimized"]["file"]).mode == "RGBA", "WebP optimized keeps alpha") + +# 6. 16-bit grayscale PNG, CMYK JPEG, palette PNG +rc16, _, _, _ = run_child(save(Image.new("I;16", (300, 300), 40000), "PNG")) +rcc, _, _, _ = run_child(save(Image.new("CMYK", (300, 300), (0, 255, 0, 0)), "JPEG")) +rcp, _, _, _ = run_child(save(Image.new("P", (300, 300), 3), "PNG", transparency=3)) +check(rc16 == 0 and rcc == 0 and rcp == 0, "16-bit PNG, CMYK JPEG, palette+transparency PNG are processed") + +# 7. limits: pixels (exit 4), not an image (exit 3), memory (exit 5), CPU (SIGXCPU) +rc, _, _, _ = run_child(save(Image.new("RGB", (100, 100)), "PNG"), max_pixels=5000) +check(rc == 4, "pixel limit enforced before decode -> image_too_large (exit 4)") +junk = tempfile.mktemp() +open(junk, "wb").write(b"\x89PNG\r\n\x1a\n" + os.urandom(2000)) +rc, _, _, _ = run_child(junk) +check(rc == 3, "corrupted PNG -> invalid_image (exit 3)") +big = save(Image.new("RGB", (6000, 6000), "green"), "PNG") +rc, _, _, _ = run_child(big, mem_mb=160) +check(rc == 5, "36 Mpx PNG with 160 MiB address-space limit -> image_too_complex (exit 5), got %s" % rc) + +# 8. largest allowed inputs under the real limits: time and child peak RSS +for name, w, h, fmt in (("png", 7900, 5000, "PNG"), ("jpeg", 7900, 5000, "JPEG")): + noise = Image.effect_noise((w // 10, h // 10), 60).convert("RGB").resize((w, h)) + path = save(noise, fmt, **({"quality": 90} if fmt == "JPEG" else {"compress_level": 1})) + rc, p, secs, rss = run_child(path) + check(rc == 0 and dims(p["thumbnail"]) == (256, 162), "%.1f Mpx %s (%d MB file) ok in %.1fs, peak child RSS %.0f MiB" + % (w * h / 1e6, name.upper(), os.path.getsize(path) >> 20, secs, rss)) + if fmt == "PNG": + rc, _, _, _ = run_child(path, cpu=0) + check(rc == -24, "CPU limit exhausted -> SIGXCPU (worker: permanent image_too_complex), got %s" % rc) + +print("RESULT:", "PASS" if not FAILS else "FAIL %s" % FAILS) +sys.exit(1 if FAILS else 0) -- 2.54.0