From cbaadb7636e18a577f8511f8fc6d27cdb4a5496c Mon Sep 17 00:00:00 2001 From: Ejur22 Date: Fri, 20 Nov 2026 16:57:31 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A05:=20=D0=B0=D1=81=D0=B8=D0=BD=D1=85?= =?UTF-8?q?=D1=80=D0=BE=D0=BD=D0=BD=D0=B0=D1=8F=20=D0=BE=D0=B1=D1=80=D0=B0?= =?UTF-8?q?=D0=B1=D0=BE=D1=82=D0=BA=D0=B0=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5?= =?UTF-8?q?=D0=BD=D0=B8=D0=B9=20(RabbitMQ=20+=20worker),=20transactional?= =?UTF-8?q?=20outbox,=20=D0=B2=D0=B0=D1=80=D0=B8=D0=B0=D0=BD=D1=82=D1=8B,?= =?UTF-8?q?=20=D1=82=D1=80=D0=B5=D0=BA=205.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 37 +- .gitattributes | 2 + .gitignore | 2 + Dockerfile | 29 + Makefile | 7 +- REPORT.md | 143 ++++- compose.yaml | 277 ++++++++ evidence/lab5-failures.txt | 26 + evidence/lab5-overload.json | 148 +++++ evidence/lab5-overload.txt | 19 + evidence/lab5-smoke.txt | 3 + evidence/lab5-student-tests.txt | 60 ++ infra/nginx.conf | 62 ++ migrations/001_init.sql | 42 ++ migrations/002_idempotency.sql | 19 + migrations/003_cache_invalidations.sql | 15 + migrations/004_attachments.sql | 31 + migrations/005_jobs.sql | 49 ++ requirements-dev.txt | 2 + requirements.txt | 11 + scripts/check-failures.py | 285 +++++++++ scripts/check-queue-overload.py | 243 +++++++ scripts/check-revocation.py | 79 +++ scripts/gen-secrets.sh | 52 ++ scripts/restart.sh | 9 + scripts/restore-api.sh | 7 + scripts/rolling-update.sh | 38 ++ scripts/start-worker.sh | 6 + scripts/stop-one.sh | 26 + scripts/stop-worker.sh | 7 + secrets.example/README.md | 20 + src/__init__.py | 1 + src/app.py | 585 +++++++++++++++++ src/broker.py | 48 ++ src/config.py | 115 ++++ src/db.py | 42 ++ src/errors.py | 20 + src/imaging.py | 106 ++++ src/instance.py | 21 + src/invalidation.py | 76 +++ src/main.py | 60 ++ src/migrate.py | 67 ++ src/outbox.py | 78 +++ src/pagination.py | 53 ++ src/reconcile.py | 82 +++ src/replay.py | 46 ++ src/repositories.py | 773 +++++++++++++++++++++++ src/s3_bootstrap.py | 57 ++ src/s3_private_check.py | 37 ++ src/security.py | 33 + src/seed.py | 87 +++ src/sessions.py | 203 ++++++ src/state.py | 19 + src/storage.py | 129 ++++ src/worker.py | 248 ++++++++ student-tests/conftest.py | 67 ++ student-tests/http_client.py | 148 +++++ student-tests/test_attachments.py | 133 ++++ student-tests/test_auth.py | 122 ++++ student-tests/test_concurrency.py | 37 ++ student-tests/test_idempotency.py | 65 ++ student-tests/test_instances.py | 23 + student-tests/test_negative.py | 65 ++ student-tests/test_pagination.py | 79 +++ student-tests/test_pipeline.py | 74 +++ student-tests/test_race.py | 34 + student-tests/test_session_revocation.py | 34 + 67 files changed, 5603 insertions(+), 20 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/lab5-failures.txt create mode 100644 evidence/lab5-overload.json create mode 100644 evidence/lab5-overload.txt create mode 100644 evidence/lab5-smoke.txt create mode 100644 evidence/lab5-student-tests.txt create mode 100644 infra/nginx.conf create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 migrations/003_cache_invalidations.sql create mode 100644 migrations/004_attachments.sql create mode 100644 migrations/005_jobs.sql create mode 100644 requirements.txt create mode 100644 scripts/check-failures.py create mode 100644 scripts/check-queue-overload.py create mode 100644 scripts/check-revocation.py create mode 100755 scripts/gen-secrets.sh create mode 100755 scripts/restart.sh create mode 100755 scripts/restore-api.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/start-worker.sh create mode 100755 scripts/stop-one.sh create mode 100755 scripts/stop-worker.sh create mode 100644 secrets.example/README.md create mode 100644 src/__init__.py create mode 100644 src/app.py create mode 100644 src/broker.py create mode 100644 src/config.py create mode 100644 src/db.py create mode 100644 src/errors.py create mode 100644 src/imaging.py create mode 100644 src/instance.py create mode 100644 src/invalidation.py create mode 100644 src/main.py create mode 100644 src/migrate.py create mode 100644 src/outbox.py create mode 100644 src/pagination.py create mode 100644 src/reconcile.py create mode 100644 src/replay.py create mode 100644 src/repositories.py create mode 100644 src/s3_bootstrap.py create mode 100644 src/s3_private_check.py create mode 100644 src/security.py create mode 100644 src/seed.py create mode 100644 src/sessions.py create mode 100644 src/state.py create mode 100644 src/storage.py create mode 100644 src/worker.py create mode 100644 student-tests/conftest.py create mode 100644 student-tests/http_client.py create mode 100644 student-tests/test_attachments.py create mode 100644 student-tests/test_auth.py create mode 100644 student-tests/test_concurrency.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_instances.py create mode 100644 student-tests/test_negative.py create mode 100644 student-tests/test_pagination.py create mode 100644 student-tests/test_pipeline.py create mode 100644 student-tests/test_race.py create mode 100644 student-tests/test_session_revocation.py diff --git a/.dockerignore b/.dockerignore index bc16e556..6b0b11b1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,10 +1,20 @@ +# Keep the image minimal: only src/, migrations/ and requirements.txt are copied. .git .github +.gitignore +.gitattributes +.dockerignore +Dockerfile +Dockerfile.example +compose*.yaml +compose*.yml +stack*.yaml + +# Secrets / local state (never bake these in). .env .env.* -!.env.example secrets -certs +secrets.example volumes data backups @@ -13,8 +23,31 @@ backups *.p12 *.pfx *.jks + +# Local tooling and caches. .venv +venv __pycache__ +*.py[cod] node_modules +.pytest_cache +.mypy_cache +.ruff_cache +*.log + +# Tests, evidence and docs are not needed to run the API. +tests +student-tests evidence +examples +contracts +scripts +infra +*.md +lab.json +Makefile + +# Editor / OS noise. +.idea +.vscode .DS_Store diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..cf6c9216 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Keep shell hooks LF so they stay executable on Linux/CI. +*.sh text eol=lf diff --git a/.gitignore b/.gitignore index b5a71896..df2af196 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,7 @@ coverage/ .DS_Store .idea/ .vscode/ +# Personal study notes, not part of the submission. +EXPLAIN.md # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c8f60042 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 +# Lab3 messenger image. Python is interpreted, so one pinned stage is enough. +# Migrations run from a separate one-shot compose service using the same image. +FROM python:3.11.9-slim-bookworm + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt ./ +RUN python -m pip install --no-cache-dir -r requirements.txt + +COPY src ./src +COPY migrations ./migrations + +RUN useradd --create-home --uid 10001 --shell /usr/sbin/nologin appuser +USER appuser + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=20s --retries=5 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + +CMD ["python", "-m", "src.main"] diff --git a/Makefile b/Makefile index 5d86d171..8c8750da 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-student help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; test-student: own pytest; load: synthetic writes" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +28,6 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-student: + $(PYTHON) -m pytest student-tests -q diff --git a/REPORT.md b/REPORT.md index e98f75e9..c77dae7b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,148 @@ # Отчёт — лабораторная 5 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Журавский Егор БВТ2301 +- Целевая оценка; выбранный трек 5, если нужен: **5**, трек **5.1 «Очередь под перегрузкой»** +- Commit/PR: TODO после публикации (ветка `solution/БВТ2301-Журавский`) +- Использованные источники, библиотеки и помощь ИИ: FastAPI 0.115.6, Pydantic 2.10.6, Uvicorn 0.34.0, psycopg 3.2.3 + psycopg-pool 3.2.4, argon2-cffi 23.1.0, redis-py 5.2.1, boto3 1.35.90, Pillow 11.1.0, **pika 1.3.2**; pytest и openapi-spec-validator только для проверок. Инфраструктура: nginx 1.27.4, PostgreSQL 16.6, Valkey 8.0.2, SeaweedFS 3.80, RabbitMQ 3.13.7. Документация RabbitMQ reliability/confirms, Pillow, contracts/attachment-states.md. Помощь ИИ: генерация каркаса и текста, проверено запуском тестов. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Версии и prerequisites** + +| Компонент | Версия / образ | +| --- | --- | +| Runtime | Python 3.11.9 | +| API-образ | `python:3.11.9-slim-bookworm` (пин) | +| Балансировщик | `nginx:1.27.4-alpine` | +| PostgreSQL | `postgres:16.6-bookworm` | +| Valkey | `valkey/valkey:8.0.2-alpine` | +| S3 | `chrislusf/seaweedfs:3.80` | +| Брокер | `rabbitmq:3.13.7-alpine` | +| Python-зависимости | `requirements.txt` (+ `pika` для брокера) | +| Стенд | Windows 10 Pro 19045, Ryzen 5 3600 (6C/12T), 15.9 GiB RAM; хуки/Compose — в WSL Ubuntu (или Linux) | + +Prerequisites: Docker с Linux-контейнерами и Compose; POSIX-shell (WSL/Git Bash/Linux) для `scripts/*.sh` и `make test-persistence/test-failover`. + +**Создание secrets (не коммитятся)** + +```sh +sh scripts/gen-secrets.sh +# ./secrets/{database_password,database_url,cache_url,broker_password,broker_url,s3_*,s3_config.json} +``` + +**Запуск, bootstrap, проверка, остановка** + +```sh +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build # api×2 + lb + worker + postgres + valkey + s3 + broker +docker compose ps # дождаться healthy (migrate и s3-init — одноразовые) + +python tests/smoke.py --base-url http://localhost:8080 # публичный smoke +.venv\Scripts\python -m pytest student-tests -v # свои тесты + +# Сценарии отказов уровня 4 (worker/broker/replay/redelivery/lease) +python scripts/check-failures.py + +# Трек 5.1 — очередь под перегрузкой +python scripts/check-queue-overload.py --count 12 + +# Persistence и failover +make test-persistence ACTION_SCRIPT=scripts/restart.sh # в WSL/Linux +make test-failover ACTION_SCRIPT=scripts/stop-one.sh # затем scripts/restore-api.sh + +# Явный replay упавших задач: +docker compose exec -T api python -m src.replay # список +docker compose exec -T api python -m src.replay # повтор + +# Безопасная остановка (данные сохраняются): +docker compose stop +# Намеренное удаление учебных данных (тома PG, Valkey, S3, RabbitMQ): +docker compose down -v +``` + +**Роль → service → версия → health → volume → примечание** + +| Роль | service | версия | healthcheck | volume | +| --- | --- | --- | --- | --- | +| load_balancer | `lb` | nginx 1.27.4 | `GET /health/live` | — (публикует `127.0.0.1:8080`) | +| api ×2 | `api` | из исходников | `GET /health/ready` (PG+S3) | — | +| worker | `worker` | из исходников | healthcheck отключён (не HTTP-процесс) | — | +| broker | `broker` | rabbitmq 3.13.7 | `rabbitmq-diagnostics ping` | `brokerdata` | +| s3 | `s3` | seaweedfs 3.80 | master `cluster/status` | `s3data` | +| postgres | `postgres` | 16.6 | `pg_isready` | `pgdata` | +| valkey | `valkey` | 8.0.2 | `valkey-cli ping` | `valkeydata` | +| migrate / s3-init | одноразовые | из исходников | — | — | ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +**Схема:** `Клиент → nginx LB → API ×2 → PostgreSQL (метаданные/состояния) + Valkey (кэш сессий) + SeaweedFS S3 (байты)`, а также `RabbitMQ (durable broker) → worker → S3 + PostgreSQL`. Наружу опубликован только порт балансировщика. + +- **Асинхронный приём (201 → 202).** `POST …/attachments` валидирует размер/формат по содержимому, пишет полный оригинал в **durable staging** S3 (`staging/{id}`) и в **одной транзакции** создаёт строку `attachments(status=queued)` и строку задания в `attachment_jobs` (outbox). Только после COMMIT отвечаем `202 queued`. Остановка API после ответа не теряет работу. +- **Transactional outbox (закрытие разрыва COMMIT→publish).** Отдельный поток в API публикует неопубликованные задания в RabbitMQ с **publisher confirms** и лишь затем ставит `published_at`. Падение между COMMIT и publish оставляет строку неопубликованной; она уходит в брокер после рестарта без ручного вмешательства. Тот же механизм доставляет отложенные повторы (`available_at`). +- **Worker отдельным процессом.** Консьюмеры (число = `WORKER_CONCURRENCY`) с `prefetch=1` и ручным ack. Идемпотентный эффект с ключом `(attachment_id, pipeline_version, variant)`: варианты пишутся `upsert`-ом, результат публикуется **атомарно** (в одной транзакции: варианты + `status=ready` + `s3_key` + `done` задания). Повторная доставка ready-задачи — no-op; старая `pipeline_version` не перезаписывает новую. +- **Варианты (уровень 4).** thumbnail JPEG (вписан в 256×256, без увеличения, EXIF ориентация, alpha на белый, floor, min 1 px), crop JPEG центр 128×128, optimized WebP (вписан в 1280×1280 без увеличения, EXIF удалён, quality=80). Оригинал неизменен (побайтно), его `sha256`/`size` соответствуют источнику. +- **Повторы и финальные ошибки.** Временная ошибка → задание возвращается в `pending` с backoff+jitter и перепубликуется через outbox; по достижении `JOB_MAX_ATTEMPTS` → `failed` с `retry_exhausted`. Постоянная (невалидное изображение) → `failed` c `invalid_image` и ack. `src.replay` — явный повтор из реестра failed. +- **Lease/восстановление.** На время обработки ставится `lease_until`; watchdog в worker переводит `processing` с истёкшим lease обратно в `pending` (без размножения эффектов). +- **Staging/cleanup.** Staging удаляется только после устойчивой фиксации результата; сверка `src.reconcile` учитывает и staging, и оригинал, и варианты. +- **Критичные зависимости.** Для синхронного пути критичны PostgreSQL и S3 (`/health/ready`=503). Брокер не критичен для API-приёма: намерение уже durable в БД и уходит через outbox. Valkey обходится (кэш сессий). + +**Соответствие `.env.example`** + +| Переменная | Роль | +| --- | --- | +| `BROKER_URL_FILE` | файл с AMQP URL брокера; `BROKER_QUEUE` — имя durable-очереди | +| `WORKER_CONCURRENCY` | число консьюмеров worker (`prefetch=1` каждый) | +| `JOB_MAX_ATTEMPTS` | предел попыток до `failed` | +| `JOB_TIMEOUT_SECONDS` | lease обработки (защита от зависшего worker) | +| `STAGING_RETENTION_SECONDS` | retention staging (очистка после фиксации) | +| `S3_*`, `MAX_UPLOAD_BYTES`, `MAX_IMAGE_PIXELS` | S3 и лимиты (как в lab4) | +| `DATABASE_URL_FILE`, `CACHE_URL_FILE`, `SESSION_*`, `IDEMPOTENCY_TTL_SECONDS`, `API_REPLICAS` | как в предыдущих лабах | ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke (уровень 3) | `python tests/smoke.py …` | `202` → `ready`, JPEG thumbnail 256×160 | `evidence/lab5-smoke.txt` | +| API 202 после durable приёма; остановка API не теряет работу | `check-failures.py` сценарий 1 | при остановленном worker `queued`; после рестарта API и старта worker → `ready` | `evidence/lab5-failures.txt` | +| Отдельные broker и worker; at-least-once; недоступность worker не блокирует API | сценарий 1 | 202 без worker; resize не выполняется синхронно | `evidence/lab5-failures.txt` | +| Состояния queued/processing/ready/failed, оригинал + thumbnail | smoke + `test_pipeline.py` | варианты thumbnail/crop/optimized, реальные размеры JPEG | `evidence/lab5-student-tests.txt` | +| Ограниченный retry, финальный failed с безопасным error_code; повтор не размножает варианты | `check-failures.py` сценарии 3,4 | `retry_exhausted`; повторный ready-job — no-op | `evidence/lab5-failures.txt` | +| Уровень 4: crop 128×128 и optimized WebP | `test_pipeline.py` | размеры/типы/checksums совпадают | `evidence/lab5-student-tests.txt` | +| Уровень 4: разрыв COMMIT/publish закрыт outbox | сценарий 2 | broker down → job не опубликован → рестарт API → broker up → `ready` | `evidence/lab5-failures.txt` | +| Уровень 4: backoff+jitter, DLQ/replay | сценарий 3 + `src.replay` | после исправления причины replay → `ready` | `evidence/lab5-failures.txt` | +| Уровень 4: lease для зависшего processing | сценарий 5 | истёкший lease → watchdog → `ready` | `evidence/lab5-failures.txt` | +| **Трек 5.1** очередь под перегрузкой | `check-queue-overload.py --count 12` | таблица throughput/lag/p95/RSS, backpressure, восстановление | `evidence/lab5-overload.txt`, `.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза (трек 5.1): рост конкуренции worker повышает пропускную способность до предела, после чего упирается в CPU (Pillow) и память; durable-очередь даёт backpressure (глубина растёт, API продолжает принимать) и последующее восстановление. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдали | Восстановление/потери | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| worker остановлен, upload, рестарт API, старт worker | задача не теряется | `queued` → `ready` (thumbnail/crop/optimized) | без потерь | +| broker недоступен на момент COMMIT | intent durable, публикация после восстановления | job не опубликован → рестарт API → broker up → `ready` | без ручных действий | +| временные сбои обработки | backoff→failed(retry_exhausted)→replay | failed, replay → `ready` | без потерь | +| повторная доставка ready-job | no-op, варианты не дублируются | варианты неизменны | без потерь | +| зависший processing (истёк lease) | watchdog повторяет | `ready` | без потерь | +| перегрузка (трек 5.1) | throughput ↑ затем saturation, RSS ↑, backpressure | см. таблицу ниже | восстановление OK | + +**Трек 5.1 (closed-loop, count=12; small 640×480 ≈ 0.92 МБ, large 2200×1500 ≈ 9.9 МБ):** + +| concurrency | size | accepted | p50 ready, c | p95 ready, c | max queue | worker peak RSS, МБ | +| --- | --- | --- | --- | --- | --- | --- | +| 1 | small | 12 | 2.24 | 9.68 | 10 | 89.7 | +| 2 | small | 12 | 81.66 | 83.11 | 12 | 77.5 | +| 4 | small | 12 | 2.75 | 3.23 | 12 | 152.5 | +| 1 | large | 12 | 27.37 | 37.37 | 11 | 151.6 | +| 2 | large | 9 (+3 трансп.) | 25.99 | 48.76 | 8 | 168.1 | +| 4 | large | 11 (+1 трансп.) | 37.55 | 39.23 | 10 | 252.2 | + +Выводы: пропускная способность **не растёт** с конкуренцией — обработка CPU-bound (Pillow) под GIL, поэтому потоки делят одно ядро; растёт потребление RSS (90→252 МБ) и появляется хвост (p95). Очередь работает как backpressure: глубина достигает ~10–12, но API всё время принимает/ставит в очередь, а после всплеска очередь дренируется (recovery `ready` за <1 c). Влияние имеют и транспортные ограничения локального стенда (единичные клиентские таймауты `0` и один `503` в конце строки). Hardware: Ryzen 5 3600, RAM 16 ГБ; worker limit cpus=1.0/512 МБ. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- `make check`/smoke на Windows требуют `PYTHONUTF8=1`; `localhost` на Windows даёт ~2 c (IPv6) — в тестах используется `127.0.0.1`. +- Измерения — closed-loop на одном Docker-host; клиент один процесс, страдает coordinated omission, поэтому p95 включает транспортный хвост. Для строгих чисел нужны k6/Locust/wrk. +- Величина прироста throughput ограничена GIL; в реальности помогли бы процессы-реплики worker (scale) или вынос resize в не-GIL путь. +- Не заявлено: exactly-once (только at-least-once + идемпотентный эффект), HA брокера/БД/S3, TLS (lab7), rate limiting. +- Что подтверждено: сохранность принятой работы, outbox-восстановление, retry/replay, lease-восстановление, backpressure и восстановление очереди. Что остаётся гипотезой — масштабирование по числу процессов-реплик worker. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..58c120d2 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,277 @@ +# Lab5 topology: nginx LB -> 2+ API replicas -> shared PostgreSQL (metadata), +# Valkey (session cache) and a private S3 store; plus a durable RabbitMQ broker +# and a separate worker that produces the attachment variants asynchronously. +# Only the balancer is published. Secrets are file-based (./secrets, gitignored). +name: messenger-lab5 + +x-api-env: &api-env + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: ${SESSION_STORE:-valkey} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + CACHE_INVALIDATION_INTERVAL_SECONDS: ${CACHE_INVALIDATION_INTERVAL_SECONDS:-0.5} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + BROKER_URL_FILE: /run/secrets/broker_url + BROKER_QUEUE: ${BROKER_QUEUE:-attachment.process.v1} + WORKER_CONCURRENCY: ${WORKER_CONCURRENCY:-2} + JOB_MAX_ATTEMPTS: ${JOB_MAX_ATTEMPTS:-5} + JOB_TIMEOUT_SECONDS: ${JOB_TIMEOUT_SECONDS:-60} + STAGING_RETENTION_SECONDS: ${STAGING_RETENTION_SECONDS:-86400} + OUTBOX_INTERVAL_SECONDS: ${OUTBOX_INTERVAL_SECONDS:-0.5} + LEASE_RECOVERY_INTERVAL_SECONDS: ${LEASE_RECOVERY_INTERVAL_SECONDS:-2} + RETRY_BACKOFF_BASE_SECONDS: ${RETRY_BACKOFF_BASE_SECONDS:-0.5} + RETRY_BACKOFF_MAX_SECONDS: ${RETRY_BACKOFF_MAX_SECONDS:-30} + PIPELINE_VERSION: ${PIPELINE_VERSION:-1} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT: ${DB_POOL_TIMEOUT:-5} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + +x-api-secrets: &api-secrets + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + - broker_url + +x-api-depends: &api-depends + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + valkey: + condition: service_healthy + +services: + lb: + image: nginx:1.27.4-alpine + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "127.0.0.1:${PUBLISH_PORT:-8080}:8080" + depends_on: + - api + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live >/dev/null 2>&1 || exit 1"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 10s + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + + api: + build: + context: . + image: messenger-api:lab5 + environment: *api-env + secrets: *api-secrets + expose: + - "8080" + depends_on: *api-depends + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + interval: 5s + timeout: 3s + retries: 5 + start_period: 20s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "0.75" + memory: 256M + + worker: + build: + context: . + image: messenger-api:lab5 + command: ["python", "-m", "src.worker"] + environment: *api-env + secrets: *api-secrets + healthcheck: + disable: true + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + broker: + condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + migrate: + build: + context: . + image: messenger-api:lab5 + command: ["python", "-m", "src.migrate"] + environment: *api-env + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + + s3-init: + build: + context: . + image: messenger-api:lab5 + command: ["python", "-m", "src.s3_bootstrap"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ROOT_ACCESS_KEY_ID_FILE: /run/secrets/s3_root_access_key_id + S3_ROOT_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_root_secret_access_key + secrets: + - s3_root_access_key_id + - s3_root_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + + broker: + image: rabbitmq:3.13.7-alpine + entrypoint: + - /bin/sh + - -c + - 'export RABBITMQ_DEFAULT_PASS="$$(cat /run/secrets/broker_password)"; exec docker-entrypoint.sh rabbitmq-server' + environment: + RABBITMQ_DEFAULT_USER: messenger + secrets: + - broker_password + volumes: + - brokerdata:/var/lib/rabbitmq + expose: + - "5672" + healthcheck: + test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"] + interval: 5s + timeout: 5s + retries: 20 + start_period: 20s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.75" + memory: 384M + + s3: + image: chrislusf/seaweedfs:3.80 + command: + - server + - -dir=/data + - -ip=0.0.0.0 + - -s3 + - -s3.port=8333 + - -s3.config=/run/secrets/s3_config + volumes: + - s3data:/data + secrets: + - s3_config + expose: + - "8333" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:9333/cluster/status || exit 1"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 15s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + postgres: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/database_password + secrets: + - database_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + + valkey: + image: valkey/valkey:8.0.2-alpine + command: ["valkey-server", "--appendonly", "yes"] + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.50" + memory: 128M + +volumes: + pgdata: + valkeydata: + s3data: + brokerdata: + +secrets: + database_url: + file: ./secrets/database_url + database_password: + file: ./secrets/database_password + cache_url: + file: ./secrets/cache_url + broker_password: + file: ./secrets/broker_password + broker_url: + file: ./secrets/broker_url + s3_config: + file: ./secrets/s3_config.json + s3_root_access_key_id: + file: ./secrets/s3_root_access_key_id + s3_root_secret_access_key: + file: ./secrets/s3_root_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/evidence/lab5-failures.txt b/evidence/lab5-failures.txt new file mode 100644 index 00000000..9e2cf23f --- /dev/null +++ b/evidence/lab5-failures.txt @@ -0,0 +1,26 @@ + +== scenario 1: accept while worker stopped, resume -> ready == +with worker stopped: status=queued (expected queued) +API restarted; work still queued (durable) +worker resumed: status=ready variants=['crop', 'optimized', 'thumbnail'] + +== scenario 2: COMMIT/publish gap (broker down), API restart, then broker == +broker down: status=queued published_at='' (expected empty) +API restarted with broker still down; intent preserved in PostgreSQL +broker back: status=ready (outbox republished without new task) + +== scenario 3: transient failures -> failed(retry_exhausted) -> replay == +injected: staging object points to a missing key (transient fetch failure) +status=failed error_code=retry_exhausted (expected retry_exhausted) +explicit replay after recovery: {"replayed": 1} +after replay: status=ready variants=['crop', 'optimized', 'thumbnail'] + +== scenario 4: redelivery of a ready job is a no-op == +re-queued the same job (duplicate delivery) +status=ready variants=['crop', 'optimized', 'thumbnail'] (must be unchanged) + +== scenario 5: stuck processing -> lease recovery == +marked attachment+job processing with an expired lease +watchdog requeued and finished the job: status=ready + +PASS: lab5 grade-4 failure scenarios diff --git a/evidence/lab5-overload.json b/evidence/lab5-overload.json new file mode 100644 index 00000000..d24738f1 --- /dev/null +++ b/evidence/lab5-overload.json @@ -0,0 +1,148 @@ +{ + "results": [ + { + "concurrency": 1, + "size": "small", + "bytes": 922218, + "submitted": 12, + "accepted": 12, + "errors": 0, + "codes": { + "202": 12 + }, + "ready": 12, + "failed": 0, + "submit_seconds": 1.42, + "drain_seconds": 11.04, + "throughput_per_s": 0.96, + "p50_ready_s": 2.24, + "p95_ready_s": 9.68, + "max_queue_depth": 10, + "worker_peak_mem_mb": 89.7 + }, + { + "concurrency": 2, + "size": "small", + "bytes": 922218, + "submitted": 12, + "accepted": 12, + "errors": 0, + "codes": { + "202": 12 + }, + "ready": 12, + "failed": 0, + "submit_seconds": 0.45, + "drain_seconds": 85.08, + "throughput_per_s": 0.14, + "p50_ready_s": 81.66, + "p95_ready_s": 83.11, + "max_queue_depth": 12, + "worker_peak_mem_mb": 77.5 + }, + { + "concurrency": 4, + "size": "small", + "bytes": 922218, + "submitted": 12, + "accepted": 12, + "errors": 0, + "codes": { + "202": 12 + }, + "ready": 12, + "failed": 0, + "submit_seconds": 2.23, + "drain_seconds": 4.1, + "throughput_per_s": 1.9, + "p50_ready_s": 2.75, + "p95_ready_s": 3.23, + "max_queue_depth": 12, + "worker_peak_mem_mb": 152.5 + }, + { + "concurrency": 1, + "size": "large", + "bytes": 9902323, + "submitted": 12, + "accepted": 12, + "errors": 0, + "codes": { + "202": 12 + }, + "ready": 12, + "failed": 0, + "submit_seconds": 46.75, + "drain_seconds": 15.97, + "throughput_per_s": 0.19, + "p50_ready_s": 27.37, + "p95_ready_s": 37.37, + "max_queue_depth": 11, + "worker_peak_mem_mb": 151.6 + }, + { + "concurrency": 2, + "size": "large", + "bytes": 9902323, + "submitted": 12, + "accepted": 9, + "errors": 3, + "codes": { + "202": 9, + "0": 2, + "503": 1 + }, + "ready": 9, + "failed": 0, + "submit_seconds": 49.83, + "drain_seconds": 7.05, + "throughput_per_s": 0.16, + "p50_ready_s": 25.99, + "p95_ready_s": 48.76, + "max_queue_depth": 8, + "worker_peak_mem_mb": 168.1 + }, + { + "concurrency": 4, + "size": "large", + "bytes": 9902323, + "submitted": 12, + "accepted": 11, + "errors": 1, + "codes": { + "202": 11, + "0": 1 + }, + "ready": 6, + "failed": 0, + "submit_seconds": 45.04, + "drain_seconds": 302.74, + "throughput_per_s": 0.02, + "p50_ready_s": 37.55, + "p95_ready_s": 39.23, + "max_queue_depth": 10, + "worker_peak_mem_mb": 252.2 + }, + { + "concurrency": 2, + "size": "recovery", + "bytes": 1440778, + "submitted": 1, + "accepted": 1, + "errors": 0, + "codes": { + "202": 1 + }, + "ready": 1, + "failed": 0, + "submit_seconds": 0.1, + "drain_seconds": 2.14, + "throughput_per_s": 0.45, + "p50_ready_s": 0.62, + "p95_ready_s": 0.62, + "max_queue_depth": 0, + "worker_peak_mem_mb": 70.0 + } + ], + "recovered": true +} \ No newline at end of file diff --git a/evidence/lab5-overload.txt b/evidence/lab5-overload.txt new file mode 100644 index 00000000..1ae32e9f --- /dev/null +++ b/evidence/lab5-overload.txt @@ -0,0 +1,19 @@ +{"concurrency": 1, "size": "small", "bytes": 922218, "submitted": 12, "accepted": 12, "errors": 0, "codes": {"202": 12}, "ready": 12, "failed": 0, "submit_seconds": 1.42, "drain_seconds": 11.04, "throughput_per_s": 0.96, "p50_ready_s": 2.24, "p95_ready_s": 9.68, "max_queue_depth": 10, "worker_peak_mem_mb": 89.7} +{"concurrency": 2, "size": "small", "bytes": 922218, "submitted": 12, "accepted": 12, "errors": 0, "codes": {"202": 12}, "ready": 12, "failed": 0, "submit_seconds": 0.45, "drain_seconds": 85.08, "throughput_per_s": 0.14, "p50_ready_s": 81.66, "p95_ready_s": 83.11, "max_queue_depth": 12, "worker_peak_mem_mb": 77.5} +{"concurrency": 4, "size": "small", "bytes": 922218, "submitted": 12, "accepted": 12, "errors": 0, "codes": {"202": 12}, "ready": 12, "failed": 0, "submit_seconds": 2.23, "drain_seconds": 4.1, "throughput_per_s": 1.9, "p50_ready_s": 2.75, "p95_ready_s": 3.23, "max_queue_depth": 12, "worker_peak_mem_mb": 152.5} +{"concurrency": 1, "size": "large", "bytes": 9902323, "submitted": 12, "accepted": 12, "errors": 0, "codes": {"202": 12}, "ready": 12, "failed": 0, "submit_seconds": 46.75, "drain_seconds": 15.97, "throughput_per_s": 0.19, "p50_ready_s": 27.37, "p95_ready_s": 37.37, "max_queue_depth": 11, "worker_peak_mem_mb": 151.6} +{"concurrency": 2, "size": "large", "bytes": 9902323, "submitted": 12, "accepted": 9, "errors": 3, "codes": {"202": 9, "0": 2, "503": 1}, "ready": 9, "failed": 0, "submit_seconds": 49.83, "drain_seconds": 7.05, "throughput_per_s": 0.16, "p50_ready_s": 25.99, "p95_ready_s": 48.76, "max_queue_depth": 8, "worker_peak_mem_mb": 168.1} +{"concurrency": 4, "size": "large", "bytes": 9902323, "submitted": 12, "accepted": 11, "errors": 1, "codes": {"202": 11, "0": 1}, "ready": 6, "failed": 0, "submit_seconds": 45.04, "drain_seconds": 302.74, "throughput_per_s": 0.02, "p50_ready_s": 37.55, "p95_ready_s": 39.23, "max_queue_depth": 10, "worker_peak_mem_mb": 252.2} +{"concurrency": 2, "size": "recovery", "bytes": 1440778, "submitted": 1, "accepted": 1, "errors": 0, "codes": {"202": 1}, "ready": 1, "failed": 0, "submit_seconds": 0.1, "drain_seconds": 2.14, "throughput_per_s": 0.45, "p50_ready_s": 0.62, "p95_ready_s": 0.62, "max_queue_depth": 0, "worker_peak_mem_mb": 70.0} + +concurrency size bytes accepted ready p50_s p95_s maxq peakRSS_MB + 1 small 922218 12 12 2.24 9.68 10 89.7 + 2 small 922218 12 12 81.66 83.11 12 77.5 + 4 small 922218 12 12 2.75 3.23 12 152.5 + 1 large 9902323 12 12 27.37 37.37 11 151.6 + 2 large 9902323 9 9 25.99 48.76 8 168.1 + 4 large 9902323 11 6 37.55 39.23 10 252.2 + 2 recovery 1440778 1 1 0.62 0.62 0 70.0 + +recovery after overload: OK +{"output": "evidence/lab5-overload.json", "recovered": true} diff --git a/evidence/lab5-smoke.txt b/evidence/lab5-smoke.txt new file mode 100644 index 00000000..daf9dac1 --- /dev/null +++ b/evidence/lab5-smoke.txt @@ -0,0 +1,3 @@ +# Public acceptance scenario (lab5) +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab5 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/lab5-student-tests.txt b/evidence/lab5-student-tests.txt new file mode 100644 index 00000000..85891158 --- /dev/null +++ b/evidence/lab5-student-tests.txt @@ -0,0 +1,60 @@ +# Student black-box tests (grade 4/5), one command +$ C:\IT-labs\VNP\mtusi2026lab5\.venv\Scripts\python.exe -m pytest student-tests -v +============================= test session starts ============================= +platform win32 -- Python 3.11.9, pytest-8.3.4, pluggy-1.7.0 -- C:\IT-labs\VNP\mtusi2026lab5\.venv\Scripts\python.exe +cachedir: .pytest_cache +rootdir: C:\IT-labs\VNP\mtusi2026lab5 +plugins: anyio-4.16.0 +collecting ... collected 49 items + +student-tests/test_attachments.py::test_upload_is_202_queued_then_ready PASSED [ 2%] +student-tests/test_attachments.py::test_content_409_while_queued PASSED [ 4%] +student-tests/test_attachments.py::test_foreign_actor_gets_404 PASSED [ 6%] +student-tests/test_attachments.py::test_cross_chat_attachment_is_404 PASSED [ 8%] +student-tests/test_attachments.py::test_oversized_upload_is_413 PASSED [ 10%] +student-tests/test_attachments.py::test_corrupt_image_is_415 PASSED [ 12%] +student-tests/test_attachments.py::test_mime_spoof_is_415 PASSED [ 14%] +student-tests/test_attachments.py::test_empty_upload_is_415 PASSED [ 16%] +student-tests/test_attachments.py::test_unsafe_filename_is_400 PASSED [ 18%] +student-tests/test_attachments.py::test_empty_text_without_attachment_is_400 PASSED [ 20%] +student-tests/test_auth.py::test_registration_requires_password PASSED [ 22%] +student-tests/test_auth.py::test_short_password_is_400 PASSED [ 24%] +student-tests/test_auth.py::test_extra_field_is_400 PASSED [ 26%] +student-tests/test_auth.py::test_malformed_json_is_400 PASSED [ 28%] +student-tests/test_auth.py::test_duplicate_username_is_409 PASSED [ 30%] +student-tests/test_auth.py::test_registration_never_returns_password PASSED [ 32%] +student-tests/test_auth.py::test_wrong_password_is_401_basic PASSED [ 34%] +student-tests/test_auth.py::test_unknown_user_is_401_basic PASSED [ 36%] +student-tests/test_auth.py::test_basic_on_business_endpoint_is_401_bearer PASSED [ 38%] +student-tests/test_auth.py::test_missing_auth_is_401_bearer PASSED [ 40%] +student-tests/test_auth.py::test_unknown_bearer_is_401 PASSED [ 42%] +student-tests/test_auth.py::test_x_user_id_is_ignored PASSED [ 44%] +student-tests/test_auth.py::test_logout_revokes_session PASSED [ 46%] +student-tests/test_concurrency.py::test_100_concurrent_messages_are_not_lost PASSED [ 48%] +student-tests/test_idempotency.py::test_replay_same_key_returns_same_message PASSED [ 51%] +student-tests/test_idempotency.py::test_same_key_different_body_is_409 PASSED [ 53%] +student-tests/test_idempotency.py::test_missing_key_is_400 PASSED [ 55%] +student-tests/test_idempotency.py::test_invalid_key_is_400 PASSED [ 57%] +student-tests/test_idempotency.py::test_concurrent_same_key_creates_exactly_one_message PASSED [ 59%] +student-tests/test_instances.py::test_instance_header_present_on_success PASSED [ 61%] +student-tests/test_instances.py::test_instance_header_present_on_error PASSED [ 63%] +student-tests/test_instances.py::test_at_least_two_instances_seen_through_lb PASSED [ 65%] +student-tests/test_negative.py::test_foreign_chat_is_404 PASSED [ 67%] +student-tests/test_negative.py::test_empty_message_is_400 PASSED [ 69%] +student-tests/test_negative.py::test_oversized_message_is_400 PASSED [ 71%] +student-tests/test_negative.py::test_limit_out_of_range_is_400 PASSED [ 73%] +student-tests/test_negative.py::test_sender_id_in_body_is_rejected PASSED [ 75%] +student-tests/test_negative.py::test_chat_without_caller_is_400 PASSED [ 77%] +student-tests/test_negative.py::test_chat_with_unknown_member_is_400 PASSED [ 79%] +student-tests/test_negative.py::test_chat_with_duplicate_members_is_400 PASSED [ 81%] +student-tests/test_pagination.py::test_full_traversal_without_duplicates PASSED [ 83%] +student-tests/test_pagination.py::test_cursor_from_other_chat_is_400 PASSED [ 85%] +student-tests/test_pagination.py::test_garbage_cursor_is_400 PASSED [ 87%] +student-tests/test_pagination.py::test_well_formed_cursor_with_wrong_types_is_400 PASSED [ 89%] +student-tests/test_pipeline.py::test_variants_match_contract PASSED [ 91%] +student-tests/test_pipeline.py::test_missing_variant_is_404 PASSED [ 93%] +student-tests/test_race.py::test_concurrent_duplicate_username_single_winner PASSED [ 95%] +student-tests/test_race.py::test_failed_chat_creation_rolls_back PASSED [ 97%] +student-tests/test_session_revocation.py::test_logout_invalidates_across_replicas PASSED [100%] + +============================= 49 passed in 15.19s ============================= diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..955e158b --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,62 @@ +# Lab4 edge load balancer. +# +# Docker's embedded DNS (127.0.0.11) returns one A record per running `api` +# replica. `resolve` + a shared `zone` make nginx re-resolve that name every +# `valid` seconds, so replicas that appear/disappear are picked up without a +# reload. Passive checks (max_fails/fail_timeout) eject a replica that refuses +# connections; proxy_next_upstream retries the same request on a healthy peer. +worker_processes auto; + +events { + worker_connections 2048; +} + +http { + resolver 127.0.0.11 valid=5s ipv6=off; + resolver_timeout 2s; + + access_log /dev/stdout; + error_log /dev/stderr warn; + + sendfile on; + tcp_nopush on; + keepalive_timeout 30; + + upstream api_backend { + zone api_backend 64k; + server api:8080 resolve max_fails=1 fail_timeout=5s; + } + + server { + listen 8080; + server_name _; + + # Allow one attachment (<= 10 MiB) plus multipart-free JSON overhead. + client_max_body_size 12m; + + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + + proxy_connect_timeout 2s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + + # Retry a request on another replica if a backend is down mid-flight. + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + location /health/live { + access_log off; + proxy_pass http://api_backend; + } + + location / { + proxy_pass http://api_backend; + } + } +} diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..41499537 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,42 @@ +-- Lab2 initial schema. Idempotent and safe to re-run via the migration runner. +create table if not exists users ( + id uuid primary key, + username text not null unique, + display_name text not null, + password_hash text not null, + created_at timestamptz not null +); + +create table if not exists chats ( + id uuid primary key, + title text not null, + created_at timestamptz not null +); + +create table if not exists chat_members ( + chat_id uuid not null references chats (id) on delete cascade, + user_id uuid not null references users (id) on delete cascade, + primary key (chat_id, user_id) +); + +create table if not exists messages ( + id uuid primary key, + chat_id uuid not null references chats (id) on delete cascade, + sender_id uuid not null references users (id), + text text not null, + created_at timestamptz not null +); + +-- Composite index prepared for keyset pagination: +-- where chat_id = $1 and (created_at, id) > ($2, $3) order by created_at, id +create index if not exists messages_chat_created_id_idx + on messages (chat_id, created_at, id); + +create table if not exists sessions ( + token_hash text primary key, + user_id uuid not null references users (id) on delete cascade, + created_at timestamptz not null, + expires_at timestamptz not null +); + +create index if not exists sessions_expires_at_idx on sessions (expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..2fba4094 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,19 @@ +-- Lab3 shared idempotency ledger for POST message retries. +-- +-- Scope is (user_id, HTTP method + chat path, key); the stored row carries the +-- canonical request hash and the exact 201 body so any replica can replay the +-- same Message. Rows expire after IDEMPOTENCY_TTL_SECONDS (default 24h). +create table if not exists idempotency_keys ( + user_id uuid not null references users (id) on delete cascade, + scope text not null, + key text not null, + request_hash text not null, + status_code integer not null, + response_body jsonb not null, + created_at timestamptz not null default now(), + expires_at timestamptz not null, + primary key (user_id, scope, key) +); + +create index if not exists idempotency_keys_expires_at_idx + on idempotency_keys (expires_at); diff --git a/migrations/003_cache_invalidations.sql b/migrations/003_cache_invalidations.sql new file mode 100644 index 00000000..3b7d274c --- /dev/null +++ b/migrations/003_cache_invalidations.sql @@ -0,0 +1,15 @@ +-- Lab3 durable cache invalidation queue. +-- +-- Logout deletes the authoritative session row in PostgreSQL and enqueues the +-- token hash here in the SAME transaction. A lightweight background drainer in +-- each API replica removes the corresponding Valkey cache key and then the row. +-- If Valkey was unreachable at logout time, the row survives and the stale +-- cache entry is removed as soon as the cache comes back, instead of being +-- trusted until its TTL. +create table if not exists cache_invalidations ( + token_hash text primary key, + created_at timestamptz not null default now() +); + +create index if not exists cache_invalidations_created_at_idx + on cache_invalidations (created_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..0d3dce7d --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,31 @@ +-- Lab4 attachments: metadata in PostgreSQL, bytes in S3. +-- +-- ``s3_key`` is server-generated and never exposed through the API. Linkage to +-- a message is a many-to-many table so a message can carry ordered attachments +-- and the same attachment can be referenced again without duplicating bytes. +create table if not exists attachments ( + id uuid primary key, + chat_id uuid not null references chats (id) on delete cascade, + owner_id uuid not null references users (id), + filename text not null, + content_type text not null, + size_bytes bigint not null, + sha256 text not null, + status text not null, + s3_key text not null, + error_code text, + created_at timestamptz not null, + updated_at timestamptz not null +); + +create index if not exists attachments_chat_id_idx on attachments (chat_id); + +create table if not exists message_attachments ( + message_id uuid not null references messages (id) on delete cascade, + attachment_id uuid not null references attachments (id) on delete cascade, + position integer not null, + primary key (message_id, attachment_id) +); + +create index if not exists message_attachments_attachment_idx + on message_attachments (attachment_id); diff --git a/migrations/005_jobs.sql b/migrations/005_jobs.sql new file mode 100644 index 00000000..91215497 --- /dev/null +++ b/migrations/005_jobs.sql @@ -0,0 +1,49 @@ +-- Lab5: asynchronous attachment processing. +-- +-- * attachments gains a staging pointer (the durable upload) and a pipeline +-- version; s3_key becomes the FINAL original and is set by the worker, so it +-- must be nullable while the attachment is queued. +-- * attachment_variants holds one row per (attachment, pipeline_version, name) +-- so reprocessing is idempotent and an older version can never overwrite a +-- newer result. +-- * attachment_jobs is both the job registry and the transactional outbox: +-- the API writes a row in the same transaction as the attachment, and a +-- publisher thread moves rows to the broker (published_at) closing the +-- COMMIT -> publish gap. +alter table attachments alter column s3_key drop not null; +alter table attachments add column if not exists staging_key text; +alter table attachments add column if not exists pipeline_version integer not null default 1; + +create table if not exists attachment_variants ( + id uuid primary key, + attachment_id uuid not null references attachments (id) on delete cascade, + pipeline_version integer not null, + name text not null, + content_type text not null, + size_bytes bigint not null, + width integer not null, + height integer not null, + sha256 text not null, + s3_key text not null, + created_at timestamptz not null, + unique (attachment_id, pipeline_version, name) +); + +create table if not exists attachment_jobs ( + job_id uuid primary key, + attachment_id uuid not null references attachments (id) on delete cascade, + pipeline_version integer not null, + attempt integer not null default 1, + status text not null default 'pending', -- pending|processing|done|failed + available_at timestamptz not null default now(), + lease_until timestamptz, + published_at timestamptz, + last_error text, + created_at timestamptz not null default now(), + updated_at timestamptz not null default now() +); + +create index if not exists attachment_jobs_outbox_idx + on attachment_jobs (published_at, available_at); +create index if not exists attachment_jobs_lease_idx + on attachment_jobs (status, lease_until); diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..a7643f94 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,4 @@ # Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 +# Student-owned black-box tests (grade 4). +pytest==8.3.4 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8fb51d1c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,11 @@ +# Runtime dependencies for the lab5 messenger. Pinned for reproducibility. +fastapi==0.115.6 +pydantic==2.10.6 +uvicorn[standard]==0.34.0 +psycopg[binary]==3.2.3 +psycopg-pool==3.2.4 +argon2-cffi==23.1.0 +redis==5.2.1 +boto3==1.35.90 +pillow==11.1.0 +pika==1.3.2 diff --git a/scripts/check-failures.py b/scripts/check-failures.py new file mode 100644 index 00000000..c262376d --- /dev/null +++ b/scripts/check-failures.py @@ -0,0 +1,285 @@ +#!/usr/bin/env python3 +"""Grade-4 failure scenarios for the lab5 attachment pipeline. + +Each scenario is explicit and reproducible: + 1. worker-stopped accept: 202 while no worker, resume -> ready; + 2. outbox gap: broker down at COMMIT -> job stays unpublished -> API restart + -> broker returns -> ready (no new task needed); + 3. retry/replay: repeated transient failures -> failed(retry_exhausted) -> + operator replay after recovery -> ready; + 4. redelivery of a ready job is a no-op (variants not duplicated); + 5. lease recovery: a stuck 'processing' job is requeued by the watchdog. + +Run from the repository root (docker CLI + running lab5 stand). +""" +from __future__ import annotations + +import argparse +import base64 +import io +import json +import os +import struct +import subprocess +import time +import urllib.error +import urllib.request +import uuid +import zlib + +BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + + +def _png_chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + +def make_png(width=320, height=200): + raw = bytearray() + pixels = os.urandom(width * height * 3) + for row in range(height): + raw.append(0) + raw += pixels[row * width * 3 : (row + 1) * width * 3] + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _png_chunk(b"IHDR", ihdr) + _png_chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _png_chunk(b"IEND", b"") + + +def http(method, path, token=None, body=None, raw=None, content_type=None): + headers = {"Accept": "application/json"} + data = raw + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if content_type: + headers["Content-Type"] = content_type + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=20) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + except (urllib.error.URLError, TimeoutError, OSError): + return 0, b"" + + +def compose(*args, check=True): + return subprocess.run(["docker", "compose", *args], check=check, capture_output=True, text=True) + + +def psql(sql): + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-tAc", sql], + capture_output=True, text=True, + ) + return result.stdout.strip() + + +def service_running(name: str) -> bool: + result = subprocess.run( + ["docker", "compose", "ps", "--status", "running", "--format", "{{.Service}}"], + capture_output=True, text=True, + ) + return name in result.stdout.split() + + +def storage_reachable() -> bool: + # Raw TCP probe (bypasses the app's circuit breaker, which cannot be used to + # decide whether the store is really down). + result = subprocess.run( + ["docker", "compose", "exec", "-T", "api", "python", "-c", + "import socket,sys; socket.create_connection(('s3',8333),1); sys.exit(0)"], + capture_output=True, text=True, timeout=30, + ) + return result.returncode == 0 + + +def wait_storage_down(timeout: float = 60.0) -> None: + deadline = time.time() + timeout + while time.time() < deadline: + if not storage_reachable(): + return + time.sleep(1) + raise SystemExit("S3 still reachable; cannot run the failure scenario") + + +def wait_ready(timeout=90.0): + deadline = time.time() + timeout + while time.time() < deadline: + if http("GET", "/health/ready")[0] == 200: + return + time.sleep(1) + raise SystemExit("stand did not become ready") + + +def new_actor(): + username = "fail_" + uuid.uuid4().hex[:8] + password = "T3st-" + uuid.uuid4().hex + http("POST", "/api/v1/users", body={"username": username, "display_name": "Fail", "password": password}) + credential = base64.b64encode(f"{username}:{password}".encode()).decode() + request = urllib.request.Request( + BASE + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential, "Accept": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=20) as response: + token = json.loads(response.read())["token"] + user_id = json.loads(http("GET", "/api/v1/users/me", token=token)[1])["id"] + chat = json.loads(http("POST", "/api/v1/chats", token=token, body={"title": "fail", "member_ids": [user_id]})[1]) + return token, chat["id"] + + +def upload(token, chat_id, data=None): + data = data if data is not None else make_png() + status, body = http("POST", f"/api/v1/chats/{chat_id}/attachments?filename=x.png", token=token, raw=data, content_type="image/png") + assert status == 202, (status, body) + return json.loads(body)["id"] + + +def status_of(token, attachment_id): + return json.loads(http("GET", f"/api/v1/attachments/{attachment_id}", token=token)[1]) + + +def wait_status(token, attachment_id, wanted, timeout=120.0): + deadline = time.time() + timeout + last = None + while time.time() < deadline: + last = status_of(token, attachment_id) + if last["status"] == wanted: + return last + time.sleep(0.5) + raise SystemExit(f"attachment {attachment_id} did not reach {wanted}: {last}") + + +def scenario_worker_stopped(log): + log("\n== scenario 1: accept while worker stopped, resume -> ready ==") + compose("stop", "worker") + try: + token, chat_id = new_actor() + att_id = upload(token, chat_id) + time.sleep(2) + state = status_of(token, att_id) + log(f"with worker stopped: status={state['status']} (expected queued)") + assert state["status"] == "queued", state + compose("restart", "api") + wait_ready() + log("API restarted; work still queued (durable)") + assert status_of(token, att_id)["status"] == "queued" + finally: + compose("start", "worker") + ready = wait_status(token, att_id, "ready") + names = sorted(v["name"] for v in ready["variants"]) + log(f"worker resumed: status=ready variants={names}") + assert {"thumbnail", "crop", "optimized"} <= set(names) + + +def scenario_outbox_gap(log): + log("\n== scenario 2: COMMIT/publish gap (broker down), API restart, then broker ==") + compose("stop", "broker") + try: + token, chat_id = new_actor() + att_id = upload(token, chat_id) + time.sleep(2) + job = psql(f"select published_at from attachment_jobs where attachment_id='{att_id}'") + log(f"broker down: status={status_of(token, att_id)['status']} published_at={job!r} (expected empty)") + assert status_of(token, att_id)["status"] in {"queued", "processing"} + compose("restart", "api") + wait_ready() + log("API restarted with broker still down; intent preserved in PostgreSQL") + finally: + compose("start", "broker") + ready = wait_status(token, att_id, "ready", timeout=180) + log(f"broker back: status={ready['status']} (outbox republished without new task)") + + +def scenario_retry_replay(log): + log("\n== scenario 3: transient failures -> failed(retry_exhausted) -> replay ==") + compose("stop", "worker") + while service_running("worker"): + time.sleep(0.5) + token, chat_id = new_actor() + att_id = upload(token, chat_id) + original_key = psql(f"select staging_key from attachments where id='{att_id}'") + # Fault injection: the durable original is (temporarily) unavailable, so the + # worker's fetch fails and is retried with backoff until attempts are spent. + psql(f"update attachments set staging_key='staging/missing-{att_id}' where id='{att_id}'") + log("injected: staging object points to a missing key (transient fetch failure)") + compose("start", "worker") + failed = wait_status(token, att_id, "failed", timeout=180) + log(f"status=failed error_code={failed['error_code']} (expected retry_exhausted)") + assert failed["error_code"] == "retry_exhausted", failed + # Operator repairs the cause and replays the failed job explicitly. + psql(f"update attachments set staging_key='{original_key}' where id='{att_id}'") + job_id = att_id_job(att_id) + replayed = subprocess.run( + ["docker", "compose", "exec", "-T", "api", "python", "-m", "src.replay", job_id], + capture_output=True, text=True, + ).stdout.strip() + log(f"explicit replay after recovery: {replayed}") + ready = wait_status(token, att_id, "ready", timeout=120) + log(f"after replay: status=ready variants={sorted(v['name'] for v in ready['variants'])}") + + +def att_id_job(attachment_id): + return psql(f"select job_id from attachment_jobs where attachment_id='{attachment_id}'") + + +def scenario_redelivery(log): + log("\n== scenario 4: redelivery of a ready job is a no-op ==") + token, chat_id = new_actor() + att_id = upload(token, chat_id) + ready = wait_status(token, att_id, "ready") + before = sorted((v["name"], v["sha256"]) for v in ready["variants"]) + # Republish the finished job (duplicate delivery). + psql(f"update attachment_jobs set status='pending', published_at=null, available_at=now() where attachment_id='{att_id}'") + log("re-queued the same job (duplicate delivery)") + time.sleep(6) + after = status_of(token, att_id) + after_variants = sorted((v["name"], v["sha256"]) for v in after["variants"]) + log(f"status={after['status']} variants={[n for n,_ in after_variants]} (must be unchanged)") + assert after["status"] == "ready" + assert after_variants == before, "duplicate delivery changed variants" + + +def scenario_lease_recovery(log): + log("\n== scenario 5: stuck processing -> lease recovery ==") + compose("stop", "worker") + token, chat_id = new_actor() + att_id = upload(token, chat_id) + job_id = att_id_job(att_id) + # Simulate a worker that died mid-processing with an already-expired lease. + psql(f"update attachments set status='processing' where id='{att_id}'") + psql(f"update attachment_jobs set status='processing', lease_until=now() - interval '5 seconds' where job_id='{job_id}'") + log("marked attachment+job processing with an expired lease") + compose("start", "worker") + ready = wait_status(token, att_id, "ready", timeout=120) + log(f"watchdog requeued and finished the job: status={ready['status']}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--only", choices=["1", "2", "3", "4", "5"], help="run one scenario") + args = parser.parse_args() + wait_ready() + + def log(message): + print(message, flush=True) + + scenarios = { + "1": scenario_worker_stopped, + "2": scenario_outbox_gap, + "3": scenario_retry_replay, + "4": scenario_redelivery, + "5": scenario_lease_recovery, + } + if args.only: + scenarios[args.only](log) + else: + for name in ("1", "2", "3", "4", "5"): + scenarios[name](log) + print("\nPASS: lab5 grade-4 failure scenarios" + (f" (scenario {args.only})" if args.only else "")) + + +if __name__ == "__main__": + main() diff --git a/scripts/check-queue-overload.py b/scripts/check-queue-overload.py new file mode 100644 index 00000000..7466b9fc --- /dev/null +++ b/scripts/check-queue-overload.py @@ -0,0 +1,243 @@ +#!/usr/bin/env python3 +"""Track 5.1 - the attachment queue under overload. + +Hypothesis: raising worker concurrency increases throughput and shortens +time-to-ready up to a point, after which CPU contention (Pillow resizing) and +memory set the ceiling; the durable queue lets the API keep accepting (202) and +bounds in-flight work, then drains and recovers afterwards. + +Method (reproducible): + * for each (concurrency, image size) pair, recreate the worker with a fixed + WORKER_CONCURRENCY, submit a burst of uploads, and poll every attachment + until ready; + * record accepted/errors, throughput, p50/p95 time-to-ready, queue depth over + time (backpressure) and peak worker RSS (docker stats); + * after the largest burst, wait for the queue to drain and confirm a fresh + upload still becomes ready quickly (recovery). + +Output: a JSON document plus a readable table. Run from the repository root. +""" +from __future__ import annotations + +import argparse +import base64 +import concurrent.futures +import json +import os +import statistics +import struct +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +import zlib + +BASE = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") +RESULTS = [] + + +def _chunk(tag, data): + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + + +def make_png(width, height): + raw = bytearray() + pixels = os.urandom(width * height * 3) + stride = width * 3 + for row in range(height): + raw.append(0) + raw += pixels[row * stride : (row + 1) * stride] + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + _chunk(b"IHDR", ihdr) + _chunk(b"IDAT", zlib.compress(bytes(raw), 0)) + _chunk(b"IEND", b"") + + +def http(method, path, token=None, body=None, raw=None, content_type=None): + headers = {"Accept": "application/json"} + data = raw + if body is not None: + data = json.dumps(body).encode() + headers["Content-Type"] = "application/json" + if content_type: + headers["Content-Type"] = content_type + if token: + headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(request, timeout=30) as response: + return response.status, response.read() + except urllib.error.HTTPError as error: + return error.code, error.read() + except (urllib.error.URLError, TimeoutError, OSError): + return 0, b"" + + +def login(): + username = "q_" + uuid.uuid4().hex[:8] + password = "T3st-" + uuid.uuid4().hex + http("POST", "/api/v1/users", body={"username": username, "display_name": "Q", "password": password}) + credential = base64.b64encode(f"{username}:{password}".encode()).decode() + request = urllib.request.Request( + BASE + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential, "Accept": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=20) as response: + token = json.loads(response.read())["token"] + user_id = json.loads(http("GET", "/api/v1/users/me", token=token)[1])["id"] + chat = json.loads(http("POST", "/api/v1/chats", token=token, body={"title": "q", "member_ids": [user_id]})[1]) + return token, chat["id"] + + +def worker_mem_mb(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}}|{{.MemUsage}}"], + capture_output=True, text=True).stdout + for line in out.splitlines(): + name, _, usage = line.partition("|") + if "worker" not in name: + continue + value = usage.split("/")[0].strip() + number = "".join(ch for ch in value if ch.isdigit() or ch == ".") + if value.endswith("GiB"): + return float(number) * 1024 + if value.endswith("MiB"): + return float(number) + return 0.0 + + +def set_worker_concurrency(value): + env = dict(os.environ, WORKER_CONCURRENCY=str(value)) + subprocess.run(["docker", "compose", "up", "-d", "--force-recreate", "worker"], + env=env, check=True, capture_output=True, text=True) + time.sleep(3) + + +def percentile(values, pct): + if not values: + return 0.0 + ordered = sorted(values) + index = min(len(ordered) - 1, int(round((pct / 100.0) * (len(ordered) - 1)))) + return ordered[index] + + +def run_case(concurrency, size_name, width, height, count, submit_workers=8): + set_worker_concurrency(concurrency) + token, chat_id = login() + payload = make_png(width, height) + + pending = {} + codes = {} + lock = threading.Lock() + + def submit(_): + status, body = http("POST", f"/api/v1/chats/{chat_id}/attachments?filename=q.png", token=token, + raw=payload, content_type="image/png") + with lock: + codes[status] = codes.get(status, 0) + 1 + if status != 202: + return + attachment_id = json.loads(body)["id"] + with lock: + pending[attachment_id] = time.time() + + queue_samples = [] + stop = threading.Event() + peak = {"value": worker_mem_mb()} + + def sampler(): + while not stop.is_set(): + with lock: + depth = len(pending) + queue_samples.append(depth) + peak["value"] = max(peak["value"], worker_mem_mb()) + time.sleep(0.2) + + thread = threading.Thread(target=sampler, daemon=True) + thread.start() + submit_start = time.time() + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, min(submit_workers, count))) as pool: + list(pool.map(submit, range(count))) + submit_done = time.time() + latencies = [] + deadline = time.time() + 300 + while pending and time.time() < deadline: + with lock: + ids = list(pending.keys()) + for attachment_id in ids: + status, body = http("GET", f"/api/v1/attachments/{attachment_id}", token=token) + if status != 200: + continue + state = json.loads(body)["status"] + if state in {"ready", "failed"}: + with lock: + started = pending.pop(attachment_id, None) + if started is not None: + latencies.append((time.time() - started, state)) + time.sleep(0.2) + stop.set() + thread.join(timeout=2) + + ready = [lat for lat, state in latencies if state == "ready"] + failed = [lat for lat, state in latencies if state == "failed"] + result = { + "concurrency": concurrency, + "size": size_name, + "bytes": len(payload), + "submitted": count, + "accepted": codes.get(202, 0), + "errors": sum(v for k, v in codes.items() if k != 202), + "codes": codes, + "ready": len(ready), + "failed": len(failed), + "submit_seconds": round(submit_done - submit_start, 2), + "drain_seconds": round(time.time() - submit_done, 2), + "throughput_per_s": round(len(ready) / max(0.001, time.time() - submit_start), 2), + "p50_ready_s": round(statistics.median(ready), 2) if ready else None, + "p95_ready_s": round(percentile(ready, 95), 2) if ready else None, + "max_queue_depth": max(queue_samples) if queue_samples else 0, + "worker_peak_mem_mb": round(peak["value"], 1), + } + RESULTS.append(result) + print(json.dumps(result, ensure_ascii=False), flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--count", type=int, default=12) + parser.add_argument("--output", default="evidence/lab5-overload.json") + args = parser.parse_args() + + os.makedirs(os.path.dirname(args.output), exist_ok=True) + plan = [ + (1, "small", 640, 480, 8), + (2, "small", 640, 480, 8), + (4, "small", 640, 480, 8), + (1, "large", 2200, 1500, 4), + (2, "large", 2200, 1500, 4), + (4, "large", 2200, 1500, 4), + ] + for concurrency, size_name, width, height, workers in plan: + run_case(concurrency, size_name, width, height, args.count, submit_workers=workers) + + # Recovery: after the heaviest cases the queue must drain and a fresh upload + # must still be served promptly. + set_worker_concurrency(2) + token, chat_id = login() + fresh = run_case(2, "recovery", 800, 600, 1) + drained = fresh["ready"] == 1 and fresh["p95_ready_s"] is not None and fresh["p95_ready_s"] < 30 + + with open(args.output, "w", encoding="utf-8") as handle: + json.dump({"results": RESULTS, "recovered": drained}, handle, ensure_ascii=False, indent=2) + + print("\nconcurrency size bytes accepted ready p50_s p95_s maxq peakRSS_MB") + for r in RESULTS: + print(f"{r['concurrency']:>11} {r['size']:<7} {r['bytes']:>7} {r['accepted']:>8} {r['ready']:>5} " + f"{str(r['p50_ready_s']):>5} {str(r['p95_ready_s']):>5} {r['max_queue_depth']:>5} {r['worker_peak_mem_mb']:>10}") + print(f"\nrecovery after overload: {'OK' if drained else 'FAILED'}") + print(json.dumps({"output": args.output, "recovered": drained})) + + +if __name__ == "__main__": + main() diff --git a/scripts/check-revocation.py b/scripts/check-revocation.py new file mode 100644 index 00000000..f0f157a4 --- /dev/null +++ b/scripts/check-revocation.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Regression check: logout must not be bypassed while Valkey is unavailable. + +Scenario (the one that used to fail): + 1. login and warm any session cache; + 2. pause Valkey; + 3. logout -> 204 (session row removed from PostgreSQL); + 4. resume Valkey; + 5. after the cache circuit breaker window, the old token must still be 401 on + every replica, and it must not be able to send a message. + +Requires the docker CLI and a running lab3 stack. +""" +import argparse +import os +import subprocess +import sys +import time +import uuid + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "tests")) +from client import Client # noqa: E402 + + +def _docker(*args: str) -> None: + subprocess.run(["docker", *args], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://localhost:8080")) + parser.add_argument("--valkey-container", default=os.environ.get("VALKEY_CONTAINER", "messenger-lab3-valkey-1")) + parser.add_argument("--wait", type=float, default=6.0, help="seconds to wait after resuming Valkey") + args = parser.parse_args() + + c = Client(args.base_url) + alice, bob, eve, chat = c.bootstrap() + + # Warm the cache on both replicas. + for _ in range(6): + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + c.message(alice, chat, "warm-up before revocation") + + print("Pausing Valkey (%s) ..." % args.valkey_container) + _docker("pause", args.valkey_container) + try: + logout = c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=alice) + assert logout["status"] == 204, logout["status"] + print("Logout returned 204 while the cache was unavailable.") + finally: + _docker("unpause", args.valkey_container) + print("Valkey resumed; waiting %.1fs for the cache circuit breaker ..." % args.wait) + time.sleep(args.wait) + + seen = set() + for _ in range(12): + response = c.call("GET", "/api/v1/users/me", 401, actor=alice, schema="Error") + seen.add(response["headers"].get("X-Instance-Id")) + assert len(seen) >= 1, "no replica observed" + + blocked = c.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 401, + actor=alice, + body={"text": "must not be written"}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + schema="Error", + ) + assert blocked["status"] == 401, blocked["status"] + + print( + "PASS: revoked session stays 401 through %d replica(s) and cannot post after a " + "failed cache invalidation." % len(seen) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/gen-secrets.sh b/scripts/gen-secrets.sh new file mode 100755 index 00000000..c4198d50 --- /dev/null +++ b/scripts/gen-secrets.sh @@ -0,0 +1,52 @@ +#!/bin/sh +set -eu +# Generate local (gitignored) compose secrets with random credentials. +# Writes database, cache and S3 credentials plus the SeaweedFS S3 config that +# declares the limited "messenger-api" identity (separate from root). +cd "$(dirname "$0")/.." +mkdir -p secrets + +BUCKET="${S3_BUCKET:-messenger-attachments}" + +rand() { head -c 24 /dev/urandom | od -An -tx1 | tr -d ' \n'; } + +PASSWORD=$(rand) +ROOT_KEY="root$(rand)" +ROOT_SECRET=$(rand) +API_KEY="api$(rand)" +API_SECRET=$(rand) +BROKER_PASSWORD=$(rand) + +printf '%s' "$PASSWORD" > secrets/database_password +printf '%s' "postgresql://messenger:${PASSWORD}@postgres:5432/messenger" > secrets/database_url +printf '%s' "redis://valkey:6379/0" > secrets/cache_url + +printf '%s' "$BROKER_PASSWORD" > secrets/broker_password +printf '%s' "amqp://messenger:${BROKER_PASSWORD}@broker:5672/%2F" > secrets/broker_url + +printf '%s' "$ROOT_KEY" > secrets/s3_root_access_key_id +printf '%s' "$ROOT_SECRET" > secrets/s3_root_secret_access_key +printf '%s' "$API_KEY" > secrets/s3_access_key_id +printf '%s' "$API_SECRET" > secrets/s3_secret_access_key + +# The API/worker identity may only Read/Write/List/Delete THIS bucket; root +# (Admin) is used solely by the one-shot s3-init bootstrap. +cat > secrets/s3_config.json </dev/null || true +echo "Wrote ./secrets/{database_password,database_url,cache_url,broker_*,s3_*,s3_config.json} (not committed)." diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..ff5c7e9b --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +# Restart the API replicas, the balancer, the worker and ALL state stores used +# by this stack (PostgreSQL, Valkey, S3, RabbitMQ). Volumes are preserved; +# schema, bucket, queues and data are never dropped or re-seeded. Used by: +# make test-persistence ACTION_SCRIPT=scripts/restart.sh +cd "$(dirname "$0")/.." +docker compose restart api lb worker postgres valkey s3 broker +exit 0 diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100755 index 00000000..42b6544e --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +# Restore the API service to the desired replica count after a failover test. +# Stopped replicas are started again; volumes and data are untouched. +cd "$(dirname "$0")/.." +docker compose up -d --no-recreate api +exit 0 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..9bbfc91b --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,38 @@ +#!/bin/sh +set -eu +# Sequential (rolling) replacement of the API replicas under load. +# +# The image is rebuilt once, then each running replica is stopped (the balancer +# drains it and keeps serving from the peers) and recreated from the new image. +# Before the next replica is touched we wait until every running replica is +# healthy, so the balancer always has somewhere to send traffic. +cd "$(dirname "$0")/.." + +N="${API_REPLICAS:-2}" + +wait_healthy() { + for _ in $(seq 1 90); do + total=$(docker compose ps -q api | wc -l) + healthy=$(docker ps --filter label=com.docker.compose.service=api --filter health=healthy --format '{{.ID}}' | wc -l) + if [ "$total" -ge "$N" ] && [ "$healthy" -ge "$total" ]; then + return 0 + fi + sleep 1 + done +} + +echo "Building new API image..." +docker compose build api + +# Wait for the current generation to be healthy before starting. +wait_healthy + +for id in $(docker compose ps -q api); do + echo "Replacing replica $id ..." + docker stop -t 10 "$id" >/dev/null + docker rm "$id" >/dev/null + docker compose up -d --no-recreate api >/dev/null + wait_healthy +done + +echo "Rolling update complete; all replicas healthy." diff --git a/scripts/start-worker.sh b/scripts/start-worker.sh new file mode 100755 index 00000000..ac2f01d6 --- /dev/null +++ b/scripts/start-worker.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +# Start the worker again after stop-worker.sh. +cd "$(dirname "$0")/.." +docker compose start worker +exit 0 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..2547395a --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,26 @@ +#!/bin/sh +set -eu +# Stop exactly ONE API replica and leave it stopped. +# +# TARGET_INSTANCE_ID is the X-Instance-Id observed by the test, which equals the +# replica's container hostname (Docker's short container id). We map it to the +# running api container and stop it with a short grace period so the hook +# returns well within the 10 second budget. PostgreSQL, Valkey and the balancer +# are never touched. +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." + +target="" +for id in $(docker ps --filter "label=com.docker.compose.service=api" --format '{{.ID}}'); do + case "$id" in + "$TARGET_INSTANCE_ID"*) target="$id" ;; + esac +done + +if [ -z "$target" ]; then + echo "no running API replica matches X-Instance-Id=$TARGET_INSTANCE_ID" >&2 + exit 1 +fi + +docker stop -t 5 "$target" >/dev/null +exit 0 diff --git a/scripts/stop-worker.sh b/scripts/stop-worker.sh new file mode 100755 index 00000000..c6cda6cc --- /dev/null +++ b/scripts/stop-worker.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu +# Stop the worker (leave it stopped). Used to demonstrate that uploads are +# accepted (202) while no worker is running, and resume when it returns. +cd "$(dirname "$0")/.." +docker compose stop worker +exit 0 diff --git a/secrets.example/README.md b/secrets.example/README.md new file mode 100644 index 00000000..23b97172 --- /dev/null +++ b/secrets.example/README.md @@ -0,0 +1,20 @@ +# Example only. Copy the whole directory to ./secrets/ (gitignored) and put +# real local values in. Never commit real credentials. +# +# mkdir -p secrets && cp secrets.example/* secrets/ +# # then edit the values below +# +# Or generate everything at once with random credentials: +# sh scripts/gen-secrets.sh +# +# Files expected by compose.yaml: +# database_password postgres password +# database_url postgresql://messenger:@postgres:5432/messenger +# cache_url redis://valkey:6379/0 +# broker_password RabbitMQ default user password +# broker_url amqp://messenger:@broker:5672/%2F +# s3_root_access_key_id SeaweedFS root access key (bootstrap only) +# s3_root_secret_access_key SeaweedFS root secret +# s3_access_key_id limited API/worker access key +# s3_secret_access_key limited API/worker secret +# s3_config.json SeaweedFS s3 config with both identities diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..19d86e09 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""MTUSI Messenger lab3 application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..4eafc14c --- /dev/null +++ b/src/app.py @@ -0,0 +1,585 @@ +"""FastAPI application implementing the lab4 HTTP contract. + +Canonical contract: ``contracts/openapi.json`` and ``contracts/README.md``. +PostgreSQL is the source of truth for metadata; bytes live in a private S3 +bucket. Valkey is a best-effort session cache. Every response carries a +non-empty ``X-Request-Id`` and ``X-Instance-Id``. Messages are idempotent per +``Idempotency-Key`` and may reference attachments of the same chat. +""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import logging +import re +import tempfile +import uuid +from contextlib import asynccontextmanager +from typing import Dict, List, Optional + +from fastapi import Depends, FastAPI, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, StreamingResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from psycopg import InterfaceError, OperationalError +from psycopg_pool import PoolTimeout +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +from . import imaging, invalidation, outbox, storage +from . import repositories as repo +from . import security, sessions, state +from .config import settings +from .db import pool +from .errors import ApiError +from .instance import INSTANCE_ID +from .pagination import bound_cursor, encode_cursor + +logger = logging.getLogger("messenger") + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +_BEARER_RE = re.compile(r"^Bearer\s+(\S+)$", re.IGNORECASE) +_IDEMPOTENCY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + +_STATUS_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 405: "invalid_request", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", +} + +@asynccontextmanager +async def lifespan(_: FastAPI): + invalidation.start_worker() + outbox.start_worker() + try: + yield + finally: + outbox.stop_worker() + invalidation.stop_worker() + + +app = FastAPI( + title="MTUSI Messenger — лабораторная 5", + version="1.5.0", + docs_url=None, + redoc_url=None, + openapi_url=None, + lifespan=lifespan, +) + + +# --------------------------------------------------------------------------- # +# Error envelope +# --------------------------------------------------------------------------- # +def _request_id(request: Request) -> str: + return getattr(request.state, "request_id", None) or uuid.uuid4().hex + + +def error_response( + request: Request, status: int, code: str, message: str, challenge: Optional[str] = None +) -> JSONResponse: + rid = _request_id(request) + headers = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID} + if challenge: + headers["WWW-Authenticate"] = challenge + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers=headers, + ) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = rid + try: + response = await call_next(request) + except Exception as exc: # noqa: BLE001 - never leak internals + logging.getLogger("messenger").error("unhandled error: %s", type(exc).__name__) + logging.getLogger("messenger").debug("traceback", exc_info=exc) + response = error_response(request, 500, "internal_error", "Внутренняя ошибка") + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse: + challenge = exc.challenge or (BEARER_CHALLENGE if exc.status == 401 else None) + return error_response(request, exc.status, exc.code, exc.message, challenge) + + +@app.exception_handler(RequestValidationError) +async def handle_validation(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Некорректный запрос") + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse: + code = _STATUS_CODES.get(exc.status_code, "invalid_request") + challenge = BEARER_CHALLENGE if exc.status_code == 401 else None + return error_response(request, exc.status_code, code, "Ошибка запроса", challenge) + + +def _handle_db_error(request: Request, exc: Exception) -> JSONResponse: + logger.warning("database unavailable: %s", type(exc).__name__) + return error_response(request, 503, "unavailable", "База данных недоступна") + + +app.add_exception_handler(OperationalError, _handle_db_error) +app.add_exception_handler(InterfaceError, _handle_db_error) +app.add_exception_handler(PoolTimeout, _handle_db_error) + + +# --------------------------------------------------------------------------- # +# Request schemas +# --------------------------------------------------------------------------- # +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def _not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("display_name must not be blank") + return value + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: List[uuid.UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def _not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title must not be blank") + return value + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + attachment_ids: List[uuid.UUID] = Field(default_factory=list, max_length=10) + + @field_validator("attachment_ids") + @classmethod + def _unique(cls, value: List[uuid.UUID]) -> List[uuid.UUID]: + if len(set(map(str, value))) != len(value): + raise ValueError("duplicate attachment_ids") + return value + + @model_validator(mode="after") + def _non_empty(self) -> "CreateMessage": + if not self.text.strip() and not self.attachment_ids: + raise ValueError("message must have text or attachments") + return self + + +# --------------------------------------------------------------------------- # +# Dependencies and helpers +# --------------------------------------------------------------------------- # +def get_conn(): + with pool().connection() as conn: + yield conn + + +def _canonical_uuid(value: str) -> str: + try: + return str(uuid.UUID(value)) + except (ValueError, AttributeError, TypeError): + raise ApiError(400, "invalid_request", "Некорректный идентификатор") + + +def _parse_basic(request: Request) -> tuple[str, str]: + header = request.headers.get("Authorization", "").strip() + if not header.lower().startswith("basic "): + raise ApiError(401, "unauthorized", "Требуется Basic-аутентификация", challenge=BASIC_CHALLENGE) + encoded = header[6:].strip() + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise ApiError(401, "unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE) + if ":" not in decoded: + raise ApiError(401, "unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE) + username, password = decoded.split(":", 1) + return username, password + + +def _bearer_token(request: Request) -> str: + header = request.headers.get("Authorization", "").strip() + match = _BEARER_RE.match(header) + if not match: + raise ApiError(401, "unauthorized", "Требуется Bearer-токен", challenge=BEARER_CHALLENGE) + return match.group(1) + + +def _idempotency_key(request: Request) -> str: + value = (request.headers.get("Idempotency-Key") or "").strip() + if not value or not _IDEMPOTENCY_RE.match(value): + raise ApiError(400, "invalid_request", "Требуется корректный Idempotency-Key") + return value + + +def current_user(request: Request, conn=Depends(get_conn)) -> Dict: + token = _bearer_token(request) + user_id = sessions.resolve(conn, token) + if user_id is None: + raise ApiError(401, "unauthorized", "Сессия недействительна", challenge=BEARER_CHALLENGE) + row = repo.get_user_by_id(conn, user_id) + if row is None: + raise ApiError(401, "unauthorized", "Сессия недействительна", challenge=BEARER_CHALLENGE) + return row + + +def _require_chat(conn, chat_id: str, user_id: str) -> str: + canonical = _canonical_uuid(chat_id) + if repo.get_chat_for(conn, canonical, user_id) is None: + raise ApiError(404, "not_found", "Чат недоступен") + return canonical + + +def _require_chat_standalone(chat_id: str, user_id: str) -> str: + """Chat-membership check on its own connection (used by the async upload).""" + with pool().connection() as conn: + return _require_chat(conn, chat_id, user_id) + + +def _validate_filename(name: str) -> str: + if not name or len(name) > 255: + raise ApiError(400, "invalid_request", "Недопустимое имя файла") + if any(ch in name for ch in ("/", "\\", "\x00")) or any( + ord(ch) < 0x20 or ord(ch) == 0x7F for ch in name + ): + raise ApiError(400, "invalid_request", "Недопустимое имя файла") + return name + + +def _validate_attachments(conn, chat_id: str, attachment_ids: List[str]) -> List[str]: + """Referenced attachments must belong to THIS chat and not be failed. + + queued/processing/ready are all linkable (lab5); a failed attachment is a + 409 and an unknown/foreign one is a 404. + """ + if not attachment_ids: + return [] + rows = repo.attachments_in_chat(conn, chat_id, attachment_ids) + found = {str(row["id"]): row["status"] for row in rows} + if len(found) != len(set(attachment_ids)): + raise ApiError(404, "not_found", "Вложение недоступно") + for attachment_id in attachment_ids: + if found.get(attachment_id) == "failed": + raise ApiError(409, "conflict", "Вложение не готово") + return attachment_ids + + +def _store_upload( + chat_id: str, owner_id: str, filename: str, fileobj, size_bytes: int, sha256: str +) -> Dict: + """Validate the image, persist the original to staging and record the job. + + Runs in a worker thread. Ordering (see contracts/attachment-states.md): + validate from bytes -> durable staging PUT -> one transaction writing the + attachment row (status=queued) AND the outbox job row -> 202. Only after the + transaction commits is the API allowed to answer 202, so an API restart + after the response never loses the accepted work. + """ + content_type, _width, _height = imaging.inspect(fileobj) + attachment_id = str(uuid.uuid4()) + job_id = str(uuid.uuid4()) + staging_key = f"staging/{attachment_id}" + fileobj.seek(0) + storage.put_object(staging_key, fileobj, content_type) + try: + with pool().connection() as conn: + now = repo.utcnow() + row = repo.insert_attachment( + conn=conn, + attachment_id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + content_type=content_type, + size_bytes=size_bytes, + sha256=sha256, + staging_key=staging_key, + status="queued", + pipeline_version=settings.pipeline_version, + error_code=None, + created_at=now, + updated_at=now, + ) + repo.create_job(conn, job_id, attachment_id, settings.pipeline_version, now) + return repo.attachment_json(row) + except Exception: + if storage.delete_object(staging_key): + logger.warning("compensated orphan staging object %s", staging_key) + raise + + +# --------------------------------------------------------------------------- # +# Health +# --------------------------------------------------------------------------- # +@app.get("/health/live") +def liveness() -> dict: + return {"status": "ok"} + + +@app.get("/health/ready") +def readiness() -> dict: + if state.is_shutting_down(): + raise ApiError(503, "unavailable", "Реплика останавливается") + try: + with pool().connection(timeout=2) as conn: + conn.execute("select 1") + except (OperationalError, InterfaceError, PoolTimeout): + raise ApiError(503, "unavailable", "База данных недоступна") + if not storage.ready(): + raise ApiError(503, "unavailable", "Хранилище недоступно") + return {"status": "ok"} + + +# --------------------------------------------------------------------------- # +# Users and sessions +# --------------------------------------------------------------------------- # +@app.post("/api/v1/users", status_code=201) +def register_user(payload: CreateUser, conn=Depends(get_conn)) -> dict: + password_hash = security.hash_password(payload.password) + row = repo.create_user(conn, payload.username, payload.display_name, password_hash) + return repo.user_json(row) + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session(request: Request, conn=Depends(get_conn)) -> dict: + username, password = _parse_basic(request) + row = repo.get_user_with_hash_by_username(conn, username) + if row is None or not security.verify_password(row["password_hash"], password): + raise ApiError(401, "unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE) + token, expires_at = sessions.create(conn, str(row["id"])) + return { + "token": token, + "token_type": "Bearer", + "expires_at": repo.rfc3339(expires_at), + "user": repo.user_json(row), + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session( + request: Request, + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> Response: + sessions.revoke(conn, _bearer_token(request)) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def current_user_endpoint(user: Dict = Depends(current_user)) -> dict: + return repo.user_json(user) + + +# --------------------------------------------------------------------------- # +# Chats +# --------------------------------------------------------------------------- # +@app.post("/api/v1/chats", status_code=201) +def create_chat(payload: CreateChat, user: Dict = Depends(current_user), conn=Depends(get_conn)) -> dict: + members = [str(member) for member in payload.member_ids] + if len(set(members)) != len(members): + raise ApiError(400, "invalid_request", "Дубликаты участников") + if str(user["id"]) not in members: + raise ApiError(400, "invalid_request", "Вызывающий должен входить в участников") + if repo.count_users(conn, members) != len(set(members)): + raise ApiError(400, "invalid_request", "Неизвестный участник") + row = repo.create_chat(conn, payload.title, members) + return repo.chat_json(row) + + +@app.get("/api/v1/chats") +def list_chats( + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> dict: + key = bound_cursor(cursor, "chats") + rows = repo.list_chats(conn, str(user["id"]), limit + 1, key) + page = rows[:limit] + next_cursor = None + if len(rows) > limit: + next_cursor = encode_cursor("chats", page[-1]["created_at"], str(page[-1]["id"])) + return {"items": [repo.chat_json(row) for row in page], "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat(chat_id: str, user: Dict = Depends(current_user), conn=Depends(get_conn)) -> dict: + canonical = _require_chat(conn, chat_id, str(user["id"])) + row = repo.get_chat_for(conn, canonical, str(user["id"])) + return repo.chat_json(row) + + +# --------------------------------------------------------------------------- # +# Messages +# --------------------------------------------------------------------------- # +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + payload: CreateMessage, + request: Request, + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> dict: + canonical = _require_chat(conn, chat_id, str(user["id"])) + key = _idempotency_key(request) + attachment_ids = _validate_attachments( + conn, canonical, [str(item) for item in payload.attachment_ids] + ) + scope = f"POST /api/v1/chats/{canonical}/messages" + return repo.create_message_idempotent( + conn, + canonical, + str(user["id"]), + payload.text, + attachment_ids, + scope, + key, + settings.idempotency_ttl_seconds, + ) + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: str, + limit: int = Query(default=50, ge=1, le=100), + cursor: Optional[str] = Query(default=None), + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> dict: + canonical = _require_chat(conn, chat_id, str(user["id"])) + key = bound_cursor(cursor, canonical) + rows = repo.list_messages(conn, canonical, limit + 1, key) + page = rows[:limit] + next_cursor = None + if len(rows) > limit: + next_cursor = encode_cursor(canonical, page[-1]["created_at"], str(page[-1]["id"])) + return {"items": [repo.message_json(row) for row in page], "next_cursor": next_cursor} + + +# --------------------------------------------------------------------------- # +# Attachments (lab5): synchronous accept into staging, async processing +# --------------------------------------------------------------------------- # +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(..., min_length=1, max_length=255), + user: Dict = Depends(current_user), +) -> dict: + canonical = await run_in_threadpool(_require_chat_standalone, chat_id, str(user["id"])) + safe_name = _validate_filename(filename) + + content_length = request.headers.get("Content-Length", "") + if content_length.isdigit() and int(content_length) > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + + # Stream the raw bytes to a bounded spill-to-disk file, hashing as we go, so + # a single upload never holds the whole body in RAM. + spool = tempfile.SpooledTemporaryFile(max_size=1024 * 1024) + size = 0 + digest = hashlib.sha256() + try: + async for chunk in request.stream(): + if not chunk: + continue + size += len(chunk) + if size > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Файл слишком большой") + digest.update(chunk) + spool.write(chunk) + if size == 0: + raise ApiError(415, "unsupported_media_type", "Пустой файл") + spool.seek(0) + result = await run_in_threadpool( + _store_upload, + canonical, + str(user["id"]), + safe_name, + spool, + size, + digest.hexdigest(), + ) + # Best-effort immediate publish for latency; the outbox guarantees it. + outbox.publish_now() + return result + finally: + spool.close() + + +@app.get("/api/v1/attachments/{attachment_id}") +def get_attachment( + attachment_id: str, user: Dict = Depends(current_user), conn=Depends(get_conn) +) -> dict: + canonical = _canonical_uuid(attachment_id) + row = repo.get_attachment_for(conn, canonical, str(user["id"])) + if row is None: + raise ApiError(404, "not_found", "Вложение недоступно") + variants = ( + repo.list_variants(conn, canonical, int(row["pipeline_version"])) + if row["status"] == "ready" + else [] + ) + return repo.attachment_json(row, variants) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +def download_attachment( + attachment_id: str, + variant: str = Query(default="original"), + user: Dict = Depends(current_user), + conn=Depends(get_conn), +) -> StreamingResponse: + canonical = _canonical_uuid(attachment_id) + row = repo.get_attachment_for(conn, canonical, str(user["id"])) + if row is None: + raise ApiError(404, "not_found", "Вложение недоступно") + if row["status"] != "ready": + # queued/processing/failed: bytes are not available yet + raise ApiError(409, "conflict", "Вложение не готово") + + if variant == "original": + key = row["s3_key"] + content_type = row["content_type"] + else: + variant_row = repo.get_variant(conn, canonical, int(row["pipeline_version"]), variant) + if variant_row is None: + raise ApiError(404, "not_found", "Вариант не найден") + key = variant_row["s3_key"] + content_type = variant_row["content_type"] + + obj = storage.open_object(key) + body = obj["Body"] + + def stream(): + try: + for chunk in body.iter_chunks(chunk_size=64 * 1024): + yield chunk + finally: + body.close() + + return StreamingResponse(stream(), media_type=content_type) diff --git a/src/broker.py b/src/broker.py new file mode 100644 index 00000000..f56ab6d9 --- /dev/null +++ b/src/broker.py @@ -0,0 +1,48 @@ +"""RabbitMQ broker access: durable queue, publisher confirms, manual consumer ack. + +The queue is declared durable and messages are published persistent. Publisher +confirms are requested so the outbox publisher only marks a job published once +the broker has taken responsibility for it. Consumers ack manually after the +durable side effect, which is a different boundary (see attachment-states.md). +""" +from __future__ import annotations + +import json +import logging + +import pika + +from .config import settings + +logger = logging.getLogger("messenger.broker") + + +def connect() -> pika.BlockingConnection: + params = pika.URLParameters(settings.broker_url) + params.heartbeat = 30 + params.blocked_connection_timeout = 5 + params.socket_timeout = 10 + return pika.BlockingConnection(params) + + +def declare(channel) -> None: + channel.queue_declare(queue=settings.broker_queue, durable=True) + + +def publish(job: dict) -> None: + """Publish one job with publisher confirms; raises on broker failure.""" + connection = connect() + try: + channel = connection.channel() + declare(channel) + channel.confirm_delivery() + channel.basic_publish( + exchange="", + routing_key=settings.broker_queue, + body=json.dumps(job, separators=(",", ":")).encode("utf-8"), + properties=pika.BasicProperties( + delivery_mode=2, content_type="application/json" + ), + ) + finally: + connection.close() diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..4e2b1a63 --- /dev/null +++ b/src/config.py @@ -0,0 +1,115 @@ +"""Runtime configuration from environment variables / secret files. + +Only logical interface names from ``.env.example`` are consumed; secret values +are read from files (``*_FILE``) so that no credential is baked into the image +or committed. The mapping is documented in REPORT.md. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +def _read_secret(file_key: str, value_key: str, default: str = "") -> str: + path = os.environ.get(file_key) + if path: + try: + return Path(path).read_text(encoding="utf-8").strip() + except OSError: + return default + return os.environ.get(value_key, default) + + +@dataclass(frozen=True) +class Settings: + app_host: str + app_port: int + app_env: str + session_ttl_seconds: int + session_store: str + session_cache_ttl_seconds: int + cache_invalidation_interval_seconds: float + idempotency_ttl_seconds: int + database_url: str + cache_url: str + db_pool_min: int + db_pool_max: int + db_pool_timeout: float + db_statement_timeout_ms: int + # S3 / attachment limits + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + # Broker / worker / jobs (lab5) + broker_url: str + broker_queue: str + worker_concurrency: int + job_max_attempts: int + job_timeout_seconds: int + staging_retention_seconds: int + outbox_interval_seconds: float + lease_recovery_interval_seconds: float + retry_backoff_base_seconds: float + retry_backoff_max_seconds: float + pipeline_version: int + thumbnail_size: int + crop_size: int + optimize_max_size: int + optimize_quality: int + + @property + def cache_enabled(self) -> bool: + return bool(self.cache_url) and self.session_store in {"valkey", "redis"} + + +def load_settings() -> Settings: + return Settings( + app_host=os.environ.get("APP_HOST", "0.0.0.0"), + app_port=int(os.environ.get("APP_PORT", "8080")), + app_env=os.environ.get("APP_ENV", "lab"), + session_ttl_seconds=int(os.environ.get("SESSION_TTL_SECONDS", "3600")), + session_store=os.environ.get("SESSION_STORE", "postgres").lower(), + session_cache_ttl_seconds=int(os.environ.get("SESSION_CACHE_TTL_SECONDS", "60")), + cache_invalidation_interval_seconds=float( + os.environ.get("CACHE_INVALIDATION_INTERVAL_SECONDS", "0.5") + ), + idempotency_ttl_seconds=int(os.environ.get("IDEMPOTENCY_TTL_SECONDS", "86400")), + database_url=_read_secret("DATABASE_URL_FILE", "DATABASE_URL"), + cache_url=_read_secret("CACHE_URL_FILE", "CACHE_URL"), + db_pool_min=int(os.environ.get("DB_POOL_MIN", "1")), + db_pool_max=int(os.environ.get("DB_POOL_MAX", "10")), + db_pool_timeout=float(os.environ.get("DB_POOL_TIMEOUT", "5")), + db_statement_timeout_ms=int(os.environ.get("DB_STATEMENT_TIMEOUT_MS", "5000")), + s3_endpoint=os.environ.get("S3_ENDPOINT", "http://s3:9000"), + s3_region=os.environ.get("S3_REGION", "us-east-1"), + s3_bucket=os.environ.get("S3_BUCKET", "messenger-attachments"), + s3_access_key_id=_read_secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID"), + s3_secret_access_key=_read_secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY"), + max_upload_bytes=int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024))), + max_image_pixels=int(os.environ.get("MAX_IMAGE_PIXELS", "40000000")), + broker_url=_read_secret("BROKER_URL_FILE", "BROKER_URL", "amqp://guest:guest@broker:5672/%2F"), + broker_queue=os.environ.get("BROKER_QUEUE", "attachment.process.v1"), + worker_concurrency=int(os.environ.get("WORKER_CONCURRENCY", "2")), + job_max_attempts=int(os.environ.get("JOB_MAX_ATTEMPTS", "5")), + job_timeout_seconds=int(os.environ.get("JOB_TIMEOUT_SECONDS", "60")), + staging_retention_seconds=int(os.environ.get("STAGING_RETENTION_SECONDS", "86400")), + outbox_interval_seconds=float(os.environ.get("OUTBOX_INTERVAL_SECONDS", "0.5")), + lease_recovery_interval_seconds=float( + os.environ.get("LEASE_RECOVERY_INTERVAL_SECONDS", "2") + ), + retry_backoff_base_seconds=float(os.environ.get("RETRY_BACKOFF_BASE_SECONDS", "0.5")), + retry_backoff_max_seconds=float(os.environ.get("RETRY_BACKOFF_MAX_SECONDS", "30")), + pipeline_version=int(os.environ.get("PIPELINE_VERSION", "1")), + thumbnail_size=int(os.environ.get("THUMBNAIL_SIZE", "256")), + crop_size=int(os.environ.get("CROP_SIZE", "128")), + optimize_max_size=int(os.environ.get("OPTIMIZE_MAX_SIZE", "1280")), + optimize_quality=int(os.environ.get("OPTIMIZE_QUALITY", "80")), + ) + + +settings = load_settings() diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..5353efc5 --- /dev/null +++ b/src/db.py @@ -0,0 +1,42 @@ +"""PostgreSQL access: a bounded connection pool with timeouts. + +All queries elsewhere are parameterized; this module only owns the pool. +A statement timeout and a bounded pool are part of the grade-4 requirements. +""" +from __future__ import annotations + +import logging +from typing import Optional + +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import settings + +logger = logging.getLogger("messenger.db") + +_pool: Optional[ConnectionPool] = None + + +def pool() -> ConnectionPool: + global _pool + if _pool is None: + _pool = ConnectionPool( + conninfo=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_pool_timeout, + kwargs={ + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.db_statement_timeout_ms}", + }, + open=True, + ) + return _pool + + +def close_pool() -> None: + global _pool + if _pool is not None: + _pool.close() + _pool = None diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..48f9a0f0 --- /dev/null +++ b/src/errors.py @@ -0,0 +1,20 @@ +"""Shared error type mapped to the contract error envelope.""" +from __future__ import annotations + +from typing import Optional + + +class ApiError(Exception): + def __init__( + self, + status: int, + code: str, + message: str, + challenge: Optional[str] = None, + ) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + # Optional WWW-Authenticate value (used by 401 responses). + self.challenge = challenge diff --git a/src/imaging.py b/src/imaging.py new file mode 100644 index 00000000..5e71032a --- /dev/null +++ b/src/imaging.py @@ -0,0 +1,106 @@ +"""Image validation and variant generation (lab5). + +The real format is decided by the bytes (Pillow), never by the client's +``Content-Type`` or by ``filename``. Pixel count is checked from the header +*before* any full decode, so a decompression bomb is rejected cheaply. + +Variant rules (contracts/README.md): + * thumbnail: JPEG fitted into 256x256, aspect kept, never upscaled, EXIF + orientation applied, alpha composited on white, floor dimensions, min 1 px; + * crop: JPEG centered square 128x128 (upscaling allowed for small sources); + * optimized: WebP fitted into 1280x1280, never upscaled, EXIF stripped. +""" +from __future__ import annotations + +import io +import logging + +from PIL import Image, ImageOps, UnidentifiedImageError + +from .config import settings +from .errors import ApiError + +logger = logging.getLogger("messenger.imaging") + +Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + +_CONTENT_TYPES = {"PNG": "image/png", "JPEG": "image/jpeg"} + + +class PermanentImageError(Exception): + """Invalid/unsupported image: retrying will not help.""" + + def __init__(self, code: str = "invalid_image") -> None: + super().__init__(code) + self.code = code + + +def inspect(fileobj) -> tuple[str, int, int]: + """Return ``(content_type, width, height)`` or raise ``ApiError`` (API path).""" + fileobj.seek(0) + try: + image = Image.open(fileobj) + fmt = image.format + width, height = image.size + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError): + raise ApiError(415, "unsupported_media_type", "Поддерживаются только PNG и JPEG") + + content_type = _CONTENT_TYPES.get(fmt) + if content_type is None: + raise ApiError(415, "unsupported_media_type", "Поддерживаются только PNG и JPEG") + if width * height > settings.max_image_pixels: + raise ApiError(413, "payload_too_large", "Слишком большое изображение") + try: + image.verify() + except Exception: # noqa: BLE001 + raise ApiError(415, "unsupported_media_type", "Повреждённое изображение") + return content_type, width, height + + +def _normalized(data: bytes) -> Image.Image: + """Open, apply EXIF orientation and composite alpha on white.""" + try: + image = Image.open(io.BytesIO(data)) + image.load() + image = ImageOps.exif_transpose(image) + except (UnidentifiedImageError, OSError, ValueError, Image.DecompressionBombError) as exc: + raise PermanentImageError("invalid_image") from exc + if image.width * image.height > settings.max_image_pixels: + raise PermanentImageError("too_many_pixels") + if image.mode in ("RGBA", "LA") or (image.mode == "P" and "transparency" in image.info): + rgba = image.convert("RGBA") + background = Image.new("RGB", rgba.size, (255, 255, 255)) + background.paste(rgba, mask=rgba.split()[-1]) + image = background + elif image.mode != "RGB": + image = image.convert("RGB") + return image + + +def _jpeg(image: Image.Image, quality: int = 85) -> bytes: + buffer = io.BytesIO() + image.save(buffer, format="JPEG", quality=quality) + return buffer.getvalue() + + +def make_thumbnail(data: bytes) -> tuple[bytes, int, int]: + image = _normalized(data) + thumb = ImageOps.contain(image, (settings.thumbnail_size, settings.thumbnail_size), Image.LANCZOS) + thumb = thumb.resize((max(1, thumb.width), max(1, thumb.height)), Image.LANCZOS) + return _jpeg(thumb), thumb.width, thumb.height + + +def make_crop(data: bytes) -> tuple[bytes, int, int]: + image = _normalized(data) + size = settings.crop_size + crop = ImageOps.fit(image, (size, size), method=Image.LANCZOS, centering=(0.5, 0.5)) + return _jpeg(crop, quality=85), crop.width, crop.height + + +def make_optimized(data: bytes) -> tuple[bytes, int, int]: + image = _normalized(data) + optimized = ImageOps.contain(image, (settings.optimize_max_size, settings.optimize_max_size), Image.LANCZOS) + buffer = io.BytesIO() + # No exif= argument -> metadata (incl. EXIF) is dropped. + optimized.save(buffer, format="WEBP", quality=settings.optimize_quality, method=4) + return buffer.getvalue(), optimized.width, optimized.height diff --git a/src/instance.py b/src/instance.py new file mode 100644 index 00000000..8411c30e --- /dev/null +++ b/src/instance.py @@ -0,0 +1,21 @@ +"""Stable identity of this API process, exposed as ``X-Instance-Id``. + +The value is fixed for the whole life of the process, so clients and the +balancer can tell replicas apart. Inside Docker the default container hostname +already is the (short) container id, which makes the value easy to map back to +a real container without exposing a routable address. +""" +from __future__ import annotations + +import os +import socket + + +def _resolve() -> str: + value = os.environ.get("INSTANCE_ID", "").strip() + if value: + return value + return socket.gethostname() or "api" + + +INSTANCE_ID = _resolve() diff --git a/src/invalidation.py b/src/invalidation.py new file mode 100644 index 00000000..aa534ac1 --- /dev/null +++ b/src/invalidation.py @@ -0,0 +1,76 @@ +"""Durable cache-invalidation drainer (lab3). + +Logout writes a pending invalidation to PostgreSQL in the same transaction as +the authoritative session delete. This tiny background thread (one per API +replica, not an external broker) removes the corresponding Valkey key and then +the queue row. Multiple replicas may process the same row; cache deletes are +idempotent, so this is safe and needs no distributed lock. + +The worker is deliberately not on the request path: it only runs when a cache +is configured, and it retries forever while Valkey is unreachable. That is what +lets a logout survive a Valkey outage: the session is already gone from +PostgreSQL, and the stale cache entry is evicted as soon as the cache returns. +""" +from __future__ import annotations + +import logging +import threading +from typing import Optional + +from . import repositories as repo +from . import sessions +from .config import settings +from .db import pool + +logger = logging.getLogger("messenger.invalidation") + +_BATCH = 200 +_stop = threading.Event() +_thread: Optional[threading.Thread] = None + + +def _drain_once() -> None: + with pool().connection() as conn: + pending = repo.list_pending_invalidations(conn, _BATCH) + if not pending: + return + + invalidated = [token_hash for token_hash in pending if sessions.invalidate_session_key(token_hash)] + if not invalidated: + # Cache still unreachable; keep the rows for the next attempt. + return + with pool().connection() as conn: + repo.delete_cache_invalidations(conn, invalidated) + logger.debug("drained %d cache invalidations", len(invalidated)) + + +def _loop() -> None: + while not _stop.is_set(): + try: + _drain_once() + except Exception as exc: # noqa: BLE001 - never kill the drainer + logger.warning("cache invalidation drain failed: %s", type(exc).__name__) + _stop.wait(settings.cache_invalidation_interval_seconds) + + +def start_worker() -> None: + global _thread + if not settings.cache_enabled: + return + if _thread is not None and _thread.is_alive(): + return + _stop.clear() + _thread = threading.Thread(target=_loop, name="cache-invalidation", daemon=True) + _thread.start() + logger.info( + "cache invalidation drainer started (interval=%.2fs)", + settings.cache_invalidation_interval_seconds, + ) + + +def stop_worker() -> None: + global _thread + _stop.set() + if _thread is not None: + _thread.join(timeout=2) + _thread = None diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..502c57ff --- /dev/null +++ b/src/main.py @@ -0,0 +1,60 @@ +"""Container entrypoint: run the HTTP server as PID 1 with graceful shutdown. + +Migrations are executed by the dedicated one-shot ``migrate`` compose service +(``python -m src.migrate``) before any replica starts, so replicas do not race +on the schema. + +On SIGTERM/SIGINT the handler marks the process as shutting down: readiness +degrades to 503 so the balancer drains this replica, then Uvicorn stops +accepting connections and waits up to ``timeout_graceful_shutdown`` for +in-flight requests to finish. +""" +from __future__ import annotations + +import logging +import os + +import uvicorn + +from . import state +from .config import settings +from .instance import INSTANCE_ID + + +def main() -> None: + logging.basicConfig( + level=os.environ.get("LOG_LEVEL", "INFO").upper(), + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + logging.getLogger("messenger").info( + "starting messenger api instance=%s env=%s port=%s sessions=%s cache=%s", + INSTANCE_ID, + settings.app_env, + settings.app_port, + settings.session_store, + "on" if settings.cache_enabled else "off", + ) + + config = uvicorn.Config( + "src.app:app", + host=settings.app_host, + port=settings.app_port, + log_level=os.environ.get("LOG_LEVEL", "info").lower(), + access_log=True, + timeout_graceful_shutdown=10, + ) + server = uvicorn.Server(config) + + def handle_exit(sig, frame=None): # noqa: ANN001 - signal handler + state.begin_shutdown() + logging.getLogger("messenger").info("received signal %s; draining", sig) + server.should_exit = True + + # Uvicorn installs this exact method as its signal handler; overriding the + # bound attribute keeps cross-platform behaviour while adding our flag. + server.handle_exit = handle_exit + server.run() + + +if __name__ == "__main__": + main() diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..76f17bb9 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,67 @@ +"""Versioned, idempotent migration runner. + +Applied versions are recorded in ``schema_migrations``; a transaction-scoped +advisory lock serializes concurrent callers. In lab3 migrations are executed by +a dedicated one-shot ``migrate`` service, not by every API replica, so schema +changes happen exactly once before any replica starts serving traffic. +""" +from __future__ import annotations + +import logging +import time +from pathlib import Path + +import psycopg +from psycopg_pool import PoolTimeout + +from .db import pool + +logger = logging.getLogger("messenger.migrate") + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" +LOCK_KEY = 776_2026 + + +def _apply_once() -> None: + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + with pool().connection() as conn: + conn.execute("select pg_advisory_xact_lock(%s)", (LOCK_KEY,)) + conn.execute( + """ + create table if not exists schema_migrations ( + version text primary key, + applied_at timestamptz not null default now() + ) + """ + ) + applied = { + row["version"] + for row in conn.execute("select version from schema_migrations").fetchall() + } + for path in files: + if path.name in applied: + continue + logger.info("applying migration %s", path.name) + conn.execute(path.read_text(encoding="utf-8")) + conn.execute( + "insert into schema_migrations (version) values (%s)", (path.name,) + ) + + +def apply_migrations(attempts: int = 30, delay: float = 1.0) -> None: + last: Exception | None = None + for attempt in range(1, attempts + 1): + try: + _apply_once() + logger.info("migrations up to date") + return + except (psycopg.OperationalError, PoolTimeout) as exc: + last = exc + logger.warning("database not ready (attempt %s/%s): %s", attempt, attempts, exc) + time.sleep(delay) + raise RuntimeError(f"database unavailable after {attempts} attempts: {last}") + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + apply_migrations() diff --git a/src/outbox.py b/src/outbox.py new file mode 100644 index 00000000..9971fbdf --- /dev/null +++ b/src/outbox.py @@ -0,0 +1,78 @@ +"""Transactional outbox publisher (runs inside the API process). + +The API writes an attachment AND its job row in one transaction. This tiny +background thread moves pending rows to RabbitMQ with publisher confirms and +only then marks ``published_at``. That closes the COMMIT -> publish gap: a crash +between the two leaves the row unpublished and it is retried after restart. +Transient processing errors are also scheduled here: the worker sets the row +back to ``pending`` with a future ``available_at`` (backoff + jitter) and the +publisher republishes it when due. +""" +from __future__ import annotations + +import logging +import threading +from typing import Optional + +from . import broker +from . import repositories as repo +from .config import settings +from .db import pool + +logger = logging.getLogger("messenger.outbox") + +_stop = threading.Event() +_thread: Optional[threading.Thread] = None + + +def _publish_once() -> int: + with pool().connection() as conn: + jobs = repo.list_outbox_jobs(conn, 100) + published = 0 + for job in jobs: + try: + broker.publish(repo.job_message(job)) + except Exception as exc: # noqa: BLE001 - broker may be down; retry later + logger.warning("broker publish failed (%s); will retry", type(exc).__name__) + break + with pool().connection() as conn: + repo.mark_job_published(conn, str(job["job_id"])) + published += 1 + if published: + logger.info("outbox published %d job(s)", published) + return published + + +def _loop() -> None: + while not _stop.is_set(): + try: + _publish_once() + except Exception as exc: # noqa: BLE001 - never kill the publisher + logger.warning("outbox drain failed: %s", type(exc).__name__) + _stop.wait(settings.outbox_interval_seconds) + + +def publish_now() -> None: + """Best-effort immediate publish (called right after a 202 for latency).""" + try: + _publish_once() + except Exception as exc: # noqa: BLE001 + logger.warning("immediate outbox publish failed: %s", type(exc).__name__) + + +def start_worker() -> None: + global _thread + if _thread is not None and _thread.is_alive(): + return + _stop.clear() + _thread = threading.Thread(target=_loop, name="outbox-publisher", daemon=True) + _thread.start() + logger.info("outbox publisher started (interval=%.2fs)", settings.outbox_interval_seconds) + + +def stop_worker() -> None: + global _thread + _stop.set() + if _thread is not None: + _thread.join(timeout=2) + _thread = None diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 00000000..c832d2ff --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,53 @@ +"""Opaque keyset cursors for ``(created_at, id)`` ascending pagination.""" +from __future__ import annotations + +import base64 +import binascii +import datetime as dt +import json +import uuid +from typing import Optional, Tuple + +from .errors import ApiError + +CursorKey = Tuple[dt.datetime, str] + + +def encode_cursor(resource_id: str, created_at: dt.datetime, item_id: str) -> str: + payload = json.dumps( + { + "r": resource_id, + "t": created_at.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ"), + "i": item_id, + }, + separators=(",", ":"), + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def decode_cursor(cursor: str) -> Tuple[str, CursorKey]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + data = json.loads(base64.urlsafe_b64decode(padded).decode("utf-8")) + if not isinstance(data, dict): + raise ValueError("cursor payload must be an object") + resource = data["r"] + item_id = data["i"] + if not isinstance(resource, str) or not isinstance(item_id, str): + raise ValueError("cursor fields must be strings") + created = dt.datetime.strptime(data["t"], "%Y-%m-%dT%H:%M:%S.%fZ").replace( + tzinfo=dt.timezone.utc + ) + return resource, (created, str(uuid.UUID(item_id))) + except (binascii.Error, ValueError, KeyError, TypeError, AttributeError, UnicodeDecodeError): + raise ApiError(400, "invalid_request", "Некорректный cursor") + + +def bound_cursor(cursor: str, expected_resource: str) -> Optional[CursorKey]: + """Validate resource binding and return the key, or None for empty cursor.""" + if cursor is None: + return None + resource, key = decode_cursor(cursor) + if resource != expected_resource: + raise ApiError(400, "invalid_request", "cursor относится к другому ресурсу") + return key diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..101ed3ce --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,82 @@ +"""Reconcile S3 objects and attachment metadata (lab4 grade-4 safety net). + +The synchronous upload spends a short window between the S3 PUT and the +PostgreSQL COMMIT. A crash there can leave an *orphan* object (in S3, no row); +the inline compensation in the API covers the ordinary failure path, and this +tool covers crash windows and any object created but never referenced. + + docker compose exec -T api python -m src.reconcile # dry-run + docker compose exec -T api python -m src.reconcile --apply # delete orphans + +Safety: + * dry-run by default; ``--apply`` is required to delete anything; + * only objects with no metadata row are candidates (referenced objects are + never touched); + * only objects older than ``--grace-seconds`` (default 3600) are deleted, so a + just-started upload is never mistaken for an orphan; + * objects present in metadata but missing in S3 are reported, never guessed. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json + +from . import repositories as repo +from . import storage +from .db import pool + + +def main() -> None: + parser = argparse.ArgumentParser(description="Reconcile S3 objects and attachment metadata") + parser.add_argument("--apply", action="store_true", help="actually delete orphan objects") + parser.add_argument("--grace-seconds", type=int, default=3600) + parser.add_argument("--limit", type=int, default=50, help="how many orphans to print") + args = parser.parse_args() + + with pool().connection() as conn: + rows = repo.list_all_attachments(conn) + variants = conn.execute("select s3_key from attachment_variants").fetchall() + known = {} + for row in rows: + for key in (row["staging_key"], row["s3_key"]): + if key: + known[str(key)] = str(row["id"]) + for row in variants: + known[str(row["s3_key"])] = "variant" + + cutoff = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=args.grace_seconds) + objects = list(storage.list_keys("")) + s3_keys = {item["Key"]: item.get("LastModified") for item in objects} + + missing = sorted(key for key in known if key not in s3_keys) + orphans = [] + for key, modified in s3_keys.items(): + if key in known: + continue + if modified is not None and modified > cutoff: + continue # too fresh: may be an in-flight upload + orphans.append(key) + + report = { + "db_attachments": len(known), + "s3_objects": len(s3_keys), + "orphans": len(orphans), + "missing": len(missing), + "apply": bool(args.apply), + "grace_seconds": args.grace_seconds, + } + print(json.dumps(report, ensure_ascii=False)) + + for key in sorted(orphans)[: args.limit]: + print("orphan:", key) + for key in missing[: args.limit]: + print("missing:", key, "attachment_id=" + known[key]) + + if args.apply and orphans: + deleted = sum(1 for key in orphans if storage.delete_object(key)) + print(json.dumps({"deleted_orphans": deleted})) + + +if __name__ == "__main__": + main() diff --git a/src/replay.py b/src/replay.py new file mode 100644 index 00000000..40ef60b4 --- /dev/null +++ b/src/replay.py @@ -0,0 +1,46 @@ +"""Explicit operator replay of failed attachment jobs (lab5 grade-4). + + docker compose exec -T api python -m src.replay # list failed + docker compose exec -T api python -m src.replay # replay one + +Replaying resets the job to ``pending`` and the attachment to ``queued``; the +outbox publisher republishes it. The idempotent effect key is +``(attachment_id, pipeline_version, variant)``, so replaying an already-succeeded +job can never duplicate variants. +""" +from __future__ import annotations + +import argparse +import json + +from . import repositories as repo +from .db import pool + + +def main() -> None: + parser = argparse.ArgumentParser(description="List or replay failed attachment jobs") + parser.add_argument("job_id", nargs="?", help="specific job id to replay") + args = parser.parse_args() + + with pool().connection() as conn: + if args.job_id is None: + failed = repo.list_failed_jobs(conn) + print(json.dumps( + [ + { + "job_id": str(row["job_id"]), + "attachment_id": str(row["attachment_id"]), + "attempt": int(row["attempt"]), + "error_code": row["last_error"], + } + for row in failed + ], + ensure_ascii=False, + )) + return + replayed = repo.replay_failed_jobs(conn, args.job_id) + print(json.dumps({"replayed": replayed})) + + +if __name__ == "__main__": + main() diff --git a/src/repositories.py b/src/repositories.py new file mode 100644 index 00000000..bdc00c79 --- /dev/null +++ b/src/repositories.py @@ -0,0 +1,773 @@ +"""Parameterized SQL for users, chats, members, messages, sessions and the +shared idempotency ledger. + +Every statement uses bound parameters (no string concatenation). Composite +writes happen inside one transaction so the caller gets all-or-nothing. +""" +from __future__ import annotations + +import datetime as dt +import hashlib +import json +import uuid +from typing import Dict, List, Optional, Sequence + +from psycopg.errors import UniqueViolation +from psycopg.types.json import Jsonb + +from .errors import ApiError +from .pagination import CursorKey + + +def utcnow() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def rfc3339(value: dt.datetime) -> str: + return value.astimezone(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f") + "Z" + + +def _members(value) -> List[str]: + if value is None: + return [] + return [str(item) for item in value] + + +def user_json(row: Dict) -> Dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": rfc3339(row["created_at"]), + } + + +def chat_json(row: Dict) -> Dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": _members(row["member_ids"]), + "created_at": rfc3339(row["created_at"]), + } + + +def message_json(row: Dict) -> Dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": rfc3339(row["created_at"]), + "attachment_ids": _members(row.get("attachment_ids")), + } + + +_MESSAGE_ATTACHMENTS_SUBQUERY = ( + "(select array_agg(ma.attachment_id order by ma.position) " + "from message_attachments ma where ma.message_id = m.id)" +) + + +def variant_json(row: Dict) -> Dict: + return { + "name": row["name"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "width": int(row["width"]), + "height": int(row["height"]), + "sha256": row["sha256"], + } + + +def attachment_json(row: Dict, variants: Sequence[Dict] = ()) -> Dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "sha256": row["sha256"], + "status": row["status"], + "variants": [variant_json(v) for v in variants], + "error_code": row["error_code"], + "created_at": rfc3339(row["created_at"]), + "updated_at": rfc3339(row["updated_at"]), + } + + +# --------------------------------------------------------------------------- # +# Users +# --------------------------------------------------------------------------- # +def create_user(conn, username: str, display_name: str, password_hash: str) -> Dict: + try: + row = conn.execute( + """ + insert into users (id, username, display_name, password_hash, created_at) + values (%s, %s, %s, %s, %s) + returning id, username, display_name, created_at + """, + (str(uuid.uuid4()), username, display_name, password_hash, utcnow()), + ).fetchone() + except UniqueViolation: + raise ApiError(409, "conflict", "Имя пользователя уже занято") + return row + + +def get_user_by_id(conn, user_id: str) -> Optional[Dict]: + return conn.execute( + "select id, username, display_name, created_at from users where id = %s", + (user_id,), + ).fetchone() + + +def get_user_with_hash_by_username(conn, username: str) -> Optional[Dict]: + return conn.execute( + "select id, username, display_name, created_at, password_hash from users where username = %s", + (username,), + ).fetchone() + + +def count_users(conn, user_ids: Sequence[str]) -> int: + row = conn.execute( + "select count(*) as n from users where id = any(%s::uuid[])", (list(user_ids),) + ).fetchone() + return int(row["n"]) + + +# --------------------------------------------------------------------------- # +# Chats +# --------------------------------------------------------------------------- # +def create_chat(conn, title: str, member_ids: Sequence[str]) -> Dict: + chat_id = str(uuid.uuid4()) + created = utcnow() + conn.execute( + "insert into chats (id, title, created_at) values (%s, %s, %s)", + (chat_id, title, created), + ) + with conn.cursor() as cur: + cur.executemany( + "insert into chat_members (chat_id, user_id) values (%s, %s)", + [(chat_id, member) for member in member_ids], + ) + return { + "id": chat_id, + "title": title, + "member_ids": list(member_ids), + "created_at": created, + } + + +_MEMBER_SUBQUERY = ( + "(select array_agg(m.user_id) from chat_members m where m.chat_id = c.id)" +) + + +def get_chat_for(conn, chat_id: str, user_id: str) -> Optional[Dict]: + return conn.execute( + f""" + select c.id, c.title, c.created_at, {_MEMBER_SUBQUERY} as member_ids + from chats c + where c.id = %s + and exists ( + select 1 from chat_members cm + where cm.chat_id = c.id and cm.user_id = %s + ) + """, + (chat_id, user_id), + ).fetchone() + + +def list_chats(conn, user_id: str, limit: int, cursor: Optional[CursorKey]) -> List[Dict]: + if cursor is None: + return conn.execute( + f""" + select c.id, c.title, c.created_at, {_MEMBER_SUBQUERY} as member_ids + from chats c + join chat_members cm on cm.chat_id = c.id + where cm.user_id = %s + order by c.created_at, c.id + limit %s + """, + (user_id, limit), + ).fetchall() + return conn.execute( + f""" + select c.id, c.title, c.created_at, {_MEMBER_SUBQUERY} as member_ids + from chats c + join chat_members cm on cm.chat_id = c.id + where cm.user_id = %s and (c.created_at, c.id) > (%s, %s) + order by c.created_at, c.id + limit %s + """, + (user_id, cursor[0], cursor[1], limit), + ).fetchall() + + +# --------------------------------------------------------------------------- # +# Messages +# --------------------------------------------------------------------------- # +def insert_message( + conn, chat_id: str, sender_id: str, text: str, attachment_ids: Sequence[str] = () +) -> Dict: + message_id = str(uuid.uuid4()) + created = utcnow() + conn.execute( + "insert into messages (id, chat_id, sender_id, text, created_at) values (%s, %s, %s, %s, %s)", + (message_id, chat_id, sender_id, text, created), + ) + attachments = list(attachment_ids) + if attachments: + with conn.cursor() as cur: + cur.executemany( + "insert into message_attachments (message_id, attachment_id, position) values (%s, %s, %s)", + [(message_id, attachment_id, position) for position, attachment_id in enumerate(attachments)], + ) + return { + "id": message_id, + "chat_id": chat_id, + "sender_id": sender_id, + "text": text, + "created_at": created, + "attachment_ids": attachments, + } + + +def list_messages( + conn, chat_id: str, limit: int, cursor: Optional[CursorKey] +) -> List[Dict]: + if cursor is None: + return conn.execute( + f""" + select m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {_MESSAGE_ATTACHMENTS_SUBQUERY} as attachment_ids + from messages m + where m.chat_id = %s + order by m.created_at, m.id + limit %s + """, + (chat_id, limit), + ).fetchall() + return conn.execute( + f""" + select m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {_MESSAGE_ATTACHMENTS_SUBQUERY} as attachment_ids + from messages m + where m.chat_id = %s and (m.created_at, m.id) > (%s, %s) + order by m.created_at, m.id + limit %s + """, + (chat_id, cursor[0], cursor[1], limit), + ).fetchall() + + +# --------------------------------------------------------------------------- # +# Shared idempotency ledger (lab3) +# --------------------------------------------------------------------------- # +def message_request_hash(text: str, attachment_ids: Sequence[str] = ()) -> str: + """Canonical hash of the semantic body. + + JSON key order and an absent ``attachment_ids`` versus ``[]`` must be + equivalent, and the order of attachment ids is not significant. The + canonical form is deterministic so any replica computes the same value. + """ + canonical = json.dumps( + {"text": text, "attachment_ids": sorted(attachment_ids or [])}, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=False, + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def get_idempotency(conn, user_id: str, scope: str, key: str) -> Optional[Dict]: + return conn.execute( + """ + select request_hash, status_code, response_body + from idempotency_keys + where user_id = %s and scope = %s and key = %s + """, + (user_id, scope, key), + ).fetchone() + + +def insert_idempotency( + conn, + user_id: str, + scope: str, + key: str, + request_hash: str, + status_code: int, + response_body: Dict, + ttl_seconds: int, +) -> None: + conn.execute( + """ + insert into idempotency_keys + (user_id, scope, key, request_hash, status_code, response_body, created_at, expires_at) + values (%s, %s, %s, %s, %s, %s, now(), now() + make_interval(secs => %s)) + """, + (user_id, scope, key, request_hash, status_code, Jsonb(response_body), ttl_seconds), + ) + + +def purge_expired_idempotency(conn) -> int: + rows = conn.execute( + "delete from idempotency_keys where expires_at <= now() returning key" + ).fetchall() + return len(rows) + + +def create_message_idempotent( + conn, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: Sequence[str], + scope: str, + key: str, + ttl_seconds: int, +) -> Dict: + """Create a message exactly once for (sender, scope, key). + + A transaction-scoped advisory lock keyed on the logical idempotency scope + serializes concurrent retries across *all* API replicas through PostgreSQL. + The first caller inserts the message (and its attachment links) and the + ledger row in one transaction; later callers observe the committed row and + replay the stored response. A reused key with different content conflicts. + """ + request_hash = message_request_hash(text, attachment_ids) + conn.execute( + "select pg_advisory_xact_lock(hashtext(%s), hashtext(%s))", + (sender_id + "|" + scope, key), + ) + existing = get_idempotency(conn, sender_id, scope, key) + if existing is not None: + if existing["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим сообщением") + return existing["response_body"] + message = insert_message(conn, chat_id, sender_id, text, attachment_ids) + body = message_json(message) + insert_idempotency(conn, sender_id, scope, key, request_hash, 201, body, ttl_seconds) + return body + + +# --------------------------------------------------------------------------- # +# Sessions (PostgreSQL is the source of truth) +# --------------------------------------------------------------------------- # +def insert_session(conn, token_hash: str, user_id: str, expires_at: dt.datetime) -> None: + conn.execute( + """ + insert into sessions (token_hash, user_id, created_at, expires_at) + values (%s, %s, %s, %s) + """, + (token_hash, user_id, utcnow(), expires_at), + ) + + +def get_valid_session(conn, token_hash: str) -> Optional[Dict]: + return conn.execute( + """ + select token_hash, user_id, expires_at + from sessions + where token_hash = %s and expires_at > now() + """, + (token_hash,), + ).fetchone() + + +def delete_session(conn, token_hash: str) -> None: + conn.execute("delete from sessions where token_hash = %s", (token_hash,)) + + +def purge_expired_sessions(conn) -> int: + row = conn.execute( + "delete from sessions where expires_at <= now() returning token_hash" + ).fetchall() + return len(row) + + +# --------------------------------------------------------------------------- # +# Durable cache invalidation queue (lab3) +# --------------------------------------------------------------------------- # +def enqueue_cache_invalidation(conn, token_hash: str) -> None: + """Record a session cache key that must be removed from Valkey. + + Written in the same transaction as the authoritative session delete, so a + committed logout can never lose the obligation to invalidate the cache. + """ + conn.execute( + """ + insert into cache_invalidations (token_hash, created_at) + values (%s, now()) + on conflict (token_hash) do nothing + """, + (token_hash,), + ) + + +def list_pending_invalidations(conn, limit: int) -> List[str]: + rows = conn.execute( + "select token_hash from cache_invalidations order by created_at limit %s", + (limit,), + ).fetchall() + return [str(row["token_hash"]) for row in rows] + + +def delete_cache_invalidations(conn, token_hashes: Sequence[str]) -> None: + if not token_hashes: + return + conn.execute( + "delete from cache_invalidations where token_hash = any(%s)", + (list(token_hashes),), + ) + + +# --------------------------------------------------------------------------- # +# Attachments (lab5): metadata in PostgreSQL, bytes in S3, async processing +# --------------------------------------------------------------------------- # +_ATTACHMENT_COLUMNS = """ + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + staging_key, s3_key, status, pipeline_version, error_code, created_at, updated_at +""" + + +def insert_attachment( + conn, + attachment_id: str, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + size_bytes: int, + sha256: str, + staging_key: str, + status: str, + pipeline_version: int, + error_code: Optional[str], + created_at: dt.datetime, + updated_at: dt.datetime, +) -> Dict: + return conn.execute( + f""" + insert into attachments + (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + staging_key, status, pipeline_version, error_code, created_at, updated_at) + values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + returning {_ATTACHMENT_COLUMNS} + """, + ( + attachment_id, chat_id, owner_id, filename, content_type, size_bytes, + sha256, staging_key, status, pipeline_version, error_code, created_at, updated_at, + ), + ).fetchone() + + +def get_attachment(conn, attachment_id: str) -> Optional[Dict]: + return conn.execute( + f"select {_ATTACHMENT_COLUMNS} from attachments where id = %s", (attachment_id,) + ).fetchone() + + +def get_attachment_for(conn, attachment_id: str, user_id: str) -> Optional[Dict]: + """Metadata only when the caller is a member of the attachment's chat.""" + return conn.execute( + f""" + select {_ATTACHMENT_COLUMNS} + from attachments a + where a.id = %s + and exists ( + select 1 from chat_members cm + where cm.chat_id = a.chat_id and cm.user_id = %s + ) + """, + (attachment_id, user_id), + ).fetchone() + + +def get_attachment_for_update(conn, attachment_id: str) -> Optional[Dict]: + return conn.execute( + f"select {_ATTACHMENT_COLUMNS} from attachments where id = %s for update", + (attachment_id,), + ).fetchone() + + +def attachments_in_chat(conn, chat_id: str, attachment_ids: Sequence[str]) -> List[Dict]: + if not attachment_ids: + return [] + return conn.execute( + "select id, status from attachments where chat_id = %s and id = any(%s::uuid[])", + (chat_id, list(attachment_ids)), + ).fetchall() + + +def mark_attachment_processing(conn, attachment_id: str) -> None: + conn.execute( + "update attachments set status = 'processing', updated_at = now() where id = %s", + (attachment_id,), + ) + + +def mark_attachment_queued(conn, attachment_id: str) -> None: + conn.execute( + "update attachments set status = 'queued', updated_at = now() where id = %s", + (attachment_id,), + ) + + +def finalize_attachment_ready( + conn, attachment_id: str, original_key: str, pipeline_version: int +) -> None: + conn.execute( + """ + update attachments + set status = 'ready', s3_key = %s, pipeline_version = %s, + error_code = null, updated_at = now() + where id = %s + """, + (original_key, pipeline_version, attachment_id), + ) + + +def mark_attachment_failed(conn, attachment_id: str, error_code: str) -> None: + conn.execute( + "update attachments set status = 'failed', error_code = %s, updated_at = now() where id = %s", + (error_code, attachment_id), + ) + + +def list_variants(conn, attachment_id: str, pipeline_version: int) -> List[Dict]: + return conn.execute( + """ + select name, content_type, size_bytes, width, height, sha256, s3_key + from attachment_variants + where attachment_id = %s and pipeline_version = %s + order by name + """, + (attachment_id, pipeline_version), + ).fetchall() + + +def get_variant(conn, attachment_id: str, pipeline_version: int, name: str) -> Optional[Dict]: + return conn.execute( + """ + select name, content_type, size_bytes, width, height, sha256, s3_key + from attachment_variants + where attachment_id = %s and pipeline_version = %s and name = %s + """, + (attachment_id, pipeline_version, name), + ).fetchone() + + +def upsert_variant( + conn, + attachment_id: str, + pipeline_version: int, + name: str, + content_type: str, + size_bytes: int, + width: int, + height: int, + sha256: str, + s3_key: str, +) -> None: + """Idempotent write keyed on (attachment_id, pipeline_version, name).""" + conn.execute( + """ + insert into attachment_variants + (id, attachment_id, pipeline_version, name, content_type, size_bytes, + width, height, sha256, s3_key, created_at) + values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, now()) + on conflict (attachment_id, pipeline_version, name) + do update set content_type = excluded.content_type, + size_bytes = excluded.size_bytes, + width = excluded.width, + height = excluded.height, + sha256 = excluded.sha256, + s3_key = excluded.s3_key + """, + (str(uuid.uuid4()), attachment_id, pipeline_version, name, content_type, + size_bytes, width, height, sha256, s3_key), + ) + + +def list_all_attachments(conn) -> List[Dict]: + return conn.execute( + "select id, s3_key, staging_key, created_at from attachments" + ).fetchall() + + +# --------------------------------------------------------------------------- # +# Attachment jobs / transactional outbox (lab5) +# --------------------------------------------------------------------------- # +def create_job( + conn, job_id: str, attachment_id: str, pipeline_version: int, created_at: dt.datetime +) -> None: + conn.execute( + """ + insert into attachment_jobs + (job_id, attachment_id, pipeline_version, attempt, status, available_at, + created_at, updated_at) + values (%s, %s, %s, 1, 'pending', now(), %s, %s) + """, + (job_id, attachment_id, pipeline_version, created_at, created_at), + ) + + +def list_outbox_jobs(conn, limit: int) -> List[Dict]: + return conn.execute( + """ + select job_id, attachment_id, pipeline_version, attempt, created_at + from attachment_jobs + where published_at is null and status = 'pending' and available_at <= now() + order by created_at + limit %s + """, + (limit,), + ).fetchall() + + +def mark_job_published(conn, job_id: str) -> None: + conn.execute( + "update attachment_jobs set published_at = now(), updated_at = now() where job_id = %s", + (job_id,), + ) + + +def claim_job(conn, job_id: str, lease_seconds: int) -> bool: + """Atomically transition pending->processing; False if already claimed.""" + row = conn.execute( + """ + update attachment_jobs + set status = 'processing', + lease_until = now() + make_interval(secs => %s), + updated_at = now() + where job_id = %s and status = 'pending' + returning attempt + """, + (lease_seconds, job_id), + ).fetchone() + return row is not None + + +def complete_job(conn, job_id: str) -> None: + conn.execute( + "update attachment_jobs set status = 'done', lease_until = null, updated_at = now() where job_id = %s", + (job_id,), + ) + + +def fail_job(conn, job_id: str, error: str) -> None: + conn.execute( + """ + update attachment_jobs + set status = 'failed', last_error = %s, lease_until = null, updated_at = now() + where job_id = %s + """, + (error, job_id), + ) + + +def requeue_job(conn, job_id: str, available_at: dt.datetime, error: str) -> None: + conn.execute( + """ + update attachment_jobs + set status = 'pending', published_at = null, available_at = %s, + attempt = attempt + 1, last_error = %s, lease_until = null, updated_at = now() + where job_id = %s + """, + (available_at, error, job_id), + ) + + +def requeue_expired_leases(conn) -> List[str]: + rows = conn.execute( + """ + update attachment_jobs + set status = 'pending', published_at = null, available_at = now(), + lease_until = null, updated_at = now() + where status = 'processing' and lease_until is not null and lease_until < now() + returning attachment_id + """ + ).fetchall() + attachment_ids = [str(row["attachment_id"]) for row in rows] + for attachment_id in attachment_ids: + conn.execute( + "update attachments set status = 'queued', updated_at = now() " + "where id = %s and status = 'processing'", + (attachment_id,), + ) + return attachment_ids + + +def job_attempt(conn, job_id: str) -> Optional[int]: + row = conn.execute( + "select attempt from attachment_jobs where job_id = %s", (job_id,) + ).fetchone() + return int(row["attempt"]) if row else None + + +def get_job_for_update(conn, job_id: str) -> Optional[Dict]: + return conn.execute( + "select job_id, attachment_id, pipeline_version, attempt, status " + "from attachment_jobs where job_id = %s for update", + (job_id,), + ).fetchone() + + +def list_failed_jobs(conn, limit: int = 50) -> List[Dict]: + return conn.execute( + """ + select job_id, attachment_id, attempt, last_error, updated_at + from attachment_jobs where status = 'failed' + order by updated_at desc limit %s + """, + (limit,), + ).fetchall() + + +def replay_failed_jobs(conn, job_id: Optional[str] = None) -> int: + if job_id is None: + rows = conn.execute( + """ + update attachment_jobs + set status = 'pending', published_at = null, available_at = now(), + attempt = 1, last_error = null, lease_until = null, updated_at = now() + where status = 'failed' + returning job_id, attachment_id + """ + ).fetchall() + else: + rows = conn.execute( + """ + update attachment_jobs + set status = 'pending', published_at = null, available_at = now(), + attempt = 1, last_error = null, lease_until = null, updated_at = now() + where status = 'failed' and job_id = %s + returning job_id, attachment_id + """ + , + (job_id,), + ).fetchall() + for row in rows: + conn.execute( + "update attachments set status = 'queued', error_code = null, updated_at = now() where id = %s", + (str(row["attachment_id"]),), + ) + return len(rows) + + +def job_message(row: Dict) -> Dict: + """Build a broker payload matching contracts/jobs.schema.json.""" + return { + "schema_version": 1, + "job_id": str(row["job_id"]), + "type": "attachment.process.v1", + "attachment_id": str(row["attachment_id"]), + "pipeline_version": int(row["pipeline_version"]), + "attempt": int(row["attempt"]), + "created_at": rfc3339(row["created_at"]), + } + + + diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 00000000..fae9b4e7 --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,57 @@ +"""Idempotent S3 bucket bootstrap (one-shot compose service). + +Creates the private attachments bucket with the root credentials before any API +replica starts. Re-running is safe; no credentials are printed. +""" +from __future__ import annotations + +import os +import time + +import boto3 +from botocore.client import Config +from botocore.exceptions import BotoCoreError, ClientError + + +def _read(env_key: str) -> str: + with open(os.environ[env_key], encoding="utf-8") as handle: + return handle.read().strip() + + +def main() -> None: + endpoint = os.environ["S3_ENDPOINT"] + region = os.environ.get("S3_REGION", "us-east-1") + bucket = os.environ["S3_BUCKET"] + client = boto3.client( + "s3", + endpoint_url=endpoint, + region_name=region, + aws_access_key_id=_read("S3_ROOT_ACCESS_KEY_ID_FILE"), + aws_secret_access_key=_read("S3_ROOT_SECRET_ACCESS_KEY_FILE"), + config=Config( + connect_timeout=2, + read_timeout=5, + retries={"max_attempts": 1}, + s3={"addressing_style": "path"}, + ), + ) + + last: Exception | None = None + for attempt in range(1, 31): + try: + try: + client.head_bucket(Bucket=bucket) + print(f"bucket {bucket} already present") + except ClientError: + client.create_bucket(Bucket=bucket) + print(f"bucket {bucket} created") + return + except (ClientError, BotoCoreError) as exc: + last = exc + print(f"bucket bootstrap not ready (attempt {attempt}/30)") + time.sleep(2) + raise SystemExit(f"bucket bootstrap failed: {last}") + + +if __name__ == "__main__": + main() diff --git a/src/s3_private_check.py b/src/s3_private_check.py new file mode 100644 index 00000000..a708a16f --- /dev/null +++ b/src/s3_private_check.py @@ -0,0 +1,37 @@ +"""Prove the bucket is private *outside* the API authorization path. + +Lists one object with the limited credentials, then requests the same object +over plain HTTP with NO credentials. A private bucket must answer AccessDenied +(403) or 404, never 200 with the bytes. + + docker compose exec -T api python -m src.s3_private_check +""" +from __future__ import annotations + +import urllib.error +import urllib.request + +from . import storage +from .config import settings + + +def main() -> None: + keys = [item["Key"] for item in storage.list_keys("")] + if not keys: + raise SystemExit("no objects to check; upload an attachment first") + key = keys[0] + url = f"{settings.s3_endpoint.rstrip('/')}/{settings.s3_bucket}/{key}" + try: + with urllib.request.urlopen(url, timeout=5) as response: + status = response.status + size = len(response.read()) + except urllib.error.HTTPError as error: + print(f"anonymous GET -> {error.code} (denied)") + assert error.code in {403, 404}, f"unexpected status {error.code}" + print("PASS: private bucket denies anonymous access") + return + raise SystemExit(f"FAIL: anonymous GET returned {status} with {size} bytes") + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..79aaa92d --- /dev/null +++ b/src/security.py @@ -0,0 +1,33 @@ +"""Password KDF (Argon2id) and opaque bearer-token helpers.""" +from __future__ import annotations + +import hashlib +import secrets + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError, VerificationError + +# Argon2id is the default of argon2-cffi's PasswordHasher. Parameters are kept +# modest so the local test suite stays fast while still using a real KDF. +_hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return _hasher.verify(password_hash, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def new_token() -> str: + """Opaque bearer token with >= 256 bits of entropy.""" + return secrets.token_urlsafe(32) + + +def token_hash(token: str) -> str: + """Sessions are identified by a SHA-256 of the token, never the token itself.""" + return hashlib.sha256(token.encode("utf-8")).hexdigest() diff --git a/src/seed.py b/src/seed.py new file mode 100644 index 00000000..c59e338c --- /dev/null +++ b/src/seed.py @@ -0,0 +1,87 @@ +"""Synthetic data seeder for EXPLAIN evidence (never used by the public smoke). + +Runs INSIDE the api container so it can reach PostgreSQL over the compose +network: + + docker compose exec -T api python -m src.seed --messages 10000 + +It writes a small number of users/chats and one "hot" chat with many messages, +then prints the hot chat id so the operator can run EXPLAIN against it. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import uuid + +from .db import pool + + +def _now() -> dt.datetime: + return dt.datetime.now(dt.timezone.utc) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Seed synthetic chat history") + parser.add_argument("--users", type=int, default=10) + parser.add_argument("--chats", type=int, default=3) + parser.add_argument("--messages", type=int, default=10000) + args = parser.parse_args() + + user_ids = [str(uuid.uuid4()) for _ in range(args.users)] + chat_ids = [str(uuid.uuid4()) for _ in range(args.chats)] + base = _now() + + with pool().connection() as conn: + with conn.cursor() as cur: + cur.executemany( + """ + insert into users (id, username, display_name, password_hash, created_at) + values (%s, %s, %s, %s, %s) + """, + [ + (uid, f"seed_{i}_{uuid.uuid4().hex[:6]}", f"Seed {i}", "seed-not-loginable", base) + for i, uid in enumerate(user_ids) + ], + ) + cur.executemany( + "insert into chats (id, title, created_at) values (%s, %s, %s)", + [(cid, f"Seed chat {i}", base) for i, cid in enumerate(chat_ids)], + ) + cur.executemany( + "insert into chat_members (chat_id, user_id) values (%s, %s)", + [(cid, uid) for cid in chat_ids for uid in user_ids], + ) + + hot_chat = chat_ids[0] + per_chat = max(1, args.messages // len(chat_ids)) + rows = [] + total = 0 + for chat_index, cid in enumerate(chat_ids): + count = args.messages - total if chat_index == len(chat_ids) - 1 else per_chat + count = max(0, count) + for _ in range(count): + rows.append( + ( + str(uuid.uuid4()), + cid, + user_ids[0], + "synthetic history message", + base, + ) + ) + total += count + cur.executemany( + """ + insert into messages (id, chat_id, sender_id, text, created_at) + values (%s, %s, %s, %s, %s) + """, + rows, + ) + + print(json.dumps({"hot_chat_id": hot_chat, "messages": args.messages, "chats": chat_ids})) + + +if __name__ == "__main__": + main() diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..be6f0376 --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,203 @@ +"""Sessions: PostgreSQL is the source of truth, Valkey is a shared session cache. + +Design (lab3): + * ``sessions`` rows in PostgreSQL decide validity (``expires_at > now()``) and + survive restarts with the named volume. + * Valkey caches ``session:{token_hash} -> {user_id, expires_at}`` for at most + ``SESSION_CACHE_TTL_SECONDS``. The TTL is a *bounded staleness* guarantee: + even if an invalidation is lost, a cached positive cannot outlive this + window. + * Logout deletes the PostgreSQL row and, in the same transaction, enqueues the + token hash in ``cache_invalidations``. A background drainer (see + ``src/invalidation.py``) removes the Valkey key and then the queue row, so + a logout that happened while Valkey was down is still honoured as soon as + the cache returns — instead of being resurrected from a stale entry. + * If Valkey is unreachable every cache operation degrades to PostgreSQL. The + API keeps serving; the cache is never trusted as the source of truth. + +Cache calls on the request path are bounded by a hard timeout and a small +circuit breaker (a stopped Valkey can make Docker DNS block for seconds). The +drainer bypasses the breaker on purpose: invalidation must not be skipped. +""" +from __future__ import annotations + +import concurrent.futures +import datetime as dt +import json +import logging +import time +from typing import Optional, Tuple + +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + +from . import repositories as repo +from . import security +from .config import settings + +logger = logging.getLogger("messenger.sessions") + +_client: Optional[redis.Redis] = None +_client_disabled = False + +_executor = concurrent.futures.ThreadPoolExecutor(max_workers=2, thread_name_prefix="cache") +_OP_TIMEOUT = 0.5 +_BREAKER_SECONDS = 5.0 +_breaker_until = 0.0 + + +def _cache() -> Optional[redis.Redis]: + global _client, _client_disabled + if not settings.cache_enabled or _client_disabled: + return None + if _client is None: + try: + _client = redis.Redis.from_url( + settings.cache_url, + socket_timeout=0.2, + socket_connect_timeout=0.2, + health_check_interval=0, + retry_on_timeout=False, + retry=Retry(NoBackoff(), 0), + ) + except (ValueError, redis.RedisError): + logger.warning("valkey client init failed; cache disabled") + _client_disabled = True + return None + return _client + + +def _guard(fn, *args): + """Run a request-path cache operation with a deadline and a small breaker.""" + global _breaker_until + if time.monotonic() < _breaker_until: + return None + try: + return _executor.submit(fn, *args).result(timeout=_OP_TIMEOUT) + except (concurrent.futures.TimeoutError, redis.RedisError, OSError): + _breaker_until = time.monotonic() + _BREAKER_SECONDS + logger.warning("valkey unavailable; cache bypassed for %.0fs", _BREAKER_SECONDS) + return None + + +def _session_key(token_hash: str) -> str: + return f"session:{token_hash}" + + +def _cache_get(key: str) -> Optional[str]: + client = _cache() + if client is None: + return None + value = _guard(lambda: client.get(key)) + if value is None: + return None + return value.decode("utf-8") if isinstance(value, bytes) else str(value) + + +def _cache_set(key: str, value: str, ttl_seconds: float) -> None: + client = _cache() + if client is None: + return + _guard(lambda: client.set(key, value, ex=max(1, int(ttl_seconds)))) + + +def _cache_delete(key: str) -> None: + client = _cache() + if client is None: + return + _guard(lambda: client.delete(key)) + + +def _encode_session(user_id: str, expires_at: dt.datetime) -> str: + return json.dumps( + {"user_id": user_id, "expires_at": repo.rfc3339(expires_at)}, + separators=(",", ":"), + ) + + +def _decode_session(raw: str) -> Optional[Tuple[str, dt.datetime]]: + try: + data = json.loads(raw) + user_id = str(data["user_id"]) + value = str(data["expires_at"]) + if value.endswith("Z"): + value = value[:-1] + "+00:00" + expires_at = dt.datetime.fromisoformat(value) + return user_id, expires_at + except (ValueError, KeyError, TypeError): + return None + + +def _cache_ttl(remaining: float) -> int: + return max(1, int(min(remaining, settings.session_cache_ttl_seconds))) + + +def create(conn, user_id: str) -> Tuple[str, dt.datetime]: + token = security.new_token() + hashed = security.token_hash(token) + expires_at = repo.utcnow() + dt.timedelta(seconds=settings.session_ttl_seconds) + repo.insert_session(conn, hashed, user_id, expires_at) + # The cache is populated by the first successful resolve() (read-through), + # so it can only ever contain sessions already committed to PostgreSQL. + return token, expires_at + + +def resolve(conn, token: str) -> Optional[str]: + """Return the session's user id, or None if the token is not valid. + + A cache hit is accepted only while the stored expiry is in the future and + only for the bounded cache TTL. A miss or any cache failure falls back to + the authoritative PostgreSQL row. + """ + hashed = security.token_hash(token) + cached = _cache_get(_session_key(hashed)) + if cached is not None: + decoded = _decode_session(cached) + if decoded is not None and decoded[1] > repo.utcnow(): + return decoded[0] + + row = repo.get_valid_session(conn, hashed) + if row is None: + # Remove any legacy/stale entry; revocation is already authoritative. + _cache_delete(_session_key(hashed)) + return None + + user_id = str(row["user_id"]) + remaining = (row["expires_at"] - repo.utcnow()).total_seconds() + if remaining > 0: + _cache_set( + _session_key(hashed), + _encode_session(user_id, row["expires_at"]), + _cache_ttl(remaining), + ) + return user_id + + +def revoke(conn, token: str) -> None: + hashed = security.token_hash(token) + repo.delete_session(conn, hashed) + if settings.cache_enabled: + # Durable obligation, committed together with the session delete. + repo.enqueue_cache_invalidation(conn, hashed) + # Best effort: the drainer retries if this fails (e.g. Valkey is down). + _cache_delete(_session_key(hashed)) + + +def invalidate_session_key(token_hash: str) -> bool: + """Delete one cache key for the drainer; True means it is safe to dequeue. + + Deliberately bypasses the circuit breaker: a transient cache outage must not + cause pending invalidations to be dropped. Socket timeouts still bound the + call so the worker cannot block forever. + """ + client = _cache() + if client is None: + # Cache deliberately disabled -> nothing to invalidate. A client that + # failed to initialise keeps its rows so a restart can retry. + return not settings.cache_enabled + try: + client.delete(_session_key(token_hash)) + return True + except (redis.RedisError, OSError): + return False diff --git a/src/state.py b/src/state.py new file mode 100644 index 00000000..276a5d95 --- /dev/null +++ b/src/state.py @@ -0,0 +1,19 @@ +"""Process-wide lifecycle flags. + +``shutting_down`` is raised from the SIGTERM/SIGINT handler before Uvicorn +stops accepting connections, so ``/health/ready`` reports ``503`` and the +balancer drains this replica instead of sending it new work. +""" +from __future__ import annotations + +import threading + +_shutting_down = threading.Event() + + +def begin_shutdown() -> None: + _shutting_down.set() + + +def is_shutting_down() -> bool: + return _shutting_down.is_set() diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 00000000..65c03140 --- /dev/null +++ b/src/storage.py @@ -0,0 +1,129 @@ +"""S3-compatible object storage client (SeaweedFS). + +Only the API process uses these limited credentials; the bucket is private and +object keys are generated by the server, never derived from ``filename``. All +connectivity failures become a bounded ``503 unavailable``. + +Every operation runs under a hard deadline with a small circuit breaker: a +stopped S3 container can make Docker DNS itself block for seconds, and without +this guard a storage outage would stall request handling instead of failing fast. +""" +from __future__ import annotations + +import concurrent.futures +import logging +import time +from typing import BinaryIO, Iterable, Optional + +import boto3 +from botocore.client import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import settings +from .errors import ApiError + +logger = logging.getLogger("messenger.storage") + +_client = None + +_BOTO_CONFIG = Config( + connect_timeout=2, + read_timeout=10, + retries={"max_attempts": 1, "mode": "standard"}, + s3={"addressing_style": "path"}, +) + +_executor = concurrent.futures.ThreadPoolExecutor(max_workers=4, thread_name_prefix="s3") +_BREAKER_SECONDS = 5.0 +_breaker_until = 0.0 + + +def _unavailable() -> ApiError: + return ApiError(503, "unavailable", "Хранилище недоступно") + + +def client(): + global _client + if _client is None: + _client = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + region_name=settings.s3_region, + aws_access_key_id=settings.s3_access_key_id or None, + aws_secret_access_key=settings.s3_secret_access_key or None, + config=_BOTO_CONFIG, + ) + return _client + + +def _run(fn, timeout: float): + """Run an S3 call under a deadline and a shared breaker.""" + global _breaker_until + if time.monotonic() < _breaker_until: + raise _unavailable() + try: + return _executor.submit(fn).result(timeout=timeout) + except (concurrent.futures.TimeoutError, BotoCoreError, ClientError, OSError): + _breaker_until = time.monotonic() + _BREAKER_SECONDS + logger.warning("S3 unavailable; operations fail fast for %.0fs", _BREAKER_SECONDS) + raise _unavailable() + + +def ready() -> bool: + """Return True if the bucket is reachable (used by /health/ready).""" + try: + _run(lambda: client().head_bucket(Bucket=settings.s3_bucket), timeout=1.5) + return True + except ApiError: + return False + + +def put_object(key: str, fileobj: BinaryIO, content_type: str) -> None: + _run( + lambda: client().upload_fileobj( + fileobj, settings.s3_bucket, key, ExtraArgs={"ContentType": content_type} + ), + timeout=30, + ) + + +def open_object(key: str): + """Return the boto3 get_object response (caller streams and closes Body).""" + + def _get(): + try: + return client().get_object(Bucket=settings.s3_bucket, Key=key) + except ClientError as exc: + if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: + raise ApiError(404, "not_found", "Объект не найден") + raise + + return _run(_get, timeout=10) + + +def get_bytes(key: str) -> bytes: + """Read a whole object (bounded: attachments are <= 10 MiB).""" + response = open_object(key) + body = response["Body"] + try: + return body.read() + finally: + body.close() + + +def delete_object(key: str) -> bool: + try: + _run(lambda: client().delete_object(Bucket=settings.s3_bucket, Key=key), timeout=10) + return True + except ApiError: + return False + + +def list_keys(prefix: str = "") -> Iterable[dict]: + paginator = client().get_paginator("list_objects_v2") + try: + for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix): + for item in page.get("Contents", []): + yield item + except (BotoCoreError, ClientError): + raise _unavailable() diff --git a/src/worker.py b/src/worker.py new file mode 100644 index 00000000..16bcaa40 --- /dev/null +++ b/src/worker.py @@ -0,0 +1,248 @@ +"""Attachment worker: consumes durable jobs and produces the variants. + +Separate process from the API. Delivery is at-least-once; the durable effect is +idempotent: + * variants are upserted on ``(attachment_id, pipeline_version, name)``; + * the original is written to a deterministic key and then metadata is + committed atomically with the variants and the job's ``done`` state; + * a redelivery of an already-``ready`` job is a no-op ack; + * an older ``pipeline_version`` never overwrites a newer result. + +Transient failures are scheduled in the outbox (``pending`` + future +``available_at`` with jitter) and the message is acked; permanent failures mark +the attachment ``failed`` with a stable ``error_code`` and ack. Lease expiry is +handled by a watchdog that requeues stuck ``processing`` jobs. +""" +from __future__ import annotations + +import hashlib +import io +import json +import logging +import random +import signal +import threading +import time +from datetime import timedelta + +from . import broker, imaging +from . import repositories as repo +from . import storage +from .config import settings +from .db import close_pool, pool +from .errors import ApiError + +logging.basicConfig( + level=__import__("os").environ.get("LOG_LEVEL", "INFO").upper(), + format="%(asctime)s %(levelname)s %(name)s %(message)s", +) +logger = logging.getLogger("messenger.worker") + +_stop = threading.Event() + +_VARIANT_SPECS = [ + ("thumbnail", "make_thumbnail", "image/jpeg"), + ("crop", "make_crop", "image/jpeg"), + ("optimized", "make_optimized", "image/webp"), +] + + +def _build_variants(chat_id: str, attachment_id: str, data: bytes) -> list[dict]: + variants = [] + for name, func_name, content_type in _VARIANT_SPECS: + func = getattr(imaging, func_name) + try: + payload, width, height = func(data) + except imaging.PermanentImageError: + raise + except Exception as exc: # noqa: BLE001 - unexpected encode failure + raise imaging.PermanentImageError("processing_failed") from exc + variants.append( + { + "name": name, + "content_type": content_type, + "bytes": payload, + "width": width, + "height": height, + "sha256": hashlib.sha256(payload).hexdigest(), + "s3_key": f"chats/{chat_id}/{attachment_id}/{name}", + } + ) + return variants + + +def _process(job: dict) -> None: + attachment_id = job["attachment_id"] + job_id = job["job_id"] + pipeline_version = int(job["pipeline_version"]) + + with pool().connection() as conn: + job_row = repo.get_job_for_update(conn, job_id) + attachment = repo.get_attachment_for_update(conn, attachment_id) + if attachment is None or job_row is None: + return + if attachment["status"] == "ready" and int(attachment["pipeline_version"]) >= pipeline_version: + repo.complete_job(conn, job_id) # redelivery of finished work + return + if pipeline_version < int(attachment["pipeline_version"]): + return # stale version must not overwrite a newer result + # Atomic claim: a duplicate delivery of an already-claimed job is a no-op. + if not repo.claim_job(conn, job_id, settings.job_timeout_seconds): + logger.info("job %s not claimable (status=%s); skipping duplicate", job_id, job_row["status"]) + return + attempt = int(job_row["attempt"]) + staging_key = attachment["staging_key"] + original_key = f"chats/{attachment['chat_id']}/{attachment_id}" + content_type = attachment["content_type"] + chat_id = str(attachment["chat_id"]) + repo.mark_attachment_processing(conn, attachment_id) + + try: + data = storage.get_bytes(staging_key) + storage.put_object(original_key, io.BytesIO(data), content_type) + variants = _build_variants(chat_id, attachment_id, data) + for variant in variants: + storage.put_object(variant["s3_key"], io.BytesIO(variant["bytes"]), variant["content_type"]) + with pool().connection() as conn: + for variant in variants: + repo.upsert_variant( + conn, attachment_id, pipeline_version, variant["name"], + variant["content_type"], len(variant["bytes"]), + variant["width"], variant["height"], variant["sha256"], variant["s3_key"], + ) + repo.finalize_attachment_ready(conn, attachment_id, original_key, pipeline_version) + repo.complete_job(conn, job_id) + _cleanup_staging(staging_key) + logger.info("attachment %s ready (pipeline=%s)", attachment_id, pipeline_version) + except imaging.PermanentImageError as exc: + with pool().connection() as conn: + repo.mark_attachment_failed(conn, attachment_id, exc.code) + repo.fail_job(conn, job_id, exc.code) + logger.warning("attachment %s permanently failed: %s", attachment_id, exc.code) + except ApiError as exc: + _retry(attachment_id, job_id, attempt, exc.code) + except Exception as exc: # noqa: BLE001 - transient by default + _retry(attachment_id, job_id, attempt, type(exc).__name__) + + +def _retry(attachment_id: str, job_id: str, attempt: int, error: str) -> None: + if attempt >= settings.job_max_attempts: + with pool().connection() as conn: + repo.mark_attachment_failed(conn, attachment_id, "retry_exhausted") + repo.fail_job(conn, job_id, "retry_exhausted") + logger.warning("attachment %s exhausted %d attempts", attachment_id, attempt) + return + base = min(settings.retry_backoff_max_seconds, settings.retry_backoff_base_seconds * (2 ** (attempt - 1))) + delay = base * (0.5 + random.random()) # jitter + available_at = repo.utcnow() + timedelta(seconds=delay) + with pool().connection() as conn: + repo.requeue_job(conn, job_id, available_at, error) + repo.mark_attachment_queued(conn, attachment_id) + logger.info("attachment %s requeued (attempt %d -> %d, in %.2fs, %s)", + attachment_id, attempt, attempt + 1, delay, error) + + +def _cleanup_staging(staging_key: str) -> None: + # Remove the staging copy only after the result is durably committed. A + # redelivery of a ready job is a no-op, so deleting is safe here. + if staging_key and storage.delete_object(staging_key): + logger.debug("removed staging object %s", staging_key) + + +def _valid(job: dict) -> bool: + return ( + isinstance(job, dict) + and job.get("schema_version") == 1 + and job.get("type") == "attachment.process.v1" + and isinstance(job.get("job_id"), str) + and isinstance(job.get("attachment_id"), str) + and int(job.get("pipeline_version", 0)) >= 1 + ) + + +def _on_message(channel, method, properties, body) -> None: + try: + job = json.loads(body) + except (ValueError, UnicodeError): + logger.warning("dropping malformed job message") + channel.basic_ack(delivery_tag=method.delivery_tag) + return + if not _valid(job): + logger.warning("dropping job with unexpected schema") + channel.basic_ack(delivery_tag=method.delivery_tag) + return + logger.info("received job %s (attempt %s)", job.get("job_id"), job.get("attempt")) + try: + _process(job) + except Exception as exc: # noqa: BLE001 - never kill the consumer + logger.warning("job %s handling error: %s", job.get("job_id"), type(exc).__name__) + # The durable intent (ready / scheduled retry / failed) lives in PostgreSQL, + # so the message is always acked; redelivery is a safe no-op. + channel.basic_ack(delivery_tag=method.delivery_tag) + + +def _consume_loop(index: int) -> None: + while not _stop.is_set(): + try: + connection = broker.connect() + channel = connection.channel() + broker.declare(channel) + channel.basic_qos(prefetch_count=1) + channel.basic_consume( + queue=settings.broker_queue, on_message_callback=_on_message, auto_ack=False + ) + logger.info("worker consumer %d ready", index) + while not _stop.is_set(): + connection.process_data_events(time_limit=1) + except Exception as exc: # noqa: BLE001 - reconnect + logger.warning("worker consumer %d broker issue: %s", index, type(exc).__name__) + time.sleep(2) + finally: + try: + connection.close() + except Exception: # noqa: BLE001 + pass + + +def _lease_loop() -> None: + while not _stop.is_set(): + try: + with pool().connection() as conn: + recovered = repo.requeue_expired_leases(conn) + if recovered: + logger.warning("recovered %d expired lease(s)", len(recovered)) + except Exception as exc: # noqa: BLE001 + logger.warning("lease recovery failed: %s", type(exc).__name__) + _stop.wait(settings.lease_recovery_interval_seconds) + + +def _handle_signal(signum, frame=None): # noqa: ANN001 + logger.info("worker received signal %s; draining", signum) + _stop.set() + + +def main() -> None: + signal.signal(signal.SIGTERM, _handle_signal) + signal.signal(signal.SIGINT, _handle_signal) + logger.info("worker starting (concurrency=%d, queue=%s)", + settings.worker_concurrency, settings.broker_queue) + + threading.Thread(target=_lease_loop, name="lease-recovery", daemon=True).start() + consumers = [ + threading.Thread(target=_consume_loop, args=(i,), name=f"consumer-{i}", daemon=True) + for i in range(settings.worker_concurrency) + ] + for thread in consumers: + thread.start() + try: + while not _stop.is_set(): + time.sleep(1) + finally: + for thread in consumers: + thread.join(timeout=2) + close_pool() + logger.info("worker stopped") + + +if __name__ == "__main__": + main() diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..17447d63 --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,67 @@ +"""Student tests run against an already-running compose stand. + +Start the stand first (secrets + `docker compose up -d --build`), then: + + .venv\\Scripts\\python -m pytest student-tests -v + # or: make test-student PYTHON=.venv/bin/python + +Override the target with BASE_URL (default http://localhost:8080). +""" +from __future__ import annotations + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080") + + +@pytest.fixture(scope="session") +def base_url() -> str: + return BASE_URL.rstrip("/") + + +@pytest.fixture(scope="session") +def client(base_url): + from http_client import Client + + return Client(base_url) + + +def _register_and_login(client, label: str) -> dict: + suffix = os.urandom(4).hex() + username = f"s_{label}_{suffix}" + password = "T3st-" + os.urandom(8).hex() + registered = client.register(username, password, "Студент " + label) + assert registered.status == 201, registered.body + user = registered.json + session = client.login(username, password) + assert session.status == 201, session.body + payload = session.json + return { + "id": user["id"], + "username": username, + "password": password, + "user": user, + "token": payload["token"], + } + + +@pytest.fixture(scope="session") +def alice(client): + return _register_and_login(client, "alice") + + +@pytest.fixture(scope="session") +def bob(client): + return _register_and_login(client, "bob") + + +@pytest.fixture(scope="session") +def chat(client, alice, bob): + response = client.create_chat(alice["token"], [alice["id"], bob["id"]]) + assert response.status == 201, response.body + return response.json diff --git a/student-tests/http_client.py b/student-tests/http_client.py new file mode 100644 index 00000000..0622ffef --- /dev/null +++ b/student-tests/http_client.py @@ -0,0 +1,148 @@ +"""Minimal stdlib HTTP client for student black-box tests (no extra deps).""" +from __future__ import annotations + +import base64 +import json +import os +import urllib.error +import urllib.request +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class Response: + status: int + headers: object + body: bytes + + @property + def json(self): + if not self.body: + return None + return json.loads(self.body.decode("utf-8")) + + @property + def content_type(self) -> str: + return self.headers.get_content_type() + + @property + def request_id(self) -> Optional[str]: + return self.headers.get("X-Request-Id") + + @property + def instance_id(self) -> Optional[str]: + return self.headers.get("X-Instance-Id") + + +class Client: + def __init__(self, base_url: str): + self.base = base_url.rstrip("/") + + def request( + self, + method: str, + path: str, + body=None, + raw: Optional[bytes] = None, + token: Optional[str] = None, + basic: Optional[str] = None, + headers: Optional[dict] = None, + ) -> Response: + data = raw + request_headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if raw is not None: + request_headers.setdefault("Content-Type", "application/json") + if token is not None: + request_headers["Authorization"] = "Bearer " + token + if basic is not None: + request_headers["Authorization"] = "Basic " + basic + if headers: + request_headers.update(headers) + request = urllib.request.Request( + self.base + path, data=data, headers=request_headers, method=method + ) + try: + with urllib.request.urlopen(request, timeout=15) as response: + return Response(response.status, response.headers, response.read()) + except urllib.error.HTTPError as error: + return Response(error.code, error.headers, error.read()) + + # -- convenience -------------------------------------------------------- + def register(self, username: str, password: str, display_name: str = "Студент") -> Response: + return self.request( + "POST", + "/api/v1/users", + body={"username": username, "display_name": display_name, "password": password}, + ) + + def login(self, username: str, password: str) -> Response: + credential = base64.b64encode(f"{username}:{password}".encode("utf-8")).decode() + return self.request("POST", "/api/v1/auth/sessions", basic=credential) + + def me(self, token: str) -> Response: + return self.request("GET", "/api/v1/users/me", token=token) + + def create_chat(self, token: str, member_ids, title: str = "Чат") -> Response: + return self.request( + "POST", + "/api/v1/chats", + body={"title": title, "member_ids": list(member_ids)}, + token=token, + ) + + def send( + self, + token: str, + chat_id: str, + text: str, + key: Optional[str] = None, + headers: Optional[dict] = None, + attachments=None, + ) -> Response: + # lab3 requires an Idempotency-Key on every message POST. + request_headers = dict(headers or {}) + request_headers.setdefault("Idempotency-Key", key or os.urandom(16).hex()) + body = {"text": text} + if attachments is not None: + body["attachment_ids"] = list(attachments) + return self.request( + "POST", + f"/api/v1/chats/{chat_id}/messages", + body=body, + token=token, + headers=request_headers, + ) + + def upload_attachment(self, token: str, chat_id: str, content: bytes, filename: str = "sample.png", + content_type: str = "image/png") -> Response: + return self.request( + "POST", + f"/api/v1/chats/{chat_id}/attachments?filename={filename}", + raw=content, + token=token, + headers={"Content-Type": content_type}, + ) + + def get_attachment(self, token: str, attachment_id: str) -> Response: + return self.request("GET", f"/api/v1/attachments/{attachment_id}", token=token) + + def download(self, token: str, attachment_id: str, variant: Optional[str] = None) -> Response: + path = f"/api/v1/attachments/{attachment_id}/content" + if variant is not None: + path += f"?variant={variant}" + return self.request("GET", path, token=token) + + def list_messages(self, token: str, chat_id: str, limit: int = 50, cursor=None) -> Response: + path = f"/api/v1/chats/{chat_id}/messages?limit={limit}" + if cursor is not None: + from urllib.parse import quote + + path += "&cursor=" + quote(cursor, safe="") + return self.request("GET", path, token=token) + + def logout(self, token: str) -> Response: + return self.request("DELETE", "/api/v1/auth/sessions/current", token=token) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..e7c30c9e --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,133 @@ +"""Attachment tests for lab5: async accept (202), variants, access control. + +Upload now returns ``202 queued``; readiness is polled via GET metadata. +""" +from __future__ import annotations + +import hashlib +import os +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SAMPLE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def _assert_error(response, status, code): + assert response.status == status, (response.status, response.body) + assert response.content_type == "application/json" + payload = response.json + assert payload["error"]["code"] == code + assert payload["error"]["request_id"] == response.request_id + + +def _fresh_actor(client, label): + from conftest import _register_and_login + + return _register_and_login(client, label) + + +def _peer_chat(client, actor): + peer = _fresh_actor(client, "att_peer") + response = client.create_chat(actor["token"], [actor["id"], peer["id"]], title="Attachment chat") + assert response.status == 201, response.body + return response.json, peer + + +def wait_ready(client, token, attachment_id, timeout=60): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + last = client.get_attachment(token, attachment_id).json + if last["status"] == "failed": + raise AssertionError(f"attachment failed: {last.get('error_code')}") + if last["status"] == "ready": + return last + time.sleep(0.3) + raise AssertionError(f"attachment not ready in time: {last}") + + +def test_upload_is_202_queued_then_ready(client, alice, bob, chat): + upload = client.upload_attachment(alice["token"], chat["id"], SAMPLE, filename="sample.png") + assert upload.status == 202, upload.body + att = upload.json + assert att["status"] == "queued" + assert att["error_code"] is None + assert att["variants"] == [] + assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"] + assert att["content_type"] == "image/png" + assert att["size_bytes"] == len(SAMPLE) + assert att["sha256"] == hashlib.sha256(SAMPLE).hexdigest() + + ready = wait_ready(client, bob["token"], att["id"]) + assert ready["status"] == "ready" and ready["error_code"] is None + names = {v["name"] for v in ready["variants"]} + assert {"thumbnail", "crop", "optimized"} <= names, names + + download = client.download(bob["token"], att["id"]) + assert download.status == 200, download.body + assert download.body == SAMPLE + assert download.content_type == "image/png" + + message = client.send(alice["token"], chat["id"], "", attachments=[att["id"]]) + assert message.status == 201, message.body + assert message.json["attachment_ids"] == [att["id"]] + + +def test_content_409_while_queued(client, alice, chat): + # A fresh attachment is queued/processing for at least a moment; if the + # worker is fast it may already be ready, so accept either 409 or 200. + att = client.upload_attachment(alice["token"], chat["id"], SAMPLE).json + response = client.download(alice["token"], att["id"]) + assert response.status in (200, 409), response.status + if response.status == 409: + _assert_error(response, 409, "conflict") + + +def test_foreign_actor_gets_404(client, alice, bob, chat): + att = client.upload_attachment(alice["token"], chat["id"], SAMPLE).json + stranger = _fresh_actor(client, "foreign") + _assert_error(client.get_attachment(stranger["token"], att["id"]), 404, "not_found") + _assert_error(client.download(stranger["token"], att["id"]), 404, "not_found") + + +def test_cross_chat_attachment_is_404(client, alice, bob, chat): + other_chat, _peer = _peer_chat(client, alice) + foreign = client.upload_attachment(alice["token"], other_chat["id"], SAMPLE).json + response = client.send(alice["token"], chat["id"], "", attachments=[foreign["id"]]) + _assert_error(response, 404, "not_found") + + +def test_oversized_upload_is_413(client, alice, chat): + too_big = b"\x89PNG\r\n\x1a\n" + b"\x00" * (10 * 1024 * 1024 + 1) + _assert_error(client.upload_attachment(alice["token"], chat["id"], too_big), 413, "payload_too_large") + + +def test_corrupt_image_is_415(client, alice, chat): + corrupt = b"\x89PNG\r\n\x1a\n" + os.urandom(256) + _assert_error(client.upload_attachment(alice["token"], chat["id"], corrupt), 415, "unsupported_media_type") + + +def test_mime_spoof_is_415(client, alice, chat): + spoof = b"GIF89a" + os.urandom(64) + _assert_error( + client.upload_attachment(alice["token"], chat["id"], spoof, content_type="image/png"), + 415, + "unsupported_media_type", + ) + + +def test_empty_upload_is_415(client, alice, chat): + _assert_error(client.upload_attachment(alice["token"], chat["id"], b""), 415, "unsupported_media_type") + + +def test_unsafe_filename_is_400(client, alice, chat): + _assert_error( + client.upload_attachment(alice["token"], chat["id"], SAMPLE, filename="a/b.png"), + 400, + "invalid_request", + ) + + +def test_empty_text_without_attachment_is_400(client, alice, chat): + _assert_error(client.send(alice["token"], chat["id"], ""), 400, "invalid_request") diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py new file mode 100644 index 00000000..646db383 --- /dev/null +++ b/student-tests/test_auth.py @@ -0,0 +1,122 @@ +"""Authentication and registration tests (lab2 auth, unchanged in lab3).""" +from __future__ import annotations + +import base64 +import os + + +def _assert_error(response, status, code, challenge_prefix=None): + assert response.status == status, (response.status, response.body) + assert response.content_type == "application/json" + payload = response.json + assert payload["error"]["code"] == code + assert response.request_id, "missing X-Request-Id" + assert payload["error"]["request_id"] == response.request_id + if challenge_prefix: + header = response.headers.get("WWW-Authenticate", "") + assert header.lower().startswith(challenge_prefix.lower()), header + + +def test_registration_requires_password(client): + response = client.request( + "POST", "/api/v1/users", body={"username": "no_pw_" + os.urandom(3).hex(), "display_name": "X"} + ) + _assert_error(response, 400, "invalid_request") + + +def test_short_password_is_400(client): + response = client.register("shortpw_" + os.urandom(3).hex(), "too-short") + _assert_error(response, 400, "invalid_request") + + +def test_extra_field_is_400(client): + response = client.request( + "POST", + "/api/v1/users", + body={ + "username": "extra_" + os.urandom(3).hex(), + "display_name": "X", + "password": "T3st-" + os.urandom(8).hex(), + "role": "admin", + }, + ) + _assert_error(response, 400, "invalid_request") + + +def test_malformed_json_is_400(client): + response = client.request("POST", "/api/v1/users", raw=b"{not json") + _assert_error(response, 400, "invalid_request") + + +def test_duplicate_username_is_409(client): + username = "dup_" + os.urandom(4).hex() + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + _assert_error(client.register(username, password), 409, "conflict") + + +def test_registration_never_returns_password(client): + response = client.register("pw_hidden_" + os.urandom(3).hex(), "T3st-" + os.urandom(8).hex()) + assert response.status == 201, response.body + assert "password" not in response.json + + +def test_wrong_password_is_401_basic(client, alice): + _assert_error( + client.login(alice["username"], "definitely-wrong-password"), + 401, + "unauthorized", + challenge_prefix="Basic", + ) + + +def test_unknown_user_is_401_basic(client): + _assert_error( + client.login("ghost_" + os.urandom(4).hex(), "T3st-" + os.urandom(8).hex()), + 401, + "unauthorized", + challenge_prefix="Basic", + ) + + +def test_basic_on_business_endpoint_is_401_bearer(client, alice): + credential = base64.b64encode(f"{alice['username']}:{alice['password']}".encode()).decode() + response = client.request("GET", "/api/v1/users/me", basic=credential) + _assert_error(response, 401, "unauthorized", challenge_prefix="Bearer") + + +def test_missing_auth_is_401_bearer(client): + _assert_error(client.request("GET", "/api/v1/users/me"), 401, "unauthorized", challenge_prefix="Bearer") + + +def test_unknown_bearer_is_401(client): + _assert_error( + client.me("not-a-real-token-" + os.urandom(8).hex()), + 401, + "unauthorized", + challenge_prefix="Bearer", + ) + + +def test_x_user_id_is_ignored(client, alice, bob): + only_header = client.request("GET", "/api/v1/users/me", headers={"X-User-Id": alice["id"]}) + _assert_error(only_header, 401, "unauthorized", challenge_prefix="Bearer") + response = client.request( + "GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]} + ) + assert response.status == 200, response.body + assert response.json["id"] == alice["id"] + + +def test_logout_revokes_session(client): + suffix = os.urandom(4).hex() + username = f"s_out_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + assert client.me(token).status == 200 + logout = client.logout(token) + assert logout.status == 204, logout.body + assert not logout.body + _assert_error(client.me(token), 401, "unauthorized", challenge_prefix="Bearer") + _assert_error(client.logout(token), 401, "unauthorized", challenge_prefix="Bearer") diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..a6506194 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,37 @@ +"""Concurrency: 100 parallel unique messages survive without loss.""" +from __future__ import annotations + +import concurrent.futures +import os + + +def _register(client, label): + username = f"s_race_{label}_{os.urandom(4).hex()}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + return {"id": client.me(token).json["id"], "token": token} + + +def test_100_concurrent_messages_are_not_lost(client, alice): + senders = [_register(client, str(i)) for i in range(10)] + members = [alice["id"]] + [sender["id"] for sender in senders] + chat = client.create_chat(alice["token"], members).json + + def worker(task): + sender, index = task + response = client.send(sender["token"], chat["id"], f"race-{index}") + assert response.status == 201, response.body + return response.json["id"] + + tasks = [(senders[index // 10], index) for index in range(100)] + with concurrent.futures.ThreadPoolExecutor(max_workers=25) as pool: + created = list(pool.map(worker, tasks)) + + assert len(created) == 100 + assert len(set(created)) == 100, "duplicate message ids" + history = client.list_messages(alice["token"], chat["id"], limit=100).json + stored = [item["id"] for item in history["items"]] + assert len(stored) == 100, f"stored {len(stored)} of 100" + assert len(set(stored)) == 100 + assert set(stored) == set(created) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..9e667e5e --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,65 @@ +"""Lab3 shared idempotency: replay, conflict and cross-replica concurrency.""" +from __future__ import annotations + +import concurrent.futures +import json +import uuid + + +def test_replay_same_key_returns_same_message(client, alice, chat): + key = uuid.uuid4().hex + first = client.send(alice["token"], chat["id"], "идемпотентно", key=key) + assert first.status == 201, first.body + replay = client.send(alice["token"], chat["id"], "идемпотентно", key=key) + assert replay.status == 201, replay.body + assert replay.json == first.json, "replay must return the same Message" + + +def test_same_key_different_body_is_409(client, alice, chat): + key = uuid.uuid4().hex + assert client.send(alice["token"], chat["id"], "первый", key=key).status == 201 + conflict = client.send(alice["token"], chat["id"], "второй", key=key) + assert conflict.status == 409, conflict.body + assert conflict.json["error"]["code"] == "conflict" + + +def test_missing_key_is_400(client, alice, chat): + response = client.request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "без ключа"}, + token=alice["token"], + ) + assert response.status == 400, response.body + assert response.json["error"]["code"] == "invalid_request" + + +def test_invalid_key_is_400(client, alice, chat): + for bad in ("bad key", "has/slash", "x" * 129, "with$sign"): + response = client.request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "плохой ключ"}, + token=alice["token"], + headers={"Idempotency-Key": bad}, + ) + assert response.status == 400, (bad, response.status, response.body) + + +def test_concurrent_same_key_creates_exactly_one_message(client, alice, chat): + key = uuid.uuid4().hex + text = "конкурентный повтор " + uuid.uuid4().hex + + def send(_): + return client.send(alice["token"], chat["id"], text, key=key) + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + responses = list(pool.map(send, range(8))) + + assert all(r.status == 201 for r in responses), [r.status for r in responses] + bodies = {json.dumps(r.json, sort_keys=True) for r in responses} + assert len(bodies) == 1, "parallel replays returned different messages" + + history = client.list_messages(alice["token"], chat["id"], limit=100).json["items"] + matching = [m for m in history if m["text"] == text] + assert len(matching) == 1, f"expected exactly one stored message, got {len(matching)}" diff --git a/student-tests/test_instances.py b/student-tests/test_instances.py new file mode 100644 index 00000000..d8cf51ef --- /dev/null +++ b/student-tests/test_instances.py @@ -0,0 +1,23 @@ +"""Lab3 balancing: X-Instance-Id is present and more than one replica serves.""" +from __future__ import annotations + + +def test_instance_header_present_on_success(client, alice): + response = client.me(alice["token"]) + assert response.status == 200, response.body + assert response.instance_id, "missing X-Instance-Id" + + +def test_instance_header_present_on_error(client): + response = client.request("GET", "/api/v1/users/me") + assert response.status == 401, response.body + assert response.instance_id, "error response missing X-Instance-Id" + + +def test_at_least_two_instances_seen_through_lb(client, alice): + seen = set() + for _ in range(60): + instance = client.me(alice["token"]).instance_id + if instance: + seen.add(instance) + assert len(seen) >= 2, f"expected >=2 API replicas behind the LB, saw {sorted(seen)}" diff --git a/student-tests/test_negative.py b/student-tests/test_negative.py new file mode 100644 index 00000000..a5af2237 --- /dev/null +++ b/student-tests/test_negative.py @@ -0,0 +1,65 @@ +"""Negative and access-control tests (carried over, lab3 keeps them).""" +from __future__ import annotations + +import os + + +def _assert_error(response, status, code): + assert response.status == status, (response.status, response.body) + payload = response.json + assert payload["error"]["code"] == code + assert payload["error"]["request_id"] == response.request_id + + +def _fresh_actor(client, label="peer"): + suffix = os.urandom(4).hex() + username = f"s_{label}_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + return {"id": client.me(token).json["id"], "token": token} + + +def test_foreign_chat_is_404(client, alice, chat): + stranger = _fresh_actor(client, "stranger") + assert client.request("GET", f"/api/v1/chats/{chat['id']}", token=stranger["token"]).status == 404 + _assert_error(client.send(stranger["token"], chat["id"], "чужое"), 404, "not_found") + _assert_error(client.list_messages(stranger["token"], chat["id"]), 404, "not_found") + + +def test_empty_message_is_400(client, alice, chat): + for text in ("", " ", "\t\n"): + _assert_error(client.send(alice["token"], chat["id"], text), 400, "invalid_request") + + +def test_oversized_message_is_400(client, alice, chat): + _assert_error(client.send(alice["token"], chat["id"], "x" * 4001), 400, "invalid_request") + + +def test_limit_out_of_range_is_400(client, alice, chat): + _assert_error(client.list_messages(alice["token"], chat["id"], limit=0), 400, "invalid_request") + _assert_error(client.list_messages(alice["token"], chat["id"], limit=101), 400, "invalid_request") + + +def test_sender_id_in_body_is_rejected(client, alice, bob, chat): + response = client.request( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + body={"text": "привет", "sender_id": bob["id"]}, + token=alice["token"], + headers={"Idempotency-Key": os.urandom(12).hex()}, + ) + _assert_error(response, 400, "invalid_request") + + +def test_chat_without_caller_is_400(client, bob, alice): + _assert_error(client.create_chat(bob["token"], [alice["id"]]), 400, "invalid_request") + + +def test_chat_with_unknown_member_is_400(client, alice): + unknown = "00000000-0000-4000-8000-000000000000" + _assert_error(client.create_chat(alice["token"], [alice["id"], unknown]), 400, "invalid_request") + + +def test_chat_with_duplicate_members_is_400(client, alice): + _assert_error(client.create_chat(alice["token"], [alice["id"], alice["id"]]), 400, "invalid_request") diff --git a/student-tests/test_pagination.py b/student-tests/test_pagination.py new file mode 100644 index 00000000..18e7188b --- /dev/null +++ b/student-tests/test_pagination.py @@ -0,0 +1,79 @@ +"""Keyset pagination tests (carried over from lab2).""" +from __future__ import annotations + +import base64 +import json +import os + + +def _fake_cursor(payload) -> str: + raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _fresh_chat(client, alice): + suffix = os.urandom(4).hex() + username = f"s_page_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + peer_token = client.login(username, password).json["token"] + peer_id = client.me(peer_token).json["id"] + response = client.create_chat(alice["token"], [alice["id"], peer_id]) + assert response.status == 201, response.body + return response.json + + +def test_full_traversal_without_duplicates(client, alice): + chat = _fresh_chat(client, alice) + expected = [] + for index in range(5): + response = client.send(alice["token"], chat["id"], f"msg-{index}") + assert response.status == 201, response.body + expected.append(response.json["id"]) + + seen = [] + cursor = None + while True: + page = client.list_messages(alice["token"], chat["id"], limit=2, cursor=cursor).json + assert len(page["items"]) <= 2 + seen.extend(item["id"] for item in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + assert len(seen) == 5 + assert len(set(seen)) == 5, "duplicate across pages" + assert seen == expected, "pagination reordered history" + + +def test_cursor_from_other_chat_is_400(client, alice): + first = _fresh_chat(client, alice) + second = _fresh_chat(client, alice) + for index in range(3): + client.send(alice["token"], first["id"], f"a{index}") + client.send(alice["token"], second["id"], f"b{index}") + cursor = client.list_messages(alice["token"], first["id"], limit=1).json["next_cursor"] + assert cursor + foreign = client.list_messages(alice["token"], second["id"], limit=1, cursor=cursor) + assert foreign.status == 400, foreign.body + assert foreign.json["error"]["code"] == "invalid_request" + + +def test_garbage_cursor_is_400(client, alice): + chat = _fresh_chat(client, alice) + client.send(alice["token"], chat["id"], "hello") + response = client.list_messages(alice["token"], chat["id"], cursor="!!!not-a-cursor!!!") + assert response.status == 400, response.body + + +def test_well_formed_cursor_with_wrong_types_is_400(client, alice): + bad_payloads = [ + {"r": "chats", "t": "2026-01-01T00:00:00.000000Z", "i": 1}, + {"r": "chats", "t": "2026-01-01T00:00:00.000000Z", "i": []}, + {"r": "chats", "t": "2026-01-01T00:00:00.000000Z", "i": True}, + {"r": "chats", "t": "not-a-time", "i": "11111111-1111-4111-8111-111111111111"}, + ] + for payload in bad_payloads: + response = client.request( + "GET", "/api/v1/chats?cursor=" + _fake_cursor(payload), token=alice["token"] + ) + assert response.status == 400, (payload, response.status, response.body) diff --git a/student-tests/test_pipeline.py b/student-tests/test_pipeline.py new file mode 100644 index 00000000..8afe95be --- /dev/null +++ b/student-tests/test_pipeline.py @@ -0,0 +1,74 @@ +"""Variant pipeline tests (lab5): thumbnail/crop/optimized correctness.""" +from __future__ import annotations + +from pathlib import Path + +from test_attachments import SAMPLE, wait_ready + +ROOT = Path(__file__).resolve().parents[1] + + +def jpeg_size(data: bytes): + assert data[:2] == b"\xff\xd8", "not JPEG" + i = 2 + sof = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} + while i < len(data): + assert data[i] == 0xFF, "invalid JPEG marker" + while i < len(data) and data[i] == 0xFF: + i += 1 + marker = data[i] + i += 1 + if marker in {0xD9, 0xDA}: + break + if marker in {0x01, 0xD8} or 0xD0 <= marker <= 0xD7: + continue + length = int.from_bytes(data[i : i + 2], "big") + if marker in sof: + return (int.from_bytes(data[i + 5 : i + 7], "big"), int.from_bytes(data[i + 3 : i + 5], "big")) + i += length + raise AssertionError("JPEG dimensions not found") + + +def _variant(ready, name): + matches = [v for v in ready["variants"] if v["name"] == name] + assert len(matches) == 1, (name, ready["variants"]) + return matches[0] + + +def test_variants_match_contract(client, alice, bob, chat): + import hashlib + + att = client.upload_attachment(alice["token"], chat["id"], SAMPLE).json + ready = wait_ready(client, bob["token"], att["id"]) + assert len(ready["variants"]) == 3 + assert {v["name"] for v in ready["variants"]} == {"thumbnail", "crop", "optimized"} + + thumb = _variant(ready, "thumbnail") + assert (thumb["width"], thumb["height"]) == (256, 160), thumb + assert thumb["content_type"] == "image/jpeg" + body = client.download(bob["token"], att["id"], variant="thumbnail") + assert body.content_type == "image/jpeg" + assert len(body.body) == thumb["size_bytes"] + assert hashlib.sha256(body.body).hexdigest() == thumb["sha256"] + assert jpeg_size(body.body) == (256, 160) + + crop = _variant(ready, "crop") + assert (crop["width"], crop["height"]) == (128, 128), crop + crop_body = client.download(bob["token"], att["id"], variant="crop").body + assert jpeg_size(crop_body) == (128, 128) + + optimized = _variant(ready, "optimized") + assert optimized["content_type"] == "image/webp" + opt_body = client.download(bob["token"], att["id"], variant="optimized").body + assert opt_body[:4] == b"RIFF" and opt_body[8:12] == b"WEBP", "not WebP" + + # original remains byte-identical + assert client.download(bob["token"], att["id"]).body == SAMPLE + + +def test_missing_variant_is_404(client, alice, bob, chat): + from test_attachments import _assert_error + + att = client.upload_attachment(alice["token"], chat["id"], SAMPLE).json + wait_ready(client, bob["token"], att["id"]) + _assert_error(client.download(bob["token"], att["id"], variant="unknown"), 404, "not_found") diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 00000000..a095feb0 --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,34 @@ +"""Grade-4 integrity checks: registration race and composite-op rollback.""" +from __future__ import annotations + +import concurrent.futures +import os + + +def test_concurrent_duplicate_username_single_winner(client): + username = "race_dup_" + os.urandom(4).hex() + password = "T3st-" + os.urandom(8).hex() + + def register(_): + return client.register(username, password).status + + with concurrent.futures.ThreadPoolExecutor(max_workers=5) as pool: + statuses = list(pool.map(register, range(5))) + + assert statuses.count(201) == 1, statuses + assert statuses.count(409) == 4, statuses + + +def test_failed_chat_creation_rolls_back(client, alice): + def chats_count(): + return len(client.request("GET", "/api/v1/chats?limit=100", token=alice["token"]).json["items"]) + + before = chats_count() + unknown = "00000000-0000-4000-8000-000000000000" + response = client.create_chat(alice["token"], [alice["id"], unknown], title="should-rollback") + assert response.status == 400, response.body + after = chats_count() + assert after == before, "failed composite operation left a partial chat" + + listed = client.request("GET", "/api/v1/chats?limit=100", token=alice["token"]).json["items"] + assert all(item["title"] != "should-rollback" for item in listed) diff --git a/student-tests/test_session_revocation.py b/student-tests/test_session_revocation.py new file mode 100644 index 00000000..e5c56547 --- /dev/null +++ b/student-tests/test_session_revocation.py @@ -0,0 +1,34 @@ +"""Lab3 revocation: logout takes effect immediately on every replica. + +Session validity is decided only by PostgreSQL, so a revoked token must be +rejected regardless of which replica answers and regardless of the cache state. +The heavier Valkey-outage reproduction lives in scripts/check-revocation.py. +""" +from __future__ import annotations + +import os + + +def _actor(client, label="revoke"): + suffix = os.urandom(4).hex() + username = f"s_{label}_{suffix}" + password = "T3st-" + os.urandom(8).hex() + assert client.register(username, password).status == 201 + token = client.login(username, password).json["token"] + return {"id": client.me(token).json["id"], "token": token} + + +def test_logout_invalidates_across_replicas(client): + actor = _actor(client) + seen = set() + for _ in range(20): + response = client.me(actor["token"]) + assert response.status == 200, response.body + seen.add(response.instance_id) + assert len(seen) >= 2, f"expected the LB to spread requests, saw {sorted(seen)}" + + logout = client.logout(actor["token"]) + assert logout.status == 204, logout.body + + for _ in range(20): + assert client.me(actor["token"]).status == 401 -- 2.54.0