diff --git a/.env.example b/.env.example index d99d26a2..a36e6092 100644 --- a/.env.example +++ b/.env.example @@ -18,3 +18,5 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# The public LB port for the local Compose stand (avoid lab3's default 8081). +API_PUBLISH_PORT=8083 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f58f0fbf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,18 @@ +# Shell scripts and configs must stay LF so they run in Linux containers. +*.sh text eol=lf +*.conf text eol=lf +nginx.conf text eol=lf +Dockerfile text eol=lf +dockerignore text eol=lf +compose.yaml text eol=lf +compose.example.yaml text eol=lf +entrypoint.sh text eol=lf + +# Python and text files are fine with platform-native line endings. +*.py text +*.md text +*.json text +*.txt text +*.sql text +.env.example text +Makefile text diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..6c31cd3d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,17 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Тимофей Александров, БВТ2301 +Целевая оценка (3/4/5): 5 +Трек 5: «Согласованность двух хранилищ» -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: две API-реплики, LB, PostgreSQL, durable sessions и scoped idempotency перенесены из lab3. Добавлены private SeaweedFS S3 bucket, PostgreSQL metadata, binary PNG/JPEG upload/download и связи attachment-message. Отдельный `s3-init` создаёт bucket bootstrap-учёткой; API имеет только data-plane credential. Reconciler трека 5 выполняет dry-run/apply orphan cleanup с grace period и сообщает missing objects. +Команды запуска из чистого clone: `sh scripts/bootstrap-secrets.sh && docker compose --env-file .env up -d --build --wait --wait-timeout 240`. +Результаты проверок и ссылка на REPORT.md: public smoke PASS; attachment validation PASS 3/3; persistence PASS; failover PASS (96 writes, 0 failures); concurrent 10×8 MiB uploads PASS (peak 273.10 MiB); reconciler evidence — `evidence/reconcile.txt`, `evidence/reconcile-missing.json`. Подробности: `REPORT.md`. +Ограничения / известные ошибки: Docker host, LB, PostgreSQL и S3 не реплицированы; missing object only reported, not reconstructed. +Источники и использование ИИ: SeaweedFS S3, MinIO SDK, Pillow, PostgreSQL и материалы задания; ИИ использован для проектирования/кода/тестов, автор проверяет результат. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Проверки и сценарии отказа воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2a2e6f86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# syntax=docker/dockerfile:1 +# Runtime image is pinned by digest, dependencies by version in requirements.txt. +FROM python:3.12.7-slim-bookworm@sha256:60d9996b6a8a3689d36db740b49f4327be3be09a21122bd02fb8895abb38b50d + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +# Unprivileged runtime user; the API never needs to write to the image. +RUN groupadd --system --gid 10001 messenger \ + && useradd --system --uid 10001 --gid messenger --no-create-home messenger + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir --requirement /app/requirements.txt + +COPY --chown=messenger:messenger src/ /app/src/ +COPY --chown=messenger:messenger contracts/ /app/contracts/ +COPY --chown=messenger:messenger --chmod=0555 entrypoint.sh /app/entrypoint.sh +# Windows hosts may store CRLF; Python and /bin/sh need LF. +RUN find /app/src -name '*.py' -print0 | xargs -0 sed -i 's/\r$//' \ + && sed -i 's/\r$//' /app/entrypoint.sh + +USER messenger +EXPOSE 8080 +STOPSIGNAL SIGTERM + +# Readiness (not just liveness) so an unhealthy container is visible to Compose. +HEALTHCHECK --interval=5s --timeout=5s --start-period=20s --retries=3 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + +# Migrations are idempotent; the entrypoint execs the server so it is PID 1. +ENTRYPOINT ["/app/entrypoint.sh"] diff --git a/Makefile b/Makefile index 5d86d171..0f6d3bee 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,13 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-student reconcile attachment-load reconcile-experiment help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "reconcile: dry-run S3/PostgreSQL consistency check; set RECONCILE_ARGS=--apply deliberately" + @echo "test-student: functional, race, idempotency and attachment negative tests" + @echo "attachment-load: >=10 concurrent uploads plus sampled API RAM; reconcile-experiment: injected missing S3 object" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +31,15 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +reconcile: + sh scripts/reconcile.sh $(RECONCILE_ARGS) + +test-student: + cd student-tests && BASE_URL="$(BASE_URL)" $(PYTHON) -B -m unittest -v test_api test_race test_idempotency test_attachments + +attachment-load: + $(PYTHON) scripts/attachment_load.py --base-url "$(BASE_URL)" --evidence evidence/attachment-load.json + +reconcile-experiment: + $(PYTHON) scripts/reconcile_experiment.py --base-url "$(BASE_URL)" --evidence evidence/reconcile-missing.json diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..594f62dd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,63 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Тимофей Александров, БВТ2301 +- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 2 «Согласованность двух хранилищ» +- Commit/PR: локальная ветка, перед публикацией заполнить hash/ссылку +- Использованные источники, библиотеки и помощь ИИ: документация SeaweedFS S3 API, MinIO Python SDK, Pillow, PostgreSQL, README/COURSE/contracts текущей и предыдущей лабораторных; ИИ использовался для проектирования, реализации, тестов и отчёта. Автор проверяет и объясняет решение. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +```sh +sh scripts/bootstrap-secrets.sh +docker compose --env-file .env up -d --build --wait --wait-timeout 240 +PYTHONUTF8=1 python3 tests/smoke.py --base-url http://127.0.0.1:8083 +PYTHONUTF8=1 python3 tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8083 +PYTHONUTF8=1 TARGET_INSTANCE_ID=api-1 python3 tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8083 +sh scripts/reconcile.sh --grace-seconds 3600 +python3 scripts/attachment_load.py --base-url http://127.0.0.1:8083 --concurrency 10 --payload-mib 8 --evidence evidence/attachment-load.json +python3 scripts/reconcile_experiment.py --base-url http://127.0.0.1:8083 --evidence evidence/reconcile-missing.json +``` + +Локальный Compose публикует только nginx на `127.0.0.1:8083`; API, PostgreSQL, Valkey и S3 не имеют опубликованных портов. `secrets/` и `.env` генерируются локально, не попадают в Git. Обычная остановка: `docker compose --env-file .env down`; намеренное удаление учебных данных: `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Сохранены две API-реплики lab3 за nginx, PostgreSQL как источник метаданных/сессий и Valkey как необязательный кэш. SeaweedFS 3.91 предоставляет приватный S3-совместимый bucket с persistent volume; отдельная S3 identity API создаётся bootstrap-скриптом. Оригинальные байты хранятся только в S3, API использует `SpooledTemporaryFile` с 1 MiB memory threshold и отправляет объект после проверки. + +`attachments` хранит chat, owner, server-generated key, filename, MIME, hash, размер и геометрию. `message_attachments` хранит упорядоченную связь; API проверяет membership, принадлежность attachment тому же чату и status до вставки сообщения. Upload сохраняет S3 object, затем metadata в одной DB-транзакции; при DB-ошибке выполняется best-effort delete S3 object. Reconciler компенсирует остаточные orphan-объекты. + +Bucket создаёт idempotent одноразовый сервис `s3-init` отдельной bootstrap-учёткой. У API-реплик только data-plane S3 credentials (`Read`, `Write`, `List`, `Tagging`), bootstrap secret им не монтируется. S3 не имеет published port; анонимные запросы не получают object. После запуска API лишь проверяет существование bucket, а не может создавать его. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `PYTHONUTF8=1 python tests/smoke.py --base-url http://127.0.0.1:8083` | PASS: upload, SHA-256 original, access control и attachment-only message | terminal output | +| Persistence | `tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | PASS: данные, сессия и original сохранены после restart API/DB/Valkey/S3 | terminal output | +| Failover | `tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | PASS: 96 successful writes, 0 failures, first recovery 2.344 s; original доступен через оставшуюся реплику | terminal output | +| Валидация вложений уровня 4 | `cd student-tests && BASE_URL=http://127.0.0.1:8083 python -m unittest -v test_attachments` | PASS 3/3: oversized, corrupt, MIME mismatch, чужой user и attachment другого chat | terminal output | +| ≥10 concurrent uploads и RAM | `scripts/attachment_load.py --concurrency 10 --payload-mib 8` | PASS: 10/10, 8 MiB каждый; sampled API peak 273.10 MiB | `evidence/attachment-load.json` | +| Private bucket вне API | anonymous S3 ListObjects из backend network | PASS: S3 вернул 403; порт S3 не published на host | `evidence/private-s3.txt` | +| Трек 5: orphan reconciler | `sh scripts/reconcile.sh [--apply] --grace-seconds N` | orphan найден в dry-run, не удалён до grace period, удалён только явным `--apply`; final scan clean | `evidence/reconcile.txt` | +| Трек 5: missing-object reconciler | `scripts/reconcile_experiment.py` | missing найден; `--apply` metadata не удалил; bytes восстановлены без пересоздания metadata | `evidence/reconcile-missing.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза.** Периодический полный обход metadata и server-owned S3 prefix безопасно обнаружит расхождения двух хранилищ; grace period и dry-run не позволят удалить свежую/подтверждённую загрузку, а missing metadata не будет удалена автоматически. + +Методика orphan: в действующий private bucket вручную добавлен synthetic orphan без строки PostgreSQL. Dry-run с grace 60 s нашёл 1 orphan, но `eligible=0`; после grace явный `--apply` удалил ровно его, следующий dry-run чист (`evidence/reconcile.txt`). + +Методика missing: скрипт создаёт isolate attachment через API, удаляет только его server-generated S3 key во внутренней Docker network, запускает dry-run и `--apply`, затем возвращает те же bytes. Наблюдение: `missing_metadata_objects=1`; download вернул `503 unavailable`; `--apply` удалил 0 metadata/orphan; после восстановления byte-for-byte download и final scan успешны (`evidence/reconcile-missing.json`). Стоимость обхода выводится как `elapsed_ms` вместе с количеством metadata/S3 objects; она относится только к локальному учебному набору и не экстраполируется на production. + +Нагрузочная методика: после restart обеих API запускаются 10 одновременных реальных PNG по 8 MiB (каждый превышает 1 MiB `SpooledTemporaryFile` memory threshold); суммарная память API sampled через `docker stats` до и во время запроса. Результат: 10/10, 25.312 s, 0.395 successful RPS, p50/p95/p99 24938/25312/25312 ms; baseline 158.34 MiB, peak 273.10 MiB, рост 114.76 MiB. Raw JSON: `evidence/attachment-load.json`. Это sampled container memory, а не точный per-request heap profile; число samples равно 11. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Stop одной API | nginx повторяет запрос на здоровую реплику; оригинал остаётся доступен | PASS, 96 writes / 0 failures, recovery 2.344 s | данные в PostgreSQL/S3 не затронуты | +| Restart стека | session, metadata и bytes сохраняются | PASS persistence | volumes сохранены | +| Orphan S3 object | dry-run находит; apply после grace удаляет только orphan | PASS: 1 → 0 orphan | нет удаления metadata/fresh objects | +| Missing S3 object | download возвращает controlled 503; reconcile сообщает missing; apply metadata не удаляет | PASS: 1 missing → restore → clean | metadata сохранена, байты восстановлены | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Один Docker host, nginx, PostgreSQL и SeaweedFS остаются SPOF; работа не заявляет переживание их потери. Upload до 10 MiB принимает файл во временное bounded storage API перед S3, это не путь для больших production files. Missing S3 object reconciler безопасно сообщает и сохраняет metadata для ручного восстановления; он не может восстановить утраченные байты. Reconciler и RAM sample показаны на малом локальном наборе, поэтому их абсолютные значения не являются production capacity claim. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..80331262 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,234 @@ +# Local laboratory stand for lab 4. +# Roles: api (2 replicas, student-implements), postgres (source of truth), +# Valkey cache, private SeaweedFS S3 and nginx (the public entrypoint). +# Secrets are generated into ./secrets by scripts/bootstrap-secrets.sh and are +# mounted as files, so no credential ever appears in this file or in `docker inspect`. +services: + api-1: + build: + context: . + image: mtusi-messenger:lab4 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_STORE: ${SESSION_STORE:-postgres} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-300} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-2} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-2} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + INSTANCE_ID: api-1 + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + interval: 5s + timeout: 5s + start_period: 20s + retries: 3 + restart: unless-stopped + networks: + - backend + + api-2: + build: + context: . + image: mtusi-messenger:lab4 + environment: + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_STORE: ${SESSION_STORE:-postgres} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-300} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-2} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-2} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + INSTANCE_ID: api-2 + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + s3-init: + condition: service_completed_successfully + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + interval: 5s + timeout: 5s + start_period: 20s + retries: 3 + restart: unless-stopped + networks: + - backend + + nginx: + image: nginx:1.27-alpine + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "127.0.0.1:${API_PUBLISH_PORT:-8081}:8080" + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget --spider -q http://127.0.0.1:8080/health/ready || exit 1"] + interval: 5s + timeout: 5s + start_period: 10s + retries: 12 + restart: unless-stopped + networks: + - backend + + postgres: + image: postgres:17.6-bookworm@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --locale=C" + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger --timeout=3"] + interval: 5s + timeout: 5s + start_period: 10s + retries: 12 + restart: unless-stopped + networks: + - backend + + valkey: + image: valkey/valkey:8.1.3@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 + command: + - sh + - -c + - 'exec valkey-server --requirepass "$$(cat /run/secrets/valkey_password)" --appendonly yes --appendfsync everysec --save "" --dir /data' + secrets: + - valkey_password + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD-SHELL", 'valkey-cli --no-auth-warning -a "$$(cat /run/secrets/valkey_password)" ping | grep -q PONG'] + interval: 5s + timeout: 5s + start_period: 5s + retries: 12 + restart: unless-stopped + networks: + - backend + + s3: + image: chrislusf/seaweedfs:3.91@sha256:ed087c76ee348cc4cf91a92c23a9b136af49654bc0f89b27621425f01250065a + command: server -dir=/data -s3 -s3.port=9000 -s3.config=/etc/seaweedfs/s3.json + volumes: + - miniodata:/data + - ./secrets/s3.json:/etc/seaweedfs/s3.json:ro + healthcheck: + # Root S3 requests correctly return 403 for an anonymous caller; test the + # listening S3 TCP endpoint instead of treating that policy as unhealthy. + test: ["CMD-SHELL", "nc -z 127.0.0.1 9000"] + interval: 5s + timeout: 5s + start_period: 10s + retries: 12 + restart: unless-stopped + networks: + - backend + + # The short-lived privileged role creates the bucket before either API starts. + # API replicas never receive this credential. + s3-init: + build: + context: . + image: mtusi-messenger:lab4 + entrypoint: ["python", "-m", "src.s3_bootstrap"] + environment: + S3_ENDPOINT: ${S3_ENDPOINT:-http://s3:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_BOOTSTRAP_ACCESS_KEY_ID_FILE: /run/secrets/s3_bootstrap_access_key_id + S3_BOOTSTRAP_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_bootstrap_secret_access_key + secrets: + - s3_bootstrap_access_key_id + - s3_bootstrap_secret_access_key + depends_on: + s3: + condition: service_healthy + restart: "no" + networks: + - backend + + +secrets: + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + s3_bootstrap_access_key_id: + file: ./secrets/s3_bootstrap_access_key_id + s3_bootstrap_secret_access_key: + file: ./secrets/s3_bootstrap_secret_access_key + postgres_password: + file: ./secrets/postgres_password + valkey_password: + file: ./secrets/valkey_password + +volumes: + pgdata: + valkeydata: + miniodata: + +networks: + backend: + driver: bridge diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 00000000..d7de1137 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Apply versioned migrations (idempotent, advisory-locked), then hand PID 1 to +# the server so SIGTERM reaches uvicorn directly and shutdown stays graceful. +set -eu + +python -B -m src.migrate +exec python -B src/app.py diff --git a/evidence/attachment-load.json b/evidence/attachment-load.json new file mode 100644 index 00000000..6d39c7ab --- /dev/null +++ b/evidence/attachment-load.json @@ -0,0 +1 @@ +{"api_memory_bytes": {"baseline_before_test": 166031522, "growth": 120334582, "peak_sample": 286366104, "samples": 11}, "concurrency": 10, "elapsed_seconds": 25.312, "failed_uploads": 0, "latency_ms": {"p50": 24938.0, "p95": 25312.0, "p99": 25312.0}, "memory_method": "sum of docker stats for api-1 and api-2; baseline before requests, then repeat after a 100 ms wait", "payload_bytes_each": 8388608, "successful_rps": 0.395, "successful_uploads": 10} diff --git a/evidence/private-s3.txt b/evidence/private-s3.txt new file mode 100644 index 00000000..be77eb64 --- /dev/null +++ b/evidence/private-s3.txt @@ -0,0 +1 @@ +anonymous_s3_list_status=403 \ No newline at end of file diff --git a/evidence/reconcile-missing.json b/evidence/reconcile-missing.json new file mode 100644 index 00000000..3da73e51 --- /dev/null +++ b/evidence/reconcile-missing.json @@ -0,0 +1 @@ +{"final_dry_run": {"deleted_orphans": 0, "elapsed_ms": 88.209, "eligible_orphans": 0, "grace_seconds": 60, "metadata_rows": 53, "missing_metadata_objects": 0, "mode": "dry_run", "orphan_objects": 0, "s3_objects": 53, "safety": "missing metadata retained; fresh and referenced objects never deleted"}, "missing_apply": {"deleted_orphans": 0, "elapsed_ms": 106.908, "eligible_orphans": 0, "grace_seconds": 60, "metadata_rows": 53, "missing_metadata_objects": 1, "mode": "apply", "orphan_objects": 0, "s3_objects": 52, "safety": "missing metadata retained; fresh and referenced objects never deleted"}, "missing_download": {"error": "unavailable", "status": 503}, "missing_dry_run": {"deleted_orphans": 0, "elapsed_ms": 174.655, "eligible_orphans": 0, "grace_seconds": 60, "metadata_rows": 53, "missing_metadata_objects": 1, "mode": "dry_run", "orphan_objects": 0, "s3_objects": 52, "safety": "missing metadata retained; fresh and referenced objects never deleted"}, "restored_download": {"bytes_match": true, "status": 200}, "safety": "apply leaves missing PostgreSQL metadata intact; object was restored without metadata recreation"} diff --git a/evidence/reconcile.txt b/evidence/reconcile.txt new file mode 100644 index 00000000..8dfd11af --- /dev/null +++ b/evidence/reconcile.txt @@ -0,0 +1,18 @@ +## Reconciler experiment — 2026-11-06 + +Synthetic unreferenced object `attachments/reconcile-orphan` was inserted directly +into the private S3 bucket. No PostgreSQL metadata row was created. + +Dry-run immediately after insertion (`--grace-seconds 60`): +{"deleted_orphans":0,"eligible_orphans":0,"grace_seconds":60,"metadata_rows":8,"missing_metadata_objects":0,"mode":"dry_run","orphan_objects":1,"s3_objects":9} + +Apply after the 60-second grace period: +{"deleted_orphans":1,"eligible_orphans":1,"grace_seconds":60,"metadata_rows":8,"missing_metadata_objects":0,"mode":"apply","orphan_objects":1,"s3_objects":9} + +Final dry-run: +{"deleted_orphans":0,"eligible_orphans":0,"grace_seconds":60,"metadata_rows":8,"missing_metadata_objects":0,"mode":"dry_run","orphan_objects":0,"s3_objects":8} + +The reconciler only deletes S3 keys under the generated `attachments/` prefix that +have no metadata and are older than the grace period. A missing S3 object is +reported but its metadata is retained for manual repair; referenced and fresh +objects are never candidates for deletion. diff --git a/nginx/nginx-scale.conf b/nginx/nginx-scale.conf new file mode 100644 index 00000000..84ef6904 --- /dev/null +++ b/nginx/nginx-scale.conf @@ -0,0 +1,31 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Used only by scripts/research-scale.py with compose.scale.yaml. + upstream api { + server api-1:8080 max_fails=3 fail_timeout=10s; + server api-2:8080 max_fails=3 fail_timeout=10s; + server api-3:8080 max_fails=3 fail_timeout=10s; + server api-4:8080 max_fails=3 fail_timeout=10s; + } + + server { + listen 8080; + server_name _; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 2s; + proxy_read_timeout 10s; + proxy_send_timeout 10s; + proxy_next_upstream error timeout http_502; + proxy_next_upstream_tries 4; + location / { proxy_pass http://api; } + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 00000000..9fedd23c --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,67 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Upstream group: both API replicas. Health-checked by Nginx so a stopped + # replica is taken out of rotation automatically (max_fails + fail_timeout). + upstream api { + server api-1:8080 max_fails=3 fail_timeout=10s; + server api-2:8080 max_fails=3 fail_timeout=10s; + } + + server { + listen 8080; + server_name _; + # API owns the 10 MiB attachment limit and its JSON error envelope; + # leave a small header margin so nginx does not generate HTML 413 first. + client_max_body_size 11m; + + # Keep the instance header so clients can see which replica answered. + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # Never wait forever on a dead replica. + # A stopped Compose container can black-hole SYNs instead of refusing + # them. Keep one failed upstream attempt well below the test/client + # timeout so the healthy replica receives the retry. + proxy_connect_timeout 500ms; + # A valid request may stream the permitted 10 MiB through a loaded + # development Docker host. Connection failure detection stays fast; + # only an established upload/download gets its bounded 60-second budget. + proxy_read_timeout 60s; + proxy_send_timeout 60s; + + # A refused connection/502 means the selected replica is unavailable and + # can be retried on its peer. A structured 503 is an application + # dependency failure (for example PostgreSQL down), not an unhealthy + # replica: propagating it preserves the API error envelope instead of + # marking both live peers down and manufacturing an Nginx 502. + proxy_next_upstream error timeout http_502; + # Two attempts cover both configured replicas: a request that lands on + # the stopped instance is retried once on its healthy peer. + proxy_next_upstream_tries 2; + + location / { + proxy_pass http://api; + } + + location /health/ready { + # Readiness is checked by Compose on the LB itself; forward to api. + proxy_pass http://api; + } + + location /version { + proxy_pass http://api; + } + + location /info { + proxy_pass http://api; + } + } +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6680137a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,29 @@ +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +fastapi==0.162.0 +h11==0.16.0 +httptools==0.9.0 +idna==3.20 +minio==7.2.20 +Pillow==11.3.0 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +pycparser==3.12 +pydantic==2.14.2 +pydantic_core==2.50.2 +python-dotenv==1.2.4 +PyYAML==6.0.3 +redis==8.1.0 +starlette==1.10.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +tzdata==2026.6 +uvicorn==0.54.0 +watchfiles==1.3.0 +websockets==17.3 diff --git a/scripts/attachment_load.py b/scripts/attachment_load.py new file mode 100755 index 00000000..82d2168f --- /dev/null +++ b/scripts/attachment_load.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Measure bounded API memory during concurrent, valid attachment uploads. + +The script uses only the standard library and Docker CLI. It creates isolated +test data, submits N uploads concurrently, samples both API containers' memory, +and prints a token-free JSON summary suitable for evidence. +""" +from __future__ import annotations + +import argparse +import base64 +import json +import re +import statistics +import subprocess +import threading +import time +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() +MEMORY_VALUE = re.compile(r"([0-9.]+)\s*([KMGT]?i?B)") +UNIT_BYTES = {"B": 1, "KB": 1000, "MB": 1000**2, "GB": 1000**3, + "TB": 1000**4, "KiB": 1024, "MiB": 1024**2, + "GiB": 1024**3, "TiB": 1024**4} + + +def request(method: str, base_url: str, path: str, *, body: dict[str, Any] | None = None, + raw: bytes | None = None, headers: dict[str, str] | None = None, + timeout: float = 90) -> tuple[int, bytes]: + data = raw + request_headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body).encode("utf-8") + request_headers["Content-Type"] = "application/json" + request_headers.update(headers or {}) + prepared = urllib.request.Request(base_url + path, data=data, headers=request_headers, method=method) + try: + response = urllib.request.urlopen(prepared, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + with response: + return response.status, response.read() + + +def require_json(status: int, data: bytes, expected: int) -> dict[str, Any]: + if status != expected: + raise RuntimeError("expected HTTP %s, received %s: %s" % (expected, status, data[:300])) + return json.loads(data) + + +def docker_api_containers(env_file: str) -> list[str]: + completed = subprocess.run( + ["docker", "compose", "--env-file", env_file, "ps", "-q", "api-1", "api-2"], + cwd=ROOT, check=True, text=True, capture_output=True, + ) + containers = completed.stdout.split() + if len(containers) != 2: + raise RuntimeError("both API replicas must be running before the experiment") + return containers + + +def memory_bytes(containers: list[str]) -> int: + completed = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.MemUsage}}", *containers], + check=True, text=True, capture_output=True, + ) + total = 0 + for line in completed.stdout.splitlines(): + match = MEMORY_VALUE.search(line) + if match is None: + raise RuntimeError("cannot parse docker memory output: " + line) + total += int(float(match.group(1)) * UNIT_BYTES[match.group(2)]) + return total + + +def percentile(values: list[float], fraction: float) -> float: + values = sorted(values) + return values[min(len(values) - 1, max(0, round((len(values) - 1) * fraction)))] + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8083") + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--payload-mib", type=int, default=8) + parser.add_argument("--env-file", default=".env") + parser.add_argument("--evidence", type=Path) + args = parser.parse_args() + if not 10 <= args.concurrency <= 40: + raise SystemExit("concurrency must be 10..40") + if not 2 <= args.payload_mib <= 10: + raise SystemExit("payload-mib must be 2..10") + base_url = args.base_url.rstrip("/") + payload = PNG + b"\0" * (args.payload_mib * 1024 * 1024 - len(PNG)) + registration = { + "username": "load_" + uuid.uuid4().hex[:16], + "display_name": "Attachment load test", + "password": "Load-" + uuid.uuid4().hex, + } + user = require_json(*request("POST", base_url, "/api/v1/users", body=registration), 201) + basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode() + session = require_json(*request("POST", base_url, "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + basic}), 201) + token = session["token"] + chat = require_json(*request("POST", base_url, "/api/v1/chats", + body={"title": "Attachment memory experiment", "member_ids": [user["id"]]}, + headers={"Authorization": "Bearer " + token}), 201) + path = "/api/v1/chats/%s/attachments?filename=memory.png" % chat["id"] + containers = docker_api_containers(args.env_file) + baseline = memory_bytes(containers) + samples: list[int] = [] + monitor_error: list[str] = [] + stop = threading.Event() + + def monitor() -> None: + while not stop.is_set(): + try: + samples.append(memory_bytes(containers)) + except (OSError, subprocess.SubprocessError, RuntimeError) as exc: + monitor_error.append(type(exc).__name__) + return + stop.wait(0.1) + + thread = threading.Thread(target=monitor, daemon=True) + thread.start() + time.sleep(0.2) + + def upload(_: int) -> tuple[int, float]: + started = time.monotonic() + status, _ = request("POST", base_url, path, raw=payload, headers={ + "Authorization": "Bearer " + token, "Content-Type": "image/png", + }) + return status, time.monotonic() - started + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.concurrency) as executor: + results = list(executor.map(upload, range(args.concurrency))) + elapsed = time.monotonic() - started + stop.set() + thread.join(timeout=3) + statuses = [status for status, _ in results] + durations_ms = [duration * 1000 for _, duration in results] + if monitor_error: + raise RuntimeError("memory sampling failed: " + monitor_error[0]) + if not samples: + raise RuntimeError("memory sampling returned no samples") + summary = { + "concurrency": args.concurrency, + "payload_bytes_each": len(payload), + "successful_uploads": statuses.count(201), + "failed_uploads": len(statuses) - statuses.count(201), + "elapsed_seconds": round(elapsed, 3), + "successful_rps": round(statuses.count(201) / elapsed, 3), + "latency_ms": {"p50": round(percentile(durations_ms, .50), 1), + "p95": round(percentile(durations_ms, .95), 1), + "p99": round(percentile(durations_ms, .99), 1)}, + "api_memory_bytes": {"baseline_before_test": baseline, "peak_sample": max([baseline, *samples]), + "growth": max([baseline, *samples]) - baseline, "samples": len(samples)}, + "memory_method": "sum of docker stats for api-1 and api-2; baseline before requests, then repeat after a 100 ms wait", + } + text = json.dumps(summary, ensure_ascii=False, sort_keys=True) + print(text) + if args.evidence is not None: + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text(text + "\n", encoding="utf-8") + if summary["failed_uploads"]: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/bootstrap-secrets.sh b/scripts/bootstrap-secrets.sh new file mode 100755 index 00000000..97a33347 --- /dev/null +++ b/scripts/bootstrap-secrets.sh @@ -0,0 +1,60 @@ +#!/bin/sh +# Generate the local, synthetic stand's secrets and .env. +# Values are written to ./secrets (git-ignored, mode 0600) and never printed. +# Re-running is safe: existing files are kept unless FORCE=1. +set -eu +cd "$(dirname "$0")/.." + +PYTHON="${PYTHON:-python}" +SECRETS_DIR="${SECRETS_DIR:-secrets}" + +if [ ! -f .env ]; then + cp .env.example .env + printf '%s\n' "created .env from .env.example" +fi + +mkdir -p "$SECRETS_DIR" +chmod 700 "$SECRETS_DIR" 2>/dev/null || true + +umask 077 + +random_password() { + "$PYTHON" -c 'import secrets; print(secrets.token_hex(24))' +} + +write_if_missing() { + path="$SECRETS_DIR/$1" + if [ -f "$path" ] && [ "${FORCE:-0}" != "1" ]; then + return 0 + fi + printf '%s' "$2" > "$path" + chmod 600 "$path" 2>/dev/null || true +} + +postgres_password="$(random_password)" +valkey_password="$(random_password)" +s3_access_key_id="messenger-api" +s3_secret_access_key="$(random_password)" +s3_bootstrap_access_key_id="messenger-bootstrap" +s3_bootstrap_secret_access_key="$(random_password)" + +write_if_missing postgres_password "$postgres_password" +write_if_missing valkey_password "$valkey_password" +write_if_missing database_url "postgresql://messenger:${postgres_password}@postgres:5432/messenger" +write_if_missing cache_url "redis://:${valkey_password}@valkey:6379/0" +write_if_missing s3_access_key_id "$s3_access_key_id" +write_if_missing s3_secret_access_key "$s3_secret_access_key" +write_if_missing s3_bootstrap_access_key_id "$s3_bootstrap_access_key_id" +write_if_missing s3_bootstrap_secret_access_key "$s3_bootstrap_secret_access_key" + +# SeaweedFS consumes an S3 identity policy file. API has only data-plane rights; +# bucket creation uses a separate identity unavailable to API processes. +if [ ! -f "$SECRETS_DIR/s3.json" ] || [ "${FORCE:-0}" = "1" ]; then + cat >"$SECRETS_DIR/s3.json" </dev/null || true +fi + +printf '%s\n' "wrote secrets to $SECRETS_DIR: postgres, valkey, restricted S3 API and bootstrap credentials" +printf '%s\n' "next: docker compose --env-file .env up -d --build" diff --git a/scripts/check_contract.py b/scripts/check_contract.py index dcb34dfa..0c75274b 100644 --- a/scripts/check_contract.py +++ b/scripts/check_contract.py @@ -8,8 +8,8 @@ import re ROOT = Path(__file__).resolve().parents[1] def main(): - lab = json.loads((ROOT/'lab.json').read_text())['number'] - spec = json.loads((ROOT/'contracts/openapi.json').read_text()) + lab = json.loads((ROOT/'lab.json').read_text(encoding='utf-8'))['number'] + spec = json.loads((ROOT/'contracts/openapi.json').read_text(encoding='utf-8')) assert spec['openapi']=='3.1.0' assert spec['info']['version']=='1.%d.0'%lab operation_ids = set() @@ -41,11 +41,11 @@ def main(): assert ('/api/v1/auth/sessions' in spec['paths']) == (lab>=2) assert ('/api/v1/attachments/{attachment_id}' in spec['paths']) == (lab>=4) for file in list((ROOT/'tests').glob('*.py'))+list((ROOT/'scripts').glob('*.py')): - ast.parse(file.read_text(),filename=str(file)) + ast.parse(file.read_text(encoding='utf-8'),filename=str(file)) for file in ROOT.rglob('*.json'): if any(part in {'.git','.venv','node_modules','vendor','output'} for part in file.parts): continue - json.loads(file.read_text()) + json.loads(file.read_text(encoding='utf-8')) for required in ('README.md','COURSE.md','CONTRIBUTING.md','REPORT.md','contracts/README.md','tests/smoke.py','tests/README.md'): assert (ROOT/required).is_file(), 'missing '+required print('PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.') diff --git a/scripts/reconcile.sh b/scripts/reconcile.sh new file mode 100755 index 00000000..394cfb16 --- /dev/null +++ b/scripts/reconcile.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file "${COMPOSE_ENV_FILE:-.env}" exec -T api-1 python -m src.reconcile "$@" diff --git a/scripts/reconcile_experiment.py b/scripts/reconcile_experiment.py new file mode 100755 index 00000000..b88e5732 --- /dev/null +++ b/scripts/reconcile_experiment.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Inject and restore a missing S3 object to prove reconciler safety. + +Creates an isolated attachment through the public API, removes only its +server-generated S3 key from inside the private network, verifies detection and +non-destructive ``--apply``, restores the exact bytes, and verifies a clean scan. +""" +from __future__ import annotations + +import argparse +import base64 +import json +import subprocess +import urllib.error +import urllib.request +import uuid +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +PNG = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def request(method: str, base_url: str, path: str, *, body: dict[str, Any] | None = None, + raw: bytes | None = None, headers: dict[str, str] | None = None) -> tuple[int, bytes]: + data = raw if body is None else json.dumps(body).encode("utf-8") + request_headers = {"Accept": "application/json"} + if body is not None: + request_headers["Content-Type"] = "application/json" + request_headers.update(headers or {}) + prepared = urllib.request.Request(base_url + path, data=data, headers=request_headers, method=method) + try: + response = urllib.request.urlopen(prepared, timeout=30) + except urllib.error.HTTPError as error: + response = error + with response: + return response.status, response.read() + + +def require_json(response: tuple[int, bytes], expected: int) -> dict[str, Any]: + status, data = response + if status != expected: + raise RuntimeError("expected HTTP %s, received %s: %s" % (expected, status, data[:300])) + return json.loads(data) + + +def reconcile(env_file: str, *args: str) -> dict[str, Any]: + result = subprocess.run( + ["docker", "compose", "--env-file", env_file, "exec", "-T", "api-1", "python", "-m", "src.reconcile", *args], + cwd=ROOT, check=True, text=True, capture_output=True, + ) + return json.loads(result.stdout) + + +def s3_command(env_file: str, source: str) -> None: + subprocess.run( + ["docker", "compose", "--env-file", env_file, "exec", "-T", "api-1", "python", "-c", source], + cwd=ROOT, check=True, text=True, capture_output=True, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8083") + parser.add_argument("--env-file", default=".env") + parser.add_argument("--evidence", type=Path) + args = parser.parse_args() + base_url = args.base_url.rstrip("/") + registration = {"username": "missing_" + uuid.uuid4().hex[:16], + "display_name": "Missing S3 experiment", "password": "Missing-" + uuid.uuid4().hex} + user = require_json(request("POST", base_url, "/api/v1/users", body=registration), 201) + basic = base64.b64encode((registration["username"] + ":" + registration["password"]).encode()).decode() + session = require_json(request("POST", base_url, "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + basic}), 201) + token = session["token"] + chat = require_json(request("POST", base_url, "/api/v1/chats", + body={"title": "Missing S3 experiment", "member_ids": [user["id"]]}, + headers={"Authorization": "Bearer " + token}), 201) + attachment = require_json(request("POST", base_url, + "/api/v1/chats/%s/attachments?filename=missing.png" % chat["id"], raw=PNG, + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}), 201) + key = "attachments/" + attachment["id"] + delete_code = "from src.config import load_settings; from src.object_store import ObjectStore; ObjectStore(load_settings()).delete(%r)" % key + s3_command(args.env_file, delete_code) + unavailable_status, unavailable_body = request("GET", base_url, "/api/v1/attachments/%s/content" % attachment["id"], + headers={"Authorization": "Bearer " + token}) + missing_dry_run = reconcile(args.env_file, "--grace-seconds", "60") + missing_apply = reconcile(args.env_file, "--apply", "--grace-seconds", "60") + # Do not put image bytes on a Windows command line: it is capped at 32 KiB. + # docker compose cp crosses the private container boundary without exposing S3. + subprocess.run( + ["docker", "compose", "--env-file", args.env_file, "cp", + str(ROOT / "tests" / "fixtures" / "sample.png"), "api-1:/tmp/reconcile-restore.png"], + cwd=ROOT, check=True, text=True, capture_output=True, + ) + restore_code = ( + "from src.config import load_settings; from src.object_store import ObjectStore; " + "source=open('/tmp/reconcile-restore.png','rb'); " + "ObjectStore(load_settings()).put(%r,source,%s,'image/png'); source.close()" + % (key, len(PNG)) + ) + s3_command(args.env_file, restore_code) + final_dry_run = reconcile(args.env_file, "--grace-seconds", "60") + restored_status, restored_bytes = request("GET", base_url, "/api/v1/attachments/%s/content" % attachment["id"], + headers={"Authorization": "Bearer " + token}) + summary = { + "missing_download": {"status": unavailable_status, "error": json.loads(unavailable_body)["error"]["code"]}, + "missing_dry_run": missing_dry_run, + "missing_apply": missing_apply, + "final_dry_run": final_dry_run, + "restored_download": {"status": restored_status, "bytes_match": restored_bytes == PNG}, + "safety": "apply leaves missing PostgreSQL metadata intact; object was restored without metadata recreation", + } + text = json.dumps(summary, ensure_ascii=False, sort_keys=True) + print(text) + if args.evidence is not None: + args.evidence.parent.mkdir(parents=True, exist_ok=True) + args.evidence.write_text(text + "\n", encoding="utf-8") + if unavailable_status != 503 or missing_dry_run["missing_metadata_objects"] < 1 or missing_apply["deleted_orphans"] != 0 or final_dry_run["missing_metadata_objects"] != 0 or restored_status != 200 or restored_bytes != PNG: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..5997afc6 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,11 @@ +@echo off +setlocal +cd /d "%~dp0.." +if "%COMPOSE_ENV_FILE%"=="" set COMPOSE_ENV_FILE=.env +echo restart: restarting API, PostgreSQL, Valkey and S3 (volumes preserved) +rem The resilience test owns readiness waiting; s3-init has already created bucket. +docker compose --env-file %COMPOSE_ENV_FILE% restart --timeout 15 postgres valkey s3 api-1 api-2 nginx +if errorlevel 1 exit /b 1 +docker compose --env-file %COMPOSE_ENV_FILE% ps +echo stack_restarted volumes_preserved +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..466cceb3 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. +# Restarts every long-lived API and storage process while preserving volumes. +set -eu +cd "$(dirname "$0")/.." + +ENV_FILE="${COMPOSE_ENV_FILE:-.env}" + +printf '%s\n' "restart: restarting API, PostgreSQL, Valkey and S3 (volumes preserved)" +# The resilience test owns readiness waiting. The persistent bucket was already +# created by the idempotent one-shot s3-init bootstrap service. +docker compose --env-file "$ENV_FILE" restart --timeout 15 postgres valkey s3 api-1 api-2 nginx + +docker compose --env-file "$ENV_FILE" ps +printf '%s\n' "stack_restarted volumes_preserved" diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..513d77a2 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,17 @@ +@echo off +setlocal +cd /d "%~dp0.." +if "%COMPOSE_ENV_FILE%"=="" set COMPOSE_ENV_FILE=.env +if "%TARGET_INSTANCE_ID%"=="" ( + echo TARGET_INSTANCE_ID is required >&2 + exit /b 2 +) +if not "%TARGET_INSTANCE_ID%"=="api-1" if not "%TARGET_INSTANCE_ID%"=="api-2" ( + echo Invalid TARGET_INSTANCE_ID: %TARGET_INSTANCE_ID% >&2 + exit /b 2 +) +echo stop-one: stopping API replica %TARGET_INSTANCE_ID% +docker compose --env-file %COMPOSE_ENV_FILE% stop --timeout 10 %TARGET_INSTANCE_ID% +if errorlevel 1 exit /b 1 +echo stop-one: %TARGET_INSTANCE_ID% stopped +exit /b 0 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..990054d6 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`. +# Stops ONE API replica (selected by TARGET_INSTANCE_ID = api-1 or api-2) and +# leaves it stopped; the load balancer keeps serving via the remaining replica. +# Does not touch postgres, valkey, nginx or any volumes. Restore the replica +# explicitly after the test (docker compose start ). +set -eu + +cd "$(dirname "$0")/.." + +ENV_FILE="${COMPOSE_ENV_FILE:-.env}" + +: "${TARGET_INSTANCE_ID:?test provides target API instance alias (api-1 or api-2)}" + +case "$TARGET_INSTANCE_ID" in + api-1|api-2) ;; + *) + printf '%s\n' "stop-one: invalid TARGET_INSTANCE_ID='$TARGET_INSTANCE_ID' (expected api-1 or api-2)" >&2 + exit 2 + ;; +esac + +printf '%s\n' "stop-one: stopping API replica $TARGET_INSTANCE_ID (nginx keeps serving via the other replica)" +docker compose --env-file "$ENV_FILE" stop --timeout 10 "$TARGET_INSTANCE_ID" + +printf '%s\n' "stop-one: $TARGET_INSTANCE_ID stopped" diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py index 1dfe4a98..1542dc6a 100644 --- a/scripts/validate_openapi.py +++ b/scripts/validate_openapi.py @@ -6,17 +6,17 @@ from openapi_spec_validator import validate from jsonschema import Draft202012Validator root = Path(__file__).resolve().parents[1] -spec = json.loads((root/'contracts/openapi.json').read_text()) +spec = json.loads((root/'contracts/openapi.json').read_text(encoding='utf-8')) validate(spec) from referencing import Registry, Resource # The base URI resolves local #/components references inside examples. resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'}) registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource)) -for name, example in json.loads((root/'contracts/examples.json').read_text()).items(): +for name, example in json.loads((root/'contracts/examples.json').read_text(encoding='utf-8')).items(): schema = {'$ref': 'urn:mtusi:openapi#/components/schemas/' + name} Draft202012Validator(schema, registry=registry).validate(example) for path in (root/'contracts').glob('*.schema.json'): - schema = json.loads(path.read_text()) + schema = json.loads(path.read_text(encoding='utf-8')) Draft202012Validator.check_schema(schema) for example in schema.get('examples',[]): Draft202012Validator(schema).validate(example) diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..a21908e8 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""MTUSI messenger, laboratory 3: multi-instance, sessions, idempotency.""" diff --git a/src/api.py b/src/api.py new file mode 100644 index 00000000..7a1a7b12 --- /dev/null +++ b/src/api.py @@ -0,0 +1,533 @@ +"""Routes, authentication and request validation for lab 2.""" + +from __future__ import annotations + +import base64 +import binascii +import hashlib +import json +import re +import tempfile +import uuid +from dataclasses import dataclass +from typing import Annotated, Any, cast + +from fastapi import APIRouter, Depends, Request, Response +from PIL import Image, UnidentifiedImageError +from psycopg_pool import AsyncConnectionPool +from pydantic import ValidationError +from starlette.concurrency import run_in_threadpool +from starlette.responses import StreamingResponse + +from . import sessions as session_service +from . import store +from .config import DEFAULT_PAGE_SIZE, MAX_JSON_BYTES, MAX_PAGE_SIZE, IDEMPOTENCY_KEY_PATTERN +from .db import ping +from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, ApiProblem +from .object_store import ObjectStore +from .models import CreateChat, CreateMessage, CreateUser, StrictModel +from .pagination import build_page, decode_cursor, format_timestamp +from .security import hash_password, verify_password, verify_password_unknown_user + + +router = APIRouter() + +USERNAME_PATTERN = re.compile(r"^[a-z0-9_]{3,32}$") + + +@dataclass(frozen=True) +class Principal: + """The authenticated identity behind a Bearer token.""" + + user_id: uuid.UUID + user: dict[str, Any] + + +def get_pool(request: Request) -> AsyncConnectionPool: + return cast(AsyncConnectionPool, request.app.state.pool) + + +def get_cache(request: Request) -> session_service.SessionCache: + return cast(session_service.SessionCache, request.app.state.cache) + + +def get_object_store(request: Request) -> ObjectStore: + return cast(ObjectStore, request.app.state.object_store) + + +async def drain_body(request: Request) -> None: + """Consume an unwanted request body without retaining it in memory.""" + async for _chunk in request.stream(): + pass + + +def declared_body_size(request: Request) -> int | None: + value = request.headers.get("Content-Length") + if value is None: + return None + try: + size = int(value) + except ValueError: + raise ApiProblem(400, "invalid_request", "Content-Length is invalid") from None + if size < 0: + raise ApiProblem(400, "invalid_request", "Content-Length is invalid") + return size + + +async def read_limited_body(request: Request) -> bytes: + chunks: list[bytes] = [] + size = 0 + too_large = False + async for chunk in request.stream(): + size += len(chunk) + if size <= MAX_JSON_BYTES: + chunks.append(chunk) + else: + too_large = True + if too_large: + raise ApiProblem(413, "payload_too_large", "Request body is too large") + return b"".join(chunks) + + +async def parse_body(request: Request, model: type[StrictModel]) -> Any: + content_length = declared_body_size(request) + if content_length is not None and content_length > MAX_JSON_BYTES: + await drain_body(request) + raise ApiProblem(413, "payload_too_large", "Request body is too large") + content_type = request.headers.get("Content-Type", "").split(";", 1)[0].strip() + if content_type.lower() != "application/json": + await drain_body(request) + raise ApiProblem( + 415, "unsupported_media_type", "Content-Type must be application/json" + ) + body = await read_limited_body(request) + try: + decoded = json.loads(body.decode("utf-8")) + except (json.JSONDecodeError, UnicodeDecodeError): + raise ApiProblem( + 400, "invalid_request", "Request body is not valid JSON" + ) from None + try: + return model.model_validate(decoded) + except ValidationError: + raise ApiProblem(400, "invalid_request", "Request fields are invalid") from None + + +def ensure_no_query(request: Request) -> None: + if request.url.query: + raise ApiProblem(400, "invalid_request", "Query parameters are not supported") + + +def parse_pagination_request(request: Request) -> tuple[int, str | None]: + """Read limit/cursor only; unknown or repeated parameters are a 400.""" + parameters = request.query_params + keys = [key for key, _ in parameters.multi_items()] + if set(keys) - {"limit", "cursor"} or any( + keys.count(key) != 1 for key in set(keys) + ): + raise ApiProblem(400, "invalid_request", "Query parameters are invalid") + limit_value = parameters.get("limit", str(DEFAULT_PAGE_SIZE)) + try: + limit = int(limit_value) + except (TypeError, ValueError): + raise ApiProblem(400, "invalid_request", "limit is invalid") from None + if not 1 <= limit <= MAX_PAGE_SIZE: + raise ApiProblem(400, "invalid_request", "limit is invalid") + cursor = parameters.get("cursor") + if cursor == "": + raise ApiProblem(400, "invalid_request", "cursor is invalid") + return limit, cursor + + +def require_chat_id(chat_id: str) -> uuid.UUID: + try: + return uuid.UUID(chat_id) + except (ValueError, AttributeError): + raise ApiProblem(400, "invalid_request", "chat_id must be a UUID") from None + + +def require_attachment_id(attachment_id: str) -> uuid.UUID: + try: + return uuid.UUID(attachment_id) + except (ValueError, AttributeError): + raise ApiProblem(400, "invalid_request", "attachment_id must be a UUID") from None + + +def unauthorized(message: str, challenge: str) -> ApiProblem: + return ApiProblem(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def bearer_token(request: Request) -> str: + """Only Bearer is accepted; Basic here is a 401, not a fallback login.""" + header = request.headers.get("Authorization") + if header is None: + raise unauthorized("Bearer session token is required", BEARER_CHALLENGE) + scheme, _, value = header.partition(" ") + if scheme.strip().lower() != "bearer" or not value.strip(): + raise unauthorized("Bearer session token is required", BEARER_CHALLENGE) + return value.strip() + + +def basic_credentials(request: Request) -> tuple[str, str]: + """Decode Basic credentials; it is an encoding, never a trusted identity.""" + header = request.headers.get("Authorization") + if header is None: + raise unauthorized("Basic credentials are required", BASIC_CHALLENGE) + scheme, _, value = header.partition(" ") + if scheme.strip().lower() != "basic" or not value.strip(): + raise unauthorized("Basic credentials are required", BASIC_CHALLENGE) + try: + raw = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError, ValueError): + raise unauthorized("Basic credentials are invalid", BASIC_CHALLENGE) from None + username, separator, password = raw.partition(":") + if not separator or not username or not password: + raise unauthorized("Basic credentials are invalid", BASIC_CHALLENGE) + if USERNAME_PATTERN.fullmatch(username) is None: + # Unknown or malformed usernames must not be distinguished from a bad password. + raise unauthorized("Invalid username or password", BASIC_CHALLENGE) + return username, password + + +async def current_principal(request: Request) -> Principal: + token = bearer_token(request) + user_id, user = await session_service.resolve_session( + get_pool(request), get_cache(request), token + ) + return Principal(user_id=user_id, user=user) + + +Actor = Annotated[Principal, Depends(current_principal)] + + +# --------------------------------------------------------------------------- # +# Health +# --------------------------------------------------------------------------- # +@router.get("/health/live") +async def liveness(request: Request) -> dict[str, str]: + ensure_no_query(request) + return {"status": "ok"} + + +@router.get("/health/ready") +async def readiness(request: Request) -> dict[str, str]: + ensure_no_query(request) + try: + await ping(get_pool(request)) + await run_in_threadpool(get_object_store(request).ensure_available) + except Exception as exc: # noqa: BLE001 - any DB failure means not ready + request.app.state.logger.warning( + "readiness_database_failed error=%s", type(exc).__name__ + ) + raise ApiProblem(503, "unavailable", "Required storage is unavailable") from None + return {"status": "ok"} + + +# --------------------------------------------------------------------------- # +# Users and sessions +# --------------------------------------------------------------------------- # +@router.post("/api/v1/users", status_code=201) +async def register_user(request: Request) -> dict[str, Any]: + ensure_no_query(request) + body = await parse_body(request, CreateUser) + assert isinstance(body, CreateUser) + password_hash = await run_in_threadpool(hash_password, body.password) + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor: + return await store.create_user( + cursor, + username=body.username, + display_name=body.display_name, + password_hash=password_hash, + ) + + +@router.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request) -> dict[str, Any]: + ensure_no_query(request) + await drain_body(request) + username, password = basic_credentials(request) + pool = get_pool(request) + async with pool.connection() as connection: + async with connection.cursor() as cursor: + row = await store.user_credentials(cursor, username) + if row is None: + await run_in_threadpool(verify_password_unknown_user, password) + raise unauthorized("Invalid username or password", BASIC_CHALLENGE) + if not await run_in_threadpool( + verify_password, password, row["password_hash"] + ): + raise unauthorized("Invalid username or password", BASIC_CHALLENGE) + token, expires_at = await session_service.create_session( + pool, + get_cache(request), + user_id=row["id"], + ttl_seconds=request.app.state.settings.session_ttl_seconds, + ) + return { + "token": token, + "token_type": "Bearer", + "expires_at": format_timestamp(expires_at), + "user": store.user_json(row), + } + + +@router.delete("/api/v1/auth/sessions/current", status_code=204) +async def delete_current_session(request: Request) -> Response: + ensure_no_query(request) + token = bearer_token(request) + await session_service.revoke_session(get_pool(request), get_cache(request), token) + return Response(status_code=204) + + +@router.get("/api/v1/users/me") +async def current_user(request: Request, actor: Actor) -> dict[str, Any]: + ensure_no_query(request) + return actor.user + + +# --------------------------------------------------------------------------- # +# Chats and messages +# --------------------------------------------------------------------------- # +@router.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request, actor: Actor) -> dict[str, Any]: + ensure_no_query(request) + body = await parse_body(request, CreateChat) + assert isinstance(body, CreateChat) + members = [uuid.UUID(member) for member in body.member_ids] + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor: + return await store.create_chat( + cursor, actor_id=actor.user_id, title=body.title, member_ids=members + ) + + +@router.get("/api/v1/chats") +async def list_chats(request: Request, actor: Actor) -> dict[str, Any]: + limit, cursor = parse_pagination_request(request) + resource = "chats:" + str(actor.user_id) + after = decode_cursor(cursor, resource) if cursor is not None else None + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor_handle: + rows = await store.chat_page( + cursor_handle, actor_id=actor.user_id, limit=limit, after=after + ) + return build_page(rows, resource, limit) + + +@router.get("/api/v1/chats/{chat_id}") +async def get_chat(request: Request, chat_id: str, actor: Actor) -> dict[str, Any]: + ensure_no_query(request) + normalized = require_chat_id(chat_id) + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor: + return await store.accessible_chat( + cursor, actor_id=actor.user_id, chat_id=normalized + ) + + +@router.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message( + request: Request, chat_id: str, actor: Actor +) -> dict[str, Any]: + normalized = require_chat_id(chat_id) + ensure_no_query(request) + body = await parse_body(request, CreateMessage) + assert isinstance(body, CreateMessage) + pool = get_pool(request) + # The lab4 contract declares an omitted attachment_ids and [] semantically + # identical. Keep the lab3 text-only canonical form as well so old replay + # records remain valid after migration. + canonical_body: dict[str, Any] = {"text": body.text} + if body.attachment_ids: + canonical_body["attachment_ids"] = body.attachment_ids + request_sha256 = hashlib.sha256( + json.dumps(canonical_body, sort_keys=True).encode("utf-8") + ).hexdigest() + async with pool.connection() as connection: + async with connection.cursor() as cursor: + # Check membership first so non-members get 404, not 400. + await store.require_membership( + cursor, actor_id=actor.user_id, chat_id=normalized + ) + key = request.headers.get("Idempotency-Key", "").strip() + if not key or not IDEMPOTENCY_KEY_PATTERN.fullmatch(key): + raise ApiProblem( + 400, "invalid_request", "Idempotency-Key must match " + "^[A-Za-z0-9._:-]{1,128}$" + ) + existing = await store.get_idempotency_key( + cursor, actor_id=actor.user_id, chat_id=normalized, key=key + ) + if existing is not None: + if existing["request_sha256"] != request_sha256: + raise ApiProblem(409, "conflict", "Idempotency-Key was already used") + # Replay: return the original message. + original = await store.message_by_id(cursor, existing["message_id"]) + if original is not None: + return original + inserted = await store.insert_idempotency_key( + cursor, + actor_id=actor.user_id, + key=key, + chat_id=normalized, + request_sha256=request_sha256, + ) + if not inserted: + # INSERT .. ON CONFLICT keeps this transaction usable. PostgreSQL + # waited for the winning transaction, so its message is committed + # and can be returned as the exact replay result. + existing = await store.get_idempotency_key( + cursor, actor_id=actor.user_id, chat_id=normalized, key=key + ) + if existing is not None: + if existing["request_sha256"] != request_sha256: + raise ApiProblem(409, "conflict", "Idempotency-Key was already used") + original = await store.message_by_id(cursor, existing["message_id"]) + if original is not None: + return original + raise ApiProblem(409, "conflict", "Idempotency-Key is still being stored") + return await store.create_message( + cursor, + actor_id=actor.user_id, + chat_id=normalized, + text=body.text, + attachment_ids=[uuid.UUID(value) for value in body.attachment_ids], + request_sha256=request_sha256, + key=key, + ) + + +def _attachment_filename(request: Request) -> str: + values = request.query_params.getlist("filename") + if len(values) != 1: + raise ApiProblem(400, "invalid_request", "filename is required exactly once") + filename = values[0] + if not 1 <= len(filename) <= 255 or any( + character in filename for character in ("/", "\\", "\x00") + ) or any(ord(character) < 32 or ord(character) == 127 for character in filename): + raise ApiProblem(400, "invalid_request", "filename is invalid") + return filename + + +async def _read_and_validate_image(request: Request) -> tuple[tempfile.SpooledTemporaryFile[bytes], int, str, str, int, int]: + settings = request.app.state.settings + declared = declared_body_size(request) + if declared is not None and declared > settings.max_upload_bytes: + await drain_body(request) + raise ApiProblem(413, "payload_too_large", "Attachment is too large") + content_type = request.headers.get("Content-Type", "").split(";", 1)[0].lower().strip() + if content_type not in {"image/png", "image/jpeg"}: + await drain_body(request) + raise ApiProblem(415, "unsupported_media_type", "Only PNG and JPEG are accepted") + payload = tempfile.SpooledTemporaryFile(max_size=1_048_576, mode="w+b") + digest = hashlib.sha256() + size = 0 + too_large = False + async for chunk in request.stream(): + size += len(chunk) + if size > settings.max_upload_bytes: + too_large = True + # Continue draining the connection but never retain more bytes. + continue + digest.update(chunk) + payload.write(chunk) + if too_large: + payload.close() + raise ApiProblem(413, "payload_too_large", "Attachment is too large") + if size == 0: + payload.close() + raise ApiProblem(415, "unsupported_media_type", "Image is empty or invalid") + try: + payload.seek(0) + image = await run_in_threadpool(Image.open, payload) + detected = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format or "") + width, height = image.size + if detected is None or detected != content_type: + raise ApiProblem(415, "unsupported_media_type", "Image type does not match Content-Type") + if width * height > settings.max_image_pixels: + raise ApiProblem(413, "payload_too_large", "Image has too many pixels") + await run_in_threadpool(image.verify) + except ApiProblem: + payload.close() + raise + except (UnidentifiedImageError, OSError, ValueError): + payload.close() + raise ApiProblem(415, "unsupported_media_type", "Image is invalid") from None + payload.seek(0) + return payload, size, digest.hexdigest(), content_type, width, height + + +@router.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(request: Request, chat_id: str, actor: Actor) -> dict[str, Any]: + normalized_chat = require_chat_id(chat_id) + filename = _attachment_filename(request) + # Do authorization before retaining upload bytes, including when Content- + # Length is missing or adversarial. + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor: + await store.require_membership(cursor, actor_id=actor.user_id, chat_id=normalized_chat) + payload, size, sha256, content_type, width, height = await _read_and_validate_image(request) + attachment_id = uuid.uuid4() + object_key = "attachments/" + str(attachment_id) + object_store = get_object_store(request) + try: + await run_in_threadpool(object_store.put, object_key, payload, size, content_type) + try: + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor: + return await store.create_attachment( + cursor, attachment_id=attachment_id, chat_id=normalized_chat, + owner_id=actor.user_id, object_key=object_key, filename=filename, + content_type=content_type, size_bytes=size, sha256=sha256, + width=width, height=height, + ) + except Exception: + await run_in_threadpool(object_store.delete, object_key) + raise + finally: + payload.close() + + +@router.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(request: Request, attachment_id: str, actor: Actor) -> dict[str, Any]: + ensure_no_query(request) + normalized = require_attachment_id(attachment_id) + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor: + row = await store.accessible_attachment(cursor, actor_id=actor.user_id, attachment_id=normalized) + return store.attachment_json(row) + + +@router.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(request: Request, attachment_id: str, actor: Actor) -> StreamingResponse: + ensure_no_query(request) + normalized = require_attachment_id(attachment_id) + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor: + row = await store.accessible_attachment(cursor, actor_id=actor.user_id, attachment_id=normalized) + await run_in_threadpool(get_object_store(request).ensure_object, row["object_key"]) + return StreamingResponse( + get_object_store(request).chunks(row["object_key"]), + media_type=row["content_type"], + headers={"Content-Length": str(row["size_bytes"])}, + ) + + +@router.get("/api/v1/chats/{chat_id}/messages") +async def list_messages( + request: Request, chat_id: str, actor: Actor +) -> dict[str, Any]: + normalized = require_chat_id(chat_id) + limit, cursor = parse_pagination_request(request) + resource = "messages:" + str(normalized) + after = decode_cursor(cursor, resource) if cursor is not None else None + async with get_pool(request).connection() as connection: + async with connection.cursor() as cursor_handle: + await store.require_membership( + cursor_handle, actor_id=actor.user_id, chat_id=normalized + ) + rows = await store.message_page( + cursor_handle, chat_id=normalized, limit=limit, after=after + ) + return build_page(rows, resource, limit) diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..372df994 --- /dev/null +++ b/src/app.py @@ -0,0 +1,34 @@ +"""Executable entry point: migrations run first, then uvicorn as PID 1.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import uvicorn + + +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from src.application import create_app # noqa: E402 +from src.config import load_address # noqa: E402 + + +app = create_app() + + +def run() -> None: + host, port = load_address() + uvicorn.run( + app, + host=host, + port=port, + access_log=False, + timeout_graceful_shutdown=8, + log_level="info", + ) + + +if __name__ == "__main__": + run() diff --git a/src/application.py b/src/application.py new file mode 100644 index 00000000..97bb2ad3 --- /dev/null +++ b/src/application.py @@ -0,0 +1,325 @@ +"""FastAPI application factory, lifespan, middleware and error mapping. + +Beta-1 of the lab 3 daemon. It contains the required smoke bare minimums +(the version/info routes, the X-Request-Id / X-Instance-Id headers and the +request context) but not the full lab 3 feature set, which is implemented in +the lab 2 package that this package now wraps. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import logging +import sys +import time +import uuid +from contextlib import asynccontextmanager +from typing import Any, AsyncIterator + +import psycopg +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, RedirectResponse +from psycopg_pool import AsyncConnectionPool, PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.responses import Response + +from . import sessions as session_service +from .api import router +from .config import CONTRACT_PATH, Settings, load_settings +from .db import create_pool +from .errors import ApiProblem +from .object_store import ObjectStore, ObjectStoreUnavailable + + +access_logger = logging.getLogger("messenger.access") +lifecycle_logger = logging.getLogger("uvicorn.error") +if not access_logger.handlers: + access_handler = logging.StreamHandler(sys.stdout) + access_handler.setFormatter( + logging.Formatter("%(asctime)s %(levelname)s %(name)s %(message)s") + ) + access_logger.addHandler(access_handler) +access_logger.setLevel(logging.INFO) +access_logger.propagate = False + + +def request_id(request: Request) -> str: + return getattr(request.state, "request_id", str(uuid.uuid4())) + + +def problem_response( + request: Request, + status: int, + code: str, + message: str, + headers: dict[str, str] | None = None, +) -> JSONResponse: + current_request_id = request_id(request) + response_headers = {"X-Request-Id": current_request_id} + response_headers.update(headers or {}) + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": current_request_id, + } + }, + headers=response_headers, + ) + + +def load_contract() -> dict[str, Any]: + with CONTRACT_PATH.open(encoding="utf-8") as contract_file: + return json.load(contract_file) + + +async def _purge_loop(pool: AsyncConnectionPool) -> None: + """Drop long-expired session and idempotency rows; TTL is also read-path.""" + while True: + try: + await asyncio.sleep(60) + purged = await session_service.purge_expired(pool) + idempotency_purged = await session_service.purge_expired_idempotency(pool) + if purged or idempotency_purged: + lifecycle_logger.info( + "housekeeping sessions_purged=%s idempotency_purged=%s", + purged, + idempotency_purged, + ) + except asyncio.CancelledError: + raise + except Exception as exc: # noqa: BLE001 - housekeeping must not stop the API + lifecycle_logger.warning( + "sessions_purge_failed error=%s", type(exc).__name__ + ) + + +@asynccontextmanager +async def lifespan(application: FastAPI) -> AsyncIterator[None]: + settings: Settings = application.state.settings + pool: AsyncConnectionPool = application.state.pool + cache: session_service.SessionCache = application.state.cache + lifecycle_logger.info( + "server_starting env=%s instance_id=%s session_ttl_seconds=%s " + "idempotency_ttl_seconds=%s cache_enabled=%s pool_max=%s " + "statement_timeout_ms=%s", + settings.app_env, + settings.instance_id, + settings.session_ttl_seconds, + settings.idempotency_ttl_seconds, + cache.enabled, + settings.pool_max_size, + settings.statement_timeout_ms, + ) + try: + await pool.open(wait=True, timeout=10) + except Exception as exc: # noqa: BLE001 - start only with a usable database + lifecycle_logger.error( + "database_unavailable_at_startup error=%s", type(exc).__name__ + ) + raise + try: + await asyncio.to_thread(application.state.object_store.ensure_available) + except ObjectStoreUnavailable as exc: + lifecycle_logger.error("s3_unavailable_at_startup error=%s", type(exc).__name__) + await pool.close() + raise + lifecycle_logger.info( + "server_started env=%s instance_id=%s", settings.app_env, settings.instance_id + ) + if cache.enabled: + if await cache.ping(): + lifecycle_logger.info("cache_ready") + else: + lifecycle_logger.warning("cache_unavailable_using_database_only") + else: + lifecycle_logger.warning("cache_not_configured_using_database_only") + idempotency_stale_rows = await session_service.purge_expired_idempotency(pool) + if idempotency_stale_rows: + lifecycle_logger.info( + "idempotency_purge_at_start count=%s", idempotency_stale_rows + ) + purge_task = asyncio.create_task(_purge_loop(pool)) + try: + yield + finally: + lifecycle_logger.info("shutdown_requested") + purge_task.cancel() + with contextlib.suppress(BaseException): + await purge_task + await cache.close() + await pool.close() + lifecycle_logger.info("server_stopped") + + +def create_app(settings: Settings | None = None) -> FastAPI: + resolved = settings or load_settings() + application = FastAPI( + title="MTUSI Messenger — лабораторная 4", + version="1.4.0", + docs_url="/docs", + redoc_url=None, + lifespan=lifespan, + ) + application.state.settings = resolved + application.state.logger = lifecycle_logger + application.state.pool = create_pool( + resolved.database_url, + min_size=resolved.pool_min_size, + max_size=resolved.pool_max_size, + timeout_seconds=resolved.pool_timeout_seconds, + connect_timeout_seconds=resolved.connect_timeout_seconds, + statement_timeout_ms=resolved.statement_timeout_ms, + ) + application.state.cache = session_service.SessionCache( + resolved.cache_url, + timeout_seconds=resolved.cache_timeout_seconds, + ttl_cap=resolved.cache_ttl_seconds, + ) + application.state.object_store = ObjectStore(resolved) + application.openapi = load_contract # type: ignore[method-assign] + + @application.get("/version", include_in_schema=False) + async def version() -> dict[str, str]: + return {"version": resolved.version, "instance_id": resolved.instance_id} + + @application.get("/info", include_in_schema=False) + async def info() -> dict[str, str]: + return {"service": "mtusi-messenger", "lab": "4", "version": resolved.version} + + @application.middleware("http") + async def request_context(request: Request, call_next: Any) -> Response: + request.state.request_id = str(uuid.uuid4()) + started_at = time.monotonic() + status = 500 + try: + response = await call_next(request) + if request.method in {"POST", "PUT", "PATCH"}: + try: + # Authentication or resource checks can finish before the body + # is read. Drain any remainder without retaining it so the + # client receives that JSON response instead of a TCP reset. + async for _chunk in request.stream(): + pass + except RuntimeError: + pass + status = response.status_code + except ApiProblem as problem: + response = problem_response( + request, problem.status, problem.code, problem.message, problem.headers + ) + status = problem.status + except Exception: # noqa: BLE001 - never leak internals to the client + lifecycle_logger.exception( + "unhandled_error request_id=%s", request.state.request_id + ) + response = problem_response( + request, 500, "internal_error", "Internal server error" + ) + status = 500 + response.headers["X-Request-Id"] = request.state.request_id + # Every response carries the instance that served it, including errors, + # health and redirects, so a client can prove which process answered. + response.headers["X-Instance-Id"] = request.app.state.settings.instance_id + # The middleware still owns the response, but the idempotency check now + # depends on the endpoint having already derived the Idempotency-Key value + # from the contract and the security model, so the header is validated in + # api.py rather than here. + client = request.client.host if request.client is not None else "unknown" + client = request.client.host if request.client is not None else "unknown" + access_logger.info( + "request_complete request_id=%s method=%s path=%s status=%s " + "client=%s elapsed_ms=%.3f", + request.state.request_id, + request.method, + request.url.path, + status, + client, + (time.monotonic() - started_at) * 1000, + ) + return response + + @application.exception_handler(ApiProblem) + async def api_problem_handler(request: Request, problem: ApiProblem) -> JSONResponse: + return problem_response( + request, problem.status, problem.code, problem.message, problem.headers + ) + + @application.exception_handler(RequestValidationError) + async def validation_handler( + request: Request, _error: RequestValidationError + ) -> JSONResponse: + return problem_response( + request, 400, "invalid_request", "Request fields are invalid" + ) + + @application.exception_handler(StarletteHTTPException) + async def http_error_handler( + request: Request, error: StarletteHTTPException + ) -> JSONResponse: + if error.status_code in {404, 405}: + return problem_response(request, 404, "not_found", "Endpoint not found") + return problem_response( + request, error.status_code, "invalid_request", "Request is invalid" + ) + + @application.exception_handler(PoolTimeout) + async def pool_timeout_handler( + request: Request, error: PoolTimeout + ) -> JSONResponse: + lifecycle_logger.warning( + "database_pool_timeout request_id=%s error=%s", + request_id(request), + type(error).__name__, + ) + return problem_response( + request, 503, "unavailable", "Database is unavailable" + ) + + @application.exception_handler(psycopg.OperationalError) + @application.exception_handler(psycopg.InterfaceError) + async def database_unavailable_handler( + request: Request, error: Exception + ) -> JSONResponse: + lifecycle_logger.warning( + "database_unavailable request_id=%s error=%s", + request_id(request), + type(error).__name__, + ) + return problem_response( + request, 503, "unavailable", "Database is unavailable" + ) + + @application.exception_handler(psycopg.Error) + async def database_error_handler( + request: Request, error: psycopg.Error + ) -> JSONResponse: + lifecycle_logger.warning( + "database_error request_id=%s error=%s", + request_id(request), + type(error).__name__, + ) + return problem_response(request, 500, "internal_error", "Internal server error") + + @application.exception_handler(ObjectStoreUnavailable) + async def object_store_unavailable_handler( + request: Request, error: ObjectStoreUnavailable + ) -> JSONResponse: + lifecycle_logger.warning( + "s3_unavailable request_id=%s error=%s", + request_id(request), + type(error).__name__, + ) + return problem_response(request, 503, "unavailable", "Object storage is unavailable") + + @application.get("/", include_in_schema=False) + async def docs_redirect() -> RedirectResponse: + return RedirectResponse(url="/docs", status_code=307) + + application.include_router(router) + return application diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..4a456be9 --- /dev/null +++ b/src/config.py @@ -0,0 +1,193 @@ +"""Runtime configuration: environment variables plus file-based secrets.""" + +from __future__ import annotations + +import os +import re +import socket +from dataclasses import dataclass +from pathlib import Path + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +CONTRACT_PATH = PROJECT_ROOT / "contracts" / "openapi.json" +MIGRATIONS_DIR = Path(__file__).resolve().parent / "migrations" + +DEFAULT_PAGE_SIZE = 50 +MAX_PAGE_SIZE = 100 +MAX_JSON_BYTES = 1_048_576 +DEFAULT_SESSION_TTL_SECONDS = 3600 +DEFAULT_CACHE_TTL_SECONDS = 300 +# Idempotency-Key records are replayed for this long, then the key is reusable. +DEFAULT_IDEMPOTENCY_TTL_SECONDS = 86_400 +DEFAULT_MAX_UPLOAD_BYTES = 10_485_760 +DEFAULT_MAX_IMAGE_PIXELS = 40_000_000 +SUPPORTED_SESSION_STORES = {"postgres"} + +__version__ = "1.4.0" + +# Key pattern of the contract: 1..128 ASCII characters of the allowed set. +IDEMPOTENCY_KEY_PATTERN = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") + + +class ConfigurationError(Exception): + """Startup configuration is unusable; the process must not serve traffic.""" + + +@dataclass(frozen=True) +class Settings: + """Everything the process needs, resolved once at startup.""" + + database_url: str + cache_url: str | None + session_store: str + session_ttl_seconds: int + cache_ttl_seconds: int + pool_min_size: int + pool_max_size: int + pool_timeout_seconds: float + connect_timeout_seconds: int + statement_timeout_ms: int + cache_timeout_seconds: float + idempotency_ttl_seconds: int + instance_id: str + app_env: str + version: str + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + max_upload_bytes: int + max_image_pixels: int + + @property + def cache_enabled(self) -> bool: + return self.cache_url is not None + + +def _secret(file_variable: str, plain_variable: str) -> str | None: + """Read a secret from a mounted file, falling back to a plain variable.""" + path_value = os.environ.get(file_variable, "").strip() + if path_value: + path = Path(path_value) + try: + value = path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise ConfigurationError( + file_variable + " points to an unreadable file (" + type(exc).__name__ + ")" + ) from None + if not value: + raise ConfigurationError(file_variable + " points to an empty file") + return value + plain = os.environ.get(plain_variable, "").strip() + return plain or None + + +def _integer(variable: str, default: int, minimum: int, maximum: int) -> int: + raw = os.environ.get(variable, "").strip() + if not raw: + return default + try: + value = int(raw) + except ValueError: + raise ConfigurationError(variable + " must be an integer") from None + if not minimum <= value <= maximum: + raise ConfigurationError( + variable + " must be between " + str(minimum) + " and " + str(maximum) + ) + return value + + +def _number(variable: str, default: float, minimum: float, maximum: float) -> float: + raw = os.environ.get(variable, "").strip() + if not raw: + return default + try: + value = float(raw) + except ValueError: + raise ConfigurationError(variable + " must be a number") from None + if not minimum <= value <= maximum: + raise ConfigurationError( + variable + " must be between " + str(minimum) + " and " + str(maximum) + ) + return value + + +def load_settings() -> Settings: + """Resolve settings once; a bad configuration fails fast before serving.""" + database_url = _secret("DATABASE_URL_FILE", "DATABASE_URL") + if database_url is None: + raise ConfigurationError("DATABASE_URL_FILE or DATABASE_URL is required") + cache_url = _secret("CACHE_URL_FILE", "CACHE_URL") + session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() + if session_store not in SUPPORTED_SESSION_STORES: + raise ConfigurationError( + "SESSION_STORE=" + + session_store + + " is not implemented: sessions are stored in PostgreSQL " + "(the durable source of truth) and cached in Valkey. Use SESSION_STORE=postgres." + ) + pool_min = _integer("DB_POOL_MIN", 1, 0, 100) + pool_max = _integer("DB_POOL_MAX", 10, 1, 200) + if pool_min > pool_max: + raise ConfigurationError("DB_POOL_MIN must not exceed DB_POOL_MAX") + s3_endpoint = os.environ.get("S3_ENDPOINT", "").strip() + if not s3_endpoint.startswith(("http://", "https://")): + raise ConfigurationError("S3_ENDPOINT must be an http(s) URL") + s3_bucket = os.environ.get("S3_BUCKET", "").strip() + if not re.fullmatch(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]", s3_bucket): + raise ConfigurationError("S3_BUCKET must be a valid bucket name") + s3_access_key_id = _secret("S3_ACCESS_KEY_ID_FILE", "S3_ACCESS_KEY_ID") + s3_secret_access_key = _secret("S3_SECRET_ACCESS_KEY_FILE", "S3_SECRET_ACCESS_KEY") + if s3_access_key_id is None or s3_secret_access_key is None: + raise ConfigurationError("S3 API credentials are required") + return Settings( + database_url=database_url, + cache_url=cache_url, + session_store=session_store, + session_ttl_seconds=_integer( + "SESSION_TTL_SECONDS", DEFAULT_SESSION_TTL_SECONDS, 1, 2_592_000 + ), + cache_ttl_seconds=_integer( + "CACHE_TTL_SECONDS", DEFAULT_CACHE_TTL_SECONDS, 1, 86_400 + ), + pool_min_size=pool_min, + pool_max_size=pool_max, + pool_timeout_seconds=_number("DB_POOL_TIMEOUT_SECONDS", 2.0, 0.1, 60.0), + connect_timeout_seconds=_integer("DB_CONNECT_TIMEOUT_SECONDS", 2, 1, 60), + statement_timeout_ms=_integer("DB_STATEMENT_TIMEOUT_MS", 5000, 100, 120_000), + cache_timeout_seconds=_number("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0), + idempotency_ttl_seconds=_integer( + "IDEMPOTENCY_TTL_SECONDS", + DEFAULT_IDEMPOTENCY_TTL_SECONDS, + 60, + 604_800, + ), + # Stable for the life of the process. Inside a container this is the + # short container id, which is what scripts/stop-one.sh maps back to a + # single replica; outside Docker it stays a readable hostname. + instance_id=os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname(), + app_env=os.environ.get("APP_ENV", "lab").strip() or "lab", + version=__version__, + s3_endpoint=s3_endpoint, + s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + s3_bucket=s3_bucket, + s3_access_key_id=s3_access_key_id, + s3_secret_access_key=s3_secret_access_key, + max_upload_bytes=_integer( + "MAX_UPLOAD_BYTES", DEFAULT_MAX_UPLOAD_BYTES, 1, DEFAULT_MAX_UPLOAD_BYTES + ), + max_image_pixels=_integer( + "MAX_IMAGE_PIXELS", DEFAULT_MAX_IMAGE_PIXELS, 1, DEFAULT_MAX_IMAGE_PIXELS + ), + ) + + +def load_address() -> tuple[str, int]: + """Read and validate the HTTP listen address from the environment.""" + host = os.environ.get("APP_HOST", "0.0.0.0").strip() + if not host: + raise SystemExit("APP_HOST must not be empty") + port = _integer("APP_PORT", 8080, 1, 65535) + return host, port diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..0705124f --- /dev/null +++ b/src/db.py @@ -0,0 +1,49 @@ +"""PostgreSQL access: a bounded async pool with explicit timeouts.""" + +from __future__ import annotations + +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool + + +def connection_options(statement_timeout_ms: int) -> str: + """Server-side limits so a slow query or an idle transaction cannot pile up.""" + return ( + "-c statement_timeout=" + + str(statement_timeout_ms) + + " -c lock_timeout=2000 -c idle_in_transaction_session_timeout=10000" + + " -c timezone=UTC" + ) + + +def create_pool( + database_url: str, + *, + min_size: int, + max_size: int, + timeout_seconds: float, + connect_timeout_seconds: int, + statement_timeout_ms: int, +) -> AsyncConnectionPool: + """Build a closed pool; the lifespan opens it and closes it on shutdown.""" + return AsyncConnectionPool( + conninfo=database_url, + min_size=min_size, + max_size=max_size, + timeout=timeout_seconds, + open=False, + check=AsyncConnectionPool.check_connection, + kwargs={ + "row_factory": dict_row, + "connect_timeout": connect_timeout_seconds, + "options": connection_options(statement_timeout_ms), + "application_name": "mtusi-messenger-api", + }, + name="messenger-pool", + ) + + +async def ping(pool: AsyncConnectionPool) -> None: + """Cheap readiness probe; raises when PostgreSQL is not usable.""" + async with pool.connection() as connection: + await connection.execute("SELECT 1") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..47ab154e --- /dev/null +++ b/src/errors.py @@ -0,0 +1,26 @@ +"""Errors that may safely be returned through the public API.""" + +from __future__ import annotations + +from typing import Mapping + + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiProblem(Exception): + """An expected API failure with a stable status, machine code and headers.""" + + def __init__( + self, + status: int, + code: str, + message: str, + headers: Mapping[str, str] | None = None, + ) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = dict(headers or {}) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..0b098e8b --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,93 @@ +"""Versioned SQL migration runner. + +Every run is one transaction guarded by a PostgreSQL advisory lock, so a second +process (or a restarted container) cannot apply the same file twice and a failed +migration leaves the schema unchanged. +""" + +from __future__ import annotations + +import sys +import time +from pathlib import Path + +import psycopg + +if __package__ in {None, ""}: + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from src.config import MIGRATIONS_DIR, ConfigurationError, load_settings # noqa: E402 + + +ADVISORY_LOCK_KEY = 915_127_001 +CONNECT_RETRY_SECONDS = 60 + + +def _version(path: Path) -> int: + prefix = path.name.split("_", 1)[0] + return int(prefix) + + +def apply_migrations(connection: psycopg.Connection, directory: Path) -> list[str]: + applied: list[str] = [] + with connection.transaction(): + connection.execute("SELECT pg_advisory_xact_lock(%s)", (ADVISORY_LOCK_KEY,)) + connection.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version integer PRIMARY KEY, + filename text NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + rows = connection.execute("SELECT version FROM schema_migrations").fetchall() + done = {row[0] for row in rows} + for path in sorted(directory.glob("*.sql"), key=_version): + version = _version(path) + if version in done: + continue + connection.execute(path.read_text(encoding="utf-8")) + connection.execute( + "INSERT INTO schema_migrations (version, filename) VALUES (%s, %s)", + (version, path.name), + ) + applied.append(path.name) + return applied + + +def main() -> None: + try: + settings = load_settings() + except ConfigurationError as error: + print("configuration_error " + str(error), file=sys.stderr) + raise SystemExit(1) from None + deadline = time.monotonic() + CONNECT_RETRY_SECONDS + while True: + try: + with psycopg.connect( + settings.database_url, + connect_timeout=5, + options="-c timezone=UTC", + application_name="mtusi-messenger-migrate", + ) as connection: + applied = apply_migrations(connection, MIGRATIONS_DIR) + break + except psycopg.OperationalError as error: + if time.monotonic() >= deadline: + print( + "database_unreachable error=" + type(error).__name__, file=sys.stderr + ) + raise SystemExit(1) from None + time.sleep(1) + except psycopg.Error as error: + print("migration_failed error=" + type(error).__name__, file=sys.stderr) + raise SystemExit(1) from None + if applied: + print("migrations_applied " + ",".join(applied)) + else: + print("migrations_up_to_date") + + +if __name__ == "__main__": + main() diff --git a/src/migrations/001_users.sql b/src/migrations/001_users.sql new file mode 100644 index 00000000..2d0d88b2 --- /dev/null +++ b/src/migrations/001_users.sql @@ -0,0 +1,13 @@ +-- Users own credentials and identity. The password column stores an Argon2id hash. +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_length CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_password_hash_present CHECK (char_length(password_hash) > 0) +); + +CREATE UNIQUE INDEX users_username_unique ON users (username); diff --git a/src/migrations/002_chats_messages.sql b/src/migrations/002_chats_messages.sql new file mode 100644 index 00000000..aeebc9ea --- /dev/null +++ b/src/migrations/002_chats_messages.sql @@ -0,0 +1,34 @@ +-- Chats, explicit membership and message history. +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT chats_title_length CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- Chat lists are ordered by (created_at, id) for the same keyset cursor. +CREATE INDEX chats_order_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + joined_at timestamptz NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +-- "Chats of the current user" and the membership check on every read. +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + text text NOT NULL, + created_at timestamptz NOT NULL, + CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Serves history keyset pagination exactly: equality on chat_id, then the +-- (created_at, id) tuple used by both ORDER BY and the cursor predicate. +-- Justification and EXPLAIN (ANALYZE, BUFFERS) are in evidence/explain-history.txt. +CREATE INDEX messages_chat_order_idx ON messages (chat_id, created_at, id); diff --git a/src/migrations/003_sessions.sql b/src/migrations/003_sessions.sql new file mode 100644 index 00000000..c55296e0 --- /dev/null +++ b/src/migrations/003_sessions.sql @@ -0,0 +1,16 @@ +-- Opaque server-side sessions: only a SHA-256 of the token is stored. +CREATE TABLE sessions ( + id uuid PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + token_hash text NOT NULL, + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_length CHECK (char_length(token_hash) = 64), + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); + +-- Every authenticated request looks a token up by hash. +CREATE UNIQUE INDEX sessions_token_hash_unique ON sessions (token_hash); +CREATE INDEX sessions_user_idx ON sessions (user_id); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/src/migrations/004_idempotency.sql b/src/migrations/004_idempotency.sql new file mode 100644 index 00000000..4330a32e --- /dev/null +++ b/src/migrations/004_idempotency.sql @@ -0,0 +1,11 @@ +-- Idempotency-Key records for the lab 3 message write path. +CREATE TABLE idempotency_keys ( + key text PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + request_sha256 text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + CONSTRAINT idempotency_key_format CHECK (char_length(key) <= 128), + CONSTRAINT idempotency_request_sha256_length CHECK (char_length(request_sha256) = 64) +); + +CREATE INDEX idempotency_chat_idx ON idempotency_keys (chat_id); diff --git a/src/migrations/005_indexes.sql b/src/migrations/005_indexes.sql new file mode 100644 index 00000000..a317002e --- /dev/null +++ b/src/migrations/005_indexes.sql @@ -0,0 +1,2 @@ +-- Composite index covering the idempotency queries: by chat and by key. +CREATE INDEX idempotency_keys_chat_order_idx ON idempotency_keys (chat_id, key); diff --git a/src/migrations/006_idempotency_message.sql b/src/migrations/006_idempotency_message.sql new file mode 100644 index 00000000..1f0e4ac0 --- /dev/null +++ b/src/migrations/006_idempotency_message.sql @@ -0,0 +1,3 @@ +-- Add message_id to idempotency_keys so replays can return the original message. +ALTER TABLE idempotency_keys ADD COLUMN IF NOT EXISTS message_id uuid REFERENCES messages (id) ON DELETE CASCADE; +CREATE INDEX IF NOT EXISTS idempotency_message_idx ON idempotency_keys (message_id); \ No newline at end of file diff --git a/src/migrations/007_idempotency_scope.sql b/src/migrations/007_idempotency_scope.sql new file mode 100644 index 00000000..81e21f64 --- /dev/null +++ b/src/migrations/007_idempotency_scope.sql @@ -0,0 +1,25 @@ +-- An idempotency key belongs to the authenticated user and target chat, not +-- to the entire service. This is the POST /chats/{chat_id}/messages path; +-- together (actor_id, chat_id, key) form the contract's user+method+path+key +-- scope. +ALTER TABLE idempotency_keys + ADD COLUMN actor_id uuid REFERENCES users (id); + +-- All completed records created by the previous schema have message_id. +-- Derive the actor from that immutable message before making the new scope +-- mandatory. Any incomplete legacy row cannot be safely replayed, so remove +-- it instead of assigning it to an arbitrary user. +UPDATE idempotency_keys AS ik +SET actor_id = m.sender_id +FROM messages AS m +WHERE ik.message_id = m.id + AND ik.actor_id IS NULL; + +DELETE FROM idempotency_keys WHERE actor_id IS NULL; + +ALTER TABLE idempotency_keys + ALTER COLUMN actor_id SET NOT NULL, + DROP CONSTRAINT idempotency_keys_pkey, + ADD PRIMARY KEY (actor_id, chat_id, key); + +CREATE INDEX idempotency_created_at_idx ON idempotency_keys (created_at); diff --git a/src/migrations/008_attachments.sql b/src/migrations/008_attachments.sql new file mode 100644 index 00000000..7ebd7ce9 --- /dev/null +++ b/src/migrations/008_attachments.sql @@ -0,0 +1,29 @@ +-- Lab4 binary objects live in private S3; this table stores only metadata. +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + object_key text NOT NULL UNIQUE, + filename text NOT NULL, + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes bigint NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 char(64) NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + width integer NOT NULL CHECK (width > 0), + height integer NOT NULL CHECK (height > 0), + status text NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + CHECK (char_length(filename) BETWEEN 1 AND 255) +); + +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position) +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/src/migrations/009_attachment_only_messages.sql b/src/migrations/009_attachment_only_messages.sql new file mode 100644 index 00000000..eb6c25c3 --- /dev/null +++ b/src/migrations/009_attachment_only_messages.sql @@ -0,0 +1,5 @@ +-- Lab4 permits an empty text only when attachment_ids is non-empty; the API +-- validates that cross-table rule before this row is inserted. +ALTER TABLE messages DROP CONSTRAINT messages_text_length; +ALTER TABLE messages + ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000); diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..7bb9cb69 --- /dev/null +++ b/src/models.py @@ -0,0 +1,98 @@ +"""Strict request models for the lab 2 API.""" + +from __future__ import annotations + +import re +import uuid + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +from .config import IDEMPOTENCY_KEY_PATTERN + + +USERNAME_PATTERN = re.compile(r"^[a-z0-9_]{3,32}$") +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +def _not_blank(value: str, field: str) -> str: + if not value.strip(): + raise ValueError(field + " must not be blank") + return value + + +class CreateUser(StrictModel): + """Registration body; the password never appears in a response.""" + + username: str = Field( + min_length=3, max_length=32, pattern=USERNAME_PATTERN.pattern + ) + display_name: str = Field(min_length=1, max_length=100) + password: str = Field( + min_length=PASSWORD_MIN_LENGTH, max_length=PASSWORD_MAX_LENGTH + ) + + @field_validator("display_name") + @classmethod + def display_name_is_not_blank(cls, value: str) -> str: + return _not_blank(value, "display_name") + + @field_validator("password") + @classmethod + def password_length_in_code_points(cls, value: str) -> str: + # Python len() counts Unicode code points, which is what the contract uses. + if not PASSWORD_MIN_LENGTH <= len(value) <= PASSWORD_MAX_LENGTH: + raise ValueError("password length is out of range") + return value + + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_is_not_blank(cls, value: str) -> str: + return _not_blank(value, "title") + + @field_validator("member_ids") + @classmethod + def members_are_valid_and_unique(cls, value: list[str]) -> list[str]: + try: + normalized = [str(uuid.UUID(member_id)) for member_id in value] + except (ValueError, AttributeError): + raise ValueError("member_ids must contain UUID strings") from None + if len(normalized) != len(set(normalized)): + raise ValueError("member_ids must be unique") + return normalized + + +class CreateMessage(StrictModel): + text: str = Field(max_length=4000) + attachment_ids: list[str] = Field(default_factory=list, max_length=10) + + @field_validator("text") + @classmethod + def text_is_not_blank(cls, value: str) -> str: + # Lab4 permits an empty text only when at least one attachment is sent; + # the model-level check below enforces that relation. + return value + + @field_validator("attachment_ids") + @classmethod + def attachment_ids_are_unique_uuids(cls, value: list[str]) -> list[str]: + try: + normalized = [str(uuid.UUID(attachment_id)) for attachment_id in value] + except (ValueError, AttributeError): + raise ValueError("attachment_ids must contain UUID strings") from None + if len(normalized) != len(set(normalized)): + raise ValueError("attachment_ids must be unique") + return normalized + + def model_post_init(self, __context: object) -> None: + if not self.text.strip() and not self.attachment_ids: + raise ValueError("text must not be blank without attachments") diff --git a/src/object_store.py b/src/object_store.py new file mode 100644 index 00000000..8b63b222 --- /dev/null +++ b/src/object_store.py @@ -0,0 +1,77 @@ +"""Small bounded wrapper around the private S3-compatible object store.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import BinaryIO +from urllib.parse import urlparse + +from minio import Minio +from minio.error import S3Error + +from .config import Settings + + +class ObjectStoreUnavailable(Exception): + """The S3 dependency cannot safely complete the requested operation.""" + + +class ObjectStore: + def __init__(self, settings: Settings) -> None: + parsed = urlparse(settings.s3_endpoint) + self.bucket = settings.s3_bucket + self.client = Minio( + parsed.netloc, + access_key=settings.s3_access_key_id, + secret_key=settings.s3_secret_access_key, + secure=parsed.scheme == "https", + region=settings.s3_region, + ) + + def ensure_available(self) -> None: + """Probe the bucket created by the separate privileged bootstrap job.""" + try: + if not self.client.bucket_exists(self.bucket): + raise ObjectStoreUnavailable("bucket_missing") + except ObjectStoreUnavailable: + raise + except Exception as exc: # MinIO also raises transport exceptions + raise ObjectStoreUnavailable(type(exc).__name__) from None + + def put(self, key: str, source: BinaryIO, size: int, content_type: str) -> None: + try: + source.seek(0) + self.client.put_object( + self.bucket, key, source, size, content_type=content_type + ) + except Exception as exc: + raise ObjectStoreUnavailable(type(exc).__name__) from None + + def delete(self, key: str) -> None: + try: + self.client.remove_object(self.bucket, key) + except Exception: + # Best-effort compensation is intentionally silent here; callers + # return the DB/S3 failure without leaking storage internals. + pass + + def ensure_object(self, key: str) -> None: + """Fail before HTTP headers if a ready metadata row lost its object.""" + try: + self.client.stat_object(self.bucket, key) + except Exception as exc: + raise ObjectStoreUnavailable(type(exc).__name__) from None + + def chunks(self, key: str, chunk_size: int = 64 * 1024) -> Iterator[bytes]: + response = None + try: + response = self.client.get_object(self.bucket, key) + yield from response.stream(chunk_size) + except S3Error as exc: + raise ObjectStoreUnavailable(type(exc).__name__) from None + except Exception as exc: + raise ObjectStoreUnavailable(type(exc).__name__) from None + finally: + if response is not None: + response.close() + response.release_conn() diff --git a/src/pagination.py b/src/pagination.py new file mode 100644 index 00000000..7b5e653c --- /dev/null +++ b/src/pagination.py @@ -0,0 +1,84 @@ +"""Opaque, resource-bound keyset cursors, timestamp formatting and page shaping.""" + +from __future__ import annotations + +import base64 +import binascii +import json +import uuid +from datetime import datetime, timezone +from typing import Any + +from .errors import ApiProblem + + +def format_timestamp(value: datetime) -> str: + """RFC 3339 UTC with microseconds and a trailing Z, as the contract requires.""" + return ( + value.astimezone(timezone.utc) + .isoformat(timespec="microseconds") + .replace("+00:00", "Z") + ) + + +def parse_timestamp(value: str) -> datetime: + """Parse an RFC 3339 timestamp from a cursor; reject anything else.""" + candidate = value[:-1] + "+00:00" if value.endswith("Z") else value + parsed = datetime.fromisoformat(candidate) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def encode_cursor(resource: str, created_at: datetime, record_id: str) -> str: + payload = json.dumps( + { + "resource": resource, + "created_at": format_timestamp(created_at), + "id": record_id, + }, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def decode_cursor(value: str, resource: str) -> tuple[datetime, uuid.UUID]: + """Decode a cursor and bind it to the resource it was issued for.""" + try: + padding = "=" * (-len(value) % 4) + raw = base64.b64decode(value + padding, altchars=b"-_", validate=True) + payload: Any = json.loads(raw.decode("utf-8")) + if not isinstance(payload, dict) or set(payload) != { + "resource", + "created_at", + "id", + }: + raise ValueError("unexpected cursor payload") + if payload["resource"] != resource: + raise ValueError("cursor belongs to another resource") + created_at = payload["created_at"] + record_id = payload["id"] + if not isinstance(created_at, str) or not isinstance(record_id, str): + raise ValueError("cursor fields have wrong types") + return parse_timestamp(created_at), uuid.UUID(record_id) + except ( + ValueError, + TypeError, + UnicodeError, + binascii.Error, + json.JSONDecodeError, + ): + raise ApiProblem(400, "invalid_request", "Cursor is invalid") from None + + +def build_page(items: list[dict[str, Any]], resource: str, limit: int) -> dict[str, Any]: + """Shape a limit+1 storage slice into the public page and its next cursor.""" + page = items[:limit] + next_cursor = None + if len(items) > limit and page: + last = page[-1] + next_cursor = encode_cursor( + resource, parse_timestamp(last["created_at"]), last["id"] + ) + return {"items": page, "next_cursor": next_cursor} diff --git a/src/reconcile.py b/src/reconcile.py new file mode 100644 index 00000000..eb0b55de --- /dev/null +++ b/src/reconcile.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Reconcile private S3 originals against PostgreSQL attachment metadata. + +Read-only dry-run is the default. ``--apply`` deletes only orphan objects under +the server-owned prefix after a grace period; missing metadata rows are reported +but never deleted automatically. +""" +from __future__ import annotations + +import argparse +import json +import time +from datetime import datetime, timezone + +import psycopg + +from .config import load_settings +from .object_store import ObjectStore, ObjectStoreUnavailable + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--apply", action="store_true") + parser.add_argument("--grace-seconds", type=int, default=3600) + args = parser.parse_args() + if not 60 <= args.grace_seconds <= 604800: + raise SystemExit("grace-seconds must be 60..604800") + settings = load_settings() + store = ObjectStore(settings) + started = time.monotonic() + try: + store.ensure_available() + with psycopg.connect(settings.database_url) as connection: + metadata_keys = {row[0] for row in connection.execute("SELECT object_key FROM attachments")} + objects = { + item.object_name: item.last_modified + for item in store.client.list_objects(settings.s3_bucket, prefix="attachments/", recursive=True) + if item.object_name and item.last_modified + } + except Exception as exc: + raise SystemExit("reconcile dependency unavailable: " + type(exc).__name__) from None + missing = metadata_keys - set(objects) + orphan = set(objects) - metadata_keys + now = datetime.now(timezone.utc) + eligible = {key for key in orphan if (now - objects[key]).total_seconds() >= args.grace_seconds} + deleted = 0 + if args.apply: + try: + for key in eligible: + store.client.remove_object(settings.s3_bucket, key) + deleted += 1 + except Exception as exc: + raise SystemExit("reconcile S3 delete failed: " + type(exc).__name__) from None + print(json.dumps({ + "mode": "apply" if args.apply else "dry_run", "grace_seconds": args.grace_seconds, + "metadata_rows": len(metadata_keys), "s3_objects": len(objects), + "missing_metadata_objects": len(missing), "orphan_objects": len(orphan), + "eligible_orphans": len(eligible), "deleted_orphans": deleted, + "elapsed_ms": round((time.monotonic() - started) * 1000, 3), + "safety": "missing metadata retained; fresh and referenced objects never deleted", + }, ensure_ascii=False, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/src/s3_bootstrap.py b/src/s3_bootstrap.py new file mode 100644 index 00000000..477a1bcb --- /dev/null +++ b/src/s3_bootstrap.py @@ -0,0 +1,37 @@ +"""Create the private bucket once, using a credential unavailable to the API.""" +from __future__ import annotations + +import os +from pathlib import Path +from urllib.parse import urlparse + +from minio import Minio + + +def secret(variable: str) -> str: + value = Path(os.environ[variable]).read_text(encoding="utf-8").strip() + if not value: + raise SystemExit(variable + " is empty") + return value + + +def main() -> None: + endpoint = os.environ.get("S3_ENDPOINT", "").strip() + bucket = os.environ.get("S3_BUCKET", "").strip() + parsed = urlparse(endpoint) + if parsed.scheme not in {"http", "https"} or not parsed.netloc or not bucket: + raise SystemExit("S3_ENDPOINT and S3_BUCKET are required") + client = Minio( + parsed.netloc, + access_key=secret("S3_BOOTSTRAP_ACCESS_KEY_ID_FILE"), + secret_key=secret("S3_BOOTSTRAP_SECRET_ACCESS_KEY_FILE"), + secure=parsed.scheme == "https", + region=os.environ.get("S3_REGION", "us-east-1"), + ) + if not client.bucket_exists(bucket): + client.make_bucket(bucket) + print("private_s3_bucket_ready") + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..791af2c4 --- /dev/null +++ b/src/security.py @@ -0,0 +1,43 @@ +"""Password hashing with Argon2id (library KDF, salted, versioned parameters).""" + +from __future__ import annotations + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + + +# OWASP-recommended Argon2id baseline: 19 MiB, 2 iterations, 1 lane. +_hasher = PasswordHasher( + time_cost=2, + memory_cost=19456, + parallelism=1, + hash_len=32, + salt_len=16, +) + +# Verified against when a username is unknown, so the failure path costs the same +# as a wrong password and does not leak username existence through timing. +_DUMMY_HASH = _hasher.hash("dummy-password-not-used-for-any-account") + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password: str, encoded: str | None) -> bool: + """Return True only for a matching password and a parseable hash.""" + if encoded is None: + verify_password_unknown_user(password) + return False + try: + return _hasher.verify(encoded, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +def verify_password_unknown_user(password: str) -> None: + """Burn comparable CPU time when no account exists for the username.""" + try: + _hasher.verify(_DUMMY_HASH, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return diff --git a/src/sessions.py b/src/sessions.py new file mode 100644 index 00000000..14000bea --- /dev/null +++ b/src/sessions.py @@ -0,0 +1,337 @@ +"""Opaque Bearer sessions: PostgreSQL is the source of truth, Valkey is a cache. + +Design (see REPORT.md): + +* ``sessions`` in PostgreSQL owns identity, expiry and revocation. +* Valkey stores the same session under ``session:`` for at most + ``CACHE_TTL_SECONDS`` and never longer than the session's remaining lifetime. +* Revocation writes a tombstone instead of relying on ``DEL`` alone, so a stale + cache entry cannot authenticate after logout. +* When Valkey is unavailable every request falls back to PostgreSQL. A cache + error is never treated as "authenticated", and a cache hit is only trusted + because it came from a session row that PostgreSQL confirmed at login time. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import secrets +import time +import uuid +from datetime import datetime, timezone +from typing import Any + +import redis.asyncio as redis +from psycopg_pool import AsyncConnectionPool +from redis.backoff import NoBackoff +from redis.retry import Retry + +from . import store +from .config import DEFAULT_IDEMPOTENCY_TTL_SECONDS as IDEMPOTENCY_TTL_SECONDS +from .errors import ApiProblem +from .pagination import format_timestamp + +logger = logging.getLogger("messenger.sessions") + +TOKEN_BYTES = 48 # 384 bits of entropy, well above the required 256. +CACHE_PREFIX = "session:" +REVOKED = "revoked" +# After a cache error, skip Valkey for a few seconds instead of paying the +# connect timeout on every request (lookup and remember would cost two each). +FAILURE_COOLDOWN_SECONDS = 5.0 +# Bound on revocations remembered while the cache cannot accept a tombstone. +MAX_PENDING_REVOCATIONS = 4096 + +HIT = "hit" +MISS = "miss" +TOMBSTONE = "revoked" +ERROR = "error" + + +def token_hash(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +class SessionCache: + """Small, failure-tolerant Valkey client used only for session lookups.""" + + def __init__(self, url: str | None, *, timeout_seconds: float, ttl_cap: int) -> None: + self.ttl_cap = max(1, ttl_cap) + self.timeout_seconds = timeout_seconds + # hashed token -> epoch at which the session would be expired anyway. + # PostgreSQL already holds the authoritative revocation; this only keeps + # logout immediate for this process while the cache is unreachable. + self._pending: dict[str, float] = {} + self._disabled_until = 0.0 + self._degraded = False + self.client: redis.Redis | None = None + if url: + self.client = redis.Redis.from_url( + url, + socket_timeout=timeout_seconds, + socket_connect_timeout=timeout_seconds, + decode_responses=True, + health_check_interval=30, + # A cache must fail fast: one attempt, no retry loop, so a dead + # Valkey costs at most one timeout before PostgreSQL answers. + retry=Retry(NoBackoff(), 0), + retry_on_error=[], + ) + + @property + def enabled(self) -> bool: + return self.client is not None + + def _ready(self) -> bool: + """True when a cache call is worth attempting right now.""" + return self.client is not None and time.monotonic() >= self._disabled_until + + def _unavailable(self, operation: str, exc: BaseException) -> None: + self._disabled_until = time.monotonic() + FAILURE_COOLDOWN_SECONDS + # One warning per outage, not one per request. + if not self._degraded: + self._degraded = True + logger.warning( + "cache_unavailable operation=%s error=%s cooldown_seconds=%s", + operation, + type(exc).__name__, + FAILURE_COOLDOWN_SECONDS, + ) + + def _ok(self) -> None: + self._degraded = False + + def remember_revocation(self, hashed: str, remaining: float) -> None: + """Ask this process to reject a token even if the cache cannot store it.""" + if len(self._pending) >= MAX_PENDING_REVOCATIONS: + # Drop the entries closest to expiry: they are the least useful. + keep = len(self._pending) // 4 + for key, _ in sorted(self._pending.items(), key=lambda item: item[1])[:keep]: + self._pending.pop(key, None) + self._pending[hashed] = time.time() + max(1.0, remaining) + + def is_locally_revoked(self, hashed: str) -> bool: + deadline = self._pending.get(hashed) + if deadline is None: + return False + if deadline <= time.time(): + self._pending.pop(hashed, None) + return False + return True + + async def _flush_revocations(self) -> None: + """Re-apply tombstones that could not be written during an outage.""" + for hashed, deadline in list(self._pending.items()): + remaining = deadline - time.time() + if remaining <= 0: + self._pending.pop(hashed, None) + continue + try: + await self.client.set(self._key(hashed), REVOKED, ex=self._ttl(remaining)) + except Exception as exc: # noqa: BLE001 - still revoked in PostgreSQL + self._unavailable("flush_revocation", exc) + return + self._pending.pop(hashed, None) + + async def ping(self) -> bool: + if self.client is None: + return False + try: + alive = bool(await self.client.ping()) + except Exception as exc: # noqa: BLE001 - any client error means "no cache" + logger.warning("cache_ping_failed error=%s", type(exc).__name__) + return False + self._disabled_until = 0.0 + self._ok() + return alive + + async def close(self) -> None: + if self.client is None: + return + try: + await self.client.aclose() + except Exception as exc: # noqa: BLE001 - shutdown must not fail on cache + logger.warning("cache_close_failed error=%s", type(exc).__name__) + + def _key(self, hashed: str) -> str: + return CACHE_PREFIX + hashed + + def _ttl(self, seconds: float) -> int: + return max(1, min(int(seconds), self.ttl_cap)) + + async def lookup(self, hashed: str) -> tuple[str, dict[str, Any] | None]: + """Return (state, payload); every cache failure degrades to a DB read.""" + if not self._ready(): + return MISS, None + if self._pending: + await self._flush_revocations() + if not self._ready(): + return MISS, None + try: + raw = await self.client.get(self._key(hashed)) + self._ok() + except Exception as exc: # noqa: BLE001 - cache must never break auth + self._unavailable("lookup", exc) + return ERROR, None + if raw is None: + return MISS, None + if raw == REVOKED: + return TOMBSTONE, None + try: + payload = json.loads(raw) + user = payload["user"] + user["id"] = str(uuid.UUID(user["id"])) + payload["expires_epoch"] = float(payload["expires_epoch"]) + payload["user"] = user + except (ValueError, KeyError, TypeError, json.JSONDecodeError): + logger.warning("cache_payload_invalid") + return MISS, None + return HIT, payload + + async def remember(self, hashed: str, user: dict[str, Any], expires_at: datetime) -> None: + if not self._ready(): + return + remaining = (expires_at - datetime.now(timezone.utc)).total_seconds() + if remaining <= 0: + return + # Cache the immutable public user snapshot, so a hit needs no SQL at all. + # Users cannot be renamed or deleted in the mandatory API of lab 2. + payload = { + "user": user, + "expires_epoch": expires_at.timestamp(), + } + try: + await self.client.set( + self._key(hashed), + json.dumps(payload, ensure_ascii=False), + ex=self._ttl(remaining), + ) + self._ok() + except Exception as exc: # noqa: BLE001 - a missed cache write is not fatal + self._unavailable("remember", exc) + + async def revoke(self, hashed: str, expires_at: datetime | None) -> None: + """Write a tombstone so a stale entry cannot authenticate after logout.""" + remaining = 60.0 + if expires_at is not None: + remaining = max(1.0, (expires_at - datetime.now(timezone.utc)).total_seconds()) + # PostgreSQL already revoked the session; this keeps the fast path honest + # even if the tombstone cannot be written right now. + self.remember_revocation(hashed, remaining) + if not self._ready(): + return + try: + await self.client.set(self._key(hashed), REVOKED, ex=self._ttl(remaining)) + self._pending.pop(hashed, None) + self._ok() + except Exception as exc: # noqa: BLE001 - PostgreSQL already revoked the session + self._unavailable("revoke", exc) + + +def _user_payload(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_timestamp(row["created_at"]), + } + + +async def create_session( + pool: AsyncConnectionPool, + cache: SessionCache, + *, + user_id: uuid.UUID, + ttl_seconds: int, +) -> tuple[str, datetime]: + token = secrets.token_urlsafe(TOKEN_BYTES) + hashed = token_hash(token) + async with pool.connection() as connection: + async with connection.cursor() as cursor: + expires_at = await store.insert_session( + cursor, user_id=user_id, token_hash=hashed, ttl_seconds=ttl_seconds + ) + user = await store.user_by_id(cursor, user_id) + if user is not None: + await cache.remember(hashed, _user_payload(user), expires_at) + return token, expires_at + + +async def resolve_session( + pool: AsyncConnectionPool, cache: SessionCache, token: str +) -> tuple[uuid.UUID, dict[str, Any]]: + """Return the authenticated user or raise 401 with a Bearer challenge.""" + hashed = token_hash(token) + if cache.is_locally_revoked(hashed): + raise ApiProblem( + 401, + "unauthorized", + "Session token is expired or revoked", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + state, _payload = await cache.lookup(hashed) + if state == TOMBSTONE: + raise ApiProblem( + 401, + "unauthorized", + "Session token is expired or revoked", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + + # PostgreSQL is the authoritative revocation store. A cache value can only + # accelerate the immutable user snapshot; it must never be enough to grant + # access on its own. This makes logout durable across API replicas and + # Valkey outages/restores, where a tombstone may not reach shared cache. + async with pool.connection() as connection: + async with connection.cursor() as cursor: + row = await store.session_user(cursor, hashed) + if row is None: + await cache.revoke(hashed, None) + raise ApiProblem( + 401, + "unauthorized", + "Session token is expired or revoked", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + user = _user_payload(row) + # Refresh an old cache hit after PostgreSQL has confirmed it is still live. + await cache.remember(hashed, user, row["expires_at"]) + return uuid.UUID(user["id"]), user + + +async def revoke_session( + pool: AsyncConnectionPool, cache: SessionCache, token: str +) -> None: + hashed = token_hash(token) + async with pool.connection() as connection: + async with connection.cursor() as cursor: + expires_at = await store.revoke_session(cursor, hashed) + if expires_at is None: + raise ApiProblem( + 401, + "unauthorized", + "Session token is expired or revoked", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + await cache.revoke(hashed, expires_at) + + +async def purge_expired(pool: AsyncConnectionPool) -> int: + async with pool.connection() as connection: + async with connection.cursor() as cursor: + return await store.purge_expired_sessions(cursor) + + +async def purge_expired_idempotency(pool: AsyncConnectionPool) -> int: + async with pool.connection() as connection: + async with connection.cursor() as cursor: + await cursor.execute( + """ + DELETE FROM idempotency_keys + WHERE created_at < now() - (%s || ' seconds')::interval + """, + (str(IDEMPOTENCY_TTL_SECONDS),), + ) + return cursor.rowcount diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..89cc8149 --- /dev/null +++ b/src/store.py @@ -0,0 +1,433 @@ +"""PostgreSQL repository: parameterised statements only, explicit transactions.""" + +from __future__ import annotations + +import uuid +from datetime import datetime +from typing import Any + +import psycopg +from psycopg import AsyncCursor + +from .errors import ApiProblem +from .pagination import format_timestamp + +# One expression keeps the member list of a chat consistent across endpoints. +CHAT_COLUMNS = """ + c.id, + c.title, + c.created_at, + ARRAY( + SELECT m.user_id::text + FROM chat_members m + WHERE m.chat_id = c.id + ORDER BY m.user_id::text + ) AS member_ids +""" + + +def user_json(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_timestamp(row["created_at"]), + } + + +def chat_json(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(member) for member in row["member_ids"]], + "created_at": format_timestamp(row["created_at"]), + } + + +def message_json(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_timestamp(row["created_at"]), + "attachment_ids": [str(attachment_id) for attachment_id in row["attachment_ids"]], + } + + +def attachment_json(row: dict[str, Any]) -> dict[str, Any]: + return { + "id": str(row["id"]), "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), "filename": row["filename"], + "content_type": row["content_type"], "size_bytes": row["size_bytes"], + "sha256": row["sha256"], "status": row["status"], "variants": [], + "error_code": None, "created_at": format_timestamp(row["created_at"]), + "updated_at": format_timestamp(row["updated_at"]), + } + + +MESSAGE_COLUMNS = """ + m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY(SELECT ma.attachment_id FROM message_attachments ma + WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids +""" + + +async def create_user( + cursor: AsyncCursor, + *, + username: str, + display_name: str, + password_hash: str, +) -> dict[str, Any]: + try: + await cursor.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, now()) + RETURNING id, username, display_name, created_at + """, + (uuid.uuid4(), username, display_name, password_hash), + ) + except psycopg.errors.UniqueViolation: + raise ApiProblem(409, "conflict", "Username is already taken") from None + row = await cursor.fetchone() + return user_json(row) + + +async def user_credentials(cursor: AsyncCursor, username: str) -> dict[str, Any] | None: + await cursor.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users + WHERE username = %s + """, + (username,), + ) + return await cursor.fetchone() + + +async def user_by_id(cursor: AsyncCursor, user_id: uuid.UUID) -> dict[str, Any] | None: + await cursor.execute( + """ + SELECT id, username, display_name, created_at + FROM users + WHERE id = %s + """, + (user_id,), + ) + return await cursor.fetchone() + + +async def create_chat( + cursor: AsyncCursor, + *, + actor_id: uuid.UUID, + title: str, + member_ids: list[uuid.UUID], +) -> dict[str, Any]: + """Insert the chat and its membership in one transaction. + + Unknown members are rejected by the foreign key, so the failure rolls the + already inserted chat row back instead of leaving a chat without members. + """ + if actor_id not in member_ids: + raise ApiProblem(400, "invalid_request", "Current user must be a chat member") + chat_id = uuid.uuid4() + try: + await cursor.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, now())", + (chat_id, title), + ) + await cursor.execute( + """ + INSERT INTO chat_members (chat_id, user_id, joined_at) + SELECT %s, member_id, now() FROM unnest(%s::uuid[]) AS member_id + """, + (chat_id, member_ids), + ) + except psycopg.errors.ForeignKeyViolation: + raise ApiProblem( + 400, "invalid_request", "One or more members do not exist" + ) from None + except psycopg.errors.UniqueViolation: + raise ApiProblem(400, "invalid_request", "Chat members must be unique") from None + await cursor.execute( + "SELECT " + CHAT_COLUMNS + " FROM chats c WHERE c.id = %s", + (chat_id,), + ) + row = await cursor.fetchone() + return chat_json(row) + + +async def accessible_chat( + cursor: AsyncCursor, *, actor_id: uuid.UUID, chat_id: uuid.UUID +) -> dict[str, Any]: + await cursor.execute( + """ + SELECT """ + + CHAT_COLUMNS + + """ + FROM chats c + JOIN chat_members membership + ON membership.chat_id = c.id AND membership.user_id = %s + WHERE c.id = %s + """, + (actor_id, chat_id), + ) + row = await cursor.fetchone() + if row is None: + raise ApiProblem(404, "not_found", "Chat not found") + return chat_json(row) + + +async def chat_page( + cursor: AsyncCursor, + *, + actor_id: uuid.UUID, + limit: int, + after: tuple[datetime, uuid.UUID] | None, +) -> list[dict[str, Any]]: + query = ( + "SELECT " + + CHAT_COLUMNS + + """ + FROM chats c + JOIN chat_members membership + ON membership.chat_id = c.id AND membership.user_id = %(actor)s + """ + ) + params: dict[str, Any] = {"actor": actor_id, "limit": limit + 1} + if after is not None: + query += " WHERE (c.created_at, c.id) > (%(after_ts)s, %(after_id)s)" + params["after_ts"], params["after_id"] = after + query += " ORDER BY c.created_at, c.id LIMIT %(limit)s" + await cursor.execute(query, params) + return [chat_json(row) for row in await cursor.fetchall()] + + +async def require_membership( + cursor: AsyncCursor, *, actor_id: uuid.UUID, chat_id: uuid.UUID +) -> None: + await cursor.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, actor_id), + ) + if await cursor.fetchone() is None: + raise ApiProblem(404, "not_found", "Chat not found") + + +async def get_idempotency_key( + cursor: AsyncCursor, *, actor_id: uuid.UUID, chat_id: uuid.UUID, key: str +) -> dict[str, Any] | None: + """Return a record only from this user's POST-message resource scope.""" + await cursor.execute( + """ + SELECT key, actor_id, chat_id, request_sha256, message_id, created_at + FROM idempotency_keys + WHERE actor_id = %s AND chat_id = %s AND key = %s + """, + (actor_id, chat_id, key), + ) + return await cursor.fetchone() + + +async def insert_idempotency_key( + cursor: AsyncCursor, + *, + actor_id: uuid.UUID, + key: str, + chat_id: uuid.UUID, + request_sha256: str, +) -> bool: + """Atomically reserve a key without poisoning a concurrent transaction.""" + await cursor.execute( + """ + INSERT INTO idempotency_keys (actor_id, key, chat_id, request_sha256, created_at) + VALUES (%s, %s, %s, %s, now()) + ON CONFLICT (actor_id, chat_id, key) DO NOTHING + RETURNING key + """, + (actor_id, key, chat_id, request_sha256), + ) + return await cursor.fetchone() is not None + + +async def message_by_id(cursor: AsyncCursor, message_id: uuid.UUID) -> dict[str, Any] | None: + await cursor.execute( + """ + SELECT """ + MESSAGE_COLUMNS + """ + FROM messages m WHERE m.id = %s + """, + (message_id,), + ) + row = await cursor.fetchone() + if row is None: + return None + return message_json(row) + + +async def create_message( + cursor: AsyncCursor, + *, + actor_id: uuid.UUID, + chat_id: uuid.UUID, + text: str, + attachment_ids: list[uuid.UUID], + request_sha256: str, + key: str, +) -> dict[str, Any]: + await require_membership(cursor, actor_id=actor_id, chat_id=chat_id) + # The SHA-256 of the replayed body is recorded with the idempotency row so a + # client cannot replay a request with different content under the same key. + message_id = uuid.uuid4() + if attachment_ids: + await cursor.execute( + """ + SELECT id FROM attachments + WHERE chat_id = %s AND status = 'ready' AND id = ANY(%s::uuid[]) + FOR UPDATE + """, + (chat_id, attachment_ids), + ) + if len(await cursor.fetchall()) != len(attachment_ids): + raise ApiProblem(404, "not_found", "Attachment not found") + await cursor.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, now()) + RETURNING id + """, + (message_id, chat_id, actor_id, text), + ) + await cursor.fetchone() + if attachment_ids: + await cursor.executemany( + """INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES (%s, %s, %s)""", + [(message_id, attachment_id, position) + for position, attachment_id in enumerate(attachment_ids)], + ) + # Record the message_id so a replay can return the original message. + await cursor.execute( + """ + UPDATE idempotency_keys + SET request_sha256 = %s, message_id = %s + WHERE actor_id = %s AND chat_id = %s AND key = %s + """, + (request_sha256, message_id, actor_id, chat_id, key), + ) + return (await message_by_id(cursor, message_id)) # type: ignore[return-value] + + +async def message_page( + cursor: AsyncCursor, + *, + chat_id: uuid.UUID, + limit: int, + after: tuple[datetime, uuid.UUID] | None, +) -> list[dict[str, Any]]: + """Keyset page over (created_at, id); the composite index serves both branches.""" + query = """ + SELECT """ + MESSAGE_COLUMNS + """ + FROM messages m + WHERE m.chat_id = %(chat)s + """ + params: dict[str, Any] = {"chat": chat_id, "limit": limit + 1} + if after is not None: + query += " AND (m.created_at, m.id) > (%(after_ts)s, %(after_id)s)" + params["after_ts"], params["after_id"] = after + query += " ORDER BY m.created_at, m.id LIMIT %(limit)s" + await cursor.execute(query, params) + return [message_json(row) for row in await cursor.fetchall()] + + +async def create_attachment( + cursor: AsyncCursor, *, attachment_id: uuid.UUID, chat_id: uuid.UUID, + owner_id: uuid.UUID, object_key: str, filename: str, content_type: str, + size_bytes: int, sha256: str, width: int, height: int, +) -> dict[str, Any]: + await require_membership(cursor, actor_id=owner_id, chat_id=chat_id) + await cursor.execute( + """INSERT INTO attachments + (id, chat_id, owner_id, object_key, filename, content_type, size_bytes, + sha256, width, height, status, created_at, updated_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'ready', now(), now()) + RETURNING *""", + (attachment_id, chat_id, owner_id, object_key, filename, content_type, + size_bytes, sha256, width, height), + ) + return attachment_json(await cursor.fetchone()) + + +async def accessible_attachment( + cursor: AsyncCursor, *, actor_id: uuid.UUID, attachment_id: uuid.UUID +) -> dict[str, Any]: + await cursor.execute( + """SELECT a.* FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s""", + (actor_id, attachment_id), + ) + row = await cursor.fetchone() + if row is None: + raise ApiProblem(404, "not_found", "Attachment not found") + return row + + +async def insert_session( + cursor: AsyncCursor, + *, + user_id: uuid.UUID, + token_hash: str, + ttl_seconds: int, +) -> datetime: + await cursor.execute( + """ + INSERT INTO sessions (id, user_id, token_hash, created_at, expires_at) + VALUES (%s, %s, %s, now(), now() + make_interval(secs => %s)) + RETURNING expires_at + """, + (uuid.uuid4(), user_id, token_hash, ttl_seconds), + ) + row = await cursor.fetchone() + return row["expires_at"] + + +async def session_user(cursor: AsyncCursor, token_hash: str) -> dict[str, Any] | None: + """Resolve a live session row; expired and revoked sessions never match.""" + await cursor.execute( + """ + SELECT s.expires_at, u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > now() + """, + (token_hash,), + ) + return await cursor.fetchone() + + +async def revoke_session(cursor: AsyncCursor, token_hash: str) -> datetime | None: + """Revoke a live session and return its expiry, or None when it is not live.""" + await cursor.execute( + """ + UPDATE sessions SET revoked_at = now() + WHERE token_hash = %s AND revoked_at IS NULL AND expires_at > now() + RETURNING expires_at + """, + (token_hash,), + ) + row = await cursor.fetchone() + return None if row is None else row["expires_at"] + + +async def purge_expired_sessions(cursor: AsyncCursor) -> int: + await cursor.execute( + """ + DELETE FROM sessions + WHERE expires_at < now() - interval '1 hour' + OR (revoked_at IS NOT NULL AND revoked_at < now() - interval '1 hour') + """ + ) + return cursor.rowcount diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 00000000..d6a7175e --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,209 @@ +"""Minimal standard-library HTTP client for the student's own tests. + +Run with the API already up: + + python -m unittest discover -s student-tests -p "test_*.py" -v + +BASE_URL selects the stand (default http://127.0.0.1:8080). No third-party +dependency is needed, so `make test-student` works on a clean clone. +""" + +from __future__ import annotations + +import base64 +import json +import os +import urllib.error +import urllib.parse +import urllib.request +import uuid +import json +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +LAB = json.loads((ROOT / "lab.json").read_text(encoding="utf-8"))["number"] + + +# 127.0.0.1 rather than "localhost": on a dual-stack host "localhost" resolves +# to ::1 first, a published Docker port may answer on IPv4 only, and every IPv6 +# attempt then costs ~2 s of SYN retry before urllib falls back to IPv4. The IPv4 +# literal keeps the client fast and never reaches the application. +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +TIMEOUT = float(os.environ.get("STUDENT_TEST_TIMEOUT", "20")) + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + """The contract never requires redirects; fail loudly instead of following.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001 + return None + + +# The lab stand is always loopback; urllib ignores the Windows proxy bypass +# list, so a corporate/system proxy would otherwise be used for localhost too. +_OPENER = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + + +class Response: + def __init__(self, status: int, headers: dict[str, str], data: bytes) -> None: + self.status = status + self.headers = headers + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except (ValueError, UnicodeDecodeError): + self.json = None + + def header(self, name: str) -> str: + for key, value in self.headers.items(): + if key.lower() == name.lower(): + return value + return "" + + def __repr__(self) -> str: # pragma: no cover - diagnostics only + return "Response(status=%s, json=%r)" % (self.status, self.json) + + +def request( + method: str, + path: str, + *, + body=None, + raw: bytes | None = None, + token: str | None = None, + headers: dict[str, str] | None = None, + content_type: str | None = None, + timeout: float | None = None, + base_url: str | None = None, +) -> Response: + url = (base_url or BASE_URL) + path + data = raw + request_headers = {"Accept": "application/json"} + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + request_headers["Content-Type"] = "application/json" + if content_type is not None: + request_headers["Content-Type"] = content_type + if token is not None: + request_headers["Authorization"] = "Bearer " + token + request_headers.update(headers or {}) + prepared = urllib.request.Request(url, data=data, headers=request_headers, method=method) + try: + response = _OPENER.open(prepared, timeout=timeout or TIMEOUT) + except urllib.error.HTTPError as error: + response = error + with response: + payload = response.read() + return Response(response.code, dict(response.headers.items()), payload) + + +def basic(username: str, password: str) -> dict[str, str]: + encoded = base64.b64encode((username + ":" + password).encode("utf-8")).decode("ascii") + return {"Authorization": "Basic " + encoded} + + +def new_registration(prefix: str = "st", password: str | None = None) -> dict[str, str]: + suffix = uuid.uuid4().hex[:12] + return { + "username": prefix + "_" + suffix, + "display_name": "Тест " + suffix, + "password": password or ("St-" + uuid.uuid4().hex), + } + + +def register(registration: dict[str, str] | None = None) -> Response: + return request("POST", "/api/v1/users", body=registration or new_registration()) + + +def login(username: str, password: str) -> Response: + return request("POST", "/api/v1/auth/sessions", headers=basic(username, password)) + + +class Actor: + """A registered user with a live session, as a client would hold it.""" + + def __init__(self, registration: dict[str, str]) -> None: + self.registration = registration + response = register(registration) + if response.status != 201: + raise AssertionError( + "registration failed: %s %r" % (response.status, response.json) + ) + self.user = response.json + self.username = registration["username"] + self.password = registration["password"] + + @property + def id(self) -> str: + return self.user["id"] + + def sign_in(self) -> "Actor": + response = login(self.username, self.password) + if response.status != 201: + raise AssertionError("login failed: %s %r" % (response.status, response.json)) + self.session = response.json + self.token = response.json["token"] + return self + + +def actor(prefix: str = "st", password: str | None = None) -> Actor: + return Actor(new_registration(prefix, password)).sign_in() + + +def create_chat(token: str, member_ids: list[str], title: str = "Тестовый чат") -> dict: + response = request( + "POST", + "/api/v1/chats", + body={"title": title, "member_ids": member_ids}, + token=token, + ) + if response.status != 201: + raise AssertionError("chat creation failed: %s %r" % (response.status, response.json)) + return response.json + + +def send(token: str, chat_id: str, text: str) -> dict: + headers = {} + if LAB >= 3: + headers["Idempotency-Key"] = uuid.uuid4().hex + response = request( + "POST", "/api/v1/chats/" + chat_id + "/messages", body={"text": text}, token=token, headers=headers + ) + if response.status != 201: + raise AssertionError("message failed: %s %r" % (response.status, response.json)) + return response.json + + +def history(token: str, chat_id: str, limit: int = 100) -> list[dict]: + """Walk every page of a chat, guarding against cursor loops.""" + items: list[dict] = [] + seen: set[str] = set() + cursor: str | None = None + while True: + path = "/api/v1/chats/" + chat_id + "/messages?limit=" + str(limit) + if cursor is not None: + path += "&cursor=" + urllib.parse.quote(cursor, safe="") + response = request("GET", path, token=token) + if response.status != 200: + raise AssertionError("history failed: %s %r" % (response.status, response.json)) + items.extend(response.json["items"]) + cursor = response.json["next_cursor"] + if cursor is None: + return items + if cursor in seen: + raise AssertionError("cursor loop detected") + seen.add(cursor) + + +def expect_error(test, response: Response, status: int, code: str) -> dict: + """Assert the public error envelope, request id echo and the status.""" + test.assertEqual(response.status, status, response.json) + test.assertIsInstance(response.json, dict) + error = response.json.get("error") + test.assertIsNotNone(error, "missing error envelope") + test.assertEqual(error["code"], code) + test.assertTrue(error["message"]) + request_id = response.header("X-Request-Id") + test.assertTrue(request_id, "missing X-Request-Id header") + test.assertEqual(error["request_id"], request_id) + return error diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..b108e533 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,329 @@ +"""Own functional tests: contract edges carried from lab 1 plus lab 2 auth. + +Every test creates unique synthetic users, so the file is safe to re-run against +the same stand (the API has no delete endpoint by design). +""" + +from __future__ import annotations + +import unittest +import uuid +from datetime import datetime, timezone + +import common + + +def timestamp(value: str) -> datetime: + return datetime.fromisoformat(value[:-1] + "+00:00") + + +class Base(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + again = common.request("POST", "/api/v1/users", body=common.new_registration()) + if again.status != 201: + raise AssertionError("stand is not serving: %r" % (again.json,)) + cls.alice = common.actor("alice") + cls.bob = common.actor("bob") + cls.eve = common.actor("eve") + cls.chat = common.create_chat( + cls.alice.token, [cls.alice.id, cls.bob.id], "Контрактный чат" + ) + + +class HealthAndEnvelope(Base): + def test_health_endpoints_and_request_id(self) -> None: + live = common.request("GET", "/health/live") + self.assertEqual(live.status, 200) + self.assertEqual(live.json, {"status": "ok"}) + self.assertTrue(live.header("X-Request-Id")) + ready = common.request("GET", "/health/ready") + self.assertEqual(ready.status, 200) + self.assertEqual(ready.json, {"status": "ok"}) + + def test_unknown_route_uses_same_error_envelope(self) -> None: + common.expect_error(self, common.request("GET", "/api/v1/nope"), 404, "not_found") + + def test_query_parameters_rejected_where_not_supported(self) -> None: + response = common.request( + "GET", "/api/v1/users/me?verbose=1", token=self.alice.token + ) + common.expect_error(self, response, 400, "invalid_request") + + +class Registration(Base): + def test_response_has_no_secret_fields(self) -> None: + response = common.register(common.new_registration()) + self.assertEqual(response.status, 201) + self.assertEqual( + set(response.json), {"id", "username", "display_name", "created_at"} + ) + self.assertNotIn("password", response.json) + self.assertNotIn("password_hash", response.json) + + def test_duplicate_username_conflict(self) -> None: + registration = common.new_registration() + self.assertEqual(common.register(registration).status, 201) + common.expect_error(self, common.register(registration), 409, "conflict") + + def test_username_rules(self) -> None: + for username in ("UPPER_case", "ab", "with-dash", "with space"): + body = common.new_registration() + body["username"] = username + common.expect_error(self, common.register(body), 400, "invalid_request") + + def test_password_rules(self) -> None: + short = common.new_registration(password="x" * 11) + common.expect_error(self, common.register(short), 400, "invalid_request") + long_password = common.new_registration(password="x" * 129) + common.expect_error(self, common.register(long_password), 400, "invalid_request") + # 12 Unicode code points are enough, even when they are multi-byte. + unicode_password = common.new_registration(password="😀" * 12) + self.assertEqual(common.register(unicode_password).status, 201) + + def test_malformed_json_extra_fields_and_wrong_types(self) -> None: + common.expect_error( + self, + common.request( + "POST", "/api/v1/users", raw=b'{"username": ', content_type="application/json" + ), + 400, + "invalid_request", + ) + body = common.new_registration() + body["is_admin"] = True + common.expect_error(self, common.register(body), 400, "invalid_request") + body = common.new_registration() + body["display_name"] = 42 + common.expect_error(self, common.register(body), 400, "invalid_request") + + def test_unsupported_media_type(self) -> None: + response = common.request( + "POST", + "/api/v1/users", + raw=b"username=alice", + content_type="application/x-www-form-urlencoded", + ) + common.expect_error(self, response, 415, "unsupported_media_type") + + +class Chats(Base): + def test_membership_validation(self) -> None: + cases = [ + [self.bob.id], # caller is missing + [self.alice.id, self.alice.id], # duplicate member + [self.alice.id, str(uuid.uuid4())], # unknown member + [], # no members + ] + for member_ids in cases: + response = common.request( + "POST", + "/api/v1/chats", + body={"title": "Плохой чат", "member_ids": member_ids}, + token=self.alice.token, + ) + common.expect_error(self, response, 400, "invalid_request") + + def test_foreign_chat_is_hidden_everywhere(self) -> None: + chat_id = self.chat["id"] + common.expect_error( + self, + common.request("GET", "/api/v1/chats/" + chat_id, token=self.eve.token), + 404, + "not_found", + ) + common.expect_error( + self, + common.request( + "GET", "/api/v1/chats/" + chat_id + "/messages", token=self.eve.token + ), + 404, + "not_found", + ) + common.expect_error( + self, + common.request( + "POST", + "/api/v1/chats/" + chat_id + "/messages", + body={"text": "чужое"}, + token=self.eve.token, + ), + 404, + "not_found", + ) + listed = common.request("GET", "/api/v1/chats?limit=100", token=self.eve.token) + self.assertEqual(listed.status, 200) + self.assertNotIn(chat_id, [item["id"] for item in listed.json["items"]]) + + def test_messages_are_validated_and_unicode_preserved(self) -> None: + chat = common.create_chat(self.alice.token, [self.alice.id, self.bob.id]) + for text in ("", " \n\t ", "x" * 4001): + response = common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": text}, + token=self.alice.token, + ) + common.expect_error(self, response, 400, "invalid_request") + prefix = "Привет 👋 世界 " + text = prefix + "x" * (4000 - len(prefix)) # exactly the contract maximum + message = common.send(self.alice.token, chat["id"], text) + self.assertEqual(message["text"], text) + self.assertEqual(message["sender_id"], self.alice.id) + + injected = common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "подмена", "sender_id": self.bob.id}, + token=self.alice.token, + ) + common.expect_error(self, injected, 400, "invalid_request") + + +class Pagination(Base): + def test_walks_every_page_without_loss_or_duplicates(self) -> None: + chat = common.create_chat(self.alice.token, [self.alice.id, self.bob.id]) + sent = [ + common.send(self.alice.token if index % 2 else self.bob.token, chat["id"], "msg-%d" % index) + for index in range(7) + ] + collected: list[dict] = [] + cursor = None + pages = 0 + while True: + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=2" + if cursor is not None: + path += "&cursor=" + cursor + page = common.request("GET", path, token=self.alice.token) + self.assertEqual(page.status, 200) + collected.extend(page.json["items"]) + pages += 1 + cursor = page.json["next_cursor"] + if cursor is None: + break + self.assertLess(pages, 10, "pagination did not terminate") + self.assertEqual(pages, 4) + self.assertEqual([item["id"] for item in collected], [item["id"] for item in sent]) + self.assertEqual( + collected, + sorted(collected, key=lambda item: (timestamp(item["created_at"]), item["id"])), + ) + + def test_cursor_is_bound_to_its_chat(self) -> None: + other = common.create_chat(self.alice.token, [self.alice.id]) + common.send(self.alice.token, other["id"], "a") + common.send(self.alice.token, other["id"], "b") + first = common.request( + "GET", "/api/v1/chats/" + other["id"] + "/messages?limit=1", token=self.alice.token + ) + cursor = first.json["next_cursor"] + self.assertIsNotNone(cursor) + response = common.request( + "GET", + "/api/v1/chats/" + self.chat["id"] + "/messages" + "?limit=1&cursor=" + cursor, + token=self.alice.token, + ) + common.expect_error(self, response, 400, "invalid_request") + + def test_limit_and_cursor_validation(self) -> None: + base = "/api/v1/chats/" + self.chat["id"] + "/messages" + for query in ("?limit=0", "?limit=101", "?limit=abc", "?cursor=", "?cursor=not-a-cursor", "?limit=1&limit=2", "?unknown=1"): + common.expect_error( + self, + common.request("GET", base + query, token=self.alice.token), + 400, + "invalid_request", + ) + + +class Authentication(Base): + def test_basic_is_not_accepted_outside_session_creation(self) -> None: + response = common.request( + "GET", "/api/v1/users/me", headers=common.basic(self.alice.username, self.alice.password) + ) + common.expect_error(self, response, 401, "unauthorized") + self.assertTrue(response.header("WWW-Authenticate").lower().startswith("bearer")) + + def test_missing_and_unknown_tokens(self) -> None: + for headers in ({}, {"Authorization": "Bearer not-a-real-token"}): + response = common.request("GET", "/api/v1/users/me", headers=headers) + common.expect_error(self, response, 401, "unauthorized") + self.assertTrue(response.header("WWW-Authenticate").lower().startswith("bearer")) + + def test_x_user_id_is_ignored(self) -> None: + response = common.request("GET", "/api/v1/users/me", headers={"X-User-Id": self.alice.id}) + common.expect_error(self, response, 401, "unauthorized") + overridden = common.request( + "GET", + "/api/v1/users/me", + token=self.alice.token, + headers={"X-User-Id": self.eve.id}, + ) + self.assertEqual(overridden.status, 200) + self.assertEqual(overridden.json["id"], self.alice.id) + + def test_session_payload_and_expiry(self) -> None: + response = common.login(self.alice.username, self.alice.password) + self.assertEqual(response.status, 201) + session = response.json + self.assertEqual(set(session), {"token", "token_type", "expires_at", "user"}) + self.assertEqual(session["token_type"], "Bearer") + self.assertGreaterEqual(len(session["token"]), 43) + self.assertNotIn(self.alice.username, session["token"]) + self.assertNotEqual(session["token"], self.alice.id) + self.assertEqual(session["user"], self.alice.user) + self.assertGreater(timestamp(session["expires_at"]), datetime.now(timezone.utc)) + me = common.request("GET", "/api/v1/users/me", token=session["token"]) + self.assertEqual(me.status, 200) + self.assertEqual(me.json, self.alice.user) + + def test_wrong_password_and_unknown_user_are_indistinguishable(self) -> None: + wrong = common.login(self.alice.username, "definitely-wrong-password") + unknown = common.login("nobody_here_" + uuid.uuid4().hex[:8], "definitely-wrong-password") + messages = [] + for response in (wrong, unknown): + error = common.expect_error(self, response, 401, "unauthorized") + self.assertTrue( + response.header("WWW-Authenticate").lower().startswith("basic") + ) + messages.append(error["message"]) + # Same message and challenge: a client cannot tell the accounts apart. + self.assertEqual(messages[0], messages[1]) + + def test_logout_revokes_only_that_session(self) -> None: + first = common.login(self.alice.username, self.alice.password).json["token"] + second = common.login(self.alice.username, self.alice.password).json["token"] + revoked = common.request("DELETE", "/api/v1/auth/sessions/current", token=first) + self.assertEqual(revoked.status, 204) + self.assertEqual(revoked.data, b"") + common.expect_error( + self, + common.request("GET", "/api/v1/users/me", token=first), + 401, + "unauthorized", + ) + common.expect_error( + self, + common.request("DELETE", "/api/v1/auth/sessions/current", token=first), + 401, + "unauthorized", + ) + still_valid = common.request("GET", "/api/v1/users/me", token=second) + self.assertEqual(still_valid.status, 200) + + def test_session_endpoint_rejects_json_body_and_bad_basic(self) -> None: + response = common.request( + "POST", + "/api/v1/auth/sessions", + body={"username": self.alice.username, "password": self.alice.password}, + ) + common.expect_error(self, response, 401, "unauthorized") + response = common.request( + "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic !!!not-base64!!!"} + ) + common.expect_error(self, response, 401, "unauthorized") + self.assertTrue(response.header("WWW-Authenticate").lower().startswith("basic")) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..5b7acdc9 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,62 @@ +"""Negative and access-control checks specific to lab4 attachments.""" +from __future__ import annotations + +import hashlib +from pathlib import Path +import unittest +import uuid + +import common + + +PNG = (Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "sample.png").read_bytes() + + +class Attachments(unittest.TestCase): + def upload(self, token: str, chat_id: str, raw: bytes = PNG, content_type: str = "image/png"): + return common.request( + "POST", "/api/v1/chats/" + chat_id + "/attachments?filename=sample.png", + raw=raw, token=token, content_type=content_type, + ) + + def test_upload_download_and_foreign_access_are_scoped_to_chat(self) -> None: + alice = common.actor("attachalice") + bob = common.actor("attachbob") + eve = common.actor("attacheve") + chat = common.create_chat(alice.token, [alice.id, bob.id], "Attachment scope") + uploaded = self.upload(alice.token, chat["id"]) + self.assertEqual(uploaded.status, 201, uploaded.json) + attachment = uploaded.json + self.assertEqual(attachment["sha256"], hashlib.sha256(PNG).hexdigest()) + downloaded = common.request("GET", "/api/v1/attachments/" + attachment["id"] + "/content", token=bob.token) + self.assertEqual(downloaded.status, 200) + self.assertEqual(downloaded.data, PNG) + common.expect_error(self, common.request("GET", "/api/v1/attachments/" + attachment["id"], token=eve.token), 404, "not_found") + common.expect_error(self, common.request("GET", "/api/v1/attachments/" + attachment["id"] + "/content", token=eve.token), 404, "not_found") + + def test_rejects_bad_filename_mime_corruption_and_oversize_declaration(self) -> None: + alice = common.actor("attachinvalid") + chat = common.create_chat(alice.token, [alice.id], "Attachment validation") + base = "/api/v1/chats/" + chat["id"] + "/attachments" + common.expect_error(self, common.request("POST", base + "?filename=../secret.png", raw=PNG, token=alice.token, content_type="image/png"), 400, "invalid_request") + common.expect_error(self, common.request("POST", base + "?filename=x.png", raw=b"not an image", token=alice.token, content_type="image/png"), 415, "unsupported_media_type") + common.expect_error(self, common.request("POST", base + "?filename=x.jpg", raw=PNG, token=alice.token, content_type="image/jpeg"), 415, "unsupported_media_type") + too_large = common.request( + "POST", base + "?filename=large.png", + raw=PNG + b"x" * (10_485_761 - len(PNG)), token=alice.token, + content_type="image/png", timeout=60, + ) + common.expect_error(self, too_large, 413, "payload_too_large") + + def test_attachment_cannot_be_bound_to_another_chat(self) -> None: + alice = common.actor("attachcross") + source_chat = common.create_chat(alice.token, [alice.id], "Attachment source") + target_chat = common.create_chat(alice.token, [alice.id], "Attachment target") + uploaded = self.upload(alice.token, source_chat["id"]) + self.assertEqual(uploaded.status, 201, uploaded.json) + rejected = common.request( + "POST", "/api/v1/chats/" + target_chat["id"] + "/messages", + body={"text": "", "attachment_ids": [uploaded.json["id"]]}, token=alice.token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + common.expect_error(self, rejected, 404, "not_found") diff --git a/student-tests/test_degradation.py b/student-tests/test_degradation.py new file mode 100644 index 00000000..bae5f94d --- /dev/null +++ b/student-tests/test_degradation.py @@ -0,0 +1,211 @@ +"""Dependency outage behaviour: Valkey down, then PostgreSQL down. + +Requires Docker access on the same host as the Compose stand and is therefore +opt-in: + + STUDENT_DOCKER_TESTS=1 python -m unittest -v test_degradation + +The test never removes volumes and restores both services in `finally`. +""" + +from __future__ import annotations + +import json +import os +import uuid +import statistics +import subprocess +import time +import unittest +from pathlib import Path + +import common + + +REPO_ROOT = Path(__file__).resolve().parents[1] +PROJECT = os.environ.get("COMPOSE_PROJECT_NAME", "mtusi2026lab3") +ENV_FILE = os.environ.get("COMPOSE_ENV_FILE", ".env") +ENABLED = os.environ.get("STUDENT_DOCKER_TESTS") == "1" + + +def compose(*arguments: str, timeout: int = 120) -> None: + result = subprocess.run( + ["docker", "compose", "-p", PROJECT, "--env-file", ENV_FILE, *arguments], + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != 0: + raise AssertionError( + "docker compose %s failed: %s" % (" ".join(arguments), result.stderr[-500:]) + ) + + +def wait_ready(expected: int, seconds: float) -> float: + started = time.monotonic() + deadline = started + seconds + last = None + while time.monotonic() < deadline: + try: + last = common.request("GET", "/health/ready", timeout=5) + if last.status == expected: + return time.monotonic() - started + except OSError: + # A replica can close the listener while Compose restarts it. + pass + time.sleep(0.5) + raise AssertionError( + "readiness did not reach %s within %.0fs (last=%s %r)" + % (expected, seconds, last.status if last else None, last.json if last else None) + ) + + +@unittest.skipUnless(ENABLED, "set STUDENT_DOCKER_TESTS=1 to stop real containers") +class DependencyOutages(unittest.TestCase): + def test_logout_remains_revoked_across_replicas_after_cache_outage(self) -> None: + """A stale shared-cache hit must not bypass PostgreSQL revocation.""" + alice = common.actor("revoked") + chat = common.create_chat(alice.token, [alice.id], "Durable logout") + self.assertEqual(common.request("GET", "/api/v1/users/me", token=alice.token).status, 200) + try: + compose("stop", "valkey") + logout = common.request("DELETE", "/api/v1/auth/sessions/current", token=alice.token) + self.assertEqual(logout.status, 204, logout.json) + + # Remove both processes' local fallback state and restore Valkey's + # old data. Requests through nginx can land on either replica. + compose("restart", "api-1", "api-2") + compose("start", "valkey") + wait_ready(200, 90) + + common.expect_error( + self, common.request("GET", "/api/v1/users/me", token=alice.token), 401, "unauthorized" + ) + common.expect_error( + self, + common.request("GET", "/api/v1/chats/" + chat["id"], token=alice.token), + 401, + "unauthorized", + ) + common.expect_error( + self, + common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "must not be accepted"}, + token=alice.token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ), + 401, + "unauthorized", + ) + finally: + compose("start", "valkey", "postgres", "api-1", "api-2") + try: + wait_ready(200, 90) + except AssertionError: + pass + + def test_valkey_loss_degrades_then_postgres_loss_is_bounded(self) -> None: + alice = common.actor("degrade") + chat = common.create_chat(alice.token, [alice.id], "Устойчивость") + # Warm the session cache. + self.assertEqual(common.request("GET", "/api/v1/users/me", token=alice.token).status, 200) + + cached = self._sample_latency(alice.token, 60) + summary = {"cached_p50_ms": round(cached["p50"], 2)} + try: + # 1. Valkey is a cache: auth still works, without DB-free fast path. + compose("stop", "valkey") + ready = common.request("GET", "/health/ready") + self.assertEqual( + ready.status, 200, "session cache must not be a readiness dependency" + ) + me = common.request("GET", "/api/v1/users/me", token=alice.token) + self.assertEqual(me.status, 200, me.json) + sent = common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "без кэша"}, + token=alice.token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + self.assertEqual(sent.status, 201, sent.json) + without_cache = self._sample_latency(alice.token, 60) + summary["cache_down_p50_ms"] = round(without_cache["p50"], 2) + summary["cache_down_p95_ms"] = round(without_cache["p95"], 2) + + compose("start", "valkey") + time.sleep(3) + self.assertEqual(common.request("GET", "/health/ready").status, 200) + + # 2. PostgreSQL is the source of truth: bounded errors, never a 200 + # for a token the server cannot verify, and never a raw 500. + compose("stop", "postgres") + ready = common.request("GET", "/health/ready") + self.assertEqual(ready.status, 503, ready.json) + self.assertEqual(ready.json["error"]["code"], "unavailable") + + started = time.monotonic() + write = common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "без базы"}, + token=alice.token, + timeout=30, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + bounded = time.monotonic() - started + self.assertEqual(write.status, 503, write.json) + self.assertEqual(write.json["error"]["code"], "unavailable") + self.assertLess(bounded, 10, "database failure must fail fast") + summary["db_down_write_seconds"] = round(bounded, 2) + + for leaked in ("psycopg", "postgresql://", "password", "Traceback"): + self.assertNotIn(leaked, json.dumps(write.json)) + unknown = common.request( + "GET", "/api/v1/users/me", token="unknown-token-value", timeout=30 + ) + self.assertIn(unknown.status, (401, 503)) + self.assertNotEqual(unknown.status, 200) + + # 3. Recovery keeps the same session and the stored chat. + compose("start", "postgres") + summary["recovery_seconds"] = round(wait_ready(200, 90), 2) + after = common.request("GET", "/api/v1/chats/" + chat["id"], token=alice.token) + self.assertEqual(after.status, 200, after.json) + recovered = common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "после восстановления"}, + token=alice.token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + self.assertEqual(recovered.status, 201, recovered.json) + messages = common.history(alice.token, chat["id"]) + self.assertIn("после восстановления", [message["text"] for message in messages]) + finally: + compose("start", "valkey", "postgres") + try: + wait_ready(200, 90) + except AssertionError: + pass + print("\n outage summary: " + json.dumps(summary, ensure_ascii=False)) + + def _sample_latency(self, token: str, count: int) -> dict[str, float]: + samples = [] + for _ in range(count): + started = time.monotonic() + response = common.request("GET", "/api/v1/users/me", token=token, timeout=30) + samples.append(time.monotonic() - started) + self.assertEqual(response.status, 200, response.json) + ordered = sorted(samples) + return { + "p50": statistics.median(samples) * 1000, + "p95": ordered[max(0, int(len(ordered) * 0.95) - 1)] * 1000, + } + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 00000000..765a2799 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,99 @@ +"""Session expiry with SESSION_TTL_SECONDS=2, plus logout and bad credentials. + +Run against the dedicated short-TTL stand: + + scripts/ttl-stack.sh start + BASE_URL=http://localhost:8081 python -m unittest -v test_expiry + scripts/ttl-stack.sh stop +""" + +from __future__ import annotations + +import statistics +import time +import unittest +from datetime import datetime, timezone + +import common + + +def expires_at(session: dict) -> datetime: + return datetime.fromisoformat(session["expires_at"][:-1] + "+00:00") + + +class ShortTtlSessions(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + ready = common.request("GET", "/health/ready") + if ready.status != 200: + raise AssertionError("stand is not ready: %r" % (ready.json,)) + + def test_expiry_is_enforced_and_cache_does_not_extend_it(self) -> None: + user = common.actor("ttl") + started = datetime.now(timezone.utc) + session = common.login(user.username, user.password) + self.assertEqual(session.status, 201) + lifetime = (expires_at(session.json) - started).total_seconds() + self.assertLessEqual(lifetime, 5.0, "stand does not use a short TTL") + + # Two reads: the second one is served from the Valkey cache, so the + # expiry check below covers the cached path as well. + for _ in range(2): + self.assertEqual( + common.request("GET", "/api/v1/users/me", token=session.json["token"]).status, + 200, + ) + + time.sleep(3.2) + expired = common.request("GET", "/api/v1/users/me", token=session.json["token"]) + common.expect_error(self, expired, 401, "unauthorized") + self.assertTrue(expired.header("WWW-Authenticate").lower().startswith("bearer")) + revoked = common.request( + "DELETE", "/api/v1/auth/sessions/current", token=session.json["token"] + ) + common.expect_error(self, revoked, 401, "unauthorized") + + # A fresh login still works, so expiry removed only the old session. + again = common.login(user.username, user.password) + self.assertEqual(again.status, 201) + self.assertEqual( + common.request("GET", "/api/v1/users/me", token=again.json["token"]).status, + 200, + ) + + def test_logout_is_immediate_even_when_the_session_is_cached(self) -> None: + user = common.actor("ttllogout") + token = user.token + self.assertEqual(common.request("GET", "/api/v1/users/me", token=token).status, 200) + self.assertEqual( + common.request("DELETE", "/api/v1/auth/sessions/current", token=token).status, + 204, + ) + outcome = common.request("GET", "/api/v1/users/me", token=token) + common.expect_error(self, outcome, 401, "unauthorized") + + def test_wrong_password_issues_no_session(self) -> None: + user = common.actor("ttlwrong") + denied = common.login(user.username, "not-the-password") + common.expect_error(self, denied, 401, "unauthorized") + self.assertTrue(denied.header("WWW-Authenticate").lower().startswith("basic")) + + def test_session_lookup_latency_is_bounded(self) -> None: + user = common.actor("ttllatency") + samples = [] + for _ in range(30): + started = time.monotonic() + response = common.request("GET", "/api/v1/users/me", token=user.token, timeout=5) + samples.append(time.monotonic() - started) + self.assertEqual(response.status, 200) + print( + "\n /users/me warm cache p50=%.1fms p95=%.1fms" + % ( + statistics.median(samples) * 1000, + sorted(samples)[int(len(samples) * 0.95) - 1] * 1000, + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..4c52a7e8 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,85 @@ +"""Idempotency scopes and concurrent replay checks for laboratory 3.""" + +from __future__ import annotations + +import threading +import unittest +import uuid + +import common + + +class IdempotencyScope(unittest.TestCase): + def test_same_key_is_independent_for_different_users_and_chats(self) -> None: + alice = common.actor("idemalice") + eve = common.actor("idemeve") + alice_chat = common.create_chat(alice.token, [alice.id], "Alice scope") + eve_chat = common.create_chat(eve.token, [eve.id], "Eve scope") + key = uuid.uuid4().hex + + first = common.request( + "POST", + "/api/v1/chats/" + alice_chat["id"] + "/messages", + body={"text": "same body"}, + token=alice.token, + headers={"Idempotency-Key": key}, + ) + second = common.request( + "POST", + "/api/v1/chats/" + eve_chat["id"] + "/messages", + body={"text": "same body"}, + token=eve.token, + headers={"Idempotency-Key": key}, + ) + self.assertEqual(first.status, 201, first.json) + self.assertEqual(second.status, 201, second.json) + self.assertNotEqual(first.json["id"], second.json["id"]) + self.assertEqual(second.json["chat_id"], eve_chat["id"]) + self.assertEqual(second.json["sender_id"], eve.id) + + # A different payload in another scope is also a new independent write. + other_scope = common.create_chat(eve.token, [eve.id], "Eve second scope") + third = common.request( + "POST", + "/api/v1/chats/" + other_scope["id"] + "/messages", + body={"text": "different body"}, + token=eve.token, + headers={"Idempotency-Key": key}, + ) + self.assertEqual(third.status, 201, third.json) + self.assertEqual(third.json["chat_id"], other_scope["id"]) + + def test_concurrent_identical_replays_return_one_message_without_500(self) -> None: + alice = common.actor("idemrace") + chat = common.create_chat(alice.token, [alice.id], "Concurrent replay") + key = uuid.uuid4().hex + workers = 32 + barrier = threading.Barrier(workers) + responses: list[common.Response] = [] + lock = threading.Lock() + + def post() -> None: + barrier.wait() + response = common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "exactly once"}, + token=alice.token, + headers={"Idempotency-Key": key}, + timeout=30, + ) + with lock: + responses.append(response) + + threads = [threading.Thread(target=post) for _ in range(workers)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=60) + + self.assertEqual(len(responses), workers) + self.assertTrue(all(response.status == 201 for response in responses), responses) + message_ids = {response.json["id"] for response in responses} + self.assertEqual(len(message_ids), 1) + messages = common.history(alice.token, chat["id"]) + self.assertEqual([message["text"] for message in messages], ["exactly once"]) diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 00000000..acc9e61f --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,142 @@ +"""Race, rollback and pool tests against the real PostgreSQL stand. + +These are the checks a code reviewer would ask for on top of the smoke test: +concurrent writers must not lose or duplicate messages, the username uniqueness +must be decided by the database (not by a read-then-write check in Python), and +a failed composite operation must leave nothing behind. +""" + +from __future__ import annotations + +import threading +import time +import unittest +import uuid + +import common + + +class ConcurrentWrites(unittest.TestCase): + def test_100_messages_from_10_clients_are_exactly_100(self) -> None: + alice = common.actor("cwalice") + bob = common.actor("cwbob") + chat = common.create_chat(alice.token, [alice.id, bob.id], "Гонка сообщений") + writers = 10 + per_writer = 10 + barrier = threading.Barrier(writers) + errors: list[str] = [] + sent: list[str] = [] + lock = threading.Lock() + + def writer(index: int) -> None: + token = alice.token if index % 2 else bob.token + barrier.wait() + for number in range(per_writer): + text = "race-%d-%d" % (index, number) + try: + common.send(token, chat["id"], text) + with lock: + sent.append(text) + except AssertionError as error: # pragma: no cover - failure detail + with lock: + errors.append(str(error)) + + threads = [threading.Thread(target=writer, args=(index,)) for index in range(writers)] + started = time.monotonic() + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=120) + elapsed = time.monotonic() - started + self.assertEqual(errors, []) + self.assertEqual(len(sent), writers * per_writer) + + messages = common.history(alice.token, chat["id"], limit=100) + self.assertEqual(len(messages), writers * per_writer) + self.assertEqual(len({message["id"] for message in messages}), len(messages)) + self.assertEqual(sorted(message["text"] for message in messages), sorted(sent)) + print("\n concurrent writes: %d messages from %d clients in %.2fs" + % (len(messages), writers, elapsed)) + + def test_duplicate_username_race_creates_one_user(self) -> None: + registration = common.new_registration("race") + attempts = 12 + barrier = threading.Barrier(attempts) + statuses: list[int] = [] + lock = threading.Lock() + + def attempt() -> None: + barrier.wait() + response = common.register(dict(registration)) + with lock: + statuses.append(response.status) + + threads = [threading.Thread(target=attempt) for _ in range(attempts)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=60) + self.assertEqual(statuses.count(201), 1, statuses) + self.assertEqual(statuses.count(409), attempts - 1, statuses) + # The single winner is usable, so exactly one row exists. + session = common.login(registration["username"], registration["password"]) + self.assertEqual(session.status, 201) + self.assertEqual(session.json["user"]["username"], registration["username"]) + + def test_chat_creation_rolls_back_on_unknown_member(self) -> None: + alice = common.actor("rollback") + before = self._chat_titles(alice.token) + response = common.request( + "POST", + "/api/v1/chats", + body={"title": "Атомарный чат", "member_ids": [alice.id, str(uuid.uuid4())]}, + token=alice.token, + ) + common.expect_error(self, response, 400, "invalid_request") + after = self._chat_titles(alice.token) + # The chat row inserted before the failing membership step is gone. + self.assertEqual(before, after) + self.assertNotIn("Атомарный чат", after) + + def _chat_titles(self, token: str) -> list[str]: + titles: list[str] = [] + cursor = None + for _ in range(50): + path = "/api/v1/chats?limit=100" + if cursor: + path += "&cursor=" + cursor + response = common.request("GET", path, token=token) + self.assertEqual(response.status, 200) + titles.extend(item["title"] for item in response.json["items"]) + cursor = response.json["next_cursor"] + if cursor is None: + return titles + self.fail("chat pagination did not terminate") + + def test_bounded_pool_serves_parallel_reads(self) -> None: + alice = common.actor("pool") + parallel = 50 + barrier = threading.Barrier(parallel) + statuses: list[int] = [] + lock = threading.Lock() + + def read() -> None: + barrier.wait() + # Chat lists are not cached, so this exercises the bounded SQL pool. + response = common.request( + "GET", "/api/v1/chats?limit=100", token=alice.token, timeout=10 + ) + with lock: + statuses.append(response.status) + + threads = [threading.Thread(target=read) for _ in range(parallel)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=60) + self.assertEqual(len(statuses), parallel) + self.assertEqual(statuses, [200] * parallel) + + +if __name__ == "__main__": + unittest.main()