diff --git a/.env.example b/.env.example index d99d26a2..dcfda857 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,37 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Replace every change-me value. The repository contains no real secrets. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +DATABASE_URL=postgresql://messenger:change-me-db-password@postgres:5432/messenger +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-db-password +CACHE_URL=redis://redis:6379/0 API_REPLICAS=2 +API_PORT=8080 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 + +# Private S3-compatible store (SeaweedFS 3.80). +S3_ENDPOINT=http://seaweedfs:8333 +# Public endpoint used only to sign direct-upload URLs for clients. +S3_PUBLIC_ENDPOINT=http://localhost:9000 +S3_PORT=9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +# Restricted identity used by API replicas: Read/Write/List only. +S3_ACCESS_KEY_ID=change-me-app-access +S3_SECRET_ACCESS_KEY=change-me-app-secret +# Administrative identity used only by the one-shot s3-init service. +S3_ADMIN_ACCESS_KEY_ID=change-me-admin-access +S3_ADMIN_SECRET_ACCESS_KEY=change-me-admin-secret + MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +MAX_DECODE_CONCURRENCY=1 +PRESIGN_TTL_SECONDS=300 +PRESIGN_GRACE_SECONDS=3600 +ORPHAN_GRACE_SECONDS=86400 +S3_CONNECT_TIMEOUT_SECONDS=3 +S3_READ_TIMEOUT_SECONDS=30 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..b225d178 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Shell entrypoints are mounted/run inside Linux containers: force LF. +*.sh text eol=lf +infra/** text eol=lf +Dockerfile text eol=lf +*.sql text eol=lf +# Windows host helpers keep native CRLF. +*.cmd text eol=crlf +*.ps1 text eol=crlf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..4e649200 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,38 @@ +# Runtime image for the lab3 API. The base tag is pinned; dependencies are +# installed from requirements.txt, never from a mutable latest tag. +# +# Every replica runs exactly the same image. The replica alias is its +# container hostname (see app/config.py), so no container_name is needed and +# Compose can scale the service freely. + +FROM python:3.12.7-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app + +COPY requirements.txt ./ +RUN python -m pip install --requirement requirements.txt + +COPY app ./app +COPY migrations ./migrations +COPY student-tests ./student-tests + +USER app +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=6 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"] + +# One worker with a bounded thread pool per replica: the unit of horizontal +# scaling is the container, which keeps instance ids and CPU limits legible. +# Gunicorn stops accepting on SIGTERM and drains in-flight requests for at most +# 8 seconds; Docker's stop_grace_period gives it room to finish. +STOPSIGNAL SIGTERM +CMD ["gunicorn", "--bind=0.0.0.0:8080", "--workers=1", "--threads=8", "--timeout=30", "--graceful-timeout=8", "--keep-alive=5", "--worker-tmp-dir=/dev/shm", "--access-logfile=-", "--error-logfile=-", "app.main:app"] diff --git a/Makefile b/Makefile index 5d86d171..2b8d3d5a 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,16 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate up down migrate logs test load test-persistence test-failover test-instance test-idempotency test-student test-attachments test-negative test-memory test-presigned test-transfer test-reconcile test-private reconcile rolling-update help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API" + @echo "up/down: local compose stack; migrate: run the one-shot migrator explicitly" + @echo "test-persistence ACTION_SCRIPT=scripts/restart.cmd: restart with volumes" + @echo "test-failover ACTION_SCRIPT=scripts/stop-one.cmd: stop one API replica" + @echo "test-attachments/test-negative/test-memory/test-presigned: lab4 evidence" + @echo "test-reconcile/test-private: S3 orphan and private-bucket checks" + @echo "reconcile: dry-run S3/PostgreSQL reconciler in the tools profile" check: $(PYTHON) scripts/check_contract.py @@ -15,6 +21,18 @@ check: validate: $(PYTHON) scripts/validate_openapi.py +up: + docker compose --env-file .env up -d --build + +down: + docker compose --env-file .env down + +migrate: + docker compose --env-file .env run --rm migrate + +logs: + docker compose --env-file .env logs --tail=200 + test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) @@ -22,9 +40,44 @@ load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-instance: + $(PYTHON) student-tests/instance_spread.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-idempotency: + $(PYTHON) student-tests/idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachments: + $(PYTHON) student-tests/attachments_negative.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-negative: test-attachments + +test-memory: + $(PYTHON) student-tests/upload_memory.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-presigned: + $(PYTHON) student-tests/presigned_flow.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-transfer: + $(PYTHON) student-tests/transfer_compare.py --base-url "$(BASE_URL)" --output evidence/transfer-compare.json $(TLS_ARGS) + +test-reconcile: + $(PYTHON) student-tests/reconcile_orphans.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-private: + $(PYTHON) student-tests/private_bucket.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +reconcile: + docker compose --env-file .env --profile tools run --rm reconcile python -m app.reconcile --dry-run + +test-student: test-instance test-idempotency test-attachments test-presigned test-transfer + +rolling-update: + docker compose --env-file .env build api + sh scripts/rolling-update.sh diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..31e0a728 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,224 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Чиняева, БВТ2301 +- Целевая оценка: **5**; выбранный трек 5: **Трек 1. Прямой upload (presigned PUT)** +- Commit/PR: коммит реализации `f39ae06`; MR `!90` в `mtusi2026/mtusi2026lab4` (ветка `solution/BVT2301-Chinyaeva`) +- Использованные источники, библиотеки и помощь ИИ: контракт и публичные тесты репозитория; SeaweedFS 3.80 (S3 API), PostgreSQL 16.4, Redis 7.4, nginx 1.27.5, Flask 3.1, boto3 1.35.76, Pillow 11.0.0. ИИ использован как помощник при написании кода, тестов и текста; все команды и результаты ниже воспроизведены локально. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования: Docker Desktop/Engine с Compose v2, Python 3.9+ для тестов на хосте, +свободные порты `8080` (API через LB) и `9000` (S3 API для presigned upload). +Версии образов закреплены в `compose.yaml`: `postgres:16.4-alpine`, +`redis:7.4.0-alpine`, `chrislusf/seaweedfs:3.80`, `nginx:1.27.5-alpine`. + +```sh +git clone mtusi2026lab4-Chinyaeva +cd mtusi2026lab4-Chinyaeva +cp .env.example .env +# Замените все change-me значения в .env. Секреты в Git не попадают. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождитесь status=healthy у api и seaweedfs: +docker compose --env-file .env ps +curl -fsS http://localhost:8080/health/ready +python tests/smoke.py --base-url http://localhost:8080 +``` + +Остановка без удаления данных: + +```sh +docker compose --env-file .env down +``` + +Намеренное удаление только учебных данных (необратимо): + +```sh +docker compose --env-file .env down -v +``` + +Секреты создаются локально в `.env` (файл в `.gitignore`). В репозитории нет +паролей, ключей, токенов и подписанных URL. S3 credentials генерируются +оператором; API получает ограниченную identity, а административная identity +используется только одноразовым сервисом `s3-init`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент ── HTTP :8080 ── nginx LB ── API-1 ─┐ + API-2 ─┤── PostgreSQL (метаданные) + ├── Redis (сессии) + └── SeaweedFS S3 (байты, private bucket) +клиент ── presigned PUT :9000 ─────────────┘ (только трек 1, optional) +``` + +- **Источник истины:** PostgreSQL для пользователей, чатов, сообщений и + метаданных вложений; Redis — только активные сессии; S3 — только байты. + Локального постоянного каталога с оригиналами у API нет: временный файл + загрузки удаляется в `finally`. +- **LB:** nginx 1.27.5 с `resolver 127.0.0.11` и `server api:8080 resolve`. + Он видит все реплики Compose-сервиса `api`, пассивно исключает недоступную + реплику и повторяет только безопасные ошибки соединения/502/504. +- **Реплики:** две и более, общий PostgreSQL и Redis, `X-Instance-Id` в каждом + ответе. Масштабирование: `API_REPLICAS=3 docker compose ... up -d --scale api=3`. +- **S3:** SeaweedFS 3.80, master/volume/filer/S3 в одном контейнере. Bucket + `messenger-attachments` создаётся идемпотентным `s3-init`; bucket приватный. + API использует identity с `Read/Write/List`, без `Admin`; `s3-init` — с + `Admin`. Порт S3 опубликован только для presigned PUT; master/filer UI + (9333/8888) не публикуются. +- **Согласованность:** proxy upload сначала пишет объект в S3, затем строку в + PostgreSQL. Если COMMIT не удался, объект удаляется компенсацией; если процесс + упал между шагами, объект остаётся orphan и удаляется reconciler'ом после + grace period. Presigned upload создаёт `pending`-строку, а `complete` + проверяет владельца, размер и SHA-256 и только затем переводит в `ready`. +- **Совместимость:** обязательный контракт `contracts/openapi.json` не изменён. + Новые endpoint'ы описаны в `contracts/extensions.openapi.json` и + `contracts/extensions.md`; proxy endpoint остаётся обязательным fallback. + +### Переменные `.env` + +`DATABASE_URL`, `CACHE_URL`, `S3_ENDPOINT` (внутренний), `S3_PUBLIC_ENDPOINT` +(для подписи), `S3_BUCKET`, `S3_ACCESS_KEY_ID`/`S3_SECRET_ACCESS_KEY` +(ограниченная identity), `S3_ADMIN_*` (только `s3-init`), `MAX_UPLOAD_BYTES`, +`MAX_IMAGE_PIXELS`, `MAX_DECODE_CONCURRENCY`, `PRESIGN_TTL_SECONDS`, +`PRESIGN_GRACE_SECONDS`, `ORPHAN_GRACE_SECONDS`. Все они проброшены в +`compose.yaml`; `*_FILE`-варианты поддерживаются в `app/config.py`. + +## Обработка вложений и лимиты + +- Принимаются только `image/png` и `image/jpeg`; MIME сверяется с реальным + форматом, а не только с заголовком. Несовпадение/повреждение — `415`. +- Тело читается чанками по 64 KiB во временный файл; лимит 10 485 760 байт + проверяется и по `Content-Length`, и в потоке. Превышение — `413`. +- `Image.open` сначала проверяет размеры; `width * height > 40 000 000` — + `413` до полного декодирования. Полный `verify()`/`load()` ограничен + семафором `MAX_DECODE_CONCURRENCY` (по умолчанию 1 на реплику). +- `filename` — только display-метаданные: запрещены `/`, `\`, NUL, control + characters, `.`/`..`; object key — `chats//`. +- Скачивание идёт потоком через API с `Content-Type` из метаданных; redirect + на S3 нет. Посторонний участник получает `404` и для метаданных, и для bytes. +- Сообщение с `attachment_ids` проверяет, что каждое вложение существует, + принадлежит этому чату и имеет `status=ready`; чужой чат/неизвестный id — + `404`, неготовое — `409`. Пустой `text` разрешён только при наличии вложений. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python tests/smoke.py --base-url http://localhost:8080` | `PASS: lab4 public API scenario` | `evidence/smoke.txt` | +| Структура/контракт не сломаны | `python scripts/check_contract.py` | `PASS: offline skeleton structure...` | `evidence/check.txt` | +| S3 в общем стенде, метаданные в PG, байты в private bucket | `docker compose ps`; `private_bucket.py` | SeaweedFS healthy; anonymous GET 403; API identity не может create/delete bucket | `evidence/compose-ps.txt`, `evidence/private-bucket.txt` | +| POST вложения → 201 ready; оригинал побайтно; message с attachment_ids | `tests/smoke.py`, `presigned_flow.py` | SHA-256 и Content-Type совпадают; участник чата читает, посторонний — 404 | `evidence/smoke.txt`, `evidence/presigned-flow.txt` | +| PNG/JPEG ≤10 MiB и ≤40 млн пикселей; MIME по содержимому; опасные filename | `student-tests/attachments_negative.py` | 413/415/400 на все негативные случаи, без 500 | `evidence/attachments-negative.txt` | +| Upload → остановка обслужившей реплики → download через другую | `tests/resilience.py --mode failover --action-script scripts/stop-one.cmd` | 2 реплики до отказа; 89 успешных записей; recovery 3.667 s; данные и сессия сохранены | `evidence/failover.txt` | +| Restart с volumes сохраняет метаданные и оригинал | `tests/resilience.py --mode persistence --action-script scripts/restart.cmd` | PASS; attachment скачан после restart | `evidence/persistence.txt` | +| Уровень 4: тесты слишком большого файла, повреждения, MIME, чужого id, другого чата | `attachments_negative.py` | все сценарии PASS | `evidence/attachments-negative.txt` | +| Уровень 4: компенсация/сверка/очистка с grace period | `student-tests/reconcile_orphans.py` | dry-run видит orphan; свежий объект не удаляется; `--apply` удаляет; stale pending удаляется | `evidence/reconcile.txt` | +| Уровень 4: ≥10 одновременных загрузок, пик RAM | `student-tests/upload_memory.py --workers 10` | 10/10 успешно; рост ~22.5 MiB на реплику при файле 4.43 MB | `evidence/upload-memory.json`, `evidence/upload-memory.txt` | +| Уровень 4: идемпотентный bootstrap, отдельные credentials, закрытая admin-консоль | `private_bucket.py`; повторный `docker compose run --rm s3-init` | повторный bootstrap видит bucket; API identity без Admin; опубликован только S3 API | `evidence/private-bucket.txt`, `evidence/s3-init-idempotent.txt`, `evidence/compose-ps.txt` | +| Трек 1: presigned upload, короткая подпись, unique key, ownership, size/checksum, incomplete | `student-tests/presigned_flow.py` | 201 presign, 200 PUT, 200 complete; wrong checksum/size 400; foreign complete 404; incomplete 409 | `evidence/presigned-flow.txt` | +| Трек 1: сравнение трафика/RAM API на ≥2 размерах | `student-tests/transfer_compare.py` | proxy ≈ 2× file bytes через API; direct ≈ 1× (только verification read); RAM не растёт линейно | `evidence/transfer-compare.json`, `evidence/transfer-compare.txt` | +| Общая репликация/идемпотентность lab3 сохранены | `instance_spread.py`, `idempotency_race.py` | ≥2 реплики; 20/20 replay → один Message | `evidence/instance-spread.txt`, `evidence/idempotency-race.txt` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Эксперименты -## Эксперименты и отказы +### 1. 10 одновременных proxy-загрузок -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Команда: + +```sh +python student-tests/upload_memory.py --base-url http://localhost:8080 \ + --workers 10 --output evidence/upload-memory.json +``` + +Файл: синтетический PNG 1400×1000, `4 431 995` байт, 10 параллельных +загрузок. API-контейнеры ограничены 512 MiB. Результат: + +```json +{"workers":10,"file_bytes":4431995,"elapsed_seconds":1.697, + "baseline_mib":{"mtusi-lab4-api-1":83.17,"mtusi-lab4-api-2":85.38}, + "peak_mib":{"mtusi-lab4-api-1":105.7,"mtusi-lab4-api-2":107.9}, + "peak_growth_mib":{"mtusi-lab4-api-1":22.53,"mtusi-lab4-api-2":22.52}, + "successful_uploads":10} +``` + +Пик не растёт как 10×размер файла: тело на диске, декодирование ограничено +семафором, поэтому рост ~22 MiB на реплику. Это измерение одного хоста; оно не +доказывает отсутствие OOM при произвольном размере, но подтверждает заявленную +границу для 10 MiB и 40 млн пикселей. + +### 2. Proxy vs presigned direct (трек 1) + +Команда: + +```sh +python student-tests/transfer_compare.py --base-url http://localhost:8080 \ + --output evidence/transfer-compare.json +``` + +Измерялись суммарные RX+TX байты API-контейнеров через `/proc/net/dev` и пик +памяти через `docker stats`. Результат: + +| Размер файла | Proxy: API network | Direct: API network | Proxy: client→API | Direct: client→S3 | +| --- | ---: | ---: | ---: | ---: | +| 552 409 B | 1 120 316 B | 564 368 B | 552 409 B | 552 409 B | +| 4 424 547 B | 8 888 758 B | 4 446 591 B | 4 424 547 B | 4 424 547 B | + +Proxy проводит тело через API дважды (вход от клиента и выход в S3); direct +освобождает публичный ingress API, но `complete` один раз читает объект из S3 +для проверки SHA-256. Поэтому direct ≈ 1× file bytes, proxy ≈ 2×. RAM API при +direct не растёт: объект читается потоком, без буферизации всего файла +(`api_memory_growth_mib=0.0` в обоих размерах). Это честная цена проверки +целостности; без неё direct был бы ещё дешевле, но weaker guarantee. + +### 3. Нагрузка сообщений (регрессия lab3) + +```sh +python tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +``` + +30.07 s, 4 closed-loop воркера, 6 789 успешных запросов, 0 ошибок, +225.78 successful RPS, p50 14.8 ms, p95 33.8 ms, p99 66.7 ms. Это не +пропускная способность production: closed-loop и один Docker-host. + +## Отказы | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка одной API-реплики (`stop-one.cmd`) | nginx исключает её, записи идут на вторую; сессия жива | 2 реплики до отказа, 89 успешных записей, 3 неуверенных retry, `first_recovery_seconds=3.667` | ~3.7 s; подтверждённые сообщения и сессия сохранены | +| `docker compose restart` (PostgreSQL, Redis, SeaweedFS, API, LB) | volumes сохраняют данные; API снова ready | persistence PASS; attachment скачан после restart | данные не потеряны; API поднялся после прогрева SeaweedFS | +| S3 недоступен | `/health/ready` = 503, API не принимает новые вложения | при старте API health был 503 до готовности S3; после готовности 200 | зависит от S3; metadata не теряется | +| Обрыв между S3 PUT и INSERT | объект не должен остаться навсегда | proxy-компенсация удаляет объект; reconciler удаляет orphan после grace period | потерян только незавершённый объект; метаданных нет | +| Незавершённый presigned upload | pending не должен жить вечно | reconciler с `--pending-grace-seconds 0` удаляет строку и объект; повторный GET = 404 | потерян только незавершённый upload | +| Чужой/неизвестный attachment_id | 404, не 500 | negative suite PASS | нет | +| Слишком большой/битый файл, MIME spoof | 413/415, не 500 | negative suite PASS | нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host не даёт отказоустойчивости к потере host; nginx, PostgreSQL, + Redis и SeaweedFS остаются single points of failure в учебном стенде. +- `complete` читает объект из S3 для проверки SHA-256. Это осознанный + компромисс: direct upload экономит публичный ingress API, но не убирает + verification egress. В production можно использовать S3 checksum API или + доверенный client checksum с отдельной моделью угроз. +- `reconcile` — операторский инструмент, не фоновый демон. В lab5 его можно + запускать по cron/worker с метриками; сейчас он запускается вручную и + поддерживает dry-run/повторный apply. +- Presigned URL публикует S3 API на `localhost:9000`; в другом окружении + `S3_PUBLIC_ENDPOINT` нужно заменить на реальный публичный адрес. Bucket + остаётся приватным, но сетевой доступ к S3 API должен быть ограничен + firewall'ом/прокси в production. +- Лимит 40 млн пикселей и 10 MiB — контрактные; при их увеличении нужно + пересмотреть `MAX_DECODE_CONCURRENCY` и memory limit API. +- Публичные тесты не заменяют защиту: сценарии ниже проверены вручную и + студенческими тестами, но оценка выставляется преподавателем. + +## Источники и ИИ + +- `contracts/openapi.json`, `contracts/README.md`, `tests/README.md` — контракт + и публичные проверки. +- SeaweedFS S3 configuration and presigned URLs; PostgreSQL deferrable + constraints; boto3 `generate_presigned_url`; Pillow decompression limits. +- ИИ-помощник использован для генерации черновиков кода/тестов и текста + отчёта. Все результаты в `evidence/` получены запуском команд на локальном + стенде; секреты и токены в отчёт не включены. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..edec7955 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +# Messenger API for MTUSI high-load course, laboratory 2. diff --git a/app/attachments.py b/app/attachments.py new file mode 100644 index 00000000..c7327e64 --- /dev/null +++ b/app/attachments.py @@ -0,0 +1,135 @@ +"""Attachment upload validation and serialisation. + +Design rules for laboratory 4: + +* raw bytes are streamed to an ephemeral file, never accumulated in RAM; +* the declared MIME type must match the format sniffed from the bytes; +* the pixel budget is checked before the image is decoded; +* a small semaphore bounds how many full decodes run at once per API replica; +* the object key is generated by the server and never contains the filename. +""" +from __future__ import annotations + +import hashlib +import os +import tempfile +import threading +import uuid +from datetime import datetime, timezone + +from PIL import Image, UnidentifiedImageError + +from .config import settings +from .errors import ApiError, payload_too_large, unsupported_media_type + +# Pillow's own bomb guard is a second line of defence; the explicit product +# check below is the contract limit and runs before verify()/load(). +Image.MAX_IMAGE_PIXELS = settings.max_image_pixels + +_decoders = threading.BoundedSemaphore(settings.max_decode_concurrency) + +ALLOWED = {"PNG": "image/png", "JPEG": "image/jpeg"} + + +def format_time(value: datetime) -> str: + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def object_key(chat_id: uuid.UUID, attachment_id: uuid.UUID) -> str: + # Random UUIDs make the key unguessable; the client filename is display + # metadata only. A stable prefix helps the reconciler and operators. + return f"chats/{chat_id}/{attachment_id}" + + +def attachment_json(row: dict) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": int(row["size_bytes"]), + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": row["error_code"], + "created_at": format_time(row["created_at"]), + "updated_at": format_time(row["updated_at"]), + } + + +def read_to_temp(stream) -> tuple[str, int, str]: + """Stream the request body to a bounded temporary file. + + The caller owns the returned path and must delete it in a finally block. + """ + digest = hashlib.sha256() + total = 0 + handle = tempfile.NamedTemporaryFile( + mode="wb", prefix="upload-", suffix=".bin", dir=settings.upload_temp_dir, delete=False + ) + path = handle.name + try: + while True: + try: + chunk = stream.read(64 * 1024) + except Exception as error: + # Flask raises RequestEntityTooLarge when MAX_CONTENT_LENGTH is + # exceeded on a chunked request. + if getattr(error, "code", None) == 413: + raise payload_too_large() from error + raise + if not chunk: + break + total += len(chunk) + if total > settings.max_upload_bytes: + raise payload_too_large("Файл больше 10 MiB") + digest.update(chunk) + handle.write(chunk) + finally: + handle.close() + if total == 0: + os.unlink(path) + raise unsupported_media_type("Пустой файл") + return path, total, digest.hexdigest() + + +def validate_image(path: str, declared_content_type: str) -> None: + """Sniff the real format, enforce the pixel budget, then fully decode.""" + try: + with Image.open(path) as image: + actual = ALLOWED.get(image.format or "") + if actual is None: + raise unsupported_media_type("Поддерживаются только PNG и JPEG") + if actual != declared_content_type: + raise unsupported_media_type("Content-Type не совпадает с содержимым") + width, height = image.size + if width < 1 or height < 1: + raise unsupported_media_type("Некорректные размеры изображения") + if width * height > settings.max_image_pixels: + raise payload_too_large("Слишком много пикселей") + # verify() parses the container and checks CRCs without keeping the + # decoded bitmap; load() below is the bounded full decode. + image.verify() + except ApiError: + raise + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error: + raise unsupported_media_type("Повреждённое изображение") from error + + with _decoders: + try: + with Image.open(path) as image: + image.load() + except (UnidentifiedImageError, OSError, ValueError, SyntaxError) as error: + raise unsupported_media_type("Повреждённое изображение") from error + + +def cleanup(path: str | None) -> None: + if not path: + return + try: + os.unlink(path) + except FileNotFoundError: + return + except OSError: + return diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..8ed1d0bd --- /dev/null +++ b/app/config.py @@ -0,0 +1,134 @@ +"""Runtime configuration. + +The application never embeds credentials. Local Compose passes environment +variables from an ignored .env file; *_FILE variants are accepted as well so a +container platform can mount secrets without copying them into the image. + +Laboratory 4 adds S3 settings. The API uses a restricted application identity; +only the one-shot bootstrap service receives the administrative identity. +""" +from __future__ import annotations + +import os +import socket +from dataclasses import dataclass +from pathlib import Path + + +def _value(name: str, fallback_name: str | None = None, default: str | None = None) -> str: + raw = os.environ.get(name) + if raw: + return raw + if fallback_name: + filename = os.environ.get(fallback_name) + if filename: + path = Path(filename) + text = path.read_text(encoding="utf-8").strip() + if text: + return text + if default is not None: + return default + raise RuntimeError(f"missing required configuration: {name}") + + +def _positive_int(name: str, default: int) -> int: + raw = os.environ.get(name, str(default)) + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be an integer") from exc + if value < 1: + raise RuntimeError(f"{name} must be positive") + return value + + +def _positive_float(name: str, default: float) -> float: + raw = os.environ.get(name, str(default)) + try: + value = float(raw) + except ValueError as exc: + raise RuntimeError(f"{name} must be a number") from exc + if value <= 0: + raise RuntimeError(f"{name} must be positive") + return value + + +def _instance_id() -> str: + """Stable public alias of one API process/container. + + Containers use their hostname (the short container id) by default. An + explicit INSTANCE_ID is useful for non-container deployments or for a + randomly generated lab7 alias. The value never contains host IP data. + """ + value = (os.environ.get("INSTANCE_ID") or socket.gethostname()).strip() + if not value or len(value) > 128 or any(not (c.isalnum() or c in "-._:") for c in value): + raise RuntimeError("INSTANCE_ID must be 1..128 safe characters") + return value + + +@dataclass(frozen=True) +class Settings: + database_url: str + cache_url: str + instance_id: str + session_ttl_seconds: int + idempotency_ttl_seconds: int + db_pool_min: int + db_pool_max: int + db_pool_acquire_timeout: float + db_connect_timeout: int + db_statement_timeout_ms: int + redis_connect_timeout: float + redis_socket_timeout: float + max_body_bytes: int + max_upload_bytes: int + max_image_pixels: int + max_decode_concurrency: int + upload_temp_dir: str + s3_endpoint: str + s3_public_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key_id: str + s3_secret_access_key: str + s3_connect_timeout: float + s3_read_timeout: float + presign_ttl_seconds: int + presign_grace_seconds: int + orphan_grace_seconds: int + + +def load_settings() -> Settings: + return Settings( + database_url=_value("DATABASE_URL", "DATABASE_URL_FILE"), + cache_url=_value("CACHE_URL", "CACHE_URL_FILE"), + instance_id=_instance_id(), + session_ttl_seconds=_positive_int("SESSION_TTL_SECONDS", 3600), + idempotency_ttl_seconds=_positive_int("IDEMPOTENCY_TTL_SECONDS", 86400), + db_pool_min=_positive_int("DB_POOL_MIN", 1), + db_pool_max=_positive_int("DB_POOL_MAX", 10), + db_pool_acquire_timeout=float(os.environ.get("DB_POOL_ACQUIRE_TIMEOUT_SECONDS", "3")), + db_connect_timeout=_positive_int("DB_CONNECT_TIMEOUT_SECONDS", 3), + db_statement_timeout_ms=_positive_int("DB_STATEMENT_TIMEOUT_MS", 5000), + redis_connect_timeout=float(os.environ.get("REDIS_CONNECT_TIMEOUT_SECONDS", "2")), + redis_socket_timeout=float(os.environ.get("REDIS_SOCKET_TIMEOUT_SECONDS", "2")), + max_body_bytes=_positive_int("MAX_BODY_BYTES", 1024 * 1024), + max_upload_bytes=_positive_int("MAX_UPLOAD_BYTES", 10 * 1024 * 1024), + max_image_pixels=_positive_int("MAX_IMAGE_PIXELS", 40_000_000), + max_decode_concurrency=_positive_int("MAX_DECODE_CONCURRENCY", 1), + upload_temp_dir=os.environ.get("UPLOAD_TEMP_DIR", "/tmp"), + s3_endpoint=_value("S3_ENDPOINT", "S3_ENDPOINT_FILE", "http://seaweedfs:8333"), + s3_public_endpoint=_value("S3_PUBLIC_ENDPOINT", "S3_PUBLIC_ENDPOINT_FILE", "http://localhost:9000"), + s3_region=_value("S3_REGION", default="us-east-1"), + s3_bucket=_value("S3_BUCKET", default="messenger-attachments"), + s3_access_key_id=_value("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE"), + s3_secret_access_key=_value("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE"), + s3_connect_timeout=_positive_float("S3_CONNECT_TIMEOUT_SECONDS", 3.0), + s3_read_timeout=_positive_float("S3_READ_TIMEOUT_SECONDS", 30.0), + presign_ttl_seconds=_positive_int("PRESIGN_TTL_SECONDS", 300), + presign_grace_seconds=_positive_int("PRESIGN_GRACE_SECONDS", 3600), + orphan_grace_seconds=_positive_int("ORPHAN_GRACE_SECONDS", 86400), + ) + + +settings = load_settings() diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..edb460bb --- /dev/null +++ b/app/db.py @@ -0,0 +1,174 @@ +"""PostgreSQL access: bounded pool, timeouts, and versioned migrations. + +Laboratory 3 separates schema management from request serving. The one-shot +``migrate`` Compose service calls :func:`apply_migrations`; API replicas only +open their pool and wait until the newest migration is visible. That prevents +N replicas from racing each other on DDL during a deploy. +""" +from __future__ import annotations + +import logging +import threading +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import settings + +log = logging.getLogger("messenger.db") +MIGRATIONS_DIR = Path(__file__).resolve().parents[1] / "migrations" +LOCK_ID = 0x4D545553 # stable application-wide advisory lock + +_pool: ConnectionPool | None = None +_pool_lock = threading.Lock() +_ready = threading.Event() +_started = False +_start_lock = threading.Lock() + + +def _create_pool() -> ConnectionPool: + options = ( + f"-c statement_timeout={settings.db_statement_timeout_ms} " + "-c idle_in_transaction_session_timeout=10000 " + "-c lock_timeout=3000" + ) + return ConnectionPool( + conninfo=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_pool_acquire_timeout, + open=False, + kwargs={ + "row_factory": dict_row, + "connect_timeout": settings.db_connect_timeout, + "application_name": "mtusi-messenger-lab3", + "options": options, + }, + ) + + +def pool() -> ConnectionPool: + global _pool + with _pool_lock: + if _pool is None: + _pool = _create_pool() + return _pool + + +def required_version() -> str: + """Newest migration shipped in this image; readiness waits for it.""" + versions = sorted(path.stem for path in MIGRATIONS_DIR.glob("*.sql")) + if not versions: + raise RuntimeError("no migrations found") + return versions[-1] + + +def open_pool(timeout: float = 60.0) -> None: + """Open the pool with bounded retries; used by migrate and API startup.""" + deadline = time.monotonic() + timeout + backoff = 0.5 + while True: + try: + pool().open(wait=True, timeout=min(5.0, max(0.5, deadline - time.monotonic()))) + return + except Exception as exc: + if time.monotonic() >= deadline: + raise + log.warning("database is not reachable yet: %s", type(exc).__name__) + time.sleep(backoff) + backoff = min(backoff * 2, 5.0) + + +def apply_migrations() -> list[str]: + """Apply every migration exactly once. + + A session-level advisory lock serialises the one-shot migrate service and + any explicit operator run. Each migration and its schema_migrations row + share one transaction, so a failed DDL statement leaves no version row. + """ + applied_now: list[str] = [] + with pool().connection() as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + try: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + """ + ) + applied = { + row["version"] + for row in conn.execute("SELECT version FROM schema_migrations").fetchall() + } + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + version = path.stem + if version in applied: + continue + log.info("applying migration %s", version) + conn.execute(path.read_text(encoding="utf-8")) + conn.execute( + "INSERT INTO schema_migrations(version) VALUES (%s)", (version,) + ) + applied_now.append(version) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + return applied_now + + +def schema_is_current() -> bool: + """Cheap readiness probe: the newest migration is committed and visible.""" + if _pool is None: + return False + try: + with pool().connection(timeout=1.0) as conn: + row = conn.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (required_version(),), + ).fetchone() + return row is not None + except Exception: + return False + + +def start_background_initialisation() -> None: + """Open the pool and wait for the one-shot migrator, without blocking the + liveness endpoint. The API never performs DDL itself.""" + global _started + with _start_lock: + if _started: + return + _started = True + + def run() -> None: + backoff = 0.5 + opened = False + while True: + try: + if not opened: + open_pool(timeout=5.0) + opened = True + if schema_is_current(): + _ready.set() + log.info("database pool is ready; schema %s is current", required_version()) + return + log.warning("waiting for migrate service: schema %s is not ready", required_version()) + except Exception as exc: + log.warning("database initialisation is not ready: %s", type(exc).__name__) + time.sleep(backoff) + backoff = min(backoff * 2, 5.0) + + threading.Thread(target=run, name="db-init", daemon=True).start() + + +def is_ready() -> bool: + return _ready.is_set() + + +def check_ready() -> bool: + """Readiness checks the live connection and the migrated schema each time.""" + return is_ready() and schema_is_current() diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..0234ee08 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,44 @@ +"""Typed public errors. Internal details never cross the HTTP boundary.""" +from __future__ import annotations + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: dict[str, str] | None = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str = "Некорректный запрос") -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(*, basic: bool = False) -> ApiError: + challenge = ( + 'Basic realm="messenger", charset="UTF-8"' + if basic + else 'Bearer realm="messenger"' + ) + return ApiError(401, "unauthorized", "Требуется аутентификация", {"WWW-Authenticate": challenge}) + + +def not_found(message: str = "Объект не найден") -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str = "Конфликт") -> ApiError: + return ApiError(409, "conflict", message) + + +def payload_too_large(message: str = "Файл слишком велик") -> ApiError: + return ApiError(413, "payload_too_large", message) + + +def unsupported_media_type(message: str = "Неподдерживаемый тип файла") -> ApiError: + return ApiError(415, "unsupported_media_type", message) + + +def unavailable(message: str = "Сервис временно недоступен") -> ApiError: + return ApiError(503, "unavailable", message) diff --git a/app/idempotency.py b/app/idempotency.py new file mode 100644 index 00000000..5427741b --- /dev/null +++ b/app/idempotency.py @@ -0,0 +1,179 @@ +"""Shared, transactional idempotency for message creation. + +The unique scope is (user, method, path, key). The idempotency row, the +message row and its attachment links are written in one PostgreSQL +transaction, so a crash or rollback cannot leave a committed key without its +message. Concurrent retries through different API replicas converge on the +same primary-key row: the loser of the INSERT ... ON CONFLICT waits for the +winner to commit and then reads the original message. + +Only the canonical body is hashed, so JSON key order and an omitted +``attachment_ids`` equal to ``[]`` are the same request. +""" +from __future__ import annotations + +import hashlib +import json +import secrets +import uuid +from datetime import datetime + +from . import db +from .config import settings +from .errors import conflict, not_found + + +def fingerprint(text: str, attachment_ids: list[str]) -> str: + payload = { + "text": text, + "attachment_ids": sorted(attachment_ids), + } + canonical = json.dumps(payload, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + +def _cleanup_expired(conn) -> None: + """Best-effort maintenance; never on the critical path of every request.""" + if secrets.randbelow(100) != 0: + return + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE ctid IN ( + SELECT ctid FROM idempotency_keys + WHERE expires_at <= clock_timestamp() + LIMIT 1000 + ) + """ + ) + + +def _validate_attachments(conn, chat_id: uuid.UUID, attachment_ids: list[str]) -> None: + if not attachment_ids: + return + parsed = [uuid.UUID(item) for item in attachment_ids] + rows = conn.execute( + """ + SELECT id, chat_id, status + FROM attachments + WHERE id = ANY(%s) + FOR SHARE + """, + (parsed,), + ).fetchall() + if len(rows) != len(parsed): + raise not_found("Вложение не найдено") + by_id = {str(row["id"]): row for row in rows} + for attachment_id in attachment_ids: + row = by_id.get(attachment_id) + if row is None or row["chat_id"] != chat_id: + raise not_found("Вложение не найдено в этом чате") + if row["status"] != "ready": + raise conflict("Вложение ещё не готово") + + +def _message_with_attachments(conn, message_id) -> dict: + row = conn.execute( + """ + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) AS attachment_ids + FROM messages m + WHERE m.id = %s + """, + (message_id,), + ).fetchone() + if row is None: + raise conflict("Результат ещё не готов, повторите запрос") + return row + + +def create_message( + *, + chat_id: uuid.UUID, + actor_id: uuid.UUID, + path: str, + key: str, + text: str, + attachment_ids: list[str], +) -> dict: + """Return the original or newly created message row, atomically.""" + request_hash = fingerprint(text, attachment_ids) + message_id = uuid.uuid4() + with db.pool().connection() as conn: + with conn.transaction(): + _validate_attachments(conn, chat_id, attachment_ids) + + # An expired key may be reused: remove it inside the same scope. + conn.execute( + """ + DELETE FROM idempotency_keys + WHERE user_id = %s AND method = 'POST' AND path = %s AND key = %s + AND expires_at <= clock_timestamp() + """, + (actor_id, path, key), + ) + _cleanup_expired(conn) + inserted = conn.execute( + """ + INSERT INTO idempotency_keys + (user_id, method, path, key, request_hash, message_id, expires_at) + VALUES + (%s, 'POST', %s, %s, %s, %s, + clock_timestamp() + (%s * interval '1 second')) + ON CONFLICT (user_id, method, path, key) DO NOTHING + RETURNING message_id + """, + ( + actor_id, + path, + key, + request_hash, + message_id, + settings.idempotency_ttl_seconds, + ), + ).fetchone() + + if inserted is not None: + conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + """, + (inserted["message_id"], chat_id, actor_id, text), + ) + if attachment_ids: + with conn.cursor() as cursor: + cursor.executemany( + """ + INSERT INTO message_attachments + (message_id, attachment_id, position) + VALUES (%s, %s, %s) + """, + [ + (inserted["message_id"], uuid.UUID(item), position) + for position, item in enumerate(attachment_ids) + ], + ) + return _message_with_attachments(conn, inserted["message_id"]) + + existing = conn.execute( + """ + SELECT request_hash, message_id, expires_at + FROM idempotency_keys + WHERE user_id = %s AND method = 'POST' AND path = %s AND key = %s + """, + (actor_id, path, key), + ).fetchone() + if existing is None: + # The row disappeared between the conflict and this read (only + # possible with a concurrent expiry cleanup). 409 asks the + # client to retry rather than creating an ambiguous duplicate. + raise conflict("Повторите запрос: ключ идемпотентности истёк") + if existing["request_hash"] != request_hash: + raise conflict("Idempotency-Key уже использован с другим сообщением") + return _message_with_attachments(conn, existing["message_id"]) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..1c2673d3 --- /dev/null +++ b/app/main.py @@ -0,0 +1,746 @@ +"""HTTP API for laboratory 3 (PostgreSQL + Redis sessions behind a LB).""" +from __future__ import annotations + +import json +import logging +import os +import time +import uuid +from datetime import datetime, timedelta, timezone +from urllib.parse import quote + +import psycopg +import psycopg_pool +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerifyMismatchError +from flask import Flask, Response, g, request + +from . import attachments as attachment_store +from . import db, idempotency, s3, sessions +from .config import settings +from .errors import ( + ApiError, + bad_request, + conflict, + not_found, + payload_too_large, + unauthorized, + unavailable, + unsupported_media_type, +) +from .validation import ( + byte_size, + cursor_time, + decode_cursor, + display_name, + encode_cursor, + exact_keys, + filename, + idempotency_key, + image_content_type, + json_object, + limit_arg, + message_text, + non_blank, + password, + sha256_hex, + username, + uuid_list, + uuid_value, +) + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger.api") + +app = Flask(__name__) +# Uploads stream through the handler; the Flask ceiling is deliberately just +# above the contract limit so a single oversized request is rejected early. +app.config["MAX_CONTENT_LENGTH"] = settings.max_upload_bytes + 64 * 1024 +hasher = PasswordHasher() + + +# --------------------------------------------------------------------------- +# Response helpers and middleware +# --------------------------------------------------------------------------- + +def _json(payload: object, status: int = 200, headers: dict[str, str] | None = None) -> Response: + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + response = Response(body, status=status, mimetype="application/json") + for key, value in (headers or {}).items(): + response.headers[key] = value + return response + + +def _error_response(error: ApiError, request_id: str) -> Response: + return _json( + { + "error": { + "code": error.code, + "message": error.message, + "request_id": request_id, + } + }, + error.status, + error.headers, + ) + + +@app.before_request +def _request_context() -> None: + g.request_id = uuid.uuid4().hex + g.started_at = time.monotonic() + db.start_background_initialisation() + + +@app.after_request +def _response_headers(response: Response) -> Response: + response.headers["X-Request-Id"] = g.get("request_id", uuid.uuid4().hex) + response.headers["X-Instance-Id"] = settings.instance_id + response.headers["Cache-Control"] = "no-store" + return response + + +@app.errorhandler(ApiError) +def _handle_api_error(error: ApiError) -> Response: + return _error_response(error, g.get("request_id", uuid.uuid4().hex)) + + +@app.errorhandler(413) +def _handle_too_large(_error: object) -> Response: + return _error_response( + ApiError(413, "payload_too_large", "Тело запроса слишком велико"), + g.get("request_id", uuid.uuid4().hex), + ) + + +@app.errorhandler(404) +def _handle_not_found(_error: object) -> Response: + return _error_response(not_found(), g.get("request_id", uuid.uuid4().hex)) + + +@app.errorhandler(405) +def _handle_method(_error: object) -> Response: + return _error_response(bad_request("Метод не поддерживается"), g.get("request_id", uuid.uuid4().hex)) + + +@app.errorhandler(Exception) +def _handle_unexpected(error: Exception) -> Response: + request_id = g.get("request_id", uuid.uuid4().hex) + if isinstance(error, (psycopg.OperationalError, psycopg_pool.PoolTimeout, psycopg_pool.PoolClosed)): + log.warning("database unavailable on %s %s: %s", request.method, request.path, type(error).__name__) + return _error_response(unavailable("База данных недоступна"), request_id) + if isinstance(error, redis.RedisError): + log.warning("session store unavailable on %s %s: %s", request.method, request.path, type(error).__name__) + return _error_response(unavailable("Хранилище сессий недоступно"), request_id) + if isinstance(error, s3.S3Unavailable): + log.warning("object store unavailable on %s %s", request.method, request.path) + return _error_response(unavailable("Объектное хранилище недоступно"), request_id) + log.exception("unhandled error on %s %s", request.method, request.path) + return _error_response( + ApiError(500, "internal_error", "Внутренняя ошибка сервера"), request_id + ) + + +# --------------------------------------------------------------------------- +# Serialisation +# --------------------------------------------------------------------------- + +def format_time(value: datetime) -> str: + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def user_json(row: dict) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": format_time(row["created_at"]), + } + + +def chat_json(row: dict) -> dict: + member_ids = sorted(str(item) for item in (row.get("member_ids") or [])) + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": format_time(row["created_at"]), + } + + +def message_json(row: dict) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": format_time(row["created_at"]), + "attachment_ids": [str(item) for item in (row.get("attachment_ids") or [])], + } + + +# --------------------------------------------------------------------------- +# Authentication +# --------------------------------------------------------------------------- + +def _bearer_token() -> str: + header = request.headers.get("Authorization", "") + parts = header.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip(): + raise unauthorized() + return parts[1].strip() + + +def current_user() -> dict: + cached = g.get("current_user") + if cached is not None: + return cached + token = _bearer_token() + user_id, token_digest = sessions.resolve(token) + if not user_id: + raise unauthorized() + try: + parsed = uuid.UUID(user_id) + except ValueError as exc: + raise unauthorized() from exc + with db.pool().connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (parsed,), + ).fetchone() + if row is None: + sessions.revoke(token_digest) + raise unauthorized() + g.current_user = row + g.session_token_digest = token_digest + return row + + +def _basic_credentials() -> tuple[str, str]: + header = request.headers.get("Authorization", "") + parts = header.split(None, 1) + if len(parts) != 2 or parts[0].lower() != "basic": + raise unauthorized(basic=True) + import base64 + + try: + decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise unauthorized(basic=True) from exc + name, separator, secret = decoded.partition(":") + if not separator or not name or not secret: + raise unauthorized(basic=True) + if len(name) > 32 or any(ord(char) > 127 for char in name): + raise unauthorized(basic=True) + return name, secret + + +def _authenticate_basic() -> dict: + name, secret = _basic_credentials() + with db.pool().connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, password_hash, created_at FROM users WHERE username = %s", + (name,), + ).fetchone() + if row is None: + raise unauthorized(basic=True) + try: + hasher.verify(row["password_hash"], secret) + except (VerifyMismatchError, InvalidHashError): + raise unauthorized(basic=True) + return row + + +# --------------------------------------------------------------------------- +# Health +# --------------------------------------------------------------------------- + +@app.get("/health/live") +def health_live() -> Response: + return _json({"status": "ok"}) + + +@app.get("/health/ready") +def health_ready() -> Response: + if not db.check_ready(): + raise unavailable("База данных ещё не готова") + sessions.ping() + if not s3.check_ready(): + raise unavailable("Объектное хранилище ещё не готово") + return _json({"status": "ok"}) + + +# --------------------------------------------------------------------------- +# Users and sessions +# --------------------------------------------------------------------------- + +@app.post("/api/v1/users") +def register_user() -> Response: + body = json_object(request, settings.max_body_bytes) + exact_keys(body, ("username", "display_name", "password")) + name = username(body["username"]) + shown_name = display_name(body["display_name"]) + secret = password(body["password"]) + user_id = uuid.uuid4() + try: + with db.pool().connection() as conn: + row = conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (user_id, name, shown_name, hasher.hash(secret)), + ).fetchone() + except psycopg.errors.UniqueViolation as exc: + raise conflict("Username уже занят") from exc + return _json(user_json(row), 201) + + +@app.post("/api/v1/auth/sessions") +def create_session() -> Response: + row = _authenticate_basic() + token, expires_at = sessions.create(str(row["id"])) + return _json( + { + "token": token, + "token_type": "Bearer", + "expires_at": format_time(expires_at), + "user": user_json(row), + }, + 201, + ) + + +@app.delete("/api/v1/auth/sessions/current") +def revoke_session() -> Response: + current_user() # authentication must happen before revocation + sessions.revoke(g.session_token_digest) + return Response(status=204) + + +@app.get("/api/v1/users/me") +def current_user_endpoint() -> Response: + return _json(user_json(current_user())) + + +# --------------------------------------------------------------------------- +# Chats +# --------------------------------------------------------------------------- + +def _require_chat_member(chat_id: uuid.UUID, user_id: uuid.UUID) -> None: + with db.pool().connection() as conn: + row = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if row is None: + raise not_found() + + +@app.post("/api/v1/chats") +def create_chat() -> Response: + actor = current_user() + body = json_object(request, settings.max_body_bytes) + exact_keys(body, ("title", "member_ids")) + title = non_blank(body["title"], "title", 1, 100) + member_ids = uuid_list(body["member_ids"], "member_ids") + actor_id = str(actor["id"]) + if actor_id not in member_ids: + raise bad_request("Создатель должен входить в member_ids") + + member_uuids = [uuid.UUID(item) for item in member_ids] + chat_id = uuid.uuid4() + with db.pool().connection() as conn: + existing = conn.execute( + "SELECT id FROM users WHERE id = ANY(%s)", (member_uuids,) + ).fetchall() + if len(existing) != len(member_uuids): + raise bad_request("Указан неизвестный участник") + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, title), + ).fetchone() + with conn.cursor() as cursor: + cursor.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member) for member in member_uuids], + ) + payload = dict(row) + payload["member_ids"] = member_uuids + return _json(chat_json(payload), 201) + + +@app.get("/api/v1/chats") +def list_chats() -> Response: + actor = current_user() + limit = limit_arg(request.args.get("limit")) + cursor = decode_cursor(request.args.get("cursor")) + parameters: list[object] = [actor["id"]] + predicate = "" + if cursor is not None: + if cursor.get("kind") != "chats" or cursor.get("user_id") != str(actor["id"]): + raise bad_request("Некорректный cursor") + try: + cursor_created = cursor_time(cursor.get("created_at")) + cursor_id = uuid_value(cursor.get("id"), "cursor") + except ApiError: + raise bad_request("Некорректный cursor") + predicate = "AND (c.created_at, c.id) > (%s, %s)" + parameters.extend([cursor_created, cursor_id]) + parameters.append(limit + 1) + query = f""" + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm.user_id FROM chat_members cm + WHERE cm.chat_id = c.id ORDER BY cm.user_id + ) AS member_ids + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE TRUE {predicate} + ORDER BY c.created_at, c.id + LIMIT %s + """ + with db.pool().connection() as conn: + rows = conn.execute(query, tuple(parameters)).fetchall() + page = rows[:limit] + next_cursor = None + if len(rows) > limit and page: + last = page[-1] + next_cursor = encode_cursor( + { + "v": 1, + "kind": "chats", + "user_id": str(actor["id"]), + "created_at": format_time(last["created_at"]), + "id": str(last["id"]), + } + ) + return _json({"items": [chat_json(row) for row in page], "next_cursor": next_cursor}) + + +@app.get("/api/v1/chats/") +def get_chat(chat_id: str) -> Response: + actor = current_user() + parsed_chat_id = uuid_value(chat_id, "chat_id") + with db.pool().connection() as conn: + row = conn.execute( + """ + SELECT c.id, c.title, c.created_at, + ARRAY( + SELECT cm.user_id FROM chat_members cm + WHERE cm.chat_id = c.id ORDER BY cm.user_id + ) AS member_ids + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s + """, + (actor["id"], parsed_chat_id), + ).fetchone() + if row is None: + raise not_found() + return _json(chat_json(row)) + + +# --------------------------------------------------------------------------- +# Messages +# --------------------------------------------------------------------------- + +@app.post("/api/v1/chats//messages") +def create_message(chat_id: str) -> Response: + actor = current_user() + parsed_chat_id = uuid_value(chat_id, "chat_id") + _require_chat_member(parsed_chat_id, actor["id"]) + body = json_object(request, settings.max_body_bytes) + exact_keys(body, ("text",), ("attachment_ids",)) + attachments = uuid_list(body.get("attachment_ids", []), "attachment_ids", 0, 10) + text = message_text(body["text"], allow_empty=bool(attachments)) + key = idempotency_key(request.headers.get("Idempotency-Key")) + row = idempotency.create_message( + chat_id=parsed_chat_id, + actor_id=actor["id"], + path=request.path, + key=key, + text=text, + attachment_ids=attachments, + ) + return _json(message_json(row), 201) + + +@app.get("/api/v1/chats//messages") +def list_messages(chat_id: str) -> Response: + actor = current_user() + parsed_chat_id = uuid_value(chat_id, "chat_id") + _require_chat_member(parsed_chat_id, actor["id"]) + limit = limit_arg(request.args.get("limit")) + cursor = decode_cursor(request.args.get("cursor")) + parameters: list[object] = [parsed_chat_id] + predicate = "" + if cursor is not None: + if cursor.get("kind") != "messages" or cursor.get("chat_id") != str(parsed_chat_id): + raise bad_request("Некорректный cursor") + try: + cursor_created = cursor_time(cursor.get("created_at")) + cursor_id = uuid_value(cursor.get("id"), "cursor") + except ApiError: + raise bad_request("Некорректный cursor") + predicate = "AND (created_at, id) > (%s, %s)" + parameters.extend([cursor_created, cursor_id]) + parameters.append(limit + 1) + query = f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + ARRAY( + SELECT ma.attachment_id + FROM message_attachments ma + WHERE ma.message_id = m.id + ORDER BY ma.position + ) AS attachment_ids + FROM messages m + WHERE m.chat_id = %s {predicate} + ORDER BY m.created_at, m.id + LIMIT %s + """ + with db.pool().connection() as conn: + rows = conn.execute(query, tuple(parameters)).fetchall() + page = rows[:limit] + next_cursor = None + if len(rows) > limit and page: + last = page[-1] + next_cursor = encode_cursor( + { + "v": 1, + "kind": "messages", + "chat_id": str(parsed_chat_id), + "created_at": format_time(last["created_at"]), + "id": str(last["id"]), + } + ) + return _json({"items": [message_json(row) for row in page], "next_cursor": next_cursor}) + + +# --------------------------------------------------------------------------- +# Attachments (lab4) +# --------------------------------------------------------------------------- + +def _attachment_for_member(attachment_id: uuid.UUID, actor_id: uuid.UUID) -> dict: + with db.pool().connection() as conn: + row = conn.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members cm + ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (actor_id, attachment_id), + ).fetchone() + if row is None: + raise not_found() + return row + + +def _attachment_owned(attachment_id: uuid.UUID, actor_id: uuid.UUID) -> dict | None: + with db.pool().connection() as conn: + return conn.execute( + """ + SELECT a.* + FROM attachments a + JOIN chat_members cm + ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s AND a.owner_id = %s + """, + (actor_id, attachment_id, actor_id), + ).fetchone() + + +@app.post("/api/v1/chats//attachments") +def upload_attachment(chat_id: str) -> Response: + """Synchronous proxy upload: 201 only after S3 PUT and PostgreSQL COMMIT.""" + actor = current_user() + parsed_chat_id = uuid_value(chat_id, "chat_id") + _require_chat_member(parsed_chat_id, actor["id"]) + name = filename(request.args.get("filename")) + declared = image_content_type(request.mimetype) + if request.content_length is not None and request.content_length > settings.max_upload_bytes: + raise payload_too_large("Файл больше 10 MiB") + + path = None + try: + path, size, digest = attachment_store.read_to_temp(request.stream) + attachment_store.validate_image(path, declared) + + attachment_id = uuid.uuid4() + key = attachment_store.object_key(parsed_chat_id, attachment_id) + with open(path, "rb") as body: + s3.put_file(key, body, declared, size, digest) + try: + with db.pool().connection() as conn: + row = conn.execute( + """ + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key, status) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'ready') + RETURNING * + """, + ( + attachment_id, + parsed_chat_id, + actor["id"], + name, + declared, + size, + digest, + key, + ), + ).fetchone() + except Exception as error: + # Compensate: an S3 object without metadata must not survive. + s3.delete(key) + if isinstance(error, psycopg.Error): + raise unavailable("Не удалось сохранить метаданные вложения") from error + raise + return _json(attachment_store.attachment_json(row), 201) + finally: + attachment_store.cleanup(path) + + +@app.post("/api/v1/chats//attachments/presign") +def presign_attachment(chat_id: str) -> Response: + """Extension: short-lived direct upload URL; the proxy endpoint stays mandatory.""" + actor = current_user() + parsed_chat_id = uuid_value(chat_id, "chat_id") + _require_chat_member(parsed_chat_id, actor["id"]) + body = json_object(request, settings.max_body_bytes) + exact_keys(body, ("filename", "content_type", "size_bytes", "sha256")) + name = filename(body["filename"]) + declared = image_content_type(body["content_type"]) + size = byte_size(body["size_bytes"], settings.max_upload_bytes) + digest = sha256_hex(body["sha256"]) + + attachment_id = uuid.uuid4() + key = attachment_store.object_key(parsed_chat_id, attachment_id) + upload_url = s3.presign_put(key, declared, digest, settings.presign_ttl_seconds) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.presign_ttl_seconds) + try: + with db.pool().connection() as conn: + conn.execute( + """ + INSERT INTO attachments + (id, chat_id, owner_id, filename, content_type, + size_bytes, sha256, object_key, status, presign_expires_at) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'pending', %s) + """, + ( + attachment_id, + parsed_chat_id, + actor["id"], + name, + declared, + size, + digest, + key, + expires_at, + ), + ) + except psycopg.Error as error: + raise unavailable("Не удалось сохранить метаданные вложения") from error + return _json( + { + "attachment_id": str(attachment_id), + "upload_url": upload_url, + "method": "PUT", + "headers": {"Content-Type": declared, "x-amz-meta-sha256": digest}, + "expires_at": format_time(expires_at), + }, + 201, + ) + + +@app.post("/api/v1/attachments//complete") +def complete_attachment(attachment_id: str) -> Response: + """Extension: verify a direct upload and make the attachment ready.""" + actor = current_user() + parsed_id = uuid_value(attachment_id, "attachment_id") + row = _attachment_owned(parsed_id, actor["id"]) + if row is None: + raise not_found() + body = json_object(request, settings.max_body_bytes) + exact_keys(body, ("sha256", "size_bytes")) + digest = sha256_hex(body["sha256"]) + size = byte_size(body["size_bytes"], settings.max_upload_bytes) + + if row["status"] == "ready": + if row["sha256"] == digest and int(row["size_bytes"]) == size: + return _json(attachment_store.attachment_json(row)) + raise conflict("Вложение уже завершено с другими параметрами") + if row["status"] != "pending": + raise conflict("Вложение не может быть завершено") + + expires_at = row.get("presign_expires_at") + if expires_at is not None and expires_at <= datetime.now(timezone.utc): + raise conflict("Срок действия загрузки истёк") + + head = s3.head(row["object_key"]) + if head is None: + raise conflict("Объект ещё не загружен") + if int(head.get("ContentLength", -1)) != size: + raise bad_request("Размер загруженного объекта не совпадает") + actual_size, actual_digest = s3.stream_sha256(row["object_key"]) + if actual_size != size or actual_digest != digest: + # Wrong bytes are never promoted; delete them so a retry starts clean. + s3.delete(row["object_key"]) + raise bad_request("Контрольная сумма загруженного объекта не совпадает") + + with db.pool().connection() as conn: + updated = conn.execute( + """ + UPDATE attachments + SET status = 'ready', error_code = NULL, updated_at = clock_timestamp() + WHERE id = %s AND status = 'pending' + RETURNING * + """, + (parsed_id,), + ).fetchone() + if updated is None: + raise conflict("Вложение уже обработано") + return _json(attachment_store.attachment_json(updated)) + + +@app.get("/api/v1/attachments/") +def get_attachment(attachment_id: str) -> Response: + actor = current_user() + parsed_id = uuid_value(attachment_id, "attachment_id") + row = _attachment_for_member(parsed_id, actor["id"]) + return _json(attachment_store.attachment_json(row)) + + +@app.get("/api/v1/attachments//content") +def download_attachment(attachment_id: str) -> Response: + actor = current_user() + parsed_id = uuid_value(attachment_id, "attachment_id") + variant = request.args.get("variant", "original") + if variant != "original": + raise bad_request("variant должен быть original") + row = _attachment_for_member(parsed_id, actor["id"]) + if row["status"] != "ready": + raise conflict("Вложение ещё не готово") + try: + body = s3.open_stream(row["object_key"]) + except s3.S3Missing as error: + raise unavailable("Оригинал вложения недоступен") from error + + def chunks(): + try: + for chunk in body.iter_chunks(chunk_size=64 * 1024): + yield chunk + finally: + body.close() + + response = Response(chunks(), mimetype=row["content_type"]) + response.headers["Content-Length"] = str(row["size_bytes"]) + response.headers["Content-Disposition"] = ( + "inline; filename*=UTF-8''" + quote(row["filename"], safe="") + ) + return response diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..1ab1f82d --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,37 @@ +"""One-shot schema migrator. + +Run as ``python -m app.migrate`` from the dedicated Compose service. It waits +for PostgreSQL, takes the application advisory lock and applies only missing +migration files. Exiting non-zero blocks API replicas via +``depends_on: service_completed_successfully``. +""" +from __future__ import annotations + +import logging +import sys + +from . import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger.migrate") + + +def main() -> int: + log.info("waiting for PostgreSQL and applying migrations") + db.open_pool(timeout=60.0) + applied = db.apply_migrations() + if applied: + log.info("applied migrations: %s", ", ".join(applied)) + else: + log.info("schema already current at %s", db.required_version()) + with db.pool().connection() as conn: + rows = conn.execute( + "SELECT version, applied_at FROM schema_migrations ORDER BY version" + ).fetchall() + for row in rows: + log.info("schema version %s applied at %s", row["version"], row["applied_at"].isoformat()) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..ab2d695e --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,136 @@ +"""Safe, repeatable S3/PostgreSQL reconciler (lab4 operational backstop). + +What it does +------------ +* finds S3 objects with no attachment row ("orphans") and deletes only those + older than a grace period; +* finds pending presigned uploads older than a grace period and removes both + the row and any object, so an interrupted direct upload cannot leak forever; +* reports ready rows whose object is missing and records ``missing_since`` for + operators without deleting user-visible metadata. + +``--dry-run`` is the default. ``--apply`` performs the deletions. Re-running +after an interruption is safe: every action is idempotent. +""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +from datetime import datetime, timezone + +from . import db, s3 +from .config import settings + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger.reconcile") + + +def _age_seconds(now: datetime, value: datetime | None) -> float | None: + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return (now - value.astimezone(timezone.utc)).total_seconds() + + +def run(*, apply: bool, orphan_grace: int, pending_grace: int, prefix: str) -> dict: + now = datetime.now(timezone.utc) + db.open_pool(timeout=60.0) + objects = s3.list_objects(prefix) + object_keys = {item["key"] for item in objects} + + with db.pool().connection() as conn: + rows = conn.execute( + "SELECT id, object_key, status, created_at, missing_since FROM attachments" + ).fetchall() + + db_keys = {row["object_key"] for row in rows} + orphans = [item for item in objects if item["key"] not in db_keys] + stale_orphans = [] + fresh_orphans = [] + for item in orphans: + age = _age_seconds(now, item["last_modified"]) + if age is None or age >= orphan_grace: + stale_orphans.append(item) + else: + fresh_orphans.append(item) + + stale_pending = [ + row + for row in rows + if row["status"] == "pending" + and (_age_seconds(now, row["created_at"]) or 0) >= pending_grace + ] + missing = [ + row + for row in rows + if row["status"] == "ready" and row["object_key"] not in object_keys + ] + + deleted_orphans = 0 + deleted_pending = 0 + marked_missing = 0 + if apply: + for item in stale_orphans: + s3.delete(item["key"]) + deleted_orphans += 1 + for row in stale_pending: + s3.delete(row["object_key"]) + with db.pool().connection() as conn: + conn.execute( + "DELETE FROM attachments WHERE id = %s AND status = 'pending'", + (row["id"],), + ) + deleted_pending += 1 + for row in missing: + if row["missing_since"] is None: + with db.pool().connection() as conn: + conn.execute( + "UPDATE attachments SET missing_since = clock_timestamp() WHERE id = %s", + (row["id"],), + ) + marked_missing += 1 + + return { + "apply": apply, + "grace_seconds": {"orphan": orphan_grace, "pending": pending_grace}, + "objects": len(objects), + "metadata_rows": len(rows), + "orphans_total": len(orphans), + "orphans_fresh_kept": len(fresh_orphans), + "orphans_stale_deleted": deleted_orphans if apply else len(stale_orphans), + "stale_pending_deleted": deleted_pending if apply else len(stale_pending), + "missing_objects_reported": len(missing), + "missing_objects_marked": marked_missing, + "sample_orphan_keys": [item["key"] for item in stale_orphans[:5]], + "sample_missing_keys": [row["object_key"] for row in missing[:5]], + } + + +def main() -> int: + parser = argparse.ArgumentParser(description="Reconcile private S3 objects with attachment metadata") + parser.add_argument("--apply", action="store_true", help="delete stale orphans/pending rows (default: dry-run)") + parser.add_argument("--dry-run", action="store_true", help="explicit dry-run (default)") + parser.add_argument("--orphan-grace-seconds", type=int, default=settings.orphan_grace_seconds) + parser.add_argument("--pending-grace-seconds", type=int, default=settings.presign_grace_seconds) + parser.add_argument("--prefix", default="") + args = parser.parse_args() + apply = args.apply and not args.dry_run + try: + summary = run( + apply=apply, + orphan_grace=max(0, args.orphan_grace_seconds), + pending_grace=max(0, args.pending_grace_seconds), + prefix=args.prefix, + ) + except s3.S3Unavailable as error: + log.error("object store is unavailable: %s", type(error).__name__) + return 2 + print(json.dumps(summary, ensure_ascii=False, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 00000000..1e70bd8b --- /dev/null +++ b/app/s3.py @@ -0,0 +1,230 @@ +"""Thin, bounded S3 client wrapper for the attachment store. + +The API identity has only Read/Write/List on the private bucket. The +administrative identity is used exclusively by ``app.s3_bootstrap``. + +All boto3 calls are wrapped so that HTTP handlers never leak stack traces or +credentials. Large bodies are streamed; no method here reads a whole object +into memory. +""" +from __future__ import annotations + +import hashlib +import threading +from datetime import datetime, timezone + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import settings + + +class S3Unavailable(Exception): + """The object store is unreachable or rejected a request.""" + + +class S3Missing(Exception): + """The requested object does not exist.""" + + +_clients: dict[bool, object] = {} +_clients_lock = threading.Lock() + + +def _make_client(admin: bool = False): + if admin: + access_key = _required_admin("S3_ADMIN_ACCESS_KEY_ID", "S3_ADMIN_ACCESS_KEY_ID_FILE") + secret_key = _required_admin("S3_ADMIN_SECRET_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY_FILE") + else: + access_key = settings.s3_access_key_id + secret_key = settings.s3_secret_access_key + config = Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=settings.s3_connect_timeout, + read_timeout=settings.s3_read_timeout, + retries={"max_attempts": 2, "mode": "standard"}, + max_pool_connections=32, + ) + return boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + region_name=settings.s3_region, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=config, + ) + + +def _required_admin(name: str, fallback: str) -> str: + import os + from pathlib import Path + + raw = os.environ.get(name) + if raw: + return raw + filename = os.environ.get(fallback) + if filename: + return Path(filename).read_text(encoding="utf-8").strip() + raise RuntimeError(f"missing required configuration: {name}") + + +def client(admin: bool = False): + with _clients_lock: + cached = _clients.get(admin) + if cached is None: + cached = _make_client(admin) + _clients[admin] = cached + return cached + + +def _error_code(error: ClientError) -> str: + return str(error.response.get("Error", {}).get("Code", "")) + + +def _missing(error: ClientError) -> bool: + return _error_code(error) in ("404", "NoSuchKey", "NoSuchBucket", "NotFound") + + +def ensure_bucket() -> None: + """Idempotent private-bucket bootstrap using the administrative identity.""" + s3 = client(admin=True) + try: + s3.head_bucket(Bucket=settings.s3_bucket) + return + except ClientError as error: + if _error_code(error) not in ("404", "NoSuchBucket", "NotFound"): + raise S3Unavailable("S3 head_bucket failed") from error + try: + s3.create_bucket(Bucket=settings.s3_bucket) + except ClientError as error: + code = _error_code(error) + if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + raise S3Unavailable("S3 create_bucket failed") from error + try: + s3.head_bucket(Bucket=settings.s3_bucket) + except (BotoCoreError, ClientError) as error: + raise S3Unavailable("bucket is not usable after bootstrap") from error + + +def check_ready() -> bool: + try: + client().head_bucket(Bucket=settings.s3_bucket) + return True + except Exception: + return False + + +def put_file(key: str, fileobj, content_type: str, size: int, sha256: str) -> None: + try: + client().put_object( + Bucket=settings.s3_bucket, + Key=key, + Body=fileobj, + ContentType=content_type, + ContentLength=size, + Metadata={"sha256": sha256}, + ) + except (BotoCoreError, ClientError) as error: + raise S3Unavailable("S3 upload failed") from error + + +def delete(key: str) -> None: + try: + client().delete_object(Bucket=settings.s3_bucket, Key=key) + except (BotoCoreError, ClientError): + # Compensation is best effort; the reconciler is the backstop. + return + + +def head(key: str) -> dict | None: + try: + return client().head_object(Bucket=settings.s3_bucket, Key=key) + except ClientError as error: + if _missing(error): + return None + raise S3Unavailable("S3 head_object failed") from error + except BotoCoreError as error: + raise S3Unavailable("S3 head_object failed") from error + + +def open_stream(key: str): + """Return a streaming body. The caller must close it.""" + try: + return client().get_object(Bucket=settings.s3_bucket, Key=key)["Body"] + except ClientError as error: + if _missing(error): + raise S3Missing("object is missing") from error + raise S3Unavailable("S3 get_object failed") from error + except BotoCoreError as error: + raise S3Unavailable("S3 get_object failed") from error + + +def stream_sha256(key: str) -> tuple[int, str]: + """Read an object once, bounded by 64 KiB chunks, and return size+digest.""" + body = open_stream(key) + digest = hashlib.sha256() + total = 0 + try: + for chunk in body.iter_chunks(chunk_size=64 * 1024): + total += len(chunk) + digest.update(chunk) + except (BotoCoreError, ClientError, OSError) as error: + raise S3Unavailable("S3 streaming read failed") from error + finally: + body.close() + return total, digest.hexdigest() + + +def presign_put(key: str, content_type: str, sha256: str, expires: int) -> str: + """Create a short-lived presigned PUT URL on the public S3 endpoint. + + The API's internal endpoint is not reachable from user clients, so the + presigner uses S3_PUBLIC_ENDPOINT. The bucket stays private: the URL is + the only grant and it expires quickly. + """ + presigner = boto3.client( + "s3", + endpoint_url=settings.s3_public_endpoint, + region_name=settings.s3_region, + aws_access_key_id=settings.s3_access_key_id, + aws_secret_access_key=settings.s3_secret_access_key, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + try: + return presigner.generate_presigned_url( + "put_object", + Params={ + "Bucket": settings.s3_bucket, + "Key": key, + "ContentType": content_type, + "Metadata": {"sha256": sha256}, + }, + ExpiresIn=expires, + ) + except (BotoCoreError, ClientError) as error: + raise S3Unavailable("could not sign upload URL") from error + + +def list_objects(prefix: str = "") -> list[dict]: + """List bucket objects with key, size and LastModified (UTC).""" + s3 = client() + result: list[dict] = [] + try: + paginator = s3.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=settings.s3_bucket, Prefix=prefix): + for item in page.get("Contents", []): + modified = item.get("LastModified") + if isinstance(modified, datetime) and modified.tzinfo is None: + modified = modified.replace(tzinfo=timezone.utc) + result.append( + { + "key": item["Key"], + "size": int(item.get("Size", 0)), + "last_modified": modified, + } + ) + except (BotoCoreError, ClientError) as error: + raise S3Unavailable("S3 list_objects failed") from error + return result diff --git a/app/s3_bootstrap.py b/app/s3_bootstrap.py new file mode 100644 index 00000000..6487d731 --- /dev/null +++ b/app/s3_bootstrap.py @@ -0,0 +1,39 @@ +"""One-shot, idempotent private-bucket bootstrap. + +Runs in the dedicated ``s3-init`` Compose service with the administrative S3 +identity. The API replicas start only after this service exits successfully, +and they use a different, restricted identity. +""" +from __future__ import annotations + +import logging +import sys +import time + +from . import s3 + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger.s3-bootstrap") + + +def main() -> int: + deadline = time.monotonic() + 90.0 + backoff = 0.5 + while True: + try: + s3.ensure_bucket() + if not s3.check_ready(): + raise RuntimeError("restricted identity cannot see the bucket") + log.info("private bucket is present and visible to the API identity") + return 0 + except Exception as error: + if time.monotonic() >= deadline: + log.error("bucket bootstrap failed: %s", type(error).__name__) + return 1 + log.warning("bucket bootstrap is not ready yet: %s", type(error).__name__) + time.sleep(backoff) + backoff = min(backoff * 2, 5.0) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..96363f6b --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,101 @@ +"""Opaque bearer sessions stored in Redis/Valkey with a TTL. + +PostgreSQL is the source of truth for users/chats/messages. Redis is the +authoritative store for *active sessions*; its append-only file and named +volume make those sessions survive a restart. The raw bearer token is never +stored: only a SHA-256 digest is used as the Redis key. +""" +from __future__ import annotations + +import hashlib +import json +import secrets +from datetime import datetime, timedelta, timezone + +import redis + +from .config import settings +from .errors import unavailable + +_client: redis.Redis | None = None + + +def client() -> redis.Redis: + global _client + if _client is None: + _client = redis.Redis.from_url( + settings.cache_url, + decode_responses=True, + socket_connect_timeout=settings.redis_connect_timeout, + socket_timeout=settings.redis_socket_timeout, + retry_on_timeout=False, + health_check_interval=30, + ) + return _client + + +def digest(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def _key(token_digest: str) -> str: + return "session:" + token_digest + + +def create(user_id: str) -> tuple[str, datetime]: + token = secrets.token_urlsafe(32) # 256 bits of entropy, opaque to clients + ttl = settings.session_ttl_seconds + expires_at = datetime.now(timezone.utc) + timedelta(seconds=ttl) + payload = json.dumps( + {"user_id": user_id, "expires_at": expires_at.isoformat().replace("+00:00", "Z")}, + separators=(",", ":"), + ) + try: + client().setex(_key(digest(token)), ttl, payload) + except redis.RedisError as exc: + raise unavailable("Хранилище сессий недоступно") from exc + return token, expires_at + + +def resolve(token: str) -> tuple[str, str]: + """Return (user_id, token_digest) or raise a bounded 503 error.""" + token_digest = digest(token) + try: + raw = client().get(_key(token_digest)) + except redis.RedisError as exc: + raise unavailable("Хранилище сессий недоступно") from exc + if not raw: + return "", token_digest + try: + data = json.loads(raw) + user_id = str(data["user_id"]) + expires_at = datetime.fromisoformat(str(data["expires_at"]).replace("Z", "+00:00")) + except (ValueError, KeyError, TypeError): + return "", token_digest + if expires_at <= datetime.now(timezone.utc): + revoke(token_digest) + return "", token_digest + return user_id, token_digest + + +def revoke(token_digest: str) -> None: + try: + client().delete(_key(token_digest)) + except redis.RedisError as exc: + raise unavailable("Хранилище сессий недоступно") from exc + + +def ping() -> None: + try: + if not client().ping(): + raise unavailable("Хранилище сессий недоступно") + except redis.RedisError as exc: + raise unavailable("Хранилище сессий недоступно") from exc + + +def seconds_left(token_digest: str) -> int: + try: + value = client().ttl(_key(token_digest)) + except redis.RedisError: + return 0 + return max(0, int(value)) diff --git a/app/validation.py b/app/validation.py new file mode 100644 index 00000000..f226ba45 --- /dev/null +++ b/app/validation.py @@ -0,0 +1,194 @@ +"""Strict parsing shared by HTTP handlers. + +The contract requires rejection of extra JSON fields, wrong types and +whitespace-only required strings. Validation happens before any SQL. +""" +from __future__ import annotations + +import base64 +import binascii +import json +import re +import uuid +from datetime import datetime, timezone + +from flask import Request + +from .errors import ApiError, bad_request + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +INTEGER_RE = re.compile(r"^[0-9]+$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +SHA256_RE = re.compile(r"^[a-f0-9]{64}$") + + +def json_object(request: Request, max_bytes: int) -> dict: + if request.mimetype != "application/json": + raise bad_request("Ожидается application/json") + raw = request.get_data(cache=False) + if not raw: + raise bad_request("Тело запроса обязательно") + if len(raw) > max_bytes: + raise ApiError(413, "payload_too_large", "Тело запроса слишком велико") + try: + value = json.loads(raw) + except (ValueError, UnicodeError) as exc: + raise bad_request("Некорректный JSON") from exc + if not isinstance(value, dict): + raise bad_request("Ожидается JSON-объект") + return value + + +def exact_keys(value: dict, required: tuple[str, ...], optional: tuple[str, ...] = ()) -> None: + allowed = set(required) | set(optional) + if not set(required).issubset(value) or set(value) - allowed: + raise bad_request("Неверный набор полей") + + +def text_field(value: object, name: str, minimum: int, maximum: int) -> str: + if not isinstance(value, str): + raise bad_request(f"Поле {name} должно быть строкой") + if not (minimum <= len(value) <= maximum): + raise bad_request(f"Поле {name} имеет недопустимую длину") + return value + + +def non_blank(value: object, name: str, minimum: int, maximum: int) -> str: + result = text_field(value, name, minimum, maximum) + if not result.strip(): + raise bad_request(f"Поле {name} не должно быть пустым") + return result + + +def username(value: object) -> str: + result = text_field(value, "username", 3, 32) + if USERNAME_RE.fullmatch(result) is None: + raise bad_request("Недопустимый username") + return result + + +def display_name(value: object) -> str: + return non_blank(value, "display_name", 1, 100) + + +def password(value: object) -> str: + result = text_field(value, "password", 12, 128) + return result + + +def message_text(value: object, allow_empty: bool = False) -> str: + if not isinstance(value, str): + raise bad_request("Поле text должно быть строкой") + if len(value) > 4000: + raise bad_request("Поле text имеет недопустимую длину") + if not allow_empty and not value.strip(): + raise bad_request("Поле text не должно быть пустым") + return value + + +def filename(value: object) -> str: + """Display-only filename; it never becomes an object key. + + The contract forbids path separators, NUL and control characters. The + limit is measured in Unicode code points (Python's len on str). + """ + if not isinstance(value, str): + raise bad_request("Параметр filename обязателен") + if not (1 <= len(value) <= 255): + raise bad_request("filename имеет недопустимую длину") + if value in (".", ".."): + raise bad_request("filename недопустим") + for char in value: + code = ord(char) + if char in "/\\" or code == 0 or code < 0x20 or code == 0x7F: + raise bad_request("filename содержит недопустимые символы") + return value + + +def image_content_type(value: object) -> str: + if not isinstance(value, str) or value not in ("image/png", "image/jpeg"): + raise ApiError(415, "unsupported_media_type", "Ожидается PNG или JPEG") + return value + + +def sha256_hex(value: object) -> str: + if not isinstance(value, str) or SHA256_RE.fullmatch(value) is None: + raise bad_request("sha256 должен быть lowercase hex SHA-256") + return value + + +def byte_size(value: object, maximum: int) -> int: + if not isinstance(value, int) or isinstance(value, bool) or not (1 <= value <= maximum): + raise bad_request("size_bytes вне допустимого диапазона") + return value + + +def idempotency_key(value: object) -> str: + if not isinstance(value, str) or IDEMPOTENCY_KEY_RE.fullmatch(value) is None: + raise bad_request("Некорректный Idempotency-Key") + return value + + +def uuid_list(value: object, name: str, minimum: int = 1, maximum: int = 100) -> list[str]: + if not isinstance(value, list): + raise bad_request(f"Поле {name} должно быть массивом") + if not (minimum <= len(value) <= maximum): + raise bad_request(f"Поле {name} имеет недопустимую длину") + parsed: list[str] = [] + for item in value: + parsed.append(str(uuid_value(item, name))) + if len(set(parsed)) != len(parsed): + raise bad_request(f"Поле {name} содержит дубликаты") + return parsed + + +def uuid_value(value: object, name: str = "id") -> uuid.UUID: + if not isinstance(value, str): + raise bad_request(f"Поле {name} должно быть UUID") + try: + return uuid.UUID(value) + except (ValueError, AttributeError) as exc: + raise bad_request(f"Поле {name} должно быть UUID") from exc + + +def limit_arg(raw: str | None, default: int = 50) -> int: + if raw is None: + return default + if INTEGER_RE.fullmatch(raw) is None: + raise bad_request("limit должен быть целым числом") + value = int(raw) + if not (1 <= value <= 100): + raise bad_request("limit должен быть в диапазоне 1..100") + return value + + +def encode_cursor(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def decode_cursor(raw: str | None) -> dict | None: + if raw is None: + return None + if not raw or len(raw) > 4096: + raise bad_request("Некорректный cursor") + padded = raw + "=" * (-len(raw) % 4) + try: + data = json.loads(base64.urlsafe_b64decode(padded.encode("ascii"))) + except (ValueError, UnicodeError, AttributeError, binascii.Error) as exc: + raise bad_request("Некорректный cursor") from exc + if not isinstance(data, dict): + raise bad_request("Некорректный cursor") + return data + + +def cursor_time(value: object) -> datetime: + if not isinstance(value, str): + raise bad_request("Некорректный cursor") + try: + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as exc: + raise bad_request("Некорректный cursor") from exc + if result.tzinfo is None: + raise bad_request("Некорректный cursor") + return result.astimezone(timezone.utc) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..7e2792e5 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,254 @@ +# Laboratory 4 stand: +# client -> nginx LB -> two or more API replicas -> PostgreSQL (metadata) +# -> private SeaweedFS S3 (bytes) +# +# Copy .env.example to .env and replace every change-me value before use. +# Only the LB and the S3 API are published; PostgreSQL, Redis, the SeaweedFS +# master/filer UIs and the API replicas stay on the internal network. +# +# The S3 port is published because the optional direct-upload extension gives +# clients a short-lived presigned PUT URL. The bucket itself is private: +# unsigned GET/PUT is rejected by SeaweedFS. + +name: mtusi-lab4 + +x-api-build: &api-build + image: mtusi-lab4-api:local + build: + context: . + dockerfile: Dockerfile + +x-api-environment: &api-environment + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env} + CACHE_URL: ${CACHE_URL:-redis://redis:6379/0} + INSTANCE_ID: ${INSTANCE_ID:-} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_ACQUIRE_TIMEOUT_SECONDS: ${DB_POOL_ACQUIRE_TIMEOUT_SECONDS:-3} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + REDIS_CONNECT_TIMEOUT_SECONDS: ${REDIS_CONNECT_TIMEOUT_SECONDS:-2} + REDIS_SOCKET_TIMEOUT_SECONDS: ${REDIS_SOCKET_TIMEOUT_SECONDS:-2} + MAX_BODY_BYTES: ${MAX_BODY_BYTES:-1048576} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + MAX_DECODE_CONCURRENCY: ${MAX_DECODE_CONCURRENCY:-1} + UPLOAD_TEMP_DIR: /tmp + S3_ENDPOINT: ${S3_ENDPOINT:-http://seaweedfs:8333} + S3_PUBLIC_ENDPOINT: ${S3_PUBLIC_ENDPOINT:-http://localhost:9000} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-3} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-30} + PRESIGN_TTL_SECONDS: ${PRESIGN_TTL_SECONDS:-300} + PRESIGN_GRACE_SECONDS: ${PRESIGN_GRACE_SECONDS:-3600} + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS:-86400} + +services: + postgres: + image: postgres:16.4-alpine + restart: unless-stopped + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?set POSTGRES_PASSWORD in .env} + command: + - postgres + - -c + - max_connections=120 + - -c + - shared_buffers=128MB + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 3s + timeout: 3s + retries: 20 + deploy: + resources: + limits: + cpus: "${POSTGRES_CPU_LIMIT:-1.0}" + memory: "${POSTGRES_MEM_LIMIT:-512M}" + networks: + - backend + + redis: + image: redis:7.4.0-alpine + restart: unless-stopped + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --dir + - /data + volumes: + - redisdata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 20 + deploy: + resources: + limits: + cpus: "${REDIS_CPU_LIMIT:-0.5}" + memory: "${REDIS_MEM_LIMIT:-256M}" + networks: + - backend + + seaweedfs: + image: chrislusf/seaweedfs:3.80 + restart: unless-stopped + entrypoint: ["/bin/sh", "/etc/seaweedfs/entrypoint.sh"] + environment: + S3_ADMIN_ACCESS_KEY_ID: ${S3_ADMIN_ACCESS_KEY_ID:?set S3_ADMIN_ACCESS_KEY_ID in .env} + S3_ADMIN_SECRET_ACCESS_KEY: ${S3_ADMIN_SECRET_ACCESS_KEY:?set S3_ADMIN_SECRET_ACCESS_KEY in .env} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + volumes: + - seaweeddata:/data + - ./infra/seaweedfs/entrypoint.sh:/etc/seaweedfs/entrypoint.sh:ro + ports: + - "${S3_PORT:-9000}:8333" + healthcheck: + test: ["CMD-SHELL", "nc -z 127.0.0.1 8333"] + interval: 3s + timeout: 3s + retries: 40 + start_period: 20s + deploy: + resources: + limits: + cpus: "${S3_CPU_LIMIT:-1.0}" + memory: "${S3_MEM_LIMIT:-768M}" + networks: + - backend + + # One-shot, serialised schema migration. API replicas never run DDL. + migrate: + <<: *api-build + restart: "no" + environment: + DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env} + CACHE_URL: ${CACHE_URL:-redis://redis:6379/0} + DB_CONNECT_TIMEOUT_SECONDS: ${DB_CONNECT_TIMEOUT_SECONDS:-3} + S3_ENDPOINT: ${S3_ENDPOINT:-http://seaweedfs:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + command: ["python", "-m", "app.migrate"] + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: "256M" + networks: + - backend + + # Idempotent private-bucket bootstrap with the administrative identity. + s3-init: + <<: *api-build + restart: "no" + environment: + DATABASE_URL: ${DATABASE_URL:?set DATABASE_URL in .env} + CACHE_URL: ${CACHE_URL:-redis://redis:6379/0} + S3_ENDPOINT: ${S3_ENDPOINT:-http://seaweedfs:8333} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID:?set S3_ACCESS_KEY_ID in .env} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY:?set S3_SECRET_ACCESS_KEY in .env} + S3_ADMIN_ACCESS_KEY_ID: ${S3_ADMIN_ACCESS_KEY_ID:?set S3_ADMIN_ACCESS_KEY_ID in .env} + S3_ADMIN_SECRET_ACCESS_KEY: ${S3_ADMIN_SECRET_ACCESS_KEY:?set S3_ADMIN_SECRET_ACCESS_KEY in .env} + command: ["python", "-m", "app.s3_bootstrap"] + depends_on: + seaweedfs: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: "256M" + networks: + - backend + + api: + <<: *api-build + restart: unless-stopped + environment: *api-environment + expose: + - "8080" + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + redis: + condition: service_healthy + stop_grace_period: 15s + security_opt: + - no-new-privileges:true + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_LIMIT:-1.0}" + memory: "${API_MEM_LIMIT:-512M}" + networks: + - backend + + lb: + image: nginx:1.27.5-alpine + restart: unless-stopped + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "${API_PORT:-8080}:8080" + healthcheck: + test: ["CMD-SHELL", "wget -q -O- http://127.0.0.1:8080/health/live >/dev/null"] + interval: 5s + timeout: 3s + retries: 10 + deploy: + resources: + limits: + cpus: "${LB_CPU_LIMIT:-0.5}" + memory: "${LB_MEM_LIMIT:-256M}" + networks: + - backend + + # Optional operator tool: docker compose --profile tools run --rm reconcile \ + # python -m app.reconcile --apply + reconcile: + <<: *api-build + profiles: ["tools"] + restart: "no" + environment: *api-environment + command: ["python", "-m", "app.reconcile", "--dry-run"] + depends_on: + migrate: + condition: service_completed_successfully + seaweedfs: + condition: service_healthy + networks: + - backend + +volumes: + pgdata: + redisdata: + seaweeddata: + +networks: + backend: + driver: bridge diff --git a/contracts/extensions.md b/contracts/extensions.md new file mode 100644 index 00000000..8d27ef9c --- /dev/null +++ b/contracts/extensions.md @@ -0,0 +1,37 @@ +# Lab4 extensions: presigned direct upload (track 1) + +The mandatory contract in `openapi.json` is unchanged. The proxy endpoint +`POST /api/v1/chats/{chat_id}/attachments` remains the required path and is the +fallback for clients that cannot reach S3. This extension adds an optional +direct path: + +1. `POST /api/v1/chats/{chat_id}/attachments/presign` with + `{filename, content_type, size_bytes, sha256}` creates a `pending` + attachment row and returns a short-lived presigned `PUT` URL. +2. The client uploads the bytes directly to the private bucket using the URL + and the returned `Content-Type` / `x-amz-meta-sha256` headers. +3. `POST /api/v1/attachments/{attachment_id}/complete` with + `{sha256, size_bytes}` verifies ownership, the stored object size and the + SHA-256 of the bytes. Only then does the attachment become `ready`. + +## Rules + +- The bucket stays private. The URL is the only grant, expires after + `PRESIGN_TTL_SECONDS` (default 300 s), and uses a unique server-generated + object key. +- Complete is owner-only and chat-member-only. A different user receives 404. +- Complete is idempotent for the same `(sha256, size_bytes)` pair. A different + pair on a ready attachment is `409`. +- Completing before the object exists, after the URL expired, or after the + pending row was cleaned up is `409`. +- A wrong size or checksum is `400`; the wrong object is deleted and is never + promoted to `ready`. +- `GET /api/v1/attachments/{id}` may return `pending` for this extension. + The mandatory proxy path always returns `ready`. +- Incomplete uploads are removed by `python -m app.reconcile --apply` after + `PRESIGN_GRACE_SECONDS` (default 3600 s). Orphan objects that never got a + metadata row are removed after `ORPHAN_GRACE_SECONDS` (default 86400 s). +- The complete call reads the object once through the API to verify its + checksum. That is a deliberate integrity cost: direct upload avoids API + ingress, but the API still verifies egress. A future S3 checksum API could + remove this read; SeaweedFS 3.80 does not provide the same portable hook. diff --git a/contracts/extensions.openapi.json b/contracts/extensions.openapi.json new file mode 100644 index 00000000..f9f41b78 --- /dev/null +++ b/contracts/extensions.openapi.json @@ -0,0 +1,158 @@ +{ + "openapi": "3.1.0", + "info": { + "title": "Messenger lab4 extensions: presigned direct upload", + "version": "1.4.0-extension.1", + "description": "Backward-compatible extension. The mandatory proxy upload in openapi.json remains supported and is the fallback." + }, + "paths": { + "/api/v1/chats/{chat_id}/attachments/presign": { + "post": { + "operationId": "presignAttachmentUpload", + "summary": "Create a short-lived presigned PUT URL for a private-bucket object", + "parameters": [ + { + "name": "chat_id", + "in": "path", + "required": true, + "schema": {"type": "string", "format": "uuid"} + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/PresignRequest"} + } + } + }, + "responses": { + "201": { + "description": "URL created; the attachment is pending until complete", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/PresignResponse"} + } + } + }, + "400": {"$ref": "#/components/responses/Error"}, + "401": {"$ref": "#/components/responses/Error"}, + "404": {"$ref": "#/components/responses/Error"}, + "413": {"$ref": "#/components/responses/Error"}, + "415": {"$ref": "#/components/responses/Error"}, + "503": {"$ref": "#/components/responses/Error"} + } + } + }, + "/api/v1/attachments/{attachment_id}/complete": { + "post": { + "operationId": "completeAttachmentUpload", + "summary": "Verify size and SHA-256 of a direct upload and make it ready", + "parameters": [ + { + "name": "attachment_id", + "in": "path", + "required": true, + "schema": {"type": "string", "format": "uuid"} + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/CompleteRequest"} + } + } + }, + "responses": { + "200": { + "description": "Ready attachment, or the same ready attachment on an idempotent retry", + "content": { + "application/json": { + "schema": {"$ref": "#/components/schemas/AttachmentExtension"} + } + } + }, + "400": {"$ref": "#/components/responses/Error"}, + "401": {"$ref": "#/components/responses/Error"}, + "404": {"$ref": "#/components/responses/Error"}, + "409": {"$ref": "#/components/responses/Error"}, + "503": {"$ref": "#/components/responses/Error"} + } + } + } + }, + "components": { + "responses": { + "Error": { + "description": "Typed error", + "content": { + "application/json": { + "schema": {"$ref": "../openapi.json#/components/schemas/Error"} + } + } + } + }, + "schemas": { + "PresignRequest": { + "type": "object", + "properties": { + "filename": {"type": "string", "minLength": 1, "maxLength": 255}, + "content_type": {"type": "string", "enum": ["image/png", "image/jpeg"]}, + "size_bytes": {"type": "integer", "minimum": 1, "maximum": 10485760}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + }, + "required": ["filename", "content_type", "size_bytes", "sha256"], + "additionalProperties": false + }, + "PresignResponse": { + "type": "object", + "properties": { + "attachment_id": {"type": "string", "format": "uuid"}, + "upload_url": {"type": "string", "format": "uri"}, + "method": {"type": "string", "const": "PUT"}, + "headers": { + "type": "object", + "properties": { + "Content-Type": {"type": "string", "enum": ["image/png", "image/jpeg"]}, + "x-amz-meta-sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"} + }, + "required": ["Content-Type", "x-amz-meta-sha256"], + "additionalProperties": false + }, + "expires_at": {"type": "string", "format": "date-time"} + }, + "required": ["attachment_id", "upload_url", "method", "headers", "expires_at"], + "additionalProperties": false + }, + "CompleteRequest": { + "type": "object", + "properties": { + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "size_bytes": {"type": "integer", "minimum": 1, "maximum": 10485760} + }, + "required": ["sha256", "size_bytes"], + "additionalProperties": false + }, + "AttachmentExtension": { + "type": "object", + "properties": { + "id": {"type": "string", "format": "uuid"}, + "chat_id": {"type": "string", "format": "uuid"}, + "owner_id": {"type": "string", "format": "uuid"}, + "filename": {"type": "string", "minLength": 1, "maxLength": 255}, + "content_type": {"type": "string", "enum": ["image/png", "image/jpeg"]}, + "size_bytes": {"type": "integer", "minimum": 1, "maximum": 10485760}, + "sha256": {"type": "string", "pattern": "^[a-f0-9]{64}$"}, + "status": {"type": "string", "enum": ["pending", "ready", "failed"]}, + "variants": {"type": "array", "items": {"type": "object"}}, + "error_code": {"type": ["string", "null"]}, + "created_at": {"type": "string", "format": "date-time"}, + "updated_at": {"type": "string", "format": "date-time"} + }, + "required": ["id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", "sha256", "status", "variants", "error_code", "created_at", "updated_at"], + "additionalProperties": false + } + } + } +} diff --git a/evidence/README.md b/evidence/README.md index 8df70015..0d8c8e25 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -1,5 +1,25 @@ # Доказательства -Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. +Небольшие обезличенные результаты запусков на локальном учебном стенде +(Windows + Docker Desktop, Linux containers; API limit 512 MiB, SeaweedFS +limit 768 MiB). Дата: 2026-11-07. Секреты, токены, cookies, signed URLs и +raw dumps в файлы не включались. -Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. +| Файл | Что доказывает | +| --- | --- | +| `check.txt` | offline-проверка структуры и синтаксиса | +| `smoke.txt` | публичный smoke lab4 | +| `attachments-negative.txt` | лимиты, MIME, повреждение, filename, чужой id/чат | +| `upload-memory.json`, `upload-memory.txt` | 10 параллельных загрузок и пик RAM | +| `transfer-compare.json`, `transfer-compare.txt` | proxy vs presigned direct: трафик/RAM на двух размерах | +| `presigned-flow.txt` | presigned PUT, complete, checksum, owner-only, negative | +| `private-bucket.txt` | private bucket, ограниченные credentials, закрытые admin UI | +| `s3-init-idempotent.txt` | повторный запуск bootstrap видит существующий bucket | +| `reconcile.txt` | dry-run/apply, grace period, orphan и stale pending | +| `persistence.txt` | restart с volumes сохраняет данные и сессию | +| `failover.txt` | остановка одной реплики, восстановление и сохранность | +| `instance-spread.txt`, `idempotency-race.txt` | регрессия lab3: ≥2 реплики, общий Idempotency-Key | +| `load.json` | closed-loop нагрузка 30 s | +| `compose-ps.txt` | состав и healthcheck'и стенда | + +Команды запуска и интерпретация — в `REPORT.md` и `student-tests/README.md`. diff --git a/evidence/attachments-negative.txt b/evidence/attachments-negative.txt new file mode 100644 index 00000000..93088590 Binary files /dev/null and b/evidence/attachments-negative.txt differ diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..80f5d361 Binary files /dev/null and b/evidence/check.txt differ diff --git a/evidence/compose-ps.txt b/evidence/compose-ps.txt new file mode 100644 index 00000000..d28cbd12 Binary files /dev/null and b/evidence/compose-ps.txt differ diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..9c68ce25 Binary files /dev/null and b/evidence/failover.txt differ diff --git a/evidence/idempotency-race.txt b/evidence/idempotency-race.txt new file mode 100644 index 00000000..621a1343 Binary files /dev/null and b/evidence/idempotency-race.txt differ diff --git a/evidence/instance-spread.txt b/evidence/instance-spread.txt new file mode 100644 index 00000000..e3dc6e9d Binary files /dev/null and b/evidence/instance-spread.txt differ diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..c94d3f4b --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.069, + "concurrency": 4, + "successful_requests": 6789, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 225.781, + "successful_latency_ms": { + "p50": 14.836, + "p95": 33.787, + "p99": 66.655 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..c25cebac Binary files /dev/null and b/evidence/persistence.txt differ diff --git a/evidence/presigned-flow.txt b/evidence/presigned-flow.txt new file mode 100644 index 00000000..0b83091f Binary files /dev/null and b/evidence/presigned-flow.txt differ diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 00000000..d3781c8f Binary files /dev/null and b/evidence/private-bucket.txt differ diff --git a/evidence/reconcile.txt b/evidence/reconcile.txt new file mode 100644 index 00000000..7d874b0d Binary files /dev/null and b/evidence/reconcile.txt differ diff --git a/evidence/s3-init-idempotent.txt b/evidence/s3-init-idempotent.txt new file mode 100644 index 00000000..1e86f8f9 Binary files /dev/null and b/evidence/s3-init-idempotent.txt differ diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..450bb652 Binary files /dev/null and b/evidence/smoke.txt differ diff --git a/evidence/transfer-compare.json b/evidence/transfer-compare.json new file mode 100644 index 00000000..b27f2949 --- /dev/null +++ b/evidence/transfer-compare.json @@ -0,0 +1,39 @@ +{ + "container_prefix": "mtusi-lab4-api", + "sizes": [ + { + "direct": { + "api_memory_growth_mib": 0.0, + "api_memory_peak_mib": 153.66, + "api_network_bytes": 564368, + "client_to_api_bytes": 0, + "client_to_s3_bytes": 552409, + "file_bytes": 552409 + }, + "proxy": { + "api_memory_growth_mib": 0.0, + "api_network_bytes": 1120316, + "client_to_api_bytes": 552409, + "client_to_s3_bytes": 0, + "file_bytes": 552409 + } + }, + { + "direct": { + "api_memory_growth_mib": 0.0, + "api_memory_peak_mib": 155.41, + "api_network_bytes": 4446591, + "client_to_api_bytes": 0, + "client_to_s3_bytes": 4424547, + "file_bytes": 4424547 + }, + "proxy": { + "api_memory_growth_mib": 1.4, + "api_network_bytes": 8888758, + "client_to_api_bytes": 4424547, + "client_to_s3_bytes": 0, + "file_bytes": 4424547 + } + } + ] +} \ No newline at end of file diff --git a/evidence/transfer-compare.txt b/evidence/transfer-compare.txt new file mode 100644 index 00000000..f32e29a1 Binary files /dev/null and b/evidence/transfer-compare.txt differ diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..037fd4e5 --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,18 @@ +{ + "baseline_mib": { + "mtusi-lab4-api-1": 83.17, + "mtusi-lab4-api-2": 85.38 + }, + "elapsed_seconds": 1.697, + "file_bytes": 4431995, + "peak_growth_mib": { + "mtusi-lab4-api-1": 22.53, + "mtusi-lab4-api-2": 22.52 + }, + "peak_mib": { + "mtusi-lab4-api-1": 105.7, + "mtusi-lab4-api-2": 107.9 + }, + "successful_uploads": 10, + "workers": 10 +} \ No newline at end of file diff --git a/evidence/upload-memory.txt b/evidence/upload-memory.txt new file mode 100644 index 00000000..b1fb55f5 Binary files /dev/null and b/evidence/upload-memory.txt differ diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..f6265152 --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,57 @@ +# Public entry point for laboratory 4. +# +# Docker's embedded DNS (127.0.0.11) returns every API replica IP for the +# Compose service name "api". The shared upstream zone + "resolve" picks up +# scale-up/scale-down and replacement containers without a reload. Passive +# checks remove a replica after failed connections; proxy_next_upstream lets a +# safe retry reach another backend. + +worker_processes auto; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + server_tokens off; + access_log /dev/stdout; + error_log /dev/stderr warn; + + resolver 127.0.0.11 valid=2s ipv6=off; + + upstream api_backend { + zone api_backend 64k; + server api:8080 resolve max_fails=3 fail_timeout=2s; + keepalive 32; + } + + server { + listen 8080; + # The API enforces 10 MiB; leave a little room for HTTP framing. + client_max_body_size 12m; + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 2s; + proxy_send_timeout 60s; + proxy_read_timeout 60s; + + # Do not retry non-idempotent requests transparently. The client + # repeats an uncertain POST with the same Idempotency-Key, which is + # the contract-level guarantee across replicas. Application 503 + # responses pass through unchanged. + proxy_next_upstream error timeout http_502 http_504; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 3s; + } + } +} diff --git a/infra/seaweedfs/entrypoint.sh b/infra/seaweedfs/entrypoint.sh new file mode 100755 index 00000000..dca37915 --- /dev/null +++ b/infra/seaweedfs/entrypoint.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# Generate the SeaweedFS S3 identity file from local Compose secrets, then run +# one master + volume + filer + S3 process. The file lives only in /tmp; the +# repository never contains credentials. +set -eu + +: "${S3_ADMIN_ACCESS_KEY_ID:?missing S3_ADMIN_ACCESS_KEY_ID}" +: "${S3_ADMIN_SECRET_ACCESS_KEY:?missing S3_ADMIN_SECRET_ACCESS_KEY}" +: "${S3_ACCESS_KEY_ID:?missing S3_ACCESS_KEY_ID}" +: "${S3_SECRET_ACCESS_KEY:?missing S3_SECRET_ACCESS_KEY}" + +case "$S3_ADMIN_ACCESS_KEY_ID$S3_ADMIN_SECRET_ACCESS_KEY$S3_ACCESS_KEY_ID$S3_SECRET_ACCESS_KEY" in + *[!A-Za-z0-9._-]*) + echo "S3 credentials must contain only A-Z a-z 0-9 . _ -" >&2 + exit 1 + ;; +esac + +umask 077 +cat > /tmp/s3.json < (?, ?) +-- ORDER BY created_at, id LIMIT ? +-- It supports the range scan without reading the whole chat history. +CREATE INDEX IF NOT EXISTS messages_chat_created_at_id_idx + ON messages (chat_id, created_at, id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..206dc6ca --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,26 @@ +-- Laboratory 3: shared idempotency for message creation. +-- Scope is (user_id, method, path, key); the stored result is the message id. +-- The row and the message are written in one transaction by the API. + +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method text NOT NULL, + path text NOT NULL, + key text NOT NULL, + request_hash text NOT NULL, + message_id uuid REFERENCES messages(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, method, path, key), + CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_method_post CHECK (method = 'POST') +); + +-- Expired rows are removed lazily; this index keeps that cleanup bounded. +CREATE INDEX IF NOT EXISTS idempotency_expires_at_idx + ON idempotency_keys (expires_at); + +-- The message id is unique per stored key, but the same message is never +-- shared across keys; a plain lookup by primary key is enough for replay. +CREATE INDEX IF NOT EXISTS idempotency_message_id_idx + ON idempotency_keys (message_id); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..1664219c --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,62 @@ +-- Laboratory 4: attachment metadata in PostgreSQL, bytes in a private S3 bucket. +-- +-- The API assigns the object key. The original filename is display metadata +-- only and is never used to build a key. A row is inserted only after the S3 +-- PUT succeeds; the proxy upload compensates a failed INSERT by deleting the +-- object, and the reconciler cleans up objects left by a process crash. + +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + filename text NOT NULL, + content_type text NOT NULL, + size_bytes bigint NOT NULL, + sha256 text NOT NULL, + object_key text NOT NULL, + status text NOT NULL DEFAULT 'ready', + error_code text, + presign_expires_at timestamptz, + missing_since timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + updated_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT attachments_object_key_key UNIQUE (object_key), + CONSTRAINT attachments_content_type_check + CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_check CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_check CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_status_check + CHECK (status IN ('pending', 'ready', 'failed')), + CONSTRAINT attachments_filename_check + CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_error_code_check + CHECK (error_code IS NULL OR error_code IN + ('invalid_image', 'processing_failed', 'retry_exhausted', 'missing_object')) +); + +CREATE INDEX IF NOT EXISTS attachments_chat_created_idx + ON attachments (chat_id, created_at, id); + +-- Reconciler access paths. +CREATE INDEX IF NOT EXISTS attachments_status_created_idx + ON attachments (status, created_at); +CREATE INDEX IF NOT EXISTS attachments_object_key_idx + ON attachments (object_key); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments(id) ON DELETE RESTRICT, + position integer NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_key UNIQUE (message_id, position), + CONSTRAINT message_attachments_position_check CHECK (position >= 0 AND position < 10) +); + +CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx + ON message_attachments (attachment_id); + +-- A message may carry an empty text only when it has at least one attachment. +-- The API enforces the semantic part; this constraint is the storage backstop. +ALTER TABLE messages DROP CONSTRAINT IF EXISTS messages_text_length; +ALTER TABLE messages + ADD CONSTRAINT messages_text_length CHECK (char_length(text) BETWEEN 0 AND 4000); diff --git a/migrations/004_defer_idempotency_fk.sql b/migrations/004_defer_idempotency_fk.sql new file mode 100644 index 00000000..1c262a57 --- /dev/null +++ b/migrations/004_defer_idempotency_fk.sql @@ -0,0 +1,10 @@ +-- The idempotency reservation is inserted before the message row inside one +-- transaction. A deferred foreign key lets both statements commit together +-- while still guaranteeing that a committed key always points at a message. +ALTER TABLE idempotency_keys + DROP CONSTRAINT IF EXISTS idempotency_keys_message_id_fkey; + +ALTER TABLE idempotency_keys + ADD CONSTRAINT idempotency_keys_message_id_fkey + FOREIGN KEY (message_id) REFERENCES messages(id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED; diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..dd0208b4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,5 @@ -# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. +# Full OpenAPI/JSON Schema validation and host-side student checks. openapi-spec-validator==0.7.2 +jsonschema==4.23.0 +Pillow==11.0.0 +boto3==1.35.76 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..20f9c74d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +Flask==3.1.0 +gunicorn==23.0.0 +psycopg[binary,pool]==3.2.3 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.35.76 +Pillow==11.0.0 diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..c48cee6f --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,10 @@ +@echo off +REM Restart API and ALL state stores without deleting named volumes. +setlocal +cd /d "%~dp0.." +if not exist .env ( + echo missing .env; copy .env.example and set local synthetic credentials>&2 + exit /b 2 +) +docker compose --env-file .env restart postgres redis seaweedfs api lb +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..965bf21b --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Restart API and ALL state stores without deleting named volumes. +# PostgreSQL and Redis keep their volumes; the public persistence test polls +# /health/ready itself, so this hook only performs the restart and returns +# non-zero on a Compose error. +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +if [ ! -f .env ]; then + echo "missing .env; copy .env.example and set local synthetic credentials" >&2 + exit 2 +fi +# Do not restart the one-shot migrate service here: the schema is already +# persisted in the PostgreSQL volume. Restart every long-running component. +docker compose --env-file .env restart postgres redis seaweedfs api lb diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..76d74042 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Equivalent of a rolling update for Docker Compose: replace one API replica +# at a time, keeping the service at the configured replica count. Run +# `docker compose build api` first when testing a new image. No volumes, +# database rows or sessions are touched. +set -eu +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +COMPOSE="docker compose --env-file .env" +COUNT=${API_REPLICAS:-2} + +wait_for_healthy() { + deadline=$(( $(date +%s) + 90 )) + while :; do + ids=$($COMPOSE ps -q api) + running=0 + healthy=0 + for id in $ids; do + state=$(docker inspect --format '{{.State.Status}}' "$id") + health=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$id") + if [ "$state" = "running" ]; then running=$((running + 1)); fi + if [ "$health" = "healthy" ] || [ "$health" = "none" ]; then healthy=$((healthy + 1)); fi + done + if [ "$running" -ge "$COUNT" ] && [ "$healthy" -ge "$COUNT" ]; then + return 0 + fi + if [ "$(date +%s)" -ge "$deadline" ]; then + echo "rolling-update: replicas did not become healthy" >&2 + return 1 + fi + sleep 1 + done +} + +wait_for_healthy +for id in $($COMPOSE ps -q api); do + echo "replacing $id" + docker stop --time 15 "$id" >/dev/null + docker rm "$id" >/dev/null + $COMPOSE up -d --no-deps --no-build --scale api="$COUNT" api + wait_for_healthy +done +echo "rolling-update: all $COUNT replicas replaced" diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..76f19b7e --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,8 @@ +# Stop exactly ONE API replica from a Windows host. +# The public failover test sets TARGET_INSTANCE_ID; the PowerShell helper maps +# the alias (container hostname) to the matching Compose container. +@echo off +setlocal +if "%TARGET_INSTANCE_ID%"=="" exit /b 2 +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.ps1 b/scripts/stop-one.ps1 new file mode 100644 index 00000000..e1c22f46 --- /dev/null +++ b/scripts/stop-one.ps1 @@ -0,0 +1,21 @@ +$ErrorActionPreference = 'Stop' +$root = Split-Path -Parent $PSScriptRoot +Set-Location $root + +$target = $env:TARGET_INSTANCE_ID +if ([string]::IsNullOrWhiteSpace($target)) { exit 2 } + +$ids = @(docker compose --env-file .env ps -q api) +foreach ($id in $ids) { + if ([string]::IsNullOrWhiteSpace($id)) { continue } + $hostName = (docker inspect --format '{{.Config.Hostname}}' $id).Trim() + $containerName = (docker inspect --format '{{.Name}}' $id).Trim().TrimStart('/') + if ($hostName -eq $target -or $hostName.StartsWith($target) -or + $target.StartsWith($hostName) -or $containerName -eq $target) { + docker stop --time 5 $id | Out-Null + exit 0 + } +} + +Write-Error 'stop-one: no API replica matches TARGET_INSTANCE_ID' +exit 3 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..fd458182 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Stop exactly ONE API replica identified by the public X-Instance-Id alias. +# The public failover test sets TARGET_INSTANCE_ID and expects this script to +# return within 10 seconds while leaving the replica stopped. It never touches +# PostgreSQL, Redis, the LB or named volumes. +set -eu + +: "${TARGET_INSTANCE_ID:?test provides the target API instance alias}" + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +ENV_FILE=${ENV_FILE:-.env} +COMPOSE="docker compose --env-file $ENV_FILE" + +target=$TARGET_INSTANCE_ID +found='' +for id in $($COMPOSE ps -q api); do + host=$(docker inspect --format '{{.Config.Hostname}}' "$id") + name=$(docker inspect --format '{{.Name}}' "$id") + case "$host" in + "$target"|"$target"*) found=$id; break ;; + esac + case "$target" in + "$host"|"$host"*) found=$id; break ;; + esac + # INSTANCE_ID may be set explicitly by an operator; then it is not the + # container hostname and the default mapping cannot be used. + if [ "$host" != "$name" ] && [ "${name#/}" = "$target" ]; then + found=$id + break + fi +done + +if [ -z "$found" ]; then + echo "stop-one: no API replica matches TARGET_INSTANCE_ID" >&2 + exit 3 +fi + +# -t 5 bounds the graceful drain; gunicorn exits earlier when idle. Docker +# does not restart a container stopped explicitly, even with restart policy. +docker stop --time 5 "$found" >/dev/null +exit 0 diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..9140aa32 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,68 @@ +# Студенческие проверки lab4 + +Все проверки, кроме `rollback_chat.py`, `explain_history.py` и `pool_settings.py`, +запускаются на хосте и обращаются к API через LB. Три проверки используют `psycopg` +и запускаются внутри одного контейнера API. Для новых проверок нужны `Pillow` и +`boto3` (см. `requirements-dev.txt`). + +```sh +# 1. Одна сессия обслуживается минимум двумя реальными репликами +python student-tests/instance_spread.py --base-url http://localhost:8080 + +# 2. Конкурентный повтор одного Idempotency-Key через разные реплики +python student-tests/idempotency_race.py --base-url http://localhost:8080 --workers 20 + +# 3. Негативная аутентификация и logout (перенесено из lab2) +python student-tests/auth_negative.py --base-url http://localhost:8080 + +# 4. Гонка регистрации одного username (перенесено из lab2) +python student-tests/race_username.py --base-url http://localhost:8080 --workers 8 + +# 5. Откат составной операции (внутри контейнера; --index обязателен при replicas>1) +docker compose exec -T --index 1 api python student-tests/rollback_chat.py + +# 6. EXPLAIN на 10 000 сообщений (внутри контейнера) +docker compose exec -T --index 1 api python student-tests/explain_history.py \ + --output /tmp/explain_history.txt + +# 7. Пул и SQL-таймауты (внутри контейнера) +docker compose exec -T --index 1 api python student-tests/pool_settings.py + +# Lab4: негативные сценарии вложений +python student-tests/attachments_negative.py --base-url http://localhost:8080 + +# Lab4: 10 одновременных загрузок и пик RAM API +python student-tests/upload_memory.py --base-url http://localhost:8080 \ + --output evidence/upload-memory.json + +# Lab4/5: presigned direct upload и обязательный proxy fallback +python student-tests/presigned_flow.py --base-url http://localhost:8080 + +# Lab5 track 1: сравнение трафика/RAM API proxy vs direct на двух размерах +python student-tests/transfer_compare.py --base-url http://localhost:8080 \ + --output evidence/transfer-compare.json + +# Lab4: приватный bucket и отсутствие опубликованных admin UI +python student-tests/private_bucket.py --base-url http://localhost:8080 + +# Lab4/5: reconciler, grace period, очистка incomplete upload +python student-tests/reconcile_orphans.py --base-url http://localhost:8080 +``` + +`instance_spread.py` проверяет не только заголовок, но и то, что запись, принятая +одной репликой, читается через другую. `idempotency_race.py` отправляет один ключ +параллельно, проверяет ровно один Message, 409 на другой body, 400 на отсутствующий +ключ и независимость scope по пользователю и чату. + +`attachments_negative.py` проверяет лимит 10 MiB, пустое/повреждённое изображение, +подмену MIME, лимит 40 млн пикселей, опасные имена файлов, чужой attachment_id и +вложение из другого чата. `upload_memory.py` измеряет пик памяти контейнеров API +через `docker stats` при 10 параллельных загрузках. `presigned_flow.py` проверяет +короткую подпись, complete, checksum, owner-only, идемпотентность и отрицательные +сценарии. `reconcile_orphans.py` внедряет orphan-объект и незавершённую загрузку, +показывает dry-run, безопасный grace period и повторный `--apply`. + +`fault_matrix.py` из lab2 не включён в основной набор: при отказе PostgreSQL/Redis +nginx исключает нездоровые API-реплики и сам возвращает 502/503 без JSON-тела, +что является корректным поведением LB, но не проверкой формата ошибки API. +Проверка отказов зависимостей для lab4 описана в `REPORT.md`. diff --git a/student-tests/attachments_negative.py b/student-tests/attachments_negative.py new file mode 100644 index 00000000..7ba6e27e --- /dev/null +++ b/student-tests/attachments_negative.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Lab4 negative attachment checks: size, corruption, MIME, ownership, chat scope.""" +from __future__ import annotations + +import argparse +import io +import struct +import uuid +import zlib +from urllib.parse import quote + +from PIL import Image + +from common import HttpClient, assert_status, error_code + +MAX_UPLOAD = 10_485_760 + + +def _png_chunk(kind: bytes, data: bytes) -> bytes: + return ( + len(data).to_bytes(4, "big") + + kind + + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + + +def png_bytes(width: int = 320, height: int = 200) -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (width, height), (30, 120, 200)).save(buffer, format="PNG") + return buffer.getvalue() + + +def jpeg_bytes(width: int = 320, height: int = 200) -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (width, height), (200, 80, 30)).save(buffer, format="JPEG") + return buffer.getvalue() + + +def png_header(width: int, height: int) -> bytes: + """Valid signature/IHDR/IEND; enough for the pixel budget check, no decode.""" + return ( + b"\x89PNG\r\n\x1a\n" + + _png_chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + _png_chunk(b"IEND", b"") + ) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + client = HttpClient(args.base_url) + + alice, alice_reg = client.register() + bob, bob_reg = client.register() + eve, _eve_reg = client.register() + alice_token = client.login(alice_reg)["token"] + bob_token = client.login(bob_reg)["token"] + eve_token = client.login(_eve_reg)["token"] + + status, _h, chat, _r = client.call( + "POST", + "/api/v1/chats", + body={"title": "Attachment negatives", "member_ids": [alice["id"], bob["id"]]}, + token=alice_token, + ) + assert_status(status, 201, "create chat") + status, _h, other_chat, _r = client.call( + "POST", + "/api/v1/chats", + body={"title": "Another chat", "member_ids": [alice["id"], bob["id"]]}, + token=alice_token, + ) + assert_status(status, 201, "create other chat") + + def upload(content: bytes, content_type: str, filename: str, token=alice_token, chat_id=None): + path = ( + "/api/v1/chats/" + + (chat_id or chat["id"]) + + "/attachments?filename=" + + quote(filename, safe="") + ) + return client.raw( + "POST", + path, + data=content, + headers={"Authorization": "Bearer " + token, "Content-Type": content_type}, + ) + + def expect(result, expected: int, label: str) -> dict: + status, headers, payload, _raw = result + assert_status(status, expected, label) + if status >= 400: + if not isinstance(payload, dict) or payload.get("error", {}).get("code") in (None, "internal_error"): + raise AssertionError(f"{label}: expected a typed user error, got {payload!r}") + if payload["error"]["request_id"] != headers.get("X-Request-Id"): + raise AssertionError(f"{label}: request_id mismatch") + return payload + + png = png_bytes() + jpeg = jpeg_bytes() + + attachment = expect(upload(png, "image/png", "ok.png"), 201, "valid PNG") + if attachment["status"] != "ready": + raise AssertionError("valid PNG did not become ready") + + expect( + upload(b"\x89PNG\r\n\x1a\n" + b"x" * (MAX_UPLOAD + 1), "image/png", "big.png"), + 413, + "oversized body", + ) + expect(upload(b"", "image/png", "empty.png"), 415, "empty body") + expect(upload(b"\x89PNG\r\n\x1a\nnot-an-image", "image/png", "broken.png"), 415, "corrupted PNG") + expect(upload(jpeg, "image/png", "spoof.png"), 415, "JPEG bytes declared as PNG") + expect(upload(png, "image/jpeg", "spoof.jpg"), 415, "PNG bytes declared as JPEG") + expect(upload(png, "text/plain", "plain.txt"), 415, "unsupported content type") + expect(upload(png_header(10_000, 5_000), "image/png", "huge.png"), 413, "pixel budget") + expect(upload(png, "image/png", "../escape.png"), 400, "path traversal filename") + expect(upload(png, "image/png", "a/b.png"), 400, "slash filename") + expect(upload(png, "image/png", "a\\b.png"), 400, "backslash filename") + + # A member of the chat still gets 404 for an unknown attachment id. + missing_id = str(uuid.uuid4()) + status, _h, payload, _r = client.call( + "GET", "/api/v1/attachments/" + missing_id, token=bob_token + ) + assert_status(status, 404, "unknown attachment metadata") + if error_code(payload) == "internal_error": + raise AssertionError("unknown attachment leaked a 500") + + # A non-member cannot read metadata or bytes. + for suffix in ("", "/content"): + status, _h, payload, _r = client.call( + "GET", "/api/v1/attachments/" + attachment["id"] + suffix, token=eve_token + ) + assert_status(status, 404, "outsider attachment read") + if error_code(payload) == "internal_error": + raise AssertionError("outsider read leaked a 500") + + # Uploading into another chat and linking it here must be a 404, not a 500. + foreign = expect(upload(png, "image/png", "foreign.png", chat_id=other_chat["id"]), 201, "other-chat upload") + status, _h, payload, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "", "attachment_ids": [foreign["id"]]}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + assert_status(status, 404, "attachment from another chat") + if error_code(payload) == "internal_error": + raise AssertionError("cross-chat attachment leaked a 500") + + # Duplicate and too-many ids are request errors, not 500. + status, _h, _p, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "", "attachment_ids": [attachment["id"], attachment["id"]]}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + assert_status(status, 400, "duplicate attachment ids") + status, _h, _p, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "", "attachment_ids": [attachment["id"]] * 11}, + token=alice_token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + assert_status(status, 400, "too many attachment ids") + + print("PASS: lab4 attachment negatives (size, corruption, MIME, filename, ownership, chat scope)") + + +if __name__ == "__main__": + main() diff --git a/student-tests/auth_negative.py b/student-tests/auth_negative.py new file mode 100644 index 00000000..3ae34f04 --- /dev/null +++ b/student-tests/auth_negative.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +"""Wrong password, Basic-on-business-endpoint, logout and reuse checks.""" +from __future__ import annotations + +import argparse +import base64 + +from common import HttpClient, assert_status, error_code + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + client = HttpClient(args.base_url) + user, registration = client.register() + + wrong = base64.b64encode( + (registration["username"] + ":wrong-password-value").encode("utf-8") + ).decode("ascii") + status, headers, payload, _ = client.call( + "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + wrong} + ) + assert_status(status, 401, "wrong password") + assert error_code(payload) == "unauthorized" + assert headers.get("WWW-Authenticate", "").lower().startswith("basic") + + good = base64.b64encode( + (registration["username"] + ":" + registration["password"]).encode("utf-8") + ).decode("ascii") + status, headers, _, _ = client.call( + "GET", "/api/v1/users/me", headers={"Authorization": "Basic " + good} + ) + assert_status(status, 401, "Basic on business endpoint") + assert headers.get("WWW-Authenticate", "").lower().startswith("bearer") + + session = client.login(registration) + token = session["token"] + status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token) + assert_status(status, 200, "login") + assert me["id"] == user["id"] + + status, _, _, raw = client.call("DELETE", "/api/v1/auth/sessions/current", token=token) + assert_status(status, 204, "logout") + assert raw == b"", "204 must have an empty body" + + status, headers, payload, _ = client.call("GET", "/api/v1/users/me", token=token) + assert_status(status, 401, "token after logout") + assert error_code(payload) == "unauthorized" + assert headers.get("WWW-Authenticate", "").lower().startswith("bearer") + + status, _, _, _ = client.call("DELETE", "/api/v1/auth/sessions/current", token=token) + assert_status(status, 401, "second logout") + print("PASS: wrong password, Basic rejection, logout and token reuse checks") + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 00000000..3308c9ec --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Small standard-library HTTP helpers for the student checks.""" +from __future__ import annotations + +import base64 +import json +import os +import uuid +import urllib.error +import urllib.request + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080").rstrip("/") + + +class HttpClient: + def __init__(self, base_url: str | None = None): + self.base = (base_url or BASE_URL).rstrip("/") + + def call(self, method: str, path: str, body=None, token: str | None = None, headers=None): + h = {"Accept": "application/json"} + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + if token: + h["Authorization"] = "Bearer " + token + h.update(headers or {}) + request = urllib.request.Request(self.base + path, data=data, headers=h, method=method) + try: + response = urllib.request.urlopen(request, timeout=10) + except urllib.error.HTTPError as error: + response = error + with response: + raw = response.read() + payload = None + if raw: + try: + payload = json.loads(raw) + except ValueError: + payload = None + return response.code, response.headers, payload, raw + + def raw(self, method: str, path: str, data: bytes | None = None, headers=None, timeout=60): + """Send raw bytes (uploads, presigned PUT) and return status/headers/json/raw.""" + h = {"Accept": "application/json"} + h.update(headers or {}) + url = path if path.startswith(("http://", "https://")) else self.base + path + request = urllib.request.Request(url, data=data, headers=h, method=method) + try: + response = urllib.request.urlopen(request, timeout=timeout) + except urllib.error.HTTPError as error: + response = error + with response: + raw = response.read() + payload = None + if raw: + try: + payload = json.loads(raw) + except ValueError: + payload = None + return response.code, response.headers, payload, raw + + def register(self) -> tuple[dict, dict]: + suffix = uuid.uuid4().hex[:12] + registration = { + "username": "st_" + suffix, + "display_name": "Студент " + suffix, + "password": "T3st-" + uuid.uuid4().hex, + } + status, _headers, user, _raw = self.call("POST", "/api/v1/users", registration) + if status != 201: + raise AssertionError(f"registration failed: HTTP {status}") + return user, registration + + def login(self, registration: dict) -> dict: + credential = base64.b64encode( + (registration["username"] + ":" + registration["password"]).encode("utf-8") + ).decode("ascii") + status, _headers, session, _raw = self.call( + "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credential} + ) + if status != 201: + raise AssertionError(f"login failed: HTTP {status}") + return session + + +def assert_status(actual: int, expected: int, label: str) -> None: + if actual != expected: + raise AssertionError(f"{label}: expected HTTP {expected}, received {actual}") + + +def error_code(payload) -> str | None: + if not isinstance(payload, dict): + return None + error = payload.get("error") + return error.get("code") if isinstance(error, dict) else None diff --git a/student-tests/expiry.py b/student-tests/expiry.py new file mode 100644 index 00000000..d48da0f6 --- /dev/null +++ b/student-tests/expiry.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Expiry check: run against an API with SESSION_TTL_SECONDS=2 (or pass --wait).""" +from __future__ import annotations + +import argparse +import time + +from common import HttpClient, assert_status, error_code + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--wait", type=float, default=3.0) + args = parser.parse_args() + + client = HttpClient(args.base_url) + user, registration = client.register() + session = client.login(registration) + token = session["token"] + + status, headers, payload, _ = client.call("GET", "/api/v1/users/me", token=token) + assert_status(status, 200, "fresh session") + assert payload["id"] == user["id"], "wrong current user" + + time.sleep(args.wait) + status, headers, payload, _ = client.call("GET", "/api/v1/users/me", token=token) + assert_status(status, 401, "expired session") + assert error_code(payload) == "unauthorized", "expired session must be unauthorized" + assert headers.get("WWW-Authenticate", "").lower().startswith("bearer"), "missing Bearer challenge" + print("PASS: expired session is rejected with 401 and Bearer challenge") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain_history.py b/student-tests/explain_history.py new file mode 100644 index 00000000..d7c7df0b --- /dev/null +++ b/student-tests/explain_history.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +"""Create 10 000 synthetic messages and record EXPLAIN (ANALYZE, BUFFERS). + +Run inside the API container (psycopg is installed there): + docker compose exec -T api python student-tests/explain_history.py \ + --output /app/evidence/explain_history.txt + +The evidence directory is mounted read-only in the container, so the example +below is normally executed with a writable bind mount or copied out. +""" +from __future__ import annotations + +import argparse +import os +import uuid +from datetime import datetime, timezone +from pathlib import Path + +import psycopg + +MESSAGES = 10_000 + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL")) + parser.add_argument("--output", default="evidence/explain_history.txt") + parser.add_argument("--messages", type=int, default=MESSAGES) + args = parser.parse_args() + if not args.database_url: + raise SystemExit("DATABASE_URL is required") + + tag = uuid.uuid4().hex[:10] + user_id = uuid.uuid4() + chat_id = uuid.uuid4() + with psycopg.connect(args.database_url) as conn: + conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s)", + (user_id, "perf_" + tag, "Performance synthetic", "not-a-real-password-hash"), + ) + conn.execute("INSERT INTO chats (id, title) VALUES (%s, %s)", (chat_id, "perf-" + tag)) + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat_id, user_id) + ) + conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), %s, %s, 'synthetic message ' || g, + timestamptz '2026-01-01 00:00:00+00' + (g || ' milliseconds')::interval + FROM generate_series(1, %s) AS g + """, + (chat_id, user_id, args.messages), + ) + total = conn.execute( + "SELECT count(*) FROM messages WHERE chat_id = %s", (chat_id,) + ).fetchone()[0] + conn.execute("ANALYZE messages") + cursor = conn.execute( + """ + SELECT created_at, id FROM messages + WHERE chat_id = %s ORDER BY created_at, id + OFFSET %s LIMIT 1 + """, + (chat_id, max(0, args.messages - 500)), + ).fetchone() + plan_rows = conn.execute( + """ + EXPLAIN (ANALYZE, BUFFERS) + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s AND (created_at, id) > (%s, %s) + ORDER BY created_at, id + LIMIT 51 + """, + (chat_id, cursor[0], cursor[1]), + ).fetchall() + conn.commit() + + plan = "\n".join(row[0] for row in plan_rows) + if "messages_chat_created_at_id_idx" not in plan: + raise AssertionError("history query did not use the keyset index:\n" + plan) + if "Seq Scan" in plan: + raise AssertionError("history query performed a sequential scan:\n" + plan) + + header = ( + "# EXPLAIN (ANALYZE, BUFFERS) for keyset history pagination\n" + f"generated_at_utc: {datetime.now(timezone.utc).isoformat().replace('+00:00', 'Z')}\n" + f"messages_in_chat: {total}\n" + f"chat_id: {chat_id}\n" + "query: WHERE chat_id = ? AND (created_at, id) > (?, ?) " + "ORDER BY created_at, id LIMIT 51\n\n" + ) + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(header + plan + "\n", encoding="utf-8") + print(header + plan) + print(f"PASS: index used on {total} messages; evidence written to {output}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/fault_matrix.py b/student-tests/fault_matrix.py new file mode 100644 index 00000000..a066fc3b --- /dev/null +++ b/student-tests/fault_matrix.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Dependency-failure matrix: Redis and PostgreSQL outages must be bounded. + +The check uses the local Compose project. It stops one dependency, verifies +503 (never 200 and never a trusted anonymous request), starts it again, waits +for readiness and confirms the original bearer session still works after the +Redis restart (AOF persistence). +""" +from __future__ import annotations + +import argparse +import os +import subprocess +import time +from pathlib import Path + +from common import HttpClient, assert_status, error_code + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args: str) -> None: + subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + +def wait_ready(client: HttpClient, expected: int, seconds: float = 90) -> None: + deadline = time.monotonic() + seconds + last = None + while time.monotonic() < deadline: + try: + status, _, _, _ = client.call("GET", "/health/ready") + last = status + if status == expected: + return + except OSError: + last = "connection error" + time.sleep(0.5) + raise AssertionError(f"readiness did not become {expected}; last={last}") + + +def expect_503(client: HttpClient, path: str, token: str | None, label: str) -> None: + status, _headers, payload, _raw = client.call("GET", path, token=token) + if status != 503: + raise AssertionError(f"{label}: expected 503, received {status}") + if error_code(payload) != "unavailable": + raise AssertionError(f"{label}: expected unavailable error code") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + client = HttpClient(args.base_url) + user, registration = client.register() + session = client.login(registration) + token = session["token"] + + results = [] + try: + compose("stop", "redis") + expect_503(client, "/health/ready", None, "Redis down readiness") + expect_503(client, "/api/v1/users/me", token, "Redis down authenticated request") + results.append("redis down: readiness=503, bearer request=503") + started = time.monotonic() + compose("start", "redis") + wait_ready(client, 200) + redis_recovery = round(time.monotonic() - started, 2) + status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token) + assert_status(status, 200, "session after Redis restart") + if me["id"] != user["id"]: + raise AssertionError("session identity changed after Redis restart") + results.append(f"redis up: readiness=200 in {redis_recovery}s, same token=200 (AOF persistence)") + + compose("stop", "postgres") + time.sleep(1) + expect_503(client, "/health/ready", None, "PostgreSQL down readiness") + expect_503(client, "/api/v1/users/me", token, "PostgreSQL down authenticated request") + results.append("postgres down: readiness=503, bearer request=503") + started = time.monotonic() + compose("start", "postgres") + wait_ready(client, 200) + postgres_recovery = round(time.monotonic() - started, 2) + status, _, me, _ = client.call("GET", "/api/v1/users/me", token=token) + assert_status(status, 200, "session after PostgreSQL restart") + results.append(f"postgres up: readiness=200 in {postgres_recovery}s, same token=200") + finally: + compose("start", "redis") + compose("start", "postgres") + + for line in results: + print("OBSERVED:", line) + print("PASS: dependency loss produced only bounded 503 responses; no anonymous trust") + + +if __name__ == "__main__": + main() diff --git a/student-tests/idempotency_race.py b/student-tests/idempotency_race.py new file mode 100644 index 00000000..33298949 --- /dev/null +++ b/student-tests/idempotency_race.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +"""Concurrent shared-key replay across replicas plus scope/conflict checks.""" +from __future__ import annotations + +import argparse +import uuid +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import HttpClient + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--workers", type=int, default=20) + args = parser.parse_args() + client = HttpClient(args.base_url) + + alice, alice_registration = client.register() + bob, bob_registration = client.register() + alice_token = client.login(alice_registration)["token"] + bob_token = client.login(bob_registration)["token"] + + status, _headers, chat, _ = client.call( + "POST", + "/api/v1/chats", + body={"title": "Idempotency race", "member_ids": [alice["id"], bob["id"]]}, + token=alice_token, + ) + if status != 201: + raise AssertionError(f"chat creation failed: HTTP {status}") + + key = uuid.uuid4().hex + text = "concurrent replay " + key + path = "/api/v1/chats/" + chat["id"] + "/messages" + + def send_same_body(_index: int): + return client.call( + "POST", path, body={"text": text}, token=alice_token, + headers={"Idempotency-Key": key}, + ) + + with ThreadPoolExecutor(max_workers=args.workers) as pool: + results = list(pool.map(send_same_body, range(args.workers))) + + statuses = Counter(status for status, _h, _p, _r in results) + statuses.pop(200, None) + if set(statuses) != {201}: + raise AssertionError(f"concurrent replay statuses are not all 201: {dict(statuses)}") + ids = {payload["id"] for _s, _h, payload, _r in results} + if len(ids) != 1: + raise AssertionError(f"concurrent replay created {len(ids)} messages instead of 1") + instances = { + headers.get("X-Instance-Id", "") + for _s, headers, _p, _r in results + if headers.get("X-Instance-Id") + } + if len(instances) < 2: + raise AssertionError("concurrent replay did not reach at least two replicas") + + message_id = next(iter(ids)) + status, _headers, page, _ = client.call( + "GET", path + "?limit=100", token=bob_token + ) + if status != 200: + raise AssertionError(f"history read failed: HTTP {status}") + matching = [m for m in page["items"] if m["id"] == message_id] + if len(matching) != 1: + raise AssertionError("history contains a duplicate or a missing replayed message") + + # Same key, different canonical body -> 409. + status, _headers, payload, _ = client.call( + "POST", path, body={"text": text + " different"}, token=alice_token, + headers={"Idempotency-Key": key}, + ) + if status != 409 or payload.get("error", {}).get("code") != "conflict": + raise AssertionError(f"different body with the same key returned HTTP {status}, expected 409") + + # Missing / malformed key -> 400. + for headers in ({}, {"Idempotency-Key": "bad key with spaces"}): + status, _headers, payload, _ = client.call( + "POST", path, body={"text": "no valid key"}, token=alice_token, headers=headers + ) + if status != 400 or payload.get("error", {}).get("code") != "invalid_request": + raise AssertionError(f"invalid Idempotency-Key returned HTTP {status}, expected 400") + + # Scope includes the user: the same key by another user is independent. + status, _headers, bob_message, _ = client.call( + "POST", path, body={"text": "bob independent"}, token=bob_token, + headers={"Idempotency-Key": key}, + ) + if status != 201 or bob_message["sender_id"] != bob["id"]: + raise AssertionError(f"another user's identical key was not independent: HTTP {status}") + + # Scope includes the path: the same key in another chat is independent. + status, _headers, other_chat, _ = client.call( + "POST", + "/api/v1/chats", + body={"title": "Second chat", "member_ids": [alice["id"], bob["id"]]}, + token=alice_token, + ) + if status != 201: + raise AssertionError(f"second chat creation failed: HTTP {status}") + status, _headers, other_message, _ = client.call( + "POST", + "/api/v1/chats/" + other_chat["id"] + "/messages", + body={"text": "same key another chat"}, + token=alice_token, + headers={"Idempotency-Key": key}, + ) + if status != 201 or other_message["id"] == message_id: + raise AssertionError(f"same key in another chat was not independent: HTTP {status}") + + print("concurrent_workers:", args.workers) + print("statuses:", dict(statuses)) + print("instances:", ", ".join(sorted(instances))) + print("PASS: shared idempotency creates exactly one message and respects all scopes") + + +if __name__ == "__main__": + main() diff --git a/student-tests/instance_spread.py b/student-tests/instance_spread.py new file mode 100644 index 00000000..7978187c --- /dev/null +++ b/student-tests/instance_spread.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Prove that one session is served by at least two real API replicas via the LB.""" +from __future__ import annotations + +import argparse +from collections import Counter + +from common import HttpClient + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--requests", type=int, default=60) + args = parser.parse_args() + client = HttpClient(args.base_url) + + alice, alice_registration = client.register() + bob, _bob_registration = client.register() + alice_session = client.login(alice_registration) + token = alice_session["token"] + + status, _headers, chat, _ = client.call( + "POST", + "/api/v1/chats", + body={"title": "Replica spread", "member_ids": [alice["id"], bob["id"]]}, + token=token, + ) + if status != 201: + raise AssertionError(f"chat creation failed: HTTP {status}") + + instances: Counter[str] = Counter() + for _ in range(args.requests): + status, headers, me, _ = client.call("GET", "/api/v1/users/me", token=token) + if status != 200 or me["id"] != alice["id"]: + raise AssertionError("existing session did not work through the LB") + instance = headers.get("X-Instance-Id", "") + if not instance: + raise AssertionError("missing X-Instance-Id on an API response") + if not headers.get("X-Request-Id"): + raise AssertionError("missing X-Request-Id on an API response") + instances[instance] += 1 + + if len(instances) < 2: + raise AssertionError( + "fewer than two API instances observed; check LB discovery / replica count" + ) + + # A write accepted by one replica must be visible from any other replica. + for index, instance in enumerate(instances): + status, headers, message, _ = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": f"replica-write-{index}"}, + token=token, + headers={"Idempotency-Key": f"spread-{index}-{alice['id']}"}, + ) + if status != 201: + raise AssertionError(f"message write failed through {instance}: HTTP {status}") + status, _headers, page, _ = client.call( + "GET", "/api/v1/chats/" + chat["id"] + "/messages?limit=100", token=token + ) + if status != 200 or not any(m["id"] == message["id"] for m in page["items"]): + raise AssertionError("message written by one replica is not readable") + + print("instances:", ", ".join(f"{name}={count}" for name, count in sorted(instances.items()))) + print("PASS: one session served by >=2 API replicas with a shared datastore") + + +if __name__ == "__main__": + main() diff --git a/student-tests/pool_settings.py b/student-tests/pool_settings.py new file mode 100644 index 00000000..21d47b24 --- /dev/null +++ b/student-tests/pool_settings.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Print the effective PostgreSQL timeouts and pool limits. + +Run inside the API container: + docker compose exec -T api python student-tests/pool_settings.py +""" +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from app import db + + +def main() -> None: + db.pool().open(wait=True, timeout=5) + db.apply_migrations() + with db.pool().connection() as conn: + statement = conn.execute("SHOW statement_timeout").fetchone()["statement_timeout"] + idle = conn.execute("SHOW idle_in_transaction_session_timeout").fetchone()[ + "idle_in_transaction_session_timeout" + ] + lock = conn.execute("SHOW lock_timeout").fetchone()["lock_timeout"] + stats = db.pool().get_stats() + print("statement_timeout =", statement) + print("idle_in_transaction_session_timeout =", idle) + print("lock_timeout =", lock) + print("pool_min =", stats["pool_min"]) + print("pool_max =", stats["pool_max"]) + print("pool_size =", stats["pool_size"]) + print("pool_available =", stats["pool_available"]) + print("PASS: bounded pool and SQL timeouts are active") + + +if __name__ == "__main__": + main() diff --git a/student-tests/presigned_flow.py b/student-tests/presigned_flow.py new file mode 100644 index 00000000..0f2e1d17 --- /dev/null +++ b/student-tests/presigned_flow.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Lab5 track 1: presigned direct upload plus the mandatory proxy fallback.""" +from __future__ import annotations + +import argparse +import hashlib +import io +import uuid + +from PIL import Image + +from common import HttpClient, assert_status, error_code + + +def png_bytes(width: int = 640, height: int = 400) -> bytes: + buffer = io.BytesIO() + Image.new("RGB", (width, height), (20, 160, 90)).save(buffer, format="PNG") + return buffer.getvalue() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + client = HttpClient(args.base_url) + + alice, alice_reg = client.register() + bob, bob_reg = client.register() + alice_token = client.login(alice_reg)["token"] + bob_token = client.login(bob_reg)["token"] + status, _h, chat, _r = client.call( + "POST", + "/api/v1/chats", + body={"title": "Presigned upload", "member_ids": [alice["id"], bob["id"]]}, + token=alice_token, + ) + assert_status(status, 201, "create chat") + + content = png_bytes() + digest = hashlib.sha256(content).hexdigest() + body = { + "filename": "direct.png", + "content_type": "image/png", + "size_bytes": len(content), + "sha256": digest, + } + status, _h, presign, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments/presign", + body=body, + token=alice_token, + ) + assert_status(status, 201, "presign") + if presign["method"] != "PUT" or not presign["upload_url"].startswith("http"): + raise AssertionError("presign response is missing a usable PUT URL") + + # The mandatory proxy endpoint still works. + status, _h, proxy, _r = client.raw( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=proxy.png", + data=content, + headers={"Authorization": "Bearer " + alice_token, "Content-Type": "image/png"}, + ) + assert_status(status, 201, "proxy upload remains available") + + # Direct PUT to the private bucket through the short-lived signature. + status, _h, _p, _r = client.raw( + "PUT", + presign["upload_url"], + data=content, + headers=presign["headers"], + ) + assert_status(status, 200, "presigned PUT") + + status, _h, ready, _r = client.call( + "POST", + "/api/v1/attachments/" + presign["attachment_id"] + "/complete", + body={"sha256": digest, "size_bytes": len(content)}, + token=alice_token, + ) + assert_status(status, 200, "complete") + if ready["status"] != "ready" or ready["sha256"] != digest: + raise AssertionError("completed attachment is not ready with the expected checksum") + + # A chat member can read the bytes through the API; no redirect is used. + status, headers, _p, raw = client.raw( + "GET", + "/api/v1/attachments/" + presign["attachment_id"] + "/content", + headers={"Authorization": "Bearer " + bob_token}, + ) + assert_status(status, 200, "download direct upload") + if raw != content or headers.get_content_type() != "image/png": + raise AssertionError("direct-upload download differs from the original") + + # Complete is idempotent for the same parameters. + status, _h, again, _r = client.call( + "POST", + "/api/v1/attachments/" + presign["attachment_id"] + "/complete", + body={"sha256": digest, "size_bytes": len(content)}, + token=alice_token, + ) + assert_status(status, 200, "idempotent complete") + if again["id"] != ready["id"]: + raise AssertionError("idempotent complete returned another attachment") + + # Negative: a ready attachment cannot be completed with different values. + for payload, label in ( + ({"sha256": "0" * 64, "size_bytes": len(content)}, "ready checksum mismatch"), + ({"sha256": digest, "size_bytes": len(content) + 1}, "ready size mismatch"), + ): + status, _h, error, _r = client.call( + "POST", + "/api/v1/attachments/" + presign["attachment_id"] + "/complete", + body=payload, + token=alice_token, + ) + assert_status(status, 409, label) + if error_code(error) == "internal_error": + raise AssertionError(label + " leaked a 500") + + # Negative: wrong bytes that are never promoted (checksum/size mismatch). + for wrong, label in ( + (png_bytes(641, 400), "checksum mismatch"), + (content + b"x", "size mismatch"), + ): + status, _h, bad_upload, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments/presign", + body=body, + token=alice_token, + ) + assert_status(status, 201, label + " presign") + status, _h, _p, _r = client.raw( + "PUT", + bad_upload["upload_url"], + data=wrong, + headers=bad_upload["headers"], + ) + assert_status(status, 200, label + " PUT") + status, _h, error, _r = client.call( + "POST", + "/api/v1/attachments/" + bad_upload["attachment_id"] + "/complete", + body={"sha256": digest, "size_bytes": len(content)}, + token=alice_token, + ) + assert_status(status, 400, label) + if error_code(error) == "internal_error": + raise AssertionError(label + " leaked a 500") + + # Negative: another user cannot complete Alice's pending upload. + status, _h, pending, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments/presign", + body=body, + token=alice_token, + ) + assert_status(status, 201, "second presign") + status, _h, error, _r = client.call( + "POST", + "/api/v1/attachments/" + pending["attachment_id"] + "/complete", + body={"sha256": digest, "size_bytes": len(content)}, + token=bob_token, + ) + assert_status(status, 404, "foreign complete") + if error_code(error) == "internal_error": + raise AssertionError("foreign complete leaked a 500") + + # Negative: completing before PUT is a typed 409. + status, _h, error, _r = client.call( + "POST", + "/api/v1/attachments/" + pending["attachment_id"] + "/complete", + body={"sha256": digest, "size_bytes": len(content)}, + token=alice_token, + ) + assert_status(status, 409, "incomplete direct upload") + if error_code(error) == "internal_error": + raise AssertionError("incomplete upload leaked a 500") + + # Negative: bad presign input. + for bad, expected in ( + ({**body, "content_type": "image/gif"}, 415), + ({**body, "size_bytes": 10_485_761}, 400), + ({**body, "sha256": "ABC"}, 400), + ({**body, "filename": "../x.png"}, 400), + ): + status, _h, error, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments/presign", + body=bad, + token=alice_token, + ) + assert_status(status, expected, "bad presign input") + if error_code(error) == "internal_error": + raise AssertionError("bad presign input leaked a 500") + + print( + "PASS: presigned upload, complete, checksum verification, proxy fallback, " + "idempotency and negative cases" + ) + + +if __name__ == "__main__": + main() diff --git a/student-tests/private_bucket.py b/student-tests/private_bucket.py new file mode 100644 index 00000000..4f4a51bc --- /dev/null +++ b/student-tests/private_bucket.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Lab4 infrastructure checks: private bucket and no published admin UIs.""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +ROOT = Path(__file__).resolve().parents[1] + + +def read_env() -> dict[str, str]: + values = dict(os.environ) + path = ROOT / ".env" + if path.is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values.setdefault(key.strip(), value.strip()) + return values + + +def s3_client(env: dict[str, str]): + return boto3.client( + "s3", + endpoint_url=env["S3_PUBLIC_ENDPOINT"], + region_name=env.get("S3_REGION", "us-east-1"), + aws_access_key_id=env["S3_ACCESS_KEY_ID"], + aws_secret_access_key=env["S3_SECRET_ACCESS_KEY"], + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + + +def check_public_ports() -> None: + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", "ps", "--format", "json"], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + ) + if result.returncode != 0: + raise AssertionError("could not inspect compose publishers") + services = [json.loads(line) for line in result.stdout.splitlines() if line.strip()] + for service in services: + name = service.get("Service") or service.get("Name", "") + publishers = service.get("Publishers") or [] + published = [p for p in publishers if p.get("PublishedPort")] + if name == "seaweedfs": + ports = {int(p["PublishedPort"]) for p in published} + if ports != {int(os.environ.get("S3_PORT", read_env().get("S3_PORT", "9000")))}: + raise AssertionError(f"unexpected published SeaweedFS ports: {ports}") + elif name in ("api", "postgres", "redis", "migrate", "s3-init"): + if published: + raise AssertionError(f"{name} publishes a host port: {published}") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + del args + env = read_env() + for key in ("S3_PUBLIC_ENDPOINT", "S3_BUCKET", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY"): + if not env.get(key): + raise AssertionError(f"missing {key} in environment/.env") + + client = s3_client(env) + key = "reconcile-tests/private-bucket-" + uuid.uuid4().hex + client.put_object(Bucket=env["S3_BUCKET"], Key=key, Body=b"private", ContentType="text/plain") + try: + url = env["S3_PUBLIC_ENDPOINT"].rstrip("/") + "/" + env["S3_BUCKET"] + "/" + key + try: + urllib.request.urlopen(url, timeout=10) + except urllib.error.HTTPError as error: + if error.code not in (401, 403): + raise AssertionError(f"anonymous GET returned HTTP {error.code}, expected 403") from error + else: + raise AssertionError("anonymous GET succeeded on a private bucket") + + for operation in ("create_bucket", "delete_bucket"): + try: + if operation == "create_bucket": + client.create_bucket(Bucket="forbidden-" + uuid.uuid4().hex[:12]) + else: + client.delete_bucket(Bucket=env["S3_BUCKET"]) + except ClientError as error: + if "AccessDenied" not in str(error): + raise AssertionError(f"{operation} failed with an unexpected error: {error}") from error + else: + raise AssertionError(f"restricted identity was allowed to {operation}") + finally: + client.delete_object(Bucket=env["S3_BUCKET"], Key=key) + + check_public_ports() + print("PASS: bucket rejects anonymous reads; API identity cannot administer it; no admin UI is published") + + +if __name__ == "__main__": + main() diff --git a/student-tests/race_username.py b/student-tests/race_username.py new file mode 100644 index 00000000..c2c30e6a --- /dev/null +++ b/student-tests/race_username.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Concurrent registration with the same username: one 201, the rest 409.""" +from __future__ import annotations + +import argparse +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import HttpClient, assert_status, error_code + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--workers", type=int, default=8) + args = parser.parse_args() + client = HttpClient(args.base_url) + username = "race_" + uuid.uuid4().hex[:16] + body = { + "username": username, + "display_name": "Race candidate", + "password": "T3st-" + uuid.uuid4().hex, + } + + def register(_index: int): + status, _, payload, _ = client.call("POST", "/api/v1/users", body) + return status, error_code(payload) + + with ThreadPoolExecutor(max_workers=args.workers) as executor: + results = list(executor.map(register, range(args.workers))) + + statuses = [status for status, _code in results] + if statuses.count(201) != 1 or statuses.count(409) != args.workers - 1: + raise AssertionError(f"expected one 201 and {args.workers - 1} conflicts, got {statuses}") + for status, code in results: + if status == 409 and code != "conflict": + raise AssertionError("409 must carry conflict code") + + session = client.login(body) + assert session["user"]["username"] == username + print(f"PASS: {args.workers} concurrent registrations produced exactly one user and one session") + + +if __name__ == "__main__": + main() diff --git a/student-tests/reconcile_orphans.py b/student-tests/reconcile_orphans.py new file mode 100644 index 00000000..d4623c0f --- /dev/null +++ b/student-tests/reconcile_orphans.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Lab4/Lab5 evidence: safe S3/PostgreSQL reconciliation with a grace period.""" +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import uuid +from pathlib import Path + +import boto3 +from botocore.config import Config + +from common import HttpClient, assert_status + +ROOT = Path(__file__).resolve().parents[1] + + +def read_env() -> dict[str, str]: + values = dict(os.environ) + path = ROOT / ".env" + if path.is_file(): + for line in path.read_text(encoding="utf-8").splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + key, _, value = line.partition("=") + values.setdefault(key.strip(), value.strip()) + return values + + +def s3_client(env: dict[str, str]): + return boto3.client( + "s3", + endpoint_url=env["S3_PUBLIC_ENDPOINT"], + region_name=env.get("S3_REGION", "us-east-1"), + aws_access_key_id=env["S3_ACCESS_KEY_ID"], + aws_secret_access_key=env["S3_SECRET_ACCESS_KEY"], + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + + +def reconcile(extra: list[str]) -> dict: + result = subprocess.run( + [ + "docker", + "compose", + "--env-file", + ".env", + "--profile", + "tools", + "run", + "--rm", + "--no-deps", + "reconcile", + "python", + "-m", + "app.reconcile", + *extra, + ], + cwd=ROOT, + capture_output=True, + text=True, + timeout=120, + ) + if result.returncode != 0: + raise AssertionError( + "reconciler failed: " + (result.stderr or result.stdout)[-500:] + ) + for line in reversed(result.stdout.splitlines()): + line = line.strip() + if line.startswith("{"): + return json.loads(line) + raise AssertionError("reconciler did not print a JSON summary") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + args = parser.parse_args() + env = read_env() + client = s3_client(env) + bucket = env["S3_BUCKET"] + + # 1. A fresh orphan is reported by a dry-run and is not deleted. + orphan_key = "reconcile-tests/orphan-" + uuid.uuid4().hex + client.put_object(Bucket=bucket, Key=orphan_key, Body=b"orphan", ContentType="text/plain") + dry = reconcile( + ["--dry-run", "--orphan-grace-seconds", "0", "--pending-grace-seconds", "0"] + ) + if dry["orphans_stale_deleted"] < 1: + raise AssertionError("dry-run did not report the injected orphan") + client.head_object(Bucket=bucket, Key=orphan_key) + + # 2. A fresh orphan inside the default grace window is kept by --apply. + kept_key = "reconcile-tests/kept-" + uuid.uuid4().hex + client.put_object(Bucket=bucket, Key=kept_key, Body=b"kept", ContentType="text/plain") + kept = reconcile( + ["--apply", "--orphan-grace-seconds", "86400", "--pending-grace-seconds", "86400"] + ) + if kept["orphans_fresh_kept"] < 1: + raise AssertionError("grace-window orphan was not reported as kept") + client.head_object(Bucket=bucket, Key=kept_key) + + # 3. --apply with grace 0 removes both injected objects. + applied = reconcile( + ["--apply", "--orphan-grace-seconds", "0", "--pending-grace-seconds", "0"] + ) + if applied["orphans_stale_deleted"] < 2: + raise AssertionError("apply did not delete the injected orphans") + for key in (orphan_key, kept_key): + try: + client.head_object(Bucket=bucket, Key=key) + except Exception: + pass + else: + raise AssertionError("orphan still exists after apply") + + # 4. An interrupted presigned upload is removed with its metadata row. + http = HttpClient(args.base_url) + alice, alice_reg = http.register() + token = http.login(alice_reg)["token"] + status, _h, chat, _r = http.call( + "POST", + "/api/v1/chats", + body={"title": "Reconcile pending", "member_ids": [alice["id"]]}, + token=token, + ) + assert_status(status, 201, "create chat") + import hashlib + import io + + from PIL import Image + + buffer = io.BytesIO() + Image.new("RGB", (32, 32), (10, 20, 30)).save(buffer, format="PNG") + png = buffer.getvalue() + digest = hashlib.sha256(png).hexdigest() + status, _h, pending, _r = http.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments/presign", + body={ + "filename": "pending.png", + "content_type": "image/png", + "size_bytes": len(png), + "sha256": digest, + }, + token=token, + ) + assert_status(status, 201, "presign pending") + pending_summary = reconcile( + ["--apply", "--orphan-grace-seconds", "86400", "--pending-grace-seconds", "0"] + ) + if pending_summary["stale_pending_deleted"] < 1: + raise AssertionError("stale pending upload was not removed") + status, _h, _p, _r = http.call( + "GET", "/api/v1/attachments/" + pending["attachment_id"], token=token + ) + assert_status(status, 404, "pending metadata removed") + + print("PASS: dry-run/apply reconciliation, grace period, orphan cleanup, stale pending cleanup") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rollback_chat.py b/student-tests/rollback_chat.py new file mode 100644 index 00000000..46804f21 --- /dev/null +++ b/student-tests/rollback_chat.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Prove that a failure after the chat insert rolls back the whole chat. + +Run inside the API container (psycopg is installed there): + docker compose exec -T api python student-tests/rollback_chat.py + +A temporary trigger aborts the second chat_members insert. The API must +return a bounded 500 and leave no chat row, because chat + membership share +one transaction. +""" +from __future__ import annotations + +import argparse +import os +import uuid + +import psycopg + +from common import HttpClient, assert_status, error_code + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--database-url", default=os.environ.get("DATABASE_URL")) + args = parser.parse_args() + if not args.database_url: + raise SystemExit("DATABASE_URL is required") + + client = HttpClient(args.base_url) + actor, actor_registration = client.register() + victim, _victim_registration = client.register() + session = client.login(actor_registration) + token = session["token"] + + title = "rollback-" + uuid.uuid4().hex + function_name = "test_fail_member_" + uuid.uuid4().hex[:8] + victim_id = str(uuid.UUID(victim["id"])) + + with psycopg.connect(args.database_url, autocommit=True) as conn: + conn.execute( + f""" + CREATE FUNCTION {function_name}() RETURNS trigger AS $$ + BEGIN + IF NEW.user_id = '{victim_id}'::uuid THEN + RAISE EXCEPTION 'injected membership failure'; + END IF; + RETURN NEW; + END + $$ LANGUAGE plpgsql + """ + ) + conn.execute( + f"CREATE TRIGGER {function_name} BEFORE INSERT ON chat_members " + f"FOR EACH ROW EXECUTE FUNCTION {function_name}()" + ) + try: + status, _, payload, _ = client.call( + "POST", + "/api/v1/chats", + {"title": title, "member_ids": [actor["id"], victim["id"]]}, + token=token, + ) + assert_status(status, 500, "injected membership failure") + if error_code(payload) != "internal_error": + raise AssertionError("unexpected error code for injected failure") + rows = conn.execute("SELECT count(*) FROM chats WHERE title = %s", (title,)).fetchone() + if rows[0] != 0: + raise AssertionError("chat survived a failed membership transaction") + finally: + conn.execute(f"DROP TRIGGER IF EXISTS {function_name} ON chat_members") + conn.execute(f"DROP FUNCTION IF EXISTS {function_name}()") + + status, _, chat, _ = client.call( + "POST", + "/api/v1/chats", + {"title": title + "-ok", "member_ids": [actor["id"], victim["id"]]}, + token=token, + ) + assert_status(status, 201, "chat after trigger cleanup") + assert set(chat["member_ids"]) == {actor["id"], victim["id"]} + print("PASS: failed membership insert rolled back the chat; subsequent create is atomic") + + +if __name__ == "__main__": + main() diff --git a/student-tests/transfer_compare.py b/student-tests/transfer_compare.py new file mode 100644 index 00000000..20585eac --- /dev/null +++ b/student-tests/transfer_compare.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Lab5 track 1: compare API traffic/RAM for proxy vs presigned direct upload.""" +from __future__ import annotations + +import argparse +import hashlib +import io +import json +import math +import os +import re +import subprocess +import threading +import time + +from PIL import Image + +from common import HttpClient, assert_status + +MEM_RE = re.compile(r"([0-9.]+)\s*(B|KiB|MiB|GiB)") + + +def make_png(target_bytes: int) -> bytes: + pixels = max(1, target_bytes // 3) + width = max(32, int(math.sqrt(pixels * 1.4))) + height = max(32, pixels // width) + image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)) + buffer = io.BytesIO() + image.save(buffer, format="PNG", compress_level=1) + return buffer.getvalue() + + +def api_containers(prefix: str) -> list[str]: + try: + result = subprocess.run( + ["docker", "ps", "--filter", "name=" + prefix, "--format", "{{.Names}}"], + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return [] + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def api_net_bytes(names: list[str]) -> dict[str, int]: + total = 0 + for name in names: + try: + result = subprocess.run( + ["docker", "exec", name, "cat", "/proc/net/dev"], + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + continue + for line in result.stdout.splitlines(): + if "eth0:" not in line: + continue + fields = line.split("eth0:")[1].split() + # RX bytes is field 0; TX bytes is field 8. + total += int(fields[0]) + int(fields[8]) + return {"total": total} + + +def memory_mib(text: str) -> float | None: + match = MEM_RE.search(text) + if not match: + return None + return float(match.group(1)) * {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[match.group(2)] + + +def api_mem_mib(prefix: str) -> float: + try: + result = subprocess.run( + ["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return 0.0 + total = 0.0 + for line in result.stdout.splitlines(): + name, _, mem = line.partition(" ") + if prefix in name: + value = memory_mib(mem) + if value is not None: + total += value + return total + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--container-prefix", default="mtusi-lab4-api") + parser.add_argument("--output", default="") + args = parser.parse_args() + + client = HttpClient(args.base_url) + alice, alice_reg = client.register() + token = client.login(alice_reg)["token"] + status, _h, chat, _r = client.call( + "POST", + "/api/v1/chats", + body={"title": "Transfer comparison", "member_ids": [alice["id"]]}, + token=token, + ) + assert_status(status, 201, "create chat") + + names = api_containers(args.container_prefix) + results = [] + for target in (512 * 1024, 4 * 1024 * 1024): + content = make_png(target) + digest = hashlib.sha256(content).hexdigest() + + # Proxy: client -> API -> S3, all bytes pass through the API process. + before = api_net_bytes(names)["total"] + mem_before = api_mem_mib(args.container_prefix) + status, _h, _p, _r = client.raw( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=proxy.png", + data=content, + headers={"Authorization": "Bearer " + token, "Content-Type": "image/png"}, + timeout=120, + ) + assert_status(status, 201, "proxy upload") + after = api_net_bytes(names)["total"] + mem_after = api_mem_mib(args.container_prefix) + proxy = { + "file_bytes": len(content), + "api_network_bytes": after - before, + "api_memory_growth_mib": round(max(0.0, mem_after - mem_before), 2), + "client_to_api_bytes": len(content), + "client_to_s3_bytes": 0, + } + + # Direct: presign, client -> S3, then complete reads the object once to + # verify the checksum. The public API never receives the file body. + status, _h, presign, _r = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments/presign", + body={ + "filename": "direct.png", + "content_type": "image/png", + "size_bytes": len(content), + "sha256": digest, + }, + token=token, + ) + assert_status(status, 201, "presign") + status, _h, _p, _r = client.raw( + "PUT", presign["upload_url"], data=content, headers=presign["headers"], timeout=120 + ) + assert_status(status, 200, "direct PUT") + + before = api_net_bytes(names)["total"] + mem_before = api_mem_mib(args.container_prefix) + peak = {"value": mem_before} + stop = threading.Event() + + def sampler() -> None: + while not stop.is_set(): + peak["value"] = max(peak["value"], api_mem_mib(args.container_prefix)) + time.sleep(0.1) + + thread = threading.Thread(target=sampler, daemon=True) + thread.start() + status, _h, _p, _r = client.call( + "POST", + "/api/v1/attachments/" + presign["attachment_id"] + "/complete", + body={"sha256": digest, "size_bytes": len(content)}, + token=token, + ) + stop.set() + thread.join(timeout=2) + assert_status(status, 200, "complete") + after = api_net_bytes(names)["total"] + direct = { + "file_bytes": len(content), + "api_network_bytes": after - before, + "api_memory_peak_mib": round(peak["value"], 2), + "api_memory_growth_mib": round(max(0.0, peak["value"] - mem_before), 2), + "client_to_api_bytes": 0, + "client_to_s3_bytes": len(content), + } + results.append({"proxy": proxy, "direct": direct}) + + summary = {"container_prefix": args.container_prefix, "sizes": results} + print(json.dumps(summary, ensure_ascii=False, sort_keys=True)) + if args.output: + with open(args.output, "w", encoding="utf-8") as handle: + json.dump(summary, handle, ensure_ascii=False, indent=2, sort_keys=True) + print("PASS: measured proxy vs direct API traffic and memory at two file sizes") + + +if __name__ == "__main__": + main() diff --git a/student-tests/upload_memory.py b/student-tests/upload_memory.py new file mode 100644 index 00000000..b1eccd90 --- /dev/null +++ b/student-tests/upload_memory.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Lab4 experiment: 10+ concurrent uploads with API container peak memory. + +The uploads are streamed to an ephemeral file and decoded under a bounded +semaphore, so this test looks for a flat memory profile rather than a linear +growth with concurrency. +""" +from __future__ import annotations + +import argparse +import io +import json +import os +import re +import subprocess +import threading +import time +from concurrent.futures import ThreadPoolExecutor + +from PIL import Image + +from common import HttpClient, assert_status + +MEM_RE = re.compile(r"([0-9.]+)\s*(B|KiB|MiB|GiB)") + + +def make_png(width: int, height: int) -> bytes: + # Random pixels compress poorly, giving a realistic multi-MiB upload. + image = Image.frombytes("RGB", (width, height), os.urandom(width * height * 3)) + buffer = io.BytesIO() + image.save(buffer, format="PNG", compress_level=1) + return buffer.getvalue() + + +def memory_mib(text: str) -> float | None: + match = MEM_RE.search(text) + if not match: + return None + value = float(match.group(1)) + unit = match.group(2) + return value * {"B": 1 / 1024 / 1024, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024}[unit] + + +def sample_api_memory(prefix: str) -> dict[str, float]: + try: + result = subprocess.run( + [ + "docker", + "stats", + "--no-stream", + "--format", + "{{.Name}} {{.MemUsage}}", + ], + capture_output=True, + text=True, + timeout=10, + ) + except (OSError, subprocess.SubprocessError): + return {} + if result.returncode != 0: + return {} + usage = {} + for line in result.stdout.splitlines(): + name, _, mem = line.partition(" ") + if prefix in name: + value = memory_mib(mem) + if value is not None: + usage[name] = value + return usage + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--workers", type=int, default=10) + parser.add_argument("--width", type=int, default=1400) + parser.add_argument("--height", type=int, default=1000) + parser.add_argument("--container-prefix", default="mtusi-lab4-api") + parser.add_argument("--output", default="") + args = parser.parse_args() + + client = HttpClient(args.base_url) + alice, alice_reg = client.register() + alice_token = client.login(alice_reg)["token"] + status, _h, chat, _r = client.call( + "POST", + "/api/v1/chats", + body={"title": "Upload memory experiment", "member_ids": [alice["id"]]}, + token=alice_token, + ) + assert_status(status, 201, "create chat") + + content = make_png(args.width, args.height) + peak: dict[str, float] = {} + stop = threading.Event() + + def sampler() -> None: + while not stop.is_set(): + for name, value in sample_api_memory(args.container_prefix).items(): + peak[name] = max(peak.get(name, 0.0), value) + time.sleep(0.2) + + sampler_thread = threading.Thread(target=sampler, name="memory-sampler", daemon=True) + sampler_thread.start() + baseline = sample_api_memory(args.container_prefix) + + def upload(index: int): + return client.raw( + "POST", + "/api/v1/chats/" + chat["id"] + "/attachments?filename=load-%d.png" % index, + data=content, + headers={"Authorization": "Bearer " + alice_token, "Content-Type": "image/png"}, + timeout=120, + ) + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=args.workers) as pool: + results = list(pool.map(upload, range(args.workers))) + elapsed = time.monotonic() - started + stop.set() + sampler_thread.join(timeout=2) + + statuses = [status for status, _h, _p, _r in results] + assert_status(min(statuses), 201, "concurrent uploads") + if max(statuses) != 201: + raise AssertionError(f"some uploads failed: {statuses}") + + summary = { + "workers": args.workers, + "file_bytes": len(content), + "elapsed_seconds": round(elapsed, 3), + "successful_uploads": len(statuses), + "baseline_mib": {k: round(v, 2) for k, v in baseline.items()}, + "peak_mib": {k: round(v, 2) for k, v in peak.items()}, + "peak_growth_mib": { + k: round(peak.get(k, 0.0) - baseline.get(k, 0.0), 2) for k in peak + }, + } + print(json.dumps(summary, ensure_ascii=False, sort_keys=True)) + if args.output: + with open(args.output, "w", encoding="utf-8") as handle: + json.dump(summary, handle, ensure_ascii=False, indent=2, sort_keys=True) + print("PASS: concurrent uploads completed; inspect peak_mib for the bounded-memory claim") + + +if __name__ == "__main__": + main()