diff --git a/.dockerignore b/.dockerignore index bc16e556..ba9ce043 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,20 +1,53 @@ +# VCS / CI .git .github + +# Secrets and local env .env .env.* !.env.example secrets certs -volumes -data -backups *.key *.pem *.p12 *.pfx *.jks + +# Local / runtime junk .venv -__pycache__ +venv +**/__pycache__ +**/*.py[cod] +*.egg-info +.pytest_cache +.mypy_cache +.ruff_cache node_modules -evidence .DS_Store +Thumbs.db +.idea +.vscode + +# Lab artifacts not needed at runtime +evidence +volumes +data +backups +docs +student-tests +tests +scripts +examples +contracts +REPORT.md +README.md +COURSE.md +CONTRIBUTING.md +MAINTAINERS.md +Dockerfile.example +compose.example.yaml +requirements-dev.txt +*.md + +# Keep only what the image needs: requirements.txt + src/ + migrations/ diff --git a/.env.example b/.env.example index d99d26a2..9be9814c 100644 --- a/.env.example +++ b/.env.example @@ -1,16 +1,17 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Secret VALUES are not here: run `python scripts/init_secrets.py` to create ./secrets/*. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Sessions live in Redis (AOF) since lab2 grade 4. +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url +# Number of interchangeable API replicas behind HAProxy (grade 3 requires ≥2). API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +# Lab4: private S3 bucket (SeaweedFS; s3:9000 is the s3-gw sidecar on the internal storage network). S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments @@ -18,3 +19,16 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Upload slots per API replica; excess waits up to 5 s, then 503 + Retry-After. +MAX_CONCURRENT_UPLOADS=8 +# Abort an upload whose body stalls this long / takes longer than the deadline. +UPLOAD_IDLE_TIMEOUT_SECONDS=10 +UPLOAD_DEADLINE_SECONDS=120 +# Reconciler: objects of unfinished uploads are removed only after this age (>= 2 * deadline). +UPLOAD_GRACE_SECONDS=900 +RECONCILE_INTERVAL_SECONDS=300 +# Graceful stop: after SIGTERM /health/ready is 503 but requests are still served for the +# drain window (≥ HAProxy inter 1s × fall 2), then in-flight requests get up to the timeout. +# Keep drain + timeout + 2 s below stop_grace_period of the api service (15 s). +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=8 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..246edb15 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +# Shell scripts run on Unix or inside Linux containers (infra/s3-gw-entrypoint.sh is +# bind-mounted into s3-gw): keep LF even on Windows checkouts with core.autocrlf=true. +*.sh text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..8594c569 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,50 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Заголовок PR: `ЛР 4 — БВТ2303 — Болотов — на 4` -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Автор / группа: Болотов Илья, БВТ2303 +Целевая оценка (3/4/5): 4 +Трек 5, если заявлен: не заявлен -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Что реализовано и что перенесено: +- Из lab3 перенесено всё: API, Redis-сессии, идемпотентность в PostgreSQL, ≥2 реплики за HAProxy, one-shot migrate, лимиты. +- Lab4: + - SeaweedFS 3.97 (digest), приватный bucket, все служебные порты закрыты; S3 API отдаётся только через сайдкар `s3-gw` во внутренней сети; + - `restart: unless-stopped` у всех долгоживущих сервисов, включая `s3` и `s3-gw`; после падения или рестарта `s3` контейнер `s3-gw` сам переходит в его новый network namespace; + - отдельный ключ API с правами только на bucket; admin-ключ есть только у `s3-init`; + - вложения: потоковое чтение с лимитами 10 MiB / 40 млн пикселей; полная проверка PNG и JPEG (`djpeg -strict` под `prlimit`); intent → PUT → COMMIT; + - компенсация, `CommitUncertain` без удаления, reconciler с grace 900 с; + - HAProxy `balance leastconn`; readiness включает S3, при общей деградации HAProxy переключается на резервный backend по liveness; + - watchdog и breaker для PostgreSQL; + - graceful shutdown: по SIGTERM 503 отдаёт только `/health/ready`, реплика обслуживает запросы ещё 3 с (пока HAProxy её не исключит), затем до 8 с дожидается идущих запросов; `stop_grace_period: 15s`; + - текст с U+0000 (BYTEA). + +Команды запуска из чистого clone: +`copy .env.example .env` → `python scripts/init_secrets.py` → `docker compose --env-file .env up -d --build --scale api=2` (или `scripts\up.cmd`) → дождаться `http://127.0.0.1:8080/health/ready`. Подробно — в REPORT.md. + +Результаты проверок и ссылка на REPORT.md ([REPORT.md](../REPORT.md), сырые данные — в `evidence/`): +- check: PASS; validate: PASS; smoke lab4: PASS. +- persistence: PASS (хук перезапускает API и все хранилища); failover: PASS (127 записей, 0 неудачных попыток; хук с окном drain возвращается за ≈3.6 с, первая засчитанная запись через 3.95 с). +- lab3 на стенде lab4: поочерёдный отказ каждой реплики — 168 + 166 записей, 0 неудачных попыток (`seq-failover.json`); rolling update под нагрузкой — обе реплики заменены, 1436 успешных, 0 ошибок, p99 62 мс (`rolling-load.json`). +- student-tests: 69 passed, 0 failed (с RUN_DOCKER_TESTS=1 RUN_DEPS_TEST=1, включая `test_shutdown`: drain после SIGTERM, `docker stop` под нагрузкой без ошибок, загрузка через закрытие listener → 201); истечение сессии при TTL 2 с: 2 passed. +- Failover вложений: первое скачивание через 3.625 / 3.64 с от начала `docker stop`, 0 ошибок. Full restart: down 15.0 с, 45.6 с до ready, SHA-256 совпал. +- Рестарт S3: `docker restart` контейнера `s3` → ready 200 через 37 с, падение процесса `weed` → через 29 с; после этого upload 201 и download 200 с тем же SHA-256 (`s3-restart.txt`). +- 10/20/40 одновременных загрузок 9.5 MiB: рост RSS API 16–31 MiB, пик LB 31.8–60.3 MiB из 128, S3 без OOM (max anon 401.3–566.4 MiB из 1024), на 40 — 18 × 201 и 22 × 503 (backpressure: 2 × 8 слотов; запросы разошлись 20/20). +- LB — `balance leastconn`: при чередовании медленных загрузок и коротких GET загрузки распределяются 4/4 (round-robin — 8/0, `lb-interleave.json`). +- Нагрузка: 265.13 RPS, 0 ошибок, p50/p95/p99 = 15/32/78 мс. + +Ограничения / известные ошибки: +- Один хост, по одному экземпляру БД, Redis, S3 и LB. +- Тело загрузки ≤10 MiB временно лежит на диске реплики (не прямой стриминг в S3). +- При зависшей БД запрос ждёт до 6 с. +- Остановка реплики занимает ≈3.6 с (окно drain); запрос, не завершившийся за 8 с после закрытия listener, обрывается. +- Windows: `PYTHONUTF8=1`, BASE_URL `127.0.0.1`. + +Источники и использование ИИ: FastAPI, psycopg3, redis-py, boto3, HAProxy, SeaweedFS, libjpeg-turbo, спецификации PNG/JPEG, документация S3; реализация и эксперименты — с помощью ИИ-ассистента (Cursor), числа взяты из `evidence/`. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] make check и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..d38b81a1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +# Pinned base image (lab3 → lab4). Update digest when bumping Python. +FROM python:3.12.8-slim-bookworm AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONPATH=/app/src \ + MIGRATIONS_DIR=/app/migrations \ + UPLOAD_SPOOL_DIR=/tmp/messenger-spool + +WORKDIR /app + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home /app --shell /usr/sbin/nologin app + +# djpeg (libjpeg-turbo) fully decodes uploaded JPEGs with -strict; prlimit (util-linux) caps it. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir --upgrade pip==24.3.1 \ + && pip install --no-cache-dir --retries 10 --timeout 60 -r requirements.txt + +# Code stays root-owned and read-only for the runtime user. +COPY src ./src +COPY migrations ./migrations + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=5 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + +CMD ["python", "-m", "messenger"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..94e37ebf 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,281 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Болотов Илья, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: **4** (трек 5 не выбран) +- Ветка: `solution/bvt2303-bolotov` +- Использованные источники, библиотеки и помощь ИИ: + - FastAPI, Uvicorn, psycopg3 (+pool), redis-py, argon2-cffi, boto3 1.35.99, HAProxy 2.9, SeaweedFS 3.97, libjpeg-turbo 2.1.5 (`djpeg`); + - спецификации PNG (W3C PNG 2nd ed.) и JPEG (ITU T.81), документация AWS S3 и SeaweedFS; + - материалы курса: README, contracts, tests; + - перенос lab3, реализация и эксперименты выполнены с помощью ИИ-ассистента (Cursor). Все числа ниже взяты из файлов `evidence/`. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Prerequisites:** + +- Docker Engine + Compose v2 (проверено: Docker 29.9.2, Compose v5.6.1, Docker Desktop/WSL2); +- Python 3.9+ на хосте для скрипта секретов и тестов (stdlib). На Windows задайте `PYTHONUTF8=1`. + +```sh +cp .env.example .env # Windows: copy .env.example .env +python scripts/init_secrets.py # создаёт ./secrets/* (идемпотентно, печатает только имена файлов) +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 +# то же одной командой: scripts\up.cmd (Windows) или ./scripts/up.sh (Unix) +``` + +Порядок старта задан в compose: + +1. `postgres` и `s3` становятся healthy. +2. One-shot `migrate` применяет `migrations/*.sql`, `s3-init` идемпотентно создаёт приватный bucket. Оба должны завершиться с кодом 0. +3. Стартуют `api` ×2, затем `lb`. + +Готовность: `GET http://127.0.0.1:8080/health/ready` → 200. На Windows используйте `127.0.0.1`, а не `localhost`: иначе каждое соединение сначала ≈2 с ждёт IPv6 `::1`, на котором порт не опубликован. + +Проверки (PowerShell; в Unix те же команды с `export`): + +```powershell +$env:PYTHONUTF8=1 +python scripts/check_contract.py # make check +.venv\Scripts\python.exe scripts/validate_openapi.py # make validate (venv с requirements-dev.txt) +python tests/smoke.py --base-url http://127.0.0.1:8080 # make test +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080 +python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080 +docker compose --env-file .env up -d --scale api=2 # вернуть остановленную реплику +python scripts/seq_failover_evidence.py # поочерёдный отказ каждой реплики (lab3) +python scripts/rolling_load_evidence.py # rolling update под нагрузкой (lab3) +python scripts/lb_interleave_evidence.py # распределение длинных загрузок по репликам (leastconn) +scripts\rolling-update.cmd # rolling update без нагрузки +$env:RUN_DOCKER_TESTS=1; $env:RUN_DEPS_TEST=1; python student-tests/run_all.py +docker compose --env-file .env exec -T api python -m messenger.selfcheck s3 +docker compose --env-file .env exec -T api python -m messenger.selfcheck consistency +docker compose --env-file .env exec -T api python -m messenger.reconcile --dry-run --check-missing +``` + +На Unix используйте `scripts/restart.sh`, `scripts/stop-one.sh` и `scripts/rolling-update.sh`. Если файлы получены без executable bit, выполните `chmod +x scripts/*.sh`. + +- Остановка без потери данных: `docker compose --env-file .env down`. +- Намеренное удаление учебных данных (PostgreSQL, Redis, S3): `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +Клиент → HAProxy lb (127.0.0.1:8080, единственная публикация) + ├─ messenger_api (check /health/ready) → API × N + └─ messenger_api_degraded (check /health/live) → те же API, если не готова ни одна +API → PostgreSQL users/chats/messages/idempotency + attachments, upload_intents (сеть labnet) + → Redis сессии (AOF) (сеть labnet) + → s3:9000 s3-gw (HAProxy TCP) → SeaweedFS S3 127.0.0.1:8333 (сеть storage, internal) +One-shot: migrate (миграции), s3-init (bucket + проверка анонимного 403). +``` + +**Перенесено из lab3:** весь `src/messenger`, Redis-сессии, идемпотентность POST сообщений в PostgreSQL, ≥2 реплики за HAProxy с `server-template`, one-shot миграции, лимиты CPU/RAM, hook-скрипты, студенческие тесты, доказательства уровня 4 lab3 (поочерёдный отказ каждой реплики и rolling update под нагрузкой, повторены на стенде lab4). + +**Решения lab4:** + +- **S3-сервер — SeaweedFS 3.97** (образ закреплён по digest), том `s3data`. Все его порты (master/filer/volume UI и gRPC) привязаны к `127.0.0.1` внутри контейнера. В internal-сеть `storage` попадает только S3 API через сайдкар `s3-gw` в том же network namespace. Сеть `storage` видят лишь `api` и `s3-init`. Анонимной identity нет. + + У `s3` и `s3-gw`, как у остальных долгоживущих сервисов, стоит `restart: unless-stopped`; у one-shot `migrate` и `s3-init` политики нет. После рестарта `s3` получает новый network namespace, а `s3-gw` остаётся в старом, где есть только `lo`. Поэтому HAProxy в `s3-gw` запущен через `infra/s3-gw-entrypoint.sh`: скрипт раз в 2 с проверяет интерфейсы. Если остался один `lo`, он выжидает 5 с и завершает контейнер, а restart policy запускает `s3-gw` уже в namespace нового `s3`. Healthcheck `s3-gw` проверяет то же условие. Перезапуски через compose (`restart s3`) перезапускают и `s3-gw` (`depends_on … restart: true`). +- **Credentials:** `init_secrets.py` генерирует две identity SeaweedFS. + - `admin` — только у `s3-init`; + - `messenger-api` — права `Read/Write/List` только на bucket вложений. + + Секреты передаются как Docker secrets (`*_FILE`), общий `.env` в контейнеры не пробрасывается. +- **Модель данных** (`003_attachments.sql`): + - `attachments` (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, width, height, object_key, status=ready); + - `upload_intents` (object_key, created_at); + - связь «сообщение — вложения» с позицией: порядок `attachment_ids` сохраняется. + + Object key = случайный UUID, назначается сервером; filename служит только для отображения. +- **Конвейер загрузки** (`attachments.py`): + 1. До чтения тела проверяются filename (400), Content-Type (415), членство в чате (404) и наличие слота (8 на реплику, ожидание 5 с, иначе 503). + 2. Тело читается кусками по 64 KiB, на каждом куске проверяются лимиты: 10 MiB (413), idle 10 с, общий дедлайн 120 с. Байты сразу идут в потоковый валидатор и во временный файл `/tmp/messenger-spool`. + 3. PNG проверяется прямо в потоке, вместе с полной распаковкой IDAT. Лимит 40 млн пикселей проверяется по IHDR/SOF до распаковки. + 4. JPEG после потоковой проверки структуры полностью декодируется: `djpeg -strict -scale 1/8` под `prlimit --as=384MiB`, таймаут 15 с, 1 декодирование на реплику. + 5. intent → S3 PUT из файла → транзакция «INSERT attachments + DELETE intent» → 201 `ready`. Временный файл удаляется в `finally`. + + **Честно о буферизации:** тело (≤10 MiB) буферизуется на диске реплики на время запроса и не стримится в S3 напрямую. Зато оно не накапливается в RAM, и после ответа на реплике ничего не остаётся (тест `test_no_original_kept_on_api_replicas`, `selfcheck-spool.json`). +- **Балансировка — `balance leastconn`** в обоих backend HAProxy. Загрузка вложения держит соединение секунды, а round-robin этого не видит и может отправлять длинные upload на занятую реплику, пока у другой свободны слоты. `leastconn` выбирает реплику с наименьшим числом открытых соединений. При равной нагрузке реплики чередуются, поэтому последовательные короткие запросы по-прежнему видят обе реплики (40 запросов → 20/20). Сравнение — в эксперименте с памятью ниже (`evidence/lb-interleave.json`). +- **Readiness = PostgreSQL + Redis + S3:** S3 стоит на обязательном синхронном пути upload/download. Чтобы падение S3 не обрушило текстовый чат, HAProxy при `nbsrv(messenger_api) = 0` переключается на backend с проверкой `/health/live`. Затронутые endpoint отвечают JSON 503 самого API, остальные работают. +- **Таймауты против зависаний:** + - S3: connect 2 с, read 5 с, 2 попытки. `dns_opt timeout:1 attempts:1`, иначе неудачный DNS-запрос к остановленному `s3` занимал 8 с. + - PostgreSQL: watchdog обрывает сокет libpq у соединения, занятого дольше 6 с, затем breaker на 3 с. `statement_timeout` при `docker pause` не срабатывает, потому что сервер заморожен. + - HAProxy: `timeout server 45s` больше худшего пути API (≈35 с). +- **Graceful shutdown реплики с окном drain** (`lifecycle.DrainingServer` — подкласс `uvicorn.Server`). Процесс API — PID 1 (exec-форма `CMD`), поэтому SIGTERM от `docker stop` приходит прямо в него. + 1. Первый SIGTERM переводит в 503 «Shutting down» только `/health/ready`. Остальные запросы обслуживаются как обычно в течение `SHUTDOWN_DRAIN_SECONDS=3`: этого хватает, чтобы HAProxy (`inter 1s fall 2`) исключил реплику. + 2. Затем uvicorn закрывает listener и до `SHUTDOWN_TIMEOUT_SECONDS=8` ждёт идущие запросы, включая загрузку (spool → S3 PUT → COMMIT). + 3. Lifespan останавливает reconciler (join ≤1 с) и закрывает Redis и пул БД (≤1 с). + + `stop_grace_period: 15s` больше 3 + 8 + 2 с; скрипты, которые зовут `docker stop` напрямую, используют `-t 15`. Если запрос всё-таки прерван по таймауту, исход тот же, что при падении процесса: intent остаётся для reconciler, ветка `CommitUncertain` не меняется. Остановка реплики длится ≈3.6 с вместо ≈0.6 с, зато HAProxy больше не получает от неё 503. +- **Переменные `.env.example` → конфигурация:** compose подставляет все параметры загрузки и S3 в `api` (`S3_`*, `MAX_*`, `UPLOAD_*`, `RECONCILE_INTERVAL_SECONDS`, `SHUTDOWN_*`). Секретные значения лежат не в `.env`, а в `./secrets/*` (gitignored) и монтируются по `*_FILE`. `API_REPLICAS` читают `scripts/up.*`. Каждая `${VAR}` из `compose.yaml` есть в `.env.example` (проверено скриптом). +- **Миграции lab4:** + - `003_attachments.sql`; + - `004_message_text_bytes.sql`: `messages.text` → BYTEA, `idempotency_records.response_body` → JSON. PostgreSQL TEXT/JSONB не хранят U+0000, а контракт разрешает его в тексте. + +### Согласованность PostgreSQL и S3 + +| Отказ | Что делает код | Итог | +| --- | --- | --- | +| БД недоступна до PUT | intent не записан → 503 | в S3 ничего нет | +| S3 PUT упал | компенсация: DELETE объекта (если S3 жив) и intent | нет объекта, нет метаданных (selfcheck A) | +| PUT прошёл, INSERT/COMMIT достоверно откатился | компенсация удаляет объект и intent | selfcheck B | +| Обрыв связи во время COMMIT (исход неизвестен) | `CommitUncertain` → 503, **объект не удаляется** | если COMMIT прошёл — `ready` с байтами; если нет — intent остаётся для reconciler (selfcheck F) | +| Процесс упал между PUT и COMMIT / компенсация не смогла удалить | остаётся intent | reconciler удаляет объект и intent после grace (selfcheck C, `reconcile-run.txt`) | +| Объект без intent и без метаданных | sweep по `LastModified` объекта | удаляется только старше grace (selfcheck D) | + +Reconciler (`messenger.reconcile`): + +- работает в фоне на каждой реплике раз в 300 с ±20%; одновременно — только один (`pg_try_advisory_lock`); +- запускается и из CLI с `--dry-run`, `--grace-seconds`, `--check-missing`; +- grace = 900 с, это ≥ 2 × дедлайна загрузки 120 с, поэтому идущая загрузка не может быть старше grace; +- intent берётся `FOR UPDATE SKIP LOCKED` и не гонится с COMMIT той же загрузки; +- объект, ключ которого есть в `attachments`, не удаляется никогда, даже при `--grace-seconds 0` (selfcheck E); +- метаданные reconciler не удаляет: отсутствующий оригинал только репортится. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Offline check / OpenAPI | `check_contract.py`, `validate_openapi.py` | PASS / PASS | `evidence/check.txt`, `evidence/validate.txt` | +| Публичный smoke lab4 | `tests/smoke.py` | PASS lab4 public API scenario (checksum, 404 постороннему) | `evidence/smoke.txt` | +| Все студенческие тесты (уровни 3–4 + lab3) | `student-tests/run_all.py`, `RUN_DOCKER_TESTS=1 RUN_DEPS_TEST=1` | **69 passed, 0 failed** (тест истечения сессии при TTL 3600 печатает SKIP; отдельно при `SESSION_TTL_SECONDS=2`: 2 passed) | `evidence/student-tests.txt`, `evidence/session-expiry.txt` | +| Байты в приватном bucket, не в БД/на API | `test_no_original_kept_on_api_replicas`, `selfcheck spool` | на репликах нет файлов с SHA-256 оригинала, spool пуст | `evidence/student-tests.txt`, `evidence/selfcheck-spool.json` | +| Upload → stop обслужившей → download через другую | `scripts/attachment_failover_evidence.py` (2 раунда) | первый 200 через 3.625 с / 3.64 с от начала `docker stop` (сама остановка с окном drain — 3.59 / 3.62 с); 20+20 скачиваний, 0 ошибок, SHA-256 совпадает, остановленная не отвечала | `evidence/attachment-failover.json` | +| Restart с сохранением volumes | `scripts/full_restart_evidence.py` (`down` без `-v` → `up`) | down 15.0 с, down→ready 45.6 с; тот же token, метаданные, SHA-256, текст с U+0000; `bucket exists` | `evidence/full-restart.json` | +| Persistence/failover (публичные) | `tests/resilience.py` | persistence PASS (хук перезапускает API, PostgreSQL, Redis, S3); failover PASS: до отказа видны 2 реплики, 127 записей, 0 неудачных попыток; тест засчитывает запись только после возврата хука, первая — через 3.95 с от его старта (хук ждёт drain ≈3.6 с, запись при этом не прерывалась) | `evidence/persistence*.txt`, `evidence/failover*.txt` | +| Поочерёдный отказ каждой реплики (lab3) | `scripts/seq_failover_evidence.py` (2 раунда, `stop-one` → запись → restore) | раунд 0: 168 записей, 0 неудачных попыток, первая запись после возврата хука через 3.906 с; раунд 1: 166 записей, 0 неудачных попыток, 3.89 с; в каждом раунде до отказа видны обе реплики, остановленная отсутствует в последних 5 ответах, после restore снова в ротации LB через 3.5 / 2.938 с | `evidence/seq-failover.json` | +| Rolling update под нагрузкой (lab3) | `scripts/rolling_load_evidence.py` (4 потока POST, `scripts/rolling-update.*` — та же процедура) | обе реплики заменены новыми контейнерами за 23.9 с: 1436 успешных, **0 ошибок**, p50/p95/p99 = 16/47/62 мс (предыдущий прогон: 1332 успешных, 0 ошибок; до graceful drain было 857 успешных и 4 × 503) | `evidence/rolling-load.json` | +| Graceful shutdown реплики | `student-tests/test_shutdown.py` (`RUN_DOCKER_TESTS=1`) | после SIGTERM 22 пробы изнутри реплики: `/health/ready` 503, `/users/me` 200, выход через 3.29 с; `docker stop` под нагрузкой — 405 × 201, 0 ошибок, 58 из них обслужила останавливаемая реплика, остановка 3.58 с; загрузка, тело которой ещё шло при закрытии listener, → 201 и скачивается через LB | `evidence/student-tests.txt` | +| Балансировка длинных загрузок | `scripts/lb_interleave_evidence.py` (8 медленных загрузок вперемешку с короткими GET) | `roundrobin`: загрузки 8/0 по репликам; `leastconn`: 4/4; все 201 | `evidence/lb-interleave.json` | +| Слишком большой файл | `test_too_large_413`, `test_exact_limit_png_through_lb_accepted` | 10 485 760 байт → 201 через LB; +1 байт и chunked без длины → JSON 413 | `evidence/student-tests.txt` | +| Пиксели | `test_too_many_pixels_413` | 8000×5000 → 201, 8000×5001 → 413 | `evidence/student-tests.txt` | +| Повреждённое изображение | `test_corrupted_images_415`, `test_reviewer_corrupted_images_415` | 11 повреждённых PNG и 7 JPEG → 415; Adam7 PNG со всеми фильтрами и progressive JPEG → 201 | `evidence/student-tests.txt` | +| Независимая перекрёстная проверка корпуса | `scripts/decoder_crosscheck.py` (Pillow 11.1 + djpeg -strict) | PASS: все повреждённые не декодируются (4 PNG — нарушения спецификации, которые Pillow терпит), все валидные декодируются | `evidence/decoder-crosscheck.txt` | +| Подмена MIME | `test_mime_spoofing_415` | PNG как `image/jpeg` и наоборот, GIF/PDF/HTML/EXE, чужие Content-Type → 415 | `evidence/student-tests.txt` | +| Чужое attachment_id / файл другого чата | `test_foreign_and_unknown_attachment_ids_404` | 404 для обоих, сообщение не создаётся | `evidence/student-tests.txt` | +| Пользовательская ошибка ≠ 500 | хелпер `upload` проверяет статус < 500 во всех негативных тестах | 0 ответов 5xx на пользовательские ошибки | `evidence/student-tests.txt` | +| Прерывание загрузки | `test_interrupted_and_stalled_uploads_leave_nothing` | обрыв и зависание тела → объектов/intent не прибавилось, сервис жив | `evidence/student-tests.txt` | +| Отказ между S3/БД, компенсация, сверка | `selfcheck consistency` (A–F), `reconcile` | все проверки ok; реальный проход: 10 зависших intent (хвосты отказов S3) удалены, повтор — 0; объектов 1382 → 1378, 0 orphan, 0 missing | `evidence/selfcheck-consistency.json`, `evidence/reconcile-run.txt` | +| ≥10 одновременных загрузок, пик памяти | `scripts/upload_memory_experiment.py --levels 10 20 40` | см. эксперимент ниже | `evidence/upload-memory.json` | +| Bucket bootstrap идемпотентен | повторный `s3-init` при каждом `up` | `bucket exists … anonymous access denied (HTTP 403); bootstrap complete` | `evidence/full-restart.json` | +| Отдельные S3 credentials с ограниченными правами | `selfcheck s3` | ключ API: ListBuckets/CreateBucket/DeleteBucket/ACL/чужой bucket → 403 AccessDenied | `evidence/selfcheck-s3.json` | +| Private bucket вне API, консоль закрыта | `selfcheck s3` (из сети storage), `test_bucket_private_and_api_key_least_privilege` | анонимный GET/LIST → 403; порты 8333/18333/8888/18888/9333/19333/8080/18080 закрыты, открыт только S3 API | `evidence/selfcheck-s3.json` | +| Недоступность S3 | `test_s3_stop_degrades_only_attachments` | ready 503; upload 503 за 0.11 с; download 503; метаданные 200 и сообщение 201; после старта S3 — download 200 | `evidence/student-tests.txt` | +| Пауза/остановка PostgreSQL, Redis | `test_postgres_pause_fails_fast`, `test_postgres_stop_returns_503`, `test_redis_stop_returns_503_or_unready` | pause: JSON 503 за 6.14 / 3.03 / 6.22 с (все < 15 с), ready 503, после unpause 200; stop: 503 < 15 с | `evidence/student-tests.txt` | +| Нагрузка (baseline) | `tests/load.py --duration 20 --concurrency 4` | 5319 успешных, 0 ошибок, 265.13 RPS, p50/p95/p99 = 15/32/78 мс | `evidence/load.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Hook-скрипты (`scripts/restart.*`, `scripts/stop-one.*`) действительно перезапускают и останавливают контейнеры. Это видно по `docker compose ps` до и после (`evidence/*-ps-*.txt`): в failover остановленная реплика `Exited (0)`, в persistence заново стартуют postgres, redis, s3 и api. Последнее закреплено тестом `test_restart_hook_restarts_api_and_stores`: он сравнивает `StartedAt` контейнеров до и после хука. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Память при одновременных загрузках** (`evidence/upload-memory.json`). + +Гипотеза: потоковая обработка (64 KiB → валидатор → файл → PUT из файла) удерживает рост RSS процесса API намного ниже N × размер файла, LB не накапливает тела, а S3-сервер выдерживает все PUT, которые пропускают слоты API. + +Методика: + +- PNG 9 960 421 байт, N загрузок одновременно (barrier) через LB; +- перед каждым уровнем — свежий restart `s3`, `s3-gw`, `api` и `lb`, прогрев каждой реплики; +- сброс VmHWM, опрос cgroup (API, LB, S3) каждые 0.25 с; +- 2 реплики по 512m / 0.5 CPU, LB 128m / 0.25 CPU, S3 1g / 0.5 CPU с `GOMEMLIMIT=768MiB`, один прогон на уровень. + +| N | статусы | p50 / max, с | рост RSS API (VmHWM − база), MiB | cgroup peak API, MiB | page cache API, MiB | cgroup peak LB, MiB (лимит 128) | S3: max anon / cgroup peak, MiB (лимит 1024) | +| --- | --- | --- | --- | --- | --- | --- | --- | +| 10 | 10 × 201 | 3.03 / 3.42 | 16 / 16 | 151.8 / 155.2 | 47.5 / 47.5 | 31.8 | 401.3 / 547.6 | +| 20 | 18 × 201, 2 × 503 | 6.16 / 6.36 | 24 / 29 | 192.1 / 181.9 | 76.0 / 76.0 | 40.7 | 541.0 / 734.0 | +| 40 | 18 × 201, 22 × 503 | 6.30 / 7.00 | 28 / 31 | 187.2 / 198.7 | 76.0 / 76.0 | 60.3 | 566.4 / 733.0 | + +Интерпретация: + +- Рост RSS — 16–31 MiB при 5–20 загрузках на реплику. Полная буферизация в RAM дала бы 47.5–76 MiB только на тела (`naive_full_buffer_estimate_mib`), поэтому тела в памяти процесса не держатся. +- Рост cgroup в основном состоит из page cache временных файлов (до 76 MiB = 8 слотов × 9.5 MiB). Это ядро, и cache вытесняем. +- При N = 20 и 40 срабатывает backpressure: слотов 2 × 8, ожидание 5 с, а лишние запросы получают JSON 503 «Too many concurrent uploads» с `Retry-After` вместо роста памяти. Текст 503 проверен отдельным прогоном (`evidence/s3-memory-limit.txt`, блок AFTER): все не-201 ответы — именно этот отказ по слотам, ошибок хранилища нет. +- Балансировка на пике не узкое место. Под `leastconn` на N = 40 — 18 × 201 / 22 × 503, запросы разошлись ровно 20/20 (`uploads_served`): пачка приходит одновременно, и у реплик в каждый момент поровну открытых соединений. Число 201 ограничивают 16 слотов: загрузка 9.5 MiB занимает слот ≈6 с, дольше 5-секундного ожидания, поэтому из очереди успевают единицы. +- `leastconn` выигрывает при смешанном трафике, когда длинные загрузки чередуются с короткими запросами (`scripts/lb_interleave_evidence.py` → `evidence/lb-interleave.json`; 8 загрузок по 1 MiB с телом, которое отправляется ≈4 с, между ними короткие GET; для прогона `roundrobin` в конфиге LB временно возвращалась прежняя стратегия). Round-robin отправил все 8 загрузок на одну реплику, а все GET — на другую (8/0). `leastconn` разложил загрузки 4/4. +- LB не падал (restarts 0, OOMKilled false). Раньше, до фиксации сокетных буферов (`tune.*buf`, sysctl `tcp_rmem/wmem`), он уходил в OOM на 40 загрузках. +- S3 не падал (`running`, OOMKilled false). Один процесс `weed` держит ≈30–40 MiB на каждый идущий PUT 10 MiB. При прежних 512m без `GOMEMLIMIT` он упирался в лимит уже на 10 загрузках и на 20 был убит OOM (exit 137) — см. «Найденные и исправленные по ходу дефекты» и `evidence/s3-memory-limit.txt`. +- Ограничение замера: один хост, один прогон на уровень, клиент на том же хосте. Пик S3 зависит от момента сборки мусора Go и меняется между прогонами: в отдельном прогоне (`s3-memory-limit.txt`, AFTER) max anon составил 481–635 MiB, в этом — 401–566 MiB. + +**Память полного декодирования JPEG** (`evidence/jpeg-decode-memory.txt`; 8000×5000, отдельный процесс на файл): + +| файл | размер | время | пик RSS djpeg | +| --- | --- | --- | --- | +| baseline 4:2:0 | 9.20 MiB | 0.10 с | 10.8 MiB | +| progressive 4:2:0 | 8.09 MiB | 0.53 с | 116.7 MiB | +| progressive 4:4:4 | 8.23 MiB | 0.70 с | 231.0 MiB | +| progressive CMYK | 18.19 MiB | 1.32 с | 307.1 MiB | + +Отсюда лимиты: + +- `prlimit --as=384MiB`, проверено: под 256MiB худший случай даёт «Insufficient memory» → API отвечает 503, а не 415; +- одно декодирование на реплику; +- `mem_limit: 512m` у API. + +CMYK-файл больше 10 MiB и через API не пройдёт. Это верхняя оценка памяти. + +**Отказы:** | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `stop-one` API (публичный failover) | запись через оставшуюся ≤10 с, без дублей | 127 записей, 0 неудачных попыток | запись не прерывалась; хук возвращается через ≈3.6 с (drain), первая засчитанная запись через 3.95 с; подтверждённые данные целы | +| `stop-one` каждой из двух реплик по очереди | запись через оставшуюся, остановленная не отвечает | 168 + 166 записей, 0 неудачных попыток | первая запись после хука через 3.906 / 3.89 с; после restore реплика в ротации через 3.5 / 2.9 с | +| Rolling update: stop → rm → новый контейнер, по одной реплике | сервис не прерывается | 1436 × 201, 0 ошибок | следующая реплика останавливается только после того, как LB отдаёт ответы новой | +| `docker stop` реплики под нагрузкой (`test_shutdown`) | ни одной ошибки, реплика дослуживает окно drain | 405 × 201, 0 ошибок, 58 ответов от останавливаемой | остановка 3.58 с, после `start` в ротации через 2.5 с | +| Stop реплики, загрузившей файл | download через другую | 0 ошибок за 2 × 20 скачиваний | 3.625 / 3.64 с от начала `docker stop` (остановка 3.59 / 3.62 с) | +| `docker compose down` → `up` (тома сохранены) | те же token, файл, текст | всё совпало | down 15.0 с, 45.6 с до ready; потерь нет | +| `docker restart` контейнера `s3` / падение процесса `weed` | пара `s3` + `s3-gw` поднимается сама | `s3-gw` выходит из старого namespace, restart policy запускает его в новом; upload 201 и download 200 с тем же SHA-256 | ready 200 через 37 / 29 с; потерь нет (`evidence/s3-restart.txt`) | +| `docker compose stop s3` | ready 503, вложения 503, чат работает | так и есть, upload 503 за 0.11 с | после `start s3` download 200, хвост intent убран reconciler | +| `docker compose pause postgres` | JSON 503 за ограниченное время, восстановление | каждый запрос — 503 не дольше ≈6.3 с: watchdog 6 с, ожидание пула 3 с, пока breaker открыт — сразу (в прогоне 6.14 / 3.03 / 6.22 с); ready 503 | после `unpause` 200, перезапусков реплик 0 | +| `docker compose stop postgres` / `stop redis` | 503 JSON, восстановление | 503 < 15 с; logout при упавшем Redis → 503 (не ложный 204) | 200 после start | +| 40 одновременных загрузок 9.5 MiB | без OOM, лишние — 503 | 18 × 201, 22 × 503; LB peak 60.3 MiB, S3 anon 566.4 MiB из 1024 | LB и S3 без рестартов | -## Ограничения и следующие шаги +Найденные и исправленные по ходу дефекты: -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Двойной `close()` дескриптора в watchdog БД.** uvloop падал с `uv__io_poll: Assertion errno == EEXIST`, реплика перезапускалась и отдавала 502. После исправления тест паузы прошёл 3 раза отдельно и ещё в двух полных прогонах, без рестартов реплик. +- **`timeout connect 3s` в HAProxy был больше 2-секундного таймаута клиента** в публичном failover-тесте. Стало `1s` + `observe layer4`. +- **Неверный путь при остановленном S3.** Раньше upload давал 504 через 30 с из-за медленного DNS, теперь 503 не дольше 2 с (в последнем прогоне — 0.11 с). +- **OOM SeaweedFS при параллельных загрузках.** Слоты API пропускают до 2 × 8 = 16 одновременных PUT, а один `weed` держит ≈30–40 MiB на каждый. С `mem_limit: 512m` без мягкого лимита Go anon доходил до 494 MiB уже при N = 10, а при N = 20 контейнер `s3` был убит OOM (exit 137). Загрузки получали 503 «Attachment storage unavailable», хвосты intent потом убирал reconciler. Стало: `mem_limit: 1g` и `GOMEMLIMIT=768MiB` — OOM нет, max anon 481–635 MiB (`evidence/s3-memory-limit.txt`, блоки BEFORE/AFTER). Тест: `test_parallel_large_uploads_keep_s3_within_memory`. +- **Ёмкость S3 была ограничена 9 томами по 512 MiB (≈4.7 GiB) при свободном диске.** Из 16 слотов `-volume.max=16` 7 занимает служебный collection по умолчанию. Когда 9 томов бакета заполнились, каждый PUT получал от SeaweedFS «No writable volumes», и API отвечал 503 при `/health/ready` = 200. Стало `-volume.max=0`: SeaweedFS считает лимит как свободное место ÷ размер тома, на стенде 1896 слотов, свободно 1873 (`evidence/s3-volumes.json`). Тест: `test_s3_can_grow_volumes`. +- **Хук `scripts/restart.*` не перезапускал API.** `up -d --scale` не трогает уже работающие реплики, поэтому публичный persistence проверял перезапуск хранилищ без перезапуска API (tests/README требует «API и всех используемых хранилищ»). Добавлен `docker compose restart api`. Тест: `test_restart_hook_restarts_api_and_stores` сравнивает `StartedAt` до и после. +- **Rolling update из lab3 не заменял реплики и не дожидался новой.** `docker stop` + `up --scale api=2` просто запускал тот же остановленный контейнер, а готовность проверялась по публичному `/health/ready`, который отвечает 200 через оставшуюся реплику. Поэтому вторая реплика останавливалась, пока первая ещё стартовала: 10 ошибок за прогон, в конце одна реплика не была healthy. Стало: `stop` → `rm` → `up --no-recreate --scale api=N` создаёт новый контейнер и не трогает работающий; следующий шаг начинается, когда LB отдаёт ответы с `X-Instance-Id` нового контейнера. То же ожидание добавлено в restore `seq_failover_evidence.py`. Итог: вместо 10 ошибок оставались единичные JSON 503 от останавливаемой реплики (их убрал graceful drain, следующий пункт), ID контейнеров до и после различаются (`evidence/rolling-load.json`, поле `replaced`). +- **Останавливаемая реплика отвечала JSON 503 на живой трафик.** По SIGTERM она сразу снимала readiness и отвечала 503 «Service unavailable» на все запросы, а uvicorn закрывал listener. HAProxy же ещё 1–2 с (`inter 1s fall 2`) направлял на неё запросы. Быстрые 503 не держат соединений, поэтому `leastconn` отправлял на неё заметную долю трафика: rolling update под нагрузкой давал 4 ошибки на 857 успешных, под round-robin — 1. Стало: окно drain (`lifecycle.DrainingServer`, см. «Решения lab4»), в нём 503 отдаёт только `/health/ready`. Результат: rolling update — 0 ошибок в двух прогонах (1332 и 1436 успешных), failover — 0 неудачных попыток вместо 1. Тесты `test_shutdown.*`: на прежней сборке `test_replica_stop_under_load_has_no_errors` получил 4 × 503, а `test_sigterm_drains_readiness_but_keeps_serving` — выход реплики сразу после SIGTERM; на текущей оба проходят. +- **Filename с невалидным UTF-8 молча переписывался.** Starlette заменяет такие байты в `query_params` на U+FFFD, поэтому `?filename=%FF%FE.png` превращался в `"\ufffd\ufffd.png"` и проходил проверку, а из повторённых `filename` молча брался последний. Теперь query разбирается строго (`attachments.query_filename`): невалидный UTF-8 (включая суррогаты `%ED%A0%80`) и повтор `filename` → 400. Тест: `test_filename_query_decoded_strictly_400`. +- **После рестарта `s3` хранилище вложений не возвращалось.** У `s3` и `s3-gw` не было restart policy, и после рестарта Docker они не поднялись бы сами. Кроме того, после `docker restart s3` контейнер `s3-gw` оставался в старом network namespace: `s3` был healthy, а `/health/ready` больше 90 с отвечал 503. Healthcheck `s3-gw` (`haproxy -c`) при этом показывал healthy. Стало: `restart: unless-stopped` и entrypoint-сторож `s3-gw` (см. «Решения lab4»). `docker restart s3` → ready 200 через 37 с, падение процесса `weed` → через 29 с; upload и download после этого работают (`evidence/s3-restart.txt`). + +## Ответы на вопросы защиты + +- **Почему файл в Docker volume одной API-реплики не равен объектному хранилищу?** + - Volume принадлежит одному контейнеру и хосту. Другая реплика его не видит: download после остановки загрузившей реплики вернул бы 404 или 500. Масштабирование и замена контейнера тоже теряют файл. + - У такого volume нет независимого жизненного цикла, ключей доступа, политики bucket и общего для всех реплик namespace. + - S3 — отдельный сервис с собственным томом, к нему одинаково обращаются все реплики. Поэтому failover вложений и работает (`attachment-failover.json`). +- **Когда можно честно вернуть 201?** + - Только когда объект целиком и успешно записан в S3 (PUT подтверждён, размер и SHA-256 посчитаны по реально прочитанным байтам) и транзакция с метаданными `ready` закоммичена в PostgreSQL. + - После этого любая реплика отдаёт файл побайтно. 201 до PUT или до COMMIT обещал бы данные, которых может не быть. +- **Что, если S3 PUT прошёл, а COMMIT в PostgreSQL — нет?** + - Если откат достоверен (ошибка до COMMIT или транзакция не начиналась), API удаляет объект (компенсация) и intent, а клиент получает 503. + - Если связь оборвалась во время самого COMMIT, исход неизвестен. Объект **не** удаляется (`CommitUncertain`), иначе могли бы остаться метаданные `ready` без байтов. + - Если COMMIT всё-таки прошёл, intent удалён в той же транзакции и объект защищён ссылкой из `attachments`. Если не прошёл, осталась строка intent, и reconciler удалит объект после grace 900 с. + - Если процесс упал между PUT и COMMIT, тоже остаётся intent → reconciler. Объект без всякой ссылки удаляет sweep, но только когда он старше grace по `LastModified`. +- **Почему случайный object key и приватный bucket не заменяют ACL приложения?** + - Приватный bucket защищает от анонимного доступа к S3, но не различает пользователей приложения: у API один ключ на всех. + - Случайный ключ — это секрет-ссылка. Его не отозвать, он может утечь (логи, пересланная ссылка), а бывший участник чата продолжал бы им пользоваться. + - Решение «кто может видеть файл» зависит от членства в чате прямо сейчас, и знает это только приложение. Поэтому каждый download идёт через API с проверкой сессии и членства в чате вложения: посторонний получает 404 и на метаданные, и на bytes. + +## Ограничения + +- **Один Docker-хост, по одному экземпляру PostgreSQL, Redis, SeaweedFS и LB.** Отказ API-реплики выдерживается, отказ хоста, БД или S3 — нет. При недоступном S3 работает только текстовая часть. +- **Тело загрузки (≤10 MiB) буферизуется во временный файл на реплике** до PUT, прямой потоковой передачи в S3 нет. Page cache этих файлов учитывается в cgroup контейнера. +- **Вызовы БД синхронные** (psycopg в threadpool, часть — в event loop). При зависшей БД запрос ждёт до 6 с (watchdog), дальше breaker отвечает сразу. +- **`s3-gw` привязан к контейнеру `s3`.** Падение или `docker restart` `s3` пара переживает сама (см. «Решения lab4»). Есть два исключения: + - после пересоздания одного `s3` (`up -d --force-recreate s3`) у него новый ID, и `s3-gw` не может к нему присоединиться. Нужен обычный `docker compose --env-file .env up -d`, который пересоздаёт и `s3-gw`; + - Docker не повторяет старт, который не удался из-за того, что `s3` ещё не запущен. Если после рестарта Docker-демона `s3-gw` окажется в `Exited`, его поднимает тот же `docker compose up -d`. Рестарт самого демона на стенде не проверялся. + + `docker compose stop s3` + `start s3` (как в `test_s3_stop_degrades_only_attachments`) — ручная остановка: restart policy на неё не действует, поэтому тест после `start s3` делает `restart s3-gw`. +- **Память и ёмкость S3.** `s3` получает 1g и `GOMEMLIMIT=768MiB` под 16 одновременных PUT (2 реплики × 8 слотов). При увеличении числа реплик или слотов лимит надо пересчитать (≈30–40 MiB на PUT). Число томов ограничено только свободным местом (`-volume.max=0`). Readiness не проверяет свободное место: при заполненном диске загрузки дадут 503 при ready 200. +- **Остановка реплики занимает ≈3.6 с** из-за окна drain, поэтому `stop-one`, rolling update и `docker compose down` идут на ≈3 с дольше. Окно рассчитано на health check HAProxy `inter 1s fall 2`: при изменении этих параметров надо пересчитать `SHUTDOWN_DRAIN_SECONDS` и `stop_grace_period`. Запрос, который не уложился в 8 с после закрытия listener (например, медленный клиент загрузки), обрывается. Это равносильно падению процесса: клиент не получает ответа и повторяет запрос (POST сообщения — с тем же `Idempotency-Key`), хвост intent убирает reconciler. Отказ реплики без SIGTERM (`kill -9`, OOM) по-прежнему может дать единичные ошибки, пока HAProxy её не исключит. +- **Конфиги HAProxy примонтированы файлом.** После правки нужен `docker compose restart lb`. +- **PNG проверяется строже Pillow** по 4 правилам спецификации (PLTE и неизвестный критический чанк), как libpng. Кроме того, отвергаются (415) PNG с данными после IEND и без IEND, хотя Pillow такие открывает. Это сознательная политика: поток PNG по спецификации заканчивается на IEND. Для JPEG данные после EOI допускаются, как в распространённых декодерах. +- **Ошибки разбора HTTP самим HAProxy (400/408 на некорректный запрос) отдаются его стандартной HTML-страницей.** `http-error` на ошибки уровня H1-парсера не действует. 413 формирует API (лимит тела у LB не задан), а 502/503/504 самого LB — `http-error` с JSON, `X-Request-Id` и `X-Instance-Id`. +- **Если в JPEG-декодере не хватило памяти, это 503, а не 415:** это ресурсный отказ, а не ошибка пользователя. +- **Windows:** публичные тесты требуют `PYTHONUTF8=1`; `make` не установлен — использовались эквивалентные команды из Makefile. +- **Трек 5 не выполнялся.** Reconciler с dry-run есть, но исследование трека 2 не заявляется. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f0faea77 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,304 @@ +# Lab4: Client → HAProxy → API×N → PostgreSQL (metadata) + Redis (sessions) +# ↘ private S3 bucket (SeaweedFS, bytes only). +# Public entrypoint only via lb:8080. API replicas and S3 publish no host ports. +services: + postgres: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: messenger + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + restart: unless-stopped + mem_limit: 512m + cpus: 0.50 + networks: + - labnet + + redis: + image: redis:7.4.2-bookworm + command: + - redis-server + - --appendonly + - "yes" + - --appendfsync + - everysec + - --dir + - /data + volumes: + - redisdata:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + restart: unless-stopped + mem_limit: 128m + cpus: 0.25 + networks: + - labnet + + # SeaweedFS all-in-one (master + volume + filer + S3 gateway). Every listener, + # including the master/filer web UIs and the unauthenticated gRPC ports, is bound + # to 127.0.0.1 inside this container; only s3-gw exposes the S3 HTTP API. + s3: + image: chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e + command: + - server + - -dir=/data + - -ip=127.0.0.1 + - -ip.bind=127.0.0.1 + - -s3 + - -s3.port=8333 + - -s3.ip.bind=127.0.0.1 + - -s3.config=/run/secrets/s3_config + - -filer.exposeDirectoryData=false + - -master.volumeSizeLimitMB=512 + - -master.telemetry=false + # 0 = free disk / volume size. A fixed count caps the bucket: with 16 slots + # (7 taken by the default collection) uploads failed after ≈4.7 GiB. + - -volume.max=0 + environment: + # One weed process holds ≈30–40 MiB per in-flight 10 MiB PUT (s3 → filer → volume); + # API slots cap S3 at replicas × MAX_CONCURRENT_UPLOADS PUTs (16 → ≈500 MiB live). + # With 512m and no soft limit 10–40 parallel uploads OOM-killed weed (exit 137). + GOMEMLIMIT: 768MiB + secrets: + - s3_config + volumes: + - s3data:/data + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8333/healthz"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 15s + stop_grace_period: 20s + restart: unless-stopped + mem_limit: 1g + cpus: 0.50 + networks: + storage: + aliases: + - s3 + + # TCP sidecar in the s3 network namespace: publishes ONLY the S3 HTTP API as s3:9000. + # A restart of s3 gives it a new network namespace; the entrypoint then exits and + # `restart: unless-stopped` starts s3-gw again inside the new one (infra/s3-gw-entrypoint.sh). + # Compose-driven restarts/recreation of s3 also restart s3-gw (depends_on restart: true). + s3-gw: + image: haproxy:2.9.14-bookworm + network_mode: "service:s3" + entrypoint: ["/bin/sh", "/usr/local/bin/s3-gw-entrypoint.sh"] + stop_signal: SIGTERM + volumes: + - ./infra/s3-gateway.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./infra/s3-gw-entrypoint.sh:/usr/local/bin/s3-gw-entrypoint.sh:ro + depends_on: + s3: + condition: service_healthy + restart: true + healthcheck: + # Config valid AND still attached to the live namespace of s3 (not just `lo`). + test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && grep -qv -e '^ *lo:' -e '|' /proc/net/dev"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 5s + restart: unless-stopped + mem_limit: 64m + cpus: 0.25 + + # One-shot, idempotent bucket bootstrap with the S3 admin identity (API never gets it). + s3-init: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab4 + command: ["python", "-m", "messenger.s3bootstrap"] + environment: + S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}" + S3_REGION: "${S3_REGION:-us-east-1}" + S3_BUCKET: "${S3_BUCKET:-messenger-attachments}" + S3_ADMIN_ACCESS_KEY_ID_FILE: /run/secrets/s3_admin_access_key_id + S3_ADMIN_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_admin_secret_access_key + secrets: + - s3_admin_access_key_id + - s3_admin_secret_access_key + depends_on: + s3-gw: + condition: service_started + restart: "no" + dns_opt: + - ndots:0 + - timeout:1 + - attempts:1 + mem_limit: 128m + cpus: 0.25 + networks: + - storage + + migrate: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab4 + command: ["python", "-m", "messenger.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + MIGRATIONS_DIR: /app/migrations + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + restart: "no" + mem_limit: 256m + cpus: 0.25 + networks: + - labnet + + api: + build: + context: . + dockerfile: Dockerfile + image: mtusi-messenger:lab4 + environment: + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + APP_ENV: lab + SESSION_TTL_SECONDS: "${SESSION_TTL_SECONDS:-3600}" + IDEMPOTENCY_TTL_SECONDS: "${IDEMPOTENCY_TTL_SECONDS:-86400}" + SESSION_STORE: redis + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + MIGRATIONS_DIR: /app/migrations + S3_ENDPOINT: "${S3_ENDPOINT:-http://s3:9000}" + S3_REGION: "${S3_REGION:-us-east-1}" + S3_BUCKET: "${S3_BUCKET:-messenger-attachments}" + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + MAX_UPLOAD_BYTES: "${MAX_UPLOAD_BYTES:-10485760}" + MAX_IMAGE_PIXELS: "${MAX_IMAGE_PIXELS:-40000000}" + MAX_CONCURRENT_UPLOADS: "${MAX_CONCURRENT_UPLOADS:-8}" + UPLOAD_IDLE_TIMEOUT_SECONDS: "${UPLOAD_IDLE_TIMEOUT_SECONDS:-10}" + UPLOAD_DEADLINE_SECONDS: "${UPLOAD_DEADLINE_SECONDS:-120}" + UPLOAD_GRACE_SECONDS: "${UPLOAD_GRACE_SECONDS:-900}" + RECONCILE_INTERVAL_SECONDS: "${RECONCILE_INTERVAL_SECONDS:-300}" + SHUTDOWN_DRAIN_SECONDS: "${SHUTDOWN_DRAIN_SECONDS:-3}" + SHUTDOWN_TIMEOUT_SECONDS: "${SHUTDOWN_TIMEOUT_SECONDS:-8}" + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + migrate: + condition: service_completed_successfully + s3-init: + condition: service_completed_successfully + redis: + condition: service_healthy + # No host ports: only reachable via labnet / HAProxy. + expose: + - "8080" + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)", + ] + interval: 5s + timeout: 3s + retries: 12 + start_period: 20s + # SIGTERM → drain 3s (ready 503, still serving until HAProxy inter 1s × fall 2 drops + # it) → in-flight requests ≤ 8s → lifespan shutdown ≤ 2s; 15s leaves margin before SIGKILL. + # Scripts that call `docker stop` directly use the same `-t 15`. + stop_grace_period: 15s + restart: unless-stopped + # While s3 is stopped its DNS alias is gone and a lookup takes ~8 s with defaults, + # i.e. >24 s over boto3 retries (HAProxy 504 instead of a fast API 503). + dns_opt: + - ndots:0 + - timeout:1 + - attempts:1 + # 512m: one full JPEG decode per replica may need ≈310 MiB (40 Mpx progressive CMYK). + mem_limit: 512m + cpus: 0.50 + # Socket buffers count against the cgroup; uncapped autotuning (MiB per connection) + # lets queued uploads push the replica towards its memory limit. + sysctls: + net.ipv4.tcp_rmem: "4096 131072 262144" + net.ipv4.tcp_wmem: "4096 16384 262144" + networks: + - labnet + - storage + + lb: + image: haproxy:2.9.14-bookworm + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + api: + condition: service_healthy + healthcheck: + # Image has no curl/wget; validate config + process. Public readiness is /health/ready via LB. + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 5 + start_period: 5s + restart: unless-stopped + mem_limit: 128m + cpus: 0.25 + sysctls: + net.ipv4.tcp_rmem: "4096 131072 262144" + net.ipv4.tcp_wmem: "4096 16384 262144" + networks: + - labnet + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_config: + file: ./secrets/s3_config.json + s3_admin_access_key_id: + file: ./secrets/s3_admin_access_key_id + s3_admin_secret_access_key: + file: ./secrets/s3_admin_secret_access_key + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key + +volumes: + pgdata: + redisdata: + s3data: + +networks: + labnet: + driver: bridge + # Storage segment: no route outside Docker, only api and s3-init can reach s3:9000. + storage: + driver: bridge + internal: true diff --git a/evidence/attachment-failover.json b/evidence/attachment-failover.json new file mode 100644 index 00000000..59151b01 --- /dev/null +++ b/evidence/attachment-failover.json @@ -0,0 +1,54 @@ +{ + "date": "2026-11-06T15:55:28Z", + "passed": true, + "rounds": [ + { + "round": 0, + "upload_served_by": "f06ef1999af2", + "stopped_container_state_after": "/mtusi2026lab4-api-21 exited", + "docker_stop_seconds": 3.59, + "first_download_ok_after_stop_s": 3.625, + "downloads": 20, + "download_instances": [ + "6fe989e050e5" + ], + "target_served_any_download": false, + "all_sha256_match": true, + "errors_during_window": 0, + "metadata_status": 200, + "message_link_preserved": true, + "containers_before": { + "f06ef1999af2": "/mtusi2026lab4-api-21 running", + "6fe989e050e5": "/mtusi2026lab4-api-20 running" + }, + "containers_after": { + "f06ef1999af2": "/mtusi2026lab4-api-21 exited", + "6fe989e050e5": "/mtusi2026lab4-api-20 running" + } + }, + { + "round": 1, + "upload_served_by": "6fe989e050e5", + "stopped_container_state_after": "/mtusi2026lab4-api-20 exited", + "docker_stop_seconds": 3.62, + "first_download_ok_after_stop_s": 3.64, + "downloads": 20, + "download_instances": [ + "f06ef1999af2" + ], + "target_served_any_download": false, + "all_sha256_match": true, + "errors_during_window": 0, + "metadata_status": 200, + "message_link_preserved": true, + "containers_before": { + "f06ef1999af2": "/mtusi2026lab4-api-21 running", + "6fe989e050e5": "/mtusi2026lab4-api-20 running" + }, + "containers_after": { + "f06ef1999af2": "/mtusi2026lab4-api-21 running", + "6fe989e050e5": "/mtusi2026lab4-api-20 exited" + } + } + ] +} \ No newline at end of file diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/decoder-crosscheck.txt b/evidence/decoder-crosscheck.txt new file mode 100644 index 00000000..143c9338 --- /dev/null +++ b/evidence/decoder-crosscheck.txt @@ -0,0 +1,28 @@ +corrupt png first_33_bytes_of_sample_png pillow_decodes=False +corrupt png sample_png_truncated_half pillow_decodes=False +corrupt png broken_idat_stream_valid_crc pillow_decodes=False +corrupt png filter_byte_05 pillow_decodes=False +corrupt png filter_byte_ff pillow_decodes=False +corrupt png interlaced_filter_byte_07 pillow_decodes=False +corrupt png plte_257_entries pillow_decodes=False +corrupt png plte_not_multiple_of_3 pillow_decodes=True (PNG spec violation tolerated by Pillow) +corrupt png plte_too_long_for_1bit pillow_decodes=True (PNG spec violation tolerated by Pillow) +corrupt png plte_in_grayscale pillow_decodes=True (PNG spec violation tolerated by Pillow) +corrupt png unknown_critical_chunk pillow_decodes=True (PNG spec violation tolerated by Pillow) +corrupt jpeg jpeg_27_bytes_no_tables djpeg_strict_decodes=False pillow_decodes=False +corrupt jpeg jpeg_18_bytes djpeg_strict_decodes=False pillow_decodes=False +corrupt jpeg jpeg_no_data_after_sos djpeg_strict_decodes=False pillow_decodes=True +corrupt jpeg jpeg_half_plus_eoi djpeg_strict_decodes=False pillow_decodes=True +corrupt jpeg progressive_half_plus_eoi djpeg_strict_decodes=False pillow_decodes=True +corrupt jpeg jpeg_truncated_no_eoi djpeg_strict_decodes=False pillow_decodes=False +corrupt jpeg jpeg_12_bit djpeg_strict_decodes=False pillow_decodes=False +valid png all_filters interlace=0 (1, 1) pillow_decodes=True +valid png all_filters interlace=0 (9, 7) pillow_decodes=True +valid png all_filters interlace=0 (33, 20) pillow_decodes=True +valid png all_filters interlace=1 (1, 1) pillow_decodes=True +valid png all_filters interlace=1 (9, 7) pillow_decodes=True +valid png all_filters interlace=1 (33, 20) pillow_decodes=True +valid jpeg sample.jpg djpeg_strict_decodes=True +valid jpeg progressive.jpg djpeg_strict_decodes=True +valid jpeg gray.jpg djpeg_strict_decodes=True +PASS diff --git a/evidence/failover-ps-after.txt b/evidence/failover-ps-after.txt new file mode 100644 index 00000000..5e1c88b2 --- /dev/null +++ b/evidence/failover-ps-after.txt @@ -0,0 +1,3 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-20 mtusi-messenger:lab4 "python -m messenger" api About a minute ago Up 56 seconds (healthy) 8080/tcp +mtusi2026lab4-api-21 mtusi-messenger:lab4 "python -m messenger" api About a minute ago Exited (0) 11 seconds ago diff --git a/evidence/failover-ps-before.txt b/evidence/failover-ps-before.txt new file mode 100644 index 00000000..d20f098b --- /dev/null +++ b/evidence/failover-ps-before.txt @@ -0,0 +1,8 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-20 mtusi-messenger:lab4 "python -m messenger" api About a minute ago Up 39 seconds (healthy) 8080/tcp +mtusi2026lab4-api-21 mtusi-messenger:lab4 "python -m messenger" api About a minute ago Up 14 seconds (healthy) 8080/tcp +mtusi2026lab4-lb-1 haproxy:2.9.14-bookworm "docker-entrypoint.s…" lb About an hour ago Up 22 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi2026lab4-postgres-1 postgres:16.6-bookworm "docker-entrypoint.s…" postgres About an hour ago Up 23 minutes (healthy) 5432/tcp +mtusi2026lab4-redis-1 redis:7.4.2-bookworm "docker-entrypoint.s…" redis About an hour ago Up 23 minutes (healthy) 6379/tcp +mtusi2026lab4-s3-1 chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e "/entrypoint.sh serv…" s3 About an hour ago Up 22 minutes (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi2026lab4-s3-gw-1 haproxy:2.9.14-bookworm "docker-entrypoint.s…" s3-gw About an hour ago Up 22 minutes (healthy) diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..e9a4fbfd --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 3.954} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/full-restart.json b/evidence/full-restart.json new file mode 100644 index 00000000..26b1b0cd --- /dev/null +++ b/evidence/full-restart.json @@ -0,0 +1,43 @@ +{ + "date": "2026-11-06T15:56:38Z", + "volumes_kept": [ + "mtusi2026lab4_pgdata", + "mtusi2026lab4_redisdata", + "mtusi2026lab4_s3data" + ], + "containers_before": [ + "api 6fe989e050e5 running", + "api f06ef1999af2 running", + "lb 183591d00904 running", + "migrate 68b4bf1c5455 exited", + "postgres f6b73da81ed7 running", + "redis 5a5ab8e1041d running", + "s3 d5b4869af5dd running", + "s3-gw c7856d235d02 running", + "s3-init cdec525d5e35 exited" + ], + "containers_after_down": [], + "containers_after_up": [ + "api 44c61b702da1 running", + "api 68b28e462825 running", + "lb 9e77728a6f63 running", + "migrate a363cd1632ff exited", + "postgres 5727ea390758 running", + "redis 1ca8a7f2bf09 running", + "s3 9fe1e4f2320a running", + "s3-gw 4b90a95b8340 running", + "s3-init 6d80e51d7aaa exited" + ], + "down_seconds": 15.0, + "down_to_ready_seconds": 45.6, + "same_token_works": true, + "metadata_unchanged": true, + "download_status": 200, + "sha256_match": true, + "message_preserved": true, + "s3_init_rerun_log_tail": [ + "2026-11-06 15:56:30,778 INFO bucket exists: messenger-attachments", + "2026-11-06 15:56:30,869 INFO anonymous access denied (HTTP 403); bootstrap complete" + ], + "passed": true +} \ No newline at end of file diff --git a/evidence/images.txt b/evidence/images.txt new file mode 100644 index 00000000..4eff19b2 --- /dev/null +++ b/evidence/images.txt @@ -0,0 +1,10 @@ +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +mtusi2026lab4-api-1 mtusi-messenger lab4 linux/amd64 60bb1bccb028 317MB 17 minutes ago +mtusi2026lab4-api-2 mtusi-messenger lab4 linux/amd64 60bb1bccb028 317MB 17 minutes ago +mtusi2026lab4-lb-1 haproxy 2.9.14-bookworm linux/amd64 2b1e58ff2368 154MB 21 months ago +mtusi2026lab4-migrate-1 mtusi-messenger lab4 linux/amd64 60bb1bccb028 317MB 17 minutes ago +mtusi2026lab4-postgres-1 postgres 16.6-bookworm linux/amd64 557fea37a744 611MB 21 months ago +mtusi2026lab4-redis-1 redis 7.4.2-bookworm linux/amd64 fbdbaea47b9a 173MB 22 months ago +mtusi2026lab4-s3-1 chrislusf/seaweedfs linux/amd64 bb05d66d2963 219MB 14 months ago +mtusi2026lab4-s3-gw-1 haproxy 2.9.14-bookworm linux/amd64 2b1e58ff2368 154MB 21 months ago +mtusi2026lab4-s3-init-1 mtusi-messenger lab4 linux/amd64 60bb1bccb028 317MB 17 minutes ago diff --git a/evidence/jpeg-decode-memory.txt b/evidence/jpeg-decode-memory.txt new file mode 100644 index 00000000..7ecb054a --- /dev/null +++ b/evidence/jpeg-decode-memory.txt @@ -0,0 +1,4 @@ +baseline_420.jpg 9.20 MiB rc=0 0.10s djpeg_peak_rss=10.8 MiB +progressive_420.jpg 8.09 MiB rc=0 0.53s djpeg_peak_rss=116.7 MiB +progressive_444.jpg 8.23 MiB rc=0 0.70s djpeg_peak_rss=231.0 MiB +progressive_cmyk.jpg 18.19 MiB rc=0 1.32s djpeg_peak_rss=307.1 MiB diff --git a/evidence/lb-interleave.json b/evidence/lb-interleave.json new file mode 100644 index 00000000..33d99a60 --- /dev/null +++ b/evidence/lb-interleave.json @@ -0,0 +1,34 @@ +{ + "roundrobin": { + "date": "2026-11-06T15:25:32Z", + "balance": "roundrobin", + "uploads": 8, + "upload_bytes": 1050888, + "uploads_per_replica": { + "bbea736d1b58": 8 + }, + "upload_statuses": { + "201": 8 + }, + "short_gets_per_replica": { + "807474b42967": 8 + } + }, + "leastconn": { + "date": "2026-11-06T15:25:44Z", + "balance": "leastconn", + "uploads": 8, + "upload_bytes": 1050888, + "uploads_per_replica": { + "bbea736d1b58": 4, + "807474b42967": 4 + }, + "upload_statuses": { + "201": 8 + }, + "short_gets_per_replica": { + "807474b42967": 4, + "bbea736d1b58": 4 + } + } +} \ No newline at end of file diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..79a63773 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 4, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 20.0, + "elapsed_seconds": 20.062, + "concurrency": 4, + "successful_requests": 5319, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 265.128, + "successful_latency_ms": { + "p50": 15.0, + "p95": 32.0, + "p99": 78.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/meta.txt b/evidence/meta.txt new file mode 100644 index 00000000..caf9f566 --- /dev/null +++ b/evidence/meta.txt @@ -0,0 +1,70 @@ +Lab4 evidence — сбор 2026-11-06 (локальная машина студента) + +Ветка: solution/bvt2303-bolotov +Хост: Windows 10.0.26300, PowerShell 5, Docker Desktop (WSL2) +Docker Engine: 29.9.2 +Docker Compose: v5.6.1 +Python (хост): 3.12.10 (PYTHONUTF8=1 для тестов) +make: не установлен — цели Makefile запускались эквивалентными командами python +BASE_URL: http://127.0.0.1:8080 (на Windows "localhost" сначала пробует ::1 → +2 с на каждое + соединение; LB опубликован только на 127.0.0.1) + +Образы (evidence/images.txt): + mtusi-messenger:lab4 python:3.12.8-slim-bookworm, boto3 1.35.99, libjpeg-turbo-progs 1:2.1.5-2 + (djpeg 2.1.5), util-linux prlimit; api, migrate, s3-init + postgres:16.6-bookworm 557fea37a744 + redis:7.4.2-bookworm fbdbaea47b9a + haproxy:2.9.14-bookworm 2b1e58ff2368 (lb и s3-gw) + chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e + +Финальная сборка API: 60bb1bccb028 (graceful shutdown с окном drain; модули src/messenger в контейнере +совпадают с рабочим деревом по SHA-256), конфигурация LB balance leastconn, api stop_grace_period 15s; +s3: 1g, GOMEMLIMIT=768MiB, -volume.max=0 — smoke.txt, persistence*.txt, failover*.txt, student-tests.txt, + seq-failover.json, rolling-load.json, attachment-failover.json, full-restart.json, load.json, images.txt; + s3/s3-gw с restart: unless-stopped и entrypoint-сторожем s3-gw — s3-restart.txt, smoke.txt, + persistence*.txt, student-tests.txt (сняты повторно после этого изменения) + Предыдущая сборка 546479b6b925 (отличается только обработкой SIGTERM: readiness и все запросы сразу 503, + без окна drain; на эти сценарии не влияет), LB balance leastconn — upload-memory.json, + lb-interleave.json (ключ "leastconn") + Та же сборка 546479b6b925, LB ещё balance roundrobin (балансировка на результат не влияет) — check.txt, + validate.txt, selfcheck-*.json, reconcile-run.txt, s3-memory-limit.txt, s3-volumes.json + lb-interleave.json, ключ "roundrobin" — тот же стенд с временно возвращённым balance roundrobin (для сравнения) + До смены -volume.max (s3 уже 1g + GOMEMLIMIT; сценарий не загружает файлы) — session-expiry.txt + Не зависят от сборки API — decoder-crosscheck.txt (корпус тестов против Pillow/djpeg), + jpeg-decode-memory.txt (djpeg 2.1.5 из того же образа) + +Лимиты: + api ×2: mem 512m, 0.5 CPU, MAX_CONCURRENT_UPLOADS=8 на реплику, ожидание слота 5 с → 503 + полная проверка JPEG: djpeg -strict под prlimit --as=384MiB, 1 одновременно на реплику, 15 с + lb 128m / 0.25 CPU; s3 1g / 0.5 CPU, GOMEMLIMIT=768MiB, -volume.max=0 (тома 512 MiB); s3-gw 64m; postgres 512m; redis 128m; migrate 256m; s3-init 128m + MAX_UPLOAD_BYTES=10485760, MAX_IMAGE_PIXELS=40000000 + UPLOAD_IDLE_TIMEOUT_SECONDS=10, UPLOAD_DEADLINE_SECONDS=120 + UPLOAD_GRACE_SECONDS=900, RECONCILE_INTERVAL_SECONDS=300 + PostgreSQL: дедлайн занятого соединения 6 с (watchdog), ожидание пула 3 с, breaker 3 с, ping 1.5 с + S3 (boto3): connect 2 с, read 5 с, 2 попытки; ping 1/1.5 с, кэш 1 с + HAProxy: balance leastconn, timeout connect 1s, server 45s, check 1.5s, inter 1s fall 2 rise 1, observe layer4, retries 2 + redispatch + Остановка API: SIGTERM → /health/ready 503, запросы обслуживаются SHUTDOWN_DRAIN_SECONDS=3 → listener закрыт, + идущие запросы ≤ SHUTDOWN_TIMEOUT_SECONDS=8 → lifespan ≤2 с; stop_grace_period 15s, docker stop -t 15 + sysctl api/lb: net.ipv4.tcp_rmem/tcp_wmem max 262144; dns_opt timeout:1 attempts:1 (api, s3-init) + +Файлы: + check.txt, validate.txt, smoke.txt — публичные make check / validate / test + persistence*.txt, failover*.txt — tests/resilience.py (+ docker compose ps до/после) + student-tests.txt — student-tests/run_all.py с RUN_DOCKER_TESTS=1 RUN_DEPS_TEST=1 (вкл. test_shutdown) + session-expiry.txt — test_sessions при api с SESSION_TTL_SECONDS=2 (затем TTL возвращён на 3600) + selfcheck-s3.json — приватность bucket и права ключа API (изнутри сети storage) + selfcheck-consistency.json — сценарии отказа S3/БД A–F, компенсация, CommitUncertain, reconciler + selfcheck-spool.json — временный каталог загрузок пуст после всех тестов + reconcile-run.txt — реальные проходы reconciler по хвостам S3-аутейджа (dry-run и повтор) + attachment-failover.json — upload → stop обслужившей реплики → download через другую (2 раунда) + full-restart.json — docker compose down (тома сохранены) → up → тот же token/файл/текст с U+0000 + s3-restart.txt — docker restart / падение / пересоздание s3: восстановление s3-gw, upload + download + upload-memory.json — 10/20/40 одновременных загрузок 9.5 MiB, пик памяти API, LB и S3 + s3-memory-limit.txt — память SeaweedFS при 10/20/40 загрузках: 512m без GOMEMLIMIT (OOM) и 1g + GOMEMLIMIT + s3-volumes.json — слоты томов SeaweedFS и заполнение по collection (-volume.max=0) + jpeg-decode-memory.txt — пик RSS djpeg на худших JPEG 40 Мп (обоснование mem_limit и prlimit) + decoder-crosscheck.txt — корпус повреждённых/валидных изображений против Pillow и djpeg -strict + load.json — базовая нагрузка POST messages (tests/load.py, 20 с, concurrency 4) + seq-failover.json, rolling-load.json — lab3 на стенде lab4: поочерёдный отказ реплик, rolling update под нагрузкой + lb-interleave.json — 8 медленных загрузок вперемешку с короткими GET: распределение по репликам, roundrobin и leastconn + images.txt — docker compose images diff --git a/evidence/persistence-ps-after.txt b/evidence/persistence-ps-after.txt new file mode 100644 index 00000000..99354595 --- /dev/null +++ b/evidence/persistence-ps-after.txt @@ -0,0 +1,8 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-1 mtusi-messenger:lab4 "python -m messenger" api 30 minutes ago Up 2 seconds (health: starting) 8080/tcp +mtusi2026lab4-api-2 mtusi-messenger:lab4 "python -m messenger" api 30 minutes ago Up 2 seconds (health: starting) 8080/tcp +mtusi2026lab4-lb-1 haproxy:2.9.14-bookworm "docker-entrypoint.s…" lb 30 minutes ago Up Less than a second (health: starting) 127.0.0.1:8080->8080/tcp +mtusi2026lab4-postgres-1 postgres:16.6-bookworm "docker-entrypoint.s…" postgres 30 minutes ago Up 45 seconds (healthy) 5432/tcp +mtusi2026lab4-redis-1 redis:7.4.2-bookworm "docker-entrypoint.s…" redis 30 minutes ago Up 45 seconds (healthy) 6379/tcp +mtusi2026lab4-s3-1 chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e "/entrypoint.sh serv…" s3 11 minutes ago Up 35 seconds (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi2026lab4-s3-gw-1 haproxy:2.9.14-bookworm "/bin/sh /usr/local/…" s3-gw 9 minutes ago Up 6 seconds (healthy) diff --git a/evidence/persistence-ps-before.txt b/evidence/persistence-ps-before.txt new file mode 100644 index 00000000..f0a68dfd --- /dev/null +++ b/evidence/persistence-ps-before.txt @@ -0,0 +1,8 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab4-api-1 mtusi-messenger:lab4 "python -m messenger" api 29 minutes ago Up 23 minutes (healthy) 8080/tcp +mtusi2026lab4-api-2 mtusi-messenger:lab4 "python -m messenger" api 29 minutes ago Up 23 minutes (healthy) 8080/tcp +mtusi2026lab4-lb-1 haproxy:2.9.14-bookworm "docker-entrypoint.s…" lb 29 minutes ago Up 23 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi2026lab4-postgres-1 postgres:16.6-bookworm "docker-entrypoint.s…" postgres 29 minutes ago Up 24 minutes (healthy) 5432/tcp +mtusi2026lab4-redis-1 redis:7.4.2-bookworm "docker-entrypoint.s…" redis 29 minutes ago Up 24 minutes (healthy) 6379/tcp +mtusi2026lab4-s3-1 chrislusf/seaweedfs:3.97@sha256:bb05d66d2963b1cc48073190781c3dd29e2c4c88a2ae2986bf58e38c86d89c6e "/entrypoint.sh serv…" s3 11 minutes ago Up About a minute (healthy) 7333/tcp, 8080/tcp, 8333/tcp, 8888/tcp, 9333/tcp, 18080/tcp, 18888/tcp, 19333/tcp +mtusi2026lab4-s3-gw-1 haproxy:2.9.14-bookworm "/bin/sh /usr/local/…" s3-gw 8 minutes ago Up About a minute (healthy) diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/reconcile-run.txt b/evidence/reconcile-run.txt new file mode 100644 index 00000000..22315fb5 --- /dev/null +++ b/evidence/reconcile-run.txt @@ -0,0 +1,12 @@ +# docker compose --env-file .env exec -T api python -m messenger.reconcile ; build 546479b6b925, 2026-11-06 +# Leftover: 10 intents of uploads that failed while S3 was down (stopped in test_s3_stop_degrades_only_attachments, +# OOM-killed in the before-fix run of s3-memory-limit.txt): compensating DELETE failed, intent kept. +# referenced_objects includes intent-held keys; 4 of the 10 objects existed in S3, hence 1382 -> 1378 after the real run. +# args: --dry-run --grace-seconds 0 --check-missing +{"dry_run": true, "grace_seconds": 0, "stale_intents": 10, "intent_objects_deleted": 10, "objects_scanned": 1382, "referenced_objects": 1382, "fresh_unreferenced_skipped": 0, "orphans_found": 0, "orphans_deleted": 0, "missing_originals": 0, "duration_ms": 624.2} +# args: --dry-run --grace-seconds 0 --check-missing (repeat: dry-run changed nothing) +{"dry_run": true, "grace_seconds": 0, "stale_intents": 10, "intent_objects_deleted": 10, "objects_scanned": 1382, "referenced_objects": 1382, "fresh_unreferenced_skipped": 0, "orphans_found": 0, "orphans_deleted": 0, "missing_originals": 0, "duration_ms": 325.0} +# args: --grace-seconds 60 --check-missing +{"dry_run": false, "grace_seconds": 60, "stale_intents": 10, "intent_objects_deleted": 10, "objects_scanned": 1378, "referenced_objects": 1378, "fresh_unreferenced_skipped": 0, "orphans_found": 0, "orphans_deleted": 0, "missing_originals": 0, "duration_ms": 341.9} +# args: --grace-seconds 60 --check-missing (repeat: nothing left) +{"dry_run": false, "grace_seconds": 60, "stale_intents": 0, "intent_objects_deleted": 0, "objects_scanned": 1378, "referenced_objects": 1378, "fresh_unreferenced_skipped": 0, "orphans_found": 0, "orphans_deleted": 0, "missing_originals": 0, "duration_ms": 311.9} diff --git a/evidence/rolling-load.json b/evidence/rolling-load.json new file mode 100644 index 00000000..22ce768e --- /dev/null +++ b/evidence/rolling-load.json @@ -0,0 +1,20 @@ +{ + "successful": 1436, + "errors": 0, + "errors_by_status": {}, + "p50_ms": 16.0, + "p95_ms": 47.0, + "p99_ms": 62.0, + "duration_s": 23.9, + "replicas_rolled": 2, + "replaced": [ + { + "old": "9ebb33521c7c", + "new": "dbd15729ffb2" + }, + { + "old": "304d00cd6af5", + "new": "d92fe1b01b2c" + } + ] +} \ No newline at end of file diff --git a/evidence/s3-memory-limit.txt b/evidence/s3-memory-limit.txt new file mode 100644 index 00000000..ce5f2f84 --- /dev/null +++ b/evidence/s3-memory-limit.txt @@ -0,0 +1,11 @@ +# SeaweedFS memory under N concurrent 9.5 MiB uploads via LB (fresh s3 + s3-gw per level); build 546479b6b925, 2026-11-06 +# Each line: n, statuses (non-201 with error.message), s3 state, s3 cgroup anon max (sampled 0.1 s), memory.peak. +# Both runs use -volume.max=0; only mem_limit/GOMEMLIMIT differ. +# BEFORE: s3 mem_limit 512m, GOMEMLIMIT off (previous compose.yaml) +{"n": 10, "statuses": {"201": 10}, "max_latency_s": 4.14, "s3_state": "running oom=false exit=0", "s3_baseline_current_mib": 138.7, "s3_sampled_max_anon_mib": 494.3, "anon_samples": 15, "s3_cgroup_peak_mib": 513.6, "s3_anon_now_mib": 494.1, "s3_file_now_mib": 3.5} +{"n": 20, "statuses": {"201": 8, "503 Too many concurrent uploads": 1, "503 Attachment storage unavailable": 11}, "max_latency_s": 17.23, "s3_state": "exited oom=true exit=137", "s3_baseline_current_mib": 138.4, "s3_sampled_max_anon_mib": 488.6, "anon_samples": 21, "s3_cgroup_peak_mib": null, "s3_anon_now_mib": 0.0, "s3_file_now_mib": 0.0} +{"n": 40, "statuses": {"201": 10, "503 Too many concurrent uploads": 22, "503 Attachment storage unavailable": 8}, "max_latency_s": 12.61, "s3_state": "running oom=false exit=0", "s3_baseline_current_mib": 139.2, "s3_sampled_max_anon_mib": 492.3, "anon_samples": 36, "s3_cgroup_peak_mib": 517.3, "s3_anon_now_mib": 492.2, "s3_file_now_mib": 4.8} +# AFTER: s3 mem_limit 1g, GOMEMLIMIT=768MiB (current compose.yaml) +{"n": 10, "statuses": {"201": 10}, "max_latency_s": 3.76, "s3_state": "running oom=false exit=0", "s3_baseline_current_mib": 138.3, "s3_sampled_max_anon_mib": 481.3, "anon_samples": 18, "s3_cgroup_peak_mib": 621.8, "s3_anon_now_mib": 177.9, "s3_file_now_mib": 95.1} +{"n": 20, "statuses": {"201": 20}, "max_latency_s": 5.99, "s3_state": "running oom=false exit=0", "s3_baseline_current_mib": 146.8, "s3_sampled_max_anon_mib": 634.8, "anon_samples": 22, "s3_cgroup_peak_mib": 860.0, "s3_anon_now_mib": 208.6, "s3_file_now_mib": 198.1} +{"n": 40, "statuses": {"201": 24, "503 Too many concurrent uploads": 16}, "max_latency_s": 8.19, "s3_state": "running oom=false exit=0", "s3_baseline_current_mib": 139.3, "s3_sampled_max_anon_mib": 584.1, "anon_samples": 29, "s3_cgroup_peak_mib": 778.1, "s3_anon_now_mib": 180.9, "s3_file_now_mib": 228.4} diff --git a/evidence/s3-restart.txt b/evidence/s3-restart.txt new file mode 100644 index 00000000..14bf63d4 --- /dev/null +++ b/evidence/s3-restart.txt @@ -0,0 +1,35 @@ +Рестарт S3 и восстановление пары s3 / s3-gw (2026-11-06, стенд lab4, сборка API 60bb1bccb028) +Проверка после каждого сценария: /health/ready через LB 200 три раза подряд, затем новый пользователь +загружает PNG (201) и другой участник чата скачивает его через LB (200, SHA-256 совпадает). + +0) До изменения (у s3 и s3-gw нет restart policy, entrypoint s3-gw — штатный haproxy): + docker restart + -> s3 healthy, но /health/ready 503 дольше 90 с; s3-gw "Up (healthy)" (healthcheck haproxy -c), + в его namespace остался только lo, на 127.0.0.1:8333 никто не слушает. + +1) restart: unless-stopped у s3 и s3-gw, entrypoint infra/s3-gw-entrypoint.sh: + docker restart (в обход compose; сам рестарт 10.4 с) + -> s3-gw: "network namespace of s3 is gone; exiting so the restart policy re-joins it" + -> s3-gw RestartCount 0 -> 1, running; ready 200 x3 через 37 с от начала рестарта + -> upload 201 + download 200, sha256 match + +2) Падение процесса S3 (weed завершается сам, контейнер поднимает restart policy): + docker exec kill -TERM 1 + -> s3 RestartCount 0 -> 1; s3-gw RestartCount 1 -> 2, running + -> ready 200 x3 через 29 с от сигнала + -> upload 201 + download 200, sha256 match + +3) docker compose --env-file .env restart s3 + -> compose перезапускает и s3-gw (depends_on s3: restart: true); ready 200 через 35 с + -> upload 201 + download 200, sha256 match + +4) docker compose --env-file .env up -d --force-recreate s3 (только s3, новый ID контейнера) + -> s3-gw: "joining network namespace of container: No such container" -> Exited (1); + upload 503 "Attachment storage unavailable" + docker compose --env-file .env up -d --scale api=2 + -> s3-gw Recreate -> Started; ready 200 через 6 с + -> upload 201 + download 200, sha256 match + Применение новой конфигурации (docker compose up -d) пересоздало s3 и s3-gw вместе — пара сразу рабочая. + +5) docker kill — Docker считает ручной остановкой: restart policy s3 не поднимает (s3 Exited, + RestartCount 0), s3-gw после попытки рестарта Exited. Восстановление: docker compose up -d. diff --git a/evidence/s3-volumes.json b/evidence/s3-volumes.json new file mode 100644 index 00000000..473fc8f1 --- /dev/null +++ b/evidence/s3-volumes.json @@ -0,0 +1,21 @@ +{ + "date": "2026-11-06T14:43:34Z", + "flags": "-master.volumeSizeLimitMB=512 -volume.max=0 (was 16)", + "max_volume_slots": 1896, + "free_volume_slots": 1873, + "volumes_by_collection": { + "messenger-attachments": { + "volumes": 16, + "size_mib": 6595, + "files": 2858, + "full_volumes": 9 + }, + "(default)": { + "volumes": 7, + "size_mib": 0, + "files": 67, + "full_volumes": 0 + } + }, + "note": "Before the fix: 16 slots, 7 taken by the default collection, all 9 bucket volumes full -> every PUT failed with 'No writable volumes' (test_s3_can_grow_volumes on that config: Max 16, Free -7)." +} \ No newline at end of file diff --git a/evidence/selfcheck-consistency.json b/evidence/selfcheck-consistency.json new file mode 100644 index 00000000..77562b16 --- /dev/null +++ b/evidence/selfcheck-consistency.json @@ -0,0 +1,165 @@ +{ + "selfcheck": "consistency", + "passed": true, + "results": [ + { + "check": "A_s3_put_failure_raises", + "ok": true, + "detail": "" + }, + { + "check": "A_no_object_no_intent_no_metadata", + "ok": true, + "detail": "" + }, + { + "check": "B_commit_failure_raises", + "ok": true, + "detail": "" + }, + { + "check": "B_compensation_removed_object_and_intent", + "ok": true, + "detail": "" + }, + { + "check": "C_crash_leaves_object_and_intent", + "ok": true, + "detail": "" + }, + { + "check": "C_fresh_intent_survives_reconcile", + "ok": true, + "detail": { + "dry_run": false, + "grace_seconds": 900, + "stale_intents": 0, + "intent_objects_deleted": 0, + "objects_scanned": 1322, + "referenced_objects": 1316, + "fresh_unreferenced_skipped": 6, + "orphans_found": 0, + "orphans_deleted": 0, + "duration_ms": 104.3 + } + }, + { + "check": "C_stale_intent_object_deleted", + "ok": true, + "detail": { + "dry_run": false, + "grace_seconds": 900, + "stale_intents": 1, + "intent_objects_deleted": 1, + "objects_scanned": 1321, + "referenced_objects": 1315, + "fresh_unreferenced_skipped": 6, + "orphans_found": 0, + "orphans_deleted": 0, + "duration_ms": 169.9 + } + }, + { + "check": "C_rerun_is_noop", + "ok": true, + "detail": { + "dry_run": false, + "grace_seconds": 900, + "stale_intents": 0, + "intent_objects_deleted": 0, + "objects_scanned": 1321, + "referenced_objects": 1315, + "fresh_unreferenced_skipped": 6, + "orphans_found": 0, + "orphans_deleted": 0, + "duration_ms": 298.5 + } + }, + { + "check": "D_fresh_orphan_not_deleted", + "ok": true, + "detail": { + "dry_run": true, + "grace_seconds": 900, + "stale_intents": 0, + "intent_objects_deleted": 0, + "objects_scanned": 1322, + "referenced_objects": 1315, + "fresh_unreferenced_skipped": 7, + "orphans_found": 0, + "orphans_deleted": 0, + "duration_ms": 299.8 + } + }, + { + "check": "D_old_orphan_deleted_by_sweep", + "ok": true, + "detail": { + "dry_run": false, + "grace_seconds": 2, + "stale_intents": 0, + "intent_objects_deleted": 0, + "objects_scanned": 1322, + "referenced_objects": 1315, + "fresh_unreferenced_skipped": 0, + "orphans_found": 7, + "orphans_deleted": 7, + "duration_ms": 107.5 + } + }, + { + "check": "F_uncertain_commit_raises", + "ok": true, + "detail": "" + }, + { + "check": "F_uncertain_commit_keeps_object_and_metadata", + "ok": true, + "detail": "" + }, + { + "check": "F_reconcile_keeps_committed_object", + "ok": true, + "detail": { + "dry_run": false, + "grace_seconds": 0, + "stale_intents": 0, + "intent_objects_deleted": 0, + "objects_scanned": 1316, + "referenced_objects": 1316, + "fresh_unreferenced_skipped": 0, + "orphans_found": 0, + "orphans_deleted": 0, + "missing_originals": 0, + "duration_ms": 107.4 + } + }, + { + "check": "E_upload_committed", + "ok": true, + "detail": "" + }, + { + "check": "E_referenced_object_kept", + "ok": true, + "detail": { + "dry_run": false, + "grace_seconds": 0, + "stale_intents": 0, + "intent_objects_deleted": 0, + "objects_scanned": 1317, + "referenced_objects": 1317, + "fresh_unreferenced_skipped": 0, + "orphans_found": 0, + "orphans_deleted": 0, + "missing_originals": 0, + "duration_ms": 104.3 + } + }, + { + "check": "E_no_missing_originals", + "ok": true, + "detail": 0 + } + ] +} diff --git a/evidence/selfcheck-s3.json b/evidence/selfcheck-s3.json new file mode 100644 index 00000000..034979e6 --- /dev/null +++ b/evidence/selfcheck-s3.json @@ -0,0 +1,103 @@ +{ + "selfcheck": "s3", + "passed": true, + "results": [ + { + "check": "api_key_put_object", + "ok": true, + "detail": "" + }, + { + "check": "api_key_get_object", + "ok": true, + "detail": "" + }, + { + "check": "anonymous_get_object_denied", + "ok": true, + "detail": 403 + }, + { + "check": "anonymous_get_object_query_denied", + "ok": true, + "detail": 403 + }, + { + "check": "anonymous_list_bucket_denied", + "ok": true, + "detail": 403 + }, + { + "check": "anonymous_list_buckets_denied", + "ok": true, + "detail": 403 + }, + { + "check": "api_key_list_buckets_denied", + "ok": true, + "detail": "403 AccessDenied" + }, + { + "check": "api_key_create_bucket_denied", + "ok": true, + "detail": "403 AccessDenied" + }, + { + "check": "api_key_delete_bucket_denied", + "ok": true, + "detail": "403 AccessDenied" + }, + { + "check": "api_key_put_bucket_acl_denied", + "ok": true, + "detail": "403 AccessDenied" + }, + { + "check": "api_key_put_object_acl_denied", + "ok": true, + "detail": "403 AccessDenied" + }, + { + "check": "api_key_other_bucket_denied", + "ok": true, + "detail": "403 AccessDenied" + }, + { + "check": "anonymous_get_still_denied_after_acl_attempts", + "ok": true, + "detail": 403 + }, + { + "check": "wrong_secret_denied", + "ok": true, + "detail": "403 SignatureDoesNotMatch" + }, + { + "check": "seaweed_admin_and_grpc_ports_unreachable", + "ok": true, + "detail": { + "open": [], + "probed": [ + 8333, + 18333, + 8888, + 18888, + 9333, + 19333, + 8080, + 18080 + ] + } + }, + { + "check": "s3_api_port_reachable", + "ok": true, + "detail": "" + }, + { + "check": "api_key_delete_object", + "ok": true, + "detail": "" + } + ] +} diff --git a/evidence/selfcheck-spool.json b/evidence/selfcheck-spool.json new file mode 100644 index 00000000..eee95539 --- /dev/null +++ b/evidence/selfcheck-spool.json @@ -0,0 +1,14 @@ +{ + "selfcheck": "spool", + "passed": true, + "results": [ + { + "check": "spool_dir_empty", + "ok": true, + "detail": { + "dir": "/tmp/messenger-spool", + "files": 0 + } + } + ] +} diff --git a/evidence/seq-failover.json b/evidence/seq-failover.json new file mode 100644 index 00000000..599436d2 --- /dev/null +++ b/evidence/seq-failover.json @@ -0,0 +1,28 @@ +[ + { + "round": 0, + "observed_before": [ + "6fe989e050e5", + "f06ef1999af2" + ], + "target": "6fe989e050e5", + "successful_writes": 168, + "failed_attempts": 0, + "first_recovery_seconds": 3.906, + "target_absent_in_last5": true, + "restore_to_lb_seconds": 3.5 + }, + { + "round": 1, + "observed_before": [ + "6fe989e050e5", + "f06ef1999af2" + ], + "target": "f06ef1999af2", + "successful_writes": 166, + "failed_attempts": 0, + "first_recovery_seconds": 3.89, + "target_absent_in_last5": true, + "restore_to_lb_seconds": 2.938 + } +] \ No newline at end of file diff --git a/evidence/session-expiry.txt b/evidence/session-expiry.txt new file mode 100644 index 00000000..906d53b7 --- /dev/null +++ b/evidence/session-expiry.txt @@ -0,0 +1,3 @@ +OK test_sessions.test_login_after_register +OK test_sessions.test_session_expiry_if_short_ttl +Summary: 2 passed, 0 failed diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..1408ce9d --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,112 @@ + Container mtusi2026lab4-postgres-1 Paused + Container mtusi2026lab4-postgres-1 Unpaused + Container mtusi2026lab4-postgres-1 Stopping + Container mtusi2026lab4-postgres-1 Stopped + Container mtusi2026lab4-postgres-1 Starting + Container mtusi2026lab4-postgres-1 Started + Container mtusi2026lab4-redis-1 Stopping + Container mtusi2026lab4-redis-1 Stopped + Container mtusi2026lab4-redis-1 Starting + Container mtusi2026lab4-redis-1 Started + Container mtusi2026lab4-redis-1 Restarting + Container mtusi2026lab4-s3-1 Restarting + Container mtusi2026lab4-postgres-1 Restarting + Container mtusi2026lab4-redis-1 Started + Container mtusi2026lab4-postgres-1 Started + Container mtusi2026lab4-s3-1 Started + Container mtusi2026lab4-s3-1 Waiting + Container mtusi2026lab4-s3-1 Healthy + Container mtusi2026lab4-s3-gw-1 Restarting + Container mtusi2026lab4-s3-gw-1 Started + Container mtusi2026lab4-s3-gw-1 Restarting + Container mtusi2026lab4-s3-gw-1 Started + Container mtusi2026lab4-api-2 Restarting + Container mtusi2026lab4-api-1 Restarting + Container mtusi2026lab4-api-2 Started + Container mtusi2026lab4-api-1 Started + Container mtusi2026lab4-api-1 Running + Container mtusi2026lab4-api-2 Running + Container mtusi2026lab4-lb-1 Restarting + Container mtusi2026lab4-lb-1 Started + Container mtusi2026lab4-s3-1 Stopping + Container mtusi2026lab4-s3-1 Stopped + Container mtusi2026lab4-s3-1 Starting + Container mtusi2026lab4-s3-1 Started + Container mtusi2026lab4-s3-gw-1 Restarting + Container mtusi2026lab4-s3-gw-1 Started +OK test_images_unit.test_all_filter_types_accepted +OK test_images_unit.test_corrupted_jpeg_rejected +OK test_images_unit.test_corrupted_png_rejected +OK test_images_unit.test_exactly_at_pixel_limit_accepted +OK test_images_unit.test_foreign_formats_rejected +OK test_images_unit.test_pixel_limit_detected_from_header_only +OK test_images_unit.test_reviewer_corrupted_jpegs_structural_part +OK test_images_unit.test_reviewer_corrupted_pngs_rejected +OK test_images_unit.test_valid_jpegs_any_chunking +OK test_images_unit.test_valid_pngs_any_chunking +OK test_api.test_basic_on_business_rejected +OK test_api.test_chat_requires_caller_member +OK test_api.test_invalid_basic_encoding_is_401 +OK test_api.test_login_rejects_json_body +OK test_api.test_logout_revokes +OK test_api.test_malformed_cursor_400 +OK test_api.test_nul_in_text_roundtrip +OK test_api.test_unknown_member_rolls_back +OK test_api.test_unsupported_media_type +OK test_api.test_username_conflict +OK test_api.test_username_trailing_newline_400 +OK test_api.test_whitespace_only_password_rejected +OK test_api.test_wrong_password +OK test_api.test_x_user_id_ignored +OK test_concurrency.test_parallel_messages_unique +OK test_concurrency.test_username_race_one_winner +OK test_sessions.test_login_after_register +SKIP expiry: SESSION_TTL_SECONDS=3600 (need <=5 for this test) +OK test_sessions.test_session_expiry_if_short_ttl +OK test_idempotency.test_empty_attachment_ids_equals_absent +OK test_idempotency.test_invalid_key_format_400 +OK test_idempotency.test_key_scoped_to_user_and_chat +OK test_idempotency.test_missing_idempotency_key_400 +OK test_idempotency.test_multiple_instance_ids_observed +OK test_idempotency.test_parallel_same_key_one_message +OK test_idempotency.test_replay_same_key_same_body +OK test_idempotency.test_same_key_different_body_409 +OK test_attachments.test_attachment_ids_validation_400 +OK test_attachments.test_corrupted_images_415 +OK test_attachments.test_dangerous_filenames_400_and_unicode_kept +OK test_attachments.test_empty_body_and_variants +OK test_attachments.test_exact_limit_png_through_lb_accepted +OK test_attachments.test_filename_query_decoded_strictly_400 +OK test_attachments.test_foreign_and_unknown_attachment_ids_404 +OK test_attachments.test_interrupted_upload_does_not_break_service +OK test_attachments.test_message_with_attachments_and_history +OK test_attachments.test_mime_spoofing_415 +OK test_attachments.test_near_limit_real_png_accepted +OK test_attachments.test_outsider_and_unknown_get_404 +OK test_attachments.test_parallel_uploads_distinct +OK test_attachments.test_reviewer_corrupted_images_415 +OK test_attachments.test_same_filename_distinct_objects +OK test_attachments.test_too_large_413 +OK test_attachments.test_too_many_pixels_413 +OK test_attachments.test_upload_auth_and_chat_checks +OK test_attachments.test_upload_roundtrip_png_and_jpeg_variants +OK test_attachments_ops.test_bucket_private_and_api_key_least_privilege +OK test_attachments_ops.test_interrupted_and_stalled_uploads_leave_nothing +OK test_attachments_ops.test_no_original_kept_on_api_replicas +OK test_attachments_ops.test_parallel_large_uploads_keep_s3_within_memory +OK test_attachments_ops.test_s3_can_grow_volumes +OK test_attachments_ops.test_s3_db_failure_compensation_and_reconcile + in-flight upload: body sent over 5.0s across listener close -> 201, download via LB OK; back in LB in 2.6s +OK test_shutdown.test_inflight_upload_finishes_after_listener_closed + stop under load: 405 x 201, 0 errors, 58 served by stopping replica; docker stop 3.58s; back in LB in 2.5s +OK test_shutdown.test_replica_stop_under_load_has_no_errors + drain: 22 probes, ready 503 / users/me 200; exit 3.29s after SIGTERM; back in LB in 2.0s +OK test_shutdown.test_sigterm_drains_readiness_but_keeps_serving + postgres paused: 503 after 6.14s, 3.03s, 6.22s +OK test_deps.test_postgres_pause_fails_fast +OK test_deps.test_postgres_stop_returns_503 +OK test_deps.test_redis_stop_returns_503_or_unready +OK test_deps.test_restart_hook_restarts_api_and_stores + s3 down: ready 503, upload 503 in 0.11s, download 503, metadata/messages OK +OK test_deps.test_s3_stop_degrades_only_attachments +Summary: 69 passed, 0 failed diff --git a/evidence/upload-memory.json b/evidence/upload-memory.json new file mode 100644 index 00000000..7043265c --- /dev/null +++ b/evidence/upload-memory.json @@ -0,0 +1,175 @@ +{ + "date": "2026-11-06T15:20:30Z", + "api_mem_limit_mib": 512, + "api_cpus": 0.5, + "max_concurrent_uploads_per_replica": 8, + "levels": [ + { + "concurrency": 10, + "payload_bytes": 9960421, + "statuses": { + "201": 10 + }, + "wall_seconds": 3.422, + "latency_ok_p50_s": 3.032, + "latency_ok_p95_s": 3.422, + "latency_ok_max_s": 3.422, + "throughput_mib_s": 27.8, + "naive_full_buffer_estimate_mib": 47.5, + "replicas": { + "bbea736d1b58": { + "uploads_served": 5, + "rss_baseline_mib": 94, + "rss_peak_vmhwm_mib": 110, + "rss_peak_growth_mib": 16, + "cgroup_peak_mib": 151.8, + "cgroup_baseline_current_mib": 76.5, + "sampled_max_anon_mib": 97.7, + "sampled_max_file_cache_mib": 47.5, + "samples": 10 + }, + "807474b42967": { + "uploads_served": 5, + "rss_baseline_mib": 94, + "rss_peak_vmhwm_mib": 110, + "rss_peak_growth_mib": 16, + "cgroup_peak_mib": 155.2, + "cgroup_baseline_current_mib": 76.7, + "sampled_max_anon_mib": 97.6, + "sampled_max_file_cache_mib": 47.5, + "samples": 9 + } + }, + "lb": { + "mem_limit_mib": 128, + "baseline_current_mib": 17.5, + "cgroup_peak_mib": 31.8, + "sampled_max_anon_mib": 14.8, + "sampled_max_current_mib": 18.7, + "restarts": 0, + "oom_killed": false + }, + "s3": { + "mem_limit_mib": 1024, + "baseline_current_mib": 143.7, + "cgroup_peak_mib": 547.6, + "sampled_max_anon_mib": 401.3, + "sampled_max_current_mib": 451.8, + "running": true, + "oom_killed": false + } + }, + { + "concurrency": 20, + "payload_bytes": 9960421, + "statuses": { + "201": 18, + "503": 2 + }, + "wall_seconds": 6.375, + "latency_ok_p50_s": 6.156, + "latency_ok_p95_s": 6.359, + "latency_ok_max_s": 6.359, + "throughput_mib_s": 26.8, + "naive_full_buffer_estimate_mib": 76.0, + "replicas": { + "bbea736d1b58": { + "uploads_served": 10, + "rss_baseline_mib": 94, + "rss_peak_vmhwm_mib": 118, + "rss_peak_growth_mib": 24, + "cgroup_peak_mib": 192.1, + "cgroup_baseline_current_mib": 76.7, + "sampled_max_anon_mib": 103.9, + "sampled_max_file_cache_mib": 76.0, + "samples": 14 + }, + "807474b42967": { + "uploads_served": 10, + "rss_baseline_mib": 94, + "rss_peak_vmhwm_mib": 123, + "rss_peak_growth_mib": 29, + "cgroup_peak_mib": 181.9, + "cgroup_baseline_current_mib": 76.6, + "sampled_max_anon_mib": 101.5, + "sampled_max_file_cache_mib": 76.0, + "samples": 14 + } + }, + "lb": { + "mem_limit_mib": 128, + "baseline_current_mib": 17.7, + "cgroup_peak_mib": 40.7, + "sampled_max_anon_mib": 15.0, + "sampled_max_current_mib": 23.0, + "restarts": 0, + "oom_killed": false + }, + "s3": { + "mem_limit_mib": 1024, + "baseline_current_mib": 140.5, + "cgroup_peak_mib": 734.0, + "sampled_max_anon_mib": 541.0, + "sampled_max_current_mib": 724.2, + "running": true, + "oom_killed": false + } + }, + { + "concurrency": 40, + "payload_bytes": 9960421, + "statuses": { + "503": 22, + "201": 18 + }, + "wall_seconds": 7.0, + "latency_ok_p50_s": 6.297, + "latency_ok_p95_s": 6.5, + "latency_ok_max_s": 7.0, + "throughput_mib_s": 24.4, + "naive_full_buffer_estimate_mib": 76.0, + "replicas": { + "bbea736d1b58": { + "uploads_served": 20, + "rss_baseline_mib": 94, + "rss_peak_vmhwm_mib": 122, + "rss_peak_growth_mib": 28, + "cgroup_peak_mib": 187.2, + "cgroup_baseline_current_mib": 76.5, + "sampled_max_anon_mib": 112.7, + "sampled_max_file_cache_mib": 76.0, + "samples": 15 + }, + "807474b42967": { + "uploads_served": 20, + "rss_baseline_mib": 94, + "rss_peak_vmhwm_mib": 125, + "rss_peak_growth_mib": 31, + "cgroup_peak_mib": 198.7, + "cgroup_baseline_current_mib": 76.3, + "sampled_max_anon_mib": 114.9, + "sampled_max_file_cache_mib": 76.0, + "samples": 15 + } + }, + "lb": { + "mem_limit_mib": 128, + "baseline_current_mib": 18.2, + "cgroup_peak_mib": 60.3, + "sampled_max_anon_mib": 15.6, + "sampled_max_current_mib": 44.0, + "restarts": 0, + "oom_killed": false + }, + "s3": { + "mem_limit_mib": 1024, + "baseline_current_mib": 140.6, + "cgroup_peak_mib": 733.0, + "sampled_max_anon_mib": 566.4, + "sampled_max_current_mib": 672.2, + "running": true, + "oom_killed": false + } + } + ] +} \ No newline at end of file diff --git a/evidence/validate.txt b/evidence/validate.txt new file mode 100644 index 00000000..248343ee --- /dev/null +++ b/evidence/validate.txt @@ -0,0 +1 @@ +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/README.md b/infra/README.md index 5a39e862..c373006a 100644 --- a/infra/README.md +++ b/infra/README.md @@ -1,7 +1,14 @@ -# Инфраструктура студента +# Инфраструктура lab4 -Здесь разместите конфигурации выбранных сервисов; основную топологию — в корневом compose.yaml/stack.yaml. `compose.example.yaml` — только worksheet ролей и намеренно не запускается. `contracts/infrastructure.json` задаёт минимальные роли, не готовые образы/пароли. +- `haproxy.cfg` — публичный балансировщик (`127.0.0.1:8080`, `balance leastconn`). + - Новый запрос уходит на реплику с наименьшим числом открытых соединений. Загрузка вложения держит соединение секунды, поэтому длинные upload не скапливаются на одной реплике, пока у другой свободны слоты. При равной нагрузке (короткие последовательные запросы) реплики чередуются, как при round-robin. + - Основной backend `messenger_api` проверяет `/health/ready` (БД + Redis + S3): неготовая реплика исключается из трафика. + - Если не готова ни одна реплика (общая зависимость недоступна), трафик идёт в `messenger_api_degraded`, который проверяет только `/health/live`. Клиент получает JSON 503 самого API на затронутых endpoint, а текстовый чат и метаданные продолжают работать. + - Ошибки, сгенерированные самим HAProxy (502/503/504), оформлены в контрактном JSON-конверте с `X-Request-Id` и `X-Instance-Id: lb`. + - DNS Docker `127.0.0.11` + `server-template` подхватывают `--scale api=N` и замену контейнеров. + - `inter 1s fall 2` + `observe layer4 … on-error mark-down`: остановленная реплика исключается после первой неудачной попытки подключения или примерно через 2–3 с по health check. При `docker stop` реплика API по SIGTERM сначала отвечает 503 только на `/health/ready` и ещё 3 с (`SHUTDOWN_DRAIN_SECONDS`) обслуживает запросы, поэтому HAProxy исключает её раньше, чем она закроет listener. `timeout connect 1s` + `option redispatch 1`: запрос, не дошедший до мёртвой реплики, уходит на другую. POST повторно не отправляется: повтор возможен только на этапе connect, когда запрос ещё не передан. + - Размер тела в HAProxy не ограничен: 10 MiB + 1 байт отклоняет API (JSON 413). `tune.*buf` фиксирует сокетные буферы, чтобы пачка загрузок не довела LB (128m) до OOM. +- `s3-gateway.cfg` — TCP-сайдкар `s3-gw` в сетевом namespace `s3`: наружу (только в internal-сеть `storage`) отдаёт один S3 HTTP API как `s3:9000`. Master/filer/volume/gRPC SeaweedFS слушают только `127.0.0.1` внутри контейнера. +- `s3-gw-entrypoint.sh` — entrypoint `s3-gw`: запускает HAProxy и раз в 2 с проверяет, что namespace `s3` жив. После падения или `docker restart` контейнера `s3` в старом namespace остаётся только `lo`. Тогда скрипт через 5 с завершает контейнер, и `restart: unless-stopped` запускает `s3-gw` уже в namespace нового `s3`. У `s3` и `s3-gw`, как у `api`, `lb`, `postgres` и `redis`, стоит `restart: unless-stopped`; у one-shot `migrate` и `s3-init` политики нет. Скрипт монтируется в Linux-контейнер, поэтому `.gitattributes` держит `*.sh` в LF и на Windows. -В REPORT.md сопоставьте роль → service name → версия/digest → health/readiness → volume → сеть/порт → способ передачи credentials. Приведите bootstrap (schema, bucket, broker topology, а с lab6 provisioning) и безопасное повторение этих команд. - -Приложение строится из исходников. Не используйте заранее существующий образ преподавательского сервера вместо своей реализации. Restart hook перезапускает все соответствующие stores с volumes; failover hook останавливает только заданную API-реплику. +Конфиги примонтированы как файлы, поэтому после правки `haproxy.cfg` нужен `docker compose --env-file .env restart lb`. `up -d` не пересоздаёт LB, если не изменился сам `compose.yaml`. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..798b8e56 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,70 @@ +# Lab4 load balancer: leastconn across Compose DNS name "api" (all replicas). +# Uploads hold a connection for seconds, so new requests go to the replica with the +# fewest open connections (equal load still rotates, like round-robin). +global + maxconn 2048 + log stdout format raw local0 + # Fixed socket buffers: a burst of 10 MiB uploads must not grow kernel buffers + # (charged to this container's 128m cgroup) until the OOM killer stops the LB. + tune.rcvbuf.client 262144 + tune.sndbuf.client 262144 + tune.rcvbuf.server 262144 + tune.sndbuf.server 262144 + +defaults + mode http + log global + option httplog + option dontlognull + # A stopped replica's IP vanishes and connects to it hang instead of being refused; + # on the local bridge a real connect takes milliseconds, so fail over after 1s. + timeout connect 1s + timeout client 30s + # After the body is forwarded the API may still verify the image and write to S3 + # (bounded at ≈35 s in the worst case: decode 15 s + S3 ≈15 s + DB 6 s). + timeout server 45s + timeout check 1500ms + timeout http-request 10s + retries 2 + # Connect-level retries go to another replica (request not sent yet, safe for POST). + option redispatch 1 + # LB-generated errors use the API error envelope and carry the contract headers. + unique-id-format %[uuid()] + http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Bad gateway","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr X-Instance-Id lb + http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"No API replica available","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr X-Instance-Id lb + http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Gateway timeout","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr X-Instance-Id lb + +resolvers docker + nameserver dns1 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 5s + hold obsolete 10s + +frontend messenger_fe + bind *:8080 + acl is_lb_live path /__lb/live + http-request return status 200 content-type text/plain string ok if is_lb_live + # Normal routing uses readiness. If NO replica is ready (shared dependency down: + # PostgreSQL, Redis or S3), live replicas still answer so clients get the API's + # own JSON 503 for affected endpoints and unaffected ones keep working. + use_backend messenger_api_degraded if { nbsrv(messenger_api) lt 1 } + default_backend messenger_api + +backend messenger_api + balance leastconn + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + # server-template expands A records for Compose service DNS name "api". + # inter=1s fall=2 → ≈2–3s to mark a backend down after failed checks; a failed + # live connect marks it down at once (observe layer4). + server-template api 8 api:8080 check inter 1s fall 2 rise 1 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + +backend messenger_api_degraded + balance leastconn + option httpchk + http-check send meth GET uri /health/live ver HTTP/1.1 hdr Host api + http-check expect status 200 + server-template live 8 api:8080 check inter 1s fall 2 rise 1 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none diff --git a/infra/s3-gateway.cfg b/infra/s3-gateway.cfg new file mode 100644 index 00000000..97393e14 --- /dev/null +++ b/infra/s3-gateway.cfg @@ -0,0 +1,22 @@ +# S3 gateway sidecar (lab4). Runs in the network namespace of the `s3` service: +# SeaweedFS listens on 127.0.0.1 only, this proxy publishes just the S3 HTTP API +# as s3:9000 on the internal `storage` network. Master/volume/filer UIs and the +# unauthenticated gRPC ports (1xxxx) stay unreachable from other containers. +global + maxconn 512 + log stdout format raw local0 + +defaults + mode tcp + log global + option tcplog + timeout connect 3s + timeout client 60s + timeout server 60s + +frontend s3_api + bind *:9000 + default_backend seaweed_s3 + +backend seaweed_s3 + server s3 127.0.0.1:8333 diff --git a/infra/s3-gw-entrypoint.sh b/infra/s3-gw-entrypoint.sh new file mode 100755 index 00000000..5bea21dc --- /dev/null +++ b/infra/s3-gw-entrypoint.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# s3-gw shares the network namespace of the `s3` container. When s3 restarts (crash, +# `docker restart`, Docker daemon restart) it gets a NEW namespace and this one is left +# with only `lo`: HAProxy keeps running but nothing can reach it. Exit in that case so +# the restart policy starts s3-gw again inside the current namespace of s3. +haproxy -W -db -f /usr/local/etc/haproxy/haproxy.cfg & +pid=$! +trap 'kill -TERM "$pid" 2>/dev/null' TERM INT USR1 + +orphaned() { + # /proc/net/dev: two header lines (with '|') and one line per interface. + ! grep -qv -e '^ *lo:' -e '|' /proc/net/dev +} + +while kill -0 "$pid" 2>/dev/null; do + if orphaned; then + echo "s3-gw: network namespace of s3 is gone; exiting so the restart policy re-joins it" >&2 + kill -TERM "$pid" 2>/dev/null + wait "$pid" + # Docker does not retry a start that fails because s3 is not running yet: + # give the restart policy of s3 time to bring it back first. + sleep 5 + exit 1 + fi + sleep 2 & + wait $! +done +wait "$pid" diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..27f63acb --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,48 @@ +-- Lab2 schema: users, chats, membership, messages. +-- Applied once via schema_migrations; safe to re-run application bootstrap. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_username_unique UNIQUE (username) +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS idx_chat_members_user + ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +-- Keyset pagination by (created_at, id) within a chat; avoids full history scan. +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at ASC, id ASC); + +-- List chats for a user ordered by chat created_at/id. +CREATE INDEX IF NOT EXISTS idx_chats_created_id + ON chats (created_at ASC, id ASC); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..a3024134 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +-- Lab3: shared idempotency records for POST /messages (across API replicas). + +CREATE TABLE IF NOT EXISTS idempotency_records ( + user_id UUID NOT NULL, + method TEXT NOT NULL, + path TEXT NOT NULL, + idem_key TEXT NOT NULL, + request_hash TEXT NOT NULL, + response_status INT NOT NULL, + response_body JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (user_id, method, path, idem_key) +); + +CREATE INDEX IF NOT EXISTS idx_idempotency_created + ON idempotency_records (created_at); diff --git a/migrations/003_attachments.sql b/migrations/003_attachments.sql new file mode 100644 index 00000000..43479825 --- /dev/null +++ b/migrations/003_attachments.sql @@ -0,0 +1,55 @@ +-- Lab4: attachment metadata (bytes live only in the private S3 bucket). +-- Additive migration: lab3 replicas keep working while it is applied. + +CREATE TABLE IF NOT EXISTS attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id), + owner_id UUID NOT NULL REFERENCES users(id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes BIGINT NOT NULL, + sha256 TEXT NOT NULL, + width INT NOT NULL, + height INT NOT NULL, + -- Server-generated S3 key; never derived from filename, never exposed by the API. + object_key TEXT NOT NULL, + status TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + CONSTRAINT attachments_object_key_unique UNIQUE (object_key), + CONSTRAINT attachments_content_type_check CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size_check CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256_check CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_pixels_check CHECK (width > 0 AND height > 0 AND width::bigint * height <= 40000000), + CONSTRAINT attachments_status_check CHECK (status IN ('ready')) +); + +CREATE INDEX IF NOT EXISTS idx_attachments_chat + ON attachments (chat_id, created_at, id); + +-- Ordered links message -> attachment (an attachment may be referenced by several messages). +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id UUID NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + attachment_id UUID NOT NULL REFERENCES attachments(id), + position SMALLINT NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position_check CHECK (position BETWEEN 0 AND 9), + CONSTRAINT message_attachments_position_unique UNIQUE (message_id, position) +); + +CREATE INDEX IF NOT EXISTS idx_message_attachments_attachment + ON message_attachments (attachment_id); + +-- Write-ahead journal of S3 objects whose metadata is not committed yet. +-- A row exists from just before the S3 PUT until the metadata commit (same +-- transaction deletes it) or until compensation/reconciliation removes the object. +CREATE TABLE IF NOT EXISTS upload_intents ( + object_key TEXT PRIMARY KEY, + attachment_id UUID NOT NULL, + chat_id UUID NOT NULL, + instance_id TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL +); + +CREATE INDEX IF NOT EXISTS idx_upload_intents_created + ON upload_intents (created_at); diff --git a/migrations/004_message_text_bytes.sql b/migrations/004_message_text_bytes.sql new file mode 100644 index 00000000..9ee94b24 --- /dev/null +++ b/migrations/004_message_text_bytes.sql @@ -0,0 +1,8 @@ +-- Lab4: message text may contain U+0000, which PostgreSQL TEXT/JSONB cannot store. +-- Text is kept as exact UTF-8 bytes; the idempotent response snapshot as JSON +-- (textual JSON keeps the "\u0000" escape, JSONB rejects it). +ALTER TABLE messages + ALTER COLUMN text TYPE BYTEA USING convert_to(text, 'UTF8'); + +ALTER TABLE idempotency_records + ALTER COLUMN response_body TYPE JSON USING response_body::json; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..de0dbcf7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.6 +uvicorn[standard]==0.32.1 +pydantic==2.10.3 +psycopg[binary,pool]==3.2.4 +redis==5.2.1 +argon2-cffi==23.1.0 +boto3==1.35.99 diff --git a/scripts/attachment_failover_evidence.py b/scripts/attachment_failover_evidence.py new file mode 100644 index 00000000..1629fae0 --- /dev/null +++ b/scripts/attachment_failover_evidence.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +"""Upload → stop the API replica that served the upload → download via another replica. + +Runs one round per replica (the stopped one is restored before the next round). +Usage: python scripts/attachment_failover_evidence.py +Writes evidence/attachment-failover.json; tokens are never printed. +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def sh(args) -> str: + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout + + +def request(method, path, body=None, token=None, raw=None, headers=None): + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + if body is not None: + raw = json.dumps(body).encode() + h["Content-Type"] = "application/json" + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=10) + except urllib.error.HTTPError as err: + resp = err + return resp.code, resp.headers, resp.read() + + +def actor(label): + username = f"fo_{label}_{uuid.uuid4().hex[:10]}" + password = "T3st-" + uuid.uuid4().hex + status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) + assert status == 201 + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert status == 201 + return json.loads(data)["id"], json.loads(data2)["token"] + + +def api_containers(): + rows = sh(["docker", "ps", "-a", "--filter", "label=com.docker.compose.service=api", + "--filter", "label=com.docker.compose.project=mtusi2026lab4", + "--format", "{{.ID}}"]).split() + return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): + sh(["docker", "inspect", "-f", "{{.Name}} {{.State.Status}}", c]).strip() for c in rows}, rows + + +def observe(token, n=30): + return {request("GET", "/api/v1/users/me", token=token)[1].get("X-Instance-Id") for _ in range(n)} + + +def wait_two(token, timeout=90): + deadline = time.time() + timeout + while time.time() < deadline: + try: + if len(observe(token, 20)) >= 2: + return + except OSError: + pass + time.sleep(1) + raise RuntimeError("two replicas not observed") + + +def one_round(index, alice, bob, chat_id, seen_targets): + data = imagegen.rgb_png(700, 500, noise=True) # ~1 MiB, incompressible + digest = hashlib.sha256(data).hexdigest() + for _ in range(40): + status, h, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=failover{index}.png", + token=alice, raw=data, headers={"Content-Type": "image/png"}) + assert status == 201, status + target = h.get("X-Instance-Id") + if target not in seen_targets: + break + att = json.loads(body) + status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "", "attachment_ids": [att["id"]]}, + token=alice, headers={"Idempotency-Key": uuid.uuid4().hex}) + assert status == 201 + before, ids = api_containers() + victim = next(c for c in ids if sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip() == target) + t0 = time.monotonic() + sh(["docker", "stop", "-t", "15", victim]) + stop_seconds = time.monotonic() - t0 + downloads, first_ok, errors = [], None, 0 + deadline = time.monotonic() + 30 + while time.monotonic() < deadline and len(downloads) < 20: + try: + status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob) + except OSError: + errors += 1 + continue + if status == 200: + if first_ok is None: + first_ok = time.monotonic() - t0 + downloads.append({"instance": h.get("X-Instance-Id"), "sha256_ok": hashlib.sha256(content).hexdigest() == digest}) + else: + errors += 1 + after, _ = api_containers() + meta_status = request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[0] + history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2]) + linked = any(m["id"] == json.loads(msg)["id"] and m["attachment_ids"] == [att["id"]] for m in history["items"]) + result = { + "round": index, + "upload_served_by": target, + "stopped_container_state_after": after.get(target), + "docker_stop_seconds": round(stop_seconds, 2), + "first_download_ok_after_stop_s": round(first_ok, 3) if first_ok is not None else None, + "downloads": len(downloads), + "download_instances": sorted({d["instance"] for d in downloads}), + "target_served_any_download": any(d["instance"] == target for d in downloads), + "all_sha256_match": all(d["sha256_ok"] for d in downloads), + "errors_during_window": errors, + "metadata_status": meta_status, + "message_link_preserved": linked, + "containers_before": before, + "containers_after": after, + } + sh(COMPOSE + ["up", "-d", "--no-deps", "--scale", "api=2", "api"]) + wait_two(alice) + seen_targets.add(target) + return result + + +def main(): + alice_id, alice = actor("a") + bob_id, bob = actor("b") + status, _, data = request("POST", "/api/v1/chats", {"title": "failover", "member_ids": [alice_id, bob_id]}, token=alice) + assert status == 201 + chat_id = json.loads(data)["id"] + wait_two(alice) + seen = set() + rounds = [one_round(i, alice, bob, chat_id, seen) for i in range(2)] + ok = all(r["downloads"] >= 20 and r["all_sha256_match"] and not r["target_served_any_download"] + and r["metadata_status"] == 200 and r["message_link_preserved"] for r in rounds) + report = {"date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "passed": ok, "rounds": rounds} + out = ROOT / "evidence" / "attachment-failover.json" + out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") + for r in rounds: + print(json.dumps({k: r[k] for k in ("round", "upload_served_by", "stopped_container_state_after", + "first_download_ok_after_stop_s", "downloads", "download_instances", + "target_served_any_download", "all_sha256_match", "errors_during_window")})) + print("PASS" if ok else "FAIL", "attachment failover;", "WROTE", out) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/decoder_crosscheck.py b/scripts/decoder_crosscheck.py new file mode 100644 index 00000000..c2f02bff --- /dev/null +++ b/scripts/decoder_crosscheck.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Cross-check the student test corpus against independent decoders. + +Corrupted inputs (student-tests/imagegen.py) must NOT decode, valid ones must. +PNG → Pillow full load(); JPEG → libjpeg-turbo `djpeg -strict` (what the API runs). +Run in a throwaway container (needs Pillow + libjpeg-turbo-progs): + + docker run --rm -v "$PWD:/w" -w /w python:3.12.8-slim-bookworm sh -c \ + "apt-get update -qq && apt-get install -y -qq libjpeg-turbo-progs >/dev/null; \ + pip install -q Pillow==11.1.0; python scripts/decoder_crosscheck.py" +""" + +import io +import os +import subprocess +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "student-tests")) +import imagegen # noqa: E402 +from PIL import Image # noqa: E402 + +# PNG spec errors (PLTE length/placement, unknown critical chunk) that Pillow tolerates; +# libpng and the API reject them. +SPEC_ONLY = {"plte_not_multiple_of_3", "plte_too_long_for_1bit", "plte_in_grayscale", "unknown_critical_chunk"} + + +def pillow_ok(data: bytes) -> bool: + try: + img = Image.open(io.BytesIO(data)) + img.load() + return True + except Exception: + return False + + +def djpeg_ok(data: bytes) -> bool: + with tempfile.NamedTemporaryFile(delete=False) as f: + f.write(data) + try: + cmd = ["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, f.name] + return subprocess.run(cmd, capture_output=True).returncode == 0 + finally: + os.unlink(f.name) + + +def main() -> int: + mismatches = 0 + for name, data in imagegen.corrupted_pngs().items(): + ok = pillow_ok(data) + note = " (PNG spec violation tolerated by Pillow)" if ok and name in SPEC_ONLY else "" + mismatches += ok and name not in SPEC_ONLY + print(f"corrupt png {name:32} pillow_decodes={ok}{note}") + for name, data in imagegen.corrupted_jpegs().items(): + ok = djpeg_ok(data) + mismatches += ok + print(f"corrupt jpeg {name:32} djpeg_strict_decodes={ok} pillow_decodes={pillow_ok(data)}") + for interlace in (0, 1): + for size in ((1, 1), (9, 7), (33, 20)): + ok = pillow_ok(imagegen.all_filters_png(*size, interlace=interlace)) + mismatches += not ok + print(f"valid png all_filters interlace={interlace} {size} pillow_decodes={ok}") + for name in ("sample.jpg", "progressive.jpg", "gray.jpg"): + ok = djpeg_ok(imagegen.fixture(name)) + mismatches += not ok + print(f"valid jpeg {name:32} djpeg_strict_decodes={ok}") + print("PASS" if mismatches == 0 else f"FAIL mismatches={mismatches}") + return 1 if mismatches else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/full_restart_evidence.py b/scripts/full_restart_evidence.py new file mode 100644 index 00000000..5ddca3ba --- /dev/null +++ b/scripts/full_restart_evidence.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Whole-stack restart: `docker compose down` (volumes kept) → `up` → same token reads the same file. + +Stronger than scripts/restart.*: every container (API, LB, PostgreSQL, Redis, SeaweedFS, +s3-gw, one-shot bootstrap jobs) is removed and recreated; only named volumes survive. +Usage: python scripts/full_restart_evidence.py (writes evidence/full-restart.json) +""" + +from __future__ import annotations + +import base64 +import hashlib +import json +import os +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def sh(args) -> str: + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True).stdout + + +def request(method, path, body=None, token=None, raw=None, headers=None): + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + if body is not None: + raw = json.dumps(body).encode() + h["Content-Type"] = "application/json" + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=10) + except urllib.error.HTTPError as err: + resp = err + return resp.code, resp.headers, resp.read() + + +def actor(label): + username = f"rs_{label}_{uuid.uuid4().hex[:10]}" + password = "T3st-" + uuid.uuid4().hex + status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) + assert status == 201 + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + status, _, data2 = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert status == 201 + return json.loads(data)["id"], json.loads(data2)["token"] + + +def snapshot(): + rows = sh(COMPOSE + ["ps", "-a", "--format", "{{.Service}} {{.ID}} {{.State}}"]).splitlines() + return sorted(rows) + + +def main(): + alice_id, alice = actor("a") + bob_id, bob = actor("b") + status, _, data = request("POST", "/api/v1/chats", {"title": "restart", "member_ids": [alice_id, bob_id]}, token=alice) + chat_id = json.loads(data)["id"] + payload = imagegen.rgb_png(600, 400, noise=True) + status, _, body = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=keep.png", token=alice, + raw=payload, headers={"Content-Type": "image/png"}) + assert status == 201, status + att = json.loads(body) + status, _, msg = request("POST", f"/api/v1/chats/{chat_id}/messages", {"text": "переживёт restart\u0000ok", "attachment_ids": [att["id"]]}, + token=alice, headers={"Idempotency-Key": uuid.uuid4().hex}) + assert status == 201 + message = json.loads(msg) + before = snapshot() + volumes = sh(["docker", "volume", "ls", "--filter", "label=com.docker.compose.project=mtusi2026lab4", "--format", "{{.Name}}"]).split() + t0 = time.monotonic() + sh(COMPOSE + ["down"]) + down_s = time.monotonic() - t0 + after_down = snapshot() + sh(COMPOSE + ["up", "-d", "--scale", "api=2"]) + deadline = time.monotonic() + 180 + while True: + try: + if request("GET", "/health/ready")[0] == 200 and request("GET", "/api/v1/users/me", token=bob)[0] == 200: + break + except OSError: + pass + if time.monotonic() > deadline: + raise SystemExit("FAIL: stack not ready after restart") + time.sleep(1) + recovery_s = time.monotonic() - t0 + after = snapshot() + meta = json.loads(request("GET", f"/api/v1/attachments/{att['id']}", token=bob)[2]) + status, h, content = request("GET", f"/api/v1/attachments/{att['id']}/content", token=bob) + history = json.loads(request("GET", f"/api/v1/chats/{chat_id}/messages?limit=100", token=bob)[2]) + s3_init_log = sh(COMPOSE + ["logs", "--no-log-prefix", "s3-init"]).strip().splitlines()[-3:] + result = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "volumes_kept": volumes, + "containers_before": before, + "containers_after_down": after_down, + "containers_after_up": after, + "down_seconds": round(down_s, 1), + "down_to_ready_seconds": round(recovery_s, 1), + "same_token_works": True, + "metadata_unchanged": meta == att, + "download_status": status, + "sha256_match": hashlib.sha256(content).hexdigest() == att["sha256"] == hashlib.sha256(payload).hexdigest(), + "message_preserved": any(m == message for m in history["items"]), + "s3_init_rerun_log_tail": s3_init_log, + } + ok = result["metadata_unchanged"] and status == 200 and result["sha256_match"] and result["message_preserved"] + result["passed"] = ok + out = ROOT / "evidence" / "full-restart.json" + out.write_text(json.dumps(result, ensure_ascii=False, indent=1), encoding="utf-8") + print(json.dumps({k: result[k] for k in ("down_to_ready_seconds", "metadata_unchanged", "download_status", + "sha256_match", "message_preserved", "passed")})) + print("WROTE", out) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..def47281 --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Create local synthetic secrets in ./secrets (idempotent, prints only file names). + +Existing files are kept, so re-running never rotates credentials of a running +stack. secrets/s3_config.json (SeaweedFS identities) is always re-derived from the +key files and S3_BUCKET (.env or environment): + * admin — Admin/Read/Write/List, used only by the one-shot s3-init; + * messenger-api — Read/Write/List on the attachments bucket only, used by API; + * no "anonymous" identity → unauthenticated S3 requests are denied. +""" + +from __future__ import annotations + +import json +import os +import secrets +import string +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" +ALNUM = string.ascii_letters + string.digits + + +def token(length: int, alphabet: str = ALNUM) -> str: + return "".join(secrets.choice(alphabet) for _ in range(length)) + + +def bucket_name() -> str: + value = os.environ.get("S3_BUCKET", "").strip() + env_file = ROOT / ".env" + if not value and env_file.is_file(): + for line in env_file.read_text(encoding="utf-8").splitlines(): + if line.startswith("S3_BUCKET="): + value = line.split("=", 1)[1].strip() + return value or "messenger-attachments" + + +def ensure(name: str, factory) -> str: + path = SECRETS / name + if path.is_file() and path.read_text(encoding="utf-8").strip(): + print("keep secrets/" + name) + return path.read_text(encoding="utf-8").strip() + value = factory() + path.write_bytes(value.encode("utf-8")) + print("create secrets/" + name) + return value + + +def main() -> int: + SECRETS.mkdir(exist_ok=True) + pg_password = ensure("postgres_password", lambda: token(32)) + ensure("database_url", lambda: f"postgresql://messenger:{pg_password}@postgres:5432/messenger") + ensure("cache_url", lambda: "redis://redis:6379/0") + admin_key = ensure("s3_admin_access_key_id", lambda: "ADM" + token(17, string.ascii_uppercase + string.digits)) + admin_secret = ensure("s3_admin_secret_access_key", lambda: token(40)) + api_key = ensure("s3_access_key_id", lambda: "API" + token(17, string.ascii_uppercase + string.digits)) + api_secret = ensure("s3_secret_access_key", lambda: token(40)) + bucket = bucket_name() + config = { + "identities": [ + { + "name": "admin", + "credentials": [{"accessKey": admin_key, "secretKey": admin_secret}], + "actions": ["Admin", "Read", "Write", "List", "Tagging"], + }, + { + "name": "messenger-api", + "credentials": [{"accessKey": api_key, "secretKey": api_secret}], + "actions": [f"Read:{bucket}", f"Write:{bucket}", f"List:{bucket}"], + }, + ] + } + (SECRETS / "s3_config.json").write_bytes(json.dumps(config, indent=1).encode("utf-8")) + print(f"write secrets/s3_config.json (bucket {bucket}; identities admin, messenger-api; no anonymous)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/jpeg_decode_memory.py b/scripts/jpeg_decode_memory.py new file mode 100644 index 00000000..8769ca2b --- /dev/null +++ b/scripts/jpeg_decode_memory.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Peak RSS of `djpeg -strict -scale 1/8` on worst-case 40 Mpx JPEGs (sizing the API limit). + +Run in a throwaway container with Pillow + libjpeg-turbo-progs: + python scripts/jpeg_decode_memory.py gen # writes /tmp/j/*.jpg (8000×5000) + python scripts/jpeg_decode_memory.py measure # one fresh child per file +The measuring parent stays tiny so RUSAGE_CHILDREN reflects djpeg only. +""" + +import os +import resource +import subprocess +import sys +import time + +OUT = "/tmp/j" + + +def gen() -> None: + from PIL import Image + + os.makedirs(OUT, exist_ok=True) + noise = Image.effect_noise((8000, 5000), 64).convert("L") + rgb = Image.merge("RGB", (noise, noise.rotate(180), noise.transpose(Image.FLIP_LEFT_RIGHT))) + for name, img, kw in ( + ("baseline_420", rgb, dict(quality=30)), + ("progressive_420", rgb, dict(quality=30, progressive=True)), + ("progressive_444", rgb, dict(quality=20, progressive=True, subsampling=0)), + ("progressive_cmyk", rgb.convert("CMYK"), dict(quality=15, progressive=True, subsampling=0)), + ): + img.save(os.path.join(OUT, name + ".jpg"), "JPEG", **kw) + + +def measure_one(path: str) -> None: + t0 = time.monotonic() + proc = subprocess.run(["djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, path], capture_output=True) + took = time.monotonic() - t0 + peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss / 1024 + print(f"{os.path.basename(path):22} {os.path.getsize(path) / 2**20:6.2f} MiB rc={proc.returncode} " + f"{took:5.2f}s djpeg_peak_rss={peak:.1f} MiB", flush=True) + + +if __name__ == "__main__": + if sys.argv[1] == "gen": + gen() + elif sys.argv[1] == "one": + measure_one(sys.argv[2]) + else: + for name in sorted(os.listdir(OUT)): + subprocess.run([sys.executable, __file__, "one", os.path.join(OUT, name)], check=True) diff --git a/scripts/lb_interleave_evidence.py b/scripts/lb_interleave_evidence.py new file mode 100644 index 00000000..c283a1ee --- /dev/null +++ b/scripts/lb_interleave_evidence.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""LB balancing under mixed traffic: slow uploads interleaved with short GETs. + +Each of 8 uploads (~1 MiB PNG) streams its body over ~4 s; between upload starts the +client sends one short GET. Records which replica served every upload and GET. +The balance algorithm is read from the running LB config. Results are merged into +evidence/lb-interleave.json keyed by algorithm. Tokens are kept in memory only. +""" +from __future__ import annotations + +import base64 +import collections +import http.client +import json +import os +import subprocess +import sys +import threading +import time +import uuid +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 + +HOST = os.environ.get("LB_HOST", "127.0.0.1") +PORT = int(os.environ.get("LB_PORT", "8080")) +UPLOADS = 8 + + +def call(method, path, body=None, token=None, headers=None): + h = {"Accept": "application/json", **(headers or {})} + data = None + if body is not None: + data = json.dumps(body).encode() + h["Content-Type"] = "application/json" + if token: + h["Authorization"] = "Bearer " + token + req = urllib.request.Request(f"http://{HOST}:{PORT}{path}", data=data, headers=h, method=method) + resp = urllib.request.urlopen(req, timeout=30) + return resp.headers, json.loads(resp.read() or b"null") + + +def actor(label): + username, password = f"ilv_{label}_{uuid.uuid4().hex[:10]}", "T3st-" + uuid.uuid4().hex + _, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + _, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + return user["id"], session["token"] + + +def balance_algorithm(): + cfg = subprocess.check_output( + ["docker", "compose", "--env-file", ".env", "exec", "-T", "lb", + "cat", "/usr/local/etc/haproxy/haproxy.cfg"], cwd=ROOT, text=True) + algos = {line.split()[1] for line in cfg.splitlines() if line.strip().startswith("balance ")} + return "/".join(sorted(algos)) + + +def main(): + algo = balance_algorithm() + alice, token = actor("a") + bob, _ = actor("b") + _, chat = call("POST", "/api/v1/chats", {"title": "interleave", "member_ids": [alice, bob]}, token=token) + payload = imagegen.rgb_png(700, 500, noise=True) + uploads, gets = [], [] + + def slow_upload(i): + conn = http.client.HTTPConnection(HOST, PORT, timeout=60) + conn.putrequest("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=ilv{i}.png") + for k, v in {"Authorization": "Bearer " + token, "Content-Type": "image/png", + "Content-Length": str(len(payload))}.items(): + conn.putheader(k, v) + conn.endheaders() + step = len(payload) // 20 + for off in range(0, len(payload), step): + conn.send(payload[off:off + step]) + time.sleep(0.2) + resp = conn.getresponse() + resp.read() + uploads.append({"upload": i, "status": resp.status, "instance": resp.getheader("X-Instance-Id")}) + + threads = [] + for i in range(UPLOADS): + th = threading.Thread(target=slow_upload, args=(i,)) + th.start() + threads.append(th) + time.sleep(0.15) + gets.append(call("GET", "/api/v1/users/me", token=token)[0]["X-Instance-Id"]) + time.sleep(0.15) + for th in threads: + th.join() + result = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "balance": algo, + "uploads": UPLOADS, + "upload_bytes": len(payload), + "uploads_per_replica": dict(collections.Counter(u["instance"] for u in uploads)), + "upload_statuses": dict(collections.Counter(str(u["status"]) for u in uploads)), + "short_gets_per_replica": dict(collections.Counter(gets)), + } + out = ROOT / "evidence" / "lb-interleave.json" + report = json.loads(out.read_text(encoding="utf-8")) if out.exists() else {} + report[algo] = result + out.write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") + print(json.dumps(result, ensure_ascii=False)) + print("WROTE", out) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..e3fbcdad --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,18 @@ +@echo off +setlocal +rem Restart API replicas and ALL state stores (PostgreSQL, Redis, SeaweedFS S3); volumes are preserved. +cd /d "%~dp0\.." +if "%API_REPLICAS%"=="" set API_REPLICAS=2 +docker compose --env-file .env restart postgres redis s3 +if errorlevel 1 exit /b 1 +rem s3-gw shares the s3 network namespace and must be restarted after it. +docker compose --env-file .env restart s3-gw +if errorlevel 1 exit /b 1 +rem "up -d" alone leaves running replicas untouched; restart them, then restore the scale. +docker compose --env-file .env restart api +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --no-deps --scale api=%API_REPLICAS% api +if errorlevel 1 exit /b 1 +docker compose --env-file .env restart lb +if errorlevel 1 exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..5c69be89 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +# Restart API replicas and ALL state stores (PostgreSQL, Redis, SeaweedFS S3); preserve volumes. +cd "$(dirname "$0")/.." +REPLICAS="${API_REPLICAS:-2}" +docker compose --env-file .env restart postgres redis s3 +# s3-gw shares the s3 network namespace and must be restarted after it. +docker compose --env-file .env restart s3-gw +# "up -d" alone leaves running replicas untouched; restart them, then restore the scale +# (migrate/s3-init already applied; skip one-shots). +docker compose --env-file .env restart api +docker compose --env-file .env up -d --no-deps --scale "api=${REPLICAS}" api +docker compose --env-file .env restart lb +exit 0 diff --git a/scripts/rolling-update.cmd b/scripts/rolling-update.cmd new file mode 100644 index 00000000..fb04e392 --- /dev/null +++ b/scripts/rolling-update.cmd @@ -0,0 +1,51 @@ +@echo off +setlocal EnableExtensions EnableDelayedExpansion +rem Grade-4 rolling replacement: replace API containers one-by-one under LB. +rem Usage: scripts\rolling-update.cmd +cd /d "%~dp0\.." +set "REPLICAS=%API_REPLICAS%" +if "%REPLICAS%"=="" set "REPLICAS=2" +set "URL=http://127.0.0.1:8080/health/ready" + +set "IDS=" +for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do set "IDS=!IDS! %%I" +set "KNOWN=%IDS%" +set /a COUNT=0 +for %%I in (%IDS%) do ( + set /a COUNT+=1 + echo rolling: replacing %%I + rem -t 15 = stop_grace_period: the replica drains ~3 s before it exits. + docker stop -t 15 %%I >nul || exit /b 1 + docker rm %%I >nul || exit /b 1 + rem --no-recreate: the surviving replica keeps serving even if the image/config changed. + docker compose --env-file .env up -d --no-deps --no-recreate --scale api=%REPLICAS% api || exit /b 1 + call :wait_new || exit /b 1 +) +echo rolling update finished (%COUNT% steps, scale=%REPLICAS%) +exit /b 0 + +:wait_new +set "NEW=" +for /f "usebackq delims=" %%J in (`docker compose --env-file .env ps -q api`) do ( + if "!KNOWN:%%J=!"=="!KNOWN!" set "NEW=%%J" +) +if "%NEW%"=="" ( + echo rolling: expected one new replica >&2 + exit /b 1 +) +set "KNOWN=%KNOWN% %NEW%" +for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %NEW%`) do set "HOST=%%H" +rem Public ready is 200 while the other replica serves; wait until the LB routes to the new one. +set /a TRIES=0 +:poll +curl.exe -s -D - -o NUL "%URL%" | findstr /I /B /C:"x-instance-id: %HOST%" >nul && goto ready +set /a TRIES+=1 +if %TRIES% geq 90 ( + echo rolling: %HOST% not in LB rotation >&2 + exit /b 1 +) +ping -n 2 127.0.0.1 >nul +goto poll +:ready +echo rolling: step %COUNT% ready (%HOST%) +exit /b 0 diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..b0865a96 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,35 @@ +#!/bin/sh +set -eu +# Grade-4 rolling replacement: replace API containers one-by-one under LB. +# Usage: scripts/rolling-update.sh +cd "$(dirname "$0")/.." +REPLICAS="${API_REPLICAS:-2}" +URL="http://127.0.0.1:8080/health/ready" + +ids=$(docker compose --env-file .env ps -q api | tr -d '\r') +known="$ids" +count=0 +for id in $ids; do + [ -n "$id" ] || continue + count=$((count + 1)) + printf '%s\n' "rolling: replacing $id" + # -t 15 = stop_grace_period: the replica drains ~3 s before it exits. + docker stop -t 15 "$id" >/dev/null + docker rm "$id" >/dev/null + # --no-recreate: the surviving replica keeps serving even if the image/config changed. + docker compose --env-file .env up -d --no-deps --no-recreate --scale "api=${REPLICAS}" api + new=$(docker compose --env-file .env ps -q api | tr -d '\r' | grep -v -F "$known" || true) + [ "$(printf '%s\n' "$new" | grep -c .)" -eq 1 ] || { printf '%s\n' "rolling: expected one new replica" >&2; exit 1; } + known="$known +$new" + host=$(docker inspect -f '{{.Config.Hostname}}' "$new" | tr -d '\r') + # Public ready is 200 while the other replica serves; wait until the LB routes to the new one. + i=0 + until curl -s -D - -o /dev/null "$URL" | tr -d '\r' | grep -qi "^x-instance-id: ${host}\$"; do + i=$((i + 1)) + [ "$i" -lt 180 ] || { printf '%s\n' "rolling: $host not in LB rotation" >&2; exit 1; } + sleep 0.5 + done + printf '%s\n' "rolling: step $count ready ($host)" +done +printf '%s\n' "rolling update finished ($count steps, scale=${REPLICAS})" diff --git a/scripts/rolling_load_evidence.py b/scripts/rolling_load_evidence.py new file mode 100644 index 00000000..64a26bf8 --- /dev/null +++ b/scripts/rolling_load_evidence.py @@ -0,0 +1,170 @@ +#!/usr/bin/env python3 +"""Grade-4: closed-loop POSTs during rolling API replacement.""" +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import threading +import time +import uuid +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +REPLICAS = os.environ.get("API_REPLICAS", "2") + + +def call(method, path, body=None, token=None, headers=None): + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + h["Content-Type"] = "application/json" + if headers: + h.update(headers) + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=5) + except urllib.error.HTTPError as err: + resp = err + except Exception as exc: + return {"status": 0, "error": type(exc).__name__} + raw = resp.read() + js = None + if raw and resp.headers.get_content_type() == "application/json": + js = json.loads(raw.decode("utf-8")) + return {"status": resp.code, "json": js, "instance": resp.headers.get("X-Instance-Id", "")} + + +def api_ids(): + out = subprocess.check_output( + ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"], cwd=ROOT, text=True + ) + return out.split() + + +def hostname(cid): + return subprocess.check_output( + ["docker", "inspect", "-f", "{{.Config.Hostname}}", cid], text=True + ).strip() + + +def replace_replica(old_id, known): + """Stop+remove one replica, create a fresh one, wait until the LB routes to it.""" + subprocess.check_call(["docker", "stop", "-t", "15", old_id], stdout=subprocess.DEVNULL) + subprocess.check_call(["docker", "rm", old_id], stdout=subprocess.DEVNULL) + # --no-recreate: the surviving replica keeps serving even if the image/config changed. + subprocess.check_call( + ["docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate", + "--scale", f"api={REPLICAS}", "api"], + cwd=ROOT, + ) + new = [cid for cid in api_ids() if cid not in known] + if len(new) != 1: + raise RuntimeError(f"expected one new replica, got {new}") + host = hostname(new[0]) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + r = call("GET", "/health/ready") + if r.get("status") == 200 and r.get("instance") == host: + return new[0], host + time.sleep(0.5) + raise RuntimeError(f"new replica {host} not in LB rotation after 90s") + + +def bootstrap(): + uname = "roll_" + uuid.uuid4().hex[:10] + pw = "T3st-" + uuid.uuid4().hex + user = call("POST", "/api/v1/users", {"username": uname, "display_name": "Roll", "password": pw}) + assert user["status"] == 201, user + cred = base64.b64encode(f"{uname}:{pw}".encode()).decode() + tok = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})["json"]["token"] + peer_u = "rollb_" + uuid.uuid4().hex[:10] + peer = call("POST", "/api/v1/users", {"username": peer_u, "display_name": "RollB", "password": pw})["json"] + chat = call( + "POST", + "/api/v1/chats", + {"title": "roll", "member_ids": [user["json"]["id"], peer["id"]]}, + token=tok, + )["json"] + return tok, chat["id"] + + +def main(): + tok, chat_id = bootstrap() + stats = {"ok": 0, "err": 0, "err_by_status": {}, "latencies_ms": []} + stop = threading.Event() + + def worker(): + while not stop.is_set(): + key = uuid.uuid4().hex + t0 = time.monotonic() + r = call( + "POST", + f"/api/v1/chats/{chat_id}/messages", + {"text": "roll-" + key}, + token=tok, + headers={"Idempotency-Key": key}, + ) + dt = (time.monotonic() - t0) * 1000 + if r.get("status") == 201: + stats["ok"] += 1 + stats["latencies_ms"].append(dt) + else: + stats["err"] += 1 + msg = ((r.get("json") or {}).get("error") or {}).get("message", "") + code = f"{r.get('status') or r.get('error')} {msg}".strip() + stats["err_by_status"][code] = stats["err_by_status"].get(code, 0) + 1 + time.sleep(0.05) + + ids = api_ids() + known = set(ids) + replaced = [] + threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)] + started = time.monotonic() + for th in threads: + th.start() + time.sleep(2) # warm-up + # Rolling: replace each original replica in turn; the next one is stopped + # only after the LB already routes requests to the fresh container. + for cid in ids: + old_host = hostname(cid) + new_id, new_host = replace_replica(cid, known) + known.add(new_id) + replaced.append({"old": old_host, "new": new_host}) + time.sleep(2) + time.sleep(2) + stop.set() + for th in threads: + th.join(timeout=5) + duration = time.monotonic() - started + lats = sorted(stats["latencies_ms"]) + def pct(p): + if not lats: + return None + return round(lats[min(len(lats) - 1, int(len(lats) * p))], 2) + summary = { + "successful": stats["ok"], + "errors": stats["err"], + "errors_by_status": stats["err_by_status"], + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + "duration_s": round(duration, 1), + "replicas_rolled": len(replaced), + "replaced": replaced, + } + out = ROOT / "evidence" / "rolling-load.json" + out.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps(summary, ensure_ascii=False)) + print("WROTE", out) + + +if __name__ == "__main__": + main() diff --git a/scripts/seq_failover_evidence.py b/scripts/seq_failover_evidence.py new file mode 100644 index 00000000..7aa1a031 --- /dev/null +++ b/scripts/seq_failover_evidence.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Grade-4 evidence: stop each observed API replica in turn; restore between steps.""" +from __future__ import annotations + +import base64 +import json +import os +import subprocess +import time +import uuid +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +def call(method, path, body=None, token=None, headers=None, expected=None): + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + h["Content-Type"] = "application/json" + if headers: + h.update(headers) + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=5) + except urllib.error.HTTPError as err: + resp = err + raw = resp.read() + status = resp.code + if expected is not None and status != expected: + raise RuntimeError(f"{method} {path}: expected {expected}, got {status}: {raw[:200]!r}") + js = None + if raw and resp.headers.get_content_type() == "application/json": + js = json.loads(raw.decode("utf-8")) + return {"status": status, "headers": resp.headers, "json": js} + + +def header(resp, name: str) -> str: + return resp["headers"].get(name) or resp["headers"].get(name.lower()) or "" + + +def restore_api(target: str): + replicas = os.environ.get("API_REPLICAS", "2") + subprocess.check_call( + [ + "docker", + "compose", + "--env-file", + ".env", + "up", + "-d", + "--no-deps", + "--scale", + f"api={replicas}", + "--remove-orphans", + "api", + ], + cwd=ROOT, + ) + # Public ready is 200 via the surviving replica; wait until the LB routes to the restored one. + for _ in range(180): + try: + if header(call("GET", "/health/ready", expected=200), "X-Instance-Id") == target: + return + except Exception: + pass + time.sleep(0.5) + raise RuntimeError(f"replica {target} not back in LB rotation after restore") + + +def one_round(round_no: int) -> dict: + uname = "seq_" + uuid.uuid4().hex[:10] + pw = "T3st-" + uuid.uuid4().hex + user = call( + "POST", + "/api/v1/users", + {"username": uname, "display_name": "Seq", "password": pw}, + expected=201, + )["json"] + cred = base64.b64encode(f"{uname}:{pw}".encode()).decode() + tok = call( + "POST", + "/api/v1/auth/sessions", + headers={"Authorization": "Basic " + cred}, + expected=201, + )["json"]["token"] + peer_u = "seqb_" + uuid.uuid4().hex[:10] + peer = call( + "POST", + "/api/v1/users", + {"username": peer_u, "display_name": "SeqB", "password": pw}, + expected=201, + )["json"] + chat = call( + "POST", + "/api/v1/chats", + {"title": "seq", "member_ids": [user["id"], peer["id"]]}, + token=tok, + expected=201, + )["json"] + + observed = set() + for _ in range(40): + observed.add(header(call("GET", "/api/v1/users/me", token=tok, expected=200), "X-Instance-Id")) + if len(observed) < 2: + raise RuntimeError(f"need ≥2 instances, got {observed}") + target = sorted(observed)[round_no % len(observed)] + + env = dict(os.environ) + env["TARGET_INSTANCE_ID"] = target + script = ROOT / ("scripts/stop-one.cmd" if os.name == "nt" else "scripts/stop-one.sh") + started = time.monotonic() + proc = subprocess.Popen([str(script)], env=env, cwd=ROOT) + ok = fail = 0 + first = None + after = [] + while time.monotonic() - started < 20: + code = proc.poll() + try: + key = uuid.uuid4().hex + r = call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + {"text": "seq-" + key}, + token=tok, + headers={"Idempotency-Key": key}, + expected=201, + ) + ok += 1 + if code == 0: + if first is None: + first = time.monotonic() - started + after.append(header(r, "X-Instance-Id")) + except Exception: + fail += 1 + time.sleep(0.1) + if proc.poll() is None: + proc.wait(timeout=10) + result = { + "round": round_no, + "observed_before": sorted(observed), + "target": target, + "successful_writes": ok, + "failed_attempts": fail, + "first_recovery_seconds": round(first, 3) if first is not None else None, + "target_absent_in_last5": target not in after[-5:], + } + t0 = time.monotonic() + restore_api(target) + result["restore_to_lb_seconds"] = round(time.monotonic() - t0, 3) + return result + + +def main(): + results = [] + for i in range(2): + results.append(one_round(i)) + print(json.dumps(results[-1], ensure_ascii=False)) + out = ROOT / "evidence" / "seq-failover.json" + out.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8") + print("WROTE", out) + + +if __name__ == "__main__": + main() diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..ed006290 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,17 @@ +@echo off +setlocal EnableExtensions +cd /d "%~dp0\.." +if "%TARGET_INSTANCE_ID%"=="" ( + echo stop-one: TARGET_INSTANCE_ID required >&2 + exit /b 1 +) +for /f "usebackq delims=" %%I in (`docker compose --env-file .env ps -q api`) do ( + for /f "usebackq delims=" %%H in (`docker inspect -f "{{.Config.Hostname}}" %%I`) do ( + if /I "%%H"=="%TARGET_INSTANCE_ID%" ( + docker stop -t 15 %%I >nul + exit /b 0 + ) + ) +) +echo stop-one: replica not found >&2 +exit /b 1 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..42e43862 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,44 @@ +#!/bin/sh +set -eu +# Stop ONE API replica matching TARGET_INSTANCE_ID (= X-Instance-Id / hostname). +# Leave it stopped; do not touch DB/Redis/LB/volumes. +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." + +found="" +# Compose project containers for service api +ids=$(docker compose --env-file .env ps -q api 2>/dev/null || true) +for id in $ids; do + [ -n "$id" ] || continue + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + if [ "$host" = "$TARGET_INSTANCE_ID" ]; then + found=$id + break + fi + iid=$(docker inspect -f '{{range .Config.Env}}{{println .}}{{end}}' "$id" | sed -n 's/^INSTANCE_ID=//p' | head -n1) + if [ -n "$iid" ] && [ "$iid" = "$TARGET_INSTANCE_ID" ]; then + found=$id + break + fi +done + +if [ -z "$found" ]; then + # Fallback: any running container with matching hostname (scaled names differ). + for id in $(docker ps -q --filter "ancestor=mtusi-messenger:lab4"); do + host=$(docker inspect -f '{{.Config.Hostname}}' "$id") + if [ "$host" = "$TARGET_INSTANCE_ID" ]; then + found=$id + break + fi + done +fi + +if [ -z "$found" ]; then + printf '%s\n' "stop-one: replica not found for TARGET_INSTANCE_ID" >&2 + exit 1 +fi + +# Graceful stop (= stop_grace_period): on SIGTERM the API reports not-ready but keeps +# serving for a 3 s drain window, then finishes in-flight requests and exits (~3.5 s). +docker stop -t 15 "$found" >/dev/null +exit 0 diff --git a/scripts/up.cmd b/scripts/up.cmd new file mode 100644 index 00000000..05e98b85 --- /dev/null +++ b/scripts/up.cmd @@ -0,0 +1,12 @@ +@echo off +setlocal +cd /d "%~dp0\.." +if "%API_REPLICAS%"=="" set API_REPLICAS=2 +python scripts\init_secrets.py +if errorlevel 1 exit /b 1 +docker compose --env-file .env config --quiet +if errorlevel 1 exit /b 1 +docker compose --env-file .env up -d --build --scale api=%API_REPLICAS% +if errorlevel 1 exit /b 1 +echo stack up: api scale=%API_REPLICAS%, entry http://127.0.0.1:8080 +exit /b 0 diff --git a/scripts/up.sh b/scripts/up.sh new file mode 100755 index 00000000..a1b90ea5 --- /dev/null +++ b/scripts/up.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +# Bootstrap lab4 stack with ≥2 API replicas behind LB, PostgreSQL, Redis and private S3. +cd "$(dirname "$0")/.." +REPLICAS="${API_REPLICAS:-2}" +python3 scripts/init_secrets.py +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale "api=${REPLICAS}" +printf '%s\n' "stack up: api scale=${REPLICAS}, entry http://127.0.0.1:8080" diff --git a/scripts/upload_memory_experiment.py b/scripts/upload_memory_experiment.py new file mode 100644 index 00000000..b0bb5858 --- /dev/null +++ b/scripts/upload_memory_experiment.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Experiment: peak memory of API replicas, LB and S3 under N simultaneous ~9.5 MiB uploads. + +Hypothesis: streaming intake (64 KiB chunks → validator → temp file → S3 PUT from +file) keeps the API process RSS growth far below N × file size; total in-flight +work per replica is capped by MAX_CONCURRENT_UPLOADS. + +Method (per concurrency level, fresh replicas): + 1. `docker compose restart s3`, then `s3-gw api lb` → new processes; wait for /health/ready; + 2. one warm-up upload per replica (imports, pools, TLS-less S3 connections); + 3. reset VmHWM of PID 1 (echo 5 > /proc/1/clear_refs) and record baseline; + 4. fire N uploads at once through the LB (barrier), sample cgroup memory.stat + (anon/file) of every replica, the LB and the s3 container from the host every ~0.25 s; + 5. read VmHWM (peak RSS of the uvicorn process) and cgroup memory.peak. + +Usage: python scripts/upload_memory_experiment.py [--levels 10 20] [--size-mib 9.5] +Writes evidence/upload-memory.json. Tokens are kept in memory only. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import statistics +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +import imagegen # noqa: E402 + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def sh(args, check=True) -> str: + proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8") + if check and proc.returncode != 0: + raise RuntimeError(f"{args[:4]} rc={proc.returncode}: {proc.stderr[-300:]}") + return proc.stdout + + +def request(method, path, body=None, token=None, raw=None, headers=None, timeout=180): + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + if body is not None: + raw = json.dumps(body).encode() + h["Content-Type"] = "application/json" + h.update(headers or {}) + req = urllib.request.Request(BASE + path, data=raw, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=timeout) + except urllib.error.HTTPError as err: + resp = err + data = resp.read() + return resp.code, resp.headers, data + + +def actor(label): + username = f"mem_{label}_{uuid.uuid4().hex[:10]}" + password = "T3st-" + uuid.uuid4().hex + status, _, data = request("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) + assert status == 201, status + user = json.loads(data) + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert status == 201, status + return user["id"], json.loads(data)["token"] + + +def wait_ready(timeout=120): + deadline = time.time() + timeout + while time.time() < deadline: + try: + if request("GET", "/health/ready", timeout=3)[0] == 200: + seen = {request("GET", "/health/live", timeout=3)[1].get("X-Instance-Id") for _ in range(20)} + if len(seen) >= 2: + return + except OSError: + pass + time.sleep(1) + raise RuntimeError("stack not ready") + + +def containers(): + ids = sh(COMPOSE + ["ps", "-q", "api"]).split() + return {sh(["docker", "inspect", "-f", "{{.Config.Hostname}}", c]).strip(): c for c in ids} + + +def proc_status(cid): + out = sh(["docker", "exec", cid, "cat", "/proc/1/status"]) + fields = {} + for line in out.splitlines(): + if line.startswith(("VmRSS", "VmHWM")): + k, v = line.split(":", 1) + fields[k] = int(v.split()[0]) // 1024 # MiB + return fields + + +def cgroup(cid): + stat = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.stat"]) + vals = dict(line.split() for line in stat.splitlines() if line) + peak = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.peak"], check=False).strip() + current = sh(["docker", "exec", cid, "cat", "/sys/fs/cgroup/memory.current"]).strip() + mib = 1024 * 1024 + return { + "anon_mib": round(int(vals["anon"]) / mib, 1), + "file_mib": round(int(vals["file"]) / mib, 1), + "current_mib": round(int(current) / mib, 1), + "peak_mib": round(int(peak) / mib, 1) if peak.isdigit() else None, + } + + +def sampler(cid, stop, out): + best = {"anon_mib": 0.0, "file_mib": 0.0, "current_mib": 0.0, "samples": 0} + while not stop.is_set(): + try: + c = cgroup(cid) + for k in ("anon_mib", "file_mib", "current_mib"): + best[k] = max(best[k], c[k]) + best["samples"] += 1 + except Exception: + pass + time.sleep(0.25) + out[cid] = best + + +def pct(values, q): + values = sorted(values) + return round(values[min(len(values) - 1, int(round(q * (len(values) - 1))))], 3) + + +def run_level(n, payload, chat_id, token): + print(f"== level {n}: restarting s3, api replicas and lb", flush=True) + # s3-gw shares the s3 network namespace and must be restarted after it. + sh(COMPOSE + ["restart", "s3"]) + sh(COMPOSE + ["restart", "s3-gw", "api", "lb"]) + wait_ready() + by_host = containers() + lb = sh(COMPOSE + ["ps", "-q", "lb"]).split()[0] + s3c = sh(COMPOSE + ["ps", "-q", "s3"]).split()[0] + # Warm-up: at least one upload served by every replica. + served = set() + for _ in range(20): + _, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=warm.png", token=token, + raw=imagegen.rgb_png(64, 64), headers={"Content-Type": "image/png"}) + served.add(h.get("X-Instance-Id")) + if served >= set(by_host): + break + baseline = {} + for host, cid in by_host.items(): + sh(["docker", "exec", cid, "sh", "-c", "echo 5 > /proc/1/clear_refs"]) + baseline[host] = {**proc_status(cid), **cgroup(cid)} + lb_baseline = cgroup(lb) + s3_baseline = cgroup(s3c) + stop = threading.Event() + sampled = {} + threads = [threading.Thread(target=sampler, args=(cid, stop, sampled)) for cid in [*by_host.values(), lb, s3c]] + for t in threads: + t.start() + time.sleep(0.5) + barrier = threading.Barrier(n) + results = [None] * n + + def one(i): + barrier.wait() + t0 = time.monotonic() + try: + status, h, _ = request("POST", f"/api/v1/chats/{chat_id}/attachments?filename=load{i}.png", + token=token, raw=payload, headers={"Content-Type": "image/png"}) + results[i] = (status, time.monotonic() - t0, h.get("X-Instance-Id")) + except OSError as exc: + results[i] = (type(exc).__name__, time.monotonic() - t0, None) + + workers = [threading.Thread(target=one, args=(i,)) for i in range(n)] + started = time.monotonic() + for w in workers: + w.start() + for w in workers: + w.join() + wall = time.monotonic() - started + time.sleep(1.0) + stop.set() + for t in threads: + t.join() + replicas = {} + for host, cid in by_host.items(): + after = proc_status(cid) + cg = cgroup(cid) + replicas[host] = { + "uploads_served": sum(1 for r in results if r[2] == host), + "rss_baseline_mib": baseline[host]["VmRSS"], + "rss_peak_vmhwm_mib": after["VmHWM"], + "rss_peak_growth_mib": after["VmHWM"] - baseline[host]["VmRSS"], + "cgroup_peak_mib": cg["peak_mib"], + "cgroup_baseline_current_mib": baseline[host]["current_mib"], + "sampled_max_anon_mib": sampled.get(cid, {}).get("anon_mib"), + "sampled_max_file_cache_mib": sampled.get(cid, {}).get("file_mib"), + "samples": sampled.get(cid, {}).get("samples"), + } + latencies = [r[1] for r in results if r[0] == 201] + statuses = {} + for r in results: + statuses[str(r[0])] = statuses.get(str(r[0]), 0) + 1 + per_replica_max = max(v["uploads_served"] for v in replicas.values()) + return { + "concurrency": n, + "payload_bytes": len(payload), + "statuses": statuses, + "wall_seconds": round(wall, 3), + "latency_ok_p50_s": pct(latencies, 0.5) if latencies else None, + "latency_ok_p95_s": pct(latencies, 0.95) if latencies else None, + "latency_ok_max_s": round(max(latencies), 3) if latencies else None, + "throughput_mib_s": round(len(latencies) * len(payload) / (1024 * 1024) / wall, 1), + "naive_full_buffer_estimate_mib": round(min(per_replica_max, 8) * len(payload) / (1024 * 1024), 1), + "replicas": replicas, + "lb": { + "mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", lb]).strip()) // (1024 * 1024), + "baseline_current_mib": lb_baseline["current_mib"], + "cgroup_peak_mib": cgroup(lb)["peak_mib"], + "sampled_max_anon_mib": sampled.get(lb, {}).get("anon_mib"), + "sampled_max_current_mib": sampled.get(lb, {}).get("current_mib"), + "restarts": int(sh(["docker", "inspect", "-f", "{{.RestartCount}}", lb]).strip()), + "oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", lb]).strip() == "true", + }, + "s3": _s3_report(s3c, s3_baseline, sampled.get(s3c, {})), + } + + +def _s3_report(cid, baseline, sampled): + running = sh(["docker", "inspect", "-f", "{{.State.Running}}", cid]).strip() == "true" + return { + "mem_limit_mib": int(sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", cid]).strip()) // (1024 * 1024), + "baseline_current_mib": baseline["current_mib"], + # memory.peak includes reclaimable page cache of the volume files; anon is the weed heap. + "cgroup_peak_mib": cgroup(cid)["peak_mib"] if running else None, + "sampled_max_anon_mib": sampled.get("anon_mib"), + "sampled_max_current_mib": sampled.get("current_mib"), + "running": running, + "oom_killed": sh(["docker", "inspect", "-f", "{{.State.OOMKilled}}", cid]).strip() == "true", + } + + +def main(): + p = argparse.ArgumentParser(description="peak memory under concurrent uploads") + p.add_argument("--levels", type=int, nargs="+", default=[10, 20]) + p.add_argument("--size-mib", type=float, default=9.5) + p.add_argument("--output", default=str(ROOT / "evidence" / "upload-memory.json")) + args = p.parse_args() + payload = imagegen.large_png(int(args.size_mib * 1024 * 1024)) + _, token = actor("a") + bob_id, _ = actor("b") + me = json.loads(request("GET", "/api/v1/users/me", token=token)[2]) + status, _, data = request("POST", "/api/v1/chats", {"title": "memory", "member_ids": [me["id"], bob_id]}, token=token) + assert status == 201, status + chat_id = json.loads(data)["id"] + limits = sh(["docker", "inspect", "-f", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", + list(containers().values())[0]]).split() + report = { + "date": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "api_mem_limit_mib": int(limits[0]) // (1024 * 1024), + "api_cpus": int(limits[1]) / 1e9, + "max_concurrent_uploads_per_replica": 8, + "levels": [], + } + for n in args.levels: + level = run_level(n, payload, chat_id, token) + report["levels"].append(level) + print(json.dumps(level, ensure_ascii=False), flush=True) + Path(args.output).write_text(json.dumps(report, ensure_ascii=False, indent=1), encoding="utf-8") + print("WROTE", args.output) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..1947ccb8 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1 @@ +"""Messenger API for lab4 (LB + shared sessions + idempotent messages + S3 attachments).""" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..cfed0da8 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,36 @@ +"""Run the messenger HTTP server.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_SRC = Path(__file__).resolve().parents[1] +if str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +import uvicorn + +from messenger import config, lifecycle + + +def main() -> None: + server = lifecycle.DrainingServer( + uvicorn.Config( + "messenger.app:app", + host=config.app_host(), + port=config.app_port(), + log_level="info", + access_log=False, + timeout_keep_alive=5, + timeout_graceful_shutdown=config.shutdown_timeout_seconds(), + ), + drain_seconds=config.shutdown_drain_seconds(), + ) + server.run() + if not server.started: + sys.exit(3) + + +if __name__ == "__main__": + main() diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..4226a59c --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,823 @@ +"""HTTP API: multi-replica messenger behind LB with attachments in private S3 (lab4).""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import logging +import re +import urllib.parse +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from typing import Any, List, Optional, Tuple + +from fastapi import FastAPI, Header, Query, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response, StreamingResponse +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.requests import ClientDisconnect + +from messenger import attachments, config, db, lifecycle, reconcile, s3, sessions +from messenger.sessions import SessionStoreError +from messenger.store import Attachment, AttachmentNotFound, IntentLost, Store, StoreUnavailable, format_ts + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}\Z") +UUID_RE = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\Z" +) +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}\Z") +BODY_READ_TIMEOUT_S = 5.0 +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 + +store = Store() +ready = False +upload_slots: Optional[asyncio.Semaphore] = None +INSTANCE_ID = config.instance_id() +VARIANT_NAMES = ("thumbnail", "crop", "optimized") +UPLOAD_SLOT_WAIT_S = 5.0 + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) +log = logging.getLogger("messenger") + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + # SIGTERM/SIGINT are handled by lifecycle.DrainingServer: readiness goes 503 first, + # requests are served through the drain window and this shutdown runs last. + global ready, upload_slots + db.init_pool() + # Migrations run once via `python -m messenger.migrate` (compose migrate service). + sessions.init_client() + s3.init_client() + attachments.prepare_spool_dir() + upload_slots = asyncio.Semaphore(config.max_concurrent_uploads()) + reconcile.start_background() + ready = True + log.info("messenger started instance_id=%s", INSTANCE_ID) + yield + ready = False + reconcile.stop_background() + sessions.close_client() + db.close_pool() + log.info("messenger stopped") + + +def request_id_from(request: Request) -> str: + incoming = request.headers.get("x-request-id") + if incoming and incoming.strip(): + return incoming.strip()[:128] + return str(uuid.uuid4()) + + +def error_body(code: str, message: str, request_id: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id}} + + +def common_headers(request_id: str, extra: Optional[dict] = None) -> dict: + headers = {"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID} + if extra: + headers.update(extra) + return headers + + +def error_response( + status: int, + code: str, + message: str, + request_id: str, + www_authenticate: Optional[str] = None, + extra_headers: Optional[dict] = None, +) -> JSONResponse: + headers = common_headers(request_id, extra_headers) + if www_authenticate: + headers["WWW-Authenticate"] = www_authenticate + return JSONResponse( + status_code=status, + content=error_body(code, message, request_id), + headers=headers, + ) + + +def is_blank(value: str) -> bool: + return value.strip() == "" + + +def has_nul(value: str) -> bool: + # Names/titles/passwords live in PostgreSQL TEXT, which cannot store U+0000. + return "\x00" in value + + +def has_lone_surrogate(value: str) -> bool: + return any(0xD800 <= ord(ch) <= 0xDFFF for ch in value) + + +def codepoint_len(value: str) -> int: + return len(value) + + +def parse_uuid(value: str) -> Optional[str]: + if not isinstance(value, str) or not UUID_RE.match(value): + return None + try: + return str(uuid.UUID(value)) + except ValueError: + return None + + +def encode_cursor(resource: str, created_at: datetime, item_id: str) -> str: + raw = f"{resource}|{created_at.strftime('%Y-%m-%dT%H:%M:%SZ')}|{item_id}" + return base64.urlsafe_b64encode(raw.encode("utf-8")).decode("ascii") + + +def decode_cursor(cursor: str, expected_resource: str) -> Tuple[datetime, uuid.UUID]: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")).decode("utf-8") + resource, ts, item_id = raw.split("|", 2) + if resource != expected_resource: + raise ValueError("wrong resource") + created_at = datetime.strptime(ts, "%Y-%m-%dT%H:%M:%SZ").replace( + tzinfo=timezone.utc + ) + return created_at, uuid.UUID(item_id) + except Exception as exc: + raise ValueError("invalid cursor") from exc + + +app = FastAPI( + title="MTUSI Messenger lab4", + docs_url=None, + redoc_url=None, + lifespan=lifespan, +) + + +@app.middleware("http") +async def add_request_id_and_gate(request: Request, call_next): + rid = request_id_from(request) + request.state.request_id = rid + + path = request.url.path + if not ready and path not in ("/health/live",): + if path == "/health/ready": + return error_response(503, "unavailable", "Shutting down", rid) + return error_response(503, "unavailable", "Service unavailable", rid) + + response = await call_next(request) + if "X-Request-Id" not in response.headers: + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = INSTANCE_ID + log.info( + "%s %s -> %s request_id=%s instance_id=%s", + request.method, + path, + response.status_code, + rid, + INSTANCE_ID, + ) + return response + + +class ApiError(Exception): + def __init__( + self, + status: int, + code: str, + message: str, + www_authenticate: Optional[str] = None, + headers: Optional[dict] = None, + ): + self.status = status + self.code = code + self.message = message + self.www_authenticate = www_authenticate + self.headers = headers + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + rid = getattr(request.state, "request_id", None) or request_id_from(request) + return error_response( + exc.status, + exc.code, + exc.message, + rid, + www_authenticate=exc.www_authenticate, + extra_headers=exc.headers, + ) + + +@app.exception_handler(RequestValidationError) +async def validation_handler(request: Request, exc: RequestValidationError): + rid = getattr(request.state, "request_id", None) or request_id_from(request) + return error_response(400, "invalid_request", "Invalid request", rid) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + rid = getattr(request.state, "request_id", None) or request_id_from(request) + if exc.status_code == 404: + return error_response(404, "not_found", "Not found", rid) + if exc.status_code == 405: + return error_response(400, "invalid_request", "Method not allowed", rid) + return error_response(exc.status_code, "invalid_request", "Request failed", rid) + + +@app.exception_handler(StoreUnavailable) +async def store_unavailable_handler(request: Request, exc: StoreUnavailable): + rid = getattr(request.state, "request_id", None) or request_id_from(request) + return error_response(503, "unavailable", "Database unavailable", rid) + + +@app.exception_handler(SessionStoreError) +async def session_store_unavailable_handler(request: Request, exc: SessionStoreError): + rid = getattr(request.state, "request_id", None) or request_id_from(request) + return error_response(503, "unavailable", "Session store unavailable", rid) + + +@app.exception_handler(s3.S3Unavailable) +async def s3_unavailable_handler(request: Request, exc: s3.S3Unavailable): + rid = getattr(request.state, "request_id", None) or request_id_from(request) + return error_response(503, "unavailable", "Attachment storage unavailable", rid) + + +@app.exception_handler(Exception) +async def unhandled_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", None) or request_id_from(request) + log.exception("unhandled error request_id=%s", rid) + return error_response(500, "internal_error", "Internal error", rid) + + +async def read_json_object(request: Request) -> Any: + content_type = request.headers.get("content-type", "") + media = content_type.split(";")[0].strip().lower() + if media and media != "application/json": + raise ApiError(415, "unsupported_media_type", "Unsupported media type") + try: + body = await asyncio.wait_for(request.body(), timeout=BODY_READ_TIMEOUT_S) + except asyncio.TimeoutError: + raise ApiError(400, "invalid_request", "Request body timeout") + if not body: + raise ApiError(400, "invalid_request", "Empty body") + try: + text = body.decode("utf-8") + # Reject trailing garbage after one JSON value (read to EOF). + data, end = json.JSONDecoder().raw_decode(text) + if text[end:].strip(): + raise ApiError(400, "invalid_request", "Malformed JSON") + except ApiError: + raise + except (UnicodeDecodeError, json.JSONDecodeError, ValueError): + raise ApiError(400, "invalid_request", "Malformed JSON") + if not isinstance(data, dict): + raise ApiError(400, "invalid_request", "JSON object required") + return data + +def require_bearer(authorization: Optional[str]) -> Tuple[str, str]: + """Return (user_id, token). X-User-Id is ignored.""" + bearer_challenge = 'Bearer realm="messenger"' + if not authorization or not authorization.strip(): + raise ApiError( + 401, "unauthorized", "Missing bearer token", www_authenticate=bearer_challenge + ) + parts = authorization.strip().split(None, 1) + if len(parts) != 2 or parts[0].lower() != "bearer" or not parts[1].strip(): + # Basic on business endpoints must not work. + raise ApiError( + 401, "unauthorized", "Invalid authorization", www_authenticate=bearer_challenge + ) + token = parts[1].strip() + try: + session = sessions.get_session(token) + except SessionStoreError: + raise ApiError(503, "unavailable", "Session store unavailable") + if session is None: + raise ApiError( + 401, "unauthorized", "Invalid or expired session", www_authenticate=bearer_challenge + ) + user_id = session["user_id"] + user = store.get_user(user_id) + if user is None: + raise ApiError( + 401, "unauthorized", "Invalid or expired session", www_authenticate=bearer_challenge + ) + return user_id, token + + +def require_chat_member(chat_id_raw: str, user_id: str): + chat_id = parse_uuid(chat_id_raw) + if chat_id is None: + raise ApiError(400, "invalid_request", "Invalid chat_id") + chat = store.get_chat(chat_id) + if chat is None or user_id not in chat.member_ids: + raise ApiError(404, "not_found", "Chat not found") + return chat + + +def parse_limit(limit: Optional[int]) -> int: + if limit is None: + return 50 + if not isinstance(limit, int) or isinstance(limit, bool) or limit < 1 or limit > 100: + raise ApiError(400, "invalid_request", "Invalid limit") + return limit + + +def validate_display_text(value: Any, field: str, max_len: int) -> str: + if not isinstance(value, str) or is_blank(value) or has_nul(value): + raise ApiError(400, "invalid_request", f"Invalid {field}") + if has_lone_surrogate(value): + raise ApiError(400, "invalid_request", f"Invalid {field}") + length = codepoint_len(value) + if length < 1 or length > max_len: + raise ApiError(400, "invalid_request", f"Invalid {field}") + return value + + +def validate_password(value: Any) -> str: + # Required strings must not be whitespace-only (contracts/README). + if not isinstance(value, str) or is_blank(value) or has_nul(value): + raise ApiError(400, "invalid_request", "Invalid password") + if has_lone_surrogate(value): + raise ApiError(400, "invalid_request", "Invalid password") + length = codepoint_len(value) + if length < PASSWORD_MIN or length > PASSWORD_MAX: + raise ApiError(400, "invalid_request", "Invalid password") + return value + + +def parse_basic_credentials(authorization: Optional[str]) -> Tuple[str, str]: + challenge = 'Basic realm="messenger", charset="UTF-8"' + if not authorization or not authorization.strip(): + raise ApiError( + 401, "unauthorized", "Invalid credentials", www_authenticate=challenge + ) + parts = authorization.strip().split(None, 1) + if len(parts) != 2 or parts[0].lower() != "basic": + raise ApiError( + 401, "unauthorized", "Invalid credentials", www_authenticate=challenge + ) + try: + decoded = base64.b64decode(parts[1].strip(), validate=True).decode("utf-8") + except Exception: + raise ApiError( + 401, "unauthorized", "Invalid credentials", www_authenticate=challenge + ) + if ":" not in decoded: + raise ApiError( + 401, "unauthorized", "Invalid credentials", www_authenticate=challenge + ) + username, password = decoded.split(":", 1) + # Username ASCII-only per contract; password may contain ':'. + if not username or any(ord(ch) > 127 for ch in username): + raise ApiError( + 401, "unauthorized", "Invalid credentials", www_authenticate=challenge + ) + return username, password + + +@app.get("/health/live") +async def liveness(request: Request): + rid = request.state.request_id + return JSONResponse(content={"status": "ok"}, headers=common_headers(rid)) + + +@app.get("/health/ready") +async def readiness(request: Request): + rid = request.state.request_id + if not ready: + return error_response(503, "unavailable", "Not ready", rid) + if lifecycle.draining: + # Only readiness reflects the drain: other endpoints keep serving until the + # LB has stopped routing here and the listener is closed. + return error_response(503, "unavailable", "Shutting down", rid) + # Mandatory synchronous path in lab4 = sessions + metadata + attachment bytes, + # so PostgreSQL, Redis and S3 are all readiness dependencies. + if not await run_in_threadpool(_dependencies_ok): + return error_response(503, "unavailable", "Dependency unavailable", rid) + return JSONResponse(content={"status": "ok"}, headers=common_headers(rid)) + + +def _dependencies_ok() -> bool: + return db.ping() and sessions.ping() and s3.ping() + + +@app.post("/api/v1/users") +async def register_user(request: Request): + rid = request.state.request_id + data = await read_json_object(request) + allowed = {"username", "display_name", "password"} + if set(data.keys()) - allowed: + raise ApiError(400, "invalid_request", "Unexpected fields") + if "username" not in data or "display_name" not in data or "password" not in data: + raise ApiError(400, "invalid_request", "Missing required fields") + username = data["username"] + display_name = data["display_name"] + if ( + not isinstance(username, str) + or is_blank(username) + or has_lone_surrogate(username) + or not USERNAME_RE.match(username) + ): + raise ApiError(400, "invalid_request", "Invalid username") + display_name = validate_display_text(display_name, "display_name", 100) + password = validate_password(data["password"]) + try: + user = store.create_user(username, display_name, password) + except KeyError: + raise ApiError(409, "conflict", "Username already taken") + return JSONResponse( + status_code=201, + content=user.to_dict(), + headers=common_headers(rid), + ) + + +@app.post("/api/v1/auth/sessions") +async def create_session(request: Request): + rid = request.state.request_id + challenge = 'Basic realm="messenger", charset="UTF-8"' + # Contract: Basic only, no JSON body (even with valid credentials). + try: + raw_body = await asyncio.wait_for(request.body(), timeout=BODY_READ_TIMEOUT_S) + except asyncio.TimeoutError: + raise ApiError(400, "invalid_request", "Request body timeout") + if raw_body and raw_body.strip(): + raise ApiError(400, "invalid_request", "Login must not include a body") + username, password = parse_basic_credentials( + request.headers.get("authorization") + ) + # Uniform 401 for wrong user / wrong password (no user enumeration). + user = store.get_user_by_username(username) + if user is None or not store.verify_password(user.password_hash, password): + raise ApiError( + 401, "unauthorized", "Invalid credentials", www_authenticate=challenge + ) + try: + session = sessions.create_session(user.id) + except SessionStoreError: + raise ApiError(503, "unavailable", "Session store unavailable") + body = { + "token": session["token"], + "token_type": "Bearer", + "expires_at": session["expires_at"], + "user": user.to_dict(), + } + return JSONResponse( + status_code=201, content=body, headers=common_headers(rid) + ) + + +@app.delete("/api/v1/auth/sessions/current") +async def logout( + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), +): + rid = request.state.request_id + _user_id, token = require_bearer(authorization) + try: + sessions.revoke_session(token) + except SessionStoreError: + raise ApiError(503, "unavailable", "Session store unavailable") + return Response(status_code=204, headers=common_headers(rid)) + + +@app.get("/api/v1/users/me") +async def current_user( + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + user = store.get_user(user_id) + assert user is not None + return JSONResponse(content=user.to_dict(), headers=common_headers(rid)) + + +@app.post("/api/v1/chats") +async def create_chat( + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + data = await read_json_object(request) + allowed = {"title", "member_ids"} + if set(data.keys()) - allowed: + raise ApiError(400, "invalid_request", "Unexpected fields") + if "title" not in data or "member_ids" not in data: + raise ApiError(400, "invalid_request", "Missing required fields") + title = validate_display_text(data["title"], "title", 100) + member_ids = data["member_ids"] + if not isinstance(member_ids, list) or not member_ids or len(member_ids) > 100: + raise ApiError(400, "invalid_request", "Invalid member_ids") + parsed: List[str] = [] + seen = set() + for mid in member_ids: + if not isinstance(mid, str): + raise ApiError(400, "invalid_request", "Invalid member_ids") + uid = parse_uuid(mid) + if uid is None: + raise ApiError(400, "invalid_request", "Invalid member_ids") + if uid in seen: + raise ApiError(400, "invalid_request", "Duplicate member_ids") + seen.add(uid) + parsed.append(uid) + if user_id not in seen: + raise ApiError(400, "invalid_request", "Caller must be a member") + try: + chat = store.create_chat(title, parsed) + except KeyError: + raise ApiError(400, "invalid_request", "Unknown member") + return JSONResponse( + status_code=201, + content=chat.to_dict(), + headers=common_headers(rid), + ) + + +@app.get("/api/v1/chats") +async def list_chats( + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), + limit: Optional[int] = Query(default=None), + cursor: Optional[str] = Query(default=None), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + lim = parse_limit(limit) + resource = f"chats:{user_id}" + cursor_key = None + if cursor is not None: + if not cursor: + raise ApiError(400, "invalid_request", "Invalid cursor") + try: + cursor_ts, cursor_id = decode_cursor(cursor, resource) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if not store.chat_exists_for_user(user_id, cursor_ts, cursor_id): + raise ApiError(400, "invalid_request", "Invalid cursor") + cursor_key = (cursor_ts, cursor_id) + chats, more = store.list_chats_for_user(user_id, lim, cursor_key) + next_cursor = None + if more and chats: + last = chats[-1] + next_cursor = encode_cursor(resource, last.created_at, last.id) + return JSONResponse( + content={"items": [c.to_dict() for c in chats], "next_cursor": next_cursor}, + headers=common_headers(rid), + ) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat( + chat_id: str, + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + chat = require_chat_member(chat_id, user_id) + return JSONResponse(content=chat.to_dict(), headers=common_headers(rid)) + + +def parse_attachment_ids(raw: Any) -> List[str]: + if not isinstance(raw, list) or len(raw) > 10: + raise ApiError(400, "invalid_request", "Invalid attachment_ids") + parsed: List[str] = [] + for item in raw: + aid = parse_uuid(item) if isinstance(item, str) else None + if aid is None or aid in parsed: + raise ApiError(400, "invalid_request", "Invalid attachment_ids") + parsed.append(aid) + return parsed + + +def parse_idempotency_key(raw: Optional[str]) -> str: + if raw is None or not raw.strip() or not IDEMPOTENCY_KEY_RE.match(raw.strip()): + raise ApiError(400, "invalid_request", "Invalid or missing Idempotency-Key") + return raw.strip() + + +@app.post("/api/v1/chats/{chat_id}/messages") +async def create_message( + chat_id: str, + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), + idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + chat = require_chat_member(chat_id, user_id) + key = parse_idempotency_key(idempotency_key) + data = await read_json_object(request) + allowed = {"text", "attachment_ids"} + if set(data.keys()) - allowed: + raise ApiError(400, "invalid_request", "Unexpected fields") + if "text" not in data: + raise ApiError(400, "invalid_request", "Missing text") + attachment_ids = parse_attachment_ids(data.get("attachment_ids", [])) + text = data["text"] + # U+0000 is a valid code point of message text (stored as UTF-8 bytes, see 004 migration). + if ( + not isinstance(text, str) + or has_lone_surrogate(text) + or codepoint_len(text) > 4000 + ): + raise ApiError(400, "invalid_request", "Invalid text") + # Empty/whitespace text is allowed only together with at least one attachment. + if not attachment_ids and is_blank(text): + raise ApiError(400, "invalid_request", "Invalid text") + path = f"/api/v1/chats/{chat.id}/messages" + try: + message, _replayed = store.create_message_idempotent( + chat.id, user_id, text, attachment_ids, "POST", path, key + ) + except KeyError: + raise ApiError(409, "conflict", "Idempotency-Key reused with different body") + except AttachmentNotFound: + raise ApiError(404, "not_found", "Attachment not found") + except ValueError: + raise ApiError(400, "invalid_request", "Invalid text") + return JSONResponse( + status_code=201, + content=message.to_dict(), + headers=common_headers(rid), + ) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages( + chat_id: str, + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), + limit: Optional[int] = Query(default=None), + cursor: Optional[str] = Query(default=None), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + chat = require_chat_member(chat_id, user_id) + lim = parse_limit(limit) + resource = f"messages:{chat.id}" + cursor_key = None + if cursor is not None: + if not cursor: + raise ApiError(400, "invalid_request", "Invalid cursor") + try: + cursor_ts, cursor_id = decode_cursor(cursor, resource) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if not store.message_exists(chat.id, cursor_ts, cursor_id): + raise ApiError(400, "invalid_request", "Invalid cursor") + cursor_key = (cursor_ts, cursor_id) + messages, has_more = store.list_messages(chat.id, lim, cursor_key) + next_cursor = None + if has_more and messages: + last = messages[-1] + next_cursor = encode_cursor(resource, last.created_at, last.id) + return JSONResponse( + content={ + "items": [m.to_dict() for m in messages], + "next_cursor": next_cursor, + }, + headers=common_headers(rid), + ) + + +def parse_content_length(raw: Optional[str]) -> Optional[int]: + if raw is None: + return None + if not raw.strip().isdigit(): + raise ApiError(400, "invalid_request", "Invalid Content-Length") + return int(raw.strip()) + + +def require_attachment(attachment_id_raw: str, user_id: str) -> Attachment: + attachment_id = parse_uuid(attachment_id_raw) + if attachment_id is None: + raise ApiError(400, "invalid_request", "Invalid attachment_id") + att = store.get_attachment_for_user(attachment_id, user_id) + if att is None: + # Unknown and foreign attachments are indistinguishable. + raise ApiError(404, "not_found", "Attachment not found") + return att + + +@app.post("/api/v1/chats/{chat_id}/attachments") +async def upload_attachment( + chat_id: str, + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + # Input is validated before the chat lookup: a bad filename never reaches S3/DB queries. + try: + filename = attachments.validate_filename(attachments.query_filename(request.scope.get("query_string", b""))) + content_type = attachments.declared_type(request.headers.get("content-type")) + except attachments.UploadRejected as exc: + raise ApiError(exc.status, exc.code, exc.message) + chat = require_chat_member(chat_id, user_id) + length = parse_content_length(request.headers.get("content-length")) + if length is not None and length > config.max_upload_bytes(): + # Rejected before taking an upload slot; the body is drained (bounded), not stored. + await attachments.drain_oversized(request.stream(), length) + raise ApiError(413, "payload_too_large", "File exceeds 10 MiB") + + assert upload_slots is not None + try: + await asyncio.wait_for(upload_slots.acquire(), timeout=UPLOAD_SLOT_WAIT_S) + except asyncio.TimeoutError: + raise ApiError(503, "unavailable", "Too many concurrent uploads", headers={"Retry-After": "1"}) + spooled = None + try: + try: + spooled = await attachments.spool_upload(request.stream(), content_type, length) + except attachments.UploadRejected as exc: + raise ApiError(exc.status, exc.code, exc.message) + except (attachments.ClientGone, ClientDisconnect): + raise ApiError(400, "invalid_request", "Upload interrupted or timed out") + try: + att = await run_in_threadpool( + attachments.persist, + store, + spooled, + chat_id=chat.id, + owner_id=user_id, + filename=filename, + instance=INSTANCE_ID, + ) + except IntentLost: + raise ApiError(503, "unavailable", "Upload expired, retry") + finally: + attachments.discard(spooled) + upload_slots.release() + return JSONResponse( + status_code=201, + content=att.to_dict(), + headers=common_headers(rid, {"Location": f"/api/v1/attachments/{att.id}"}), + ) + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment( + attachment_id: str, + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + att = require_attachment(attachment_id, user_id) + return JSONResponse(content=att.to_dict(), headers=common_headers(rid)) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment( + attachment_id: str, + request: Request, + authorization: Optional[str] = Header(default=None, alias="Authorization"), + variant: Optional[str] = Query(default=None), +): + rid = request.state.request_id + user_id, _token = require_bearer(authorization) + att = require_attachment(attachment_id, user_id) + if variant is not None and variant != "original": + if variant in VARIANT_NAMES: + raise ApiError(404, "not_found", "Variant not found") + raise ApiError(400, "invalid_request", "Invalid variant") + try: + length, chunks, body = await run_in_threadpool(s3.open_object, att.object_key) + except s3.S3ObjectMissing: + log.error("original missing in S3 attachment_id=%s request_id=%s", att.id, rid) + raise ApiError(503, "unavailable", "Attachment content unavailable") + if length != att.size_bytes: + body.close() + log.error("original size mismatch attachment_id=%s request_id=%s", att.id, rid) + raise ApiError(503, "unavailable", "Attachment content unavailable") + + def stream(): + try: + yield from chunks + finally: + body.close() + + headers = common_headers( + rid, + { + "Content-Length": str(att.size_bytes), + "ETag": f'"{att.sha256}"', + "Cache-Control": "private, no-store", + "X-Content-Type-Options": "nosniff", + "Content-Disposition": "inline; filename*=UTF-8''" + urllib.parse.quote(att.filename, safe=""), + }, + ) + return StreamingResponse(stream(), media_type=att.content_type, headers=headers) + + +def create_app() -> FastAPI: + return app diff --git a/src/messenger/attachments.py b/src/messenger/attachments.py new file mode 100644 index 00000000..06a378ab --- /dev/null +++ b/src/messenger/attachments.py @@ -0,0 +1,326 @@ +"""Attachment upload pipeline: bounded streaming intake, then intent → S3 PUT → metadata commit.""" + +from __future__ import annotations + +import asyncio +import hashlib +import logging +import os +import shutil +import tempfile +import time +import unicodedata +import urllib.parse +import uuid +from dataclasses import dataclass +from typing import AsyncIterator, Callable, Optional + +from messenger import config, images, s3 +from messenger.store import Attachment, CommitUncertain, Store, StoreUnavailable, utc_now + +log = logging.getLogger("messenger.attachments") + +ALLOWED_TYPES = ("image/png", "image/jpeg") +# Rejected bodies are drained (never stored) up to this many bytes so that the +# client can read the error response instead of a connection reset. +DRAIN_FACTOR = 2 +DECODE_TIMEOUT_S = 15 +DECODE_MEMORY_BYTES = 384 * 1024 * 1024 +_decode_slots = asyncio.Semaphore(1) + + +class UploadRejected(Exception): + def __init__(self, status: int, code: str, message: str): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +class ClientGone(Exception): + """Client disconnected or stalled mid-upload; nothing was written to S3/DB.""" + + +class SimulatedCrash(BaseException): + """Fault-injection only: bypasses compensation like a killed process would.""" + + +def query_filename(query_string: bytes) -> Optional[str]: + """The single `filename` query value, percent-decoded strictly. + + Starlette's query_params replace invalid UTF-8 with U+FFFD; such a name must be + rejected, not silently rewritten. + """ + try: + pairs = urllib.parse.parse_qsl( + query_string.decode("ascii"), keep_blank_values=True, encoding="utf-8", errors="strict" + ) + except (UnicodeDecodeError, ValueError): + raise UploadRejected(400, "invalid_request", "Invalid filename") from None + values = [v for k, v in pairs if k == "filename"] + if len(values) > 1: + raise UploadRejected(400, "invalid_request", "Invalid filename") + return values[0] if values else None + + +def validate_filename(value: Optional[str]) -> str: + if value is None or not isinstance(value, str): + raise UploadRejected(400, "invalid_request", "filename is required") + if not 1 <= len(value) <= 255 or value.strip() == "" or value in (".", ".."): + raise UploadRejected(400, "invalid_request", "Invalid filename") + for ch in value: + if ch in "/\\" or 0xD800 <= ord(ch) <= 0xDFFF: + raise UploadRejected(400, "invalid_request", "Invalid filename") + # Cc: NUL/control characters; Cf: bidi overrides (U+202E) and other invisible format chars. + if unicodedata.category(ch) in ("Cc", "Cf"): + raise UploadRejected(400, "invalid_request", "Invalid filename") + return value + + +def declared_type(header: Optional[str]) -> str: + media = (header or "").split(";")[0].strip().lower() + if media not in ALLOWED_TYPES: + raise UploadRejected(415, "unsupported_media_type", "Only image/png and image/jpeg are accepted") + return media + + +def new_object_key() -> str: + # Random, server-generated; carries no user input (filename/chat title). + return s3.ORIGINAL_PREFIX + uuid.uuid4().hex + + +@dataclass +class Spooled: + path: str + size: int + sha256: str + content_type: str + width: int + height: int + + +def prepare_spool_dir() -> None: + """Remove leftovers of requests interrupted by a crash; buffers never outlive a request.""" + root = config.spool_dir() + if root.exists(): + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True, exist_ok=True) + + +async def _next_chunk(it: AsyncIterator[bytes], timeout: float) -> Optional[bytes]: + try: + return await asyncio.wait_for(it.__anext__(), timeout=timeout) + except StopAsyncIteration: + return None + + +async def _drain(it: AsyncIterator[bytes], already: int, cap: int, deadline: float) -> None: + """Discard the rest of a rejected body (bounded) so the error reaches the client.""" + received = already + try: + while received <= cap and time.monotonic() < deadline: + chunk = await _next_chunk(it, min(2.0, max(0.1, deadline - time.monotonic()))) + if chunk is None: + return + received += len(chunk) + except Exception: + return + + +async def drain_oversized(stream: AsyncIterator[bytes], content_length: int) -> None: + cap = config.max_upload_bytes() * DRAIN_FACTOR + if content_length <= cap: + await _drain(stream.__aiter__(), 0, cap, time.monotonic() + 30) + + +async def spool_upload( + stream: AsyncIterator[bytes], + content_type: str, + content_length: Optional[int], +) -> Spooled: + """Read the body chunk by chunk into a private temp file while validating it. + + Memory per upload is bounded by the transport chunk size (≈64 KiB) plus small + parser state; size and pixel limits are enforced before the body is fully read. + """ + max_bytes = config.max_upload_bytes() + idle = float(config.upload_idle_timeout_seconds()) + deadline = time.monotonic() + config.upload_deadline_seconds() + it = stream.__aiter__() + drain_cap = max_bytes * DRAIN_FACTOR + + if content_length is not None and content_length > max_bytes: + await drain_oversized(it, content_length) + raise UploadRejected(413, "payload_too_large", "File exceeds 10 MiB") + + validator = images.validator_for(content_type, config.max_image_pixels()) + digest = hashlib.sha256() + size = 0 + prefix = b"" + fd, path = tempfile.mkstemp(prefix="up-", dir=str(config.spool_dir())) + ok = False + try: + with os.fdopen(fd, "wb") as out: + while True: + if time.monotonic() > deadline: + raise ClientGone("upload deadline exceeded") + try: + chunk = await _next_chunk(it, idle) + except asyncio.TimeoutError: + raise ClientGone("client stalled") from None + if chunk is None: + break + if not chunk: + continue + size += len(chunk) + if size > max_bytes: + await _drain(it, size, drain_cap, min(deadline, time.monotonic() + 30)) + raise UploadRejected(413, "payload_too_large", "File exceeds 10 MiB") + try: + if len(prefix) < 8: + # MIME is decided by content: the declared type must match the magic bytes. + prefix = (prefix + chunk)[:8] + sniffed = images.sniff(prefix) if len(prefix) >= 3 else None + if (len(prefix) >= 8 or sniffed == "image/jpeg") and sniffed != content_type: + raise images.ImageRejected(415, "unsupported_media_type", "Content does not match declared image type") + validator.feed(chunk) + except images.ImageRejected: + await _drain(it, size, drain_cap, min(deadline, time.monotonic() + 30)) + raise + digest.update(chunk) + out.write(chunk) + if size == 0: + raise UploadRejected(400, "invalid_request", "Empty body") + if len(prefix) < 8 and images.sniff(prefix) != content_type: + raise UploadRejected(415, "unsupported_media_type", "Content does not match declared image type") + try: + width, height = validator.finish() + except images.ImageRejected as exc: + raise UploadRejected(exc.status, exc.code, exc.message) from None + if content_type == "image/jpeg": + await verify_jpeg(path) + ok = True + return Spooled(path, size, digest.hexdigest(), content_type, width, height) + except images.ImageRejected as exc: + raise UploadRejected(exc.status, exc.code, exc.message) from None + finally: + if not ok: + _unlink(path) + + +async def verify_jpeg(path: str) -> None: + """Fully entropy-decode the JPEG with libjpeg-turbo; any corruption warning is fatal. + + `-scale 1/8` keeps the output tiny, but every Huffman-coded block is still decoded. + Progressive JPEGs need a whole-image coefficient buffer (≈2 bytes/pixel/component, + ≈310 MiB for a 40 Mpx CMYK image), so decodes run one at a time per replica in a + separate process with an address-space cap and a wall-clock timeout. + """ + async with _decode_slots: + proc = await asyncio.create_subprocess_exec( + "prlimit", f"--as={DECODE_MEMORY_BYTES}", "--", + "djpeg", "-strict", "-scale", "1/8", "-outfile", os.devnull, path, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + try: + _, err = await asyncio.wait_for(proc.communicate(), DECODE_TIMEOUT_S) + except asyncio.TimeoutError: + proc.kill() + await proc.wait() + log.warning("jpeg verification timed out") + raise UploadRejected(503, "unavailable", "Image verification timed out, retry later") from None + if proc.returncode == 0: + return + message = err.decode("utf-8", "replace") + if proc.returncode < 0 or "Insufficient memory" in message: + log.warning("jpeg verification aborted rc=%s", proc.returncode) + raise UploadRejected(503, "unavailable", "Image verification failed, retry later") + raise UploadRejected(415, "unsupported_media_type", "Corrupted JPEG image data") + + +def _unlink(path: str) -> None: + try: + os.unlink(path) + except FileNotFoundError: + pass + except OSError as exc: + log.warning("spool cleanup failed: %s", type(exc).__name__) + + +def persist( + store: Store, + spooled: Spooled, + *, + chat_id: str, + owner_id: str, + filename: str, + instance: str, + fault: Optional[Callable[[str, str], None]] = None, +) -> Attachment: + """Durably store bytes + metadata. Blocking: run in a worker thread. + + `fault` is a fault-injection hook used only by messenger.selfcheck; it is + called with (step, object_key) and may raise to simulate failures. + """ + attachment_id = str(uuid.uuid4()) + key = new_object_key() + now = utc_now() + att = Attachment( + id=attachment_id, + chat_id=chat_id, + owner_id=owner_id, + filename=filename, + content_type=spooled.content_type, + size_bytes=spooled.size, + sha256=spooled.sha256, + width=spooled.width, + height=spooled.height, + object_key=key, + status="uploading", + created_at=now, + updated_at=now, + ) + # 1. Intent first: if PostgreSQL is down nothing reaches S3 (fast 503). + store.record_intent(key, attachment_id, chat_id, instance) + try: + if fault: + fault("before_put", key) + # 2. Bytes to the private bucket. + with open(spooled.path, "rb") as body: + s3.put_object(key, body, spooled.size, spooled.content_type, spooled.sha256) + if fault: + fault("after_put", key) + # 3. Metadata + intent removal in one transaction. + committed = store.commit_attachment(att) + if fault: + fault("after_commit", key) + return committed + except CommitUncertain: + # The row may be committed: deleting the object could leave `ready` metadata + # without bytes. The intent (if the commit was lost) lets the reconciler decide. + log.warning("commit outcome unknown, object kept key_suffix=%s", key[-8:]) + raise + except Exception as exc: + # 4. Compensation. Any step may fail again; the intent row then lets the + # reconciler finish the cleanup after the grace period. + log.warning("upload failed, compensating key_suffix=%s err=%s", key[-8:], type(exc).__name__) + _compensate(store, key) + raise + + +def _compensate(store: Store, key: str) -> None: + try: + s3.delete_object(key) + except s3.S3Unavailable: + return + try: + store.drop_intent(key) + except StoreUnavailable: + pass + + +def discard(spooled: Optional[Spooled]) -> None: + if spooled is not None: + _unlink(spooled.path) diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..d068c9c3 --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,161 @@ +"""Runtime configuration from environment / secret files.""" + +from __future__ import annotations + +import os +import socket +from pathlib import Path + + +def _read_secret(env_key: str, file_key: str, default: str | None = None) -> str: + file_path = os.environ.get(file_key, "").strip() + if file_path: + path = Path(file_path) + if path.is_file(): + return path.read_text(encoding="utf-8").strip() + value = os.environ.get(env_key, "").strip() + if value: + return value + if default is not None: + return default + raise RuntimeError(f"Missing secret: set {env_key} or {file_key}") + + +def _int_env(key: str, default: int, minimum: int = 1) -> int: + raw = os.environ.get(key, str(default)).strip() + try: + value = int(raw) + except ValueError as exc: + raise RuntimeError(f"{key} must be an integer") from exc + if value < minimum: + raise RuntimeError(f"{key} must be >= {minimum}") + return value + + +def database_url() -> str: + return _read_secret("DATABASE_URL", "DATABASE_URL_FILE") + + +def cache_url() -> str: + return _read_secret("CACHE_URL", "CACHE_URL_FILE") + + +def session_ttl_seconds() -> int: + return _int_env("SESSION_TTL_SECONDS", 3600) + + +def idempotency_ttl_seconds() -> int: + return _int_env("IDEMPOTENCY_TTL_SECONDS", 86400) + + +def instance_id() -> str: + """Stable for process lifetime; used as X-Instance-Id (maps to container hostname).""" + override = os.environ.get("INSTANCE_ID", "").strip() + if override: + return override[:128] + return socket.gethostname()[:128] + + +def app_host() -> str: + return os.environ.get("APP_HOST", "0.0.0.0") + + +def app_port() -> int: + return int(os.environ.get("APP_PORT", "8080")) + + +def migrations_dir() -> Path: + override = os.environ.get("MIGRATIONS_DIR", "").strip() + if override: + return Path(override) + here = Path(__file__).resolve() + candidates = [ + Path("/app/migrations"), + here.parents[2] / "migrations", + Path.cwd() / "migrations", + ] + for path in candidates: + if path.is_dir(): + return path + return candidates[0] + + +# --- S3 / attachments (lab4) ------------------------------------------------- + +CONTRACT_MAX_UPLOAD_BYTES = 10 * 1024 * 1024 +CONTRACT_MAX_IMAGE_PIXELS = 40_000_000 + + +def s3_endpoint() -> str: + return os.environ.get("S3_ENDPOINT", "http://s3:9000").strip() + + +def s3_region() -> str: + return os.environ.get("S3_REGION", "us-east-1").strip() + + +def s3_bucket() -> str: + return os.environ.get("S3_BUCKET", "messenger-attachments").strip() + + +def s3_access_key_id() -> str: + return _read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE") + + +def s3_secret_access_key() -> str: + return _read_secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE") + + +def s3_admin_access_key_id() -> str: + return _read_secret("S3_ADMIN_ACCESS_KEY_ID", "S3_ADMIN_ACCESS_KEY_ID_FILE") + + +def s3_admin_secret_access_key() -> str: + return _read_secret("S3_ADMIN_SECRET_ACCESS_KEY", "S3_ADMIN_SECRET_ACCESS_KEY_FILE") + + +def max_upload_bytes() -> int: + # The contract caps uploads at 10 MiB; the environment may only lower it. + return min(_int_env("MAX_UPLOAD_BYTES", CONTRACT_MAX_UPLOAD_BYTES), CONTRACT_MAX_UPLOAD_BYTES) + + +def max_image_pixels() -> int: + return min(_int_env("MAX_IMAGE_PIXELS", CONTRACT_MAX_IMAGE_PIXELS), CONTRACT_MAX_IMAGE_PIXELS) + + +def max_concurrent_uploads() -> int: + return _int_env("MAX_CONCURRENT_UPLOADS", 8) + + +def upload_idle_timeout_seconds() -> int: + return _int_env("UPLOAD_IDLE_TIMEOUT_SECONDS", 10) + + +def upload_deadline_seconds() -> int: + return _int_env("UPLOAD_DEADLINE_SECONDS", 120) + + +def upload_grace_seconds() -> int: + value = _int_env("UPLOAD_GRACE_SECONDS", 900) + if value < 2 * upload_deadline_seconds(): + raise RuntimeError("UPLOAD_GRACE_SECONDS must be >= 2 * UPLOAD_DEADLINE_SECONDS") + return value + + +def reconcile_interval_seconds() -> int: + return _int_env("RECONCILE_INTERVAL_SECONDS", 300, minimum=0) + + +def shutdown_drain_seconds() -> int: + # After SIGTERM: readiness 503 but normal requests served, until HAProxy + # (inter 1s, fall 2) has taken the replica out of rotation. + return _int_env("SHUTDOWN_DRAIN_SECONDS", 3, minimum=0) + + +def shutdown_timeout_seconds() -> int: + # Upper bound for in-flight requests once the listener is closed. + return _int_env("SHUTDOWN_TIMEOUT_SECONDS", 8) + + +def spool_dir() -> Path: + return Path(os.environ.get("UPLOAD_SPOOL_DIR", "/tmp/messenger-spool")) diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..700b5797 --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,218 @@ +"""PostgreSQL connection pool and versioned migrations.""" + +from __future__ import annotations + +import itertools +import logging +import os +import re +import socket +import threading +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Dict, Iterator, List, Optional, Tuple + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +from messenger import config + +log = logging.getLogger("messenger.db") + +# statement_timeout is enforced by the server and does nothing when the server is +# frozen (docker pause, stuck host): the client would wait forever. The watchdog +# bounds every borrowed connection on the client side instead. +QUERY_DEADLINE_S = 6.0 +POOL_WAIT_S = 3.0 +BREAKER_OPEN_S = 3.0 + +_pool: Optional[ConnectionPool] = None +_guarded: Optional["GuardedPool"] = None + + +class _Watchdog: + """One thread; shuts down the socket of a connection that outlives its deadline.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._armed: Dict[int, Tuple[float, psycopg.Connection]] = {} + self._fired: set = set() + self._ids = itertools.count() + self._thread: Optional[threading.Thread] = None + + def arm(self, conn: psycopg.Connection, deadline_s: float) -> int: + token = next(self._ids) + with self._lock: + self._armed[token] = (time.monotonic() + deadline_s, conn) + if self._thread is None: + self._thread = threading.Thread(target=self._run, name="db-watchdog", daemon=True) + self._thread.start() + return token + + def disarm(self, token: int) -> bool: + with self._lock: + self._armed.pop(token, None) + if token in self._fired: + self._fired.discard(token) + return True + return False + + def _run(self) -> None: + while True: + time.sleep(0.25) + now = time.monotonic() + # Kill under the lock: the borrower must disarm (same lock) before the + # connection goes back to the pool, so its fd cannot be closed and reused + # by the event loop while we touch it. + with self._lock: + overdue = [(t, c) for t, (d, c) in self._armed.items() if d <= now] + for token, conn in overdue: + self._armed.pop(token, None) + self._fired.add(token) + _kill_socket(conn) + + +def _kill_socket(conn: psycopg.Connection) -> None: + # The blocked libpq call sees EOF, psycopg raises OperationalError and marks the + # connection broken; the pool then discards it instead of reusing a stuck one. + try: + sock = socket.socket(fileno=os.dup(conn.pgconn.socket)) + except Exception: + return + # The socket object owns the dup'ed fd and closes it exactly once. + with sock: + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + log.warning("postgres call exceeded its deadline; connection aborted") + + +_watchdog = _Watchdog() + + +class GuardedPool: + """Pool facade: bounded wait for a connection, bounded time per borrow, breaker.""" + + def __init__(self, pool: ConnectionPool): + self.raw = pool + self._open_until = 0.0 + + def _trip(self) -> None: + self._open_until = time.monotonic() + BREAKER_OPEN_S + + @contextmanager + def connection(self, deadline: Optional[float] = QUERY_DEADLINE_S) -> Iterator[psycopg.Connection]: + if time.monotonic() < self._open_until: + # Fail fast while PostgreSQL recently hung or vanished; a later call probes again. + raise psycopg.OperationalError("database circuit open") + try: + with self.raw.connection(timeout=POOL_WAIT_S) as conn: + token = _watchdog.arm(conn, deadline) if deadline else None + try: + yield conn + except psycopg.Error: + if conn.broken: + self._trip() + raise + finally: + if token is not None and _watchdog.disarm(token): + self._trip() + except PoolTimeout: + self._trip() + raise + + +def get_pool() -> GuardedPool: + if _guarded is None: + raise RuntimeError("Database pool is not initialized") + return _guarded + + +def init_pool() -> GuardedPool: + global _pool, _guarded + if _guarded is not None: + return _guarded + url = config.database_url() + _pool = ConnectionPool( + conninfo=url, + min_size=1, + max_size=10, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 3, + "options": "-c statement_timeout=5000", + }, + open=True, + timeout=POOL_WAIT_S, + ) + _guarded = GuardedPool(_pool) + return _guarded + + +def close_pool() -> None: + global _pool, _guarded + if _pool is not None: + # Bounded: the whole shutdown must fit into the container's stop_grace_period. + _pool.close(timeout=1.0) + _pool = None + _guarded = None + + +def _split_sql(script: str) -> List[str]: + """Split SQL file into statements (no procedure bodies in our migrations).""" + # Drop line comments. + cleaned = re.sub(r"--[^\n]*", "", script) + parts = [] + for chunk in cleaned.split(";"): + stmt = chunk.strip() + if stmt: + parts.append(stmt) + return parts + + +def apply_migrations(migrations_path: Optional[Path] = None) -> None: + path = migrations_path or config.migrations_dir() + sql_files = sorted(path.glob("*.sql")) + if not sql_files: + raise RuntimeError(f"No migrations found in {path}") + pool = get_pool() + with pool.connection(deadline=None) as conn: + with conn.transaction(): + conn.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + for sql_file in sql_files: + version = sql_file.name + row = conn.execute( + "SELECT 1 AS ok FROM schema_migrations WHERE version = %s", + (version,), + ).fetchone() + if row: + log.info("migration already applied: %s", version) + continue + script = sql_file.read_text(encoding="utf-8") + for stmt in _split_sql(script): + conn.execute(stmt) + conn.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (version,), + ) + log.info("applied migration: %s", version) + + +def ping() -> bool: + try: + with get_pool().connection(deadline=1.5) as conn: + conn.execute("SELECT 1") + return True + except Exception as exc: + log.warning("postgres ping failed: %s", type(exc).__name__) + return False diff --git a/src/messenger/images.py b/src/messenger/images.py new file mode 100644 index 00000000..52f732fb --- /dev/null +++ b/src/messenger/images.py @@ -0,0 +1,391 @@ +"""Incremental PNG/JPEG validation with bounded memory. + +The validators consume the upload chunk by chunk. Dimensions are taken from the +header (PNG IHDR, JPEG SOFn) and the pixel limit is enforced as soon as they are +known, before the rest of the body is read. + +PNG is decoded here in full except for the arithmetic unfiltering step (which +cannot fail): chunk CRCs, PLTE rules, the whole zlib stream inflated in bounded +slices, its exact size for the declared geometry (incl. Adam7) and the filter +type byte of every scanline. JPEG is only checked structurally here; the entropy +coded data is verified afterwards by a real decoder (see attachments.verify_jpeg). +""" + +from __future__ import annotations + +import struct +import zlib +from typing import Optional, Tuple + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +JPEG_SOI = b"\xff\xd8\xff" + +_INFLATE_SLICE = 64 * 1024 +_MAX_PNG_CHUNK = 0x7FFFFFFF +_PNG_CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +_PNG_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +_ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +# Baseline, extended sequential and progressive Huffman JPEG: what browsers decode. +_JPEG_SOF_SUPPORTED = {0xC0, 0xC1, 0xC2} +_JPEG_SOF_ALL = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} + + +class ImageRejected(Exception): + """Upload content is not an acceptable image; maps to an HTTP status.""" + + def __init__(self, status: int, code: str, message: str): + super().__init__(message) + self.status = status + self.code = code + self.message = message + + +def _unsupported(message: str) -> ImageRejected: + return ImageRejected(415, "unsupported_media_type", message) + + +def sniff(prefix: bytes) -> Optional[str]: + if prefix.startswith(PNG_SIGNATURE): + return "image/png" + if prefix.startswith(JPEG_SOI): + return "image/jpeg" + return None + + +class _Validator: + content_type = "" + + def __init__(self, max_pixels: int): + self.max_pixels = max_pixels + self.width: Optional[int] = None + self.height: Optional[int] = None + + def _set_dimensions(self, width: int, height: int) -> None: + if width < 1 or height < 1: + raise _unsupported("Invalid image dimensions") + if width * height > self.max_pixels: + raise ImageRejected(413, "payload_too_large", "Image has too many pixels") + self.width, self.height = width, height + + def feed(self, data: bytes) -> None: + raise NotImplementedError + + def finish(self) -> Tuple[int, int]: + raise NotImplementedError + + +class PngValidator(_Validator): + content_type = "image/png" + + def __init__(self, max_pixels: int): + super().__init__(max_pixels) + self._buf = bytearray() + self._state = "signature" + self._chunk_type = b"" + self._chunk_left = 0 + self._chunk_small = bytearray() + self._crc = 0 + self._seen_plte = False + self._seen_idat = False + self._idat_done = False + self._color_type = 0 + self._depth = 0 + self._expected_raw = 0 + # (start, end, row_len) of every pass in the inflated stream; a scanline + # starts with its filter type byte (0..4). + self._rows: list = [] + self._raw = 0 + self._inflater = zlib.decompressobj() + + def _layout(self, width: int, height: int, depth: int, color: int, interlace: int) -> None: + bpp = _PNG_CHANNELS[color] * depth + + def row_bytes(w: int) -> int: + return 1 + (w * bpp + 7) // 8 + + passes = [(0, 0, 1, 1)] if interlace == 0 else _ADAM7 + offset = 0 + for x0, y0, dx, dy in passes: + pw = (width - x0 + dx - 1) // dx if width > x0 else 0 + ph = (height - y0 + dy - 1) // dy if height > y0 else 0 + if pw and ph: + size = ph * row_bytes(pw) + self._rows.append((offset, offset + size, row_bytes(pw))) + offset += size + self._expected_raw = offset + + def _check_filters(self, out: bytes, base: int) -> None: + end = base + len(out) + for start, stop, row_len in self._rows: + lo, hi = max(start, base), min(stop, end) + if lo >= hi: + continue + first = start + -(-(lo - start) // row_len) * row_len + if first < hi and max(out[first - base:hi - base:row_len]) > 4: + raise _unsupported("Invalid PNG scanline filter") + + def _inflate(self, data: bytes) -> None: + if self._inflater.eof: + if data: + raise _unsupported("Data after end of PNG image stream") + return + try: + out = self._inflater.decompress(data, _INFLATE_SLICE) + self._check_filters(out, self._raw) + self._raw += len(out) + while self._inflater.unconsumed_tail and self._raw <= self._expected_raw: + out = self._inflater.decompress(self._inflater.unconsumed_tail, _INFLATE_SLICE) + self._check_filters(out, self._raw) + self._raw += len(out) + except zlib.error: + raise _unsupported("Corrupted PNG image data") from None + if self._raw > self._expected_raw: + raise _unsupported("PNG image data larger than declared dimensions") + if self._inflater.eof and self._inflater.unused_data: + raise _unsupported("Data after end of PNG image stream") + + def _begin_chunk(self, length: int, ctype: bytes) -> None: + if length > _MAX_PNG_CHUNK or not ctype.isalpha(): + raise _unsupported("Corrupted PNG chunk") + if self.width is None and ctype != b"IHDR": + raise _unsupported("PNG must start with IHDR") + # Uppercase first letter = critical: a decoder must not skip one it does not know. + if ctype[:1].isupper() and ctype not in (b"IHDR", b"PLTE", b"IDAT", b"IEND"): + raise _unsupported("Unknown critical PNG chunk") + if ctype == b"IHDR" and (self.width is not None or length != 13): + raise _unsupported("Invalid PNG IHDR") + if ctype == b"IDAT": + if self._idat_done: + raise _unsupported("Non-consecutive PNG IDAT chunks") + if self._color_type == 3 and not self._seen_plte: + raise _unsupported("Palette PNG without PLTE") + self._seen_idat = True + elif self._seen_idat: + self._idat_done = True + if ctype == b"PLTE": + entries, rest = divmod(length, 3) + if ( + self._seen_plte + or self._seen_idat + or self._color_type in (0, 4) + or rest + or not 1 <= entries <= 256 + or (self._color_type == 3 and entries > 1 << self._depth) + ): + raise _unsupported("Invalid PNG palette") + self._seen_plte = True + if ctype == b"IEND" and length != 0: + raise _unsupported("Invalid PNG IEND") + self._chunk_type = ctype + self._chunk_left = length + self._chunk_small = bytearray() + self._crc = zlib.crc32(ctype) + + def _end_chunk(self) -> None: + ctype = self._chunk_type + if ctype == b"IHDR": + width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", bytes(self._chunk_small)) + if color not in _PNG_DEPTHS or depth not in _PNG_DEPTHS[color] or comp != 0 or filt != 0 or interlace not in (0, 1): + raise _unsupported("Unsupported PNG header") + if width > _MAX_PNG_CHUNK or height > _MAX_PNG_CHUNK: + raise _unsupported("Invalid image dimensions") + self._set_dimensions(width, height) + self._color_type = color + self._depth = depth + self._layout(width, height, depth, color, interlace) + elif ctype == b"IEND": + self._state = "end" + + def feed(self, data: bytes) -> None: + view = memoryview(data) + pos = 0 + while pos < len(view): + if self._state == "end": + raise _unsupported("Data after PNG IEND") + if self._state == "signature": + need = 8 - len(self._buf) + self._buf += view[pos:pos + need] + pos += min(need, len(view) - pos) + if len(self._buf) == 8: + if bytes(self._buf) != PNG_SIGNATURE: + raise _unsupported("Not a PNG image") + self._buf.clear() + self._state = "header" + elif self._state == "header": + need = 8 - len(self._buf) + self._buf += view[pos:pos + need] + pos += min(need, len(view) - pos) + if len(self._buf) == 8: + length, ctype = struct.unpack(">I4s", bytes(self._buf)) + self._buf.clear() + self._begin_chunk(length, ctype) + self._state = "data" if length else "crc" + elif self._state == "data": + take = min(self._chunk_left, len(view) - pos) + piece = view[pos:pos + take] + pos += take + self._chunk_left -= take + self._crc = zlib.crc32(piece, self._crc) + if self._chunk_type == b"IDAT": + self._inflate(bytes(piece)) + elif self._chunk_type == b"IHDR": + self._chunk_small += piece + if self._chunk_left == 0: + self._state = "crc" + elif self._state == "crc": + need = 4 - len(self._buf) + self._buf += view[pos:pos + need] + pos += min(need, len(view) - pos) + if len(self._buf) == 4: + (crc,) = struct.unpack(">I", bytes(self._buf)) + self._buf.clear() + if crc != (self._crc & 0xFFFFFFFF): + raise _unsupported("PNG chunk checksum mismatch") + self._state = "header" + self._end_chunk() + + def finish(self) -> Tuple[int, int]: + if self._state != "end" or self.width is None or self.height is None: + raise _unsupported("Truncated PNG image") + if not self._seen_idat or not self._inflater.eof or self._raw != self._expected_raw: + raise _unsupported("Incomplete PNG image data") + return self.width, self.height + + +class JpegValidator(_Validator): + content_type = "image/jpeg" + + def __init__(self, max_pixels: int): + super().__init__(max_pixels) + self._buf = bytearray() + self._state = "soi" + self._marker = 0 + self._seg_left = 0 + self._seg_small = bytearray() + self._pending_ff = False + self._seen_sof = False + self._seen_sos = False + self._done = False + + def _begin_segment(self, marker: int, length: int) -> None: + if length < 2: + raise _unsupported("Corrupted JPEG segment") + if marker in _JPEG_SOF_ALL: + if marker not in _JPEG_SOF_SUPPORTED: + raise _unsupported("Unsupported JPEG coding") + if self._seen_sof: + raise _unsupported("Multiple JPEG frames") + if marker == 0xDA and not self._seen_sof: + raise _unsupported("JPEG scan before frame header") + self._marker = marker + self._seg_left = length - 2 + self._seg_small = bytearray() + + def _end_segment(self) -> None: + marker = self._marker + if marker in _JPEG_SOF_SUPPORTED: + seg = bytes(self._seg_small) + if len(seg) < 6: + raise _unsupported("Corrupted JPEG frame header") + precision, height, width, components = struct.unpack(">BHHB", seg[:6]) + # 8-bit only: what browsers and the verifying decoder (libjpeg-turbo 2.1) support. + if precision != 8 or components not in (1, 3, 4) or len(seg) != 6 + 3 * components: + raise _unsupported("Unsupported JPEG frame header") + if height == 0: + raise _unsupported("JPEG without explicit height is not supported") + self._set_dimensions(width, height) + self._seen_sof = True + if marker == 0xDA: + self._seen_sos = True + self._state = "entropy" + else: + self._state = "marker" + + def feed(self, data: bytes) -> None: + pos = 0 + n = len(data) + while pos < n: + if self._done: + # Bytes after EOI are tolerated but never interpreted. + return + state = self._state + if state == "soi": + need = 2 - len(self._buf) + self._buf += data[pos:pos + need] + pos += min(need, n - pos) + if len(self._buf) == 2: + if bytes(self._buf) != b"\xff\xd8": + raise _unsupported("Not a JPEG image") + self._buf.clear() + self._state = "marker" + elif state == "marker": + byte = data[pos] + pos += 1 + if not self._pending_ff: + if byte != 0xFF: + raise _unsupported("Corrupted JPEG marker") + self._pending_ff = True + continue + if byte == 0xFF: + continue + self._pending_ff = False + if byte == 0xD9: + if not self._seen_sos: + raise _unsupported("JPEG without image data") + self._done = True + elif byte == 0x01 or 0xD0 <= byte <= 0xD7: + continue + elif byte in (0x00, 0xD8): + raise _unsupported("Corrupted JPEG marker") + else: + self._marker = byte + self._state = "length" + elif state == "length": + need = 2 - len(self._buf) + self._buf += data[pos:pos + need] + pos += min(need, n - pos) + if len(self._buf) == 2: + (length,) = struct.unpack(">H", bytes(self._buf)) + self._buf.clear() + self._begin_segment(self._marker, length) + self._state = "segment" + if self._seg_left == 0: + self._end_segment() + elif state == "segment": + take = min(self._seg_left, n - pos) + if self._marker in _JPEG_SOF_SUPPORTED: + self._seg_small += data[pos:pos + take] + pos += take + self._seg_left -= take + if self._seg_left == 0: + self._end_segment() + elif state == "entropy": + if self._pending_ff: + byte = data[pos] + if byte == 0x00 or 0xD0 <= byte <= 0xD7: + pos += 1 + self._pending_ff = False + elif byte == 0xFF: + pos += 1 + else: + # A real marker ends the scan; reprocess it in marker state. + self._state = "marker" + continue + idx = data.find(b"\xff", pos) + if idx < 0: + return + pos = idx + 1 + self._pending_ff = True + + def finish(self) -> Tuple[int, int]: + if not self._done or self.width is None or self.height is None: + raise _unsupported("Truncated JPEG image") + return self.width, self.height + + +def validator_for(content_type: str, max_pixels: int) -> _Validator: + if content_type == "image/png": + return PngValidator(max_pixels) + if content_type == "image/jpeg": + return JpegValidator(max_pixels) + raise _unsupported("Only image/png and image/jpeg are accepted") diff --git a/src/messenger/lifecycle.py b/src/messenger/lifecycle.py new file mode 100644 index 00000000..e8445735 --- /dev/null +++ b/src/messenger/lifecycle.py @@ -0,0 +1,50 @@ +"""Graceful shutdown: readiness drain window before uvicorn stops accepting connections. + +Plain uvicorn closes its listening socket on the first SIGTERM, while HAProxy keeps +routing to the replica until `/health/ready` fails `fall` times (inter 1s, fall 2). +Here the first SIGTERM/SIGINT only switches readiness to 503; the replica keeps +serving normally for SHUTDOWN_DRAIN_SECONDS, then uvicorn closes the listener and +waits up to SHUTDOWN_TIMEOUT_SECONDS for in-flight requests before the lifespan +shutdown (reconciler, Redis, DB pool). +""" + +from __future__ import annotations + +import logging +import time +from types import FrameType +from typing import Optional + +import uvicorn + +log = logging.getLogger("messenger") + +draining = False + + +class DrainingServer(uvicorn.Server): + def __init__(self, config: uvicorn.Config, drain_seconds: float) -> None: + super().__init__(config) + self.drain_seconds = drain_seconds + self._drain_until: Optional[float] = None + self._drain_logged = False + + def handle_exit(self, sig: int, frame: Optional[FrameType]) -> None: + global draining + draining = True + if self._drain_until is None and not self.should_exit and self.drain_seconds > 0: + # Runs as a signal handler: only set flags, logging happens in on_tick. + self._drain_until = time.monotonic() + self.drain_seconds + return + # A second signal ends the drain window at once; a repeated SIGINT forces exit. + super().handle_exit(sig, frame) + + async def on_tick(self, counter: int) -> bool: + if self._drain_until is not None and not self.should_exit: + if not self._drain_logged: + self._drain_logged = True + log.info("shutdown signal: readiness 503, serving for %ss drain window", self.drain_seconds) + if time.monotonic() >= self._drain_until: + log.info("drain window over; closing listener, waiting for in-flight requests") + self.should_exit = True + return await super().on_tick(counter) diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..345af3eb --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,28 @@ +"""One-shot migrations entrypoint (run once before API replicas).""" + +from __future__ import annotations + +import logging +import sys + +from messenger import db + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(message)s", +) +log = logging.getLogger("messenger.migrate") + + +def main() -> int: + db.init_pool() + try: + db.apply_migrations() + log.info("migrations complete") + return 0 + finally: + db.close_pool() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/messenger/reconcile.py b/src/messenger/reconcile.py new file mode 100644 index 00000000..44fcbb50 --- /dev/null +++ b/src/messenger/reconcile.py @@ -0,0 +1,222 @@ +"""DB/S3 reconciler: removes objects of interrupted uploads after a safe grace period. + +Rules (never violated, also with --grace-seconds 0): + * an object whose key is in `attachments` is never deleted; + * metadata is never deleted (a missing original is only reported); + * an intent is resolved under FOR UPDATE SKIP LOCKED, so it cannot race with the + commit transaction of the same upload (that one locks the intent too). +The grace period protects uploads that are still in flight from being aborted. + +CLI: python -m messenger.reconcile [--dry-run] [--grace-seconds N] [--check-missing] +""" + +from __future__ import annotations + +import argparse +import json +import logging +import random +import sys +import threading +import time +from typing import Dict, List, Optional + +from messenger import config, db, s3 + +log = logging.getLogger("messenger.reconcile") + +LOCK_KEY = 0x4C414234 # "LAB4": one reconciler at a time across replicas +BATCH = 500 + + +def _referenced(conn, keys: List[str]) -> set: + rows = conn.execute( + """ + SELECT object_key FROM attachments WHERE object_key = ANY(%s) + UNION + SELECT object_key FROM upload_intents WHERE object_key = ANY(%s) + """, + (keys, keys), + ).fetchall() + return {r["object_key"] for r in rows} + + +def _resolve_intents(conn, grace: int, dry_run: bool, report: Dict[str, int]) -> None: + if dry_run: + rows = conn.execute( + """ + SELECT i.object_key, (a.id IS NOT NULL) AS committed + FROM upload_intents i LEFT JOIN attachments a ON a.object_key = i.object_key + WHERE i.created_at < now() - make_interval(secs => %s) + """, + (grace,), + ).fetchall() + report["stale_intents"] += len(rows) + report["intent_objects_deleted"] += sum(1 for r in rows if not r["committed"]) + return + while True: + with conn.transaction(): + rows = conn.execute( + """ + SELECT object_key FROM upload_intents + WHERE created_at < now() - make_interval(secs => %s) + ORDER BY created_at + LIMIT %s + FOR UPDATE SKIP LOCKED + """, + (grace, BATCH), + ).fetchall() + if not rows: + return + for row in rows: + key = row["object_key"] + report["stale_intents"] += 1 + committed = conn.execute( + "SELECT 1 AS ok FROM attachments WHERE object_key = %s", (key,) + ).fetchone() + if not committed: + s3.delete_object(key) + report["intent_objects_deleted"] += 1 + conn.execute("DELETE FROM upload_intents WHERE object_key = %s", (key,)) + if len(rows) < BATCH: + return + + +def _sweep_objects(conn, grace: int, dry_run: bool, check_missing: bool, report: Dict[str, int]) -> None: + db_now = conn.execute("SELECT now() AS now").fetchone()["now"] + listed = set() if check_missing else None + page: List[dict] = [] + + def flush() -> None: + keys = [o["Key"] for o in page] + refs = _referenced(conn, keys) + for obj in page: + key = obj["Key"] + if key in refs: + report["referenced_objects"] += 1 + continue + age = (db_now - obj["LastModified"]).total_seconds() + if age < grace: + report["fresh_unreferenced_skipped"] += 1 + continue + report["orphans_found"] += 1 + if not dry_run: + # Re-check under the reconciler lock right before deleting. + if not _referenced(conn, [key]): + s3.delete_object(key) + report["orphans_deleted"] += 1 + page.clear() + + for obj in s3.list_objects(): + report["objects_scanned"] += 1 + if listed is not None: + listed.add(obj["Key"]) + page.append(obj) + if len(page) >= BATCH: + flush() + if page: + flush() + + if listed is not None: + rows = conn.execute( + "SELECT object_key FROM attachments WHERE created_at < %s", (db_now,) + ).fetchall() + report["missing_originals"] = sum(1 for r in rows if r["object_key"] not in listed) + + +def run( + grace_seconds: int, + dry_run: bool = False, + check_missing: bool = False, + resolve_intents: bool = True, +) -> dict: + started = time.monotonic() + report: Dict[str, int] = { + "stale_intents": 0, + "intent_objects_deleted": 0, + "objects_scanned": 0, + "referenced_objects": 0, + "fresh_unreferenced_skipped": 0, + "orphans_found": 0, + "orphans_deleted": 0, + } + # Holds the session advisory lock across the whole S3 listing: no per-borrow deadline. + with db.get_pool().connection(deadline=None) as conn: + conn.autocommit = True + locked = conn.execute("SELECT pg_try_advisory_lock(%s) AS ok", (LOCK_KEY,)).fetchone()["ok"] + if not locked: + return {"skipped": "another reconciler holds the lock"} + try: + if resolve_intents: + _resolve_intents(conn, grace_seconds, dry_run, report) + _sweep_objects(conn, grace_seconds, dry_run, check_missing, report) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_KEY,)) + conn.autocommit = False + result = {"dry_run": dry_run, "grace_seconds": grace_seconds, **report} + result["duration_ms"] = round((time.monotonic() - started) * 1000, 1) + return result + + +_stop = threading.Event() +_thread: Optional[threading.Thread] = None + + +def _loop(interval: int, grace: int) -> None: + # Jitter spreads replicas; the advisory lock still guarantees one runner. + if _stop.wait(random.uniform(5, 15)): + return + while not _stop.is_set(): + try: + result = run(grace) + if result.get("intent_objects_deleted") or result.get("orphans_deleted"): + log.info("reconcile %s", json.dumps(result)) + except Exception as exc: + log.warning("reconcile failed: %s", type(exc).__name__) + if _stop.wait(interval * random.uniform(0.8, 1.2)): + return + + +def start_background() -> Optional[threading.Thread]: + global _thread + interval = config.reconcile_interval_seconds() + if interval == 0: + return None + grace = config.upload_grace_seconds() + _stop.clear() + _thread = threading.Thread(target=_loop, args=(interval, grace), name="reconciler", daemon=True) + _thread.start() + return _thread + + +def stop_background(timeout: float = 1.0) -> None: + # A pass cut short by process exit is safe (intents and the lock are re-checked + # next time); the bounded join only lets a nearly finished pass release its connection. + _stop.set() + if _thread is not None: + _thread.join(timeout) + if _thread.is_alive(): + log.info("reconciler still running at shutdown; leaving it to the next pass") + + +def main(argv: Optional[List[str]] = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--dry-run", action="store_true", help="report only, change nothing") + parser.add_argument("--grace-seconds", type=int, default=None, + help="override UPLOAD_GRACE_SECONDS (lab demos only; in-flight uploads younger than this survive)") + parser.add_argument("--check-missing", action="store_true", help="also report metadata whose original is absent") + args = parser.parse_args(argv) + grace = config.upload_grace_seconds() if args.grace_seconds is None else args.grace_seconds + if grace < 0: + parser.error("--grace-seconds must be >= 0") + db.init_pool() + try: + print(json.dumps(run(grace, dry_run=args.dry_run, check_missing=args.check_missing))) + finally: + db.close_pool() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/s3.py b/src/messenger/s3.py new file mode 100644 index 00000000..30c8d261 --- /dev/null +++ b/src/messenger/s3.py @@ -0,0 +1,161 @@ +"""Private S3 bucket access (SeaweedFS S3 gateway) with bounded timeouts.""" + +from __future__ import annotations + +import logging +import threading +import time +from typing import BinaryIO, Iterator, Optional, Tuple + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from messenger import config + +log = logging.getLogger("messenger.s3") + +_client = None + +ORIGINAL_PREFIX = "original/" +DOWNLOAD_CHUNK = 64 * 1024 + + +class S3Unavailable(Exception): + """S3 is unreachable, timed out or refused the request.""" + + +class S3ObjectMissing(Exception): + """Object referenced by metadata does not exist in the bucket.""" + + +# Worst case per call: 2 attempts × (2 s connect + 5 s read) + backoff ≈ 15 s, well +# under the LB `timeout server` (45 s), so clients get a JSON 503, not an LB 504. +CONNECT_TIMEOUT_S = 2 +READ_TIMEOUT_S = 5 +TOTAL_ATTEMPTS = 2 +PING_TTL_S = 1.0 + +_ping_client = None +_ping_lock = threading.Lock() +_ping_cache: Tuple[float, bool] = (0.0, False) + + +def build_client(access_key: str, secret_key: str, *, connect: float = CONNECT_TIMEOUT_S, + read: float = READ_TIMEOUT_S, attempts: int = TOTAL_ATTEMPTS): + return boto3.client( + "s3", + endpoint_url=config.s3_endpoint(), + region_name=config.s3_region(), + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect, + read_timeout=read, + retries={"total_max_attempts": attempts, "mode": "standard"}, + max_pool_connections=32, + ), + ) + + +def init_client(): + global _client, _ping_client + if _client is None: + access, secret = config.s3_access_key_id(), config.s3_secret_access_key() + _client = build_client(access, secret) + _ping_client = build_client(access, secret, connect=1, read=1.5, attempts=1) + return _client + + +def client(): + return init_client() + + +def _error_code(exc: ClientError) -> str: + return str(exc.response.get("Error", {}).get("Code", "")) + + +def put_object(key: str, body: BinaryIO, size: int, content_type: str, sha256_hex: str) -> None: + try: + client().put_object( + Bucket=config.s3_bucket(), + Key=key, + Body=body, + ContentLength=size, + ContentType=content_type, + Metadata={"sha256": sha256_hex}, + ) + except (BotoCoreError, ClientError) as exc: + log.warning("s3 put failed: %s", type(exc).__name__) + raise S3Unavailable("s3 put failed") from exc + + +def delete_object(key: str) -> None: + """Idempotent: deleting an absent key succeeds.""" + try: + client().delete_object(Bucket=config.s3_bucket(), Key=key) + except (BotoCoreError, ClientError) as exc: + log.warning("s3 delete failed: %s", type(exc).__name__) + raise S3Unavailable("s3 delete failed") from exc + + +def head_object(key: str) -> Optional[dict]: + try: + return client().head_object(Bucket=config.s3_bucket(), Key=key) + except ClientError as exc: + if _error_code(exc) in ("404", "NoSuchKey", "NotFound"): + return None + raise S3Unavailable("s3 head failed") from exc + except BotoCoreError as exc: + raise S3Unavailable("s3 head failed") from exc + + +def open_object(key: str) -> Tuple[int, Iterator[bytes], object]: + """Start a GET before response headers are sent so failures can still map to JSON errors.""" + try: + resp = client().get_object(Bucket=config.s3_bucket(), Key=key) + except ClientError as exc: + if _error_code(exc) in ("404", "NoSuchKey", "NotFound"): + raise S3ObjectMissing(key) from exc + raise S3Unavailable("s3 get failed") from exc + except BotoCoreError as exc: + raise S3Unavailable("s3 get failed") from exc + body = resp["Body"] + return int(resp["ContentLength"]), body.iter_chunks(DOWNLOAD_CHUNK), body + + +def list_objects(prefix: str = ORIGINAL_PREFIX) -> Iterator[dict]: + token = None + while True: + kwargs = {"Bucket": config.s3_bucket(), "Prefix": prefix, "MaxKeys": 1000} + if token: + kwargs["ContinuationToken"] = token + try: + page = client().list_objects_v2(**kwargs) + except (BotoCoreError, ClientError) as exc: + raise S3Unavailable("s3 list failed") from exc + for item in page.get("Contents", []) or []: + yield item + if not page.get("IsTruncated"): + return + token = page.get("NextContinuationToken") + + +def ping() -> bool: + """Readiness probe: one short request, result shared for PING_TTL_S across callers.""" + global _ping_cache + with _ping_lock: + checked_at, ok = _ping_cache + if time.monotonic() - checked_at < PING_TTL_S: + return ok + init_client() + try: + _ping_client.list_objects_v2(Bucket=config.s3_bucket(), Prefix=ORIGINAL_PREFIX, MaxKeys=1) + ok = True + except Exception as exc: + log.warning("s3 ping failed: %s", type(exc).__name__) + ok = False + _ping_cache = (time.monotonic(), ok) + return ok diff --git a/src/messenger/s3bootstrap.py b/src/messenger/s3bootstrap.py new file mode 100644 index 00000000..1894638c --- /dev/null +++ b/src/messenger/s3bootstrap.py @@ -0,0 +1,67 @@ +"""Idempotent bucket bootstrap (one-shot `s3-init` service, admin credentials). + +Creates the private bucket if absent; re-running is a no-op. Access rights are +not stored in the bucket: SeaweedFS reads identities (admin / messenger-api, no +anonymous identity) from the s3 config secret, so the bucket stays private. +""" + +from __future__ import annotations + +import logging +import sys +import time +import urllib.error +import urllib.request + +from botocore.exceptions import BotoCoreError, ClientError + +from messenger import config, s3 + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") +log = logging.getLogger("messenger.s3bootstrap") + + +def _anonymous_status(url: str) -> int: + try: + with urllib.request.urlopen(url, timeout=5) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + + +def main() -> int: + bucket = config.s3_bucket() + admin = s3.build_client(config.s3_admin_access_key_id(), config.s3_admin_secret_access_key()) + deadline = time.monotonic() + 90 + while True: + try: + admin.head_bucket(Bucket=bucket) + log.info("bucket exists: %s", bucket) + break + except ClientError as exc: + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if status != 404: + raise + try: + admin.create_bucket(Bucket=bucket) + log.info("bucket created: %s", bucket) + except ClientError as create_exc: + code = create_exc.response.get("Error", {}).get("Code", "") + if code not in ("BucketAlreadyOwnedByYou", "BucketAlreadyExists"): + raise + log.info("bucket appeared concurrently: %s", bucket) + except BotoCoreError as exc: + if time.monotonic() > deadline: + raise + log.info("waiting for S3 (%s)", type(exc).__name__) + time.sleep(2) + status = _anonymous_status(config.s3_endpoint().rstrip("/") + "/" + bucket + "?list-type=2") + if status != 403: + log.error("bucket is not private: anonymous list returned %s", status) + return 1 + log.info("anonymous access denied (HTTP %s); bootstrap complete", status) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/selfcheck.py b/src/messenger/selfcheck.py new file mode 100644 index 00000000..fc4e6420 --- /dev/null +++ b/src/messenger/selfcheck.py @@ -0,0 +1,283 @@ +"""Operator self-checks executed inside an API container (docker compose exec api ...). + + python -m messenger.selfcheck s3 private bucket + least-privilege API key + python -m messenger.selfcheck consistency fault injection between S3 PUT and DB commit + python -m messenger.selfcheck spool upload buffers left on the API container + +Prints one JSON document; exit code 1 if any check fails. Never prints credentials. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import socket +import struct +import sys +import tempfile +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +import zlib +from typing import Callable, Dict, List + +from botocore.exceptions import ClientError + +from messenger import attachments, config, db, reconcile, s3 +from messenger.store import CommitUncertain, Store + +INTERNAL_SEAWEED_PORTS = (8333, 18333, 8888, 18888, 9333, 19333, 8080, 18080) + + +def _png(width: int = 4, height: int = 3) -> bytes: + def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + raw = b"".join(b"\x00" + bytes((x * 40) % 256 for x in range(width * 3)) for _ in range(height)) + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw)) + chunk(b"IEND", b"") + + +class Checks: + def __init__(self) -> None: + self.results: List[Dict[str, object]] = [] + + def check(self, name: str, ok: bool, detail: object = "") -> bool: + self.results.append({"check": name, "ok": bool(ok), "detail": detail}) + return ok + + def expect_denied(self, name: str, fn: Callable[[], object]) -> None: + try: + fn() + self.check(name, False, "allowed") + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + self.check(name, status in (403, 400, 405) and code != "", f"{status} {code}") + + def report(self, title: str) -> int: + ok = all(r["ok"] for r in self.results) + print(json.dumps({"selfcheck": title, "passed": ok, "results": self.results}, ensure_ascii=False, indent=1)) + return 0 if ok else 1 + + +def _anonymous_get(url: str) -> object: + try: + with urllib.request.urlopen(url, timeout=5) as resp: + return resp.status + except urllib.error.HTTPError as exc: + return exc.code + except OSError as exc: + return type(exc).__name__ + + +def _port_open(host: str, port: int) -> bool: + try: + with socket.create_connection((host, port), timeout=2): + return True + except OSError: + return False + + +def check_s3() -> int: + c = Checks() + bucket = config.s3_bucket() + endpoint = config.s3_endpoint().rstrip("/") + host = urllib.parse.urlparse(endpoint).hostname or "s3" + app = s3.client() + key = s3.ORIGINAL_PREFIX + "selfcheck-" + uuid.uuid4().hex + body = _png() + + app.put_object(Bucket=bucket, Key=key, Body=body, ContentType="image/png") + c.check("api_key_put_object", True) + got = app.get_object(Bucket=bucket, Key=key)["Body"].read() + c.check("api_key_get_object", got == body) + + status = _anonymous_get(f"{endpoint}/{bucket}/{key}") + c.check("anonymous_get_object_denied", status == 403, status) + status = _anonymous_get(f"{endpoint}/{bucket}/{key}?x-id=GetObject") + c.check("anonymous_get_object_query_denied", status == 403, status) + status = _anonymous_get(f"{endpoint}/{bucket}?list-type=2") + c.check("anonymous_list_bucket_denied", status == 403, status) + status = _anonymous_get(f"{endpoint}/") + c.check("anonymous_list_buckets_denied", status == 403, status) + + c.expect_denied("api_key_list_buckets_denied", lambda: app.list_buckets()) + c.expect_denied("api_key_create_bucket_denied", lambda: app.create_bucket(Bucket="lab4-escalation-" + uuid.uuid4().hex[:8])) + c.expect_denied("api_key_delete_bucket_denied", lambda: app.delete_bucket(Bucket=bucket)) + c.expect_denied("api_key_put_bucket_acl_denied", lambda: app.put_bucket_acl(Bucket=bucket, ACL="public-read")) + c.expect_denied("api_key_put_object_acl_denied", lambda: app.put_object_acl(Bucket=bucket, Key=key, ACL="public-read")) + c.expect_denied("api_key_other_bucket_denied", lambda: app.put_object(Bucket="lab4-other", Key="x", Body=b"x")) + status = _anonymous_get(f"{endpoint}/{bucket}/{key}") + c.check("anonymous_get_still_denied_after_acl_attempts", status == 403, status) + + wrong = s3.build_client(config.s3_access_key_id(), "wrong-" + uuid.uuid4().hex) + c.expect_denied("wrong_secret_denied", lambda: wrong.get_object(Bucket=bucket, Key=key)) + + open_ports = [p for p in INTERNAL_SEAWEED_PORTS if _port_open(host, p)] + c.check("seaweed_admin_and_grpc_ports_unreachable", not open_ports, {"open": open_ports, "probed": list(INTERNAL_SEAWEED_PORTS)}) + c.check("s3_api_port_reachable", _port_open(host, urllib.parse.urlparse(endpoint).port or 80)) + + app.delete_object(Bucket=bucket, Key=key) + c.check("api_key_delete_object", s3.head_object(key) is None) + return c.report("s3") + + +def _reconcile(**kwargs) -> dict: + for _ in range(20): + result = reconcile.run(**kwargs) + if "skipped" not in result: + return result + time.sleep(0.5) + raise RuntimeError("reconciler lock busy") + + +def _intent_exists(key: str) -> bool: + with db.get_pool().connection() as conn: + return conn.execute("SELECT 1 AS ok FROM upload_intents WHERE object_key = %s", (key,)).fetchone() is not None + + +def _attachment_exists(key: str) -> bool: + with db.get_pool().connection() as conn: + return conn.execute("SELECT 1 AS ok FROM attachments WHERE object_key = %s", (key,)).fetchone() is not None + + +def _spool(body: bytes) -> attachments.Spooled: + config.spool_dir().mkdir(parents=True, exist_ok=True) + fd, path = tempfile.mkstemp(prefix="selfcheck-", dir=str(config.spool_dir())) + with os.fdopen(fd, "wb") as out: + out.write(body) + return attachments.Spooled(path, len(body), hashlib.sha256(body).hexdigest(), "image/png", 4, 3) + + +def check_consistency() -> int: + c = Checks() + db.init_pool() + store = Store() + grace = config.upload_grace_seconds() + suffix = uuid.uuid4().hex[:10] + user = store.create_user("selfcheck_" + suffix, "Selfcheck", "Selfcheck-" + uuid.uuid4().hex) + chat = store.create_chat("selfcheck", [user.id]) + body = _png() + keys: Dict[str, str] = {} + + def persist(fault=None): + spooled = _spool(body) + try: + return attachments.persist(store, spooled, chat_id=chat.id, owner_id=user.id, + filename="selfcheck.png", instance="selfcheck", fault=fault) + finally: + attachments.discard(spooled) + + # A. S3 PUT fails: intent is dropped, nothing stays in the bucket. + def fail_put(step: str, key: str) -> None: + keys["put"] = key + if step == "before_put": + raise s3.S3Unavailable("injected") + + try: + persist(fail_put) + c.check("A_s3_put_failure_raises", False) + except s3.S3Unavailable: + c.check("A_s3_put_failure_raises", True) + k = keys["put"] + c.check("A_no_object_no_intent_no_metadata", + s3.head_object(k) is None and not _intent_exists(k) and not _attachment_exists(k)) + + # B. PUT succeeded, DB commit fails: compensation deletes the object at once. + def fail_commit(step: str, key: str) -> None: + keys["commit"] = key + if step == "after_put": + raise RuntimeError("injected commit failure") + + try: + persist(fail_commit) + c.check("B_commit_failure_raises", False) + except RuntimeError: + c.check("B_commit_failure_raises", True) + k = keys["commit"] + c.check("B_compensation_removed_object_and_intent", + s3.head_object(k) is None and not _intent_exists(k) and not _attachment_exists(k)) + + # C. Process dies after PUT (no compensation): object + intent remain until grace expires. + def crash(step: str, key: str) -> None: + keys["crash"] = key + if step == "after_put": + raise attachments.SimulatedCrash() + + try: + persist(crash) + except attachments.SimulatedCrash: + pass + k = keys["crash"] + c.check("C_crash_leaves_object_and_intent", s3.head_object(k) is not None and _intent_exists(k)) + first = _reconcile(grace_seconds=grace) + c.check("C_fresh_intent_survives_reconcile", s3.head_object(k) is not None and _intent_exists(k), first) + with db.get_pool().connection() as conn: + with conn.transaction(): + conn.execute( + "UPDATE upload_intents SET created_at = now() - make_interval(secs => %s) WHERE object_key = %s", + (grace + 60, k), + ) + second = _reconcile(grace_seconds=grace) + c.check("C_stale_intent_object_deleted", s3.head_object(k) is None and not _intent_exists(k), second) + again = _reconcile(grace_seconds=grace) + c.check("C_rerun_is_noop", s3.head_object(k) is None, again) + + # D. Orphan object without intent/metadata (e.g. intent lost): only the age rule applies. + orphan = attachments.new_object_key() + s3.client().put_object(Bucket=config.s3_bucket(), Key=orphan, Body=body, ContentType="image/png") + dry = _reconcile(grace_seconds=grace, resolve_intents=False, dry_run=True) + c.check("D_fresh_orphan_not_deleted", s3.head_object(orphan) is not None, dry) + time.sleep(3) + swept = _reconcile(grace_seconds=2, resolve_intents=False) + c.check("D_old_orphan_deleted_by_sweep", s3.head_object(orphan) is None, swept) + + # F. COMMIT succeeded but its result was lost: the object must NOT be compensated. + def lose_commit_reply(step: str, key: str) -> None: + keys["uncertain"] = key + if step == "after_commit": + raise CommitUncertain("injected lost COMMIT reply") + + try: + persist(lose_commit_reply) + c.check("F_uncertain_commit_raises", False) + except CommitUncertain: + c.check("F_uncertain_commit_raises", True) + k = keys["uncertain"] + c.check("F_uncertain_commit_keeps_object_and_metadata", + s3.head_object(k) is not None and _attachment_exists(k) and not _intent_exists(k)) + swept = _reconcile(grace_seconds=0, resolve_intents=False, check_missing=True) + c.check("F_reconcile_keeps_committed_object", s3.head_object(k) is not None, swept) + + # E. Committed attachments are never touched, even with grace 0. + att = persist() + c.check("E_upload_committed", _attachment_exists(att.object_key) and not _intent_exists(att.object_key)) + swept = _reconcile(grace_seconds=0, resolve_intents=False, check_missing=True) + c.check("E_referenced_object_kept", s3.head_object(att.object_key) is not None, swept) + c.check("E_no_missing_originals", swept.get("missing_originals") == 0, swept.get("missing_originals")) + db.close_pool() + return c.report("consistency") + + +def check_spool() -> int: + c = Checks() + root = config.spool_dir() + files = [p.name for p in root.iterdir()] if root.exists() else [] + c.check("spool_dir_empty", not files, {"dir": str(root), "files": len(files)}) + return c.report("spool") + + +def main(argv: List[str]) -> int: + commands = {"s3": check_s3, "consistency": check_consistency, "spool": check_spool} + if len(argv) != 1 or argv[0] not in commands: + print(__doc__) + return 2 + return commands[argv[0]]() + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 00000000..21d7520a --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,136 @@ +"""Opaque Bearer sessions in Redis with TTL (durable via Redis AOF).""" + +from __future__ import annotations + +import json +import logging +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional + +import redis + +from messenger import config + +log = logging.getLogger("messenger.sessions") + +_client: Optional[redis.Redis] = None + +TOKEN_BYTES = 32 # 256-bit entropy +KEY_PREFIX = "sess:" + + +def utc_now() -> datetime: + return datetime.now(timezone.utc).replace(microsecond=0) + + +def format_ts(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_ts(value: str) -> datetime: + return datetime.strptime(value, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + + +def init_client() -> redis.Redis: + global _client + if _client is not None: + return _client + url = config.cache_url() + _client = redis.Redis.from_url( + url, + decode_responses=True, + socket_connect_timeout=3, + socket_timeout=3, + health_check_interval=30, + ) + return _client + + +def close_client() -> None: + global _client + if _client is not None: + try: + _client.close() + except Exception: + pass + _client = None + + +def get_client() -> redis.Redis: + if _client is None: + raise RuntimeError("Redis client is not initialized") + return _client + + +def ping() -> bool: + try: + return bool(get_client().ping()) + except Exception as exc: + log.warning("redis ping failed: %s", type(exc).__name__) + return False + + +class SessionStoreError(Exception): + """Raised when Redis is unavailable for session operations.""" + + +def create_session(user_id: str) -> dict: + ttl = config.session_ttl_seconds() + token = secrets.token_urlsafe(TOKEN_BYTES) + now = utc_now() + expires_at = now + timedelta(seconds=ttl) + payload = { + "user_id": user_id, + "expires_at": format_ts(expires_at), + "created_at": format_ts(now), + } + key = KEY_PREFIX + token + try: + client = get_client() + # SET with EX: Redis is sole session store; AOF keeps it across restart. + client.set(key, json.dumps(payload), ex=ttl) + except redis.RedisError as exc: + raise SessionStoreError("session store unavailable") from exc + return { + "token": token, + "token_type": "Bearer", + "expires_at": payload["expires_at"], + "user_id": user_id, + } + + +def get_session(token: str) -> Optional[dict]: + if not token: + return None + key = KEY_PREFIX + token + try: + raw = get_client().get(key) + except redis.RedisError as exc: + raise SessionStoreError("session store unavailable") from exc + if not raw: + return None + try: + data = json.loads(raw) + expires_at = parse_ts(data["expires_at"]) + except (ValueError, KeyError, TypeError, json.JSONDecodeError): + return None + if expires_at <= utc_now(): + try: + get_client().delete(key) + except redis.RedisError: + pass + return None + return data + + +def revoke_session(token: str) -> bool: + if not token: + return False + key = KEY_PREFIX + token + try: + return bool(get_client().delete(key)) + except redis.RedisError as exc: + raise SessionStoreError("session store unavailable") from exc diff --git a/src/messenger/store.py b/src/messenger/store.py new file mode 100644 index 00000000..031acf38 --- /dev/null +++ b/src/messenger/store.py @@ -0,0 +1,645 @@ +"""PostgreSQL repository: users, chats, membership, messages, idempotency, attachment metadata.""" + +from __future__ import annotations + +import hashlib +import json +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from typing import Any, List, Optional, Tuple + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError +from psycopg import Error as PsycopgError +from psycopg import errors as pg_errors +from psycopg.types.json import Json + +from messenger import config +from messenger.db import get_pool + +_ph = PasswordHasher() + + +class StoreUnavailable(Exception): + """Raised when PostgreSQL is unreachable or timed out.""" + + +def utc_now() -> datetime: + return datetime.now(timezone.utc).replace(microsecond=0) + + +def format_ts(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +@dataclass +class User: + id: str + username: str + display_name: str + created_at: datetime + password_hash: str = "" + + def to_dict(self) -> dict: + return { + "id": self.id, + "username": self.username, + "display_name": self.display_name, + "created_at": format_ts(self.created_at), + } + + +@dataclass +class Chat: + id: str + title: str + member_ids: List[str] + created_at: datetime + + def to_dict(self) -> dict: + return { + "id": self.id, + "title": self.title, + "member_ids": list(self.member_ids), + "created_at": format_ts(self.created_at), + } + + +@dataclass +class Message: + id: str + chat_id: str + sender_id: str + text: str + created_at: datetime + attachment_ids: List[str] = field(default_factory=list) + + def to_dict(self) -> dict: + return { + "id": self.id, + "chat_id": self.chat_id, + "sender_id": self.sender_id, + "text": self.text, + "created_at": format_ts(self.created_at), + "attachment_ids": list(self.attachment_ids), + } + + +@dataclass +class Attachment: + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + width: int + height: int + object_key: str + status: str + created_at: datetime + updated_at: datetime + + def to_dict(self) -> dict: + # object_key is internal: the public API never exposes bucket keys. + return { + "id": self.id, + "chat_id": self.chat_id, + "owner_id": self.owner_id, + "filename": self.filename, + "content_type": self.content_type, + "size_bytes": self.size_bytes, + "sha256": self.sha256, + "status": self.status, + "variants": [], + "error_code": None, + "created_at": format_ts(self.created_at), + "updated_at": format_ts(self.updated_at), + } + + +class AttachmentNotFound(Exception): + """Referenced attachment is unknown, not ready or belongs to another chat.""" + + +class IntentLost(Exception): + """Upload intent vanished (reconciled) before metadata commit.""" + + +class CommitUncertain(StoreUnavailable): + """COMMIT was sent but its result is unknown: the row may or may not exist.""" + + +def _row_user(row: dict) -> User: + return User( + id=str(row["id"]), + username=row["username"], + display_name=row["display_name"], + created_at=row["created_at"], + password_hash=row.get("password_hash", ""), + ) + + +def _row_message(row: dict) -> Message: + return Message( + id=str(row["id"]), + chat_id=str(row["chat_id"]), + sender_id=str(row["sender_id"]), + text=bytes(row["text"]).decode("utf-8"), + created_at=row["created_at"], + attachment_ids=[str(a) for a in (row.get("attachment_ids") or [])], + ) + + +def _row_attachment(row: dict) -> Attachment: + return Attachment( + id=str(row["id"]), + chat_id=str(row["chat_id"]), + owner_id=str(row["owner_id"]), + filename=row["filename"], + content_type=row["content_type"], + size_bytes=int(row["size_bytes"]), + sha256=row["sha256"], + width=int(row["width"]), + height=int(row["height"]), + object_key=row["object_key"], + status=row["status"], + created_at=row["created_at"], + updated_at=row["updated_at"], + ) + + +_ATTACHMENT_COLUMNS = ( + "a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, " + "a.width, a.height, a.object_key, a.status, a.created_at, a.updated_at" +) +_MESSAGE_ATTACHMENTS = ( + "ARRAY(SELECT ma.attachment_id FROM message_attachments ma " + "WHERE ma.message_id = m.id ORDER BY ma.position) AS attachment_ids" +) + + +class Store: + def hash_password(self, password: str) -> str: + return _ph.hash(password) + + def verify_password(self, password_hash: str, password: str) -> bool: + try: + return _ph.verify(password_hash, password) + except VerifyMismatchError: + return False + except Exception: + return False + + def create_user(self, username: str, display_name: str, password: str) -> User: + user_id = uuid.uuid4() + created = utc_now() + password_hash = self.hash_password(password) + try: + with get_pool().connection() as conn: + with conn.transaction(): + conn.execute( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (user_id, username, display_name, password_hash, created), + ) + except (pg_errors.UniqueViolation, pg_errors.IntegrityError) as exc: + raise KeyError("username_taken") from exc + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return User( + id=str(user_id), + username=username, + display_name=display_name, + created_at=created, + ) + + def get_user(self, user_id: str) -> Optional[User]: + try: + with get_pool().connection() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE id = %s + """, + (user_id,), + ).fetchone() + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return _row_user(row) if row else None + + def get_user_by_username(self, username: str) -> Optional[User]: + try: + with get_pool().connection() as conn: + row = conn.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return _row_user(row) if row else None + + def create_chat(self, title: str, member_ids: List[str]) -> Chat: + chat_id = uuid.uuid4() + created = utc_now() + try: + with get_pool().connection() as conn: + with conn.transaction(): + # Atomic membership check + insert; rollback if any member missing. + rows = conn.execute( + "SELECT id FROM users WHERE id = ANY(%s::uuid[])", + (member_ids,), + ).fetchall() + found = {str(r["id"]) for r in rows} + if found != set(member_ids): + raise KeyError("unknown_member") + conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)", + (chat_id, title, created), + ) + for mid in member_ids: + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (chat_id, mid), + ) + except KeyError: + raise + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return Chat( + id=str(chat_id), + title=title, + member_ids=list(member_ids), + created_at=created, + ) + + def get_chat(self, chat_id: str) -> Optional[Chat]: + try: + with get_pool().connection() as conn: + row = conn.execute( + "SELECT id, title, created_at FROM chats WHERE id = %s", + (chat_id,), + ).fetchone() + if not row: + return None + members = conn.execute( + """ + SELECT user_id FROM chat_members + WHERE chat_id = %s + ORDER BY user_id + """, + (chat_id,), + ).fetchall() + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return Chat( + id=str(row["id"]), + title=row["title"], + member_ids=[str(m["user_id"]) for m in members], + created_at=row["created_at"], + ) + + def list_chats_for_user( + self, + user_id: str, + limit: int, + cursor: Optional[Tuple[datetime, uuid.UUID]], + ) -> Tuple[List[Chat], Optional[str]]: + params: list = [user_id] + where = "cm.user_id = %s" + if cursor is not None: + where += " AND (c.created_at, c.id) > (%s, %s)" + params.extend([cursor[0], cursor[1]]) + params.append(limit + 1) + try: + with get_pool().connection() as conn: + rows = conn.execute( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE {where} + ORDER BY c.created_at ASC, c.id ASC + LIMIT %s + """, + params, + ).fetchall() + chats: List[Chat] = [] + for row in rows[:limit]: + members = conn.execute( + """ + SELECT user_id FROM chat_members + WHERE chat_id = %s + ORDER BY user_id + """, + (row["id"],), + ).fetchall() + chats.append( + Chat( + id=str(row["id"]), + title=row["title"], + member_ids=[str(m["user_id"]) for m in members], + created_at=row["created_at"], + ) + ) + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + has_more = len(rows) > limit + return chats, "more" if has_more else None + + @staticmethod + def request_hash_for_message(text: str, attachment_ids: List[str]) -> str: + # Canonical form: sorted keys; absent attachment_ids ≡ []; array order is significant. + canonical = json.dumps( + {"attachment_ids": list(attachment_ids), "text": text}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(canonical.encode("utf-8")).hexdigest() + + def create_message_idempotent( + self, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: List[str], + method: str, + path: str, + idem_key: str, + ) -> Tuple[Message, bool]: + """ + Create message with shared idempotency across replicas. + Returns (message, replayed). + Raises KeyError('conflict') when same key was used with another body, + AttachmentNotFound when an attachment is unknown or belongs to another chat. + """ + req_hash = self.request_hash_for_message(text, attachment_ids) + ttl = timedelta(seconds=config.idempotency_ttl_seconds()) + # Transaction-scoped advisory lock serializes same-key concurrent POSTs. + lock_material = f"{sender_id}|{method}|{path}|{idem_key}" + lock_key = int( + hashlib.sha256(lock_material.encode("utf-8")).hexdigest()[:15], 16 + ) + + def _load_existing(conn) -> Optional[dict]: + return conn.execute( + """ + SELECT request_hash, response_status, response_body, created_at + FROM idempotency_records + WHERE user_id = %s AND method = %s AND path = %s AND idem_key = %s + """, + (sender_id, method, path, idem_key), + ).fetchone() + + def _message_from_body(body: Any) -> Message: + return Message( + id=str(body["id"]), + chat_id=str(body["chat_id"]), + sender_id=str(body["sender_id"]), + text=body["text"], + created_at=datetime.strptime( + body["created_at"], "%Y-%m-%dT%H:%M:%SZ" + ).replace(tzinfo=timezone.utc), + attachment_ids=[str(a) for a in body.get("attachment_ids", [])], + ) + + try: + with get_pool().connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_key,)) + existing = _load_existing(conn) + if existing is not None: + if existing["created_at"] >= utc_now() - ttl: + if existing["request_hash"] != req_hash: + raise KeyError("conflict") + return _message_from_body(existing["response_body"]), True + conn.execute( + """ + DELETE FROM idempotency_records + WHERE user_id = %s AND method = %s AND path = %s AND idem_key = %s + """, + (sender_id, method, path, idem_key), + ) + + if attachment_ids: + # Only ready attachments of THIS chat may be referenced; others look absent. + found = conn.execute( + """ + SELECT id FROM attachments + WHERE id = ANY(%s::uuid[]) AND chat_id = %s AND status = 'ready' + FOR SHARE + """, + (attachment_ids, chat_id), + ).fetchall() + if {str(r["id"]) for r in found} != set(attachment_ids): + raise AttachmentNotFound() + + msg_id = uuid.uuid4() + created = utc_now() + message = Message( + id=str(msg_id), + chat_id=chat_id, + sender_id=sender_id, + text=text, + created_at=created, + attachment_ids=list(attachment_ids), + ) + conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + (msg_id, chat_id, sender_id, text.encode("utf-8"), created), + ) + for position, attachment_id in enumerate(attachment_ids): + conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES (%s, %s, %s) + """, + (msg_id, attachment_id, position), + ) + conn.execute( + """ + INSERT INTO idempotency_records ( + user_id, method, path, idem_key, + request_hash, response_status, response_body, created_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, + ( + sender_id, + method, + path, + idem_key, + req_hash, + 201, + Json(message.to_dict()), + created, + ), + ) + return message, False + except (KeyError, AttachmentNotFound): + raise + except pg_errors.DataError as exc: + raise ValueError("invalid_text") from exc + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + + def list_messages( + self, + chat_id: str, + limit: int, + cursor: Optional[Tuple[datetime, uuid.UUID]], + ) -> Tuple[List[Message], bool]: + params: list = [chat_id] + where = "m.chat_id = %s" + if cursor is not None: + where += " AND (m.created_at, m.id) > (%s, %s)" + params.extend([cursor[0], cursor[1]]) + params.append(limit + 1) + try: + with get_pool().connection() as conn: + rows = conn.execute( + f""" + SELECT m.id, m.chat_id, m.sender_id, m.text, m.created_at, + {_MESSAGE_ATTACHMENTS} + FROM messages m + WHERE {where} + ORDER BY m.created_at ASC, m.id ASC + LIMIT %s + """, + params, + ).fetchall() + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + messages = [_row_message(r) for r in rows[:limit]] + return messages, len(rows) > limit + + def message_exists(self, chat_id: str, created_at: datetime, item_id: uuid.UUID) -> bool: + try: + with get_pool().connection() as conn: + row = conn.execute( + """ + SELECT 1 AS ok FROM messages + WHERE chat_id = %s AND created_at = %s AND id = %s + """, + (chat_id, created_at, item_id), + ).fetchone() + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return bool(row) + + def chat_exists_for_user( + self, user_id: str, created_at: datetime, item_id: uuid.UUID + ) -> bool: + try: + with get_pool().connection() as conn: + row = conn.execute( + """ + SELECT 1 AS ok + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = %s AND c.created_at = %s AND c.id = %s + """, + (user_id, created_at, item_id), + ).fetchone() + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return bool(row) + + # --- attachments (lab4) --------------------------------------------------- + # + # Write protocol (see REPORT.md, "Согласованность БД/S3"): + # 1. record_intent() — durable intent row BEFORE any byte reaches S3; + # 2. S3 PUT of a server-generated key; + # 3. commit_attachment() — one transaction locks the intent, inserts the + # ready metadata and deletes the intent; + # 4. on failure: compensate (S3 DELETE + drop intent); whatever survives a + # crash is collected by messenger.reconcile after the grace period. + + def record_intent(self, object_key: str, attachment_id: str, chat_id: str, instance: str) -> None: + try: + with get_pool().connection() as conn: + with conn.transaction(): + conn.execute( + """ + INSERT INTO upload_intents (object_key, attachment_id, chat_id, instance_id, created_at) + VALUES (%s, %s, %s, %s, now()) + """, + (object_key, attachment_id, chat_id, instance), + ) + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + + def drop_intent(self, object_key: str) -> None: + try: + with get_pool().connection() as conn: + with conn.transaction(): + conn.execute("DELETE FROM upload_intents WHERE object_key = %s", (object_key,)) + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + + def commit_attachment(self, att: Attachment) -> Attachment: + committing = False + try: + with get_pool().connection() as conn: + with conn.transaction(): + intent = conn.execute( + "SELECT object_key FROM upload_intents WHERE object_key = %s FOR UPDATE", + (att.object_key,), + ).fetchone() + if intent is None: + raise IntentLost(att.object_key) + row = conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + width, height, object_key, status, created_at, updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'ready', now(), now()) + RETURNING created_at, updated_at + """, + ( + att.id, att.chat_id, att.owner_id, att.filename, att.content_type, + att.size_bytes, att.sha256, att.width, att.height, att.object_key, + ), + ).fetchone() + conn.execute("DELETE FROM upload_intents WHERE object_key = %s", (att.object_key,)) + # Leaving the block sends COMMIT; a failure from here on is ambiguous. + committing = True + except IntentLost: + raise + except PsycopgError as exc: + if committing: + raise CommitUncertain("commit outcome unknown") from exc + raise StoreUnavailable("database unavailable") from exc + att.status = "ready" + att.created_at = row["created_at"] + att.updated_at = row["updated_at"] + return att + + def get_attachment_for_user(self, attachment_id: str, user_id: str) -> Optional[Attachment]: + """Return the attachment only if user_id is a member of its chat (else None → 404).""" + try: + with get_pool().connection() as conn: + row = conn.execute( + f""" + SELECT {_ATTACHMENT_COLUMNS} + FROM attachments a + JOIN chat_members cm ON cm.chat_id = a.chat_id AND cm.user_id = %s + WHERE a.id = %s + """, + (user_id, attachment_id), + ).fetchone() + except PsycopgError as exc: + raise StoreUnavailable("database unavailable") from exc + return _row_attachment(row) if row else None diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..ac15abc4 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,19 @@ +# Собственные проверки (lab4) + +Нужен уже запущенный стенд (`scripts/up.cmd` / `scripts/up.sh`, ≥2 API за LB). Только stdlib Python. + +```sh +# из корня репозитория (Windows PowerShell: $env:PYTHONUTF8=1; $env:RUN_DOCKER_TESTS=1; ...) +BASE_URL=http://127.0.0.1:8080 RUN_DOCKER_TESTS=1 RUN_DEPS_TEST=1 python student-tests/run_all.py +# отдельный модуль: +python student-tests/run_all.py test_attachments +``` + +- `test_images_unit` — офлайн-проверка потокового валидатора PNG/JPEG (любое разбиение на куски, граница 40 млн пикселей, корпус повреждённых входов ревьюеров). +- `test_attachments` — upload/download/ACL, лимиты 413 (ровно 10 485 760 байт → 201, +1 байт → 413, 8000×5001 → 413), повреждённые изображения и подмена MIME → 415, опасные filename и невалидный UTF-8/повтор `filename` в query → 400, чужое или неизвестное `attachment_id` → 404, порядок и идемпотентность `attachment_ids`. +- `test_attachments_ops` (`RUN_DOCKER_TESTS=1`) — приватность bucket и права ключа API, отсутствие оригиналов на репликах, прерванные/зависшие загрузки, компенсация и reconciler, 16 параллельных загрузок 9.5 MiB без OOM у `s3`. +- `test_shutdown` (`RUN_DOCKER_TESTS=1`, по очереди останавливает и возвращает одну API-реплику) — graceful shutdown: после SIGTERM `/health/ready` сразу 503, а обычные запросы обслуживаются всё окно drain (пробы изнутри реплики, процесс API — PID 1); `docker stop` реплики под нагрузкой через LB — 0 ошибок; загрузка, тело которой ещё идёт при закрытии listener, завершается 201. +- `test_deps` (`RUN_DEPS_TEST=1`, останавливает зависимости!) — stop Redis, stop/pause PostgreSQL, stop S3: JSON 503 за ограниченное время и восстановление; хук `scripts/restart.*` действительно перезапускает API, PostgreSQL, Redis и S3. +- `test_idempotency`, `test_sessions`, `test_concurrency`, `test_api` — перенесены из lab3 и дополнены (область ключа, формат ключа, текст с U+0000). + +Для проверки истечения сессии поднимите API с `SESSION_TTL_SECONDS=2` и повторите `test_sessions`. diff --git a/student-tests/fixtures/gray.jpg b/student-tests/fixtures/gray.jpg new file mode 100644 index 00000000..3b88afb2 Binary files /dev/null and b/student-tests/fixtures/gray.jpg differ diff --git a/student-tests/fixtures/make_jpeg_fixtures.py b/student-tests/fixtures/make_jpeg_fixtures.py new file mode 100644 index 00000000..2bc3175c --- /dev/null +++ b/student-tests/fixtures/make_jpeg_fixtures.py @@ -0,0 +1,17 @@ +"""Regenerate synthetic JPEG fixtures (needs Pillow; run once in a throwaway container): + +docker run --rm -v "$PWD/student-tests/fixtures:/out" python:3.12.8-slim-bookworm \ + sh -c "pip install -q pillow==11.0.0 && python /out/make_jpeg_fixtures.py" +""" + +from PIL import Image + +im = Image.new("RGB", (320, 200)) +px = im.load() +for y in range(200): + for x in range(320): + px[x, y] = (x * 255 // 319, y * 255 // 199, (x + y) % 256) +im.save("/out/sample.jpg", "JPEG", quality=85) +im.save("/out/progressive.jpg", "JPEG", quality=80, progressive=True) +im.convert("L").save("/out/gray.jpg", "JPEG", quality=80) +print("ok") diff --git a/student-tests/fixtures/progressive.jpg b/student-tests/fixtures/progressive.jpg new file mode 100644 index 00000000..1dcb9f54 Binary files /dev/null and b/student-tests/fixtures/progressive.jpg differ diff --git a/student-tests/fixtures/sample.jpg b/student-tests/fixtures/sample.jpg new file mode 100644 index 00000000..d3f0c2d5 Binary files /dev/null and b/student-tests/fixtures/sample.jpg differ diff --git a/student-tests/helpers.py b/student-tests/helpers.py new file mode 100644 index 00000000..d9b00015 --- /dev/null +++ b/student-tests/helpers.py @@ -0,0 +1,94 @@ +"""Minimal HTTP helpers for student tests (stdlib only).""" + +from __future__ import annotations + +import base64 +import json +import os +import urllib.error +import urllib.request +import uuid + + +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +def call(method, path, expected=None, body=None, token=None, headers=None, raw=None): + h = {"Accept": "application/json"} + if token: + h["Authorization"] = "Bearer " + token + if body is not None: + raw = json.dumps(body, ensure_ascii=False).encode("utf-8") + h["Content-Type"] = "application/json" + # Lab3+: POST messages require Idempotency-Key unless caller supplies headers. + if ( + method == "POST" + and path.rstrip("/").endswith("/messages") + and (headers is None or "Idempotency-Key" not in headers) + ): + h["Idempotency-Key"] = uuid.uuid4().hex + h.update(headers or {}) + req = urllib.request.Request(BASE_URL + path, data=raw, headers=h, method=method) + try: + resp = urllib.request.urlopen(req, timeout=60) + except urllib.error.HTTPError as err: + resp = err + data = resp.read() + status = resp.code + if expected is not None: + assert status == expected, ( + f"{method} {path}: expected {expected}, got {status}: {data[:200]!r}" + ) + payload = None + ctype = resp.headers.get_content_type() + if ctype == "application/json" and data: + payload = json.loads(data.decode("utf-8")) + assert resp.headers.get("X-Request-Id"), "missing X-Request-Id" + assert resp.headers.get("X-Instance-Id"), "missing X-Instance-Id" + return {"status": status, "headers": resp.headers, "json": payload, "bytes": data} + + +def upload(token, chat_id, data, filename="photo.png", content_type="image/png", expected=None, headers=None): + from urllib.parse import quote + + h = {} if content_type is None else {"Content-Type": content_type} + h.update(headers or {}) + query = "" if filename is None else "?filename=" + quote(filename, safe="") + resp = call("POST", f"/api/v1/chats/{chat_id}/attachments{query}", expected, token=token, raw=data, headers=h) + if expected is None or expected < 500: + assert resp["status"] < 500, f"user error became {resp['status']}: {resp['json']}" + return resp + + +def chat_for(*actors, title="attachments"): + return call( + "POST", + "/api/v1/chats", + 201, + token=actors[0]["token"], + body={"title": title, "member_ids": [a["id"] for a in actors]}, + )["json"] + + +def register(label: str): + username = f"st_{label}_{uuid.uuid4().hex[:10]}" + password = "T3st-" + uuid.uuid4().hex + body = { + "username": username, + "display_name": f"Student {label}", + "password": password, + } + user = call("POST", "/api/v1/users", 201, body=body)["json"] + credential = base64.b64encode(f"{username}:{password}".encode()).decode() + session = call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + credential}, + )["json"] + return { + "id": user["id"], + "user": user, + "token": session["token"], + "registration": body, + } diff --git a/student-tests/imagegen.py b/student-tests/imagegen.py new file mode 100644 index 00000000..2b6b8324 --- /dev/null +++ b/student-tests/imagegen.py @@ -0,0 +1,160 @@ +"""Synthetic PNG generation (stdlib only) for student tests and experiments.""" + +from __future__ import annotations + +import os +import struct +import zlib +from pathlib import Path + +FIXTURES = Path(__file__).resolve().parent / "fixtures" +PUBLIC_PNG = Path(__file__).resolve().parents[1] / "tests" / "fixtures" / "sample.png" +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) + + +def chunk(kind: bytes, data: bytes) -> bytes: + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + + +def png_from_raw(width: int, height: int, raw: bytes, *, depth: int = 8, color: int = 2, + interlace: int = 0, level: int = 6, extra: bytes = b"", plte: bytes = b"") -> bytes: + ihdr = struct.pack(">IIBBBBB", width, height, depth, color, 0, 0, interlace) + body = chunk(b"IHDR", ihdr) + extra + if plte: + body += chunk(b"PLTE", plte) + return b"\x89PNG\r\n\x1a\n" + body + chunk(b"IDAT", zlib.compress(raw, level)) + chunk(b"IEND", b"") + + +def rgb_png(width: int, height: int, *, noise: bool = False, level: int = 6) -> bytes: + if noise: + rows = [b"\x00" + os.urandom(width * 3) for _ in range(height)] + else: + rows = [b"\x00" + bytes(((x + y) * 7) % 256 for x in range(width * 3)) for y in range(height)] + return png_from_raw(width, height, b"".join(rows), level=level) + + +def interlaced_rgb_png(width: int, height: int) -> bytes: + def pixel(x: int, y: int) -> bytes: + return bytes(((x * 31) % 256, (y * 17) % 256, ((x + y) * 5) % 256)) + + raw = bytearray() + for x0, y0, dx, dy in ADAM7: + xs = list(range(x0, width, dx)) + ys = list(range(y0, height, dy)) + if not xs or not ys: + continue + for y in ys: + raw += b"\x00" + b"".join(pixel(x, y) for x in xs) + return png_from_raw(width, height, bytes(raw), interlace=1) + + +def gray1_png(width: int, height: int) -> bytes: + """1-bit grayscale: huge pixel counts in a tiny, perfectly valid file.""" + row = b"\x00" + b"\x55" * ((width + 7) // 8) + return png_from_raw(width, height, row * height, depth=1, color=0, level=9) + + +def palette_png(width: int, height: int) -> bytes: + raw = b"".join(b"\x00" + bytes((x + y) % 4 for x in range(width)) for y in range(height)) + return png_from_raw(width, height, raw, depth=8, color=3, plte=bytes(range(12))) + + +def png16_rgba(width: int, height: int) -> bytes: + raw = b"".join(b"\x00" + bytes((x * 3 + y) % 256 for x in range(width * 8)) for y in range(height)) + return png_from_raw(width, height, raw, depth=16, color=6) + + +def large_png(target_bytes: int) -> bytes: + """Incompressible RGB PNG of roughly target_bytes (stored with zlib level 0).""" + width = 1024 + row = 1 + width * 3 + height = max(1, target_bytes // row) + return rgb_png(width, height, noise=True, level=0) + + +def exact_size_png(total_bytes: int) -> bytes: + """Valid PNG of exactly total_bytes: large_png padded with a tEXt chunk before IEND.""" + base = large_png(total_bytes - 64 * 1024) + pad = total_bytes - len(base) - 12 - len(b"Comment\x00") + assert pad >= 0 + text = chunk(b"tEXt", b"Comment\x00" + b"x" * pad) + data = base[:-12] + text + base[-12:] + assert len(data) == total_bytes + return data + + +def unknown_critical_chunk_png() -> bytes: + return png_from_raw(2, 2, b"\x00" + b"\x10" * 6 + b"\x00" + b"\x20" * 6, extra=chunk(b"ABCD", b"x")) + + +def fixture(name: str) -> bytes: + return (FIXTURES / name).read_bytes() + + +def all_filters_png(width: int, height: int, *, interlace: int = 0) -> bytes: + """Valid PNG whose scanlines cycle through filter types 0..4 (any bytes unfilter fine).""" + passes = ADAM7 if interlace else ((0, 0, 1, 1),) + raw = bytearray() + row_no = 0 + for x0, y0, dx, dy in passes: + pw = len(range(x0, width, dx)) + ph = len(range(y0, height, dy)) + if not pw or not ph: + continue + for _ in range(ph): + raw.append(row_no % 5) + raw += bytes((row_no * 13 + i) % 256 for i in range(pw * 3)) + row_no += 1 + return png_from_raw(width, height, bytes(raw), interlace=interlace) + + +def _with_bad_idat(png: bytes) -> bytes: + """Same PNG, IDAT payload damaged in the middle, CRC recomputed (valid chunk framing).""" + i = png.index(b"IDAT") + length = struct.unpack(">I", png[i - 4:i])[0] + data = bytearray(png[i + 4:i + 4 + length]) + data[len(data) // 2] ^= 0xFF + data[len(data) // 2 + 1] ^= 0x55 + return png[:i - 4] + chunk(b"IDAT", bytes(data)) + chunk(b"IEND", b"") + + +def corrupted_pngs() -> dict: + """Inputs used by reviewers with an independent decoder: all must be 415.""" + public = PUBLIC_PNG.read_bytes() + raw = bytearray() + for x0, y0, dx, dy in ADAM7: + pw, ph = len(range(x0, 9, dx)), len(range(y0, 9, dy)) + for _ in range(ph if pw else 0): + raw += b"\x00" + bytes(pw * 3) + raw[-(1 + 9 * 3)] = 7 # last scanline of the last Adam7 pass + return { + "first_33_bytes_of_sample_png": public[:33], + "sample_png_truncated_half": public[: len(public) // 2], + "broken_idat_stream_valid_crc": _with_bad_idat(rgb_png(64, 64)), + "filter_byte_05": png_from_raw(1, 1, b"\x05\x00\x00\x00"), + "filter_byte_ff": png_from_raw(1, 1, b"\xff\x00\x00\x00"), + "interlaced_filter_byte_07": png_from_raw(9, 9, bytes(raw), interlace=1), + "plte_257_entries": png_from_raw(1, 1, b"\x00\x00", color=3, plte=bytes(771)), + "plte_not_multiple_of_3": png_from_raw(1, 1, b"\x00\x00", color=3, plte=bytes(4)), + "plte_too_long_for_1bit": png_from_raw(1, 1, b"\x00\x00", depth=1, color=3, plte=bytes(9)), + "plte_in_grayscale": png_from_raw(1, 1, b"\x00\x00", depth=8, color=0, plte=bytes(3)), + "unknown_critical_chunk": unknown_critical_chunk_png(), + } + + +def corrupted_jpegs() -> dict: + sample = fixture("sample.jpg") + progressive = fixture("progressive.jpg") + sos = sample.index(b"\xff\xda") + sos_len = struct.unpack(">H", sample[sos + 2:sos + 4])[0] + twelve_bit = bytearray(sample) + twelve_bit[sample.index(b"\xff\xc0") + 4] = 12 + return { + "jpeg_27_bytes_no_tables": bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), + "jpeg_18_bytes": bytes.fromhex("ffd8ffc00008080001000101ffda0002ffd9"), + "jpeg_no_data_after_sos": sample[: sos + 2 + sos_len] + b"\xff\xd9", + "jpeg_half_plus_eoi": sample[: len(sample) // 2] + b"\xff\xd9", + "progressive_half_plus_eoi": progressive[: len(progressive) // 2] + b"\xff\xd9", + "jpeg_truncated_no_eoi": sample[:-200], + "jpeg_12_bit": bytes(twelve_bit), + } diff --git a/student-tests/run_all.py b/student-tests/run_all.py new file mode 100644 index 00000000..0aa5d00c --- /dev/null +++ b/student-tests/run_all.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Run all student tests against a live API (BASE_URL).""" + +from __future__ import annotations + +import importlib +import sys +import traceback + + +MODULES = [ + "test_images_unit", + "test_api", + "test_concurrency", + "test_sessions", + "test_idempotency", + "test_attachments", + "test_attachments_ops", + "test_shutdown", + "test_deps", +] + + +def main() -> int: + failed = 0 + passed = 0 + only = set(sys.argv[1:]) + for name in MODULES: + if only and name not in only: + continue + mod = importlib.import_module(name) + tests = [getattr(mod, n) for n in dir(mod) if n.startswith("test_")] + for fn in tests: + label = f"{name}.{fn.__name__}" + try: + fn() + print("OK", label) + passed += 1 + except Exception as exc: + failed += 1 + print("FAIL", label, ":", exc) + traceback.print_exc() + print(f"Summary: {passed} passed, {failed} failed") + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.path.insert(0, ".") + raise SystemExit(main()) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..c9da1df0 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,185 @@ +"""Grade 3/4 API checks: auth, conflict, X-User-Id ignored, logout.""" + +from __future__ import annotations + +import base64 +import time +import uuid + +from helpers import call, register + + +def test_username_conflict(): + a = register("conflict") + body = dict(a["registration"]) + call("POST", "/api/v1/users", 409, body=body) + + +def test_wrong_password(): + a = register("wrongpw") + bad = base64.b64encode( + (a["registration"]["username"] + ":wrong-password-xx").encode() + ).decode() + resp = call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + bad}, + ) + assert resp["headers"].get("WWW-Authenticate", "").lower().startswith("basic") + + +def test_x_user_id_ignored(): + a = register("xu") + b = register("xv") + me = call( + "GET", + "/api/v1/users/me", + 200, + token=a["token"], + headers={"X-User-Id": b["id"]}, + )["json"] + assert me["id"] == a["id"] + + +def test_basic_on_business_rejected(): + a = register("basicbiz") + cred = base64.b64encode( + (a["registration"]["username"] + ":" + a["registration"]["password"]).encode() + ).decode() + call( + "GET", + "/api/v1/users/me", + 401, + headers={"Authorization": "Basic " + cred}, + ) + + +def test_logout_revokes(): + a = register("logout") + call("DELETE", "/api/v1/auth/sessions/current", 204, token=a["token"]) + call("GET", "/api/v1/users/me", 401, token=a["token"]) + + +def test_chat_requires_caller_member(): + a = register("owner") + b = register("peer") + call( + "POST", + "/api/v1/chats", + 400, + token=a["token"], + body={"title": "no me", "member_ids": [b["id"]]}, + ) + + +def test_unknown_member_rolls_back(): + a = register("atom") + fake = str(uuid.uuid4()) + call( + "POST", + "/api/v1/chats", + 400, + token=a["token"], + body={"title": "bad", "member_ids": [a["id"], fake]}, + ) + + +def test_whitespace_only_password_rejected(): + # Maintainer remark: required strings must not be whitespace-only. + body = { + "username": "ws_" + uuid.uuid4().hex[:10], + "display_name": "Spaces", + "password": " " * 12, + } + call("POST", "/api/v1/users", 400, body=body) + + +def test_username_trailing_newline_400(): + # The pattern must match the whole value: "name\n" is not a valid username. + for username in ("review123\n", "review123\r\n", "\nreview123"): + resp = call("POST", "/api/v1/users", 400, + body={"username": username, "display_name": "x", "password": "T3st-" + uuid.uuid4().hex}) + assert resp["json"]["error"]["code"] == "invalid_request", repr(username) + + +def test_malformed_cursor_400(): + a = register("cursor") + chat = call("POST", "/api/v1/chats", 201, token=a["token"], body={"title": "c", "member_ids": [a["id"]]})["json"] + + def b64(raw: str) -> str: + return base64.urlsafe_b64encode(raw.encode()).decode() + + bad = ["not-base64!!", b64("messages|not-a-date|" + str(uuid.uuid4())), + b64("messages|2026-01-01T00:00:00Z|not-a-uuid"), b64("chats|2026-01-01T00:00:00Z|" + str(uuid.uuid4()))] + for cursor in bad: + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={cursor}", 400, token=a["token"]) + for cursor in bad[:3]: + call("GET", f"/api/v1/chats?cursor={cursor}", 400, token=a["token"]) + + +def test_nul_in_text_roundtrip(): + # U+0000 is valid message text: exact in the response, history and idempotent replay. + a = register("nul") + peer = register("nulb") + chat = call( + "POST", + "/api/v1/chats", + 201, + token=a["token"], + body={"title": "nul-chat", "member_ids": [a["id"], peer["id"]]}, + )["json"] + text = "abc\u0000def \\0 \\u0000" + key = {"Idempotency-Key": "nul-" + uuid.uuid4().hex} + path = f"/api/v1/chats/{chat['id']}/messages" + first = call("POST", path, 201, token=a["token"], body={"text": text}, headers=key)["json"] + assert first["text"] == text + again = call("POST", path, 201, token=a["token"], body={"text": text}, headers=key)["json"] + assert again == first + items = call("GET", path, 200, token=peer["token"])["json"]["items"] + assert [m["text"] for m in items if m["id"] == first["id"]] == [text] + + +def test_login_rejects_json_body(): + a = register("bodylogin") + cred = base64.b64encode( + (a["registration"]["username"] + ":" + a["registration"]["password"]).encode() + ).decode() + call( + "POST", + "/api/v1/auth/sessions", + 400, + body={"unexpected": True}, + headers={"Authorization": "Basic " + cred}, + ) + + +def test_invalid_basic_encoding_is_401(): + # Maintainer remark: malformed Basic must be 401, not 500. + call( + "POST", + "/api/v1/auth/sessions", + 401, + headers={"Authorization": "Basic " + base64.b64encode("é".encode()).decode()}, + ) + + +def test_unsupported_media_type(): + a = register("ctype") + peer = register("ctypeb") + chat = call( + "POST", + "/api/v1/chats", + 201, + token=a["token"], + body={"title": "ctype", "member_ids": [a["id"], peer["id"]]}, + )["json"] + raw = b'{"text":"x"}' + call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 415, + token=a["token"], + raw=raw, + headers={"Content-Type": "text/plain"}, + ) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..2d49b578 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,355 @@ +"""Lab4 grade 3/4: attachment upload/download, ACL, limits, MIME sniffing, filenames.""" + +from __future__ import annotations + +import concurrent.futures +import hashlib +import http.client +import socket +import time +import urllib.parse +import uuid + +import imagegen +from helpers import BASE_URL, call, chat_for, register, upload + +MIB = 1024 * 1024 +LIMIT = 10 * MIB + + +def _trio(): + alice, bob, eve = register("att_a"), register("att_b"), register("att_e") + return alice, bob, eve, chat_for(alice, bob) + + +def _png(seed: int = 0) -> bytes: + return imagegen.rgb_png(16 + seed % 7, 9 + seed % 5) + + +def _download(token, att_id, expected=200, query=""): + return call("GET", f"/api/v1/attachments/{att_id}/content{query}", expected, token=token) + + +def _message(actor, chat, text, attachment_ids=None, expected=201, key=None): + body = {"text": text} + if attachment_ids is not None: + body["attachment_ids"] = attachment_ids + return call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + expected, + token=actor["token"], + body=body, + headers={"Idempotency-Key": key or uuid.uuid4().hex}, + ) + + +def test_upload_roundtrip_png_and_jpeg_variants(): + alice, bob, _eve, chat = _trio() + cases = [ + (imagegen.PUBLIC_PNG.read_bytes(), "image/png", "sample.png"), + (imagegen.fixture("sample.jpg"), "image/jpeg", "sample.jpg"), + (imagegen.fixture("progressive.jpg"), "image/jpeg", "progressive.jpg"), + (imagegen.fixture("gray.jpg"), "image/jpeg", "gray.jpg"), + (imagegen.interlaced_rgb_png(41, 29), "image/png", "adam7.png"), + (imagegen.palette_png(12, 7), "image/png", "palette.png"), + ] + for data, ctype, name in cases: + resp = upload(alice["token"], chat["id"], data, name, ctype, 201) + att = resp["json"] + assert att["status"] == "ready" and att["variants"] == [] and att["error_code"] is None, att + assert att["chat_id"] == chat["id"] and att["owner_id"] == alice["id"] + assert att["content_type"] == ctype and att["filename"] == name + assert att["size_bytes"] == len(data) and att["sha256"] == hashlib.sha256(data).hexdigest() + assert set(att) == {"id", "chat_id", "owner_id", "filename", "content_type", "size_bytes", + "sha256", "status", "variants", "error_code", "created_at", "updated_at"} + assert resp["headers"].get("Location") == "/api/v1/attachments/" + att["id"] + meta = call("GET", "/api/v1/attachments/" + att["id"], 200, token=bob["token"])["json"] + assert meta == att + dl = _download(bob["token"], att["id"]) + assert dl["bytes"] == data, name + assert dl["headers"].get_content_type() == ctype + assert dl["headers"].get("Content-Length") == str(len(data)) + assert dl["headers"].get("X-Content-Type-Options") == "nosniff" + assert dl["headers"].get("ETag") == '"' + att["sha256"] + '"' + + +def test_outsider_and_unknown_get_404(): + alice, _bob, eve, chat = _trio() + att = upload(alice["token"], chat["id"], _png(), "secret.png", "image/png", 201)["json"] + for suffix in ("", "/content", "/content?variant=original"): + resp = call("GET", "/api/v1/attachments/" + att["id"] + suffix, 404, token=eve["token"]) + assert resp["json"]["error"]["code"] == "not_found" + unknown = str(uuid.uuid4()) + call("GET", "/api/v1/attachments/" + unknown, 404, token=alice["token"]) + call("GET", "/api/v1/attachments/" + unknown + "/content", 404, token=alice["token"]) + call("GET", "/api/v1/attachments/not-a-uuid", 400, token=alice["token"]) + call("GET", "/api/v1/attachments/" + att["id"], 401) + call("GET", "/api/v1/attachments/" + att["id"] + "/content", 401) + # Outsider cannot upload into the chat either. + upload(eve["token"], chat["id"], _png(), "x.png", "image/png", 404) + + +def test_upload_auth_and_chat_checks(): + alice, _bob, _eve, chat = _trio() + upload(None, chat["id"], _png(), "x.png", "image/png", 401) + upload(alice["token"], str(uuid.uuid4()), _png(), "x.png", "image/png", 404) + upload(alice["token"], "not-a-uuid", _png(), "x.png", "image/png", 400) + + +def test_message_with_attachments_and_history(): + alice, bob, _eve, chat = _trio() + a1 = upload(alice["token"], chat["id"], _png(1), "one.png", "image/png", 201)["json"] + a2 = upload(bob["token"], chat["id"], _png(2), "two.png", "image/png", 201)["json"] + key = uuid.uuid4().hex + msg = _message(bob, chat, "", [a2["id"], a1["id"]], key=key)["json"] + assert msg["attachment_ids"] == [a2["id"], a1["id"]] and msg["text"] == "" + replay = _message(bob, chat, "", [a2["id"], a1["id"]], key=key)["json"] + assert replay == msg + _message(bob, chat, "", [a1["id"], a2["id"]], expected=409, key=key) + plain_key = uuid.uuid4().hex + plain = _message(alice, chat, "без вложений", key=plain_key)["json"] + assert plain["attachment_ids"] == [] + # Absent attachment_ids ≡ []: same key replays the same message, another set conflicts. + assert _message(alice, chat, "без вложений", [], key=plain_key)["json"] == plain + _message(alice, chat, "без вложений", [a1["id"]], expected=409, key=plain_key) + # Any member may reference any attachment of the chat, also repeatedly. + again = _message(alice, chat, "ещё раз", [a2["id"]])["json"] + assert again["attachment_ids"] == [a2["id"]] + page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=alice["token"])["json"] + by_id = {m["id"]: m for m in page["items"]} + assert by_id[msg["id"]]["attachment_ids"] == [a2["id"], a1["id"]] + assert by_id[plain["id"]]["attachment_ids"] == [] + assert by_id[again["id"]]["attachment_ids"] == [a2["id"]] + + +def test_foreign_and_unknown_attachment_ids_404(): + alice, bob, eve, chat = _trio() + other = chat_for(alice, eve, title="other chat") + foreign = upload(alice["token"], other["id"], _png(3), "other.png", "image/png", 201)["json"] + own = upload(alice["token"], chat["id"], _png(4), "own.png", "image/png", 201)["json"] + # File of another chat (even uploaded by the same user) cannot be attached. + resp = _message(alice, chat, "чужой файл", [foreign["id"]], expected=404) + assert resp["json"]["error"]["code"] == "not_found" + _message(alice, chat, "смесь", [own["id"], foreign["id"]], expected=404) + _message(bob, chat, "неизвестный", [str(uuid.uuid4())], expected=404) + # Eve is in `other` only: cannot attach chat's file there. + _message(eve, other, "украсть", [own["id"]], expected=404) + # Failed attempts did not create messages. + page = call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"])["json"] + assert page["items"] == [] + + +def test_attachment_ids_validation_400(): + alice, _bob, _eve, chat = _trio() + att = upload(alice["token"], chat["id"], _png(), "v.png", "image/png", 201)["json"] + many = [str(uuid.uuid4()) for _ in range(11)] + for bad in (many, [att["id"], att["id"]], ["not-a-uuid"], [123], "x", {"id": att["id"]}): + _message(alice, chat, "x", bad, expected=400) + call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + token=alice["token"], + body={"text": "x", "attachment_ids": None}, + ) + _message(alice, chat, "", [], expected=400) + _message(alice, chat, " ", None, expected=400) + _message(alice, chat, " ", [att["id"]]) + + +def test_too_large_413(): + alice, _bob, _eve, chat = _trio() + big = imagegen.large_png(LIMIT + 64 * 1024) + assert len(big) > LIMIT + resp = upload(alice["token"], chat["id"], big, "big.png", "image/png", 413) + assert resp["json"]["error"]["code"] == "payload_too_large" + # Same oversized body without Content-Length (chunked): detected while streaming. + parsed = urllib.parse.urlparse(BASE_URL) + conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=60) + + def body(): + for i in range(0, len(big), 256 * 1024): + yield big[i:i + 256 * 1024] + + conn.request( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=big.png", + body=body(), + headers={"Authorization": "Bearer " + alice["token"], "Content-Type": "image/png"}, + encode_chunked=True, + ) + r = conn.getresponse() + r.read() + conn.close() + assert r.status == 413, r.status + + +def test_near_limit_real_png_accepted(): + alice, bob, _eve, chat = _trio() + data = imagegen.large_png(LIMIT - 256 * 1024) + assert len(data) <= LIMIT + att = upload(alice["token"], chat["id"], data, "near-limit.png", "image/png", 201)["json"] + dl = _download(bob["token"], att["id"]) + assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"] == hashlib.sha256(data).hexdigest() + + +def test_exact_limit_png_through_lb_accepted(): + # 10 485 760 bytes exactly passes the LB and the API; one more byte is JSON 413 from the API. + alice, bob, _eve, chat = _trio() + data = imagegen.exact_size_png(LIMIT) + att = upload(alice["token"], chat["id"], data, "exact.png", "image/png", 201)["json"] + assert att["size_bytes"] == LIMIT + assert hashlib.sha256(_download(bob["token"], att["id"])["bytes"]).hexdigest() == hashlib.sha256(data).hexdigest() + resp = upload(alice["token"], chat["id"], imagegen.exact_size_png(LIMIT + 1), "plus1.png", "image/png", 413) + assert resp["json"]["error"]["code"] == "payload_too_large" + + +def test_too_many_pixels_413(): + alice, _bob, _eve, chat = _trio() + bomb = imagegen.gray1_png(8000, 5001) # 40 008 000 px in a few KiB + assert len(bomb) < 64 * 1024 + resp = upload(alice["token"], chat["id"], bomb, "bomb.png", "image/png", 413) + assert resp["json"]["error"]["code"] == "payload_too_large" + upload(alice["token"], chat["id"], imagegen.gray1_png(8000, 5000), "limit.png", "image/png", 201) + + +def test_corrupted_images_415(): + alice, _bob, _eve, chat = _trio() + png = _png(5) + flipped = bytearray(png) + flipped[-20] ^= 0xFF + jpeg = imagegen.fixture("sample.jpg") + for data, ctype in ( + (png[: len(png) // 2], "image/png"), + (bytes(flipped), "image/png"), + (png + b"", "image/png"), + (b"MZ\x90\x00\x03\x00\x00\x00", "image/jpeg"), + (jpeg, "image/png"), + (png, "image/jpeg"), + (png, "text/plain"), + (png, "image/gif"), + (png, "multipart/form-data; boundary=x"), + (png, "application/octet-stream"), + ] + for data, ctype in cases: + resp = upload(alice["token"], chat["id"], data, "spoof.png", ctype, 415) + assert resp["json"]["error"]["code"] == "unsupported_media_type", ctype + + +def test_dangerous_filenames_400_and_unicode_kept(): + alice, bob, _eve, chat = _trio() + png = _png(7) + for name in ("../etc/passwd", "../x.png", "a/b.png", "a\\b.png", "x\x00.png", "x\n.png", "x\r.png", + "del\x7f.png", "c1\x85.png", + "tab\t.png", "\u202egnp.exe", "zero\u200bwidth.png", "", " ", ".", "..", "a" * 256): + resp = upload(alice["token"], chat["id"], png, name, "image/png", 400) + assert resp["json"]["error"]["code"] == "invalid_request", repr(name) + upload(alice["token"], chat["id"], png, None, "image/png", 400) + for name in ("фото отпуска 😀.png", "b" * 255, "with space & ampersand;=.png"): + att = upload(alice["token"], chat["id"], png, name, "image/png", 201)["json"] + assert att["filename"] == name + call("GET", "/api/v1/attachments/" + att["id"], 200, token=bob["token"]) + + +def test_filename_query_decoded_strictly_400(): + # Invalid UTF-8 must not be "sanitized" into U+FFFD; an ambiguous repeated filename is rejected too. + alice, _bob, _eve, chat = _trio() + png = _png(10) + path = f"/api/v1/chats/{chat['id']}/attachments" + for query in ("?filename=%FF%FE.png", "?filename=%C3.png", "?filename=%ED%A0%80.png", + "?filename=ok.png&filename=a%2Fb", "?filename=a.png&filename=b.png"): + resp = call("POST", path + query, 400, token=alice["token"], raw=png, headers={"Content-Type": "image/png"}) + assert resp["json"]["error"]["code"] == "invalid_request", query + att = call("POST", path + "?filename=%D1%84%2B1+2.png", 201, token=alice["token"], raw=png, + headers={"Content-Type": "image/png"})["json"] + assert att["filename"] == "ф+1 2.png" + + +def test_same_filename_distinct_objects(): + alice, bob, _eve, chat = _trio() + first, second = _png(8), _png(9) + a = upload(alice["token"], chat["id"], first, "same.png", "image/png", 201)["json"] + b = upload(alice["token"], chat["id"], second, "same.png", "image/png", 201)["json"] + assert a["id"] != b["id"] + assert _download(bob["token"], a["id"])["bytes"] == first + assert _download(bob["token"], b["id"])["bytes"] == second + + +def test_empty_body_and_variants(): + alice, bob, _eve, chat = _trio() + upload(alice["token"], chat["id"], b"", "empty.png", "image/png", 400) + att = upload(alice["token"], chat["id"], _png(), "v.png", "image/png", 201)["json"] + _download(bob["token"], att["id"], 200, "?variant=original") + _download(bob["token"], att["id"], 404, "?variant=thumbnail") + _download(bob["token"], att["id"], 400, "?variant=../../etc") + + +def test_parallel_uploads_distinct(): + alice, bob, _eve, chat = _trio() + blobs = [_png(i) + b"" for i in range(12)] + + def send(i): + return upload(alice["token"], chat["id"], blobs[i], f"p{i}.png", "image/png", 201)["json"] + + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + results = list(pool.map(send, range(12))) + assert len({r["id"] for r in results}) == 12 + for i, att in enumerate(results): + assert att["sha256"] == hashlib.sha256(blobs[i]).hexdigest() + + +def _raw_partial_upload(token: str, chat_id: str, declared: int, payload: bytes, hold: float = 0.0) -> None: + parsed = urllib.parse.urlparse(BASE_URL) + sock = socket.create_connection((parsed.hostname, parsed.port or 80), timeout=30) + head = ( + f"POST /api/v1/chats/{chat_id}/attachments?filename=cut.png HTTP/1.1\r\n" + f"Host: {parsed.hostname}\r\nAuthorization: Bearer {token}\r\n" + f"Content-Type: image/png\r\nContent-Length: {declared}\r\n\r\n" + ).encode() + sock.sendall(head + payload) + if hold: + time.sleep(hold) + sock.close() + + +def test_interrupted_upload_does_not_break_service(): + alice, bob, _eve, chat = _trio() + big = imagegen.large_png(2 * MIB) + for _ in range(3): + _raw_partial_upload(alice["token"], chat["id"], len(big), big[: 300 * 1024]) + att = upload(alice["token"], chat["id"], _png(), "after.png", "image/png", 201)["json"] + _download(bob["token"], att["id"]) diff --git a/student-tests/test_attachments_ops.py b/student-tests/test_attachments_ops.py new file mode 100644 index 00000000..4ccbf4a3 --- /dev/null +++ b/student-tests/test_attachments_ops.py @@ -0,0 +1,162 @@ +"""Lab4 grade 4 operator checks that need the Docker CLI (RUN_DOCKER_TESTS=1). + +Non-destructive: nothing is stopped; checks run inside the API containers. +""" + +from __future__ import annotations + +import concurrent.futures +import json +import os +import subprocess +import threading +import time + +import imagegen +from helpers import call, chat_for, register, upload +from test_attachments import _raw_partial_upload + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") + + +def _enabled(name: str) -> bool: + if os.environ.get("RUN_DOCKER_TESTS") != "1": + print(f"SKIP {name} (set RUN_DOCKER_TESTS=1)") + return False + return True + + +def _run(args, check=True): + proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8") + if check and proc.returncode != 0: + raise AssertionError(f"{' '.join(args[:6])} rc={proc.returncode}: {proc.stdout[-800:]}{proc.stderr[-400:]}") + return proc + + +def _api_containers(): + out = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).stdout.split() + assert len(out) >= 2, out + return out + + +def _exec_json(container: str, *args): + proc = _run(["docker", "exec", container, "python", "-m", *args]) + return json.loads(proc.stdout[proc.stdout.index("{"):]) + + +def _bucket_state(container: str) -> dict: + return _exec_json(container, "messenger.reconcile", "--dry-run", "--grace-seconds", "0") + + +def test_bucket_private_and_api_key_least_privilege(): + if not _enabled("test_bucket_private_and_api_key_least_privilege"): + return + report = _exec_json(_api_containers()[0], "messenger.selfcheck", "s3") + failed = [r for r in report["results"] if not r["ok"]] + assert report["passed"] and not failed, failed + + +def test_s3_db_failure_compensation_and_reconcile(): + if not _enabled("test_s3_db_failure_compensation_and_reconcile"): + return + report = _exec_json(_api_containers()[0], "messenger.selfcheck", "consistency") + failed = [r for r in report["results"] if not r["ok"]] + assert report["passed"] and not failed, failed + + +def test_interrupted_and_stalled_uploads_leave_nothing(): + if not _enabled("test_interrupted_and_stalled_uploads_leave_nothing"): + return + containers = _api_containers() + alice, bob = register("cut_a"), register("cut_b") + chat = chat_for(alice, bob) + before = _bucket_state(containers[0]) + big = imagegen.large_png(3 * 1024 * 1024) + for _ in range(4): + _raw_partial_upload(alice["token"], chat["id"], len(big), big[: 512 * 1024]) + # Stalled client: headers + part of body, then silence past UPLOAD_IDLE_TIMEOUT_SECONDS (10 s). + _raw_partial_upload(alice["token"], chat["id"], len(big), big[: 256 * 1024], hold=12) + time.sleep(1) + after = _bucket_state(containers[0]) + assert after["objects_scanned"] == before["objects_scanned"], (before, after) + # Leftovers of earlier S3-outage runs (intents kept for the reconciler) are not ours. + assert after["stale_intents"] == before["stale_intents"], (before, after) + assert after["orphans_found"] == before["orphans_found"], (before, after) + for container in containers: + spool = _exec_json(container, "messenger.selfcheck", "spool") + assert spool["passed"], (container, spool) + att = upload(alice["token"], chat["id"], imagegen.rgb_png(8, 8), "ok.png", "image/png", 201)["json"] + call("GET", "/api/v1/attachments/" + att["id"] + "/content", 200, token=bob["token"]) + + +def test_parallel_large_uploads_keep_s3_within_memory(): + # 2 replicas × 8 slots = 16 PUTs in flight; SeaweedFS holds ≈30–40 MiB per 10 MiB PUT. + # With mem_limit 512m it ran at the limit and was OOM-killed (exit 137) under 40 uploads. + if not _enabled("test_parallel_large_uploads_keep_s3_within_memory"): + return + s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0] + limit = int(_run(["docker", "inspect", "-f", "{{.HostConfig.Memory}}", s3]).stdout.strip()) + alice = register("par_a") + chat = chat_for(alice) + payload = imagegen.large_png(int(9.5 * 1024 * 1024)) + peak = [0] + stop = threading.Event() + + def sample(): + while not stop.is_set(): + stat = _run(["docker", "exec", s3, "cat", "/sys/fs/cgroup/memory.stat"], check=False).stdout + for line in stat.splitlines(): + if line.startswith("anon "): + peak[0] = max(peak[0], int(line.split()[1])) + time.sleep(0.1) + + def send(i): + return call("POST", f"/api/v1/chats/{chat['id']}/attachments?filename=par{i}.png", token=alice["token"], + raw=payload, headers={"Content-Type": "image/png"})["status"] + + sampler = threading.Thread(target=sample) + sampler.start() + try: + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as pool: + statuses = list(pool.map(send, range(16))) + finally: + time.sleep(1) + stop.set() + sampler.join() + state = _run(["docker", "inspect", "-f", "{{.State.Running}} {{.State.OOMKilled}}", s3]).stdout.split() + assert state == ["true", "false"], state + # 503 is the documented backpressure (slot wait > 5 s), never an S3 failure. + assert set(statuses) <= {201, 503} and statuses.count(201) >= 10, statuses + assert peak[0] < 0.8 * limit, (peak[0] // 2**20, limit // 2**20) + call("GET", "/health/ready", 200) + + +def test_s3_can_grow_volumes(): + # SeaweedFS grows a collection by 7 volumes at a time; when no slot is left every PUT + # fails ("No writable volumes") while readiness stays 200. -volume.max=16 hit that at ≈4.7 GiB. + if not _enabled("test_s3_can_grow_volumes"): + return + s3 = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "s3"]).stdout.split()[0] + status = json.loads(_run(["docker", "exec", s3, "wget", "-q", "-O", "-", "http://127.0.0.1:9333/dir/status"]).stdout) + topology = status["Topology"] + assert topology["Free"] >= 8, {k: topology[k] for k in ("Max", "Free")} + + +def test_no_original_kept_on_api_replicas(): + if not _enabled("test_no_original_kept_on_api_replicas"): + return + alice, bob = register("loc_a"), register("loc_b") + chat = chat_for(alice, bob) + for i in range(4): + upload(alice["token"], chat["id"], imagegen.rgb_png(10 + i, 10), f"l{i}.png", "image/png", 201) + for container in _api_containers(): + spool = _exec_json(container, "messenger.selfcheck", "spool") + assert spool["passed"], (container, spool) + # Any regular file created after the container started, anywhere on its writable layer + # (/etc/hosts, /etc/resolv.conf, /etc/hostname are bind mounts written by Docker itself). + started = _run(["docker", "inspect", "-f", "{{.State.StartedAt}}", container]).stdout.strip() + found = _run(["docker", "exec", "-u", "0", container, "find", "/", "-xdev", "-type", "f", + "-newermt", started, "-not", "-path", "/proc/*", + "-not", "-path", "/etc/hosts", "-not", "-path", "/etc/resolv.conf", + "-not", "-path", "/etc/hostname"]).stdout.split() + assert not found, (container, found[:5]) diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py new file mode 100644 index 00000000..8920d294 --- /dev/null +++ b/student-tests/test_concurrency.py @@ -0,0 +1,66 @@ +"""Concurrent registration race and parallel messages.""" + +from __future__ import annotations + +import concurrent.futures +import uuid + +from helpers import BASE_URL, call, register + + +def test_parallel_messages_unique(): + a = register("par") + b = register("parb") + chat = call( + "POST", + "/api/v1/chats", + 201, + token=a["token"], + body={"title": "parallel", "member_ids": [a["id"], b["id"]]}, + )["json"] + + def send(i: int): + return call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=a["token"], + body={"text": f"msg-{i}"}, + )["json"]["id"] + + with concurrent.futures.ThreadPoolExecutor(max_workers=10) as pool: + ids = list(pool.map(send, range(100))) + assert len(ids) == len(set(ids)) == 100 + + +def test_username_race_one_winner(): + username = "race_" + uuid.uuid4().hex[:12] + password = "T3st-" + uuid.uuid4().hex + body = { + "username": username, + "display_name": "Race", + "password": password, + } + + def attempt(_): + import json + import urllib.error + import urllib.request + + raw = json.dumps(body).encode("utf-8") + req = urllib.request.Request( + BASE_URL + "/api/v1/users", + data=raw, + headers={"Content-Type": "application/json", "Accept": "application/json"}, + method="POST", + ) + try: + resp = urllib.request.urlopen(req, timeout=30) + except urllib.error.HTTPError as err: + resp = err + return resp.code + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + codes = list(pool.map(attempt, range(8))) + assert codes.count(201) == 1, codes + assert codes.count(409) == 7, codes diff --git a/student-tests/test_deps.py b/student-tests/test_deps.py new file mode 100644 index 00000000..b1b56067 --- /dev/null +++ b/student-tests/test_deps.py @@ -0,0 +1,176 @@ +"""Dependency failure checks (Redis/Postgres/S3). Optional: RUN_DEPS_TEST=1.""" + +from __future__ import annotations + +import hashlib +import os +import subprocess +import time +import uuid + +import imagegen +from helpers import call, chat_for, register, upload + + +ROOT = os.path.join(os.path.dirname(__file__), "..") + + +def _compose(*args: str) -> None: + cmd = ["docker", "compose", "--env-file", ".env", *args] + subprocess.check_call(cmd, cwd=ROOT) + + +def _started_at(service: str) -> dict: + ids = subprocess.check_output(["docker", "compose", "--env-file", ".env", "ps", "-q", service], cwd=ROOT, text=True).split() + assert ids, service + return { + c: subprocess.check_output(["docker", "inspect", "-f", "{{.State.StartedAt}}", c], text=True).strip() + for c in ids + } + + +def _wait_ready(timeout_s: int = 60, stable: int = 6) -> None: + # Several consecutive 200s: HAProxy may still finish health checks that started + # during the outage and briefly mark every replica DOWN after the first 200. + deadline = time.time() + timeout_s + streak = 0 + while time.time() < deadline: + try: + streak = streak + 1 if call("GET", "/health/ready")["status"] == 200 else 0 + except Exception: + streak = 0 + if streak >= stable: + return + time.sleep(1) + raise AssertionError("stack did not become ready in time") + + +def _unavailable(method: str, path: str, token: str | None = None, body=None) -> dict: + # Every 503 — from a replica or from HAProxy itself — is the JSON error envelope + # with X-Request-Id / X-Instance-Id (checked inside call()). + resp = call(method, path, 503, token=token, body=body) + assert resp["json"]["error"]["code"] == "unavailable", resp["json"] + return resp + + +def _skip(name: str) -> bool: + if os.environ.get("RUN_DEPS_TEST") != "1": + print(f"SKIP {name} (set RUN_DEPS_TEST=1)") + return True + return False + + +def test_redis_stop_returns_503_or_unready(): + if _skip("test_redis_stop_returns_503_or_unready"): + return + actor = register("deps") + _compose("stop", "redis") + time.sleep(2) + try: + _unavailable("GET", "/health/ready") + _unavailable("GET", "/api/v1/users/me", actor["token"]) + _unavailable("DELETE", "/api/v1/auth/sessions/current", actor["token"]) + finally: + _compose("start", "redis") + _wait_ready() + call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_s3_stop_degrades_only_attachments(): + # S3 is on the synchronous attachment path, so readiness reports 503; HAProxy then + # falls back to the liveness-checked backend and text chat/metadata keep working. + if _skip("test_s3_stop_degrades_only_attachments"): + return + alice, bob = register("s3down_a"), register("s3down_b") + chat = chat_for(alice, bob) + data = imagegen.rgb_png(24, 12) + att = upload(alice["token"], chat["id"], data, "before.png", "image/png", 201)["json"] + _compose("stop", "s3") + try: + time.sleep(3) + _unavailable("GET", "/health/ready") + started = time.monotonic() + up = upload(alice["token"], chat["id"], data, "during.png", "image/png", 503) + elapsed = time.monotonic() - started + assert up["json"]["error"]["code"] == "unavailable", up["json"] + assert elapsed < 15, elapsed + _unavailable("GET", f"/api/v1/attachments/{att['id']}/content", bob["token"]) + meta = call("GET", f"/api/v1/attachments/{att['id']}", 200, token=bob["token"])["json"] + assert meta["sha256"] == att["sha256"] + msg = call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=bob["token"], + body={"text": "S3 недоступен, но чат работает", "attachment_ids": [att["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + )["json"] + assert msg["attachment_ids"] == [att["id"]] + print(" s3 down: ready 503, upload 503 in %.2fs, download 503, metadata/messages OK" % elapsed) + finally: + _compose("start", "s3") + _compose("restart", "s3-gw") + _wait_ready() + dl = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"] + upload(alice["token"], chat["id"], data, "after.png", "image/png", 201) + + +def test_postgres_stop_returns_503(): + if _skip("test_postgres_stop_returns_503"): + return + actor = register("deps_pg") + _compose("stop", "postgres") + time.sleep(2) + try: + _unavailable("GET", "/health/ready") + started = time.monotonic() + _unavailable("GET", "/api/v1/users/me", actor["token"]) + _unavailable("GET", "/api/v1/chats", actor["token"]) + assert time.monotonic() - started < 15 + finally: + _compose("start", "postgres") + _wait_ready(90) + call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_restart_hook_restarts_api_and_stores(): + # tests/README: the persistence hook must restart the API and every store. + if _skip("test_restart_hook_restarts_api_and_stores"): + return + actor = register("deps_hook") + services = ("api", "postgres", "redis", "s3") + before = {s: _started_at(s) for s in services} + hook = ["cmd", "/c", r"scripts\restart.cmd"] if os.name == "nt" else ["sh", "scripts/restart.sh"] + subprocess.check_call(hook, cwd=ROOT, stdout=subprocess.DEVNULL) + _wait_ready(90) + for s in services: + after = _started_at(s) + if s == "api": + assert len(after) >= 2, after + stale = [c for c, t in after.items() if before[s].get(c) == t] + assert not stale, (s, stale) + call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_postgres_pause_fails_fast(): + # A paused (not stopped) database never answers: no RST, statement_timeout is never + # evaluated. Requests must still end as JSON 503 within a bounded time. + if _skip("test_postgres_pause_fails_fast"): + return + actor = register("deps_pause") + _compose("pause", "postgres") + try: + timings = [] + for path in ("/api/v1/users/me", "/api/v1/chats", "/api/v1/users/me"): + started = time.monotonic() + _unavailable("GET", path, actor["token"]) + timings.append(time.monotonic() - started) + assert max(timings) < 15, timings + time.sleep(4) + _unavailable("GET", "/health/ready") + print(" postgres paused: 503 after " + ", ".join("%.2fs" % t for t in timings)) + finally: + _compose("unpause", "postgres") + _wait_ready(90) + call("GET", "/api/v1/users/me", 200, token=actor["token"]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..52801ed8 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,165 @@ +"""Lab3: shared Idempotency-Key and multi-replica visibility.""" + +from __future__ import annotations + +import concurrent.futures +import uuid + +from helpers import call, register + + +def _chat_pair(): + a = register("idem_a") + b = register("idem_b") + chat = call( + "POST", + "/api/v1/chats", + 201, + token=a["token"], + body={"title": "idem", "member_ids": [a["id"], b["id"]]}, + )["json"] + return a, b, chat + + +def test_missing_idempotency_key_400(): + a, _b, chat = _chat_pair() + call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 400, + token=a["token"], + body={"text": "no-key"}, + headers={"Idempotency-Key": ""}, + ) + + +def test_replay_same_key_same_body(): + a, _b, chat = _chat_pair() + key = uuid.uuid4().hex + first = call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=a["token"], + body={"text": "same"}, + headers={"Idempotency-Key": key}, + )["json"] + second = call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=a["token"], + body={"text": "same"}, + headers={"Idempotency-Key": key}, + )["json"] + assert first == second + + +def test_empty_attachment_ids_equals_absent(): + # Peer remark: attachment_ids: [] ≡ absent field; must not 400; same idempotency hash. + a, _b, chat = _chat_pair() + key = uuid.uuid4().hex + first = call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=a["token"], + body={"text": "with-empty-attachments", "attachment_ids": []}, + headers={"Idempotency-Key": key}, + )["json"] + second = call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=a["token"], + body={"text": "with-empty-attachments"}, + headers={"Idempotency-Key": key}, + )["json"] + assert first == second + + +def test_same_key_different_body_409(): + a, _b, chat = _chat_pair() + key = uuid.uuid4().hex + call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=a["token"], + body={"text": "one"}, + headers={"Idempotency-Key": key}, + ) + call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 409, + token=a["token"], + body={"text": "two"}, + headers={"Idempotency-Key": key}, + ) + + +def test_key_scoped_to_user_and_chat(): + # Bob reusing Alice's key must get his own message, not her stored response. + a, b, chat = _chat_pair() + key = uuid.uuid4().hex + first = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=a["token"], + body={"text": "same"}, headers={"Idempotency-Key": key})["json"] + other = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=b["token"], + body={"text": "same"}, headers={"Idempotency-Key": key})["json"] + assert other["id"] != first["id"] and other["sender_id"] == b["id"] + second_chat = call("POST", "/api/v1/chats", 201, token=a["token"], + body={"title": "idem2", "member_ids": [a["id"], b["id"]]})["json"] + elsewhere = call("POST", f"/api/v1/chats/{second_chat['id']}/messages", 201, token=a["token"], + body={"text": "same"}, headers={"Idempotency-Key": key})["json"] + assert elsewhere["id"] != first["id"] and elsewhere["chat_id"] == second_chat["id"] + + +def test_invalid_key_format_400(): + a, _b, chat = _chat_pair() + for key in ("has space", "x" * 129, "a/b", "a,b", "ключ".encode().decode("latin-1")): + resp = call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=a["token"], + body={"text": "bad key"}, headers={"Idempotency-Key": key}) + assert resp["json"]["error"]["code"] == "invalid_request", key + call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=a["token"], + body={"text": "ok key"}, headers={"Idempotency-Key": "A-z0.9_:" + "x" * 120}) + + +def test_parallel_same_key_one_message(): + a, b, chat = _chat_pair() + key = uuid.uuid4().hex + text = "parallel-idem-" + key + + def send(_): + return call( + "POST", + f"/api/v1/chats/{chat['id']}/messages", + 201, + token=a["token"], + body={"text": text}, + headers={"Idempotency-Key": key}, + )["json"] + + with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(send, range(8))) + ids = {m["id"] for m in results} + assert len(ids) == 1, ids + page = call( + "GET", + f"/api/v1/chats/{chat['id']}/messages?limit=100", + 200, + token=b["token"], + )["json"] + assert sum(1 for m in page["items"] if m["text"] == text) == 1 + + +def test_multiple_instance_ids_observed(): + a = register("inst") + seen = set() + for _ in range(40): + seen.add( + call("GET", "/api/v1/users/me", 200, token=a["token"])["headers"].get( + "X-Instance-Id" + ) + ) + assert len(seen) >= 2, f"expected ≥2 replicas via LB, got {seen}" diff --git a/student-tests/test_images_unit.py b/student-tests/test_images_unit.py new file mode 100644 index 00000000..0f067976 --- /dev/null +++ b/student-tests/test_images_unit.py @@ -0,0 +1,139 @@ +"""Offline checks of the streaming PNG/JPEG validators (no server needed).""" + +from __future__ import annotations + +import struct +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from messenger import images # noqa: E402 + +import imagegen # noqa: E402 + +LIMIT = 40_000_000 + + +def _validate(data: bytes, content_type: str, step: int = 0): + v = images.validator_for(content_type, LIMIT) + if step: + for i in range(0, len(data), step): + v.feed(data[i:i + step]) + else: + v.feed(data) + return v.finish() + + +def _rejected(data: bytes, content_type: str, step: int = 0) -> int: + try: + _validate(data, content_type, step) + except images.ImageRejected as exc: + return exc.status + raise AssertionError("image was accepted") + + +def test_valid_pngs_any_chunking(): + cases = { + "public": (imagegen.PUBLIC_PNG.read_bytes(), (320, 200)), + "rgb": (imagegen.rgb_png(33, 17), (33, 17)), + "interlaced": (imagegen.interlaced_rgb_png(37, 23), (37, 23)), + "interlaced_tiny": (imagegen.interlaced_rgb_png(1, 1), (1, 1)), + "gray1": (imagegen.gray1_png(13, 9), (13, 9)), + "palette": (imagegen.palette_png(10, 6), (10, 6)), + "rgba16": (imagegen.png16_rgba(5, 4), (5, 4)), + } + for name, (data, size) in cases.items(): + for step in (0, 1, 7, 4096): + assert _validate(data, "image/png", step) == size, (name, step) + + +def test_valid_jpegs_any_chunking(): + for name in ("sample.jpg", "progressive.jpg", "gray.jpg"): + data = imagegen.fixture(name) + for step in (0, 1, 5, 4096): + assert _validate(data, "image/jpeg", step) == (320, 200), (name, step) + + +def test_pixel_limit_detected_from_header_only(): + data = imagegen.gray1_png(8000, 5001) + v = images.validator_for("image/png", LIMIT) + try: + v.feed(data[:33]) # signature + IHDR only + except images.ImageRejected as exc: + assert exc.status == 413 + else: + raise AssertionError("pixel limit not enforced on IHDR") + jpeg = bytearray(imagegen.fixture("sample.jpg")) + sof = jpeg.find(b"\xff\xc0") + struct.pack_into(">HH", jpeg, sof + 5, 6000, 8000) + v = images.validator_for("image/jpeg", LIMIT) + try: + v.feed(bytes(jpeg[: sof + 32])) + except images.ImageRejected as exc: + assert exc.status == 413 + else: + raise AssertionError("pixel limit not enforced on SOF") + + +def test_exactly_at_pixel_limit_accepted(): + assert _validate(imagegen.gray1_png(8000, 5000), "image/png") == (8000, 5000) + + +def test_corrupted_png_rejected(): + good = imagegen.rgb_png(20, 10) + flipped = bytearray(good) + flipped[-20] ^= 0xFF # inside IDAT → CRC mismatch + assert _rejected(bytes(flipped), "image/png") == 415 + assert _rejected(good[: len(good) // 2], "image/png") == 415 + assert _rejected(good + b"trailing", "image/png") == 415 + ihdr = imagegen.chunk(b"IHDR", struct.pack(">IIBBBBB", 4, 4, 8, 2, 0, 0, 0)) + bad_zlib = b"\x89PNG\r\n\x1a\n" + ihdr + imagegen.chunk(b"IDAT", b"\x78\x9c" + b"\xff" * 40) + imagegen.chunk(b"IEND", b"") + assert _rejected(bad_zlib, "image/png") == 415 + short = imagegen.png_from_raw(10, 10, b"\x00" * 10) # fewer rows than IHDR declares + assert _rejected(short, "image/png") == 415 + long = imagegen.png_from_raw(2, 2, b"\x00" * 100) # decompression bomb style overflow + assert _rejected(long, "image/png") == 415 + assert _rejected(imagegen.palette_png(4, 4).replace(b"PLTE", b"tEXt"), "image/png") == 415 + + +def test_corrupted_jpeg_rejected(): + good = imagegen.fixture("sample.jpg") + assert _rejected(good[: len(good) // 2], "image/jpeg") == 415 + assert _rejected(b"\xff\xd8\xff\xe0" + bytes(range(256)) * 4, "image/jpeg") == 415 + no_sof = b"\xff\xd8\xff\xda\x00\x08\x01\x01\x00\x00\x3f\x00" + b"\x12" * 64 + b"\xff\xd9" + assert _rejected(no_sof, "image/jpeg") == 415 + assert _rejected(b"\xff\xd8\xff\xd9", "image/jpeg") == 415 + arithmetic = bytearray(good) + arithmetic[good.find(b"\xff\xc0") + 1] = 0xC9 + assert _rejected(bytes(arithmetic), "image/jpeg") == 415 + + +def test_reviewer_corrupted_pngs_rejected(): + for name, data in imagegen.corrupted_pngs().items(): + for step in (0, 1, 4096): + assert _rejected(data, "image/png", step) == 415, (name, step) + + +def test_all_filter_types_accepted(): + for interlace in (0, 1): + for size in ((1, 1), (9, 7), (33, 20)): + data = imagegen.all_filters_png(*size, interlace=interlace) + for step in (0, 1, 7): + assert _validate(data, "image/png", step) == size, (interlace, size, step) + + +def test_reviewer_corrupted_jpegs_structural_part(): + # The rest (no tables, missing scan data, truncated + EOI) needs a real decoder: + # covered over HTTP by test_attachments.test_reviewer_corrupted_images_415. + corpus = imagegen.corrupted_jpegs() + for name in ("jpeg_18_bytes", "jpeg_truncated_no_eoi", "jpeg_12_bit"): + assert _rejected(corpus[name], "image/jpeg") == 415, name + + +def test_foreign_formats_rejected(): + for blob in (b"GIF89a" + b"\x00" * 100, b"%PDF-1.7\n", b"", b"MZ\x90\x00", b""): + assert _rejected(blob, "image/png") == 415 + assert _rejected(blob, "image/jpeg") == 415 + assert images.sniff(imagegen.fixture("sample.jpg")[:8]) == "image/jpeg" + assert images.sniff(imagegen.PUBLIC_PNG.read_bytes()[:8]) == "image/png" diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..dbbb4703 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,34 @@ +"""Session expiry (short TTL) — run against stack with SESSION_TTL_SECONDS=2.""" + +from __future__ import annotations + +import base64 +import os +import time + +from helpers import call, register + + +def test_session_expiry_if_short_ttl(): + ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + if ttl > 5: + print("SKIP expiry: SESSION_TTL_SECONDS=%s (need <=5 for this test)" % ttl) + return + a = register("expire") + time.sleep(ttl + 1.5) + call("GET", "/api/v1/users/me", 401, token=a["token"]) + + +def test_login_after_register(): + a = register("relogin") + cred = base64.b64encode( + (a["registration"]["username"] + ":" + a["registration"]["password"]).encode() + ).decode() + session = call( + "POST", + "/api/v1/auth/sessions", + 201, + headers={"Authorization": "Basic " + cred}, + )["json"] + assert session["token_type"] == "Bearer" + assert session["user"]["id"] == a["id"] diff --git a/student-tests/test_shutdown.py b/student-tests/test_shutdown.py new file mode 100644 index 00000000..5991fe93 --- /dev/null +++ b/student-tests/test_shutdown.py @@ -0,0 +1,249 @@ +"""Graceful shutdown of an API replica (RUN_DOCKER_TESTS=1). + +Each test stops ONE API replica and brings it back; dependencies are not touched. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import threading +import time +import uuid +from datetime import datetime + +import imagegen +from helpers import call, chat_for, register + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +STOP_TIMEOUT_S = 15 # = stop_grace_period of the api service + +# Runs inside the replica: SIGTERM to PID 1 (as `docker stop` does), then probes the +# same process directly while HAProxy is still expected to route to it. +PROBE = r""" +import json, os, signal, time, urllib.error, urllib.request + +def get(path, token=None): + req = urllib.request.Request("http://127.0.0.1:8080" + path) + if token: + req.add_header("Authorization", "Bearer " + token) + try: + with urllib.request.urlopen(req, timeout=2) as resp: + return resp.status + except urllib.error.HTTPError as err: + return err.code + except Exception as exc: + return type(exc).__name__ + +token = os.environ["DRAIN_TOKEN"] +drain = float(os.environ.get("SHUTDOWN_DRAIN_SECONDS", "3")) +with open("/proc/1/cmdline", "rb") as f: + pid1 = f.read().replace(b"\0", b" ").decode().strip() +before = [get("/health/ready"), get("/api/v1/users/me", token)] +sent_at = time.time() +t0 = time.monotonic() +os.kill(1, signal.SIGTERM) +samples = [] +while time.monotonic() - t0 < drain - 0.8: + samples.append([round(time.monotonic() - t0, 2), get("/health/ready"), get("/api/v1/users/me", token)]) + time.sleep(0.1) +print(json.dumps({"pid1": pid1, "drain": drain, "before": before, "sent_at": sent_at, "samples": samples})) +""" + +# Runs inside the replica: a slow upload (body from stdin) whose body is still being sent +# when the drain window ends and the listener is closed. +SLOW_UPLOAD = r""" +import http.client, json, os, signal, sys, time + +body = sys.stdin.buffer.read() +drain = float(os.environ.get("SHUTDOWN_DRAIN_SECONDS", "3")) +conn = http.client.HTTPConnection("127.0.0.1", 8080, timeout=30) +conn.putrequest("POST", "/api/v1/chats/%s/attachments?filename=drain.png" % os.environ["DRAIN_CHAT"]) +conn.putheader("Authorization", "Bearer " + os.environ["DRAIN_TOKEN"]) +conn.putheader("Content-Type", "image/png") +conn.putheader("Content-Length", str(len(body))) +conn.endheaders() +t0 = time.monotonic() +step = len(body) // 40 + 1 +for i in range(0, len(body), step): + conn.send(body[i:i + step]) + if i == 0: + os.kill(1, signal.SIGTERM) + time.sleep((drain + 2) / 40) +resp = conn.getresponse() +print(json.dumps({"status": resp.status, "body_sent_s": round(time.monotonic() - t0, 2), + "json": json.loads(resp.read() or b"null")})) +""" + + +def _enabled(name: str) -> bool: + if os.environ.get("RUN_DOCKER_TESTS") != "1": + print(f"SKIP {name} (set RUN_DOCKER_TESTS=1)") + return False + return True + + +def _run(args, check=True): + proc = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, encoding="utf-8") + if check and proc.returncode != 0: + raise AssertionError(f"{' '.join(args[:6])} rc={proc.returncode}: {proc.stdout[-800:]}{proc.stderr[-400:]}") + return proc + + +def _api_containers(): + out = _run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).stdout.split() + assert len(out) >= 2, out + return out + + +def _inspect(container: str, fmt: str) -> str: + return _run(["docker", "inspect", "-f", fmt, container]).stdout.strip() + + +def _docker_time(value: str) -> float: + # 2026-11-06T15:37:12.123456789Z → POSIX seconds (nanoseconds cut to microseconds). + head, _, frac = value.rstrip("Z").partition(".") + stamp = datetime.fromisoformat(head + "+00:00").timestamp() + return stamp + (float("0." + frac[:6]) if frac else 0.0) + + +def _wait_in_rotation(host: str, timeout_s: int = 90) -> float: + started = time.monotonic() + while time.monotonic() - started < timeout_s: + try: + if call("GET", "/health/ready")["headers"].get("X-Instance-Id") == host: + return time.monotonic() - started + except Exception: + pass + time.sleep(0.5) + raise AssertionError(f"replica {host} not back in LB rotation after {timeout_s}s") + + +def _wait_restarted(container: str, restarts: int) -> None: + # unless-stopped restarts the container after a SIGTERM sent from inside it. + deadline = time.monotonic() + 60 + while int(_inspect(container, "{{.RestartCount}}")) == restarts: + assert time.monotonic() < deadline, "replica did not exit after SIGTERM" + time.sleep(0.5) + + +def test_sigterm_drains_readiness_but_keeps_serving(): + # After SIGTERM the replica must report /health/ready 503 at once (HAProxy takes it + # out after `inter 1s fall 2`) but keep answering normal requests until then. + if not _enabled("test_sigterm_drains_readiness_but_keeps_serving"): + return + actor = register("drain") + target = _api_containers()[0] + host = _inspect(target, "{{.Config.Hostname}}") + restarts = int(_inspect(target, "{{.RestartCount}}")) + proc = subprocess.run(["docker", "exec", "-e", "DRAIN_TOKEN", target, "python", "-c", PROBE], + cwd=ROOT, capture_output=True, text=True, encoding="utf-8", + env=dict(os.environ, DRAIN_TOKEN=actor["token"])) + if proc.returncode != 0: + _wait_in_rotation(host) + raise AssertionError(f"replica exited during the drain window (probe rc={proc.returncode}) {proc.stderr[-300:]}") + probe = json.loads(proc.stdout[proc.stdout.index("{"):]) + samples = probe["samples"] + assert probe["pid1"].startswith("python -m messenger"), probe["pid1"] + assert probe["drain"] >= 2, f"drain window {probe['drain']}s < HAProxy inter 1s x fall 2" + assert probe["before"] == [200, 200], probe["before"] + assert len(samples) >= 5, samples + not_served = [s for s in samples if s[2] != 200] + assert not not_served, f"requests failed during the drain window: {not_served}" + still_ready = [s for s in samples if s[0] >= 0.3 and s[1] != 503] + assert not still_ready, f"/health/ready not 503 while draining: {still_ready}" + + _wait_restarted(target, restarts) + # FinishedAt is the exit of the drained process (same clock as the probe in the container). + stop_s = _docker_time(_inspect(target, "{{.State.FinishedAt}}")) - probe["sent_at"] + assert probe["drain"] - 0.2 <= stop_s < STOP_TIMEOUT_S, stop_s + back_s = _wait_in_rotation(host) + print(" drain: %d probes, ready 503 / users/me 200; exit %.2fs after SIGTERM; back in LB in %.1fs" + % (len(samples), stop_s, back_s)) + + +def test_inflight_upload_finishes_after_listener_closed(): + # The body is still arriving when the drain window ends: the listener closes, but the + # in-flight upload (spool → S3 PUT → DB commit) must complete before the process exits. + if not _enabled("test_inflight_upload_finishes_after_listener_closed"): + return + alice, bob = register("drain_up_a"), register("drain_up_b") + chat = chat_for(alice, bob, title="drain upload") + data = imagegen.large_png(2 * 1024 * 1024) + target = _api_containers()[0] + host = _inspect(target, "{{.Config.Hostname}}") + restarts = int(_inspect(target, "{{.RestartCount}}")) + proc = subprocess.run(["docker", "exec", "-i", "-e", "DRAIN_TOKEN", "-e", "DRAIN_CHAT", target, + "python", "-c", SLOW_UPLOAD], + cwd=ROOT, input=data, capture_output=True, + env=dict(os.environ, DRAIN_TOKEN=alice["token"], DRAIN_CHAT=chat["id"])) + out = proc.stdout.decode("utf-8", "replace") + _wait_restarted(target, restarts) + back_s = _wait_in_rotation(host) + assert proc.returncode == 0, f"upload probe rc={proc.returncode}: {proc.stderr[-300:]!r}" + result = json.loads(out[out.index("{"):]) + assert result["status"] == 201, result + att = result["json"] + assert att["sha256"] == hashlib.sha256(data).hexdigest(), att + dl = call("GET", f"/api/v1/attachments/{att['id']}/content", 200, token=bob["token"]) + assert hashlib.sha256(dl["bytes"]).hexdigest() == att["sha256"] + print(" in-flight upload: body sent over %.1fs across listener close -> 201, download via LB OK; back in LB in %.1fs" + % (result["body_sent_s"], back_s)) + + +def test_replica_stop_under_load_has_no_errors(): + # Closed-loop POSTs through the LB while one replica gets `docker stop`: requests + # that HAProxy still sends to it during the drain window must succeed. + if not _enabled("test_replica_stop_under_load_has_no_errors"): + return + alice, bob = register("drain_load_a"), register("drain_load_b") + chat = chat_for(alice, bob, title="drain") + target = _api_containers()[0] + host = _inspect(target, "{{.Config.Hostname}}") + stats = {"ok": 0, "target_after_stop": 0, "errors": []} + lock = threading.Lock() + stop = threading.Event() + stopping = threading.Event() + + def worker(): + while not stop.is_set(): + key = uuid.uuid4().hex + try: + r = call("POST", f"/api/v1/chats/{chat['id']}/messages", token=alice["token"], + body={"text": "drain-" + key}, headers={"Idempotency-Key": key}) + status, instance = r["status"], r["headers"].get("X-Instance-Id") + except Exception as exc: + status, instance = type(exc).__name__, None + with lock: + if status == 201: + stats["ok"] += 1 + if stopping.is_set() and instance == host: + stats["target_after_stop"] += 1 + else: + stats["errors"].append(status) + time.sleep(0.05) + + threads = [threading.Thread(target=worker, daemon=True) for _ in range(4)] + for th in threads: + th.start() + try: + time.sleep(1.5) + stopping.set() + started = time.monotonic() + _run(["docker", "stop", "-t", str(STOP_TIMEOUT_S), target]) + stop_s = time.monotonic() - started + time.sleep(1.5) + finally: + stop.set() + for th in threads: + th.join(timeout=10) + _run(["docker", "start", target]) + back_s = _wait_in_rotation(host) + assert not stats["errors"], f"{len(stats['errors'])} errors during replica stop: {stats['errors'][:10]}" + assert stats["ok"] >= 20, stats + assert stats["target_after_stop"] >= 1, "stopped replica served nothing during its drain window" + assert stop_s < STOP_TIMEOUT_S, stop_s + print(" stop under load: %d x 201, 0 errors, %d served by stopping replica; docker stop %.2fs; back in LB in %.1fs" + % (stats["ok"], stats["target_after_stop"], stop_s, back_s))