diff --git a/.env.example b/.env.example index d99d26a2..4acaae9b 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,50 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Copy to .env for your LOCAL synthetic test environment; never commit .env. Values below are the ones the evidence was produced with. +# Secrets are NOT here: they are files in ./secrets (see REPORT.md), referenced by *_FILE paths. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab -SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +GRACEFUL_TIMEOUT=8 API_REPLICAS=2 + +# Sessions / idempotency +SESSION_TTL_SECONDS=3600 +SESSION_STORE=valkey +SESSION_CACHE_TTL_SECONDS=30 +SESSION_REVOKED_TTL_SECONDS=60 +SESSION_SWEEP_SECONDS=60 +CACHE_TIMEOUT=0.2 IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 + +# Database +DATABASE_URL_FILE=/run/secrets/database_url +CACHE_URL_FILE=/run/secrets/cache_url +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT=3 +DB_CONNECT_TIMEOUT=3 +DB_STATEMENT_TIMEOUT_MS=5000 +DB_OP_TIMEOUT=6 +DB_READY_TIMEOUT=1.5 +READY_CHECK_SECONDS=1 +READY_FAILURES=2 + +# Request limits (REQUEST_TIMEOUT must exceed S3 timeouts + upload time) +MAX_INFLIGHT_REQUESTS=64 +MAX_BODY_BYTES=65536 +MAX_RSS_MB=0 +REQUEST_TIMEOUT=30 + +# Object storage and attachments +S3_ENDPOINT=http://minio:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_CONNECT_TIMEOUT=3 +S3_READ_TIMEOUT=6 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +UPLOAD_SPOOL_BYTES=1048576 +UPLOAD_CONCURRENCY=2 +RECONCILE_INTERVAL_SECONDS=300 +RECONCILE_GRACE_SECONDS=900 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a3553194 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \ + && rm -rf /var/lib/apt/lists/* + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-deps -r requirements.txt + +COPY app/ app/ +COPY migrations/ migrations/ +COPY scripts/migrate.py scripts/reconcile.py scripts/reconcile_experiment.py scripts/jpeg_check_experiment.py scripts/ + +USER 10001:10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + +STOPSIGNAL SIGTERM + +CMD ["sh", "-c", "exec uvicorn app.main:app --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..f154bab5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,146 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Алёна Курлова, БВТ2301 +- Целевая оценка; выбранный трек 5: **5, трек 2 «Согласованность двух хранилищ»** +- Commit/PR: PR [#49](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/49), ветка `solution/bvt2301-kurlova` +- Источники, библиотеки, помощь ИИ: FastAPI, psycopg, boto3, Pillow, MinIO; код лабы 3 перенесён и расширен. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора; результаты прогонов ниже получены на локальном стенде. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Desktop (Compose v2), Python 3.9+, вход `docker login` (Docker Hub). Проверено на macOS, arm64. + +```sh +cp .env.example .env # значения в примере совпадают с теми, на которых получены evidence/ +mkdir -p secrets +printf 'messenger' > secrets/postgres_password +printf 'postgresql://messenger:messenger@postgres:5432/messenger' > secrets/database_url +printf 'valkey://valkey:6379/0' > secrets/cache_url +printf 'messenger_root' > secrets/minio_root_user +python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/minio_root_password +printf 'messenger_app' > secrets/s3_access_key_id +python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/s3_secret_access_key +docker compose --env-file .env up -d --build # postgres, valkey, minio, minio-init, migrate, api-1, api-2, nginx +make test # ждёт готовности по /health/ready +``` + +Остановка с сохранением данных: `docker compose stop`. Намеренное удаление учебных данных: `docker compose down -v`. Каталог `secrets/` и `.env` в git не попадают. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +| Роль | Сервис | Версия | Health | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | nginx | 1.27.3-alpine | `/health/live`; health-скрипт опрашивает `/health/ready` реплик | — | `127.0.0.1:8080` | — | +| API ×2 | api-1, api-2 | собираются из исходников, python 3.12.8 | `/health/ready` | нет (stateless) | только внутренняя сеть | Docker secrets | +| БД | postgres | 17.7-alpine@sha256 | `pg_isready` | `pgdata` | внутренняя | secret | +| Кэш сессий | valkey | 8.1-alpine@sha256 | `valkey-cli ping` | нет | внутренняя | — | +| S3 | minio | `madlabcg/minio@sha256:ad134b2d…` | `mc ready local` | `miniodata` | внутренняя, **порты и консоль не опубликованы** (`MINIO_BROWSER=off`) | root-secret только у minio и minio-init | +| Bootstrap S3 | minio-init | тот же образ | — | — | — | root + app secrets | + +**Источник истины.** Метаданные — PostgreSQL (`attachments`, `message_attachments`), байты — приватный bucket. Object key назначает сервер (`attachments/`), `filename` хранится только как отображаемое имя. + +**Порядок записи и границы согласованности** (ответ на «что если PUT прошёл, а COMMIT нет»): +1. тело читается потоком в `SpooledTemporaryFile` (1 MiB в RAM, остальное во временный файл), считается SHA-256, лимит 10 MiB применяется на лету; MIME проверяется по magic bytes и Pillow, лимит пикселей проверяется по заголовку до декодирования; целостность проверяет [app/imagecheck.py](app/imagecheck.py): для PNG собственный потоковый сканер (CRC всех chunk'ов, поля IHDR, zlib-поток разжимается порциями по 1 MiB, filter-байт каждой строки ∈ 0..4, размер данных точно равен вычисленному по IHDR, Adam7 учитывается), для JPEG дочерний процесс `djpeg -strict` (libjpeg-turbo 2.1.5) декодирует весь поток в масштабе 1/8 и считает любое предупреждение libjpeg ошибкой, поэтому обрезанный поток отвергается и с дописанным EOI; одновременно идёт одна такая проверка на реплику, память процесса ограничена `-maxmemory`, число сканов `-maxscans 100`; +2. в БД вставляется строка `status='uploading'` (коммит); +3. PUT в S3; +4. `UPDATE … SET status='ready' WHERE status='uploading'` — только после этого клиент получает **201**. + +Любой сбой между шагами оставляет либо `uploading`-строку (с объектом или без), либо ничего. Клиентам такие строки не видны (404), прикрепить их к сообщению нельзя. Объект без строки получиться не может, потому что строка вставляется раньше. При ошибке API сам пытается компенсировать (удалить объект и строку); то, что не удалось, собирает reconciler. + +**Reconciler** ([app/reconcile.py](app/reconcile.py), CLI [scripts/reconcile.py](scripts/reconcile.py), фоновый цикл в каждом API раз в `RECONCILE_INTERVAL_SECONDS`=300 с, grace `RECONCILE_GRACE_SECONDS`=900 с). Работает под `pg_try_advisory_lock`, поэтому две реплики не дублируют работу. Правила: +- `uploading` старше grace **и** объект (если есть) старше grace по `LastModified` → удалить объект и строку; старая строка со свежим объектом не трогается (повторная проверка HEAD перед удалением); +- объект без строки и старше grace → удалить (перед удалением строка перепроверяется); +- `ready`-строка, подтверждённая (`updated_at`) раньше grace, без объекта (перепроверка HEAD) → `status='failed'`, `error_code='object_missing'`; строка **не удаляется**, даже если на неё ссылается сообщение; +- по умолчанию dry-run (`--apply` меняет данные), повторный запуск идемпотентен. + +**Что изменено в lab3-коде.** Сообщение принимает `attachment_ids` (0–10) и допускает пустой `text` при наличии вложений (миграция 0004 ослабляет CHECK). Хеш идемпотентности включает `attachment_ids`; повтор без ключа `attachment_ids` равен `[]`. **Критичные зависимости `/health/ready`: PostgreSQL и S3.** Недоступность любой из них даёт 503 в общем формате ошибки с `X-Instance-Id`; состояние S3 обновляется фоновой пробой раз в секунду (отдельный клиент с таймаутами 1 с / 1.5 с, чтобы зависший S3 не вешал проверку). При этом недоступный S3 **не** блокирует остальной API: текстовые сообщения и метаданные продолжают работать (запросы блокируются только при недоступной БД), а вложения отвечают 503. Чтобы 503 не выбивал реплики из upstream и клиенты не видели 502, nginx не использует `http_503` как признак сбоя для `/health/`, `/attachments/` и upload; если обе реплики не готовы, health-скрипт оставляет обе в ротации. **Лимиты тела и память LB.** nginx сам отклоняет слишком большие тела JSON-ошибкой 413 (с `X-Request-Id`): 10 MiB для загрузки вложений и 64 KiB для остальных запросов, то есть те же границы, что в API. Загрузка идёт потоком до API (`proxy_request_buffering off`), скачивание — без временных файлов (`proxy_max_temp_file_size 0`), поэтому тела файлов не попадают в память и page cache контейнера nginx; воркеров 2, а не `auto`, чтобы потребление не зависело от числа ядер хоста. API больше не отправляет `Connection: close` на отказах с недочитанным телом: это закрывало соединение раньше, чем nginx успевал прочитать ответ, и давало HTML 502; uvicorn дочитывает и отбрасывает остаток тела. Лимит памяти контейнера API поднят до 512 MiB, потому что проверка законного прогрессивного CMYK-JPEG на 40 Мпикс требует 307 MiB. S3-клиент: без повторов, connect 3 с, read 6 с; компенсация при сбое PUT выполняется в фоне. + +**Дополнительная валидация входных данных.** filename: 1..255 code points, не только пробелы, не `.`/`..`, без `/`, `\` и любых символов категорий Cc/Cs/Zl/Zp (в том числе DEL и C1). `Idempotency-Key` не обрезается (HTTP-парсер уже убирает крайние пробелы); порядок `attachment_ids` сохраняется в ответе, повторе и истории, переставленный список с тем же ключом — 409. + +**Права S3.** API использует отдельного пользователя `messenger_app` с политикой [infra/minio/app-policy.json](infra/minio/app-policy.json): Get/Put/DeleteObject только на `attachments/*` и ListBucket. Bucket создаёт и делает приватным [infra/minio/init.sh](infra/minio/init.sh); скрипт идемпотентен (`mb --ignore-existing`, повтор политики и пользователя безвреден), запускается при каждом `up`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `make test` | PASS (загрузка, привязка, скачивание вторым участником, 404 постороннему) | [evidence/smoke.txt](evidence/smoke.txt) | +| Байты в S3, метаданные в БД, 201 после сохранения | код `upload_attachment`; `student-tests/test_attachments.py` | sha256/размер совпадают, `Content-Length` скачивания равен оригиналу | [evidence/attachments_tests.txt](evidence/attachments_tests.txt) | +| Upload → остановка реплики → download через другую; restart с volumes | `student-tests/test_attachment_resilience.py` | загрузил api-1, скачал api-2, sha256 совпал; после рестарта postgres/valkey/minio/api/nginx метаданные и оригинал целы | [evidence/attachment_resilience.txt](evidence/attachment_resilience.txt) | +| Общие persistence / failover | `make test-persistence`, `make test-failover` (хуки `scripts/restart.sh`, `scripts/stop-one.sh`) | PASS; failover: 131 успешная запись, 0 ошибок, восстановление 0.44 с | [evidence/persistence.txt](evidence/persistence.txt), [evidence/failover.txt](evidence/failover.txt) | +| Лимиты 10 MiB / 40 Mpx, MIME по содержимому, опасные filename | `test_attachments.py` | 10 MiB+ → 413; PNG 8000×6000 → 413 до декодирования; подмена MIME и мусор → 415; `../`, `/`, `\`, NUL, `\n`, пустое, 256 символов → 400 | [evidence/attachments_tests.txt](evidence/attachments_tests.txt) | +| Устойчивость к некорректным входным данным | `student-tests/test_input_hardening.py`, `scripts/jpeg_check_experiment.py` (запуск: `docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py`) | усечённый PNG, filter-байт 5/0xFF, битый IDAT, 27-байтный JPEG, настоящий JPEG, обрезанный наполовину с дописанным `FF D9` и без него → 415 (PNG-дефекты также на 6 и 36 Mpx); валидные baseline, progressive, grayscale, CMYK, файл с данными после EOI и 40 Mpx принимаются; filename из пробелов, DEL, C1, `.` → 400; порядок вложений; `[]` ≡ отсутствует; 12 MiB → JSON 413 | [evidence/input_hardening.txt](evidence/input_hardening.txt), [evidence/jpeg_check.txt](evidence/jpeg_check.txt) | +| Чужое / неизвестное / чужого чата attachment_id, ошибки не 500 | `test_attachments.py` | 404 / 404 / 404; дубликаты, не-UUID, >10 → 400 | там же | +| Недоступность и зависание S3 | `student-tests/test_s3_outage.py` | остановка: upload/download/ready → 503 JSON с `X-Instance-Id`, метаданные и текст работают, без 502; `docker pause`: upload отвечает 503 за 7 с; восстановление за секунды | [evidence/s3_outage.txt](evidence/s3_outage.txt) | +| Прерывание загрузки | обрыв сокета на половине тела в `test_attachments.py` | сервер здоров, следующий upload проходит | там же | +| Компенсация / сверка, безопасный grace | трек 2, ниже | 18/18 проверок | [evidence/reconciler_experiment.txt](evidence/reconciler_experiment.txt) | +| Пик памяти при ≥10 одновременных загрузках | `student-tests/memory_experiment.py`, `student-tests/test_lb_stress.py` | см. ниже | [evidence/memory_experiment.txt](evidence/memory_experiment.txt), [evidence/lb_stress.txt](evidence/lb_stress.txt) | +| Нагрузка на LB: нет OOM и падений, отказы в JSON | `python3 student-tests/test_lb_stress.py --concurrency 10 --rounds 3 --oversize 100` и то же с 30 параллельными | загрузки 201, скачивания 200, 800 запросов-отказов дают правильный JSON без единого 502, событий `oom`/`die` контейнеров нет, пик nginx 28–34 MiB при лимите 64 | [evidence/lb_stress.txt](evidence/lb_stress.txt), [evidence/lb_stress_30.txt](evidence/lb_stress_30.txt) | +| Private bucket вне API, консоль не открыта | `student-tests/check_private_bucket.sh` | анонимные GET/LIST из сети compose → 403, порты 9000/9001 на хосте не опубликованы, `mc anonymous get` → private | [evidence/private_bucket.txt](evidence/private_bucket.txt) | +| Отдельные S3 credentials с ограниченными правами | `infra/minio/app-policy.json`, `init.sh` | API работает под `messenger_app`; root-secret API не получает | `compose.yaml` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Проверка hook'ов подтверждает перезапуск реальных процессов: `restart.sh` перезапускает postgres, valkey, minio, api-1, api-2, nginx без `down -v`; `stop-one.sh` останавливает только указанную реплику. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### 1. Память при 10 одновременных загрузках -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Гипотеза: основная память уходит на параллельное декодирование Pillow и S3 PUT, а не на хранение тел запросов; ограничение числа одновременных тяжёлых стадий снизит пик. + +Методика: [student-tests/memory_experiment.py](student-tests/memory_experiment.py), PNG 8.27 MiB (1700×1700, несжимаемый шум), 10 параллельных загрузок через LB, 3 раунда, замер `docker stats` каждого API (0.5 CPU; лимит контейнера был 256 MiB, в итоговой версии 512 MiB). Mac arm64, Docker Desktop. + +| Вариант | Простой, MiB | Пик api-1 / api-2, MiB | Динамика по раундам | 201 / всего | +| --- | --- | --- | --- | --- | +| До: без ограничения тяжёлых стадий | 88 | 182 / 155 | рост 144 → 166 → 182 | 30 / 30 | +| После: `UPLOAD_CONCURRENCY=2` на реплику | 87 | 110 / 116 | 110 → 109 → 110, роста нет | 30 / 30 | +| После + потоковый PNG-сканер (итоговая версия) | см. файл | 116 / 107 | роста нет | 30 / 30 | + +**Цифры таблицы занижены.** `docker stats` не учитывает page cache и даёт срезы раз в 1–2 с. Точный показатель лимита — `memory.peak` из cgroup: в этом же сценарии 154–163 MiB, `oom_kill=0`. Худший сценарий — стресс-тест с 10–30 параллельными прогрессивными CMYK-JPEG на 40 Мпикс: `memory.peak` API 444–491 MiB из 512 (вместе с page cache, который ядро может вытеснить), `oom_kill=0`, рестартов нет. Постоянная память процесса API после такой нагрузки около 133 MiB, один процесс проверки до 307 MiB ([evidence/jpeg_check.txt](evidence/jpeg_check.txt)). Запас в худшем случае небольшой. + +Сырые данные: [evidence/memory_experiment_before_semaphore.txt](evidence/memory_experiment_before_semaphore.txt), [evidence/memory_experiment.txt](evidence/memory_experiment.txt). Ограничения: одна конфигурация железа; один размер файла; медиана длительности загрузки выросла с 0.82 до 1.15 с (цена ограничения), в итоговой версии 0.63 с. Причину роста «до» (фрагментация аллокатора или реальная утечка) я не выяснял. + +### 2. Трек 2 — reconciler с внедрением ошибок + +Гипотеза: сверка по правилам выше находит и исправляет расхождения, не трогая свежие и используемые вложения. + +Методика: [scripts/reconcile_experiment.py](scripts/reconcile_experiment.py) внутри api-контейнера снимает базовую линию (в БД могут быть остатки прошлых тестов), создаёт строки/объекты, имитирующие сбои на обеих сторонах, и запускает dry-run, apply, apply после истечения grace, повторный apply. Счётчики сравниваются с базовой линией. `LastModified` объекта в MinIO задним числом не изменить, поэтому «старый объект» моделируется ожиданием grace. + +| Внедрённая ошибка | Ожидание | Наблюдение | +| --- | --- | --- | +| `uploading` старше grace, объект есть, но моложе grace (падение после PUT) | не трогать до истечения grace | сохранены; собраны после grace | +| `uploading` старше grace, но под его ключом свежий объект | не трогать | не тронуты; собраны после grace | +| `ready`, подтверждённая только что, объект ещё не виден | не помечать missing | не помечена | +| `uploading` старше grace, объекта нет (падение до PUT) | строка удалена | удалена | +| `uploading` свежее grace (загрузка идёт) | не трогать | не тронута; после истечения grace собрана | +| объект без строки внутри grace | не трогать | не тронут; удалён после grace | +| `ready`, объект потерян | пометить `failed/object_missing` | помечено | +| `ready`, привязано к сообщению, объект на месте | не трогать | не тронуто | +| `ready`, привязано к сообщению, объект потерян | строку не удалять, пометить | сохранена, `failed` | +| dry-run | ничего не менять, но показать счётчики | счётчики верны, состояние не изменилось | +| повторный apply | нулевые изменения | 0 / 0 / 0 | + +Итог: 18/18 проверок ([evidence/reconciler_experiment.txt](evidence/reconciler_experiment.txt)). **Стоимость обхода:** 1000 `ready`-вложений (около 1100 объектов вместе с остатками прошлых тестов) — 1.25 с в dry-run (в основном 1000 HEAD-запросов к S3; листинг идёт страницами по 500 с одним запросом к БД на страницу). Оценка масштабирования линейна, на больших объёмах не измерялась. + +### 3. Отказы + +| Воздействие | Ожидаем | Наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка обслужившей API-реплики | download через другую реплику, тот же файл | api-1 → api-2, sha256 совпал | без потерь | +| Остановка MinIO | 503 на вложениях и ready, текст работает | upload/download/ready → 503 JSON, сообщения читаются, 502 нет | возврат за секунды после старта, без потерь | +| Зависание MinIO (`docker pause`) | быстрый 503 без вечного ожидания | upload → 503 за 7 с, download → 503 | восстановление после `unpause` | +| Остановка PostgreSQL | 503 JSON с `X-Instance-Id` на ready и бизнес-эндпоинтах | так и есть | возврат после старта | +| Рестарт postgres, valkey, minio, API, nginx с volumes | метаданные и байты на месте | сохранены | без потерь | +| 10 и 30 одновременных загрузок 8.3 MiB через LB | ограниченная обработка без падения стенда | все 201, событий `oom`/`die` нет, пик nginx 28–34 MiB | потерь нет | +| Обрыв клиента на середине upload | нет 500 и мусора | сервер здоров; строка в БД ещё не создана | — | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Образ MinIO.** Официальные образы `minio/minio` на Docker Hub и `quay.io/minio/minio` в моей среде недоступны (pull denied / 401), поэтому использован сторонний `madlabcg/minio`, закреплённый по digest. Его происхождение и содержимое я не проверял; для защиты нужно заменить образ на проверенный (собрать из исходников MinIO или взять образ, указанный преподавателем). +- **Валидация изображений.** `djpeg -strict` считает ошибкой любое предупреждение libjpeg, поэтому будет отвергнут и JPEG с мелким дефектом, который обычные программы показывают (например, лишние байты перед маркером). Это сознательный выбор в пользу строгости. Подмена байтов внутри данных, не нарушающая структуру потока, может не обнаруживаться: такой файл корректно декодируется в другую картинку. PNG-сканер не проверяет содержимое ancillary chunk'ов, кроме CRC. +- **OOM балансировщика.** Само падение nginx по OOM на моей машине (10 ядер, Docker Desktop) не воспроизвелось, воспроизвелся пик у лимита: 67–80 MiB при 64. Устранена причина (тела файлов через nginx больше не спулятся на диск, число воркеров фиксировано); после правок пик 28–34 MiB. На конфигурации, где наблюдалось падение, правка отдельно не проверялась. +- **Потоковая загрузка через LB.** С `proxy_request_buffering off` медленный клиент занимает слот API на всё время передачи, но не дольше `REQUEST_TIMEOUT` (30 с) и в пределах `MAX_INFLIGHT_REQUESTS`. Ошибка 413, выданная самим nginx, содержит `X-Instance-Id: lb`, потому что запрос до реплики не доходил. +- Проверка размера по `Content-Length` невозможна для chunked-запросов: тогда лимит применяется по факту чтения потока. +- Проверка PNG ограничена по времени (`SCAN_SECONDS`=10 с); на очень медленной машине валидный файл с огромной степенью сжатия может получить 415. +- Вложения, загруженные, но так и не прикреплённые к сообщению, не считаются мусором и не удаляются; политики TTL для них нет. +- Reconciler при проверке `ready`-строк делает HEAD на каждую; для очень больших объёмов нужен инкрементальный режим по курсору или S3 Inventory. +- Нагрузочное тестирование с заданным arrival rate, p95/p99 не выполнялось; измерялась только память. +- Не реализованы: трек 1 (presigned upload), варианты изображений (lab5), TLS (локальный стенд с синтетическими данными). +- Секреты в `secrets/` сгенерированы локально и не коммитятся; учётные данные Postgres в примере выше — учебные. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..8c3f91f4 --- /dev/null +++ b/app/config.py @@ -0,0 +1,64 @@ +import os +import socket + + +def _int(name, default): + value = os.environ.get(name) + return int(value) if value not in (None, "") else default + + +def _float(name, default): + value = os.environ.get(name) + return float(value) if value not in (None, "") else default + + +def _secret(name, default=""): + path = os.environ.get(name + "_FILE") + if path: + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + return os.environ.get(name, default) + + +DATABASE_URL = _secret("DATABASE_URL") +CACHE_URL = _secret("CACHE_URL") +SESSION_STORE = os.environ.get("SESSION_STORE", "postgres") +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30) +SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60) +DB_POOL_MIN = _int("DB_POOL_MIN", 1) +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0) +DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3) +DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000) +DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0) +REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0) +DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5) +READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0) +READY_FAILURES = _int("READY_FAILURES", 2) +CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2) +MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64) +MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536) +MAX_RSS_MB = _int("MAX_RSS_MB", 0) +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 +SESSION_REVOKED_TTL_SECONDS = max( + _int("SESSION_REVOKED_TTL_SECONDS", 60), + SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5, +) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400) + +S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000") +S3_REGION = os.environ.get("S3_REGION", "us-east-1") +S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments") +S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID") +S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY") +MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760) +MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000) +S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0) +S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0) +UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576) +RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300) +RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900) +UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2) diff --git a/app/cursor.py b/app/cursor.py new file mode 100644 index 00000000..113aabc8 --- /dev/null +++ b/app/cursor.py @@ -0,0 +1,43 @@ +import base64 +import json +import re +import uuid +from datetime import datetime, timezone + +TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z") +TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class BadCursor(Exception): + pass + + +def fmt(ts: datetime) -> str: + return ts.astimezone(timezone.utc).strftime(TS_FORMAT) + + +def encode_cursor(scope: str, created_at: str, item_id: str) -> str: + raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def decode_cursor(scope: str, cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)) + data = json.loads(raw) + except Exception: + raise BadCursor() from None + if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope: + raise BadCursor() + ts, item_id = data["t"], data["i"] + if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str): + raise BadCursor() + try: + parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc) + if str(uuid.UUID(item_id)) != item_id: + raise BadCursor() + except ValueError: + raise BadCursor() from None + if encode_cursor(scope, ts, item_id) != cursor: + raise BadCursor() + return parsed, item_id diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..0bd63089 --- /dev/null +++ b/app/db.py @@ -0,0 +1,99 @@ +import asyncio +from contextlib import asynccontextmanager + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from app import config +from app.errors import unavailable + +BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout) + + +async def discard(conn) -> None: + try: + await conn.close() + except Exception: + pass + + +class Guarded: + def __init__(self, conn, timeout: float): + self.conn = conn + self.timeout = timeout + + async def execute(self, *args, **kwargs): + try: + return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout) + except asyncio.TimeoutError: + await discard(self.conn) + raise unavailable("Хранилище не ответило за отведённое время") from None + + def transaction(self): + return self.conn.transaction() + + +class Database: + def __init__(self) -> None: + options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % ( + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS * 2, + ) + self.pool = AsyncConnectionPool( + conninfo=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + timeout=config.DB_POOL_TIMEOUT, + max_lifetime=600.0, + open=False, + kwargs={ + "autocommit": True, + "row_factory": dict_row, + "connect_timeout": config.DB_CONNECT_TIMEOUT, + "options": options, + }, + ) + + async def open(self) -> None: + await self.pool.open(wait=False) + + async def close(self) -> None: + await self.pool.close() + + @asynccontextmanager + async def connection(self): + try: + async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn: + yield Guarded(conn, config.DB_OP_TIMEOUT) + except BROKEN: + raise unavailable("Хранилище недоступно") from None + + @asynccontextmanager + async def transaction(self): + async with self.connection() as conn: + async with conn.transaction(): + yield conn + + async def in_transaction(self, action): + async def run(): + async with self.transaction() as conn: + return await action(conn) + + try: + return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT) + except asyncio.TimeoutError: + raise unavailable("Хранилище не ответило за отведённое время") from None + + async def ready(self) -> bool: + try: + async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn: + try: + await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT) + except asyncio.TimeoutError: + await discard(conn) + return False + return True + except Exception: + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..8110d273 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,37 @@ +from typing import Optional + +from fastapi.responses import JSONResponse + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None): + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def bad(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(scheme: str = "Bearer") -> ApiError: + challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE + message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия" + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Объект не найден") + + +def unavailable(message: str) -> ApiError: + return ApiError(503, "unavailable", message, {"Retry-After": "1"}) + + +def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse: + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})}, + ) diff --git a/app/imagecheck.py b/app/imagecheck.py new file mode 100644 index 00000000..fe698c55 --- /dev/null +++ b/app/imagecheck.py @@ -0,0 +1,213 @@ +"""Integrity checks for uploaded PNG/JPEG with bounded memory. + +PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in +0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy +stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is +only a warning. Both finish before anything is stored. +""" +import logging +import os +import struct +import subprocess +import threading +import time +import zlib + +from PIL import Image + +from app import config +from app.errors import ApiError, unavailable + +log = logging.getLogger("messenger.imagecheck") + +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)} +CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4} +ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2)) +Image.MAX_IMAGE_PIXELS = None +FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"} +FULL_DECODE_MAX_PIXELS = 4000000 +SCAN_SECONDS = 10.0 +OUT_STEP = 1 << 20 +# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit. +DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null") +jpeg_slot = threading.Lock() + + +class Invalid(Exception): + pass + + +def invalid() -> ApiError: + return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG") + + +class RowPlan: + def __init__(self, segments): + self.segments = [s for s in segments if s[0] > 1 and s[1] > 0] + self.seg = 0 + self.done = 0 + self.pos = 0 + + def feed(self, data: bytes) -> None: + i, n = 0, len(data) + while i < n: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + if self.seg >= len(self.segments): + raise Invalid("data after the last row") + length, rows = self.segments[self.seg] + if self.pos > 0: + take = min(length - self.pos, n - i) + i += take + self.pos += take + if self.pos == length: + self.pos = 0 + self.done += 1 + continue + full = min((n - i) // length, rows - self.done) + if full > 0: + if max(data[i:i + full * length:length]) > 4: + raise Invalid("bad filter type") + i += full * length + self.done += full + else: + if data[i] > 4: + raise Invalid("bad filter type") + self.pos = n - i + i = n + + def complete(self) -> bool: + while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]: + self.seg += 1 + self.done = 0 + return self.seg >= len(self.segments) and self.pos == 0 + + +def png_plan(width, height, depth, color, interlace): + bits = CHANNELS[color] * depth + if not interlace: + return RowPlan([(1 + (width * bits + 7) // 8, height)]) + segments = [] + for x0, y0, dx, dy in ADAM7: + w = (width - x0 + dx - 1) // dx if width > x0 else 0 + h = (height - y0 + dy - 1) // dy if height > y0 else 0 + if w and h: + segments.append((1 + (w * bits + 7) // 8, h)) + return RowPlan(segments) + + +def scan_png(fileobj) -> None: + deadline = time.monotonic() + SCAN_SECONDS + fileobj.seek(0) + if fileobj.read(8) != PNG_SIGNATURE: + raise Invalid("signature") + plan = decomp = None + palette = seen_idat = idat_closed = False + color = None + first = True + while True: + if time.monotonic() > deadline: + raise Invalid("scan too slow") + head = fileobj.read(8) + if len(head) < 8: + raise Invalid("truncated before IEND") + length, kind = struct.unpack(">I4s", head) + if length > 0x7FFFFFFF: + raise Invalid("chunk length") + if first and kind != b"IHDR": + raise Invalid("IHDR must be first") + crc = zlib.crc32(kind) + if kind == b"IHDR": + body = fileobj.read(length) + if not first or length != 13 or len(body) != 13: + raise Invalid("IHDR") + width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body) + if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1): + raise Invalid("IHDR fields") + if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]: + raise Invalid("color type / depth") + plan = png_plan(width, height, depth, color, interlace) + decomp = zlib.decompressobj() + crc = zlib.crc32(body, crc) + else: + remaining = length + while remaining: + buf = fileobj.read(min(remaining, 65536)) + if not buf: + raise Invalid("truncated chunk") + remaining -= len(buf) + crc = zlib.crc32(buf, crc) + if kind == b"IDAT": + if idat_closed or (color == 3 and not palette): + raise Invalid("IDAT order") + seen_idat = True + out = decomp.decompress(buf, OUT_STEP) + while True: + plan.feed(out) + if not decomp.unconsumed_tail: + break + if time.monotonic() > deadline: + raise Invalid("scan too slow") + out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP) + if kind == b"PLTE": + palette = True + if color in (0, 4) or length % 3 or not 3 <= length <= 768: + raise Invalid("PLTE") + elif kind != b"IDAT" and seen_idat: + idat_closed = True + tail = fileobj.read(4) + if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF: + raise Invalid("crc") + first = False + if kind == b"IEND": + if length or not seen_idat: + raise Invalid("IEND") + break + if not decomp.eof or decomp.unused_data or not plan.complete(): + raise Invalid("pixel data does not match IHDR") + + +def check_jpeg(fileobj) -> None: + fileobj.flush() + descriptor = fileobj.fileno() + os.lseek(descriptor, 0, os.SEEK_SET) + try: + with jpeg_slot: + done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=SCAN_SECONDS, check=False) + except subprocess.TimeoutExpired: + log.error("jpeg_check_timeout") + raise unavailable("Проверка изображения не успела завершиться") from None + # A signal means the checker died (e.g. OOM killer), which says nothing about the file. + if done.returncode < 0: + log.error("jpeg_check_killed signal=%d", -done.returncode) + raise unavailable("Проверка изображения прервана") + if done.returncode != 0: + raise invalid() + + +def inspect_image(fileobj, size: int, content_type: str) -> None: + """Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type).""" + try: + fileobj.seek(0) + img = Image.open(fileobj) + width, height = img.size + if img.format != FORMATS[content_type]: + raise invalid() + if width * height > config.MAX_IMAGE_PIXELS: + raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей") + if content_type == "image/png": + scan_png(fileobj) + if width * height <= FULL_DECODE_MAX_PIXELS: + fileobj.seek(0) + Image.open(fileobj).load() + else: + check_jpeg(fileobj) + except ApiError: + raise + except Exception: + raise invalid() from None + finally: + fileobj.seek(0) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..12a8d3d5 --- /dev/null +++ b/app/main.py @@ -0,0 +1,640 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import logging +import os +import re +import resource +import signal +import tempfile +import unicodedata +import sys +import uuid +from contextlib import asynccontextmanager +from typing import List, Optional, Tuple + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from starlette.concurrency import iterate_in_threadpool, run_in_threadpool +from starlette.responses import StreamingResponse + +from app import config, passwords, reconcile, repo +from app.cursor import BadCursor, decode_cursor, encode_cursor +from app.db import Database +from app.imagecheck import inspect_image +from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable +from app.s3 import S3Storage +from app.sessions import SessionStore + +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger") + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"}) +EXEMPT_PATHS = ("/health/live", "/health/ready") +PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096 +ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"} +PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +JPEG_MAGIC = b"\xff\xd8\xff" + +upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY) +background = set() +db = Database() +sessions = SessionStore(db) +s3 = S3Storage() +state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}} + + +def rss_mb() -> float: + try: + with open("/proc/self/statm") as handle: + return int(handle.read().split()[1]) * PAGE_SIZE / 1048576 + except OSError: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak / 1048576 if sys.platform == "darwin" else peak / 1024 + + +def count_rejection(code: str) -> None: + state["rejected"][code] = state["rejected"].get(code, 0) + 1 + + +async def sweeper() -> None: + while True: + await asyncio.sleep(config.SESSION_SWEEP_SECONDS) + try: + removed = await sessions.sweep() + if removed: + log.info("sessions_expired removed=%d", removed) + except Exception: + log.warning("session sweep skipped: storage unavailable") + try: + async with db.connection() as conn: + removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS) + if removed: + log.info("idempotency_cleanup removed=%d", removed) + except Exception: + log.warning("idempotency cleanup skipped") + + +async def reconciler_loop() -> None: + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + while True: + try: + result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False) + if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))): + log.info("reconcile %s", result) + except Exception: + log.warning("reconcile skipped: dependency unavailable") + await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS) + + +async def readiness_watch() -> None: + failures = 0 + while True: + if await db.ready(): + failures = 0 + if not state["ready"]: + state["ready"] = True + log.info("ready again: database reachable") + else: + failures += 1 + if failures >= config.READY_FAILURES and state["ready"]: + state["ready"] = False + log.warning("not ready: database unreachable, business requests get 503") + s3_ok = await run_in_threadpool(s3.probe) + if s3_ok != state["s3_ok"]: + state["s3_ok"] = s3_ok + log.info("object storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503") + await asyncio.sleep(config.READY_CHECK_SECONDS) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + previous = {} + + def on_signal(signum, frame): + if not state["draining"]: + state["draining"] = True + log.info("signal %s: draining, new requests get 503", signal.Signals(signum).name) + handler = previous.get(signum) + if callable(handler): + handler(signum, frame) + + for sig in (signal.SIGTERM, signal.SIGINT): + previous[sig] = signal.getsignal(sig) + signal.signal(sig, on_signal) + await db.open() + await sessions.open() + await run_in_threadpool(s3.open) + state["s3_ok"] = await run_in_threadpool(s3.probe) + tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop())] + log.info( + "started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s", + config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS, + config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES, + config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET, + ) + yield + for task in tasks: + task.cancel() + await sessions.close() + await db.close() + log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"]) + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +@app.middleware("http") +async def guard(request: Request, call_next): + rid = uuid.uuid4().hex + request.state.rid = rid + exempt = request.url.path in EXEMPT_PATHS + if state["draining"] and request.url.path != "/health/live": + count_rejection("draining") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"}) + if not exempt and not state["ready"]: + count_rejection("not_ready") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"}) + if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS: + count_rejection("inflight") + return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"}) + state["inflight"] += 1 + try: + response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT) + except asyncio.TimeoutError: + count_rejection("request_timeout") + log.warning("request timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path) + response = error_body( + rid, config.INSTANCE_ID, 503, "unavailable", + "Запрос не удалось обработать за отведённое время", {"Retry-After": "1"}, + ) + except Exception: + log.exception("unhandled error request_id=%s", rid) + response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера") + finally: + state["inflight"] -= 1 + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + if exc.status in (413, 429, 503): + count_rejection(exc.code) + return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers) + + +def parse_uuid(value: object) -> Optional[str]: + if not isinstance(value, str): + return None + try: + return str(uuid.UUID(value)) + except ValueError: + return None + + +def non_blank(value: object, max_len: int) -> bool: + if not isinstance(value, str) or not 1 <= len(value) <= max_len: + return False + return value.translate(ZERO_WIDTH).strip() != "" + + +async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict: + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое") + declared = request.headers.get("content-length") + if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES: + raise too_large + raw = bytearray() + async for chunk in request.stream(): + raw += chunk + if len(raw) > config.MAX_BODY_BYTES: + raise too_large + try: + body = json.loads(bytes(raw).decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad("Некорректный JSON") from None + if not isinstance(body, dict): + raise bad("Ожидается JSON-объект") + allowed = set(required) | set(optional) + if set(body) - allowed: + raise bad("Лишние поля в запросе") + if set(required) - set(body): + raise bad("Отсутствуют обязательные поля") + return body + + +def ensure_capacity() -> None: + if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB: + raise unavailable("Достигнут лимит памяти процесса, запись недоступна") + + +def bearer_token(request: Request) -> str: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token: + raise unauthorized() + return token + + +async def actor(request: Request) -> dict: + user = await sessions.resolve(bearer_token(request)) + if user is None: + raise unauthorized() + return user + + +def basic_credentials(request: Request) -> Tuple[str, str]: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized("Basic") + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized("Basic") from None + username, separator, password = decoded.partition(":") + if not separator: + raise unauthorized("Basic") + return username, password + + +async def member_chat(user: dict, chat_id: str) -> dict: + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + async with db.connection() as conn: + chat = await repo.chat_for_member(conn, cid, user["id"]) + if chat is None: + raise not_found() + return chat + + +def page_params(request: Request) -> Tuple[int, Optional[str]]: + query = request.query_params + limit = 50 + if "limit" in query: + if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100: + raise bad("limit должен быть целым 1..100") + limit = int(query["limit"]) + cursor = query.get("cursor") + if cursor is not None and cursor == "": + raise bad("Некорректный cursor") + return limit, cursor + + +def keyset(scope: str, cursor: Optional[str]): + if cursor is None: + return None + try: + return decode_cursor(scope, cursor) + except BadCursor: + raise bad("Некорректный cursor") from None + + +def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse: + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None + return JSONResponse({"items": items, "next_cursor": next_cursor}) + + +def detect_content_type(head: bytes) -> Optional[str]: + if head[:8] == PNG_MAGIC: + return "image/png" + if head[:3] == JPEG_MAGIC: + return "image/jpeg" + return None + + +def validate_filename(filename: str) -> None: + if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."): + raise bad("filename: 1..255 символов, не только пробелы") + if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename): + raise bad("filename содержит недопустимые символы") + + +@app.get("/health/live") +async def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def ready(): + if not await db.ready(): + raise unavailable("База данных недоступна") + if not state["s3_ok"]: + raise unavailable("Хранилище файлов недоступно") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + body = await read_json(request, ("username", "display_name", "password")) + if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]): + raise bad("username должен соответствовать [a-z0-9_]{3,32}") + if not non_blank(body["display_name"], 100): + raise bad("display_name: 1..100 символов, не только пробелы") + password = body["password"] + if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX)) + ensure_capacity() + password_hash = await run_in_threadpool(passwords.hash_password, password) + async with db.connection() as conn: + user = await repo.create_user(conn, body["username"], body["display_name"], password_hash) + if user is None: + raise ApiError(409, "conflict", "username уже занят") + log.info("user_created id=%s", user["id"]) + return JSONResponse(user, status_code=201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + if await request.body(): + raise bad("POST /auth/sessions не принимает тело запроса") + username, password = basic_credentials(request) + row = None + if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + async with db.connection() as conn: + row = await repo.credentials(conn, username) + stored = row["password_hash"] if row else passwords.DUMMY_HASH + matches = await run_in_threadpool(passwords.verify_password, stored, password) + if row is None or not matches: + log.info("login_failed username_known=%s", row is not None) + raise unauthorized("Basic") + user = repo.public_user(row) + token, expires_at = await sessions.create(user) + log.info("session_created user=%s", user["id"]) + return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request): + token = bearer_token(request) + user = await sessions.resolve(token) + if user is None: + raise unauthorized() + await sessions.revoke(token) + log.info("session_revoked user=%s", user["id"]) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + return await actor(request) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await actor(request) + body = await read_json(request, ("title", "member_ids")) + if not non_blank(body["title"], 100): + raise bad("title: 1..100 символов, не только пробелы") + raw_ids = body["member_ids"] + if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100: + raise bad("member_ids: массив из 1..100 UUID") + ids: List[str] = [] + for value in raw_ids: + parsed = parse_uuid(value) + if parsed is None: + raise bad("member_ids содержит некорректный UUID") + ids.append(parsed) + if len(set(ids)) != len(ids): + raise bad("member_ids содержит дубликаты") + if user["id"] not in ids: + raise bad("Создатель должен входить в member_ids") + ensure_capacity() + + async def insert(conn): + return await repo.create_chat(conn, body["title"], ids) + + try: + chat = await db.in_transaction(insert) + except repo.UnknownMember: + raise bad("member_ids содержит неизвестного пользователя") from None + log.info("chat_created id=%s members=%d", chat["id"], len(ids)) + return JSONResponse(chat, status_code=201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await actor(request) + limit, cursor = page_params(request) + scope = "chats:" + user["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_chats(conn, user["id"], limit, after) + return page(scope, items, has_more) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + return await member_chat(await actor(request), chat_id) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + user = await actor(request) + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + + idem_key = request.headers.get("idempotency-key", "") + if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key): + raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}") + + body = await read_json(request, ("text",), ("attachment_ids",)) + + text = body["text"] + attachment_ids_raw = body.get("attachment_ids", []) + + if not isinstance(attachment_ids_raw, list): + raise bad("attachment_ids должен быть массивом UUID") + if len(attachment_ids_raw) > 10: + raise bad("attachment_ids: максимум 10 вложений") + + attachment_ids: List[str] = [] + for value in attachment_ids_raw: + parsed = parse_uuid(value) + if parsed is None: + raise bad("attachment_ids содержит некорректный UUID") + attachment_ids.append(parsed) + + if len(set(attachment_ids)) != len(attachment_ids): + raise bad("attachment_ids содержит дубликаты") + + has_text = isinstance(text, str) and non_blank(text, 4000) + has_attachments = len(attachment_ids) > 0 + + if not has_text and not has_attachments: + raise bad("text: 1..4000 символов, не только пробелы") + + if isinstance(text, str) and len(text) > 4000: + raise bad("text: 1..4000 символов, не только пробелы") + + if not isinstance(text, str): + raise bad("text должен быть строкой") + + ensure_capacity() + + normalized = {"text": text} + if attachment_ids: + normalized["attachment_ids"] = attachment_ids + body_hash = hashlib.sha256( + json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode() + ).hexdigest() + + async def do(conn): + return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash) + + status, message = await db.in_transaction(do) + + if status == "conflict": + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса") + if status == "not_member": + raise not_found() + if status == "attachment_not_found": + raise not_found() + if status == "attachment_failed": + raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить") + return JSONResponse(message, status_code=201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + chat = await member_chat(await actor(request), chat_id) + limit, cursor = page_params(request) + scope = "messages:" + chat["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_messages(conn, chat["id"], limit, after) + return page(scope, items, has_more) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload_attachment(chat_id: str, request: Request): + user = await actor(request) + chat = await member_chat(user, chat_id) + + filename = request.query_params.get("filename", "") + validate_filename(filename) + + declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower() + if declared_ct not in ALLOWED_CONTENT_TYPES: + raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg") + + declared_length = request.headers.get("content-length") + too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB") + if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES: + raise too_large + ensure_capacity() + + sha = hashlib.sha256() + size = 0 + with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool: + head = b"" + async for chunk in request.stream(): + size += len(chunk) + if size > config.MAX_UPLOAD_BYTES: + raise too_large + if len(head) < 8: + head += chunk[: 8 - len(head)] + if len(head) == 8 and detect_content_type(head) is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + sha.update(chunk) + spool.write(chunk) + if size == 0: + raise bad("Пустой файл") + actual_ct = detect_content_type(head) + if actual_ct is None: + raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG") + if actual_ct != declared_ct: + raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла") + async with upload_slots: + await run_in_threadpool(inspect_image, spool, size, actual_ct) + + attachment_id = str(uuid.uuid4()) + object_key = "attachments/" + attachment_id + async with db.connection() as conn: + await repo.begin_attachment( + conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(), object_key, + ) + try: + async with upload_slots: + await run_in_threadpool(s3.put, object_key, spool, size, actual_ct) + except BaseException: + defer(compensate(attachment_id, object_key)) + raise + + try: + async with db.connection() as conn: + att = await repo.finish_attachment(conn, attachment_id) + except BaseException: + await compensate(attachment_id, object_key) + raise + if att is None: + raise unavailable("Загрузка отменена сверкой, повторите запрос") + log.info("attachment_uploaded id=%s chat=%s size=%d", att["id"], chat["id"], size) + return JSONResponse(att, status_code=201) + + +def defer(coro) -> None: + task = asyncio.ensure_future(coro) + background.add(task) + task.add_done_callback(background.discard) + + +async def compensate(attachment_id: str, object_key: str) -> None: + """Best effort; whatever is left behind is collected by the reconciler after the grace period.""" + try: + async with db.connection() as conn: + await repo.abort_attachment(conn, attachment_id) + await run_in_threadpool(s3.delete, object_key) + except BaseException: + log.warning("compensation incomplete attachment=%s, left for reconciler", attachment_id) + + +async def accessible_attachment(request: Request, attachment_id: str) -> dict: + user = await actor(request) + aid = parse_uuid(attachment_id) + if aid is None: + raise bad("attachment_id должен быть UUID") + async with db.connection() as conn: + row = await repo.get_attachment(conn, aid) + if row is None: + raise not_found() + chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"]) + if chat is None: + raise not_found() + return row + + +@app.get("/api/v1/attachments/{attachment_id}") +async def get_attachment(attachment_id: str, request: Request): + return repo._attachment(await accessible_attachment(request, attachment_id)) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def download_attachment(attachment_id: str, request: Request): + row = await accessible_attachment(request, attachment_id) + variant = request.query_params.get("variant", "original") + if variant != "original": + raise not_found() + if row["status"] != "ready": + raise ApiError(409, "conflict", "Вложение недоступно для скачивания") + opened = await run_in_threadpool(s3.open_stream, row["object_key"]) + if opened is None: + log.error("object_missing attachment=%s", row["id"]) + raise unavailable("Оригинал временно недоступен") + chunks, length = opened + return StreamingResponse( + iterate_in_threadpool(chunks), media_type=row["content_type"], headers={"Content-Length": str(length)}, + ) + + +@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def fallback(request: Request): + raise not_found() diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..b509e4a7 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,16 @@ +from argon2 import PasswordHasher +from argon2.exceptions import Argon2Error, VerifyMismatchError + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash("there-is-no-such-user-password") + + +def hash_password(password: str) -> str: + return HASHER.hash(password) + + +def verify_password(stored: str, password: str) -> bool: + try: + return HASHER.verify(stored, password) + except (VerifyMismatchError, Argon2Error, ValueError): + return False diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..f5313978 --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,115 @@ +"""DB <-> S3 reconciliation for attachments. + +Invariant: a row is inserted as 'uploading' BEFORE the object is written, and becomes 'ready' only AFTER the +PUT succeeded. So every object has a row, and a client only ever sees 'ready' rows. + +Cases (all gated by a grace period so in-flight uploads are never touched): + stale_uploading row 'uploading' older than grace AND its object (if any) older than grace -> delete both; + a fresh object under an old row is left alone (re-checked via LastModified) + orphan_object object without any row, older than grace -> delete object + missing_object 'ready' row confirmed longer than grace ago whose object is gone (re-checked by HEAD) -> mark 'failed'/object_missing, never delete the row +Rows referenced by messages are never deleted: only 'uploading' rows can be removed, and those cannot be linked. +""" +import logging +import time +from datetime import datetime, timedelta, timezone + +from starlette.concurrency import run_in_threadpool + +log = logging.getLogger("messenger.reconcile") + +LOCK_ID = 2026_0104 +PREFIX = "attachments/" +BATCH = 500 + + +async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict: + started = time.monotonic() + result = { + "dry_run": dry_run, "grace_seconds": grace_seconds, + "stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0, + "deleted_objects": 0, "removed_rows": 0, "marked_failed": 0, + "objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0, + } + async with db.connection() as conn: + cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,)) + if not (await cur.fetchone())["got"]: + return {"skipped": "another reconciler is running"} + try: + cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds) + await _stale_uploading(conn, s3, cutoff, dry_run, result) + await _orphan_objects(conn, s3, cutoff, dry_run, result) + await _missing_objects(conn, s3, cutoff, dry_run, result) + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + result["seconds"] = round(time.monotonic() - started, 3) + return result + + +async def _stale_uploading(conn, s3, cutoff, dry_run, result): + cur = await conn.execute( + "SELECT id, object_key FROM attachments WHERE status = 'uploading' AND created_at < %s", (cutoff,), + ) + for row in await cur.fetchall(): + modified = await run_in_threadpool(s3.head, row["object_key"]) + if modified is not None and modified >= cutoff: + result["protected_fresh_objects"] += 1 + continue + result["stale_uploading"] += 1 + if dry_run: + continue + if modified is not None: + await run_in_threadpool(s3.delete, row["object_key"]) + result["deleted_objects"] += 1 + done = await conn.execute( + "DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff), + ) + result["removed_rows"] += done.rowcount + + +async def _orphan_objects(conn, s3, cutoff, dry_run, result): + pages = await run_in_threadpool(lambda: iter(s3.list_pages(PREFIX))) + while True: + page = await run_in_threadpool(next, pages, None) + if page is None: + break + result["objects_scanned"] += len(page) + keys = [o["Key"] for o in page] + if not keys: + continue + cur = await conn.execute("SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,)) + known = {row["object_key"] for row in await cur.fetchall()} + for obj in page: + if obj["Key"] in known or obj["LastModified"] >= cutoff: + continue + result["orphan_objects"] += 1 + if not dry_run: + again = await conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (obj["Key"],)) + if await again.fetchone() is None: + await run_in_threadpool(s3.delete, obj["Key"]) + result["deleted_objects"] += 1 + + +async def _missing_objects(conn, s3, cutoff, dry_run, result): + last = None + while True: + cur = await conn.execute( + "SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s" + " AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s", + (cutoff, last, last, BATCH), + ) + rows = await cur.fetchall() + if not rows: + return + for row in rows: + result["rows_scanned"] += 1 + if await run_in_threadpool(s3.exists, row["object_key"]): + continue + result["missing_objects"] += 1 + if not dry_run: + done = await conn.execute( + "UPDATE attachments SET status = 'failed', error_code = 'object_missing'," + " updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],), + ) + result["marked_failed"] += done.rowcount + last = str(rows[-1]["id"]) diff --git a/app/repo.py b/app/repo.py new file mode 100644 index 00000000..299eb3e2 --- /dev/null +++ b/app/repo.py @@ -0,0 +1,293 @@ +import hashlib +import json +from typing import List, Optional, Tuple + +import psycopg + +from app.cursor import fmt + +MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" +CHAT_QUERY = ( + "SELECT c.id, c.title, c.created_at," + " ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids" +) + + +class UnknownMember(Exception): + pass + + +def decoded(value) -> str: + return bytes(value).decode("utf-8") + + +def public_user(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": decoded(row["display_name"]), + "created_at": fmt(row["created_at"]), + } + + +def _chat(row) -> dict: + return { + "id": str(row["id"]), + "title": decoded(row["title"]), + "member_ids": [str(value) for value in row["member_ids"]], + "created_at": fmt(row["created_at"]), + } + + +def _message(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": decoded(row["text"]), + "created_at": fmt(row["created_at"]), + } + + +def _message_with_attachments(row, attachment_ids: List[str]) -> dict: + msg = _message(row) + msg["attachment_ids"] = attachment_ids + return msg + + +def _attachment(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": row["error_code"], + "created_at": fmt(row["created_at"]), + "updated_at": fmt(row["updated_at"]), + } + + +async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]: + try: + cur = await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at)" + " VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())" + " RETURNING id, username, display_name, created_at", + (username, display_name.encode("utf-8"), password_hash), + ) + except psycopg.errors.UniqueViolation: + return None + return public_user(await cur.fetchone()) + + +async def credentials(conn, username: str): + cur = await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,), + ) + return await cur.fetchone() + + +async def create_chat(conn, title: str, member_ids: List[str]) -> dict: + cur = await conn.execute( + "INSERT INTO chats (id, title, created_at)" + " VALUES (gen_random_uuid(), %s, clock_timestamp())" + " RETURNING id, title, created_at", + (title.encode("utf-8"),), + ) + chat = await cur.fetchone() + cur = await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])", + (chat["id"], member_ids), + ) + if cur.rowcount != len(member_ids): + raise UnknownMember() + chat["member_ids"] = list(member_ids) + return _chat(chat) + + +async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]: + cur = await conn.execute( + CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s", + (user_id, chat_id), + ) + row = await cur.fetchone() + return _chat(row) if row else None + + +async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + CHAT_QUERY + + " FROM chats c JOIN chat_members m ON m.chat_id = c.id" + " WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))" + " ORDER BY c.created_at, c.id LIMIT %s", + (user_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + return [_chat(row) for row in rows[:limit]], len(rows) > limit + + +async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]: + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id), + ) + row = await cur.fetchone() + return _message(row) if row else None + + +async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]: + cur = await conn.execute( + "SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position", + (message_id,), + ) + rows = await cur.fetchall() + return [str(row["attachment_id"]) for row in rows] + + +async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None: + for pos, aid in enumerate(attachment_ids): + await conn.execute( + "INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)", + (message_id, aid, pos), + ) + + +async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + "SELECT " + MESSAGE_COLUMNS + " FROM messages" + " WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))" + " ORDER BY created_at, id LIMIT %s", + (chat_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + messages = [] + for row in rows[:limit]: + aids = await _fetch_message_attachment_ids(conn, row["id"]) + messages.append(_message_with_attachments(row, aids)) + return messages, len(rows) > limit + + +def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int: + raw = "%s:%s:%s" % (user_id, chat_id, key) + return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16) + + +async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str, + attachment_ids: List[str], idem_key: str, body_hash: str): + lock_id = _idem_lock_id(sender_id, chat_id, idem_key) + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,)) + + cur = await conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ) + if await cur.fetchone() is None: + return "not_member", None + + cur = await conn.execute( + "SELECT request_body_hash, response_json FROM idempotency_keys" + " WHERE user_id = %s AND chat_id = %s AND key = %s", + (sender_id, chat_id, idem_key), + ) + existing = await cur.fetchone() + + if existing: + if existing["request_body_hash"] != body_hash: + return "conflict", None + return "replay", json.loads(existing["response_json"]) + + if attachment_ids: + cur = await conn.execute( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'", + (attachment_ids,), + ) + att_rows = await cur.fetchall() + found_ids = {str(row["id"]) for row in att_rows} + for aid in attachment_ids: + if aid not in found_ids: + return "attachment_not_found", None + for row in att_rows: + if str(row["chat_id"]) != chat_id: + return "attachment_not_found", None + if row["status"] == "failed": + return "attachment_failed", None + + text_bytes = text.encode("utf-8") if text else b"" + + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text_bytes, chat_id, sender_id), + ) + row = await cur.fetchone() + if row is None: + return "not_member", None + + if attachment_ids: + await _link_attachments(conn, row["id"], chat_id, attachment_ids) + + message = _message_with_attachments(row, list(attachment_ids)) + + await conn.execute( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)" + " VALUES (%s, %s, %s, %s, %s, clock_timestamp())", + (sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)), + ) + + return "created", message + + +async def cleanup_idempotency(conn, ttl_seconds: int) -> int: + cur = await conn.execute( + "DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)", + (ttl_seconds,), + ) + return cur.rowcount + + +async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str, + content_type: str, size_bytes: int, sha256: str, object_key: str) -> None: + await conn.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key," + " status, error_code, created_at, updated_at)" + " VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, clock_timestamp(), clock_timestamp())", + (attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key), + ) + + +async def finish_attachment(conn, attachment_id: str) -> Optional[dict]: + cur = await conn.execute( + "UPDATE attachments SET status = 'ready', updated_at = clock_timestamp()" + " WHERE id = %s AND status = 'uploading'" + " RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code," + " created_at, updated_at", + (attachment_id,), + ) + row = await cur.fetchone() + return _attachment(row) if row else None + + +async def abort_attachment(conn, attachment_id: str) -> None: + await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,)) + + +async def get_attachment(conn, attachment_id: str): + cur = await conn.execute( + "SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, error_code," + " created_at, updated_at FROM attachments WHERE id = %s AND status <> 'uploading'", + (attachment_id,), + ) + return await cur.fetchone() diff --git a/app/s3.py b/app/s3.py new file mode 100644 index 00000000..613411a1 --- /dev/null +++ b/app/s3.py @@ -0,0 +1,97 @@ +import logging +from typing import Iterator, Optional, Tuple + +import boto3 +from botocore.config import Config as BotoConfig +from botocore.exceptions import BotoCoreError, ClientError + +from app import config +from app.errors import unavailable + +log = logging.getLogger("messenger.s3") + +MISSING = ("NoSuchKey", "404", "NotFound") + + +class S3Storage: + def __init__(self) -> None: + self.client = None + self.probe_client = None + + def open(self) -> None: + self.probe_client = self._client(1.0, 1.5) + self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT) + + def _client(self, connect: float, read: float): + return boto3.client( + "s3", + endpoint_url=config.S3_ENDPOINT, + region_name=config.S3_REGION, + aws_access_key_id=config.S3_ACCESS_KEY_ID, + aws_secret_access_key=config.S3_SECRET_ACCESS_KEY, + config=BotoConfig( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=connect, + read_timeout=read, + retries={"max_attempts": 0}, + ), + ) + + def probe(self) -> bool: + try: + self.probe_client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False + + def put(self, key: str, fileobj, size: int, content_type: str) -> None: + try: + fileobj.seek(0) + self.client.put_object( + Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type, + ) + except (BotoCoreError, ClientError) as exc: + log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + + def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]: + try: + response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key) + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + except BotoCoreError as exc: + log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__) + raise unavailable("Хранилище файлов недоступно") from None + body = response["Body"] + return body.iter_chunks(65536), int(response["ContentLength"]) + + def head(self, key: str): + """LastModified of the object, or None when it does not exist.""" + try: + return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"] + except ClientError as exc: + if exc.response["Error"].get("Code", "") in MISSING: + return None + raise + + def exists(self, key: str) -> bool: + return self.head(key) is not None + + def delete(self, key: str) -> None: + self.client.delete_object(Bucket=config.S3_BUCKET, Key=key) + + def list_pages(self, prefix: str): + paginator = self.client.get_paginator("list_objects_v2") + for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}): + yield page.get("Contents", []) + + def ready(self) -> bool: + try: + self.client.head_bucket(Bucket=config.S3_BUCKET) + return True + except Exception: + return False diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..31d472f4 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,154 @@ +import hashlib +import json +import logging +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional + +import valkey.asyncio as valkey + +from app import config, repo +from app.cursor import fmt +from app.errors import unavailable + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionStore: + def __init__(self, db) -> None: + self.db = db + self.client = None + self.stats = {"hits": 0, "misses": 0, "errors": 0} + + async def open(self) -> None: + if not config.CACHE_URL: + log.info("session cache disabled: CACHE_URL is empty") + return + self.client = valkey.Valkey.from_url( + config.CACHE_URL, + socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, + decode_responses=True, + ) + + async def close(self) -> None: + if self.client is not None: + await self.client.aclose() + self.client = None + + @staticmethod + def digest(token: str) -> bytes: + return hashlib.sha256(token.encode("utf-8")).digest() + + @staticmethod + def key(digest: bytes) -> str: + return "session:" + digest.hex() + + async def create(self, user: dict) -> tuple: + token = secrets.token_urlsafe(32) + digest = self.digest(token) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + async with self.db.connection() as conn: + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at)" + " VALUES (%s, %s, clock_timestamp(), %s)", + (digest, user["id"], expires_at), + ) + await self._cache_put(digest, user, expires_at) + return token, fmt(expires_at) + + async def resolve(self, token: str) -> Optional[dict]: + digest = self.digest(token) + cached = await self._cache_get(digest) + if cached == REVOKED: + self.stats["hits"] += 1 + return None + if cached is not None: + self.stats["hits"] += 1 + return cached + self.stats["misses"] += 1 + async with self.db.connection() as conn: + cur = await conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at" + " FROM sessions s JOIN users u ON u.id = s.user_id" + " WHERE s.token_hash = %s AND s.expires_at > now()", + (digest,), + ) + row = await cur.fetchone() + if row is None: + return None + user = repo.public_user(row) + await self._cache_put(digest, user, row["expires_at"]) + return user + + async def revoke(self, token: str) -> None: + digest = self.digest(token) + await self._cache_revoke(digest) + try: + async with self.db.connection() as conn: + await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,)) + except Exception: + await self._cache_release(digest) + raise + + async def sweep(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + return cur.rowcount + + def _cache_ttl(self, expires_at: datetime) -> int: + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + return min(config.SESSION_CACHE_TTL_SECONDS, remaining) + + async def _cache_get(self, digest: bytes) -> Optional[dict]: + if self.client is None: + return None + try: + raw = await self.client.get(self.key(digest)) + except Exception: + self._cache_failed("get") + return None + if not raw: + return None + if raw == REVOKED: + return REVOKED + try: + return json.loads(raw) + except ValueError: + return None + + async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None: + if self.client is None: + return + ttl = self._cache_ttl(expires_at) + if ttl <= 0: + return + payload = json.dumps(user) + try: + await self.client.set(self.key(digest), payload, ex=ttl, nx=True) + except Exception: + self._cache_failed("set") + + async def _cache_release(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.delete(self.key(digest)) + except Exception: + self._cache_failed("release") + + async def _cache_revoke(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS) + except Exception: + self._cache_failed("revoke") + raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None + + def _cache_failed(self, operation: str) -> None: + self.stats["errors"] += 1 + if self.stats["errors"] % 50 == 1: + log.warning("session cache %s failed, falling back to postgres (errors=%d)", operation, self.stats["errors"]) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..4fc08479 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,229 @@ +name: mtusi-lab4 + +x-api: &api-defaults + build: + context: . + image: mtusi-messenger:lab4 + env_file: + - .env + secrets: + - database_url + - cache_url + - s3_access_key_id + - s3_secret_access_key + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + minio-init: + condition: service_completed_successfully + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""] + interval: 5s + timeout: 3s + retries: 3 + start_period: 15s + restart: unless-stopped + +services: + nginx: + image: nginx:1.27.3-alpine + command: ["/bin/sh", "/opt/nginx-health.sh"] + environment: + API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2} + HEALTH_INTERVAL: "1" + HEALTH_FALL: "2" + HEALTH_RISE: "2" + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx-health.sh:/opt/nginx-health.sh:ro + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s + restart: unless-stopped + + api-1: + <<: *api-defaults + environment: + INSTANCE_ID: api-1 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://minio:9000 + + api-2: + <<: *api-defaults + environment: + INSTANCE_ID: api-2 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + S3_ENDPOINT: http://minio:9000 + + postgres: + image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: --data-checksums + command: + - postgres + - -c + - synchronous_commit=on + - -c + - shared_buffers=128MB + - -c + - work_mem=8MB + - -c + - max_connections=50 + - -c + - track_io_timing=on + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: + - valkey-server + - --save + - "" + - --appendonly + - "no" + - --maxmemory + - 64mb + - --maxmemory-policy + - allkeys-lru + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + restart: unless-stopped + + minio: + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + command: ["server", "/data"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + MINIO_BROWSER: "off" + secrets: + - minio_root_user + - minio_root_password + volumes: + - miniodata:/data + deploy: + resources: + limits: + cpus: "0.50" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "mc ready local || exit 1"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 10s + restart: unless-stopped + + minio-init: + image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a + entrypoint: ["/bin/sh", "/opt/init.sh"] + environment: + S3_BUCKET: messenger-attachments + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio/init.sh:/opt/init.sh:ro + - ./infra/minio/app-policy.json:/etc/minio-policy/app-policy.json:ro + depends_on: + minio: + condition: service_healthy + restart: "no" + + migrate: + build: + context: . + image: mtusi-messenger:lab4 + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: "no" + +volumes: + pgdata: + miniodata: + +secrets: + minio_root_user: + file: ./secrets/minio_root_user + minio_root_password: + file: ./secrets/minio_root_password + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url + s3_access_key_id: + file: ./secrets/s3_access_key_id + s3_secret_access_key: + file: ./secrets/s3_secret_access_key diff --git a/evidence/attachment_resilience.txt b/evidence/attachment_resilience.txt new file mode 100644 index 00000000..e586827e --- /dev/null +++ b/evidence/attachment_resilience.txt @@ -0,0 +1,4 @@ +metadata served by api-1 +ok uploaded via api-2, downloaded via api-1, identical sha256 +ok restart of all stateful services keeps metadata and original +PASS: attachment failover + persistence diff --git a/evidence/attachments_tests.txt b/evidence/attachments_tests.txt new file mode 100644 index 00000000..25655141 --- /dev/null +++ b/evidence/attachments_tests.txt @@ -0,0 +1,9 @@ +ok auth and membership on upload +ok file over 10 MiB -> 413 +ok 48 Mpx image -> 413 before decoding +ok corrupted image -> 415 +ok MIME spoofing and unsupported types -> 415 +ok dangerous filenames -> 400 +ok unknown / foreign / other-chat attachment ids rejected, no 500 +ok client abort mid-upload sent; server must stay healthy +PASS: attachment negative scenarios diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..af74f339 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,2 @@ +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 0.441} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/input_hardening.txt b/evidence/input_hardening.txt new file mode 100644 index 00000000..8e04d88e --- /dev/null +++ b/evidence/input_hardening.txt @@ -0,0 +1,11 @@ +ok truncated PNG, bad filter byte, broken IDAT, 27-byte JPEG stub -> 415 +ok real JPEG accepted; first half + EOI and first half alone -> 415 +ok same defects in images above the full-decode threshold (6 Mpx) -> 415 +ok valid 6 Mpx PNG accepted +ok valid 36 Mpx PNG (116 KiB on the wire, 108 MiB decoded) accepted in 0.1s +ok 36 Mpx PNG with bad filter byte -> 415 +ok whitespace-only, DEL, C1 and dot filenames -> 400 +ok attachment order preserved in POST, replay and history; reordered body -> 409 +ok idempotency: [] == absent, invalid keys -> 400, scoped per user +ok 12 MiB body -> JSON 413 +PASS: input hardening diff --git a/evidence/jpeg_check.txt b/evidence/jpeg_check.txt new file mode 100644 index 00000000..12aa6505 --- /dev/null +++ b/evidence/jpeg_check.txt @@ -0,0 +1,19 @@ +valid baseline 675893 bytes accepted 0.01s ok +valid progressive 616591 bytes accepted 0.02s ok +valid grayscale 359415 bytes accepted 0.00s ok +valid CMYK 1262509 bytes accepted 0.01s ok +valid, junk after EOI 675901 bytes accepted 0.01s ok +valid 40 Mpx baseline 626630 bytes accepted 0.01s ok +valid 40 Mpx progressive 352914 bytes accepted 0.07s ok +valid 40 Mpx progressive 444 704086 bytes accepted 0.13s ok +valid 40 Mpx progressive CMYK 626581 bytes accepted 0.17s ok +27-byte stub 27 bytes rejected 415 0.00s ok +first half + EOI 337948 bytes rejected 415 0.00s ok +first half, no EOI 337946 bytes rejected 415 0.00s ok +progressive half + EOI 308297 bytes rejected 415 0.00s ok +headers only + EOI 611 bytes rejected 415 0.01s ok + checker peak RSS, 40 Mpx baseline 1.7 MiB + checker peak RSS, 40 Mpx progressive 116.3 MiB + checker peak RSS, 40 Mpx progressive 444 230.6 MiB + checker peak RSS, 40 Mpx progressive CMYK 306.9 MiB +RESULT: 14/14 diff --git a/evidence/lb_stress.txt b/evidence/lb_stress.txt new file mode 100644 index 00000000..c3a1b04c --- /dev/null +++ b/evidence/lb_stress.txt @@ -0,0 +1,25 @@ +== 10 parallel uploads of 8674403 bytes, 3 rounds == +upload statuses: {201: 30} +== 10 parallel downloads == +download statuses: {200: 20} +== 10 parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), 3 rounds == +upload statuses: {201: 30} +== 10 MiB + 1 byte, Content-Length: 100 requests, 10 at a time == + {'413 application/json': 100} +== 10 MiB + 1 byte, chunked: 100 requests, 10 at a time == + {'413 application/json': 100} +== 10 MiB + 1 byte, chunked in 64 KiB pieces: 100 requests, 10 at a time == + {'413 application/json': 100} +== 8 MiB to a foreign chat: 100 requests, 10 at a time == + {'404 application/json': 100} +== 8 MiB without session: 100 requests, 10 at a time == + {'401 application/json': 100} +== 8 MiB with wrong magic: 100 requests, 10 at a time == + {'415 application/json': 100} +== 8 MiB with wrong Content-Type: 100 requests, 10 at a time == + {'415 application/json': 100} +== 70 KiB JSON message: 100 requests, 10 at a time == + {'413 application/json': 100} +container events during the run: none +cgroup memory.peak, MiB: {'mtusi-lab4-api-1-1': 475.1, 'mtusi-lab4-api-2-1': 490.5, 'mtusi-lab4-nginx-1': 34.3} +PASS: LB stress diff --git a/evidence/lb_stress_30.txt b/evidence/lb_stress_30.txt new file mode 100644 index 00000000..fe3a9c17 --- /dev/null +++ b/evidence/lb_stress_30.txt @@ -0,0 +1,25 @@ +== 30 parallel uploads of 8674403 bytes, 2 rounds == +upload statuses: {201: 60} +== 30 parallel downloads == +download statuses: {200: 60} +== 30 parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), 2 rounds == +upload statuses: {201: 60} +== 10 MiB + 1 byte, Content-Length: 30 requests, 30 at a time == + {'413 application/json': 30} +== 10 MiB + 1 byte, chunked: 30 requests, 30 at a time == + {'413 application/json': 30} +== 10 MiB + 1 byte, chunked in 64 KiB pieces: 30 requests, 30 at a time == + {'413 application/json': 30} +== 8 MiB to a foreign chat: 30 requests, 30 at a time == + {'404 application/json': 30} +== 8 MiB without session: 30 requests, 30 at a time == + {'401 application/json': 30} +== 8 MiB with wrong magic: 30 requests, 30 at a time == + {'415 application/json': 30} +== 8 MiB with wrong Content-Type: 30 requests, 30 at a time == + {'415 application/json': 30} +== 70 KiB JSON message: 30 requests, 30 at a time == + {'413 application/json': 30} +container events during the run: none +cgroup memory.peak, MiB: {'mtusi-lab4-api-1-1': 465.3, 'mtusi-lab4-api-2-1': 463.0, 'mtusi-lab4-nginx-1': 34.3} +PASS: LB stress diff --git a/evidence/memory_experiment.txt b/evidence/memory_experiment.txt new file mode 100644 index 00000000..eabf91dc --- /dev/null +++ b/evidence/memory_experiment.txt @@ -0,0 +1,25 @@ +payload bytes: 8674403 +round 1 peak MiB per api: {'mtusi-lab4-api-2-1': 95.5, 'mtusi-lab4-api-1-1': 115.6} statuses=[201] +round 2 peak MiB per api: {'mtusi-lab4-api-2-1': 104.8, 'mtusi-lab4-api-1-1': 115.6} statuses=[201] +round 3 peak MiB per api: {'mtusi-lab4-api-2-1': 106.6, 'mtusi-lab4-api-1-1': 115.3} statuses=[201] +{ + "payload_mib": 8.27, + "concurrency": 10, + "rounds": 3, + "idle_mib": { + "mtusi-lab4-api-2-1": 87.2, + "mtusi-lab4-api-1-1": 87.8 + }, + "peak_mib": { + "mtusi-lab4-api-2-1": 106.6, + "mtusi-lab4-api-1-1": 115.6 + }, + "container_limit_mib": 512, + "statuses": { + "201": 30 + }, + "upload_seconds_median": 0.63, + "upload_seconds_max": 0.82 +} +peak=163 MiB oom_kill 0 +peak=154 MiB oom_kill 0 diff --git a/evidence/memory_experiment_before_semaphore.txt b/evidence/memory_experiment_before_semaphore.txt new file mode 100644 index 00000000..c568c0df --- /dev/null +++ b/evidence/memory_experiment_before_semaphore.txt @@ -0,0 +1,23 @@ +payload bytes: 8674403 +round 1 peak MiB per api: {'mtusi-lab4-api-1-1': 143.7, 'mtusi-lab4-api-2-1': 142.4} statuses=[201] +round 2 peak MiB per api: {'mtusi-lab4-api-1-1': 166.1, 'mtusi-lab4-api-2-1': 153.2} statuses=[201] +round 3 peak MiB per api: {'mtusi-lab4-api-1-1': 182.3, 'mtusi-lab4-api-2-1': 154.7} statuses=[201] +{ + "payload_mib": 8.27, + "concurrency": 10, + "rounds": 3, + "idle_mib": { + "mtusi-lab4-api-1-1": 88.8, + "mtusi-lab4-api-2-1": 88.4 + }, + "peak_mib": { + "mtusi-lab4-api-1-1": 182.3, + "mtusi-lab4-api-2-1": 154.7 + }, + "container_limit_mib": 256, + "statuses": { + "201": 30 + }, + "upload_seconds_median": 0.82, + "upload_seconds_max": 1.28 +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..fc5a2b06 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1 @@ +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/private_bucket.txt b/evidence/private_bucket.txt new file mode 100644 index 00000000..760bb1a1 --- /dev/null +++ b/evidence/private_bucket.txt @@ -0,0 +1,12 @@ +== published ports of minio (expect none) == +no port 9000/tcp for container mtusi-lab4-minio-1: +no port 9001/tcp for container mtusi-lab4-minio-1: +== anonymous requests from inside the network == +GET /messenger-attachments/ -> 403 +GET /messenger-attachments/attachments/does-not-matter -> 403 +GET /messenger-attachments?list-type=2 -> 403 +== bucket policy == + Container mtusi-lab4-minio-init-run-85f57841572d Creating + Container mtusi-lab4-minio-init-run-85f57841572d Created +Access permission for `local/messenger-attachments` is `private` +PASS: bucket is private diff --git a/evidence/reconciler_experiment.txt b/evidence/reconciler_experiment.txt new file mode 100644 index 00000000..a9438feb --- /dev/null +++ b/evidence/reconciler_experiment.txt @@ -0,0 +1,30 @@ +baseline (data left by earlier tests): {"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 366, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 203, "seconds": 0.253, "stale_uploading": 0} +== dry-run, grace=900 == +{"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 2, "objects_scanned": 371, "orphan_objects": 0, "protected_fresh_objects": 2, "removed_rows": 0, "rows_scanned": 206, "seconds": 0.264, "stale_uploading": 1} +ok dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h) +ok dry-run: 2 new missing objects (e, g) +ok dry-run changes nothing +== apply, grace=900 == +{"deleted_objects": 0, "dry_run": false, "grace_seconds": 900, "marked_failed": 2, "missing_objects": 2, "objects_scanned": 371, "orphan_objects": 0, "protected_fresh_objects": 2, "removed_rows": 1, "rows_scanned": 206, "seconds": 0.273, "stale_uploading": 1} +ok old row whose object is young is kept (a) +ok stale row without object removed +ok fresh in-flight upload untouched +ok fresh orphan object kept inside grace +ok old row with a fresh object is left alone +ok recently confirmed row is not marked missing +ok missing unused object marked failed +ok used attachment with object untouched +ok used attachment with lost object kept, marked failed +== apply after grace expired (grace=2 after 3s) == +{"deleted_objects": 4, "dry_run": false, "grace_seconds": 2, "marked_failed": 1, "missing_objects": 1, "objects_scanned": 368, "orphan_objects": 1, "protected_fresh_objects": 0, "removed_rows": 3, "rows_scanned": 368, "seconds": 0.428, "stale_uploading": 3} +ok orphan object deleted after grace +ok expired in-flight row collected after grace +ok old row + object collected once the object is older than grace (a, h) +ok used attachment still untouched +== second apply is idempotent == +{"deleted_objects": 0, "dry_run": false, "grace_seconds": 2, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 367, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 367, "seconds": 0.457, "stale_uploading": 0} +ok repeat run changes nothing +== cost of a full walk, 1000 ready attachments == +{"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 1367, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 1204, "seconds": 1.605, "stale_uploading": 0} +ok bulk walk finds nothing wrong +RESULT: 18/18 checks passed diff --git a/evidence/s3_outage.txt b/evidence/s3_outage.txt new file mode 100644 index 00000000..64231192 --- /dev/null +++ b/evidence/s3_outage.txt @@ -0,0 +1,5 @@ +ok S3 down: upload/download/ready -> 503 JSON, metadata and text messages still served through the LB (no 502) +ok recovered 2s after S3 start +ok ready is 200 again +ok hung S3: upload answered 503 in 7.0s +PASS: S3 outage behaviour diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c554fd6f --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/minio/app-policy.json b/infra/minio/app-policy.json new file mode 100644 index 00000000..000dfc3f --- /dev/null +++ b/infra/minio/app-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/infra/minio/init.sh b/infra/minio/init.sh new file mode 100755 index 00000000..2b69c79f --- /dev/null +++ b/infra/minio/init.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Idempotent: safe to run on every `compose up`. Root credentials are used only here, never by the API. +set -eu +BUCKET="${S3_BUCKET:-messenger-attachments}" +ROOT_USER="$(cat /run/secrets/minio_root_user)" +ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)" +APP_USER="$(cat /run/secrets/s3_access_key_id)" +APP_PASSWORD="$(cat /run/secrets/s3_secret_access_key)" + +until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do + echo "waiting for minio"; sleep 2 +done +mc mb --ignore-existing "local/$BUCKET" +mc anonymous set none "local/$BUCKET" +mc admin policy create local app-attachments /etc/minio-policy/app-policy.json 2>/dev/null \ + || mc admin policy create local app-attachments /etc/minio-policy/app-policy.json --overwrite 2>/dev/null || true +mc admin user add local "$APP_USER" "$APP_PASSWORD" 2>/dev/null || true +mc admin policy attach local app-attachments --user "$APP_USER" 2>/dev/null || true +echo "minio bootstrap complete: bucket=$BUCKET private, app user limited to attachments/*" diff --git a/infra/nginx-health.sh b/infra/nginx-health.sh new file mode 100644 index 00000000..692cab75 --- /dev/null +++ b/infra/nginx-health.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -u + +SERVERS="${API_UPSTREAMS:-api-1 api-2}" +INTERVAL="${HEALTH_INTERVAL:-1}" +FALL="${HEALTH_FALL:-2}" +RISE="${HEALTH_RISE:-2}" +OUT=/etc/nginx/upstream/servers.conf + +mkdir -p "$(dirname "$OUT")" + +var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; } + +render() { + any_up=0 + for s in $SERVERS; do + eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1 + done + for s in $SERVERS; do + mark="" + if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then + mark=" down" + fi + echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;" + done +} + +for s in $SERVERS; do + v=$(var "$s") + eval "up_$v=1 ok_$v=0 bad_$v=0" +done +render > "$OUT" + +watch() { + pending=0 + while sleep "$INTERVAL"; do + changed=0 + for s in $SERVERS; do + v=$(var "$s") + if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then + eval "ok_$v=\$((ok_$v + 1)) bad_$v=0" + if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then + eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream" + fi + else + eval "bad_$v=\$((bad_$v + 1)) ok_$v=0" + if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then + eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down" + fi + fi + done + if [ "$changed" = 1 ]; then + render > "$OUT" + pending=1 + fi + if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then + pending=0 + fi + done +} + +watch & +exec nginx -g 'daemon off;' diff --git a/infra/nginx.conf b/infra/nginx.conf new file mode 100644 index 00000000..5d791ed9 --- /dev/null +++ b/infra/nginx.conf @@ -0,0 +1,75 @@ +worker_processes 2; + +events { + worker_connections 256; +} + +http { + resolver 127.0.0.11 valid=5s ipv6=off; + + upstream api { + zone api 64k; + include /etc/nginx/upstream/servers.conf; + keepalive 32; + keepalive_timeout 4s; + } + + log_format main '$remote_addr - $upstream_addr [$time_local] ' + '"$request" $status $body_bytes_sent ' + '"$upstream_response_time" "$request_time"'; + + access_log /var/log/nginx/access.log main; + + server { + listen 8080; + client_max_body_size 64k; + error_page 413 @payload_too_large; + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_connect_timeout 1s; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 5s; + + location ~ ^/api/v1/chats/[^/]+/messages$ { + proxy_pass http://api; + proxy_next_upstream error timeout http_502 http_503 non_idempotent; + } + + location ~ ^/api/v1/chats/[^/]+/attachments$ { + client_max_body_size 10m; + proxy_request_buffering off; + proxy_pass http://api; + proxy_next_upstream off; + } + + location @payload_too_large { + default_type application/json; + add_header X-Request-Id $request_id always; + add_header X-Instance-Id lb always; + add_header Connection close always; + return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}'; + } + + location /health/ { + proxy_pass http://api; + proxy_next_upstream error timeout; + } + + location ~ ^/api/v1/attachments/ { + proxy_max_temp_file_size 0; + proxy_pass http://api; + proxy_next_upstream error timeout; + } + + location / { + proxy_pass http://api; + proxy_next_upstream error timeout http_502 http_503; + } + } +} diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..4de361a9 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,46 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name bytea NOT NULL + CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400), + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title bytea NOT NULL + CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400), + created_at timestamptz NOT NULL +); + +CREATE INDEX chats_created_idx ON chats (created_at, id); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + text bytea NOT NULL + CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000), + created_at timestamptz NOT NULL +); + +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL +); + +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..79818618 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + key text NOT NULL + CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'), + request_body_hash text NOT NULL, + response_json text NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/0003_attachments.sql b/migrations/0003_attachments.sql new file mode 100644 index 00000000..1dc55632 --- /dev/null +++ b/migrations/0003_attachments.sql @@ -0,0 +1,28 @@ +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT, + filename text NOT NULL + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL + CONSTRAINT attachments_ct CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL + CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'), + object_key text NOT NULL, + status text NOT NULL DEFAULT 'ready' + CONSTRAINT attachments_status CHECK (status IN ('ready', 'failed')), + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT, + position smallint NOT NULL, + PRIMARY KEY (message_id, attachment_id) +); diff --git a/migrations/0004_allow_empty_text.sql b/migrations/0004_allow_empty_text.sql new file mode 100644 index 00000000..74aeab41 --- /dev/null +++ b/migrations/0004_allow_empty_text.sql @@ -0,0 +1,2 @@ +ALTER TABLE messages DROP CONSTRAINT messages_text_size; +ALTER TABLE messages ADD CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 0 AND 16000); diff --git a/migrations/0005_attachment_upload_state.sql b/migrations/0005_attachment_upload_state.sql new file mode 100644 index 00000000..f0489c5c --- /dev/null +++ b/migrations/0005_attachment_upload_state.sql @@ -0,0 +1,5 @@ +ALTER TABLE attachments DROP CONSTRAINT attachments_status; +ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('uploading', 'ready', 'failed')); +ALTER TABLE attachments ALTER COLUMN status DROP DEFAULT; +CREATE UNIQUE INDEX attachments_object_key_idx ON attachments (object_key); +CREATE INDEX attachments_status_created_idx ON attachments (status, created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e6c069d8 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,30 @@ +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +boto3==1.35.81 +botocore==1.35.81 +cffi==2.1.1 +click==8.5.0 +fastapi==0.115.6 +h11==0.16.0 +httptools==0.9.0 +idna==3.20 +jmespath==1.0.1 +pillow==11.0.0 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +pycparser==3.11 +pydantic==2.14.1 +pydantic-core==2.50.1 +python-dateutil==2.9.0.post0 +s3transfer==0.10.4 +six==1.17.0 +starlette==0.41.3 +typing-extensions==4.16.0 +typing-inspection==0.4.4 +urllib3==2.3.0 +uvicorn==0.34.0 +uvloop==0.23.0 +valkey==6.2.0 diff --git a/scripts/jpeg_check_experiment.py b/scripts/jpeg_check_experiment.py new file mode 100644 index 00000000..4d1c8a82 --- /dev/null +++ b/scripts/jpeg_check_experiment.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""JPEG integrity matrix and peak memory of the checker. + docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py""" +import io +import os +import subprocess +import sys +import tempfile +import time + +sys.path.insert(0, "/app") + +from PIL import Image # noqa: E402 + +from app.errors import ApiError # noqa: E402 +from app.imagecheck import DJPEG, inspect_image # noqa: E402 + + +def encode(img, **kw): + out = io.BytesIO() + img.save(out, "JPEG", **kw) + return out.getvalue() + + +def verdict(data): + with tempfile.SpooledTemporaryFile(max_size=1048576) as spool: + spool.write(data) + try: + inspect_image(spool, len(data), "image/jpeg") + return "accepted" + except ApiError as exc: + return "rejected %d" % exc.status + + +def child_peak_kib(data): + with tempfile.NamedTemporaryFile() as handle: + handle.write(data) + handle.flush() + handle.seek(0) + proc = subprocess.Popen(DJPEG, stdin=handle) + peak = 0 + while proc.poll() is None: + try: + with open("/proc/%d/status" % proc.pid) as status: + for line in status: + if line.startswith("VmHWM"): + peak = max(peak, int(line.split()[1])) + except OSError: + pass + time.sleep(0.002) + return peak + + +noise = Image.frombytes("RGB", (1000, 750), os.urandom(1000 * 750 * 3)) +base = encode(noise, quality=90) +progressive = encode(noise, quality=90, progressive=True) +flat = Image.new("RGB", (8000, 5000), (120, 30, 200)) +big_base = encode(flat, quality=90) +big_progressive = encode(flat, quality=90, progressive=True) +big_444 = encode(flat, quality=90, progressive=True, subsampling=0) +big_cmyk = encode(flat.convert("CMYK"), quality=90, progressive=True) +del flat +sos = base.index(b"\xff\xda") +cases = [ + ("valid baseline", base, "accepted"), + ("valid progressive", progressive, "accepted"), + ("valid grayscale", encode(noise.convert("L")), "accepted"), + ("valid CMYK", encode(noise.convert("CMYK")), "accepted"), + ("valid, junk after EOI", base + b"trailing", "accepted"), + ("valid 40 Mpx baseline", big_base, "accepted"), + ("valid 40 Mpx progressive", big_progressive, "accepted"), + ("valid 40 Mpx progressive 444", big_444, "accepted"), + ("valid 40 Mpx progressive CMYK", big_cmyk, "accepted"), + ("27-byte stub", bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "rejected 415"), + ("first half + EOI", base[: len(base) // 2] + b"\xff\xd9", "rejected 415"), + ("first half, no EOI", base[: len(base) // 2], "rejected 415"), + ("progressive half + EOI", progressive[: len(progressive) // 2] + b"\xff\xd9", "rejected 415"), + ("headers only + EOI", base[:sos] + b"\xff\xd9", "rejected 415"), +] +failed = 0 +for name, data, expected in cases: + started = time.monotonic() + got = verdict(data) + ok = got == expected + failed += not ok + print("%-28s %9d bytes %-13s %.2fs %s" % (name, len(data), got, time.monotonic() - started, "ok" if ok else "FAIL")) +for name, data in (("40 Mpx baseline", big_base), ("40 Mpx progressive", big_progressive), + ("40 Mpx progressive 444", big_444), ("40 Mpx progressive CMYK", big_cmyk)): + print("checker peak RSS, %-24s %6.1f MiB" % (name, child_peak_kib(data) / 1024)) +print("RESULT: %d/%d" % (len(cases) - failed, len(cases))) +sys.exit(1 if failed else 0) diff --git a/scripts/migrate.py b/scripts/migrate.py new file mode 100644 index 00000000..5c19e228 --- /dev/null +++ b/scripts/migrate.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run.""" +import hashlib +import os +import pathlib +import sys + +import psycopg + +ROOT = pathlib.Path(__file__).resolve().parents[1] +MIGRATIONS = ROOT / "migrations" +LOCK_ID = 2026_0004 + + +def database_url() -> str: + path = os.environ.get("DATABASE_URL_FILE") + if path: + return pathlib.Path(path).read_text(encoding="utf-8").strip() + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + sys.exit("DATABASE_URL or DATABASE_URL_FILE is required") + return url + + +def main() -> None: + files = sorted(MIGRATIONS.glob("*.sql")) + if not files: + sys.exit("no migration files found in %s" % MIGRATIONS) + with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY," + " checksum text NOT NULL," + " applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall() + applied = {row[0]: row[1] for row in rows} + for path in files: + version = path.stem + body = path.read_text(encoding="utf-8") + checksum = hashlib.sha256(body.encode("utf-8")).hexdigest() + if version in applied: + if applied[version] != checksum: + sys.exit("migration %s changed after it was applied" % version) + print("skip %s" % version) + continue + with conn.transaction(): + conn.execute(body) + conn.execute( + "INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", + (version, checksum), + ) + print("apply %s" % version) + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + print("migrations complete: %d file(s)" % len(files)) + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile.py b/scripts/reconcile.py new file mode 100644 index 00000000..10cd7b68 --- /dev/null +++ b/scripts/reconcile.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything.""" +import argparse +import asyncio +import json +import sys + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import config, reconcile # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + + +async def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports") + parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS) + args = parser.parse_args() + db, s3 = Database(), S3Storage() + await db.open() + await run_in_threadpool(s3.open) + try: + print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True)) + finally: + await db.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/scripts/reconcile_experiment.py b/scripts/reconcile_experiment.py new file mode 100644 index 00000000..4638db7f --- /dev/null +++ b/scripts/reconcile_experiment.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +"""Fault injection for the DB<->S3 reconciler. Run inside an api container: + docker compose exec -T api-1 python scripts/reconcile_experiment.py [--bulk 1000]""" +import argparse +import asyncio +import io +import json +import sys +import time +import uuid + +sys.path.insert(0, "/app") + +from starlette.concurrency import run_in_threadpool # noqa: E402 + +from app import config, reconcile # noqa: E402 +from app.db import Database # noqa: E402 +from app.s3 import S3Storage # noqa: E402 + +ZERO = "0" * 64 +checks = [] + + +def check(name, condition): + checks.append((name, bool(condition))) + print(("ok " if condition else "FAIL ") + name) + + +async def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--bulk", type=int, default=1000) + args = parser.parse_args() + db, s3 = Database(), S3Storage() + await db.open() + await run_in_threadpool(s3.open) + put = lambda key: run_in_threadpool(s3.put, key, io.BytesIO(b"x"), 1, "image/png") + exists = lambda key: run_in_threadpool(s3.exists, key) + + baseline = await reconcile.run(db, s3, 900, dry_run=True) + print("baseline (data left by earlier tests):", json.dumps(baseline, sort_keys=True)) + + async with db.connection() as conn: + suffix = uuid.uuid4().hex[:10] + user = (await (await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at)" + " VALUES (gen_random_uuid(), %s, %s, 'x', now()) RETURNING id", ("rx_" + suffix, b"rx"))).fetchone())["id"] + chat = (await (await conn.execute( + "INSERT INTO chats (id, title, created_at) VALUES (gen_random_uuid(), %s, now()) RETURNING id", (b"rx",))).fetchone())["id"] + await conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat, user)) + + async def row(status, age_s, with_object, linked=False): + aid = str(uuid.uuid4()) + key = "attachments/" + aid + await conn.execute( + "INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key," + " status, created_at, updated_at) VALUES (%s,%s,%s,'f.png','image/png',1,%s,%s,%s," + " now() - make_interval(secs => %s), now() - make_interval(secs => %s))", (aid, chat, user, ZERO, key, status, age_s, age_s)) + if with_object: + await put(key) + if linked: + mid = (await (await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " VALUES (gen_random_uuid(), %s, %s, ''::bytea, now()) RETURNING id", (chat, user))).fetchone())["id"] + await conn.execute("INSERT INTO message_attachments VALUES (%s,%s,0)", (mid, aid)) + return aid, key + + async def status_of(aid): + r = await (await conn.execute("SELECT status, error_code FROM attachments WHERE id=%s", (aid,))).fetchone() + return (r["status"], r["error_code"]) if r else None + + old = 7200 + a_id, a_key = await row("uploading", old, True) # crash after PUT, before ready (object is young) + b_id, b_key = await row("uploading", old, False) # crash before PUT + c_id, c_key = await row("uploading", 5, True) # in-flight upload, must survive + e_id, e_key = await row("ready", old, False) # object lost + f_id, f_key = await row("ready", old, True, linked=True) # healthy, used + g_id, g_key = await row("ready", old, False, linked=True) # object lost, used + h_id, h_key = await row("uploading", old, False) # old intent row, but a NEW object under its key + await put(h_key) + i_id, i_key = await row("ready", 5, False) # freshly confirmed, object not yet visible: must not be marked + d_key = "attachments/" + str(uuid.uuid4()) # PUT done, row never written + await put(d_key) + + print("== dry-run, grace=900 ==") + r = await reconcile.run(db, s3, 900, dry_run=True) + print(json.dumps(r, sort_keys=True)) + check("dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)", + r["stale_uploading"] - baseline["stale_uploading"] == 1 + and r["protected_fresh_objects"] - baseline["protected_fresh_objects"] == 2) + check("dry-run: 2 new missing objects (e, g)", r["missing_objects"] - baseline["missing_objects"] == 2) + check("dry-run changes nothing", await exists(a_key) and await status_of(a_id) == ("uploading", None) + and await status_of(e_id) == ("ready", None)) + + print("== apply, grace=900 ==") + r = await reconcile.run(db, s3, 900, dry_run=False) + print(json.dumps(r, sort_keys=True)) + check("old row whose object is young is kept (a)", await status_of(a_id) == ("uploading", None) and await exists(a_key)) + check("stale row without object removed", await status_of(b_id) is None) + check("fresh in-flight upload untouched", await status_of(c_id) == ("uploading", None) and await exists(c_key)) + check("fresh orphan object kept inside grace", await exists(d_key)) + check("old row with a fresh object is left alone", await status_of(h_id) == ("uploading", None) and await exists(h_key)) + check("recently confirmed row is not marked missing", await status_of(i_id) == ("ready", None)) + check("missing unused object marked failed", await status_of(e_id) == ("failed", "object_missing")) + check("used attachment with object untouched", await status_of(f_id) == ("ready", None) and await exists(f_key)) + check("used attachment with lost object kept, marked failed", await status_of(g_id) == ("failed", "object_missing")) + + print("== apply after grace expired (grace=2 after 3s) ==") + await asyncio.sleep(3) + r = await reconcile.run(db, s3, 2, dry_run=False) + print(json.dumps(r, sort_keys=True)) + check("orphan object deleted after grace", not await exists(d_key)) + check("expired in-flight row collected after grace", await status_of(c_id) is None and not await exists(c_key)) + check("old row + object collected once the object is older than grace (a, h)", + await status_of(a_id) is None and not await exists(a_key) + and await status_of(h_id) is None and not await exists(h_key)) + check("used attachment still untouched", await status_of(f_id) == ("ready", None) and await exists(f_key)) + + print("== second apply is idempotent ==") + r = await reconcile.run(db, s3, 2, dry_run=False) + print(json.dumps(r, sort_keys=True)) + check("repeat run changes nothing", r["deleted_objects"] == 0 and r["removed_rows"] == 0 and r["marked_failed"] == 0) + + print("== cost of a full walk, %d ready attachments ==" % args.bulk) + keys = [] + for _ in range(args.bulk): + aid, key = await row("ready", old, True) + keys.append(aid) + r = await reconcile.run(db, s3, 900, dry_run=True) + print(json.dumps(r, sort_keys=True)) + check("bulk walk finds nothing wrong", r["missing_objects"] == 0 and r["orphan_objects"] == 0) + + for aid in keys + [e_id, f_id, g_id, h_id, i_id]: + await run_in_threadpool(s3.delete, "attachments/" + aid) + await conn.execute("DELETE FROM chats WHERE id = %s", (chat,)) + await conn.execute("DELETE FROM users WHERE id = %s", (user,)) + await db.close() + failed = [n for n, ok in checks if not ok] + print("RESULT: %d/%d checks passed" % (len(checks) - len(failed), len(checks))) + sys.exit(1 if failed else 0) + + +asyncio.run(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..9df18d16 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +docker compose restart -t 5 postgres valkey minio api-1 api-2 nginx diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..23c94b55 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +docker compose stop -t 3 "$TARGET_INSTANCE_ID" diff --git a/student-tests/check_private_bucket.sh b/student-tests/check_private_bucket.sh new file mode 100755 index 00000000..b28c1278 --- /dev/null +++ b/student-tests/check_private_bucket.sh @@ -0,0 +1,27 @@ +#!/bin/sh +# Proves the bucket is private without going through the API: anonymous S3 requests from inside the compose +# network are refused, and the MinIO console/ports are not published on the host. +set -eu +cd "$(dirname "$0")/.." +NET="$(docker compose ps -q minio | xargs docker inspect --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}')" +BUCKET=messenger-attachments +echo "== published ports of minio (expect none) ==" +docker compose port minio 9000 2>&1 || true +docker compose port minio 9001 2>&1 || true +echo "== anonymous requests from inside the network ==" +for path in "$BUCKET/" "$BUCKET/attachments/does-not-matter" "$BUCKET?list-type=2"; do + code="$(docker run --rm -i --network "$NET" python:3.12.8-slim-bookworm python - "$path" <<'PY' +import sys, urllib.request, urllib.error +try: + print(urllib.request.urlopen("http://minio:9000/" + sys.argv[1], timeout=5).status) +except urllib.error.HTTPError as e: + print(e.code) +PY +)" + echo "GET /$path -> $code" + [ "$code" = 403 ] || { echo "FAIL: expected 403"; exit 1; } +done +echo "== bucket policy ==" +docker compose run --rm --no-deps --entrypoint sh minio-init -c \ + 'mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null && mc anonymous get local/messenger-attachments' +echo "PASS: bucket is private" diff --git a/student-tests/memory_experiment.py b/student-tests/memory_experiment.py new file mode 100644 index 00000000..ce691667 --- /dev/null +++ b/student-tests/memory_experiment.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Peak API memory under N concurrent ~8.7 MB uploads. Samples `docker stats` for the api containers. + python3 student-tests/memory_experiment.py --concurrency 10 --rounds 3""" +import argparse +import json +import os +import re +import statistics +import subprocess +import sys +import threading +import time +import zlib +import struct +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 + + +def png(width, height): + raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height)) + def chunk(kind, body): + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")) + + +UNITS = {"B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824} + + +def mib(text): + value, unit = re.match(r"([\d.]+)\s*([A-Za-z]+)", text).groups() + return float(value) * UNITS[unit] / 1048576 + + +def sample(stop, out): + while not stop.is_set(): + result = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"], + cwd=ROOT, capture_output=True, text=True) + for line in result.stdout.splitlines(): + name, usage = line.split(" ", 1) + if "api-" in name: + out.setdefault(name, []).append(mib(usage.split("/")[0].strip())) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--rounds", type=int, default=3) + args = parser.parse_args() + body = png(1700, 1700) + print("payload bytes:", len(body)) + c = Client(timeout=60) + alice = c.user("mem") + chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "mem", "member_ids": [alice["id"]]}, schema="Chat")["json"] + path = "/api/v1/chats/%s/attachments?filename=big.png" % chat["id"] + + def one(_): + r = c.call("POST", path, [201, 429, 503], actor=alice, raw=body, headers={"Content-Type": "image/png"}) + return r["status"], r["elapsed"] + + idle, peaks, statuses, times = {}, [], [], [] + probe = {} + stop = threading.Event() + t = threading.Thread(target=sample, args=(stop, probe)); t.start(); time.sleep(3); stop.set(); t.join() + idle = {k: max(v) for k, v in probe.items()} + for round_no in range(args.rounds): + probe = {} + stop = threading.Event() + t = threading.Thread(target=sample, args=(stop, probe)); t.start() + with ThreadPoolExecutor(args.concurrency) as pool: + results = list(pool.map(one, range(args.concurrency))) + stop.set(); t.join() + peak = {k: max(v) for k, v in probe.items()} + peaks.append(peak) + statuses += [s for s, _ in results] + times += [e for _, e in results] + print("round %d peak MiB per api: %s statuses=%s" % (round_no + 1, {k: round(v, 1) for k, v in peak.items()}, sorted(set(s for s, _ in results)))) + summary = { + "payload_mib": round(len(body) / 1048576, 2), "concurrency": args.concurrency, "rounds": args.rounds, + "idle_mib": {k: round(v, 1) for k, v in idle.items()}, + "peak_mib": {k: round(max(p.get(k, 0) for p in peaks), 1) for k in idle}, + "container_limit_mib": 512, + "statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))}, + "upload_seconds_median": round(statistics.median(times), 2), "upload_seconds_max": round(max(times), 2), + } + print(json.dumps(summary, indent=2)) + + +main() diff --git a/student-tests/test_attachment_resilience.py b/student-tests/test_attachment_resilience.py new file mode 100644 index 00000000..0804b652 --- /dev/null +++ b/student-tests/test_attachment_resilience.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Upload -> stop the serving replica -> download via the other; then restart everything and download again.""" +import hashlib +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, run # noqa: E402 + + +def compose(*args): + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True) + + +def wait_ready(c): + for _ in range(60): + try: + c.call("GET", "/health/ready") + return + except Exception: + time.sleep(2) + check(False, "stack did not become ready") + + +def main(): + c = Client() + alice, bob, _, chat = c.bootstrap() + att, content = c.attachment(alice, chat) + served_by = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["headers"]["X-Instance-Id"] + print("metadata served by", served_by) + first = c.call("POST", "/api/v1/chats/%s/attachments?filename=x.png" % chat["id"], 201, actor=alice, raw=content, + headers={"Content-Type": "image/png"}, schema="Attachment") + uploader = first["headers"]["X-Instance-Id"] + compose("stop", "-t", "3", uploader) + try: + time.sleep(3) + got = c.call("GET", "/api/v1/attachments/%s/content" % first["json"]["id"], actor=bob) + check(got["headers"]["X-Instance-Id"] != uploader, "download must be served by another replica") + check(hashlib.sha256(got["bytes"]).hexdigest() == first["json"]["sha256"], "bytes differ after uploader stopped") + print("ok uploaded via %s, downloaded via %s, identical sha256" % (uploader, got["headers"]["X-Instance-Id"])) + finally: + compose("start", uploader) + wait_ready(c) + compose("restart", "-t", "5", "postgres", "valkey", "minio", "api-1", "api-2", "nginx") + wait_ready(c) + got = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob) + check(got["bytes"] == content, "original lost after full restart") + meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"] + check(meta == att, "metadata changed after restart") + print("ok restart of all stateful services keeps metadata and original") + print("PASS: attachment failover + persistence") + + +run(main) diff --git a/student-tests/test_attachments.py b/student-tests/test_attachments.py new file mode 100644 index 00000000..dd2780a9 --- /dev/null +++ b/student-tests/test_attachments.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Negative and boundary scenarios for attachments (grade 4). Needs the stack from README on BASE_URL.""" +import hashlib +import os +import socket +import struct +import sys +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote, urlparse + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, run # noqa: E402 + +SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes() + + +def chunk(kind, body): + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF) + + +def header_only_png(width, height): + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"") + + +def upload(c, actor, chat, body, ctype, expected, filename="a.png"): + path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe="")) + return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype}, + schema="Attachment" if expected == 201 else None) + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + other = c.call("POST", "/api/v1/chats", 201, actor=alice, + body={"title": "Второй", "member_ids": [alice["id"]]}, schema="Chat")["json"] + eve_chat = c.call("POST", "/api/v1/chats", 201, actor=eve, + body={"title": "Чат Евы", "member_ids": [eve["id"]]}, schema="Chat")["json"] + + ok = upload(c, alice, chat, SAMPLE, "image/png", 201)["json"] + check(ok["status"] == "ready" and ok["variants"] == [] and ok["error_code"] is None, "ready upload shape") + got = c.call("GET", "/api/v1/attachments/" + ok["id"] + "/content", actor=bob) + check(hashlib.sha256(got["bytes"]).hexdigest() == ok["sha256"], "bob downloads identical bytes") + check(int(got["headers"]["Content-Length"]) == len(SAMPLE), "download Content-Length") + c.call("GET", "/api/v1/attachments/" + ok["id"] + "/content?variant=thumbnail", 404, actor=bob) + + upload(c, None, chat, SAMPLE, "image/png", 401) + upload(c, eve, chat, SAMPLE, "image/png", 404) + print("ok auth and membership on upload") + + big = header_only_png(10, 10) + b"\0" * (10 * 1024 * 1024) + upload(c, alice, chat, big, "image/png", 413) + print("ok file over 10 MiB -> 413") + upload(c, alice, chat, header_only_png(8000, 6000), "image/png", 413) + print("ok 48 Mpx image -> 413 before decoding") + upload(c, alice, chat, SAMPLE[: len(SAMPLE) // 2], "image/png", 415) + upload(c, alice, chat, SAMPLE[:40] + b"garbage" * 50, "image/png", 415) + print("ok corrupted image -> 415") + upload(c, alice, chat, SAMPLE, "image/jpeg", 415) + upload(c, alice, chat, b"\xff\xd8\xff\xe0" + b"\0" * 200, "image/png", 415) + upload(c, alice, chat, b"MZ not an image at all", "image/png", 415) + upload(c, alice, chat, b"", "image/png", 415) + upload(c, alice, chat, SAMPLE, "text/plain", 415) + upload(c, alice, chat, SAMPLE, "image/gif", 415) + print("ok MIME spoofing and unsupported types -> 415") + upload(c, alice, chat, b"", "image/png", 400) + for bad in ("../x.png", "a/b.png", "a\\b.png", "a\x00b.png", "a\nb.png", "", "x" * 256): + upload(c, alice, chat, SAMPLE, "image/png", 400, filename=bad) + print("ok dangerous filenames -> 400") + unicode_name = upload(c, alice, chat, SAMPLE, "image/png", 201, filename="фото 🙂.png")["json"] + check(unicode_name["filename"] == "фото 🙂.png", "display filename preserved") + + own = upload(c, alice, other, SAMPLE, "image/png", 201)["json"] + foreign = upload(c, eve, eve_chat, SAMPLE, "image/png", 201)["json"] + msg = lambda actor, ch, ids, expected: c.call( + "POST", "/api/v1/chats/%s/messages" % ch["id"], expected, actor=actor, + body={"text": "", "attachment_ids": ids}, headers={"Idempotency-Key": uuid.uuid4().hex}, schema="Message" if expected == 201 else None) + msg(alice, chat, [str(uuid.uuid4())], 404) + msg(alice, chat, [foreign["id"]], 404) + msg(alice, chat, [own["id"]], 404) + msg(alice, chat, [ok["id"], ok["id"]], 400) + msg(alice, chat, ["not-a-uuid"], 400) + msg(alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400) + msg(eve, chat, [ok["id"]], 404) + c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice, body={"text": " "}, + headers={"Idempotency-Key": uuid.uuid4().hex}) + print("ok unknown / foreign / other-chat attachment ids rejected, no 500") + sent = msg(bob, chat, [ok["id"]], 201)["json"] + check(sent["attachment_ids"] == [ok["id"]] and sent["text"] == "", "attachment-only message") + history = c.call("GET", "/api/v1/chats/%s/messages" % chat["id"], actor=alice, schema="MessagePage")["json"] + check(any(m["attachment_ids"] == [ok["id"]] for m in history["items"]), "history keeps attachment_ids") + for suffix in ("", "/content"): + c.call("GET", "/api/v1/attachments/" + foreign["id"] + suffix, 404, actor=alice) + c.call("GET", "/api/v1/attachments/" + str(uuid.uuid4()) + suffix, 404, actor=alice) + c.call("GET", "/api/v1/attachments/not-a-uuid" + suffix, 400, actor=alice) + c.call("GET", "/api/v1/attachments/" + ok["id"] + suffix, 401) + + parsed = urlparse(c.base) + sock = socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10) + request = ("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n" + "Content-Type: image/png\r\nContent-Length: %d\r\n\r\n" % (chat["id"], alice["token"], len(SAMPLE))) + sock.sendall(request.encode() + SAMPLE[: len(SAMPLE) // 2]) + sock.close() + print("ok client abort mid-upload sent; server must stay healthy") + c.call("GET", "/health/ready", schema="Health") + after = upload(c, alice, chat, SAMPLE, "image/png", 201)["json"] + check(after["sha256"] == hashlib.sha256(SAMPLE).hexdigest(), "upload works after aborted one") + print("PASS: attachment negative scenarios") + + +if __name__ == "__main__": + run(main) diff --git a/student-tests/test_input_hardening.py b/student-tests/test_input_hardening.py new file mode 100644 index 00000000..8ae649ab --- /dev/null +++ b/student-tests/test_input_hardening.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Hardening scenarios: corrupted images, filenames, attachment order, idempotency keys, oversized bodies.""" +import hashlib +import os +import struct +import subprocess +import sys +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "student-tests")) +from client import Client, check, run # noqa: E402 +from test_attachments import SAMPLE, chunk, header_only_png # noqa: E402 + +JPEG_STUB = bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9") + + +def png(width, height, filter_byte=0, level=1, corrupt_idat=False, noise=False): + row = bytes([filter_byte]) + (os.urandom(width * 3) if noise else b"\x10" * (width * 3)) + raw = row * height + data = zlib.compress(raw, level) + if corrupt_idat: + mid = len(data) // 2 + data = data[:mid] + bytes(b ^ 0xFF for b in data[mid:mid + 40]) + data[mid + 40:] + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", data) + chunk(b"IEND", b"") + + +def main(): + c = Client() + alice, bob, eve, chat = c.bootstrap() + path = "/api/v1/chats/%s/attachments?filename=" % chat["id"] + + def up(body, ctype, expected, name="a.png", **kw): + return c.call("POST", path + quote(name, safe=""), expected, actor=alice, raw=body, + headers={"Content-Type": ctype}, schema="Attachment" if expected == 201 else None) + + up(SAMPLE[:33], "image/png", 415) + up(png(1, 1, filter_byte=5), "image/png", 415) + up(png(1, 1, filter_byte=0xFF), "image/png", 415) + up(png(64, 64, corrupt_idat=True), "image/png", 415) + up(JPEG_STUB, "image/jpeg", 415) + print("ok truncated PNG, bad filter byte, broken IDAT, 27-byte JPEG stub -> 415") + real = subprocess.run( + ["docker", "run", "--rm", "--entrypoint", "python", "mtusi-messenger:lab4", "-c", + "import io,os,sys;from PIL import Image;b=io.BytesIO();" + "Image.frombytes('RGB',(1000,750),os.urandom(2250000)).save(b,'JPEG',quality=90);" + "sys.stdout.buffer.write(b.getvalue())"], capture_output=True, check=True).stdout + whole = up(real, "image/jpeg", 201, name="real.jpg")["json"] + check(c.call("GET", "/api/v1/attachments/" + whole["id"] + "/content", actor=bob)["bytes"] == real, "JPEG bytes differ") + up(real[: len(real) // 2] + b"\xff\xd9", "image/jpeg", 415, name="cut.jpg") + up(real[: len(real) // 2], "image/jpeg", 415, name="cut.jpg") + print("ok real JPEG accepted; first half + EOI and first half alone -> 415") + up(png(3000, 2000, filter_byte=7), "image/png", 415) + up(png(3000, 2000, corrupt_idat=True, noise=True), "image/png", 415) + print("ok same defects in images above the full-decode threshold (6 Mpx) -> 415") + up(png(3000, 2000), "image/png", 201) + print("ok valid 6 Mpx PNG accepted") + import time + big = png(6000, 6000, level=9) + started = time.monotonic() + r = up(big, "image/png", 201, name="36mpx.png") + took = time.monotonic() - started + check(took < 15, "36 Mpx PNG must be checked quickly, took %.1fs" % took) + print("ok valid 36 Mpx PNG (%d KiB on the wire, 108 MiB decoded) accepted in %.1fs" % (len(big) // 1024, took)) + up(png(6000, 6000, filter_byte=9, level=9), "image/png", 415) + print("ok 36 Mpx PNG with bad filter byte -> 415") + + for name in (" ", "\t", "bad\x7f.png", "bad\x85.png", ".", "..", "a
b.png"): + up(SAMPLE, "image/png", 400, name=name) + print("ok whitespace-only, DEL, C1 and dot filenames -> 400") + + a = up(SAMPLE, "image/png", 201, name="a.png")["json"] + b = up(SAMPLE, "image/png", 201, name="b.png")["json"] + url = "/api/v1/chats/%s/messages" % chat["id"] + key = uuid.uuid4().hex + first = c.call("POST", url, 201, actor=alice, body={"text": "", "attachment_ids": [b["id"], a["id"]]}, + headers={"Idempotency-Key": key}, schema="Message")["json"] + check(first["attachment_ids"] == [b["id"], a["id"]], "POST keeps attachment order") + replay = c.call("POST", url, 201, actor=alice, body={"attachment_ids": [b["id"], a["id"]], "text": ""}, + headers={"Idempotency-Key": key}, schema="Message")["json"] + check(replay == first, "replay returns the same message") + c.call("POST", url, 409, actor=alice, body={"text": "", "attachment_ids": [a["id"], b["id"]]}, + headers={"Idempotency-Key": key}) + history = c.call("GET", url, actor=bob, schema="MessagePage")["json"]["items"] + check([m for m in history if m["id"] == first["id"]][0]["attachment_ids"] == [b["id"], a["id"]], + "history keeps attachment order") + print("ok attachment order preserved in POST, replay and history; reordered body -> 409") + + key2 = uuid.uuid4().hex + plain = c.call("POST", url, 201, actor=alice, body={"text": "x"}, headers={"Idempotency-Key": key2}, schema="Message")["json"] + same = c.call("POST", url, 201, actor=alice, body={"text": "x", "attachment_ids": []}, + headers={"Idempotency-Key": key2}, schema="Message")["json"] + check(plain == same, "absent attachment_ids equals []") + for bad in ("a b", "a/b", "k" * 129, ""): + c.call("POST", url, 400, actor=alice, body={"text": "k"}, headers={"Idempotency-Key": bad}) + other = c.call("POST", url, 201, actor=bob, body={"text": "x"}, headers={"Idempotency-Key": key2}, schema="Message")["json"] + check(other["id"] != plain["id"] and other["sender_id"] == bob["id"], "idempotency key is per user") + print("ok idempotency: [] == absent, invalid keys -> 400, scoped per user") + + huge = b"\x89PNG\r\n\x1a\n" + b"\0" * (12 * 1024 * 1024) + r = c.call("POST", path + "big.png", 413, actor=alice, raw=huge, headers={"Content-Type": "image/png"}) + check(r["headers"].get_content_type() == "application/json", "413 above the nginx limit is API-shaped JSON") + print("ok 12 MiB body -> JSON 413") + print("PASS: input hardening") + + +run(main) diff --git a/student-tests/test_lb_stress.py b/student-tests/test_lb_stress.py new file mode 100644 index 00000000..fa605d9f --- /dev/null +++ b/student-tests/test_lb_stress.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""LB under upload pressure: no OOM/restart of any container, no HTML 502, every rejection is API-shaped JSON. + python3 student-tests/test_lb_stress.py --concurrency 10 --rounds 3 --oversize 200""" +import argparse +import base64 +import collections +import http.client +import json +import os +import struct +import subprocess +import sys +import threading +import uuid +import zlib +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +HOST, PORT = "localhost", 8080 +LIMIT = 10485760 + + +def noise_png(side): + raw = b"".join(b"\0" + os.urandom(side * 3) for _ in range(side)) + + def chunk(kind, body): + return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF) + + ihdr = struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"") + + +def request(method, path, body=None, headers=None, timeout=90): + """Returns (status, content_type, payload) or ('EXC:', '', b'').""" + conn = http.client.HTTPConnection(HOST, PORT, timeout=timeout) + try: + conn.request(method, path, body=body, headers=headers or {}) + response = conn.getresponse() + return response.status, response.getheader("Content-Type", ""), response.read() + except Exception as exc: + return "EXC:" + type(exc).__name__, "", b"" + finally: + conn.close() + + +def actor(label): + username = "st_%s_%s" % (label, uuid.uuid4().hex[:10]) + password = "T3st-" + uuid.uuid4().hex + body = {"username": username, "display_name": label, "password": password} + status, _, _ = request("POST", "/api/v1/users", json.dumps(body), {"Content-Type": "application/json"}) + assert status == 201, status + basic = base64.b64encode((username + ":" + password).encode()).decode() + status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic}) + assert status == 201, status + session = json.loads(data) + return session["user"]["id"], {"Authorization": "Bearer " + session["token"]} + + +def container_ids(): + out = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split() + names = {} + for cid in out: + name = subprocess.run(["docker", "inspect", "--format", "{{.Name}}", cid], capture_output=True, text=True).stdout + names[cid] = name.strip().lstrip("/") + return names + + +def peak_mib(name): + out = subprocess.run(["docker", "exec", name, "cat", "/sys/fs/cgroup/memory.peak"], capture_output=True, text=True) + return round(int(out.stdout) / 1048576, 1) if out.stdout.strip().isdigit() else None + + +class Events(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.lines = [] + self.proc = subprocess.Popen( + ["docker", "events", "--filter", "type=container", "--filter", "event=oom", "--filter", "event=die", + "--filter", "event=restart", "--format", "{{.Action}} {{.Actor.Attributes.name}} exit={{.Actor.Attributes.exitCode}}"], + stdout=subprocess.PIPE, text=True, + ) + + def run(self): + for line in self.proc.stdout: + self.lines.append(line.strip()) + + def stop(self): + self.proc.terminate() + + +def is_json_error(result, code): + status, ctype, data = result + if status != code or not ctype.startswith("application/json"): + return False + return json.loads(data)["error"]["code"] + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=int, default=10) + parser.add_argument("--rounds", type=int, default=3) + parser.add_argument("--oversize", type=int, default=200) + args = parser.parse_args() + # memory.peak is cumulative since container start, so read it after a fresh recreate. + + failures = [] + events = Events() + events.start() + uid, auth = actor("a") + _, outsider = actor("b") + status, _, data = request("POST", "/api/v1/chats", json.dumps({"title": "stress", "member_ids": [uid]}), + {**auth, "Content-Type": "application/json"}) + assert status == 201, status + chat = json.loads(data)["id"] + upload = "/api/v1/chats/%s/attachments?filename=s.png" % chat + png_headers = {**auth, "Content-Type": "image/png"} + + body = noise_png(1700) + print("== %d parallel uploads of %d bytes, %d rounds ==" % (args.concurrency, len(body), args.rounds)) + ids, tally = [], collections.Counter() + for _ in range(args.rounds): + with ThreadPoolExecutor(args.concurrency) as pool: + results = list(pool.map(lambda _: request("POST", upload, body, png_headers), range(args.concurrency))) + for status, _, data in results: + tally[status] += 1 + if status == 201: + ids.append(json.loads(data)["id"]) + print("upload statuses:", dict(tally)) + if set(tally) != {201}: + failures.append("parallel uploads: %s" % dict(tally)) + + print("== %d parallel downloads ==" % args.concurrency) + with ThreadPoolExecutor(args.concurrency) as pool: + results = list(pool.map(lambda i: request("GET", "/api/v1/attachments/%s/content" % i, headers=auth), + ids[:args.concurrency * 2])) + tally = collections.Counter(r[0] for r in results) + print("download statuses:", dict(tally)) + if set(tally) != {200} or any(r[2] != body for r in results): + failures.append("parallel downloads: %s" % dict(tally)) + + print("== %d parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), %d rounds ==" % (args.concurrency, args.rounds)) + jpeg = subprocess.run( + ["docker", "run", "--rm", "--entrypoint", "python", "mtusi-messenger:lab4", "-c", + "import io,sys;from PIL import Image;b=io.BytesIO();" + "Image.new('CMYK',(8000,5000),(120,30,200,5)).save(b,'JPEG',quality=90,progressive=True);" + "sys.stdout.buffer.write(b.getvalue())"], capture_output=True, check=True).stdout + tally = collections.Counter() + for _ in range(args.rounds): + with ThreadPoolExecutor(args.concurrency) as pool: + results = list(pool.map(lambda _: request("POST", upload, jpeg, {**auth, "Content-Type": "image/jpeg"}), + range(args.concurrency))) + tally.update(r[0] for r in results) + print("upload statuses:", dict(tally)) + if set(tally) != {201}: + failures.append("parallel progressive JPEG: %s" % dict(tally)) + + cases = [ + ("10 MiB + 1 byte, Content-Length", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (LIMIT - 7), png_headers, 413, "payload_too_large"), + ("10 MiB + 1 byte, chunked", upload, iter([b"\x89PNG\r\n\x1a\n" + b"\0" * (LIMIT - 7)]), png_headers, 413, "payload_too_large"), + ("10 MiB + 1 byte, chunked in 64 KiB pieces", upload, + [b"\x89PNG\r\n\x1a\n"] + [b"\0" * 65536] * 160, png_headers, 413, "payload_too_large"), + ("8 MiB to a foreign chat", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (8 << 20), {**outsider, "Content-Type": "image/png"}, 404, "not_found"), + ("8 MiB without session", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (8 << 20), {"Content-Type": "image/png"}, 401, "unauthorized"), + ("8 MiB with wrong magic", upload, b"X" * (8 << 20), png_headers, 415, "unsupported_media_type"), + ("8 MiB with wrong Content-Type", upload, b"X" * (8 << 20), {**auth, "Content-Type": "image/gif"}, 415, "unsupported_media_type"), + ("70 KiB JSON message", "/api/v1/chats/%s/messages" % chat, b'{"text":"' + b"x" * (70 << 10) + b'"}', + {**auth, "Content-Type": "application/json", "Idempotency-Key": "k"}, 413, "payload_too_large"), + ] + for name, path, payload, headers, code, error in cases: + print("== %s: %d requests, %d at a time ==" % (name, args.oversize, args.concurrency)) + materialized = list(payload) if not isinstance(payload, bytes) else None + + def one(_): + data = payload if materialized is None else iter(materialized) + extra = {} if materialized is None else {"Transfer-Encoding": "chunked"} + conn_headers = {**headers, **extra} + conn = http.client.HTTPConnection(HOST, PORT, timeout=90) + try: + conn.request("POST", path, body=data, headers=conn_headers, encode_chunked=materialized is not None) + response = conn.getresponse() + return response.status, response.getheader("Content-Type", ""), response.read() + except Exception as exc: + return "EXC:" + type(exc).__name__, "", b"" + finally: + conn.close() + + with ThreadPoolExecutor(args.concurrency) as pool: + results = list(pool.map(one, range(args.oversize))) + tally = collections.Counter((r[0], r[1].split(";")[0]) for r in results) + print(" ", {"%s %s" % k: v for k, v in tally.items()}) + if not all(is_json_error(r, code) == error for r in results): + failures.append("%s: %s" % (name, dict(tally))) + + status, _, _ = request("GET", "/health/ready") + if status != 200: + failures.append("not ready after the run: %s" % status) + events.stop() + bad = [line for line in events.lines if " mtusi-lab4-" in line] + print("container events during the run:", bad or "none") + if bad: + failures.append("container events: %s" % bad) + print("cgroup memory.peak, MiB:", {n: peak_mib(n) for n in container_ids().values() if "api-" in n or "nginx" in n}) + for failure in failures: + print("FAIL:", failure) + print("PASS: LB stress" if not failures else "FAILED: %d" % len(failures)) + sys.exit(1 if failures else 0) + + +main() diff --git a/student-tests/test_s3_outage.py b/student-tests/test_s3_outage.py new file mode 100644 index 00000000..de86c473 --- /dev/null +++ b/student-tests/test_s3_outage.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""S3 down: upload/download answer 503 (never 500), no partial rows are exposed, service recovers.""" +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check, run # noqa: E402 + + +def compose(*args): + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True) + + +def main(): + c = Client() + alice, bob, _, chat = c.bootstrap() + att, content = c.attachment(alice, chat) + path = "/api/v1/chats/%s/attachments?filename=x.png" % chat["id"] + compose("stop", "-t", "2", "minio") + try: + time.sleep(4) + c.call("POST", path, 503, actor=alice, raw=content, headers={"Content-Type": "image/png"}) + c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 503, actor=bob) + c.call("GET", "/health/ready", 503) + c.call("GET", "/health/ready", 503) + c.message(alice, chat, "текст без S3") + c.call("GET", "/health/live", schema="Health") + c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment") + c.call("GET", "/api/v1/chats/%s/messages" % chat["id"], actor=bob, schema="MessagePage") + print("ok S3 down: upload/download/ready -> 503 JSON, metadata and text messages still served through the LB (no 502)") + finally: + compose("start", "minio") + got = None + started = time.monotonic() + while time.monotonic() - started < 90: + got = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", [200, 503], actor=bob) + if got["status"] == 200: + break + time.sleep(2) + check(got["status"] == 200, "download did not recover after S3 came back") + print("ok recovered %.0fs after S3 start" % (time.monotonic() - started)) + check(got["bytes"] == content, "original intact after S3 outage") + streak = 0 + for _ in range(60): + streak = streak + 1 if c.call("GET", "/health/ready", [200, 503])["status"] == 200 else 0 + if streak >= 6: + break + time.sleep(0.5) + check(streak >= 6, "both replicas must report ready again") + print("ok ready is 200 again") + + compose("pause", "minio") + try: + started = time.monotonic() + c.timeout = 40 + c.call("POST", path, 503, actor=alice, raw=content, headers={"Content-Type": "image/png"}) + took = time.monotonic() - started + check(took < 20, "upload to a hung S3 must fail fast, took %.1fs" % took) + c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 503, actor=bob) + print("ok hung S3: upload answered 503 in %.1fs" % took) + finally: + compose("unpause", "minio") + time.sleep(3) + c.call("POST", path, 201, actor=alice, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment") + print("PASS: S3 outage behaviour") + + +run(main)