ЛР 4 — БВТ2301 — Курлова — на 5 #49

Closed
alenccha wants to merge 3 commits from alenccha/mtusi2026lab4:solution/bvt2301-kurlova into main
53 changed files with 3705 additions and 25 deletions
+39 -9
View File
@@ -1,20 +1,50 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
# Copy to .env for your LOCAL synthetic test environment; never commit .env. Values below are the ones the evidence was produced with.
# Secrets are NOT here: they are files in ./secrets (see REPORT.md), referenced by *_FILE paths.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
GRACEFUL_TIMEOUT=8
API_REPLICAS=2
# Sessions / idempotency
SESSION_TTL_SECONDS=3600
SESSION_STORE=valkey
SESSION_CACHE_TTL_SECONDS=30
SESSION_REVOKED_TTL_SECONDS=60
SESSION_SWEEP_SECONDS=60
CACHE_TIMEOUT=0.2
IDEMPOTENCY_TTL_SECONDS=86400
S3_ENDPOINT=http://s3:9000
# Database
DATABASE_URL_FILE=/run/secrets/database_url
CACHE_URL_FILE=/run/secrets/cache_url
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_POOL_TIMEOUT=3
DB_CONNECT_TIMEOUT=3
DB_STATEMENT_TIMEOUT_MS=5000
DB_OP_TIMEOUT=6
DB_READY_TIMEOUT=1.5
READY_CHECK_SECONDS=1
READY_FAILURES=2
# Request limits (REQUEST_TIMEOUT must exceed S3 timeouts + upload time)
MAX_INFLIGHT_REQUESTS=64
MAX_BODY_BYTES=65536
MAX_RSS_MB=0
REQUEST_TIMEOUT=30
# Object storage and attachments
S3_ENDPOINT=http://minio:9000
S3_REGION=us-east-1
S3_BUCKET=messenger-attachments
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
S3_CONNECT_TIMEOUT=3
S3_READ_TIMEOUT=6
MAX_UPLOAD_BYTES=10485760
MAX_IMAGE_PIXELS=40000000
UPLOAD_SPOOL_BYTES=1048576
UPLOAD_CONCURRENCY=2
RECONCILE_INTERVAL_SECONDS=300
RECONCILE_GRACE_SECONDS=900
+35
View File
@@ -0,0 +1,35 @@
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1 \
PIP_DEFAULT_TIMEOUT=120 \
PIP_RETRIES=10
RUN apt-get update \
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-deps -r requirements.txt
COPY app/ app/
COPY migrations/ migrations/
COPY scripts/migrate.py scripts/reconcile.py scripts/reconcile_experiment.py scripts/jpeg_check_experiment.py scripts/
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"]
STOPSIGNAL SIGTERM
CMD ["sh", "-c", "exec uvicorn app.main:app --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""]
+125 -16
View File
@@ -1,37 +1,146 @@
# Отчёт — лабораторная 4
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Алёна Курлова, БВТ2301
- Целевая оценка; выбранный трек 5: **5, трек 2 «Согласованность двух хранилищ»**
- Commit/PR: PR [#49](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/49), ветка `solution/bvt2301-kurlova`
- Источники, библиотеки, помощь ИИ: FastAPI, psycopg, boto3, Pillow, MinIO; код лабы 3 перенесён и расширен. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора; результаты прогонов ниже получены на локальном стенде.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Prerequisites: Docker Desktop (Compose v2), Python 3.9+, вход `docker login` (Docker Hub). Проверено на macOS, arm64.
```sh
cp .env.example .env # значения в примере совпадают с теми, на которых получены evidence/
mkdir -p secrets
printf 'messenger' > secrets/postgres_password
printf 'postgresql://messenger:messenger@postgres:5432/messenger' > secrets/database_url
printf 'valkey://valkey:6379/0' > secrets/cache_url
printf 'messenger_root' > secrets/minio_root_user
python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/minio_root_password
printf 'messenger_app' > secrets/s3_access_key_id
python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/s3_secret_access_key
docker compose --env-file .env up -d --build # postgres, valkey, minio, minio-init, migrate, api-1, api-2, nginx
make test # ждёт готовности по /health/ready
```
Остановка с сохранением данных: `docker compose stop`. Намеренное удаление учебных данных: `docker compose down -v`. Каталог `secrets/` и `.env` в git не попадают.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
| Роль | Сервис | Версия | Health | Volume | Сеть / порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| LB | nginx | 1.27.3-alpine | `/health/live`; health-скрипт опрашивает `/health/ready` реплик | — | `127.0.0.1:8080` | — |
| API ×2 | api-1, api-2 | собираются из исходников, python 3.12.8 | `/health/ready` | нет (stateless) | только внутренняя сеть | Docker secrets |
| БД | postgres | 17.7-alpine@sha256 | `pg_isready` | `pgdata` | внутренняя | secret |
| Кэш сессий | valkey | 8.1-alpine@sha256 | `valkey-cli ping` | нет | внутренняя | — |
| S3 | minio | `madlabcg/minio@sha256:ad134b2d…` | `mc ready local` | `miniodata` | внутренняя, **порты и консоль не опубликованы** (`MINIO_BROWSER=off`) | root-secret только у minio и minio-init |
| Bootstrap S3 | minio-init | тот же образ | — | — | — | root + app secrets |
**Источник истины.** Метаданные — PostgreSQL (`attachments`, `message_attachments`), байты — приватный bucket. Object key назначает сервер (`attachments/<uuid4>`), `filename` хранится только как отображаемое имя.
**Порядок записи и границы согласованности** (ответ на «что если PUT прошёл, а COMMIT нет»):
1. тело читается потоком в `SpooledTemporaryFile` (1 MiB в RAM, остальное во временный файл), считается SHA-256, лимит 10 MiB применяется на лету; MIME проверяется по magic bytes и Pillow, лимит пикселей проверяется по заголовку до декодирования; целостность проверяет [app/imagecheck.py](app/imagecheck.py): для PNG собственный потоковый сканер (CRC всех chunk'ов, поля IHDR, zlib-поток разжимается порциями по 1 MiB, filter-байт каждой строки ∈ 0..4, размер данных точно равен вычисленному по IHDR, Adam7 учитывается), для JPEG дочерний процесс `djpeg -strict` (libjpeg-turbo 2.1.5) декодирует весь поток в масштабе 1/8 и считает любое предупреждение libjpeg ошибкой, поэтому обрезанный поток отвергается и с дописанным EOI; одновременно идёт одна такая проверка на реплику, память процесса ограничена `-maxmemory`, число сканов `-maxscans 100`;
2. в БД вставляется строка `status='uploading'` (коммит);
3. PUT в S3;
4. `UPDATE … SET status='ready' WHERE status='uploading'` — только после этого клиент получает **201**.
Любой сбой между шагами оставляет либо `uploading`-строку (с объектом или без), либо ничего. Клиентам такие строки не видны (404), прикрепить их к сообщению нельзя. Объект без строки получиться не может, потому что строка вставляется раньше. При ошибке API сам пытается компенсировать (удалить объект и строку); то, что не удалось, собирает reconciler.
**Reconciler** ([app/reconcile.py](app/reconcile.py), CLI [scripts/reconcile.py](scripts/reconcile.py), фоновый цикл в каждом API раз в `RECONCILE_INTERVAL_SECONDS`=300 с, grace `RECONCILE_GRACE_SECONDS`=900 с). Работает под `pg_try_advisory_lock`, поэтому две реплики не дублируют работу. Правила:
- `uploading` старше grace **и** объект (если есть) старше grace по `LastModified` → удалить объект и строку; старая строка со свежим объектом не трогается (повторная проверка HEAD перед удалением);
- объект без строки и старше grace → удалить (перед удалением строка перепроверяется);
- `ready`-строка, подтверждённая (`updated_at`) раньше grace, без объекта (перепроверка HEAD) → `status='failed'`, `error_code='object_missing'`; строка **не удаляется**, даже если на неё ссылается сообщение;
- по умолчанию dry-run (`--apply` меняет данные), повторный запуск идемпотентен.
**Что изменено в lab3-коде.** Сообщение принимает `attachment_ids` (0–10) и допускает пустой `text` при наличии вложений (миграция 0004 ослабляет CHECK). Хеш идемпотентности включает `attachment_ids`; повтор без ключа `attachment_ids` равен `[]`. **Критичные зависимости `/health/ready`: PostgreSQL и S3.** Недоступность любой из них даёт 503 в общем формате ошибки с `X-Instance-Id`; состояние S3 обновляется фоновой пробой раз в секунду (отдельный клиент с таймаутами 1 с / 1.5 с, чтобы зависший S3 не вешал проверку). При этом недоступный S3 **не** блокирует остальной API: текстовые сообщения и метаданные продолжают работать (запросы блокируются только при недоступной БД), а вложения отвечают 503. Чтобы 503 не выбивал реплики из upstream и клиенты не видели 502, nginx не использует `http_503` как признак сбоя для `/health/`, `/attachments/` и upload; если обе реплики не готовы, health-скрипт оставляет обе в ротации. **Лимиты тела и память LB.** nginx сам отклоняет слишком большие тела JSON-ошибкой 413 (с `X-Request-Id`): 10 MiB для загрузки вложений и 64 KiB для остальных запросов, то есть те же границы, что в API. Загрузка идёт потоком до API (`proxy_request_buffering off`), скачивание — без временных файлов (`proxy_max_temp_file_size 0`), поэтому тела файлов не попадают в память и page cache контейнера nginx; воркеров 2, а не `auto`, чтобы потребление не зависело от числа ядер хоста. API больше не отправляет `Connection: close` на отказах с недочитанным телом: это закрывало соединение раньше, чем nginx успевал прочитать ответ, и давало HTML 502; uvicorn дочитывает и отбрасывает остаток тела. Лимит памяти контейнера API поднят до 512 MiB, потому что проверка законного прогрессивного CMYK-JPEG на 40 Мпикс требует 307 MiB. S3-клиент: без повторов, connect 3 с, read 6 с; компенсация при сбое PUT выполняется в фоне.
**Дополнительная валидация входных данных.** filename: 1..255 code points, не только пробелы, не `.`/`..`, без `/`, `\` и любых символов категорий Cc/Cs/Zl/Zp (в том числе DEL и C1). `Idempotency-Key` не обрезается (HTTP-парсер уже убирает крайние пробелы); порядок `attachment_ids` сохраняется в ответе, повторе и истории, переставленный список с тем же ключом — 409.
**Права S3.** API использует отдельного пользователя `messenger_app` с политикой [infra/minio/app-policy.json](infra/minio/app-policy.json): Get/Put/DeleteObject только на `attachments/*` и ListBucket. Bucket создаёт и делает приватным [infra/minio/init.sh](infra/minio/init.sh); скрипт идемпотентен (`mb --ignore-existing`, повтор политики и пользователя безвреден), запускается при каждом `up`.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Публичный smoke | `make test` | PASS (загрузка, привязка, скачивание вторым участником, 404 постороннему) | [evidence/smoke.txt](evidence/smoke.txt) |
| Байты в S3, метаданные в БД, 201 после сохранения | код `upload_attachment`; `student-tests/test_attachments.py` | sha256/размер совпадают, `Content-Length` скачивания равен оригиналу | [evidence/attachments_tests.txt](evidence/attachments_tests.txt) |
| Upload → остановка реплики → download через другую; restart с volumes | `student-tests/test_attachment_resilience.py` | загрузил api-1, скачал api-2, sha256 совпал; после рестарта postgres/valkey/minio/api/nginx метаданные и оригинал целы | [evidence/attachment_resilience.txt](evidence/attachment_resilience.txt) |
| Общие persistence / failover | `make test-persistence`, `make test-failover` (хуки `scripts/restart.sh`, `scripts/stop-one.sh`) | PASS; failover: 131 успешная запись, 0 ошибок, восстановление 0.44 с | [evidence/persistence.txt](evidence/persistence.txt), [evidence/failover.txt](evidence/failover.txt) |
| Лимиты 10 MiB / 40 Mpx, MIME по содержимому, опасные filename | `test_attachments.py` | 10 MiB+ → 413; PNG 8000×6000 → 413 до декодирования; подмена MIME и мусор → 415; `../`, `/`, `\`, NUL, `\n`, пустое, 256 символов → 400 | [evidence/attachments_tests.txt](evidence/attachments_tests.txt) |
| Устойчивость к некорректным входным данным | `student-tests/test_input_hardening.py`, `scripts/jpeg_check_experiment.py` (запуск: `docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py`) | усечённый PNG, filter-байт 5/0xFF, битый IDAT, 27-байтный JPEG, настоящий JPEG, обрезанный наполовину с дописанным `FF D9` и без него → 415 (PNG-дефекты также на 6 и 36 Mpx); валидные baseline, progressive, grayscale, CMYK, файл с данными после EOI и 40 Mpx принимаются; filename из пробелов, DEL, C1, `.` → 400; порядок вложений; `[]` ≡ отсутствует; 12 MiB → JSON 413 | [evidence/input_hardening.txt](evidence/input_hardening.txt), [evidence/jpeg_check.txt](evidence/jpeg_check.txt) |
| Чужое / неизвестное / чужого чата attachment_id, ошибки не 500 | `test_attachments.py` | 404 / 404 / 404; дубликаты, не-UUID, >10 → 400 | там же |
| Недоступность и зависание S3 | `student-tests/test_s3_outage.py` | остановка: upload/download/ready → 503 JSON с `X-Instance-Id`, метаданные и текст работают, без 502; `docker pause`: upload отвечает 503 за 7 с; восстановление за секунды | [evidence/s3_outage.txt](evidence/s3_outage.txt) |
| Прерывание загрузки | обрыв сокета на половине тела в `test_attachments.py` | сервер здоров, следующий upload проходит | там же |
| Компенсация / сверка, безопасный grace | трек 2, ниже | 18/18 проверок | [evidence/reconciler_experiment.txt](evidence/reconciler_experiment.txt) |
| Пик памяти при ≥10 одновременных загрузках | `student-tests/memory_experiment.py`, `student-tests/test_lb_stress.py` | см. ниже | [evidence/memory_experiment.txt](evidence/memory_experiment.txt), [evidence/lb_stress.txt](evidence/lb_stress.txt) |
| Нагрузка на LB: нет OOM и падений, отказы в JSON | `python3 student-tests/test_lb_stress.py --concurrency 10 --rounds 3 --oversize 100` и то же с 30 параллельными | загрузки 201, скачивания 200, 800 запросов-отказов дают правильный JSON без единого 502, событий `oom`/`die` контейнеров нет, пик nginx 28–34 MiB при лимите 64 | [evidence/lb_stress.txt](evidence/lb_stress.txt), [evidence/lb_stress_30.txt](evidence/lb_stress_30.txt) |
| Private bucket вне API, консоль не открыта | `student-tests/check_private_bucket.sh` | анонимные GET/LIST из сети compose → 403, порты 9000/9001 на хосте не опубликованы, `mc anonymous get` → private | [evidence/private_bucket.txt](evidence/private_bucket.txt) |
| Отдельные S3 credentials с ограниченными правами | `infra/minio/app-policy.json`, `init.sh` | API работает под `messenger_app`; root-secret API не получает | `compose.yaml` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Проверка hook'ов подтверждает перезапуск реальных процессов: `restart.sh` перезапускает postgres, valkey, minio, api-1, api-2, nginx без `down -v`; `stop-one.sh` останавливает только указанную реплику.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
### 1. Память при 10 одновременных загрузках
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
Гипотеза: основная память уходит на параллельное декодирование Pillow и S3 PUT, а не на хранение тел запросов; ограничение числа одновременных тяжёлых стадий снизит пик.
Методика: [student-tests/memory_experiment.py](student-tests/memory_experiment.py), PNG 8.27 MiB (1700×1700, несжимаемый шум), 10 параллельных загрузок через LB, 3 раунда, замер `docker stats` каждого API (0.5 CPU; лимит контейнера был 256 MiB, в итоговой версии 512 MiB). Mac arm64, Docker Desktop.
| Вариант | Простой, MiB | Пик api-1 / api-2, MiB | Динамика по раундам | 201 / всего |
| --- | --- | --- | --- | --- |
| До: без ограничения тяжёлых стадий | 88 | 182 / 155 | рост 144 → 166 → 182 | 30 / 30 |
| После: `UPLOAD_CONCURRENCY=2` на реплику | 87 | 110 / 116 | 110 → 109 → 110, роста нет | 30 / 30 |
| После + потоковый PNG-сканер (итоговая версия) | см. файл | 116 / 107 | роста нет | 30 / 30 |
**Цифры таблицы занижены.** `docker stats` не учитывает page cache и даёт срезы раз в 1–2 с. Точный показатель лимита — `memory.peak` из cgroup: в этом же сценарии 154–163 MiB, `oom_kill=0`. Худший сценарий — стресс-тест с 10–30 параллельными прогрессивными CMYK-JPEG на 40 Мпикс: `memory.peak` API 444–491 MiB из 512 (вместе с page cache, который ядро может вытеснить), `oom_kill=0`, рестартов нет. Постоянная память процесса API после такой нагрузки около 133 MiB, один процесс проверки до 307 MiB ([evidence/jpeg_check.txt](evidence/jpeg_check.txt)). Запас в худшем случае небольшой.
Сырые данные: [evidence/memory_experiment_before_semaphore.txt](evidence/memory_experiment_before_semaphore.txt), [evidence/memory_experiment.txt](evidence/memory_experiment.txt). Ограничения: одна конфигурация железа; один размер файла; медиана длительности загрузки выросла с 0.82 до 1.15 с (цена ограничения), в итоговой версии 0.63 с. Причину роста «до» (фрагментация аллокатора или реальная утечка) я не выяснял.
### 2. Трек 2 — reconciler с внедрением ошибок
Гипотеза: сверка по правилам выше находит и исправляет расхождения, не трогая свежие и используемые вложения.
Методика: [scripts/reconcile_experiment.py](scripts/reconcile_experiment.py) внутри api-контейнера снимает базовую линию (в БД могут быть остатки прошлых тестов), создаёт строки/объекты, имитирующие сбои на обеих сторонах, и запускает dry-run, apply, apply после истечения grace, повторный apply. Счётчики сравниваются с базовой линией. `LastModified` объекта в MinIO задним числом не изменить, поэтому «старый объект» моделируется ожиданием grace.
| Внедрённая ошибка | Ожидание | Наблюдение |
| --- | --- | --- |
| `uploading` старше grace, объект есть, но моложе grace (падение после PUT) | не трогать до истечения grace | сохранены; собраны после grace |
| `uploading` старше grace, но под его ключом свежий объект | не трогать | не тронуты; собраны после grace |
| `ready`, подтверждённая только что, объект ещё не виден | не помечать missing | не помечена |
| `uploading` старше grace, объекта нет (падение до PUT) | строка удалена | удалена |
| `uploading` свежее grace (загрузка идёт) | не трогать | не тронута; после истечения grace собрана |
| объект без строки внутри grace | не трогать | не тронут; удалён после grace |
| `ready`, объект потерян | пометить `failed/object_missing` | помечено |
| `ready`, привязано к сообщению, объект на месте | не трогать | не тронуто |
| `ready`, привязано к сообщению, объект потерян | строку не удалять, пометить | сохранена, `failed` |
| dry-run | ничего не менять, но показать счётчики | счётчики верны, состояние не изменилось |
| повторный apply | нулевые изменения | 0 / 0 / 0 |
Итог: 18/18 проверок ([evidence/reconciler_experiment.txt](evidence/reconciler_experiment.txt)). **Стоимость обхода:** 1000 `ready`-вложений (около 1100 объектов вместе с остатками прошлых тестов) — 1.25 с в dry-run (в основном 1000 HEAD-запросов к S3; листинг идёт страницами по 500 с одним запросом к БД на страницу). Оценка масштабирования линейна, на больших объёмах не измерялась.
### 3. Отказы
| Воздействие | Ожидаем | Наблюдали | Восстановление / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Остановка обслужившей API-реплики | download через другую реплику, тот же файл | api-1 → api-2, sha256 совпал | без потерь |
| Остановка MinIO | 503 на вложениях и ready, текст работает | upload/download/ready → 503 JSON, сообщения читаются, 502 нет | возврат за секунды после старта, без потерь |
| Зависание MinIO (`docker pause`) | быстрый 503 без вечного ожидания | upload → 503 за 7 с, download → 503 | восстановление после `unpause` |
| Остановка PostgreSQL | 503 JSON с `X-Instance-Id` на ready и бизнес-эндпоинтах | так и есть | возврат после старта |
| Рестарт postgres, valkey, minio, API, nginx с volumes | метаданные и байты на месте | сохранены | без потерь |
| 10 и 30 одновременных загрузок 8.3 MiB через LB | ограниченная обработка без падения стенда | все 201, событий `oom`/`die` нет, пик nginx 28–34 MiB | потерь нет |
| Обрыв клиента на середине upload | нет 500 и мусора | сервер здоров; строка в БД ещё не создана | — |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **Образ MinIO.** Официальные образы `minio/minio` на Docker Hub и `quay.io/minio/minio` в моей среде недоступны (pull denied / 401), поэтому использован сторонний `madlabcg/minio`, закреплённый по digest. Его происхождение и содержимое я не проверял; для защиты нужно заменить образ на проверенный (собрать из исходников MinIO или взять образ, указанный преподавателем).
- **Валидация изображений.** `djpeg -strict` считает ошибкой любое предупреждение libjpeg, поэтому будет отвергнут и JPEG с мелким дефектом, который обычные программы показывают (например, лишние байты перед маркером). Это сознательный выбор в пользу строгости. Подмена байтов внутри данных, не нарушающая структуру потока, может не обнаруживаться: такой файл корректно декодируется в другую картинку. PNG-сканер не проверяет содержимое ancillary chunk'ов, кроме CRC.
- **OOM балансировщика.** Само падение nginx по OOM на моей машине (10 ядер, Docker Desktop) не воспроизвелось, воспроизвелся пик у лимита: 67–80 MiB при 64. Устранена причина (тела файлов через nginx больше не спулятся на диск, число воркеров фиксировано); после правок пик 28–34 MiB. На конфигурации, где наблюдалось падение, правка отдельно не проверялась.
- **Потоковая загрузка через LB.** С `proxy_request_buffering off` медленный клиент занимает слот API на всё время передачи, но не дольше `REQUEST_TIMEOUT` (30 с) и в пределах `MAX_INFLIGHT_REQUESTS`. Ошибка 413, выданная самим nginx, содержит `X-Instance-Id: lb`, потому что запрос до реплики не доходил.
- Проверка размера по `Content-Length` невозможна для chunked-запросов: тогда лимит применяется по факту чтения потока.
- Проверка PNG ограничена по времени (`SCAN_SECONDS`=10 с); на очень медленной машине валидный файл с огромной степенью сжатия может получить 415.
- Вложения, загруженные, но так и не прикреплённые к сообщению, не считаются мусором и не удаляются; политики TTL для них нет.
- Reconciler при проверке `ready`-строк делает HEAD на каждую; для очень больших объёмов нужен инкрементальный режим по курсору или S3 Inventory.
- Нагрузочное тестирование с заданным arrival rate, p95/p99 не выполнялось; измерялась только память.
- Не реализованы: трек 1 (presigned upload), варианты изображений (lab5), TLS (локальный стенд с синтетическими данными).
- Секреты в `secrets/` сгенерированы локально и не коммитятся; учётные данные Postgres в примере выше — учебные.
View File
+64
View File
@@ -0,0 +1,64 @@
import os
import socket
def _int(name, default):
value = os.environ.get(name)
return int(value) if value not in (None, "") else default
def _float(name, default):
value = os.environ.get(name)
return float(value) if value not in (None, "") else default
def _secret(name, default=""):
path = os.environ.get(name + "_FILE")
if path:
with open(path, encoding="utf-8") as handle:
return handle.read().strip()
return os.environ.get(name, default)
DATABASE_URL = _secret("DATABASE_URL")
CACHE_URL = _secret("CACHE_URL")
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
READY_FAILURES = _int("READY_FAILURES", 2)
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
PASSWORD_MIN = 12
PASSWORD_MAX = 128
SESSION_REVOKED_TTL_SECONDS = max(
_int("SESSION_REVOKED_TTL_SECONDS", 60),
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
)
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID")
S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY")
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760)
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000)
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0)
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0)
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576)
RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300)
RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900)
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2)
+43
View File
@@ -0,0 +1,43 @@
import base64
import json
import re
import uuid
from datetime import datetime, timezone
TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
class BadCursor(Exception):
pass
def fmt(ts: datetime) -> str:
return ts.astimezone(timezone.utc).strftime(TS_FORMAT)
def encode_cursor(scope: str, created_at: str, item_id: str) -> str:
raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
def decode_cursor(scope: str, cursor: str):
try:
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
data = json.loads(raw)
except Exception:
raise BadCursor() from None
if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope:
raise BadCursor()
ts, item_id = data["t"], data["i"]
if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str):
raise BadCursor()
try:
parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc)
if str(uuid.UUID(item_id)) != item_id:
raise BadCursor()
except ValueError:
raise BadCursor() from None
if encode_cursor(scope, ts, item_id) != cursor:
raise BadCursor()
return parsed, item_id
+99
View File
@@ -0,0 +1,99 @@
import asyncio
from contextlib import asynccontextmanager
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from app import config
from app.errors import unavailable
BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout)
async def discard(conn) -> None:
try:
await conn.close()
except Exception:
pass
class Guarded:
def __init__(self, conn, timeout: float):
self.conn = conn
self.timeout = timeout
async def execute(self, *args, **kwargs):
try:
return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout)
except asyncio.TimeoutError:
await discard(self.conn)
raise unavailable("Хранилище не ответило за отведённое время") from None
def transaction(self):
return self.conn.transaction()
class Database:
def __init__(self) -> None:
options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % (
config.DB_STATEMENT_TIMEOUT_MS,
config.DB_STATEMENT_TIMEOUT_MS,
config.DB_STATEMENT_TIMEOUT_MS * 2,
)
self.pool = AsyncConnectionPool(
conninfo=config.DATABASE_URL,
min_size=config.DB_POOL_MIN,
max_size=config.DB_POOL_MAX,
timeout=config.DB_POOL_TIMEOUT,
max_lifetime=600.0,
open=False,
kwargs={
"autocommit": True,
"row_factory": dict_row,
"connect_timeout": config.DB_CONNECT_TIMEOUT,
"options": options,
},
)
async def open(self) -> None:
await self.pool.open(wait=False)
async def close(self) -> None:
await self.pool.close()
@asynccontextmanager
async def connection(self):
try:
async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn:
yield Guarded(conn, config.DB_OP_TIMEOUT)
except BROKEN:
raise unavailable("Хранилище недоступно") from None
@asynccontextmanager
async def transaction(self):
async with self.connection() as conn:
async with conn.transaction():
yield conn
async def in_transaction(self, action):
async def run():
async with self.transaction() as conn:
return await action(conn)
try:
return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT)
except asyncio.TimeoutError:
raise unavailable("Хранилище не ответило за отведённое время") from None
async def ready(self) -> bool:
try:
async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn:
try:
await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT)
except asyncio.TimeoutError:
await discard(conn)
return False
return True
except Exception:
return False
+37
View File
@@ -0,0 +1,37 @@
from typing import Optional
from fastapi.responses import JSONResponse
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None):
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
def bad(message: str) -> ApiError:
return ApiError(400, "invalid_request", message)
def unauthorized(scheme: str = "Bearer") -> ApiError:
challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE
message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия"
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
def not_found() -> ApiError:
return ApiError(404, "not_found", "Объект не найден")
def unavailable(message: str) -> ApiError:
return ApiError(503, "unavailable", message, {"Retry-After": "1"})
def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": rid}},
headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})},
)
+213
View File
@@ -0,0 +1,213 @@
"""Integrity checks for uploaded PNG/JPEG with bounded memory.
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy
stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is
only a warning. Both finish before anything is stored.
"""
import logging
import os
import struct
import subprocess
import threading
import time
import zlib
from PIL import Image
from app import config
from app.errors import ApiError, unavailable
log = logging.getLogger("messenger.imagecheck")
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
Image.MAX_IMAGE_PIXELS = None
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
FULL_DECODE_MAX_PIXELS = 4000000
SCAN_SECONDS = 10.0
OUT_STEP = 1 << 20
# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit.
DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null")
jpeg_slot = threading.Lock()
class Invalid(Exception):
pass
def invalid() -> ApiError:
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
class RowPlan:
def __init__(self, segments):
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
self.seg = 0
self.done = 0
self.pos = 0
def feed(self, data: bytes) -> None:
i, n = 0, len(data)
while i < n:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
if self.seg >= len(self.segments):
raise Invalid("data after the last row")
length, rows = self.segments[self.seg]
if self.pos > 0:
take = min(length - self.pos, n - i)
i += take
self.pos += take
if self.pos == length:
self.pos = 0
self.done += 1
continue
full = min((n - i) // length, rows - self.done)
if full > 0:
if max(data[i:i + full * length:length]) > 4:
raise Invalid("bad filter type")
i += full * length
self.done += full
else:
if data[i] > 4:
raise Invalid("bad filter type")
self.pos = n - i
i = n
def complete(self) -> bool:
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
self.seg += 1
self.done = 0
return self.seg >= len(self.segments) and self.pos == 0
def png_plan(width, height, depth, color, interlace):
bits = CHANNELS[color] * depth
if not interlace:
return RowPlan([(1 + (width * bits + 7) // 8, height)])
segments = []
for x0, y0, dx, dy in ADAM7:
w = (width - x0 + dx - 1) // dx if width > x0 else 0
h = (height - y0 + dy - 1) // dy if height > y0 else 0
if w and h:
segments.append((1 + (w * bits + 7) // 8, h))
return RowPlan(segments)
def scan_png(fileobj) -> None:
deadline = time.monotonic() + SCAN_SECONDS
fileobj.seek(0)
if fileobj.read(8) != PNG_SIGNATURE:
raise Invalid("signature")
plan = decomp = None
palette = seen_idat = idat_closed = False
color = None
first = True
while True:
if time.monotonic() > deadline:
raise Invalid("scan too slow")
head = fileobj.read(8)
if len(head) < 8:
raise Invalid("truncated before IEND")
length, kind = struct.unpack(">I4s", head)
if length > 0x7FFFFFFF:
raise Invalid("chunk length")
if first and kind != b"IHDR":
raise Invalid("IHDR must be first")
crc = zlib.crc32(kind)
if kind == b"IHDR":
body = fileobj.read(length)
if not first or length != 13 or len(body) != 13:
raise Invalid("IHDR")
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
raise Invalid("IHDR fields")
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
raise Invalid("color type / depth")
plan = png_plan(width, height, depth, color, interlace)
decomp = zlib.decompressobj()
crc = zlib.crc32(body, crc)
else:
remaining = length
while remaining:
buf = fileobj.read(min(remaining, 65536))
if not buf:
raise Invalid("truncated chunk")
remaining -= len(buf)
crc = zlib.crc32(buf, crc)
if kind == b"IDAT":
if idat_closed or (color == 3 and not palette):
raise Invalid("IDAT order")
seen_idat = True
out = decomp.decompress(buf, OUT_STEP)
while True:
plan.feed(out)
if not decomp.unconsumed_tail:
break
if time.monotonic() > deadline:
raise Invalid("scan too slow")
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
if kind == b"PLTE":
palette = True
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
raise Invalid("PLTE")
elif kind != b"IDAT" and seen_idat:
idat_closed = True
tail = fileobj.read(4)
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
raise Invalid("crc")
first = False
if kind == b"IEND":
if length or not seen_idat:
raise Invalid("IEND")
break
if not decomp.eof or decomp.unused_data or not plan.complete():
raise Invalid("pixel data does not match IHDR")
def check_jpeg(fileobj) -> None:
fileobj.flush()
descriptor = fileobj.fileno()
os.lseek(descriptor, 0, os.SEEK_SET)
try:
with jpeg_slot:
done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
timeout=SCAN_SECONDS, check=False)
except subprocess.TimeoutExpired:
log.error("jpeg_check_timeout")
raise unavailable("Проверка изображения не успела завершиться") from None
# A signal means the checker died (e.g. OOM killer), which says nothing about the file.
if done.returncode < 0:
log.error("jpeg_check_killed signal=%d", -done.returncode)
raise unavailable("Проверка изображения прервана")
if done.returncode != 0:
raise invalid()
def inspect_image(fileobj, size: int, content_type: str) -> None:
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
try:
fileobj.seek(0)
img = Image.open(fileobj)
width, height = img.size
if img.format != FORMATS[content_type]:
raise invalid()
if width * height > config.MAX_IMAGE_PIXELS:
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
if content_type == "image/png":
scan_png(fileobj)
if width * height <= FULL_DECODE_MAX_PIXELS:
fileobj.seek(0)
Image.open(fileobj).load()
else:
check_jpeg(fileobj)
except ApiError:
raise
except Exception:
raise invalid() from None
finally:
fileobj.seek(0)
+640
View File
@@ -0,0 +1,640 @@
import asyncio
import base64
import binascii
import hashlib
import json
import logging
import os
import re
import resource
import signal
import tempfile
import unicodedata
import sys
import uuid
from contextlib import asynccontextmanager
from typing import List, Optional, Tuple
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
from starlette.responses import StreamingResponse
from app import config, passwords, reconcile, repo
from app.cursor import BadCursor, decode_cursor, encode_cursor
from app.db import Database
from app.imagecheck import inspect_image
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
from app.s3 import S3Storage
from app.sessions import SessionStore
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
log = logging.getLogger("messenger")
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"})
EXEMPT_PATHS = ("/health/live", "/health/ready")
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
JPEG_MAGIC = b"\xff\xd8\xff"
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
background = set()
db = Database()
sessions = SessionStore(db)
s3 = S3Storage()
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}}
def rss_mb() -> float:
try:
with open("/proc/self/statm") as handle:
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
except OSError:
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
def count_rejection(code: str) -> None:
state["rejected"][code] = state["rejected"].get(code, 0) + 1
async def sweeper() -> None:
while True:
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
try:
removed = await sessions.sweep()
if removed:
log.info("sessions_expired removed=%d", removed)
except Exception:
log.warning("session sweep skipped: storage unavailable")
try:
async with db.connection() as conn:
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
if removed:
log.info("idempotency_cleanup removed=%d", removed)
except Exception:
log.warning("idempotency cleanup skipped")
async def reconciler_loop() -> None:
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
while True:
try:
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
log.info("reconcile %s", result)
except Exception:
log.warning("reconcile skipped: dependency unavailable")
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
async def readiness_watch() -> None:
failures = 0
while True:
if await db.ready():
failures = 0
if not state["ready"]:
state["ready"] = True
log.info("ready again: database reachable")
else:
failures += 1
if failures >= config.READY_FAILURES and state["ready"]:
state["ready"] = False
log.warning("not ready: database unreachable, business requests get 503")
s3_ok = await run_in_threadpool(s3.probe)
if s3_ok != state["s3_ok"]:
state["s3_ok"] = s3_ok
log.info("object storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
await asyncio.sleep(config.READY_CHECK_SECONDS)
@asynccontextmanager
async def lifespan(_app: FastAPI):
previous = {}
def on_signal(signum, frame):
if not state["draining"]:
state["draining"] = True
log.info("signal %s: draining, new requests get 503", signal.Signals(signum).name)
handler = previous.get(signum)
if callable(handler):
handler(signum, frame)
for sig in (signal.SIGTERM, signal.SIGINT):
previous[sig] = signal.getsignal(sig)
signal.signal(sig, on_signal)
await db.open()
await sessions.open()
await run_in_threadpool(s3.open)
state["s3_ok"] = await run_in_threadpool(s3.probe)
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop())]
log.info(
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
)
yield
for task in tasks:
task.cancel()
await sessions.close()
await db.close()
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
@app.middleware("http")
async def guard(request: Request, call_next):
rid = uuid.uuid4().hex
request.state.rid = rid
exempt = request.url.path in EXEMPT_PATHS
if state["draining"] and request.url.path != "/health/live":
count_rejection("draining")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
if not exempt and not state["ready"]:
count_rejection("not_ready")
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
count_rejection("inflight")
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
state["inflight"] += 1
try:
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
except asyncio.TimeoutError:
count_rejection("request_timeout")
log.warning("request timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path)
response = error_body(
rid, config.INSTANCE_ID, 503, "unavailable",
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
)
except Exception:
log.exception("unhandled error request_id=%s", rid)
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
finally:
state["inflight"] -= 1
response.headers["X-Request-Id"] = rid
response.headers["X-Instance-Id"] = config.INSTANCE_ID
return response
@app.exception_handler(ApiError)
async def api_error_handler(request: Request, exc: ApiError):
if exc.status in (413, 429, 503):
count_rejection(exc.code)
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
def parse_uuid(value: object) -> Optional[str]:
if not isinstance(value, str):
return None
try:
return str(uuid.UUID(value))
except ValueError:
return None
def non_blank(value: object, max_len: int) -> bool:
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
return False
return value.translate(ZERO_WIDTH).strip() != ""
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое")
declared = request.headers.get("content-length")
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
raise too_large
raw = bytearray()
async for chunk in request.stream():
raw += chunk
if len(raw) > config.MAX_BODY_BYTES:
raise too_large
try:
body = json.loads(bytes(raw).decode("utf-8"))
except (UnicodeDecodeError, ValueError, RecursionError):
raise bad("Некорректный JSON") from None
if not isinstance(body, dict):
raise bad("Ожидается JSON-объект")
allowed = set(required) | set(optional)
if set(body) - allowed:
raise bad("Лишние поля в запросе")
if set(required) - set(body):
raise bad("Отсутствуют обязательные поля")
return body
def ensure_capacity() -> None:
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
def bearer_token(request: Request) -> str:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
token = value.strip()
if scheme.lower() != "bearer" or not token:
raise unauthorized()
return token
async def actor(request: Request) -> dict:
user = await sessions.resolve(bearer_token(request))
if user is None:
raise unauthorized()
return user
def basic_credentials(request: Request) -> Tuple[str, str]:
scheme, _, value = request.headers.get("authorization", "").partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized("Basic")
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized("Basic") from None
username, separator, password = decoded.partition(":")
if not separator:
raise unauthorized("Basic")
return username, password
async def member_chat(user: dict, chat_id: str) -> dict:
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
async with db.connection() as conn:
chat = await repo.chat_for_member(conn, cid, user["id"])
if chat is None:
raise not_found()
return chat
def page_params(request: Request) -> Tuple[int, Optional[str]]:
query = request.query_params
limit = 50
if "limit" in query:
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
raise bad("limit должен быть целым 1..100")
limit = int(query["limit"])
cursor = query.get("cursor")
if cursor is not None and cursor == "":
raise bad("Некорректный cursor")
return limit, cursor
def keyset(scope: str, cursor: Optional[str]):
if cursor is None:
return None
try:
return decode_cursor(scope, cursor)
except BadCursor:
raise bad("Некорректный cursor") from None
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
return JSONResponse({"items": items, "next_cursor": next_cursor})
def detect_content_type(head: bytes) -> Optional[str]:
if head[:8] == PNG_MAGIC:
return "image/png"
if head[:3] == JPEG_MAGIC:
return "image/jpeg"
return None
def validate_filename(filename: str) -> None:
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
raise bad("filename: 1..255 символов, не только пробелы")
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
raise bad("filename содержит недопустимые символы")
@app.get("/health/live")
async def live():
return {"status": "ok"}
@app.get("/health/ready")
async def ready():
if not await db.ready():
raise unavailable("База данных недоступна")
if not state["s3_ok"]:
raise unavailable("Хранилище файлов недоступно")
return {"status": "ok"}
@app.post("/api/v1/users", status_code=201)
async def create_user(request: Request):
body = await read_json(request, ("username", "display_name", "password"))
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
if not non_blank(body["display_name"], 100):
raise bad("display_name: 1..100 символов, не только пробелы")
password = body["password"]
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
ensure_capacity()
password_hash = await run_in_threadpool(passwords.hash_password, password)
async with db.connection() as conn:
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
if user is None:
raise ApiError(409, "conflict", "username уже занят")
log.info("user_created id=%s", user["id"])
return JSONResponse(user, status_code=201)
@app.post("/api/v1/auth/sessions", status_code=201)
async def login(request: Request):
if await request.body():
raise bad("POST /auth/sessions не принимает тело запроса")
username, password = basic_credentials(request)
row = None
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
async with db.connection() as conn:
row = await repo.credentials(conn, username)
stored = row["password_hash"] if row else passwords.DUMMY_HASH
matches = await run_in_threadpool(passwords.verify_password, stored, password)
if row is None or not matches:
log.info("login_failed username_known=%s", row is not None)
raise unauthorized("Basic")
user = repo.public_user(row)
token, expires_at = await sessions.create(user)
log.info("session_created user=%s", user["id"])
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
@app.delete("/api/v1/auth/sessions/current", status_code=204)
async def logout(request: Request):
token = bearer_token(request)
user = await sessions.resolve(token)
if user is None:
raise unauthorized()
await sessions.revoke(token)
log.info("session_revoked user=%s", user["id"])
return Response(status_code=204)
@app.get("/api/v1/users/me")
async def me(request: Request):
return await actor(request)
@app.post("/api/v1/chats", status_code=201)
async def create_chat(request: Request):
user = await actor(request)
body = await read_json(request, ("title", "member_ids"))
if not non_blank(body["title"], 100):
raise bad("title: 1..100 символов, не только пробелы")
raw_ids = body["member_ids"]
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
raise bad("member_ids: массив из 1..100 UUID")
ids: List[str] = []
for value in raw_ids:
parsed = parse_uuid(value)
if parsed is None:
raise bad("member_ids содержит некорректный UUID")
ids.append(parsed)
if len(set(ids)) != len(ids):
raise bad("member_ids содержит дубликаты")
if user["id"] not in ids:
raise bad("Создатель должен входить в member_ids")
ensure_capacity()
async def insert(conn):
return await repo.create_chat(conn, body["title"], ids)
try:
chat = await db.in_transaction(insert)
except repo.UnknownMember:
raise bad("member_ids содержит неизвестного пользователя") from None
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
return JSONResponse(chat, status_code=201)
@app.get("/api/v1/chats")
async def list_chats(request: Request):
user = await actor(request)
limit, cursor = page_params(request)
scope = "chats:" + user["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
return page(scope, items, has_more)
@app.get("/api/v1/chats/{chat_id}")
async def get_chat(chat_id: str, request: Request):
return await member_chat(await actor(request), chat_id)
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
async def create_message(chat_id: str, request: Request):
user = await actor(request)
cid = parse_uuid(chat_id)
if cid is None:
raise bad("chat_id должен быть UUID")
idem_key = request.headers.get("idempotency-key", "")
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
body = await read_json(request, ("text",), ("attachment_ids",))
text = body["text"]
attachment_ids_raw = body.get("attachment_ids", [])
if not isinstance(attachment_ids_raw, list):
raise bad("attachment_ids должен быть массивом UUID")
if len(attachment_ids_raw) > 10:
raise bad("attachment_ids: максимум 10 вложений")
attachment_ids: List[str] = []
for value in attachment_ids_raw:
parsed = parse_uuid(value)
if parsed is None:
raise bad("attachment_ids содержит некорректный UUID")
attachment_ids.append(parsed)
if len(set(attachment_ids)) != len(attachment_ids):
raise bad("attachment_ids содержит дубликаты")
has_text = isinstance(text, str) and non_blank(text, 4000)
has_attachments = len(attachment_ids) > 0
if not has_text and not has_attachments:
raise bad("text: 1..4000 символов, не только пробелы")
if isinstance(text, str) and len(text) > 4000:
raise bad("text: 1..4000 символов, не только пробелы")
if not isinstance(text, str):
raise bad("text должен быть строкой")
ensure_capacity()
normalized = {"text": text}
if attachment_ids:
normalized["attachment_ids"] = attachment_ids
body_hash = hashlib.sha256(
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
).hexdigest()
async def do(conn):
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
status, message = await db.in_transaction(do)
if status == "conflict":
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
if status == "not_member":
raise not_found()
if status == "attachment_not_found":
raise not_found()
if status == "attachment_failed":
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
return JSONResponse(message, status_code=201)
@app.get("/api/v1/chats/{chat_id}/messages")
async def list_messages(chat_id: str, request: Request):
chat = await member_chat(await actor(request), chat_id)
limit, cursor = page_params(request)
scope = "messages:" + chat["id"]
after = keyset(scope, cursor)
async with db.connection() as conn:
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
return page(scope, items, has_more)
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
async def upload_attachment(chat_id: str, request: Request):
user = await actor(request)
chat = await member_chat(user, chat_id)
filename = request.query_params.get("filename", "")
validate_filename(filename)
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
if declared_ct not in ALLOWED_CONTENT_TYPES:
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
declared_length = request.headers.get("content-length")
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB")
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
raise too_large
ensure_capacity()
sha = hashlib.sha256()
size = 0
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
head = b""
async for chunk in request.stream():
size += len(chunk)
if size > config.MAX_UPLOAD_BYTES:
raise too_large
if len(head) < 8:
head += chunk[: 8 - len(head)]
if len(head) == 8 and detect_content_type(head) is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
sha.update(chunk)
spool.write(chunk)
if size == 0:
raise bad("Пустой файл")
actual_ct = detect_content_type(head)
if actual_ct is None:
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
if actual_ct != declared_ct:
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
async with upload_slots:
await run_in_threadpool(inspect_image, spool, size, actual_ct)
attachment_id = str(uuid.uuid4())
object_key = "attachments/" + attachment_id
async with db.connection() as conn:
await repo.begin_attachment(
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(), object_key,
)
try:
async with upload_slots:
await run_in_threadpool(s3.put, object_key, spool, size, actual_ct)
except BaseException:
defer(compensate(attachment_id, object_key))
raise
try:
async with db.connection() as conn:
att = await repo.finish_attachment(conn, attachment_id)
except BaseException:
await compensate(attachment_id, object_key)
raise
if att is None:
raise unavailable("Загрузка отменена сверкой, повторите запрос")
log.info("attachment_uploaded id=%s chat=%s size=%d", att["id"], chat["id"], size)
return JSONResponse(att, status_code=201)
def defer(coro) -> None:
task = asyncio.ensure_future(coro)
background.add(task)
task.add_done_callback(background.discard)
async def compensate(attachment_id: str, object_key: str) -> None:
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
try:
async with db.connection() as conn:
await repo.abort_attachment(conn, attachment_id)
await run_in_threadpool(s3.delete, object_key)
except BaseException:
log.warning("compensation incomplete attachment=%s, left for reconciler", attachment_id)
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
user = await actor(request)
aid = parse_uuid(attachment_id)
if aid is None:
raise bad("attachment_id должен быть UUID")
async with db.connection() as conn:
row = await repo.get_attachment(conn, aid)
if row is None:
raise not_found()
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
if chat is None:
raise not_found()
return row
@app.get("/api/v1/attachments/{attachment_id}")
async def get_attachment(attachment_id: str, request: Request):
return repo._attachment(await accessible_attachment(request, attachment_id))
@app.get("/api/v1/attachments/{attachment_id}/content")
async def download_attachment(attachment_id: str, request: Request):
row = await accessible_attachment(request, attachment_id)
variant = request.query_params.get("variant", "original")
if variant != "original":
raise not_found()
if row["status"] != "ready":
raise ApiError(409, "conflict", "Вложение недоступно для скачивания")
opened = await run_in_threadpool(s3.open_stream, row["object_key"])
if opened is None:
log.error("object_missing attachment=%s", row["id"])
raise unavailable("Оригинал временно недоступен")
chunks, length = opened
return StreamingResponse(
iterate_in_threadpool(chunks), media_type=row["content_type"], headers={"Content-Length": str(length)},
)
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
async def fallback(request: Request):
raise not_found()
+16
View File
@@ -0,0 +1,16 @@
from argon2 import PasswordHasher
from argon2.exceptions import Argon2Error, VerifyMismatchError
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = HASHER.hash("there-is-no-such-user-password")
def hash_password(password: str) -> str:
return HASHER.hash(password)
def verify_password(stored: str, password: str) -> bool:
try:
return HASHER.verify(stored, password)
except (VerifyMismatchError, Argon2Error, ValueError):
return False
+115
View File
@@ -0,0 +1,115 @@
"""DB <-> S3 reconciliation for attachments.
Invariant: a row is inserted as 'uploading' BEFORE the object is written, and becomes 'ready' only AFTER the
PUT succeeded. So every object has a row, and a client only ever sees 'ready' rows.
Cases (all gated by a grace period so in-flight uploads are never touched):
stale_uploading row 'uploading' older than grace AND its object (if any) older than grace -> delete both;
a fresh object under an old row is left alone (re-checked via LastModified)
orphan_object object without any row, older than grace -> delete object
missing_object 'ready' row confirmed longer than grace ago whose object is gone (re-checked by HEAD) -> mark 'failed'/object_missing, never delete the row
Rows referenced by messages are never deleted: only 'uploading' rows can be removed, and those cannot be linked.
"""
import logging
import time
from datetime import datetime, timedelta, timezone
from starlette.concurrency import run_in_threadpool
log = logging.getLogger("messenger.reconcile")
LOCK_ID = 2026_0104
PREFIX = "attachments/"
BATCH = 500
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
started = time.monotonic()
result = {
"dry_run": dry_run, "grace_seconds": grace_seconds,
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0,
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
}
async with db.connection() as conn:
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
if not (await cur.fetchone())["got"]:
return {"skipped": "another reconciler is running"}
try:
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
await _stale_uploading(conn, s3, cutoff, dry_run, result)
await _orphan_objects(conn, s3, cutoff, dry_run, result)
await _missing_objects(conn, s3, cutoff, dry_run, result)
finally:
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
result["seconds"] = round(time.monotonic() - started, 3)
return result
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'uploading' AND created_at < %s", (cutoff,),
)
for row in await cur.fetchall():
modified = await run_in_threadpool(s3.head, row["object_key"])
if modified is not None and modified >= cutoff:
result["protected_fresh_objects"] += 1
continue
result["stale_uploading"] += 1
if dry_run:
continue
if modified is not None:
await run_in_threadpool(s3.delete, row["object_key"])
result["deleted_objects"] += 1
done = await conn.execute(
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
)
result["removed_rows"] += done.rowcount
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
pages = await run_in_threadpool(lambda: iter(s3.list_pages(PREFIX)))
while True:
page = await run_in_threadpool(next, pages, None)
if page is None:
break
result["objects_scanned"] += len(page)
keys = [o["Key"] for o in page]
if not keys:
continue
cur = await conn.execute("SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,))
known = {row["object_key"] for row in await cur.fetchall()}
for obj in page:
if obj["Key"] in known or obj["LastModified"] >= cutoff:
continue
result["orphan_objects"] += 1
if not dry_run:
again = await conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (obj["Key"],))
if await again.fetchone() is None:
await run_in_threadpool(s3.delete, obj["Key"])
result["deleted_objects"] += 1
async def _missing_objects(conn, s3, cutoff, dry_run, result):
last = None
while True:
cur = await conn.execute(
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
(cutoff, last, last, BATCH),
)
rows = await cur.fetchall()
if not rows:
return
for row in rows:
result["rows_scanned"] += 1
if await run_in_threadpool(s3.exists, row["object_key"]):
continue
result["missing_objects"] += 1
if not dry_run:
done = await conn.execute(
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
)
result["marked_failed"] += done.rowcount
last = str(rows[-1]["id"])
+293
View File
@@ -0,0 +1,293 @@
import hashlib
import json
from typing import List, Optional, Tuple
import psycopg
from app.cursor import fmt
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
CHAT_QUERY = (
"SELECT c.id, c.title, c.created_at,"
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
)
class UnknownMember(Exception):
pass
def decoded(value) -> str:
return bytes(value).decode("utf-8")
def public_user(row) -> dict:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": decoded(row["display_name"]),
"created_at": fmt(row["created_at"]),
}
def _chat(row) -> dict:
return {
"id": str(row["id"]),
"title": decoded(row["title"]),
"member_ids": [str(value) for value in row["member_ids"]],
"created_at": fmt(row["created_at"]),
}
def _message(row) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": decoded(row["text"]),
"created_at": fmt(row["created_at"]),
}
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
msg = _message(row)
msg["attachment_ids"] = attachment_ids
return msg
def _attachment(row) -> dict:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"owner_id": str(row["owner_id"]),
"filename": row["filename"],
"content_type": row["content_type"],
"size_bytes": row["size_bytes"],
"sha256": row["sha256"],
"status": row["status"],
"variants": [],
"error_code": row["error_code"],
"created_at": fmt(row["created_at"]),
"updated_at": fmt(row["updated_at"]),
}
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
try:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
" RETURNING id, username, display_name, created_at",
(username, display_name.encode("utf-8"), password_hash),
)
except psycopg.errors.UniqueViolation:
return None
return public_user(await cur.fetchone())
async def credentials(conn, username: str):
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,),
)
return await cur.fetchone()
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_at)"
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
" RETURNING id, title, created_at",
(title.encode("utf-8"),),
)
chat = await cur.fetchone()
cur = await conn.execute(
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
(chat["id"], member_ids),
)
if cur.rowcount != len(member_ids):
raise UnknownMember()
chat["member_ids"] = list(member_ids)
return _chat(chat)
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
cur = await conn.execute(
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id),
)
row = await cur.fetchone()
return _chat(row) if row else None
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
CHAT_QUERY
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
" ORDER BY c.created_at, c.id LIMIT %s",
(user_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
return [_chat(row) for row in rows[:limit]], len(rows) > limit
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
)
row = await cur.fetchone()
return _message(row) if row else None
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
cur = await conn.execute(
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
(message_id,),
)
rows = await cur.fetchall()
return [str(row["attachment_id"]) for row in rows]
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
for pos, aid in enumerate(attachment_ids):
await conn.execute(
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
(message_id, aid, pos),
)
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
ts, last_id = after if after else (None, None)
cur = await conn.execute(
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
" ORDER BY created_at, id LIMIT %s",
(chat_id, ts, ts, last_id, limit + 1),
)
rows = await cur.fetchall()
messages = []
for row in rows[:limit]:
aids = await _fetch_message_attachment_ids(conn, row["id"])
messages.append(_message_with_attachments(row, aids))
return messages, len(rows) > limit
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
raw = "%s:%s:%s" % (user_id, chat_id, key)
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
attachment_ids: List[str], idem_key: str, body_hash: str):
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
)
if await cur.fetchone() is None:
return "not_member", None
cur = await conn.execute(
"SELECT request_body_hash, response_json FROM idempotency_keys"
" WHERE user_id = %s AND chat_id = %s AND key = %s",
(sender_id, chat_id, idem_key),
)
existing = await cur.fetchone()
if existing:
if existing["request_body_hash"] != body_hash:
return "conflict", None
return "replay", json.loads(existing["response_json"])
if attachment_ids:
cur = await conn.execute(
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
(attachment_ids,),
)
att_rows = await cur.fetchall()
found_ids = {str(row["id"]) for row in att_rows}
for aid in attachment_ids:
if aid not in found_ids:
return "attachment_not_found", None
for row in att_rows:
if str(row["chat_id"]) != chat_id:
return "attachment_not_found", None
if row["status"] == "failed":
return "attachment_failed", None
text_bytes = text.encode("utf-8") if text else b""
cur = await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
" RETURNING " + MESSAGE_COLUMNS,
(chat_id, sender_id, text_bytes, chat_id, sender_id),
)
row = await cur.fetchone()
if row is None:
return "not_member", None
if attachment_ids:
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
message = _message_with_attachments(row, list(attachment_ids))
await conn.execute(
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
)
return "created", message
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
cur = await conn.execute(
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
(ttl_seconds,),
)
return cur.rowcount
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
content_type: str, size_bytes: int, sha256: str, object_key: str) -> None:
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
" status, error_code, created_at, updated_at)"
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, clock_timestamp(), clock_timestamp())",
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key),
)
async def finish_attachment(conn, attachment_id: str) -> Optional[dict]:
cur = await conn.execute(
"UPDATE attachments SET status = 'ready', updated_at = clock_timestamp()"
" WHERE id = %s AND status = 'uploading'"
" RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code,"
" created_at, updated_at",
(attachment_id,),
)
row = await cur.fetchone()
return _attachment(row) if row else None
async def abort_attachment(conn, attachment_id: str) -> None:
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
async def get_attachment(conn, attachment_id: str):
cur = await conn.execute(
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, error_code,"
" created_at, updated_at FROM attachments WHERE id = %s AND status <> 'uploading'",
(attachment_id,),
)
return await cur.fetchone()
+97
View File
@@ -0,0 +1,97 @@
import logging
from typing import Iterator, Optional, Tuple
import boto3
from botocore.config import Config as BotoConfig
from botocore.exceptions import BotoCoreError, ClientError
from app import config
from app.errors import unavailable
log = logging.getLogger("messenger.s3")
MISSING = ("NoSuchKey", "404", "NotFound")
class S3Storage:
def __init__(self) -> None:
self.client = None
self.probe_client = None
def open(self) -> None:
self.probe_client = self._client(1.0, 1.5)
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
def _client(self, connect: float, read: float):
return boto3.client(
"s3",
endpoint_url=config.S3_ENDPOINT,
region_name=config.S3_REGION,
aws_access_key_id=config.S3_ACCESS_KEY_ID,
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
config=BotoConfig(
signature_version="s3v4",
s3={"addressing_style": "path"},
connect_timeout=connect,
read_timeout=read,
retries={"max_attempts": 0},
),
)
def probe(self) -> bool:
try:
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
try:
fileobj.seek(0)
self.client.put_object(
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
)
except (BotoCoreError, ClientError) as exc:
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
try:
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
except BotoCoreError as exc:
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
raise unavailable("Хранилище файлов недоступно") from None
body = response["Body"]
return body.iter_chunks(65536), int(response["ContentLength"])
def head(self, key: str):
"""LastModified of the object, or None when it does not exist."""
try:
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
except ClientError as exc:
if exc.response["Error"].get("Code", "") in MISSING:
return None
raise
def exists(self, key: str) -> bool:
return self.head(key) is not None
def delete(self, key: str) -> None:
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
def list_pages(self, prefix: str):
paginator = self.client.get_paginator("list_objects_v2")
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
yield page.get("Contents", [])
def ready(self) -> bool:
try:
self.client.head_bucket(Bucket=config.S3_BUCKET)
return True
except Exception:
return False
+154
View File
@@ -0,0 +1,154 @@
import hashlib
import json
import logging
import secrets
from datetime import datetime, timedelta, timezone
from typing import Optional
import valkey.asyncio as valkey
from app import config, repo
from app.cursor import fmt
from app.errors import unavailable
log = logging.getLogger("messenger.sessions")
REVOKED = "revoked"
class SessionStore:
def __init__(self, db) -> None:
self.db = db
self.client = None
self.stats = {"hits": 0, "misses": 0, "errors": 0}
async def open(self) -> None:
if not config.CACHE_URL:
log.info("session cache disabled: CACHE_URL is empty")
return
self.client = valkey.Valkey.from_url(
config.CACHE_URL,
socket_timeout=config.CACHE_TIMEOUT,
socket_connect_timeout=config.CACHE_TIMEOUT,
decode_responses=True,
)
async def close(self) -> None:
if self.client is not None:
await self.client.aclose()
self.client = None
@staticmethod
def digest(token: str) -> bytes:
return hashlib.sha256(token.encode("utf-8")).digest()
@staticmethod
def key(digest: bytes) -> str:
return "session:" + digest.hex()
async def create(self, user: dict) -> tuple:
token = secrets.token_urlsafe(32)
digest = self.digest(token)
expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
async with self.db.connection() as conn:
await conn.execute(
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at)"
" VALUES (%s, %s, clock_timestamp(), %s)",
(digest, user["id"], expires_at),
)
await self._cache_put(digest, user, expires_at)
return token, fmt(expires_at)
async def resolve(self, token: str) -> Optional[dict]:
digest = self.digest(token)
cached = await self._cache_get(digest)
if cached == REVOKED:
self.stats["hits"] += 1
return None
if cached is not None:
self.stats["hits"] += 1
return cached
self.stats["misses"] += 1
async with self.db.connection() as conn:
cur = await conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
" FROM sessions s JOIN users u ON u.id = s.user_id"
" WHERE s.token_hash = %s AND s.expires_at > now()",
(digest,),
)
row = await cur.fetchone()
if row is None:
return None
user = repo.public_user(row)
await self._cache_put(digest, user, row["expires_at"])
return user
async def revoke(self, token: str) -> None:
digest = self.digest(token)
await self._cache_revoke(digest)
try:
async with self.db.connection() as conn:
await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
except Exception:
await self._cache_release(digest)
raise
async def sweep(self) -> int:
async with self.db.connection() as conn:
cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()")
return cur.rowcount
def _cache_ttl(self, expires_at: datetime) -> int:
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
return min(config.SESSION_CACHE_TTL_SECONDS, remaining)
async def _cache_get(self, digest: bytes) -> Optional[dict]:
if self.client is None:
return None
try:
raw = await self.client.get(self.key(digest))
except Exception:
self._cache_failed("get")
return None
if not raw:
return None
if raw == REVOKED:
return REVOKED
try:
return json.loads(raw)
except ValueError:
return None
async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None:
if self.client is None:
return
ttl = self._cache_ttl(expires_at)
if ttl <= 0:
return
payload = json.dumps(user)
try:
await self.client.set(self.key(digest), payload, ex=ttl, nx=True)
except Exception:
self._cache_failed("set")
async def _cache_release(self, digest: bytes) -> None:
if self.client is None:
return
try:
await self.client.delete(self.key(digest))
except Exception:
self._cache_failed("release")
async def _cache_revoke(self, digest: bytes) -> None:
if self.client is None:
return
try:
await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS)
except Exception:
self._cache_failed("revoke")
raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None
def _cache_failed(self, operation: str) -> None:
self.stats["errors"] += 1
if self.stats["errors"] % 50 == 1:
log.warning("session cache %s failed, falling back to postgres (errors=%d)", operation, self.stats["errors"])
+229
View File
@@ -0,0 +1,229 @@
name: mtusi-lab4
x-api: &api-defaults
build:
context: .
image: mtusi-messenger:lab4
env_file:
- .env
secrets:
- database_url
- cache_url
- s3_access_key_id
- s3_secret_access_key
depends_on:
postgres:
condition: service_healthy
valkey:
condition: service_healthy
minio-init:
condition: service_completed_successfully
migrate:
condition: service_completed_successfully
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
healthcheck:
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""]
interval: 5s
timeout: 3s
retries: 3
start_period: 15s
restart: unless-stopped
services:
nginx:
image: nginx:1.27.3-alpine
command: ["/bin/sh", "/opt/nginx-health.sh"]
environment:
API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2}
HEALTH_INTERVAL: "1"
HEALTH_FALL: "2"
HEALTH_RISE: "2"
ports:
- "127.0.0.1:8080:8080"
volumes:
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/nginx-health.sh:/opt/nginx-health.sh:ro
depends_on:
api-1:
condition: service_healthy
api-2:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.25"
memory: 64M
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"]
interval: 5s
timeout: 3s
retries: 3
start_period: 5s
restart: unless-stopped
api-1:
<<: *api-defaults
environment:
INSTANCE_ID: api-1
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://minio:9000
api-2:
<<: *api-defaults
environment:
INSTANCE_ID: api-2
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
S3_ENDPOINT: http://minio:9000
postgres:
image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_INITDB_ARGS: --data-checksums
command:
- postgres
- -c
- synchronous_commit=on
- -c
- shared_buffers=128MB
- -c
- work_mem=8MB
- -c
- max_connections=50
- -c
- track_io_timing=on
secrets:
- postgres_password
volumes:
- pgdata:/var/lib/postgresql/data
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
restart: unless-stopped
valkey:
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
command:
- valkey-server
- --save
- ""
- --appendonly
- "no"
- --maxmemory
- 64mb
- --maxmemory-policy
- allkeys-lru
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
start_period: 5s
restart: unless-stopped
minio:
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
command: ["server", "/data"]
environment:
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
MINIO_BROWSER: "off"
secrets:
- minio_root_user
- minio_root_password
volumes:
- miniodata:/data
deploy:
resources:
limits:
cpus: "0.50"
memory: 512M
healthcheck:
test: ["CMD-SHELL", "mc ready local || exit 1"]
interval: 5s
timeout: 3s
retries: 10
start_period: 10s
restart: unless-stopped
minio-init:
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
entrypoint: ["/bin/sh", "/opt/init.sh"]
environment:
S3_BUCKET: messenger-attachments
secrets:
- minio_root_user
- minio_root_password
- s3_access_key_id
- s3_secret_access_key
volumes:
- ./infra/minio/init.sh:/opt/init.sh:ro
- ./infra/minio/app-policy.json:/etc/minio-policy/app-policy.json:ro
depends_on:
minio:
condition: service_healthy
restart: "no"
migrate:
build:
context: .
image: mtusi-messenger:lab4
command: ["python", "scripts/migrate.py"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets:
- database_url
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
restart: "no"
volumes:
pgdata:
miniodata:
secrets:
minio_root_user:
file: ./secrets/minio_root_user
minio_root_password:
file: ./secrets/minio_root_password
postgres_password:
file: ./secrets/postgres_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
s3_access_key_id:
file: ./secrets/s3_access_key_id
s3_secret_access_key:
file: ./secrets/s3_secret_access_key
+4
View File
@@ -0,0 +1,4 @@
metadata served by api-1
ok uploaded via api-2, downloaded via api-1, identical sha256
ok restart of all stateful services keeps metadata and original
PASS: attachment failover + persistence
+9
View File
@@ -0,0 +1,9 @@
ok auth and membership on upload
ok file over 10 MiB -> 413
ok 48 Mpx image -> 413 before decoding
ok corrupted image -> 415
ok MIME spoofing and unsupported types -> 415
ok dangerous filenames -> 400
ok unknown / foreign / other-chat attachment ids rejected, no 500
ok client abort mid-upload sent; server must stay healthy
PASS: attachment negative scenarios
+2
View File
@@ -0,0 +1,2 @@
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 0.441}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+11
View File
@@ -0,0 +1,11 @@
ok truncated PNG, bad filter byte, broken IDAT, 27-byte JPEG stub -> 415
ok real JPEG accepted; first half + EOI and first half alone -> 415
ok same defects in images above the full-decode threshold (6 Mpx) -> 415
ok valid 6 Mpx PNG accepted
ok valid 36 Mpx PNG (116 KiB on the wire, 108 MiB decoded) accepted in 0.1s
ok 36 Mpx PNG with bad filter byte -> 415
ok whitespace-only, DEL, C1 and dot filenames -> 400
ok attachment order preserved in POST, replay and history; reordered body -> 409
ok idempotency: [] == absent, invalid keys -> 400, scoped per user
ok 12 MiB body -> JSON 413
PASS: input hardening
+19
View File
@@ -0,0 +1,19 @@
valid baseline 675893 bytes accepted 0.01s ok
valid progressive 616591 bytes accepted 0.02s ok
valid grayscale 359415 bytes accepted 0.00s ok
valid CMYK 1262509 bytes accepted 0.01s ok
valid, junk after EOI 675901 bytes accepted 0.01s ok
valid 40 Mpx baseline 626630 bytes accepted 0.01s ok
valid 40 Mpx progressive 352914 bytes accepted 0.07s ok
valid 40 Mpx progressive 444 704086 bytes accepted 0.13s ok
valid 40 Mpx progressive CMYK 626581 bytes accepted 0.17s ok
27-byte stub 27 bytes rejected 415 0.00s ok
first half + EOI 337948 bytes rejected 415 0.00s ok
first half, no EOI 337946 bytes rejected 415 0.00s ok
progressive half + EOI 308297 bytes rejected 415 0.00s ok
headers only + EOI 611 bytes rejected 415 0.01s ok
␍ ␍checker peak RSS, 40 Mpx baseline 1.7 MiB
␍ ␍checker peak RSS, 40 Mpx progressive 116.3 MiB
␍ ␍checker peak RSS, 40 Mpx progressive 444 230.6 MiB
␍ ␍checker peak RSS, 40 Mpx progressive CMYK 306.9 MiB
RESULT: 14/14
+25
View File
@@ -0,0 +1,25 @@
== 10 parallel uploads of 8674403 bytes, 3 rounds ==
upload statuses: {201: 30}
== 10 parallel downloads ==
download statuses: {200: 20}
== 10 parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), 3 rounds ==
upload statuses: {201: 30}
== 10 MiB + 1 byte, Content-Length: 100 requests, 10 at a time ==
{'413 application/json': 100}
== 10 MiB + 1 byte, chunked: 100 requests, 10 at a time ==
{'413 application/json': 100}
== 10 MiB + 1 byte, chunked in 64 KiB pieces: 100 requests, 10 at a time ==
{'413 application/json': 100}
== 8 MiB to a foreign chat: 100 requests, 10 at a time ==
{'404 application/json': 100}
== 8 MiB without session: 100 requests, 10 at a time ==
{'401 application/json': 100}
== 8 MiB with wrong magic: 100 requests, 10 at a time ==
{'415 application/json': 100}
== 8 MiB with wrong Content-Type: 100 requests, 10 at a time ==
{'415 application/json': 100}
== 70 KiB JSON message: 100 requests, 10 at a time ==
{'413 application/json': 100}
container events during the run: none
cgroup memory.peak, MiB: {'mtusi-lab4-api-1-1': 475.1, 'mtusi-lab4-api-2-1': 490.5, 'mtusi-lab4-nginx-1': 34.3}
PASS: LB stress
+25
View File
@@ -0,0 +1,25 @@
== 30 parallel uploads of 8674403 bytes, 2 rounds ==
upload statuses: {201: 60}
== 30 parallel downloads ==
download statuses: {200: 60}
== 30 parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), 2 rounds ==
upload statuses: {201: 60}
== 10 MiB + 1 byte, Content-Length: 30 requests, 30 at a time ==
{'413 application/json': 30}
== 10 MiB + 1 byte, chunked: 30 requests, 30 at a time ==
{'413 application/json': 30}
== 10 MiB + 1 byte, chunked in 64 KiB pieces: 30 requests, 30 at a time ==
{'413 application/json': 30}
== 8 MiB to a foreign chat: 30 requests, 30 at a time ==
{'404 application/json': 30}
== 8 MiB without session: 30 requests, 30 at a time ==
{'401 application/json': 30}
== 8 MiB with wrong magic: 30 requests, 30 at a time ==
{'415 application/json': 30}
== 8 MiB with wrong Content-Type: 30 requests, 30 at a time ==
{'415 application/json': 30}
== 70 KiB JSON message: 30 requests, 30 at a time ==
{'413 application/json': 30}
container events during the run: none
cgroup memory.peak, MiB: {'mtusi-lab4-api-1-1': 465.3, 'mtusi-lab4-api-2-1': 463.0, 'mtusi-lab4-nginx-1': 34.3}
PASS: LB stress
+25
View File
@@ -0,0 +1,25 @@
payload bytes: 8674403
round 1 peak MiB per api: {'mtusi-lab4-api-2-1': 95.5, 'mtusi-lab4-api-1-1': 115.6} statuses=[201]
round 2 peak MiB per api: {'mtusi-lab4-api-2-1': 104.8, 'mtusi-lab4-api-1-1': 115.6} statuses=[201]
round 3 peak MiB per api: {'mtusi-lab4-api-2-1': 106.6, 'mtusi-lab4-api-1-1': 115.3} statuses=[201]
{
"payload_mib": 8.27,
"concurrency": 10,
"rounds": 3,
"idle_mib": {
"mtusi-lab4-api-2-1": 87.2,
"mtusi-lab4-api-1-1": 87.8
},
"peak_mib": {
"mtusi-lab4-api-2-1": 106.6,
"mtusi-lab4-api-1-1": 115.6
},
"container_limit_mib": 512,
"statuses": {
"201": 30
},
"upload_seconds_median": 0.63,
"upload_seconds_max": 0.82
}
peak=163 MiB oom_kill 0
peak=154 MiB oom_kill 0
@@ -0,0 +1,23 @@
payload bytes: 8674403
round 1 peak MiB per api: {'mtusi-lab4-api-1-1': 143.7, 'mtusi-lab4-api-2-1': 142.4} statuses=[201]
round 2 peak MiB per api: {'mtusi-lab4-api-1-1': 166.1, 'mtusi-lab4-api-2-1': 153.2} statuses=[201]
round 3 peak MiB per api: {'mtusi-lab4-api-1-1': 182.3, 'mtusi-lab4-api-2-1': 154.7} statuses=[201]
{
"payload_mib": 8.27,
"concurrency": 10,
"rounds": 3,
"idle_mib": {
"mtusi-lab4-api-1-1": 88.8,
"mtusi-lab4-api-2-1": 88.4
},
"peak_mib": {
"mtusi-lab4-api-1-1": 182.3,
"mtusi-lab4-api-2-1": 154.7
},
"container_limit_mib": 256,
"statuses": {
"201": 30
},
"upload_seconds_median": 0.82,
"upload_seconds_max": 1.28
}
+1
View File
@@ -0,0 +1 @@
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+12
View File
@@ -0,0 +1,12 @@
== published ports of minio (expect none) ==
no port 9000/tcp for container mtusi-lab4-minio-1:
no port 9001/tcp for container mtusi-lab4-minio-1:
== anonymous requests from inside the network ==
GET /messenger-attachments/ -> 403
GET /messenger-attachments/attachments/does-not-matter -> 403
GET /messenger-attachments?list-type=2 -> 403
== bucket policy ==
Container mtusi-lab4-minio-init-run-85f57841572d Creating
Container mtusi-lab4-minio-init-run-85f57841572d Created
Access permission for `local/messenger-attachments` is `private`
PASS: bucket is private
+30
View File
@@ -0,0 +1,30 @@
baseline (data left by earlier tests): {"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 366, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 203, "seconds": 0.253, "stale_uploading": 0}
== dry-run, grace=900 ==
{"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 2, "objects_scanned": 371, "orphan_objects": 0, "protected_fresh_objects": 2, "removed_rows": 0, "rows_scanned": 206, "seconds": 0.264, "stale_uploading": 1}
ok dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)
ok dry-run: 2 new missing objects (e, g)
ok dry-run changes nothing
== apply, grace=900 ==
{"deleted_objects": 0, "dry_run": false, "grace_seconds": 900, "marked_failed": 2, "missing_objects": 2, "objects_scanned": 371, "orphan_objects": 0, "protected_fresh_objects": 2, "removed_rows": 1, "rows_scanned": 206, "seconds": 0.273, "stale_uploading": 1}
ok old row whose object is young is kept (a)
ok stale row without object removed
ok fresh in-flight upload untouched
ok fresh orphan object kept inside grace
ok old row with a fresh object is left alone
ok recently confirmed row is not marked missing
ok missing unused object marked failed
ok used attachment with object untouched
ok used attachment with lost object kept, marked failed
== apply after grace expired (grace=2 after 3s) ==
{"deleted_objects": 4, "dry_run": false, "grace_seconds": 2, "marked_failed": 1, "missing_objects": 1, "objects_scanned": 368, "orphan_objects": 1, "protected_fresh_objects": 0, "removed_rows": 3, "rows_scanned": 368, "seconds": 0.428, "stale_uploading": 3}
ok orphan object deleted after grace
ok expired in-flight row collected after grace
ok old row + object collected once the object is older than grace (a, h)
ok used attachment still untouched
== second apply is idempotent ==
{"deleted_objects": 0, "dry_run": false, "grace_seconds": 2, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 367, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 367, "seconds": 0.457, "stale_uploading": 0}
ok repeat run changes nothing
== cost of a full walk, 1000 ready attachments ==
{"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 1367, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 1204, "seconds": 1.605, "stale_uploading": 0}
ok bulk walk finds nothing wrong
RESULT: 18/18 checks passed
+5
View File
@@ -0,0 +1,5 @@
ok S3 down: upload/download/ready -> 503 JSON, metadata and text messages still served through the LB (no 502)
ok recovered 2s after S3 start
ok ready is 200 again
ok hung S3: upload answered 503 in 7.0s
PASS: S3 outage behaviour
+1
View File
@@ -0,0 +1 @@
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
+15
View File
@@ -0,0 +1,15 @@
{
"Version": "2012-10-17",
"Statement": [
{
"Effect": "Allow",
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"]
},
{
"Effect": "Allow",
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
"Resource": ["arn:aws:s3:::messenger-attachments"]
}
]
}
+19
View File
@@ -0,0 +1,19 @@
#!/bin/sh
# Idempotent: safe to run on every `compose up`. Root credentials are used only here, never by the API.
set -eu
BUCKET="${S3_BUCKET:-messenger-attachments}"
ROOT_USER="$(cat /run/secrets/minio_root_user)"
ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
APP_USER="$(cat /run/secrets/s3_access_key_id)"
APP_PASSWORD="$(cat /run/secrets/s3_secret_access_key)"
until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do
echo "waiting for minio"; sleep 2
done
mc mb --ignore-existing "local/$BUCKET"
mc anonymous set none "local/$BUCKET"
mc admin policy create local app-attachments /etc/minio-policy/app-policy.json 2>/dev/null \
|| mc admin policy create local app-attachments /etc/minio-policy/app-policy.json --overwrite 2>/dev/null || true
mc admin user add local "$APP_USER" "$APP_PASSWORD" 2>/dev/null || true
mc admin policy attach local app-attachments --user "$APP_USER" 2>/dev/null || true
echo "minio bootstrap complete: bucket=$BUCKET private, app user limited to attachments/*"
+63
View File
@@ -0,0 +1,63 @@
#!/bin/sh
set -u
SERVERS="${API_UPSTREAMS:-api-1 api-2}"
INTERVAL="${HEALTH_INTERVAL:-1}"
FALL="${HEALTH_FALL:-2}"
RISE="${HEALTH_RISE:-2}"
OUT=/etc/nginx/upstream/servers.conf
mkdir -p "$(dirname "$OUT")"
var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; }
render() {
any_up=0
for s in $SERVERS; do
eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1
done
for s in $SERVERS; do
mark=""
if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then
mark=" down"
fi
echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;"
done
}
for s in $SERVERS; do
v=$(var "$s")
eval "up_$v=1 ok_$v=0 bad_$v=0"
done
render > "$OUT"
watch() {
pending=0
while sleep "$INTERVAL"; do
changed=0
for s in $SERVERS; do
v=$(var "$s")
if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then
eval "ok_$v=\$((ok_$v + 1)) bad_$v=0"
if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then
eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream"
fi
else
eval "bad_$v=\$((bad_$v + 1)) ok_$v=0"
if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then
eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down"
fi
fi
done
if [ "$changed" = 1 ]; then
render > "$OUT"
pending=1
fi
if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then
pending=0
fi
done
}
watch &
exec nginx -g 'daemon off;'
+75
View File
@@ -0,0 +1,75 @@
worker_processes 2;
events {
worker_connections 256;
}
http {
resolver 127.0.0.11 valid=5s ipv6=off;
upstream api {
zone api 64k;
include /etc/nginx/upstream/servers.conf;
keepalive 32;
keepalive_timeout 4s;
}
log_format main '$remote_addr - $upstream_addr [$time_local] '
'"$request" $status $body_bytes_sent '
'"$upstream_response_time" "$request_time"';
access_log /var/log/nginx/access.log main;
server {
listen 8080;
client_max_body_size 64k;
error_page 413 @payload_too_large;
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_connect_timeout 1s;
proxy_read_timeout 30s;
proxy_send_timeout 30s;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 5s;
location ~ ^/api/v1/chats/[^/]+/messages$ {
proxy_pass http://api;
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
}
location ~ ^/api/v1/chats/[^/]+/attachments$ {
client_max_body_size 10m;
proxy_request_buffering off;
proxy_pass http://api;
proxy_next_upstream off;
}
location @payload_too_large {
default_type application/json;
add_header X-Request-Id $request_id always;
add_header X-Instance-Id lb always;
add_header Connection close always;
return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}';
}
location /health/ {
proxy_pass http://api;
proxy_next_upstream error timeout;
}
location ~ ^/api/v1/attachments/ {
proxy_max_temp_file_size 0;
proxy_pass http://api;
proxy_next_upstream error timeout;
}
location / {
proxy_pass http://api;
proxy_next_upstream error timeout http_502 http_503;
}
}
}
+46
View File
@@ -0,0 +1,46 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name bytea NOT NULL
CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400),
password_hash text NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title bytea NOT NULL
CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400),
created_at timestamptz NOT NULL
);
CREATE INDEX chats_created_idx ON chats (created_at, id);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
text bytea NOT NULL
CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000),
created_at timestamptz NOT NULL
);
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL,
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
+12
View File
@@ -0,0 +1,12 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
key text NOT NULL
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'),
request_body_hash text NOT NULL,
response_json text NOT NULL,
created_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
+28
View File
@@ -0,0 +1,28 @@
CREATE TABLE attachments (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
filename text NOT NULL
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
content_type text NOT NULL
CONSTRAINT attachments_ct CHECK (content_type IN ('image/png', 'image/jpeg')),
size_bytes integer NOT NULL
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
sha256 text NOT NULL
CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
object_key text NOT NULL,
status text NOT NULL DEFAULT 'ready'
CONSTRAINT attachments_status CHECK (status IN ('ready', 'failed')),
error_code text,
created_at timestamptz NOT NULL,
updated_at timestamptz NOT NULL
);
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
CREATE TABLE message_attachments (
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
position smallint NOT NULL,
PRIMARY KEY (message_id, attachment_id)
);
+2
View File
@@ -0,0 +1,2 @@
ALTER TABLE messages DROP CONSTRAINT messages_text_size;
ALTER TABLE messages ADD CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 0 AND 16000);
@@ -0,0 +1,5 @@
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('uploading', 'ready', 'failed'));
ALTER TABLE attachments ALTER COLUMN status DROP DEFAULT;
CREATE UNIQUE INDEX attachments_object_key_idx ON attachments (object_key);
CREATE INDEX attachments_status_created_idx ON attachments (status, created_at);
+30
View File
@@ -0,0 +1,30 @@
annotated-types==0.8.0
anyio==4.15.1
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
boto3==1.35.81
botocore==1.35.81
cffi==2.1.1
click==8.5.0
fastapi==0.115.6
h11==0.16.0
httptools==0.9.0
idna==3.20
jmespath==1.0.1
pillow==11.0.0
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg-pool==3.3.3
pycparser==3.11
pydantic==2.14.1
pydantic-core==2.50.1
python-dateutil==2.9.0.post0
s3transfer==0.10.4
six==1.17.0
starlette==0.41.3
typing-extensions==4.16.0
typing-inspection==0.4.4
urllib3==2.3.0
uvicorn==0.34.0
uvloop==0.23.0
valkey==6.2.0
+91
View File
@@ -0,0 +1,91 @@
#!/usr/bin/env python3
"""JPEG integrity matrix and peak memory of the checker.
docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py"""
import io
import os
import subprocess
import sys
import tempfile
import time
sys.path.insert(0, "/app")
from PIL import Image # noqa: E402
from app.errors import ApiError # noqa: E402
from app.imagecheck import DJPEG, inspect_image # noqa: E402
def encode(img, **kw):
out = io.BytesIO()
img.save(out, "JPEG", **kw)
return out.getvalue()
def verdict(data):
with tempfile.SpooledTemporaryFile(max_size=1048576) as spool:
spool.write(data)
try:
inspect_image(spool, len(data), "image/jpeg")
return "accepted"
except ApiError as exc:
return "rejected %d" % exc.status
def child_peak_kib(data):
with tempfile.NamedTemporaryFile() as handle:
handle.write(data)
handle.flush()
handle.seek(0)
proc = subprocess.Popen(DJPEG, stdin=handle)
peak = 0
while proc.poll() is None:
try:
with open("/proc/%d/status" % proc.pid) as status:
for line in status:
if line.startswith("VmHWM"):
peak = max(peak, int(line.split()[1]))
except OSError:
pass
time.sleep(0.002)
return peak
noise = Image.frombytes("RGB", (1000, 750), os.urandom(1000 * 750 * 3))
base = encode(noise, quality=90)
progressive = encode(noise, quality=90, progressive=True)
flat = Image.new("RGB", (8000, 5000), (120, 30, 200))
big_base = encode(flat, quality=90)
big_progressive = encode(flat, quality=90, progressive=True)
big_444 = encode(flat, quality=90, progressive=True, subsampling=0)
big_cmyk = encode(flat.convert("CMYK"), quality=90, progressive=True)
del flat
sos = base.index(b"\xff\xda")
cases = [
("valid baseline", base, "accepted"),
("valid progressive", progressive, "accepted"),
("valid grayscale", encode(noise.convert("L")), "accepted"),
("valid CMYK", encode(noise.convert("CMYK")), "accepted"),
("valid, junk after EOI", base + b"trailing", "accepted"),
("valid 40 Mpx baseline", big_base, "accepted"),
("valid 40 Mpx progressive", big_progressive, "accepted"),
("valid 40 Mpx progressive 444", big_444, "accepted"),
("valid 40 Mpx progressive CMYK", big_cmyk, "accepted"),
("27-byte stub", bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "rejected 415"),
("first half + EOI", base[: len(base) // 2] + b"\xff\xd9", "rejected 415"),
("first half, no EOI", base[: len(base) // 2], "rejected 415"),
("progressive half + EOI", progressive[: len(progressive) // 2] + b"\xff\xd9", "rejected 415"),
("headers only + EOI", base[:sos] + b"\xff\xd9", "rejected 415"),
]
failed = 0
for name, data, expected in cases:
started = time.monotonic()
got = verdict(data)
ok = got == expected
failed += not ok
print("%-28s %9d bytes %-13s %.2fs %s" % (name, len(data), got, time.monotonic() - started, "ok" if ok else "FAIL"))
for name, data in (("40 Mpx baseline", big_base), ("40 Mpx progressive", big_progressive),
("40 Mpx progressive 444", big_444), ("40 Mpx progressive CMYK", big_cmyk)):
print("checker peak RSS, %-24s %6.1f MiB" % (name, child_peak_kib(data) / 1024))
print("RESULT: %d/%d" % (len(cases) - failed, len(cases)))
sys.exit(1 if failed else 0)
+60
View File
@@ -0,0 +1,60 @@
#!/usr/bin/env python3
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
import hashlib
import os
import pathlib
import sys
import psycopg
ROOT = pathlib.Path(__file__).resolve().parents[1]
MIGRATIONS = ROOT / "migrations"
LOCK_ID = 2026_0004
def database_url() -> str:
path = os.environ.get("DATABASE_URL_FILE")
if path:
return pathlib.Path(path).read_text(encoding="utf-8").strip()
url = os.environ.get("DATABASE_URL", "").strip()
if not url:
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
return url
def main() -> None:
files = sorted(MIGRATIONS.glob("*.sql"))
if not files:
sys.exit("no migration files found in %s" % MIGRATIONS)
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY,"
" checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
applied = {row[0]: row[1] for row in rows}
for path in files:
version = path.stem
body = path.read_text(encoding="utf-8")
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
if version in applied:
if applied[version] != checksum:
sys.exit("migration %s changed after it was applied" % version)
print("skip %s" % version)
continue
with conn.transaction():
conn.execute(body)
conn.execute(
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
(version, checksum),
)
print("apply %s" % version)
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
print("migrations complete: %d file(s)" % len(files))
if __name__ == "__main__":
main()
+32
View File
@@ -0,0 +1,32 @@
#!/usr/bin/env python3
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
import argparse
import asyncio
import json
import sys
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import config, reconcile # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
async def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
args = parser.parse_args()
db, s3 = Database(), S3Storage()
await db.open()
await run_in_threadpool(s3.open)
try:
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
finally:
await db.close()
if __name__ == "__main__":
asyncio.run(main())
+142
View File
@@ -0,0 +1,142 @@
#!/usr/bin/env python3
"""Fault injection for the DB<->S3 reconciler. Run inside an api container:
docker compose exec -T api-1 python scripts/reconcile_experiment.py [--bulk 1000]"""
import argparse
import asyncio
import io
import json
import sys
import time
import uuid
sys.path.insert(0, "/app")
from starlette.concurrency import run_in_threadpool # noqa: E402
from app import config, reconcile # noqa: E402
from app.db import Database # noqa: E402
from app.s3 import S3Storage # noqa: E402
ZERO = "0" * 64
checks = []
def check(name, condition):
checks.append((name, bool(condition)))
print(("ok " if condition else "FAIL ") + name)
async def main():
parser = argparse.ArgumentParser()
parser.add_argument("--bulk", type=int, default=1000)
args = parser.parse_args()
db, s3 = Database(), S3Storage()
await db.open()
await run_in_threadpool(s3.open)
put = lambda key: run_in_threadpool(s3.put, key, io.BytesIO(b"x"), 1, "image/png")
exists = lambda key: run_in_threadpool(s3.exists, key)
baseline = await reconcile.run(db, s3, 900, dry_run=True)
print("baseline (data left by earlier tests):", json.dumps(baseline, sort_keys=True))
async with db.connection() as conn:
suffix = uuid.uuid4().hex[:10]
user = (await (await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
" VALUES (gen_random_uuid(), %s, %s, 'x', now()) RETURNING id", ("rx_" + suffix, b"rx"))).fetchone())["id"]
chat = (await (await conn.execute(
"INSERT INTO chats (id, title, created_at) VALUES (gen_random_uuid(), %s, now()) RETURNING id", (b"rx",))).fetchone())["id"]
await conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat, user))
async def row(status, age_s, with_object, linked=False):
aid = str(uuid.uuid4())
key = "attachments/" + aid
await conn.execute(
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
" status, created_at, updated_at) VALUES (%s,%s,%s,'f.png','image/png',1,%s,%s,%s,"
" now() - make_interval(secs => %s), now() - make_interval(secs => %s))", (aid, chat, user, ZERO, key, status, age_s, age_s))
if with_object:
await put(key)
if linked:
mid = (await (await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
" VALUES (gen_random_uuid(), %s, %s, ''::bytea, now()) RETURNING id", (chat, user))).fetchone())["id"]
await conn.execute("INSERT INTO message_attachments VALUES (%s,%s,0)", (mid, aid))
return aid, key
async def status_of(aid):
r = await (await conn.execute("SELECT status, error_code FROM attachments WHERE id=%s", (aid,))).fetchone()
return (r["status"], r["error_code"]) if r else None
old = 7200
a_id, a_key = await row("uploading", old, True) # crash after PUT, before ready (object is young)
b_id, b_key = await row("uploading", old, False) # crash before PUT
c_id, c_key = await row("uploading", 5, True) # in-flight upload, must survive
e_id, e_key = await row("ready", old, False) # object lost
f_id, f_key = await row("ready", old, True, linked=True) # healthy, used
g_id, g_key = await row("ready", old, False, linked=True) # object lost, used
h_id, h_key = await row("uploading", old, False) # old intent row, but a NEW object under its key
await put(h_key)
i_id, i_key = await row("ready", 5, False) # freshly confirmed, object not yet visible: must not be marked
d_key = "attachments/" + str(uuid.uuid4()) # PUT done, row never written
await put(d_key)
print("== dry-run, grace=900 ==")
r = await reconcile.run(db, s3, 900, dry_run=True)
print(json.dumps(r, sort_keys=True))
check("dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)",
r["stale_uploading"] - baseline["stale_uploading"] == 1
and r["protected_fresh_objects"] - baseline["protected_fresh_objects"] == 2)
check("dry-run: 2 new missing objects (e, g)", r["missing_objects"] - baseline["missing_objects"] == 2)
check("dry-run changes nothing", await exists(a_key) and await status_of(a_id) == ("uploading", None)
and await status_of(e_id) == ("ready", None))
print("== apply, grace=900 ==")
r = await reconcile.run(db, s3, 900, dry_run=False)
print(json.dumps(r, sort_keys=True))
check("old row whose object is young is kept (a)", await status_of(a_id) == ("uploading", None) and await exists(a_key))
check("stale row without object removed", await status_of(b_id) is None)
check("fresh in-flight upload untouched", await status_of(c_id) == ("uploading", None) and await exists(c_key))
check("fresh orphan object kept inside grace", await exists(d_key))
check("old row with a fresh object is left alone", await status_of(h_id) == ("uploading", None) and await exists(h_key))
check("recently confirmed row is not marked missing", await status_of(i_id) == ("ready", None))
check("missing unused object marked failed", await status_of(e_id) == ("failed", "object_missing"))
check("used attachment with object untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
check("used attachment with lost object kept, marked failed", await status_of(g_id) == ("failed", "object_missing"))
print("== apply after grace expired (grace=2 after 3s) ==")
await asyncio.sleep(3)
r = await reconcile.run(db, s3, 2, dry_run=False)
print(json.dumps(r, sort_keys=True))
check("orphan object deleted after grace", not await exists(d_key))
check("expired in-flight row collected after grace", await status_of(c_id) is None and not await exists(c_key))
check("old row + object collected once the object is older than grace (a, h)",
await status_of(a_id) is None and not await exists(a_key)
and await status_of(h_id) is None and not await exists(h_key))
check("used attachment still untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
print("== second apply is idempotent ==")
r = await reconcile.run(db, s3, 2, dry_run=False)
print(json.dumps(r, sort_keys=True))
check("repeat run changes nothing", r["deleted_objects"] == 0 and r["removed_rows"] == 0 and r["marked_failed"] == 0)
print("== cost of a full walk, %d ready attachments ==" % args.bulk)
keys = []
for _ in range(args.bulk):
aid, key = await row("ready", old, True)
keys.append(aid)
r = await reconcile.run(db, s3, 900, dry_run=True)
print(json.dumps(r, sort_keys=True))
check("bulk walk finds nothing wrong", r["missing_objects"] == 0 and r["orphan_objects"] == 0)
for aid in keys + [e_id, f_id, g_id, h_id, i_id]:
await run_in_threadpool(s3.delete, "attachments/" + aid)
await conn.execute("DELETE FROM chats WHERE id = %s", (chat,))
await conn.execute("DELETE FROM users WHERE id = %s", (user,))
await db.close()
failed = [n for n, ok in checks if not ok]
print("RESULT: %d/%d checks passed" % (len(checks) - len(failed), len(checks)))
sys.exit(1 if failed else 0)
asyncio.run(main())
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
docker compose restart -t 5 postgres valkey minio api-1 api-2 nginx
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
+27
View File
@@ -0,0 +1,27 @@
#!/bin/sh
# Proves the bucket is private without going through the API: anonymous S3 requests from inside the compose
# network are refused, and the MinIO console/ports are not published on the host.
set -eu
cd "$(dirname "$0")/.."
NET="$(docker compose ps -q minio | xargs docker inspect --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}')"
BUCKET=messenger-attachments
echo "== published ports of minio (expect none) =="
docker compose port minio 9000 2>&1 || true
docker compose port minio 9001 2>&1 || true
echo "== anonymous requests from inside the network =="
for path in "$BUCKET/" "$BUCKET/attachments/does-not-matter" "$BUCKET?list-type=2"; do
code="$(docker run --rm -i --network "$NET" python:3.12.8-slim-bookworm python - "$path" <<'PY'
import sys, urllib.request, urllib.error
try:
print(urllib.request.urlopen("http://minio:9000/" + sys.argv[1], timeout=5).status)
except urllib.error.HTTPError as e:
print(e.code)
PY
)"
echo "GET /$path -> $code"
[ "$code" = 403 ] || { echo "FAIL: expected 403"; exit 1; }
done
echo "== bucket policy =="
docker compose run --rm --no-deps --entrypoint sh minio-init -c \
'mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null && mc anonymous get local/messenger-attachments'
echo "PASS: bucket is private"
+93
View File
@@ -0,0 +1,93 @@
#!/usr/bin/env python3
"""Peak API memory under N concurrent ~8.7 MB uploads. Samples `docker stats` for the api containers.
python3 student-tests/memory_experiment.py --concurrency 10 --rounds 3"""
import argparse
import json
import os
import re
import statistics
import subprocess
import sys
import threading
import time
import zlib
import struct
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
def png(width, height):
raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height))
def chunk(kind, body):
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
+ chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b""))
UNITS = {"B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824}
def mib(text):
value, unit = re.match(r"([\d.]+)\s*([A-Za-z]+)", text).groups()
return float(value) * UNITS[unit] / 1048576
def sample(stop, out):
while not stop.is_set():
result = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
cwd=ROOT, capture_output=True, text=True)
for line in result.stdout.splitlines():
name, usage = line.split(" ", 1)
if "api-" in name:
out.setdefault(name, []).append(mib(usage.split("/")[0].strip()))
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--rounds", type=int, default=3)
args = parser.parse_args()
body = png(1700, 1700)
print("payload bytes:", len(body))
c = Client(timeout=60)
alice = c.user("mem")
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "mem", "member_ids": [alice["id"]]}, schema="Chat")["json"]
path = "/api/v1/chats/%s/attachments?filename=big.png" % chat["id"]
def one(_):
r = c.call("POST", path, [201, 429, 503], actor=alice, raw=body, headers={"Content-Type": "image/png"})
return r["status"], r["elapsed"]
idle, peaks, statuses, times = {}, [], [], []
probe = {}
stop = threading.Event()
t = threading.Thread(target=sample, args=(stop, probe)); t.start(); time.sleep(3); stop.set(); t.join()
idle = {k: max(v) for k, v in probe.items()}
for round_no in range(args.rounds):
probe = {}
stop = threading.Event()
t = threading.Thread(target=sample, args=(stop, probe)); t.start()
with ThreadPoolExecutor(args.concurrency) as pool:
results = list(pool.map(one, range(args.concurrency)))
stop.set(); t.join()
peak = {k: max(v) for k, v in probe.items()}
peaks.append(peak)
statuses += [s for s, _ in results]
times += [e for _, e in results]
print("round %d peak MiB per api: %s statuses=%s" % (round_no + 1, {k: round(v, 1) for k, v in peak.items()}, sorted(set(s for s, _ in results))))
summary = {
"payload_mib": round(len(body) / 1048576, 2), "concurrency": args.concurrency, "rounds": args.rounds,
"idle_mib": {k: round(v, 1) for k, v in idle.items()},
"peak_mib": {k: round(max(p.get(k, 0) for p in peaks), 1) for k in idle},
"container_limit_mib": 512,
"statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))},
"upload_seconds_median": round(statistics.median(times), 2), "upload_seconds_max": round(max(times), 2),
}
print(json.dumps(summary, indent=2))
main()
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Upload -> stop the serving replica -> download via the other; then restart everything and download again."""
import hashlib
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check, run # noqa: E402
def compose(*args):
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True)
def wait_ready(c):
for _ in range(60):
try:
c.call("GET", "/health/ready")
return
except Exception:
time.sleep(2)
check(False, "stack did not become ready")
def main():
c = Client()
alice, bob, _, chat = c.bootstrap()
att, content = c.attachment(alice, chat)
served_by = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["headers"]["X-Instance-Id"]
print("metadata served by", served_by)
first = c.call("POST", "/api/v1/chats/%s/attachments?filename=x.png" % chat["id"], 201, actor=alice, raw=content,
headers={"Content-Type": "image/png"}, schema="Attachment")
uploader = first["headers"]["X-Instance-Id"]
compose("stop", "-t", "3", uploader)
try:
time.sleep(3)
got = c.call("GET", "/api/v1/attachments/%s/content" % first["json"]["id"], actor=bob)
check(got["headers"]["X-Instance-Id"] != uploader, "download must be served by another replica")
check(hashlib.sha256(got["bytes"]).hexdigest() == first["json"]["sha256"], "bytes differ after uploader stopped")
print("ok uploaded via %s, downloaded via %s, identical sha256" % (uploader, got["headers"]["X-Instance-Id"]))
finally:
compose("start", uploader)
wait_ready(c)
compose("restart", "-t", "5", "postgres", "valkey", "minio", "api-1", "api-2", "nginx")
wait_ready(c)
got = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)
check(got["bytes"] == content, "original lost after full restart")
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
check(meta == att, "metadata changed after restart")
print("ok restart of all stateful services keeps metadata and original")
print("PASS: attachment failover + persistence")
run(main)
+115
View File
@@ -0,0 +1,115 @@
#!/usr/bin/env python3
"""Negative and boundary scenarios for attachments (grade 4). Needs the stack from README on BASE_URL."""
import hashlib
import os
import socket
import struct
import sys
import uuid
import zlib
from pathlib import Path
from urllib.parse import quote, urlparse
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check, run # noqa: E402
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
def chunk(kind, body):
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
def header_only_png(width, height):
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"")
def upload(c, actor, chat, body, ctype, expected, filename="a.png"):
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe=""))
return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype},
schema="Attachment" if expected == 201 else None)
def main():
c = Client()
alice, bob, eve, chat = c.bootstrap()
other = c.call("POST", "/api/v1/chats", 201, actor=alice,
body={"title": "Второй", "member_ids": [alice["id"]]}, schema="Chat")["json"]
eve_chat = c.call("POST", "/api/v1/chats", 201, actor=eve,
body={"title": "Чат Евы", "member_ids": [eve["id"]]}, schema="Chat")["json"]
ok = upload(c, alice, chat, SAMPLE, "image/png", 201)["json"]
check(ok["status"] == "ready" and ok["variants"] == [] and ok["error_code"] is None, "ready upload shape")
got = c.call("GET", "/api/v1/attachments/" + ok["id"] + "/content", actor=bob)
check(hashlib.sha256(got["bytes"]).hexdigest() == ok["sha256"], "bob downloads identical bytes")
check(int(got["headers"]["Content-Length"]) == len(SAMPLE), "download Content-Length")
c.call("GET", "/api/v1/attachments/" + ok["id"] + "/content?variant=thumbnail", 404, actor=bob)
upload(c, None, chat, SAMPLE, "image/png", 401)
upload(c, eve, chat, SAMPLE, "image/png", 404)
print("ok auth and membership on upload")
big = header_only_png(10, 10) + b"\0" * (10 * 1024 * 1024)
upload(c, alice, chat, big, "image/png", 413)
print("ok file over 10 MiB -> 413")
upload(c, alice, chat, header_only_png(8000, 6000), "image/png", 413)
print("ok 48 Mpx image -> 413 before decoding")
upload(c, alice, chat, SAMPLE[: len(SAMPLE) // 2], "image/png", 415)
upload(c, alice, chat, SAMPLE[:40] + b"garbage" * 50, "image/png", 415)
print("ok corrupted image -> 415")
upload(c, alice, chat, SAMPLE, "image/jpeg", 415)
upload(c, alice, chat, b"\xff\xd8\xff\xe0" + b"\0" * 200, "image/png", 415)
upload(c, alice, chat, b"MZ not an image at all", "image/png", 415)
upload(c, alice, chat, b"<svg xmlns='http://www.w3.org/2000/svg'/>", "image/png", 415)
upload(c, alice, chat, SAMPLE, "text/plain", 415)
upload(c, alice, chat, SAMPLE, "image/gif", 415)
print("ok MIME spoofing and unsupported types -> 415")
upload(c, alice, chat, b"", "image/png", 400)
for bad in ("../x.png", "a/b.png", "a\\b.png", "a\x00b.png", "a\nb.png", "", "x" * 256):
upload(c, alice, chat, SAMPLE, "image/png", 400, filename=bad)
print("ok dangerous filenames -> 400")
unicode_name = upload(c, alice, chat, SAMPLE, "image/png", 201, filename="фото 🙂.png")["json"]
check(unicode_name["filename"] == "фото 🙂.png", "display filename preserved")
own = upload(c, alice, other, SAMPLE, "image/png", 201)["json"]
foreign = upload(c, eve, eve_chat, SAMPLE, "image/png", 201)["json"]
msg = lambda actor, ch, ids, expected: c.call(
"POST", "/api/v1/chats/%s/messages" % ch["id"], expected, actor=actor,
body={"text": "", "attachment_ids": ids}, headers={"Idempotency-Key": uuid.uuid4().hex}, schema="Message" if expected == 201 else None)
msg(alice, chat, [str(uuid.uuid4())], 404)
msg(alice, chat, [foreign["id"]], 404)
msg(alice, chat, [own["id"]], 404)
msg(alice, chat, [ok["id"], ok["id"]], 400)
msg(alice, chat, ["not-a-uuid"], 400)
msg(alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400)
msg(eve, chat, [ok["id"]], 404)
c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice, body={"text": " "},
headers={"Idempotency-Key": uuid.uuid4().hex})
print("ok unknown / foreign / other-chat attachment ids rejected, no 500")
sent = msg(bob, chat, [ok["id"]], 201)["json"]
check(sent["attachment_ids"] == [ok["id"]] and sent["text"] == "", "attachment-only message")
history = c.call("GET", "/api/v1/chats/%s/messages" % chat["id"], actor=alice, schema="MessagePage")["json"]
check(any(m["attachment_ids"] == [ok["id"]] for m in history["items"]), "history keeps attachment_ids")
for suffix in ("", "/content"):
c.call("GET", "/api/v1/attachments/" + foreign["id"] + suffix, 404, actor=alice)
c.call("GET", "/api/v1/attachments/" + str(uuid.uuid4()) + suffix, 404, actor=alice)
c.call("GET", "/api/v1/attachments/not-a-uuid" + suffix, 400, actor=alice)
c.call("GET", "/api/v1/attachments/" + ok["id"] + suffix, 401)
parsed = urlparse(c.base)
sock = socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10)
request = ("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n"
"Content-Type: image/png\r\nContent-Length: %d\r\n\r\n" % (chat["id"], alice["token"], len(SAMPLE)))
sock.sendall(request.encode() + SAMPLE[: len(SAMPLE) // 2])
sock.close()
print("ok client abort mid-upload sent; server must stay healthy")
c.call("GET", "/health/ready", schema="Health")
after = upload(c, alice, chat, SAMPLE, "image/png", 201)["json"]
check(after["sha256"] == hashlib.sha256(SAMPLE).hexdigest(), "upload works after aborted one")
print("PASS: attachment negative scenarios")
if __name__ == "__main__":
run(main)
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/env python3
"""Hardening scenarios: corrupted images, filenames, attachment order, idempotency keys, oversized bodies."""
import hashlib
import os
import struct
import subprocess
import sys
import uuid
import zlib
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "student-tests"))
from client import Client, check, run # noqa: E402
from test_attachments import SAMPLE, chunk, header_only_png # noqa: E402
JPEG_STUB = bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9")
def png(width, height, filter_byte=0, level=1, corrupt_idat=False, noise=False):
row = bytes([filter_byte]) + (os.urandom(width * 3) if noise else b"\x10" * (width * 3))
raw = row * height
data = zlib.compress(raw, level)
if corrupt_idat:
mid = len(data) // 2
data = data[:mid] + bytes(b ^ 0xFF for b in data[mid:mid + 40]) + data[mid + 40:]
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", data) + chunk(b"IEND", b"")
def main():
c = Client()
alice, bob, eve, chat = c.bootstrap()
path = "/api/v1/chats/%s/attachments?filename=" % chat["id"]
def up(body, ctype, expected, name="a.png", **kw):
return c.call("POST", path + quote(name, safe=""), expected, actor=alice, raw=body,
headers={"Content-Type": ctype}, schema="Attachment" if expected == 201 else None)
up(SAMPLE[:33], "image/png", 415)
up(png(1, 1, filter_byte=5), "image/png", 415)
up(png(1, 1, filter_byte=0xFF), "image/png", 415)
up(png(64, 64, corrupt_idat=True), "image/png", 415)
up(JPEG_STUB, "image/jpeg", 415)
print("ok truncated PNG, bad filter byte, broken IDAT, 27-byte JPEG stub -> 415")
real = subprocess.run(
["docker", "run", "--rm", "--entrypoint", "python", "mtusi-messenger:lab4", "-c",
"import io,os,sys;from PIL import Image;b=io.BytesIO();"
"Image.frombytes('RGB',(1000,750),os.urandom(2250000)).save(b,'JPEG',quality=90);"
"sys.stdout.buffer.write(b.getvalue())"], capture_output=True, check=True).stdout
whole = up(real, "image/jpeg", 201, name="real.jpg")["json"]
check(c.call("GET", "/api/v1/attachments/" + whole["id"] + "/content", actor=bob)["bytes"] == real, "JPEG bytes differ")
up(real[: len(real) // 2] + b"\xff\xd9", "image/jpeg", 415, name="cut.jpg")
up(real[: len(real) // 2], "image/jpeg", 415, name="cut.jpg")
print("ok real JPEG accepted; first half + EOI and first half alone -> 415")
up(png(3000, 2000, filter_byte=7), "image/png", 415)
up(png(3000, 2000, corrupt_idat=True, noise=True), "image/png", 415)
print("ok same defects in images above the full-decode threshold (6 Mpx) -> 415")
up(png(3000, 2000), "image/png", 201)
print("ok valid 6 Mpx PNG accepted")
import time
big = png(6000, 6000, level=9)
started = time.monotonic()
r = up(big, "image/png", 201, name="36mpx.png")
took = time.monotonic() - started
check(took < 15, "36 Mpx PNG must be checked quickly, took %.1fs" % took)
print("ok valid 36 Mpx PNG (%d KiB on the wire, 108 MiB decoded) accepted in %.1fs" % (len(big) // 1024, took))
up(png(6000, 6000, filter_byte=9, level=9), "image/png", 415)
print("ok 36 Mpx PNG with bad filter byte -> 415")
for name in (" ", "\t", "bad\x7f.png", "bad\x85.png", ".", "..", "a
b.png"):
up(SAMPLE, "image/png", 400, name=name)
print("ok whitespace-only, DEL, C1 and dot filenames -> 400")
a = up(SAMPLE, "image/png", 201, name="a.png")["json"]
b = up(SAMPLE, "image/png", 201, name="b.png")["json"]
url = "/api/v1/chats/%s/messages" % chat["id"]
key = uuid.uuid4().hex
first = c.call("POST", url, 201, actor=alice, body={"text": "", "attachment_ids": [b["id"], a["id"]]},
headers={"Idempotency-Key": key}, schema="Message")["json"]
check(first["attachment_ids"] == [b["id"], a["id"]], "POST keeps attachment order")
replay = c.call("POST", url, 201, actor=alice, body={"attachment_ids": [b["id"], a["id"]], "text": ""},
headers={"Idempotency-Key": key}, schema="Message")["json"]
check(replay == first, "replay returns the same message")
c.call("POST", url, 409, actor=alice, body={"text": "", "attachment_ids": [a["id"], b["id"]]},
headers={"Idempotency-Key": key})
history = c.call("GET", url, actor=bob, schema="MessagePage")["json"]["items"]
check([m for m in history if m["id"] == first["id"]][0]["attachment_ids"] == [b["id"], a["id"]],
"history keeps attachment order")
print("ok attachment order preserved in POST, replay and history; reordered body -> 409")
key2 = uuid.uuid4().hex
plain = c.call("POST", url, 201, actor=alice, body={"text": "x"}, headers={"Idempotency-Key": key2}, schema="Message")["json"]
same = c.call("POST", url, 201, actor=alice, body={"text": "x", "attachment_ids": []},
headers={"Idempotency-Key": key2}, schema="Message")["json"]
check(plain == same, "absent attachment_ids equals []")
for bad in ("a b", "a/b", "k" * 129, ""):
c.call("POST", url, 400, actor=alice, body={"text": "k"}, headers={"Idempotency-Key": bad})
other = c.call("POST", url, 201, actor=bob, body={"text": "x"}, headers={"Idempotency-Key": key2}, schema="Message")["json"]
check(other["id"] != plain["id"] and other["sender_id"] == bob["id"], "idempotency key is per user")
print("ok idempotency: [] == absent, invalid keys -> 400, scoped per user")
huge = b"\x89PNG\r\n\x1a\n" + b"\0" * (12 * 1024 * 1024)
r = c.call("POST", path + "big.png", 413, actor=alice, raw=huge, headers={"Content-Type": "image/png"})
check(r["headers"].get_content_type() == "application/json", "413 above the nginx limit is API-shaped JSON")
print("ok 12 MiB body -> JSON 413")
print("PASS: input hardening")
run(main)
+210
View File
@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""LB under upload pressure: no OOM/restart of any container, no HTML 502, every rejection is API-shaped JSON.
python3 student-tests/test_lb_stress.py --concurrency 10 --rounds 3 --oversize 200"""
import argparse
import base64
import collections
import http.client
import json
import os
import struct
import subprocess
import sys
import threading
import uuid
import zlib
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
HOST, PORT = "localhost", 8080
LIMIT = 10485760
def noise_png(side):
raw = b"".join(b"\0" + os.urandom(side * 3) for _ in range(side))
def chunk(kind, body):
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
ihdr = struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0)
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")
def request(method, path, body=None, headers=None, timeout=90):
"""Returns (status, content_type, payload) or ('EXC:<name>', '', b'')."""
conn = http.client.HTTPConnection(HOST, PORT, timeout=timeout)
try:
conn.request(method, path, body=body, headers=headers or {})
response = conn.getresponse()
return response.status, response.getheader("Content-Type", ""), response.read()
except Exception as exc:
return "EXC:" + type(exc).__name__, "", b""
finally:
conn.close()
def actor(label):
username = "st_%s_%s" % (label, uuid.uuid4().hex[:10])
password = "T3st-" + uuid.uuid4().hex
body = {"username": username, "display_name": label, "password": password}
status, _, _ = request("POST", "/api/v1/users", json.dumps(body), {"Content-Type": "application/json"})
assert status == 201, status
basic = base64.b64encode((username + ":" + password).encode()).decode()
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
assert status == 201, status
session = json.loads(data)
return session["user"]["id"], {"Authorization": "Bearer " + session["token"]}
def container_ids():
out = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
names = {}
for cid in out:
name = subprocess.run(["docker", "inspect", "--format", "{{.Name}}", cid], capture_output=True, text=True).stdout
names[cid] = name.strip().lstrip("/")
return names
def peak_mib(name):
out = subprocess.run(["docker", "exec", name, "cat", "/sys/fs/cgroup/memory.peak"], capture_output=True, text=True)
return round(int(out.stdout) / 1048576, 1) if out.stdout.strip().isdigit() else None
class Events(threading.Thread):
def __init__(self):
super().__init__(daemon=True)
self.lines = []
self.proc = subprocess.Popen(
["docker", "events", "--filter", "type=container", "--filter", "event=oom", "--filter", "event=die",
"--filter", "event=restart", "--format", "{{.Action}} {{.Actor.Attributes.name}} exit={{.Actor.Attributes.exitCode}}"],
stdout=subprocess.PIPE, text=True,
)
def run(self):
for line in self.proc.stdout:
self.lines.append(line.strip())
def stop(self):
self.proc.terminate()
def is_json_error(result, code):
status, ctype, data = result
if status != code or not ctype.startswith("application/json"):
return False
return json.loads(data)["error"]["code"]
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--concurrency", type=int, default=10)
parser.add_argument("--rounds", type=int, default=3)
parser.add_argument("--oversize", type=int, default=200)
args = parser.parse_args()
# memory.peak is cumulative since container start, so read it after a fresh recreate.
failures = []
events = Events()
events.start()
uid, auth = actor("a")
_, outsider = actor("b")
status, _, data = request("POST", "/api/v1/chats", json.dumps({"title": "stress", "member_ids": [uid]}),
{**auth, "Content-Type": "application/json"})
assert status == 201, status
chat = json.loads(data)["id"]
upload = "/api/v1/chats/%s/attachments?filename=s.png" % chat
png_headers = {**auth, "Content-Type": "image/png"}
body = noise_png(1700)
print("== %d parallel uploads of %d bytes, %d rounds ==" % (args.concurrency, len(body), args.rounds))
ids, tally = [], collections.Counter()
for _ in range(args.rounds):
with ThreadPoolExecutor(args.concurrency) as pool:
results = list(pool.map(lambda _: request("POST", upload, body, png_headers), range(args.concurrency)))
for status, _, data in results:
tally[status] += 1
if status == 201:
ids.append(json.loads(data)["id"])
print("upload statuses:", dict(tally))
if set(tally) != {201}:
failures.append("parallel uploads: %s" % dict(tally))
print("== %d parallel downloads ==" % args.concurrency)
with ThreadPoolExecutor(args.concurrency) as pool:
results = list(pool.map(lambda i: request("GET", "/api/v1/attachments/%s/content" % i, headers=auth),
ids[:args.concurrency * 2]))
tally = collections.Counter(r[0] for r in results)
print("download statuses:", dict(tally))
if set(tally) != {200} or any(r[2] != body for r in results):
failures.append("parallel downloads: %s" % dict(tally))
print("== %d parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), %d rounds ==" % (args.concurrency, args.rounds))
jpeg = subprocess.run(
["docker", "run", "--rm", "--entrypoint", "python", "mtusi-messenger:lab4", "-c",
"import io,sys;from PIL import Image;b=io.BytesIO();"
"Image.new('CMYK',(8000,5000),(120,30,200,5)).save(b,'JPEG',quality=90,progressive=True);"
"sys.stdout.buffer.write(b.getvalue())"], capture_output=True, check=True).stdout
tally = collections.Counter()
for _ in range(args.rounds):
with ThreadPoolExecutor(args.concurrency) as pool:
results = list(pool.map(lambda _: request("POST", upload, jpeg, {**auth, "Content-Type": "image/jpeg"}),
range(args.concurrency)))
tally.update(r[0] for r in results)
print("upload statuses:", dict(tally))
if set(tally) != {201}:
failures.append("parallel progressive JPEG: %s" % dict(tally))
cases = [
("10 MiB + 1 byte, Content-Length", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (LIMIT - 7), png_headers, 413, "payload_too_large"),
("10 MiB + 1 byte, chunked", upload, iter([b"\x89PNG\r\n\x1a\n" + b"\0" * (LIMIT - 7)]), png_headers, 413, "payload_too_large"),
("10 MiB + 1 byte, chunked in 64 KiB pieces", upload,
[b"\x89PNG\r\n\x1a\n"] + [b"\0" * 65536] * 160, png_headers, 413, "payload_too_large"),
("8 MiB to a foreign chat", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (8 << 20), {**outsider, "Content-Type": "image/png"}, 404, "not_found"),
("8 MiB without session", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (8 << 20), {"Content-Type": "image/png"}, 401, "unauthorized"),
("8 MiB with wrong magic", upload, b"X" * (8 << 20), png_headers, 415, "unsupported_media_type"),
("8 MiB with wrong Content-Type", upload, b"X" * (8 << 20), {**auth, "Content-Type": "image/gif"}, 415, "unsupported_media_type"),
("70 KiB JSON message", "/api/v1/chats/%s/messages" % chat, b'{"text":"' + b"x" * (70 << 10) + b'"}',
{**auth, "Content-Type": "application/json", "Idempotency-Key": "k"}, 413, "payload_too_large"),
]
for name, path, payload, headers, code, error in cases:
print("== %s: %d requests, %d at a time ==" % (name, args.oversize, args.concurrency))
materialized = list(payload) if not isinstance(payload, bytes) else None
def one(_):
data = payload if materialized is None else iter(materialized)
extra = {} if materialized is None else {"Transfer-Encoding": "chunked"}
conn_headers = {**headers, **extra}
conn = http.client.HTTPConnection(HOST, PORT, timeout=90)
try:
conn.request("POST", path, body=data, headers=conn_headers, encode_chunked=materialized is not None)
response = conn.getresponse()
return response.status, response.getheader("Content-Type", ""), response.read()
except Exception as exc:
return "EXC:" + type(exc).__name__, "", b""
finally:
conn.close()
with ThreadPoolExecutor(args.concurrency) as pool:
results = list(pool.map(one, range(args.oversize)))
tally = collections.Counter((r[0], r[1].split(";")[0]) for r in results)
print(" ", {"%s %s" % k: v for k, v in tally.items()})
if not all(is_json_error(r, code) == error for r in results):
failures.append("%s: %s" % (name, dict(tally)))
status, _, _ = request("GET", "/health/ready")
if status != 200:
failures.append("not ready after the run: %s" % status)
events.stop()
bad = [line for line in events.lines if " mtusi-lab4-" in line]
print("container events during the run:", bad or "none")
if bad:
failures.append("container events: %s" % bad)
print("cgroup memory.peak, MiB:", {n: peak_mib(n) for n in container_ids().values() if "api-" in n or "nginx" in n})
for failure in failures:
print("FAIL:", failure)
print("PASS: LB stress" if not failures else "FAILED: %d" % len(failures))
sys.exit(1 if failures else 0)
main()
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""S3 down: upload/download answer 503 (never 500), no partial rows are exposed, service recovers."""
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, check, run # noqa: E402
def compose(*args):
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True)
def main():
c = Client()
alice, bob, _, chat = c.bootstrap()
att, content = c.attachment(alice, chat)
path = "/api/v1/chats/%s/attachments?filename=x.png" % chat["id"]
compose("stop", "-t", "2", "minio")
try:
time.sleep(4)
c.call("POST", path, 503, actor=alice, raw=content, headers={"Content-Type": "image/png"})
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 503, actor=bob)
c.call("GET", "/health/ready", 503)
c.call("GET", "/health/ready", 503)
c.message(alice, chat, "текст без S3")
c.call("GET", "/health/live", schema="Health")
c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")
c.call("GET", "/api/v1/chats/%s/messages" % chat["id"], actor=bob, schema="MessagePage")
print("ok S3 down: upload/download/ready -> 503 JSON, metadata and text messages still served through the LB (no 502)")
finally:
compose("start", "minio")
got = None
started = time.monotonic()
while time.monotonic() - started < 90:
got = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", [200, 503], actor=bob)
if got["status"] == 200:
break
time.sleep(2)
check(got["status"] == 200, "download did not recover after S3 came back")
print("ok recovered %.0fs after S3 start" % (time.monotonic() - started))
check(got["bytes"] == content, "original intact after S3 outage")
streak = 0
for _ in range(60):
streak = streak + 1 if c.call("GET", "/health/ready", [200, 503])["status"] == 200 else 0
if streak >= 6:
break
time.sleep(0.5)
check(streak >= 6, "both replicas must report ready again")
print("ok ready is 200 again")
compose("pause", "minio")
try:
started = time.monotonic()
c.timeout = 40
c.call("POST", path, 503, actor=alice, raw=content, headers={"Content-Type": "image/png"})
took = time.monotonic() - started
check(took < 20, "upload to a hung S3 must fail fast, took %.1fs" % took)
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 503, actor=bob)
print("ok hung S3: upload answered 503 in %.1fs" % took)
finally:
compose("unpause", "minio")
time.sleep(3)
c.call("POST", path, 201, actor=alice, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment")
print("PASS: S3 outage behaviour")
run(main)