ЛР 4 — БВТ2301 — Курлова — на 5 #49
+39
-9
@@ -1,20 +1,50 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env. Values below are the ones the evidence was produced with.
|
||||
# Secrets are NOT here: they are files in ./secrets (see REPORT.md), referenced by *_FILE paths.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
GRACEFUL_TIMEOUT=8
|
||||
API_REPLICAS=2
|
||||
|
||||
# Sessions / idempotency
|
||||
SESSION_TTL_SECONDS=3600
|
||||
SESSION_STORE=valkey
|
||||
SESSION_CACHE_TTL_SECONDS=30
|
||||
SESSION_REVOKED_TTL_SECONDS=60
|
||||
SESSION_SWEEP_SECONDS=60
|
||||
CACHE_TIMEOUT=0.2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
|
||||
# Database
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_TIMEOUT=3
|
||||
DB_CONNECT_TIMEOUT=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
DB_OP_TIMEOUT=6
|
||||
DB_READY_TIMEOUT=1.5
|
||||
READY_CHECK_SECONDS=1
|
||||
READY_FAILURES=2
|
||||
|
||||
# Request limits (REQUEST_TIMEOUT must exceed S3 timeouts + upload time)
|
||||
MAX_INFLIGHT_REQUESTS=64
|
||||
MAX_BODY_BYTES=65536
|
||||
MAX_RSS_MB=0
|
||||
REQUEST_TIMEOUT=30
|
||||
|
||||
# Object storage and attachments
|
||||
S3_ENDPOINT=http://minio:9000
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_CONNECT_TIMEOUT=3
|
||||
S3_READ_TIMEOUT=6
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
UPLOAD_SPOOL_BYTES=1048576
|
||||
UPLOAD_CONCURRENCY=2
|
||||
RECONCILE_INTERVAL_SECONDS=300
|
||||
RECONCILE_GRACE_SECONDS=900
|
||||
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_DEFAULT_TIMEOUT=120 \
|
||||
PIP_RETRIES=10
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libjpeg-turbo-progs=1:2.1.5-2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-deps -r requirements.txt
|
||||
|
||||
COPY app/ app/
|
||||
COPY migrations/ migrations/
|
||||
COPY scripts/migrate.py scripts/reconcile.py scripts/reconcile_experiment.py scripts/jpeg_check_experiment.py scripts/
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
CMD ["sh", "-c", "exec uvicorn app.main:app --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""]
|
||||
@@ -1,37 +1,146 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Алёна Курлова, БВТ2301
|
||||
- Целевая оценка; выбранный трек 5: **5, трек 2 «Согласованность двух хранилищ»**
|
||||
- Commit/PR: PR [#49](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/49), ветка `solution/bvt2301-kurlova`
|
||||
- Источники, библиотеки, помощь ИИ: FastAPI, psycopg, boto3, Pillow, MinIO; код лабы 3 перенесён и расширен. Реализацию, тесты и черновик отчёта выполнял Claude Code (ИИ) под руководством автора; результаты прогонов ниже получены на локальном стенде.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker Desktop (Compose v2), Python 3.9+, вход `docker login` (Docker Hub). Проверено на macOS, arm64.
|
||||
|
||||
```sh
|
||||
cp .env.example .env # значения в примере совпадают с теми, на которых получены evidence/
|
||||
mkdir -p secrets
|
||||
printf 'messenger' > secrets/postgres_password
|
||||
printf 'postgresql://messenger:messenger@postgres:5432/messenger' > secrets/database_url
|
||||
printf 'valkey://valkey:6379/0' > secrets/cache_url
|
||||
printf 'messenger_root' > secrets/minio_root_user
|
||||
python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/minio_root_password
|
||||
printf 'messenger_app' > secrets/s3_access_key_id
|
||||
python3 -c "import secrets;print(secrets.token_urlsafe(24),end='')" > secrets/s3_secret_access_key
|
||||
docker compose --env-file .env up -d --build # postgres, valkey, minio, minio-init, migrate, api-1, api-2, nginx
|
||||
make test # ждёт готовности по /health/ready
|
||||
```
|
||||
|
||||
Остановка с сохранением данных: `docker compose stop`. Намеренное удаление учебных данных: `docker compose down -v`. Каталог `secrets/` и `.env` в git не попадают.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
| Роль | Сервис | Версия | Health | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| LB | nginx | 1.27.3-alpine | `/health/live`; health-скрипт опрашивает `/health/ready` реплик | — | `127.0.0.1:8080` | — |
|
||||
| API ×2 | api-1, api-2 | собираются из исходников, python 3.12.8 | `/health/ready` | нет (stateless) | только внутренняя сеть | Docker secrets |
|
||||
| БД | postgres | 17.7-alpine@sha256 | `pg_isready` | `pgdata` | внутренняя | secret |
|
||||
| Кэш сессий | valkey | 8.1-alpine@sha256 | `valkey-cli ping` | нет | внутренняя | — |
|
||||
| S3 | minio | `madlabcg/minio@sha256:ad134b2d…` | `mc ready local` | `miniodata` | внутренняя, **порты и консоль не опубликованы** (`MINIO_BROWSER=off`) | root-secret только у minio и minio-init |
|
||||
| Bootstrap S3 | minio-init | тот же образ | — | — | — | root + app secrets |
|
||||
|
||||
**Источник истины.** Метаданные — PostgreSQL (`attachments`, `message_attachments`), байты — приватный bucket. Object key назначает сервер (`attachments/<uuid4>`), `filename` хранится только как отображаемое имя.
|
||||
|
||||
**Порядок записи и границы согласованности** (ответ на «что если PUT прошёл, а COMMIT нет»):
|
||||
1. тело читается потоком в `SpooledTemporaryFile` (1 MiB в RAM, остальное во временный файл), считается SHA-256, лимит 10 MiB применяется на лету; MIME проверяется по magic bytes и Pillow, лимит пикселей проверяется по заголовку до декодирования; целостность проверяет [app/imagecheck.py](app/imagecheck.py): для PNG собственный потоковый сканер (CRC всех chunk'ов, поля IHDR, zlib-поток разжимается порциями по 1 MiB, filter-байт каждой строки ∈ 0..4, размер данных точно равен вычисленному по IHDR, Adam7 учитывается), для JPEG дочерний процесс `djpeg -strict` (libjpeg-turbo 2.1.5) декодирует весь поток в масштабе 1/8 и считает любое предупреждение libjpeg ошибкой, поэтому обрезанный поток отвергается и с дописанным EOI; одновременно идёт одна такая проверка на реплику, память процесса ограничена `-maxmemory`, число сканов `-maxscans 100`;
|
||||
2. в БД вставляется строка `status='uploading'` (коммит);
|
||||
3. PUT в S3;
|
||||
4. `UPDATE … SET status='ready' WHERE status='uploading'` — только после этого клиент получает **201**.
|
||||
|
||||
Любой сбой между шагами оставляет либо `uploading`-строку (с объектом или без), либо ничего. Клиентам такие строки не видны (404), прикрепить их к сообщению нельзя. Объект без строки получиться не может, потому что строка вставляется раньше. При ошибке API сам пытается компенсировать (удалить объект и строку); то, что не удалось, собирает reconciler.
|
||||
|
||||
**Reconciler** ([app/reconcile.py](app/reconcile.py), CLI [scripts/reconcile.py](scripts/reconcile.py), фоновый цикл в каждом API раз в `RECONCILE_INTERVAL_SECONDS`=300 с, grace `RECONCILE_GRACE_SECONDS`=900 с). Работает под `pg_try_advisory_lock`, поэтому две реплики не дублируют работу. Правила:
|
||||
- `uploading` старше grace **и** объект (если есть) старше grace по `LastModified` → удалить объект и строку; старая строка со свежим объектом не трогается (повторная проверка HEAD перед удалением);
|
||||
- объект без строки и старше grace → удалить (перед удалением строка перепроверяется);
|
||||
- `ready`-строка, подтверждённая (`updated_at`) раньше grace, без объекта (перепроверка HEAD) → `status='failed'`, `error_code='object_missing'`; строка **не удаляется**, даже если на неё ссылается сообщение;
|
||||
- по умолчанию dry-run (`--apply` меняет данные), повторный запуск идемпотентен.
|
||||
|
||||
**Что изменено в lab3-коде.** Сообщение принимает `attachment_ids` (0–10) и допускает пустой `text` при наличии вложений (миграция 0004 ослабляет CHECK). Хеш идемпотентности включает `attachment_ids`; повтор без ключа `attachment_ids` равен `[]`. **Критичные зависимости `/health/ready`: PostgreSQL и S3.** Недоступность любой из них даёт 503 в общем формате ошибки с `X-Instance-Id`; состояние S3 обновляется фоновой пробой раз в секунду (отдельный клиент с таймаутами 1 с / 1.5 с, чтобы зависший S3 не вешал проверку). При этом недоступный S3 **не** блокирует остальной API: текстовые сообщения и метаданные продолжают работать (запросы блокируются только при недоступной БД), а вложения отвечают 503. Чтобы 503 не выбивал реплики из upstream и клиенты не видели 502, nginx не использует `http_503` как признак сбоя для `/health/`, `/attachments/` и upload; если обе реплики не готовы, health-скрипт оставляет обе в ротации. **Лимиты тела и память LB.** nginx сам отклоняет слишком большие тела JSON-ошибкой 413 (с `X-Request-Id`): 10 MiB для загрузки вложений и 64 KiB для остальных запросов, то есть те же границы, что в API. Загрузка идёт потоком до API (`proxy_request_buffering off`), скачивание — без временных файлов (`proxy_max_temp_file_size 0`), поэтому тела файлов не попадают в память и page cache контейнера nginx; воркеров 2, а не `auto`, чтобы потребление не зависело от числа ядер хоста. API больше не отправляет `Connection: close` на отказах с недочитанным телом: это закрывало соединение раньше, чем nginx успевал прочитать ответ, и давало HTML 502; uvicorn дочитывает и отбрасывает остаток тела. Лимит памяти контейнера API поднят до 512 MiB, потому что проверка законного прогрессивного CMYK-JPEG на 40 Мпикс требует 307 MiB. S3-клиент: без повторов, connect 3 с, read 6 с; компенсация при сбое PUT выполняется в фоне.
|
||||
|
||||
**Дополнительная валидация входных данных.** filename: 1..255 code points, не только пробелы, не `.`/`..`, без `/`, `\` и любых символов категорий Cc/Cs/Zl/Zp (в том числе DEL и C1). `Idempotency-Key` не обрезается (HTTP-парсер уже убирает крайние пробелы); порядок `attachment_ids` сохраняется в ответе, повторе и истории, переставленный список с тем же ключом — 409.
|
||||
|
||||
**Права S3.** API использует отдельного пользователя `messenger_app` с политикой [infra/minio/app-policy.json](infra/minio/app-policy.json): Get/Put/DeleteObject только на `attachments/*` и ListBucket. Bucket создаёт и делает приватным [infra/minio/init.sh](infra/minio/init.sh); скрипт идемпотентен (`mb --ignore-existing`, повтор политики и пользователя безвреден), запускается при каждом `up`.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke | `make test` | PASS (загрузка, привязка, скачивание вторым участником, 404 постороннему) | [evidence/smoke.txt](evidence/smoke.txt) |
|
||||
| Байты в S3, метаданные в БД, 201 после сохранения | код `upload_attachment`; `student-tests/test_attachments.py` | sha256/размер совпадают, `Content-Length` скачивания равен оригиналу | [evidence/attachments_tests.txt](evidence/attachments_tests.txt) |
|
||||
| Upload → остановка реплики → download через другую; restart с volumes | `student-tests/test_attachment_resilience.py` | загрузил api-1, скачал api-2, sha256 совпал; после рестарта postgres/valkey/minio/api/nginx метаданные и оригинал целы | [evidence/attachment_resilience.txt](evidence/attachment_resilience.txt) |
|
||||
| Общие persistence / failover | `make test-persistence`, `make test-failover` (хуки `scripts/restart.sh`, `scripts/stop-one.sh`) | PASS; failover: 131 успешная запись, 0 ошибок, восстановление 0.44 с | [evidence/persistence.txt](evidence/persistence.txt), [evidence/failover.txt](evidence/failover.txt) |
|
||||
| Лимиты 10 MiB / 40 Mpx, MIME по содержимому, опасные filename | `test_attachments.py` | 10 MiB+ → 413; PNG 8000×6000 → 413 до декодирования; подмена MIME и мусор → 415; `../`, `/`, `\`, NUL, `\n`, пустое, 256 символов → 400 | [evidence/attachments_tests.txt](evidence/attachments_tests.txt) |
|
||||
| Устойчивость к некорректным входным данным | `student-tests/test_input_hardening.py`, `scripts/jpeg_check_experiment.py` (запуск: `docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py`) | усечённый PNG, filter-байт 5/0xFF, битый IDAT, 27-байтный JPEG, настоящий JPEG, обрезанный наполовину с дописанным `FF D9` и без него → 415 (PNG-дефекты также на 6 и 36 Mpx); валидные baseline, progressive, grayscale, CMYK, файл с данными после EOI и 40 Mpx принимаются; filename из пробелов, DEL, C1, `.` → 400; порядок вложений; `[]` ≡ отсутствует; 12 MiB → JSON 413 | [evidence/input_hardening.txt](evidence/input_hardening.txt), [evidence/jpeg_check.txt](evidence/jpeg_check.txt) |
|
||||
| Чужое / неизвестное / чужого чата attachment_id, ошибки не 500 | `test_attachments.py` | 404 / 404 / 404; дубликаты, не-UUID, >10 → 400 | там же |
|
||||
| Недоступность и зависание S3 | `student-tests/test_s3_outage.py` | остановка: upload/download/ready → 503 JSON с `X-Instance-Id`, метаданные и текст работают, без 502; `docker pause`: upload отвечает 503 за 7 с; восстановление за секунды | [evidence/s3_outage.txt](evidence/s3_outage.txt) |
|
||||
| Прерывание загрузки | обрыв сокета на половине тела в `test_attachments.py` | сервер здоров, следующий upload проходит | там же |
|
||||
| Компенсация / сверка, безопасный grace | трек 2, ниже | 18/18 проверок | [evidence/reconciler_experiment.txt](evidence/reconciler_experiment.txt) |
|
||||
| Пик памяти при ≥10 одновременных загрузках | `student-tests/memory_experiment.py`, `student-tests/test_lb_stress.py` | см. ниже | [evidence/memory_experiment.txt](evidence/memory_experiment.txt), [evidence/lb_stress.txt](evidence/lb_stress.txt) |
|
||||
| Нагрузка на LB: нет OOM и падений, отказы в JSON | `python3 student-tests/test_lb_stress.py --concurrency 10 --rounds 3 --oversize 100` и то же с 30 параллельными | загрузки 201, скачивания 200, 800 запросов-отказов дают правильный JSON без единого 502, событий `oom`/`die` контейнеров нет, пик nginx 28–34 MiB при лимите 64 | [evidence/lb_stress.txt](evidence/lb_stress.txt), [evidence/lb_stress_30.txt](evidence/lb_stress_30.txt) |
|
||||
| Private bucket вне API, консоль не открыта | `student-tests/check_private_bucket.sh` | анонимные GET/LIST из сети compose → 403, порты 9000/9001 на хосте не опубликованы, `mc anonymous get` → private | [evidence/private_bucket.txt](evidence/private_bucket.txt) |
|
||||
| Отдельные S3 credentials с ограниченными правами | `infra/minio/app-policy.json`, `init.sh` | API работает под `messenger_app`; root-secret API не получает | `compose.yaml` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Проверка hook'ов подтверждает перезапуск реальных процессов: `restart.sh` перезапускает postgres, valkey, minio, api-1, api-2, nginx без `down -v`; `stop-one.sh` останавливает только указанную реплику.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### 1. Память при 10 одновременных загрузках
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
Гипотеза: основная память уходит на параллельное декодирование Pillow и S3 PUT, а не на хранение тел запросов; ограничение числа одновременных тяжёлых стадий снизит пик.
|
||||
|
||||
Методика: [student-tests/memory_experiment.py](student-tests/memory_experiment.py), PNG 8.27 MiB (1700×1700, несжимаемый шум), 10 параллельных загрузок через LB, 3 раунда, замер `docker stats` каждого API (0.5 CPU; лимит контейнера был 256 MiB, в итоговой версии 512 MiB). Mac arm64, Docker Desktop.
|
||||
|
||||
| Вариант | Простой, MiB | Пик api-1 / api-2, MiB | Динамика по раундам | 201 / всего |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| До: без ограничения тяжёлых стадий | 88 | 182 / 155 | рост 144 → 166 → 182 | 30 / 30 |
|
||||
| После: `UPLOAD_CONCURRENCY=2` на реплику | 87 | 110 / 116 | 110 → 109 → 110, роста нет | 30 / 30 |
|
||||
| После + потоковый PNG-сканер (итоговая версия) | см. файл | 116 / 107 | роста нет | 30 / 30 |
|
||||
|
||||
**Цифры таблицы занижены.** `docker stats` не учитывает page cache и даёт срезы раз в 1–2 с. Точный показатель лимита — `memory.peak` из cgroup: в этом же сценарии 154–163 MiB, `oom_kill=0`. Худший сценарий — стресс-тест с 10–30 параллельными прогрессивными CMYK-JPEG на 40 Мпикс: `memory.peak` API 444–491 MiB из 512 (вместе с page cache, который ядро может вытеснить), `oom_kill=0`, рестартов нет. Постоянная память процесса API после такой нагрузки около 133 MiB, один процесс проверки до 307 MiB ([evidence/jpeg_check.txt](evidence/jpeg_check.txt)). Запас в худшем случае небольшой.
|
||||
|
||||
Сырые данные: [evidence/memory_experiment_before_semaphore.txt](evidence/memory_experiment_before_semaphore.txt), [evidence/memory_experiment.txt](evidence/memory_experiment.txt). Ограничения: одна конфигурация железа; один размер файла; медиана длительности загрузки выросла с 0.82 до 1.15 с (цена ограничения), в итоговой версии 0.63 с. Причину роста «до» (фрагментация аллокатора или реальная утечка) я не выяснял.
|
||||
|
||||
### 2. Трек 2 — reconciler с внедрением ошибок
|
||||
|
||||
Гипотеза: сверка по правилам выше находит и исправляет расхождения, не трогая свежие и используемые вложения.
|
||||
|
||||
Методика: [scripts/reconcile_experiment.py](scripts/reconcile_experiment.py) внутри api-контейнера снимает базовую линию (в БД могут быть остатки прошлых тестов), создаёт строки/объекты, имитирующие сбои на обеих сторонах, и запускает dry-run, apply, apply после истечения grace, повторный apply. Счётчики сравниваются с базовой линией. `LastModified` объекта в MinIO задним числом не изменить, поэтому «старый объект» моделируется ожиданием grace.
|
||||
|
||||
| Внедрённая ошибка | Ожидание | Наблюдение |
|
||||
| --- | --- | --- |
|
||||
| `uploading` старше grace, объект есть, но моложе grace (падение после PUT) | не трогать до истечения grace | сохранены; собраны после grace |
|
||||
| `uploading` старше grace, но под его ключом свежий объект | не трогать | не тронуты; собраны после grace |
|
||||
| `ready`, подтверждённая только что, объект ещё не виден | не помечать missing | не помечена |
|
||||
| `uploading` старше grace, объекта нет (падение до PUT) | строка удалена | удалена |
|
||||
| `uploading` свежее grace (загрузка идёт) | не трогать | не тронута; после истечения grace собрана |
|
||||
| объект без строки внутри grace | не трогать | не тронут; удалён после grace |
|
||||
| `ready`, объект потерян | пометить `failed/object_missing` | помечено |
|
||||
| `ready`, привязано к сообщению, объект на месте | не трогать | не тронуто |
|
||||
| `ready`, привязано к сообщению, объект потерян | строку не удалять, пометить | сохранена, `failed` |
|
||||
| dry-run | ничего не менять, но показать счётчики | счётчики верны, состояние не изменилось |
|
||||
| повторный apply | нулевые изменения | 0 / 0 / 0 |
|
||||
|
||||
Итог: 18/18 проверок ([evidence/reconciler_experiment.txt](evidence/reconciler_experiment.txt)). **Стоимость обхода:** 1000 `ready`-вложений (около 1100 объектов вместе с остатками прошлых тестов) — 1.25 с в dry-run (в основном 1000 HEAD-запросов к S3; листинг идёт страницами по 500 с одним запросом к БД на страницу). Оценка масштабирования линейна, на больших объёмах не измерялась.
|
||||
|
||||
### 3. Отказы
|
||||
|
||||
| Воздействие | Ожидаем | Наблюдали | Восстановление / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Остановка обслужившей API-реплики | download через другую реплику, тот же файл | api-1 → api-2, sha256 совпал | без потерь |
|
||||
| Остановка MinIO | 503 на вложениях и ready, текст работает | upload/download/ready → 503 JSON, сообщения читаются, 502 нет | возврат за секунды после старта, без потерь |
|
||||
| Зависание MinIO (`docker pause`) | быстрый 503 без вечного ожидания | upload → 503 за 7 с, download → 503 | восстановление после `unpause` |
|
||||
| Остановка PostgreSQL | 503 JSON с `X-Instance-Id` на ready и бизнес-эндпоинтах | так и есть | возврат после старта |
|
||||
| Рестарт postgres, valkey, minio, API, nginx с volumes | метаданные и байты на месте | сохранены | без потерь |
|
||||
| 10 и 30 одновременных загрузок 8.3 MiB через LB | ограниченная обработка без падения стенда | все 201, событий `oom`/`die` нет, пик nginx 28–34 MiB | потерь нет |
|
||||
| Обрыв клиента на середине upload | нет 500 и мусора | сервер здоров; строка в БД ещё не создана | — |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Образ MinIO.** Официальные образы `minio/minio` на Docker Hub и `quay.io/minio/minio` в моей среде недоступны (pull denied / 401), поэтому использован сторонний `madlabcg/minio`, закреплённый по digest. Его происхождение и содержимое я не проверял; для защиты нужно заменить образ на проверенный (собрать из исходников MinIO или взять образ, указанный преподавателем).
|
||||
- **Валидация изображений.** `djpeg -strict` считает ошибкой любое предупреждение libjpeg, поэтому будет отвергнут и JPEG с мелким дефектом, который обычные программы показывают (например, лишние байты перед маркером). Это сознательный выбор в пользу строгости. Подмена байтов внутри данных, не нарушающая структуру потока, может не обнаруживаться: такой файл корректно декодируется в другую картинку. PNG-сканер не проверяет содержимое ancillary chunk'ов, кроме CRC.
|
||||
- **OOM балансировщика.** Само падение nginx по OOM на моей машине (10 ядер, Docker Desktop) не воспроизвелось, воспроизвелся пик у лимита: 67–80 MiB при 64. Устранена причина (тела файлов через nginx больше не спулятся на диск, число воркеров фиксировано); после правок пик 28–34 MiB. На конфигурации, где наблюдалось падение, правка отдельно не проверялась.
|
||||
- **Потоковая загрузка через LB.** С `proxy_request_buffering off` медленный клиент занимает слот API на всё время передачи, но не дольше `REQUEST_TIMEOUT` (30 с) и в пределах `MAX_INFLIGHT_REQUESTS`. Ошибка 413, выданная самим nginx, содержит `X-Instance-Id: lb`, потому что запрос до реплики не доходил.
|
||||
- Проверка размера по `Content-Length` невозможна для chunked-запросов: тогда лимит применяется по факту чтения потока.
|
||||
- Проверка PNG ограничена по времени (`SCAN_SECONDS`=10 с); на очень медленной машине валидный файл с огромной степенью сжатия может получить 415.
|
||||
- Вложения, загруженные, но так и не прикреплённые к сообщению, не считаются мусором и не удаляются; политики TTL для них нет.
|
||||
- Reconciler при проверке `ready`-строк делает HEAD на каждую; для очень больших объёмов нужен инкрементальный режим по курсору или S3 Inventory.
|
||||
- Нагрузочное тестирование с заданным arrival rate, p95/p99 не выполнялось; измерялась только память.
|
||||
- Не реализованы: трек 1 (presigned upload), варианты изображений (lab5), TLS (локальный стенд с синтетическими данными).
|
||||
- Секреты в `secrets/` сгенерированы локально и не коммитятся; учётные данные Postgres в примере выше — учебные.
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import os
|
||||
import socket
|
||||
|
||||
|
||||
def _int(name, default):
|
||||
value = os.environ.get(name)
|
||||
return int(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _float(name, default):
|
||||
value = os.environ.get(name)
|
||||
return float(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _secret(name, default=""):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
return os.environ.get(name, default)
|
||||
|
||||
|
||||
DATABASE_URL = _secret("DATABASE_URL")
|
||||
CACHE_URL = _secret("CACHE_URL")
|
||||
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
|
||||
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
|
||||
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
|
||||
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
|
||||
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
||||
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
|
||||
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
|
||||
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
|
||||
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
|
||||
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
|
||||
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
|
||||
READY_FAILURES = _int("READY_FAILURES", 2)
|
||||
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
|
||||
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
|
||||
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
||||
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
|
||||
PASSWORD_MIN = 12
|
||||
PASSWORD_MAX = 128
|
||||
SESSION_REVOKED_TTL_SECONDS = max(
|
||||
_int("SESSION_REVOKED_TTL_SECONDS", 60),
|
||||
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
|
||||
)
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
|
||||
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY_ID = _secret("S3_ACCESS_KEY_ID")
|
||||
S3_SECRET_ACCESS_KEY = _secret("S3_SECRET_ACCESS_KEY")
|
||||
MAX_UPLOAD_BYTES = _int("MAX_UPLOAD_BYTES", 10485760)
|
||||
MAX_IMAGE_PIXELS = _int("MAX_IMAGE_PIXELS", 40000000)
|
||||
S3_CONNECT_TIMEOUT = _float("S3_CONNECT_TIMEOUT", 3.0)
|
||||
S3_READ_TIMEOUT = _float("S3_READ_TIMEOUT", 6.0)
|
||||
UPLOAD_SPOOL_BYTES = _int("UPLOAD_SPOOL_BYTES", 1048576)
|
||||
RECONCILE_INTERVAL_SECONDS = _int("RECONCILE_INTERVAL_SECONDS", 300)
|
||||
RECONCILE_GRACE_SECONDS = _int("RECONCILE_GRACE_SECONDS", 900)
|
||||
UPLOAD_CONCURRENCY = _int("UPLOAD_CONCURRENCY", 2)
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
|
||||
TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
|
||||
|
||||
class BadCursor(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fmt(ts: datetime) -> str:
|
||||
return ts.astimezone(timezone.utc).strftime(TS_FORMAT)
|
||||
|
||||
|
||||
def encode_cursor(scope: str, created_at: str, item_id: str) -> str:
|
||||
raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(scope: str, cursor: str):
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
raise BadCursor() from None
|
||||
if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope:
|
||||
raise BadCursor()
|
||||
ts, item_id = data["t"], data["i"]
|
||||
if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str):
|
||||
raise BadCursor()
|
||||
try:
|
||||
parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc)
|
||||
if str(uuid.UUID(item_id)) != item_id:
|
||||
raise BadCursor()
|
||||
except ValueError:
|
||||
raise BadCursor() from None
|
||||
if encode_cursor(scope, ts, item_id) != cursor:
|
||||
raise BadCursor()
|
||||
return parsed, item_id
|
||||
@@ -0,0 +1,99 @@
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool, PoolTimeout
|
||||
|
||||
from app import config
|
||||
from app.errors import unavailable
|
||||
|
||||
BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout)
|
||||
|
||||
|
||||
async def discard(conn) -> None:
|
||||
try:
|
||||
await conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class Guarded:
|
||||
def __init__(self, conn, timeout: float):
|
||||
self.conn = conn
|
||||
self.timeout = timeout
|
||||
|
||||
async def execute(self, *args, **kwargs):
|
||||
try:
|
||||
return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(self.conn)
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
def transaction(self):
|
||||
return self.conn.transaction()
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self) -> None:
|
||||
options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % (
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS * 2,
|
||||
)
|
||||
self.pool = AsyncConnectionPool(
|
||||
conninfo=config.DATABASE_URL,
|
||||
min_size=config.DB_POOL_MIN,
|
||||
max_size=config.DB_POOL_MAX,
|
||||
timeout=config.DB_POOL_TIMEOUT,
|
||||
max_lifetime=600.0,
|
||||
open=False,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": config.DB_CONNECT_TIMEOUT,
|
||||
"options": options,
|
||||
},
|
||||
)
|
||||
|
||||
async def open(self) -> None:
|
||||
await self.pool.open(wait=False)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.pool.close()
|
||||
|
||||
@asynccontextmanager
|
||||
async def connection(self):
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn:
|
||||
yield Guarded(conn, config.DB_OP_TIMEOUT)
|
||||
except BROKEN:
|
||||
raise unavailable("Хранилище недоступно") from None
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
async with self.connection() as conn:
|
||||
async with conn.transaction():
|
||||
yield conn
|
||||
|
||||
async def in_transaction(self, action):
|
||||
async def run():
|
||||
async with self.transaction() as conn:
|
||||
return await action(conn)
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn:
|
||||
try:
|
||||
await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(conn)
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,37 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None):
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def bad(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> ApiError:
|
||||
challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE
|
||||
message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия"
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def not_found() -> ApiError:
|
||||
return ApiError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, "unavailable", message, {"Retry-After": "1"})
|
||||
|
||||
|
||||
def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": rid}},
|
||||
headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})},
|
||||
)
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Integrity checks for uploaded PNG/JPEG with bounded memory.
|
||||
|
||||
PNG: own streaming scanner (chunk CRCs, IHDR rules, zlib stream decoded 1 MiB at a time, every row filter byte in
|
||||
0..4, decoded size equal to the size implied by IHDR, IEND present). JPEG: `djpeg -strict` decodes the whole entropy
|
||||
stream in a child process at 1/8 scale; Pillow hides libjpeg warnings, and a scan cut short with an appended EOI is
|
||||
only a warning. Both finish before anything is stored.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app import config
|
||||
from app.errors import ApiError, unavailable
|
||||
|
||||
log = logging.getLogger("messenger.imagecheck")
|
||||
|
||||
PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"
|
||||
VALID_DEPTHS = {0: (1, 2, 4, 8, 16), 2: (8, 16), 3: (1, 2, 4, 8), 4: (8, 16), 6: (8, 16)}
|
||||
CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
|
||||
ADAM7 = ((0, 0, 8, 8), (4, 0, 8, 8), (0, 4, 4, 8), (2, 0, 4, 4), (0, 2, 2, 4), (1, 0, 2, 2), (0, 1, 1, 2))
|
||||
Image.MAX_IMAGE_PIXELS = None
|
||||
FORMATS = {"image/png": "PNG", "image/jpeg": "JPEG"}
|
||||
FULL_DECODE_MAX_PIXELS = 4000000
|
||||
SCAN_SECONDS = 10.0
|
||||
OUT_STEP = 1 << 20
|
||||
# Progressive 40 Mpx CMYK needs ~320 MiB of coefficients; the cap keeps a hostile file from reaching the container limit.
|
||||
DJPEG = ("djpeg", "-strict", "-scale", "1/8", "-maxscans", "100", "-maxmemory", "358400", "-outfile", "/dev/null")
|
||||
jpeg_slot = threading.Lock()
|
||||
|
||||
|
||||
class Invalid(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def invalid() -> ApiError:
|
||||
return ApiError(415, "unsupported_media_type", "Файл не является корректным PNG/JPEG")
|
||||
|
||||
|
||||
class RowPlan:
|
||||
def __init__(self, segments):
|
||||
self.segments = [s for s in segments if s[0] > 1 and s[1] > 0]
|
||||
self.seg = 0
|
||||
self.done = 0
|
||||
self.pos = 0
|
||||
|
||||
def feed(self, data: bytes) -> None:
|
||||
i, n = 0, len(data)
|
||||
while i < n:
|
||||
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
||||
self.seg += 1
|
||||
self.done = 0
|
||||
if self.seg >= len(self.segments):
|
||||
raise Invalid("data after the last row")
|
||||
length, rows = self.segments[self.seg]
|
||||
if self.pos > 0:
|
||||
take = min(length - self.pos, n - i)
|
||||
i += take
|
||||
self.pos += take
|
||||
if self.pos == length:
|
||||
self.pos = 0
|
||||
self.done += 1
|
||||
continue
|
||||
full = min((n - i) // length, rows - self.done)
|
||||
if full > 0:
|
||||
if max(data[i:i + full * length:length]) > 4:
|
||||
raise Invalid("bad filter type")
|
||||
i += full * length
|
||||
self.done += full
|
||||
else:
|
||||
if data[i] > 4:
|
||||
raise Invalid("bad filter type")
|
||||
self.pos = n - i
|
||||
i = n
|
||||
|
||||
def complete(self) -> bool:
|
||||
while self.seg < len(self.segments) and self.done == self.segments[self.seg][1]:
|
||||
self.seg += 1
|
||||
self.done = 0
|
||||
return self.seg >= len(self.segments) and self.pos == 0
|
||||
|
||||
|
||||
def png_plan(width, height, depth, color, interlace):
|
||||
bits = CHANNELS[color] * depth
|
||||
if not interlace:
|
||||
return RowPlan([(1 + (width * bits + 7) // 8, height)])
|
||||
segments = []
|
||||
for x0, y0, dx, dy in ADAM7:
|
||||
w = (width - x0 + dx - 1) // dx if width > x0 else 0
|
||||
h = (height - y0 + dy - 1) // dy if height > y0 else 0
|
||||
if w and h:
|
||||
segments.append((1 + (w * bits + 7) // 8, h))
|
||||
return RowPlan(segments)
|
||||
|
||||
|
||||
def scan_png(fileobj) -> None:
|
||||
deadline = time.monotonic() + SCAN_SECONDS
|
||||
fileobj.seek(0)
|
||||
if fileobj.read(8) != PNG_SIGNATURE:
|
||||
raise Invalid("signature")
|
||||
plan = decomp = None
|
||||
palette = seen_idat = idat_closed = False
|
||||
color = None
|
||||
first = True
|
||||
while True:
|
||||
if time.monotonic() > deadline:
|
||||
raise Invalid("scan too slow")
|
||||
head = fileobj.read(8)
|
||||
if len(head) < 8:
|
||||
raise Invalid("truncated before IEND")
|
||||
length, kind = struct.unpack(">I4s", head)
|
||||
if length > 0x7FFFFFFF:
|
||||
raise Invalid("chunk length")
|
||||
if first and kind != b"IHDR":
|
||||
raise Invalid("IHDR must be first")
|
||||
crc = zlib.crc32(kind)
|
||||
if kind == b"IHDR":
|
||||
body = fileobj.read(length)
|
||||
if not first or length != 13 or len(body) != 13:
|
||||
raise Invalid("IHDR")
|
||||
width, height, depth, color, comp, filt, interlace = struct.unpack(">IIBBBBB", body)
|
||||
if not (0 < width <= 0x7FFFFFFF and 0 < height <= 0x7FFFFFFF) or comp or filt or interlace not in (0, 1):
|
||||
raise Invalid("IHDR fields")
|
||||
if color not in VALID_DEPTHS or depth not in VALID_DEPTHS[color]:
|
||||
raise Invalid("color type / depth")
|
||||
plan = png_plan(width, height, depth, color, interlace)
|
||||
decomp = zlib.decompressobj()
|
||||
crc = zlib.crc32(body, crc)
|
||||
else:
|
||||
remaining = length
|
||||
while remaining:
|
||||
buf = fileobj.read(min(remaining, 65536))
|
||||
if not buf:
|
||||
raise Invalid("truncated chunk")
|
||||
remaining -= len(buf)
|
||||
crc = zlib.crc32(buf, crc)
|
||||
if kind == b"IDAT":
|
||||
if idat_closed or (color == 3 and not palette):
|
||||
raise Invalid("IDAT order")
|
||||
seen_idat = True
|
||||
out = decomp.decompress(buf, OUT_STEP)
|
||||
while True:
|
||||
plan.feed(out)
|
||||
if not decomp.unconsumed_tail:
|
||||
break
|
||||
if time.monotonic() > deadline:
|
||||
raise Invalid("scan too slow")
|
||||
out = decomp.decompress(decomp.unconsumed_tail, OUT_STEP)
|
||||
if kind == b"PLTE":
|
||||
palette = True
|
||||
if color in (0, 4) or length % 3 or not 3 <= length <= 768:
|
||||
raise Invalid("PLTE")
|
||||
elif kind != b"IDAT" and seen_idat:
|
||||
idat_closed = True
|
||||
tail = fileobj.read(4)
|
||||
if len(tail) < 4 or struct.unpack(">I", tail)[0] != crc & 0xFFFFFFFF:
|
||||
raise Invalid("crc")
|
||||
first = False
|
||||
if kind == b"IEND":
|
||||
if length or not seen_idat:
|
||||
raise Invalid("IEND")
|
||||
break
|
||||
if not decomp.eof or decomp.unused_data or not plan.complete():
|
||||
raise Invalid("pixel data does not match IHDR")
|
||||
|
||||
|
||||
def check_jpeg(fileobj) -> None:
|
||||
fileobj.flush()
|
||||
descriptor = fileobj.fileno()
|
||||
os.lseek(descriptor, 0, os.SEEK_SET)
|
||||
try:
|
||||
with jpeg_slot:
|
||||
done = subprocess.run(DJPEG, stdin=descriptor, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=SCAN_SECONDS, check=False)
|
||||
except subprocess.TimeoutExpired:
|
||||
log.error("jpeg_check_timeout")
|
||||
raise unavailable("Проверка изображения не успела завершиться") from None
|
||||
# A signal means the checker died (e.g. OOM killer), which says nothing about the file.
|
||||
if done.returncode < 0:
|
||||
log.error("jpeg_check_killed signal=%d", -done.returncode)
|
||||
raise unavailable("Проверка изображения прервана")
|
||||
if done.returncode != 0:
|
||||
raise invalid()
|
||||
|
||||
|
||||
def inspect_image(fileobj, size: int, content_type: str) -> None:
|
||||
"""Raises ApiError 413 (too many pixels) or 415 (not a decodable image of the declared type)."""
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
img = Image.open(fileobj)
|
||||
width, height = img.size
|
||||
if img.format != FORMATS[content_type]:
|
||||
raise invalid()
|
||||
if width * height > config.MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "Изображение превышает 40 млн пикселей")
|
||||
if content_type == "image/png":
|
||||
scan_png(fileobj)
|
||||
if width * height <= FULL_DECODE_MAX_PIXELS:
|
||||
fileobj.seek(0)
|
||||
Image.open(fileobj).load()
|
||||
else:
|
||||
check_jpeg(fileobj)
|
||||
except ApiError:
|
||||
raise
|
||||
except Exception:
|
||||
raise invalid() from None
|
||||
finally:
|
||||
fileobj.seek(0)
|
||||
+640
@@ -0,0 +1,640 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import signal
|
||||
import tempfile
|
||||
import unicodedata
|
||||
import sys
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from starlette.concurrency import iterate_in_threadpool, run_in_threadpool
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from app import config, passwords, reconcile, repo
|
||||
from app.cursor import BadCursor, decode_cursor, encode_cursor
|
||||
from app.db import Database
|
||||
from app.imagecheck import inspect_image
|
||||
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
|
||||
from app.s3 import S3Storage
|
||||
from app.sessions import SessionStore
|
||||
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
log = logging.getLogger("messenger")
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
ZERO_WIDTH = str.maketrans({char: None for char in ""})
|
||||
EXEMPT_PATHS = ("/health/live", "/health/ready")
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
||||
ALLOWED_CONTENT_TYPES = {"image/png", "image/jpeg"}
|
||||
PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
JPEG_MAGIC = b"\xff\xd8\xff"
|
||||
|
||||
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
||||
background = set()
|
||||
db = Database()
|
||||
sessions = SessionStore(db)
|
||||
s3 = S3Storage()
|
||||
state = {"draining": False, "ready": True, "s3_ok": False, "inflight": 0, "rejected": {}}
|
||||
|
||||
|
||||
def rss_mb() -> float:
|
||||
try:
|
||||
with open("/proc/self/statm") as handle:
|
||||
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
|
||||
except OSError:
|
||||
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
|
||||
|
||||
|
||||
def count_rejection(code: str) -> None:
|
||||
state["rejected"][code] = state["rejected"].get(code, 0) + 1
|
||||
|
||||
|
||||
async def sweeper() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
|
||||
try:
|
||||
removed = await sessions.sweep()
|
||||
if removed:
|
||||
log.info("sessions_expired removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("session sweep skipped: storage unavailable")
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
|
||||
if removed:
|
||||
log.info("idempotency_cleanup removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("idempotency cleanup skipped")
|
||||
|
||||
|
||||
async def reconciler_loop() -> None:
|
||||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||||
while True:
|
||||
try:
|
||||
result = await reconcile.run(db, s3, config.RECONCILE_GRACE_SECONDS, dry_run=False)
|
||||
if result.get("skipped") or any(v for k, v in result.items() if k.startswith(("deleted", "marked", "removed"))):
|
||||
log.info("reconcile %s", result)
|
||||
except Exception:
|
||||
log.warning("reconcile skipped: dependency unavailable")
|
||||
await asyncio.sleep(config.RECONCILE_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
async def readiness_watch() -> None:
|
||||
failures = 0
|
||||
while True:
|
||||
if await db.ready():
|
||||
failures = 0
|
||||
if not state["ready"]:
|
||||
state["ready"] = True
|
||||
log.info("ready again: database reachable")
|
||||
else:
|
||||
failures += 1
|
||||
if failures >= config.READY_FAILURES and state["ready"]:
|
||||
state["ready"] = False
|
||||
log.warning("not ready: database unreachable, business requests get 503")
|
||||
s3_ok = await run_in_threadpool(s3.probe)
|
||||
if s3_ok != state["s3_ok"]:
|
||||
state["s3_ok"] = s3_ok
|
||||
log.info("object storage %s", "reachable" if s3_ok else "UNREACHABLE: uploads/downloads get 503, /health/ready 503")
|
||||
await asyncio.sleep(config.READY_CHECK_SECONDS)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
previous = {}
|
||||
|
||||
def on_signal(signum, frame):
|
||||
if not state["draining"]:
|
||||
state["draining"] = True
|
||||
log.info("signal %s: draining, new requests get 503", signal.Signals(signum).name)
|
||||
handler = previous.get(signum)
|
||||
if callable(handler):
|
||||
handler(signum, frame)
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
previous[sig] = signal.getsignal(sig)
|
||||
signal.signal(sig, on_signal)
|
||||
await db.open()
|
||||
await sessions.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
state["s3_ok"] = await run_in_threadpool(s3.probe)
|
||||
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch()), asyncio.create_task(reconciler_loop())]
|
||||
log.info(
|
||||
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d s3=%s bucket=%s",
|
||||
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
|
||||
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
|
||||
config.IDEMPOTENCY_TTL_SECONDS, config.S3_ENDPOINT, config.S3_BUCKET,
|
||||
)
|
||||
yield
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await sessions.close()
|
||||
await db.close()
|
||||
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def guard(request: Request, call_next):
|
||||
rid = uuid.uuid4().hex
|
||||
request.state.rid = rid
|
||||
exempt = request.url.path in EXEMPT_PATHS
|
||||
if state["draining"] and request.url.path != "/health/live":
|
||||
count_rejection("draining")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
|
||||
if not exempt and not state["ready"]:
|
||||
count_rejection("not_ready")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
|
||||
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
|
||||
count_rejection("inflight")
|
||||
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
|
||||
state["inflight"] += 1
|
||||
try:
|
||||
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
count_rejection("request_timeout")
|
||||
log.warning("request timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path)
|
||||
response = error_body(
|
||||
rid, config.INSTANCE_ID, 503, "unavailable",
|
||||
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
|
||||
)
|
||||
except Exception:
|
||||
log.exception("unhandled error request_id=%s", rid)
|
||||
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
finally:
|
||||
state["inflight"] -= 1
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
if exc.status in (413, 429, 503):
|
||||
count_rejection(exc.code)
|
||||
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
def parse_uuid(value: object) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def non_blank(value: object, max_len: int) -> bool:
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
|
||||
return False
|
||||
return value.translate(ZERO_WIDTH).strip() != ""
|
||||
|
||||
|
||||
async def read_json(request: Request, required: Tuple[str, ...], optional: Tuple[str, ...] = ()) -> dict:
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое")
|
||||
declared = request.headers.get("content-length")
|
||||
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
raw += chunk
|
||||
if len(raw) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
try:
|
||||
body = json.loads(bytes(raw).decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad("Некорректный JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise bad("Ожидается JSON-объект")
|
||||
allowed = set(required) | set(optional)
|
||||
if set(body) - allowed:
|
||||
raise bad("Лишние поля в запросе")
|
||||
if set(required) - set(body):
|
||||
raise bad("Отсутствуют обязательные поля")
|
||||
return body
|
||||
|
||||
|
||||
def ensure_capacity() -> None:
|
||||
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
|
||||
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
|
||||
|
||||
|
||||
def bearer_token(request: Request) -> str:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
token = value.strip()
|
||||
if scheme.lower() != "bearer" or not token:
|
||||
raise unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
async def actor(request: Request) -> dict:
|
||||
user = await sessions.resolve(bearer_token(request))
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
|
||||
|
||||
def basic_credentials(request: Request) -> Tuple[str, str]:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized("Basic")
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized("Basic") from None
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator:
|
||||
raise unauthorized("Basic")
|
||||
return username, password
|
||||
|
||||
|
||||
async def member_chat(user: dict, chat_id: str) -> dict:
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
async with db.connection() as conn:
|
||||
chat = await repo.chat_for_member(conn, cid, user["id"])
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
|
||||
def page_params(request: Request) -> Tuple[int, Optional[str]]:
|
||||
query = request.query_params
|
||||
limit = 50
|
||||
if "limit" in query:
|
||||
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
|
||||
raise bad("limit должен быть целым 1..100")
|
||||
limit = int(query["limit"])
|
||||
cursor = query.get("cursor")
|
||||
if cursor is not None and cursor == "":
|
||||
raise bad("Некорректный cursor")
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def keyset(scope: str, cursor: Optional[str]):
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
return decode_cursor(scope, cursor)
|
||||
except BadCursor:
|
||||
raise bad("Некорректный cursor") from None
|
||||
|
||||
|
||||
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
|
||||
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
|
||||
return JSONResponse({"items": items, "next_cursor": next_cursor})
|
||||
|
||||
|
||||
def detect_content_type(head: bytes) -> Optional[str]:
|
||||
if head[:8] == PNG_MAGIC:
|
||||
return "image/png"
|
||||
if head[:3] == JPEG_MAGIC:
|
||||
return "image/jpeg"
|
||||
return None
|
||||
|
||||
|
||||
def validate_filename(filename: str) -> None:
|
||||
if not 1 <= len(filename) <= 255 or filename.strip() == "" or filename in (".", ".."):
|
||||
raise bad("filename: 1..255 символов, не только пробелы")
|
||||
if "/" in filename or "\\" in filename or any(unicodedata.category(ch) in ("Cc", "Cs", "Zl", "Zp") for ch in filename):
|
||||
raise bad("filename содержит недопустимые символы")
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
async def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def ready():
|
||||
if not await db.ready():
|
||||
raise unavailable("База данных недоступна")
|
||||
if not state["s3_ok"]:
|
||||
raise unavailable("Хранилище файлов недоступно")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(request: Request):
|
||||
body = await read_json(request, ("username", "display_name", "password"))
|
||||
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
|
||||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
if not non_blank(body["display_name"], 100):
|
||||
raise bad("display_name: 1..100 символов, не только пробелы")
|
||||
password = body["password"]
|
||||
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
|
||||
ensure_capacity()
|
||||
password_hash = await run_in_threadpool(passwords.hash_password, password)
|
||||
async with db.connection() as conn:
|
||||
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
|
||||
if user is None:
|
||||
raise ApiError(409, "conflict", "username уже занят")
|
||||
log.info("user_created id=%s", user["id"])
|
||||
return JSONResponse(user, status_code=201)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def login(request: Request):
|
||||
if await request.body():
|
||||
raise bad("POST /auth/sessions не принимает тело запроса")
|
||||
username, password = basic_credentials(request)
|
||||
row = None
|
||||
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
async with db.connection() as conn:
|
||||
row = await repo.credentials(conn, username)
|
||||
stored = row["password_hash"] if row else passwords.DUMMY_HASH
|
||||
matches = await run_in_threadpool(passwords.verify_password, stored, password)
|
||||
if row is None or not matches:
|
||||
log.info("login_failed username_known=%s", row is not None)
|
||||
raise unauthorized("Basic")
|
||||
user = repo.public_user(row)
|
||||
token, expires_at = await sessions.create(user)
|
||||
log.info("session_created user=%s", user["id"])
|
||||
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
async def logout(request: Request):
|
||||
token = bearer_token(request)
|
||||
user = await sessions.resolve(token)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
await sessions.revoke(token)
|
||||
log.info("session_revoked user=%s", user["id"])
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def me(request: Request):
|
||||
return await actor(request)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
user = await actor(request)
|
||||
body = await read_json(request, ("title", "member_ids"))
|
||||
if not non_blank(body["title"], 100):
|
||||
raise bad("title: 1..100 символов, не только пробелы")
|
||||
raw_ids = body["member_ids"]
|
||||
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
|
||||
raise bad("member_ids: массив из 1..100 UUID")
|
||||
ids: List[str] = []
|
||||
for value in raw_ids:
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
raise bad("member_ids содержит некорректный UUID")
|
||||
ids.append(parsed)
|
||||
if len(set(ids)) != len(ids):
|
||||
raise bad("member_ids содержит дубликаты")
|
||||
if user["id"] not in ids:
|
||||
raise bad("Создатель должен входить в member_ids")
|
||||
ensure_capacity()
|
||||
|
||||
async def insert(conn):
|
||||
return await repo.create_chat(conn, body["title"], ids)
|
||||
|
||||
try:
|
||||
chat = await db.in_transaction(insert)
|
||||
except repo.UnknownMember:
|
||||
raise bad("member_ids содержит неизвестного пользователя") from None
|
||||
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
|
||||
return JSONResponse(chat, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(request: Request):
|
||||
user = await actor(request)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "chats:" + user["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(chat_id: str, request: Request):
|
||||
return await member_chat(await actor(request), chat_id)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def create_message(chat_id: str, request: Request):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
|
||||
idem_key = request.headers.get("idempotency-key", "")
|
||||
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
|
||||
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
body = await read_json(request, ("text",), ("attachment_ids",))
|
||||
|
||||
text = body["text"]
|
||||
attachment_ids_raw = body.get("attachment_ids", [])
|
||||
|
||||
if not isinstance(attachment_ids_raw, list):
|
||||
raise bad("attachment_ids должен быть массивом UUID")
|
||||
if len(attachment_ids_raw) > 10:
|
||||
raise bad("attachment_ids: максимум 10 вложений")
|
||||
|
||||
attachment_ids: List[str] = []
|
||||
for value in attachment_ids_raw:
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
raise bad("attachment_ids содержит некорректный UUID")
|
||||
attachment_ids.append(parsed)
|
||||
|
||||
if len(set(attachment_ids)) != len(attachment_ids):
|
||||
raise bad("attachment_ids содержит дубликаты")
|
||||
|
||||
has_text = isinstance(text, str) and non_blank(text, 4000)
|
||||
has_attachments = len(attachment_ids) > 0
|
||||
|
||||
if not has_text and not has_attachments:
|
||||
raise bad("text: 1..4000 символов, не только пробелы")
|
||||
|
||||
if isinstance(text, str) and len(text) > 4000:
|
||||
raise bad("text: 1..4000 символов, не только пробелы")
|
||||
|
||||
if not isinstance(text, str):
|
||||
raise bad("text должен быть строкой")
|
||||
|
||||
ensure_capacity()
|
||||
|
||||
normalized = {"text": text}
|
||||
if attachment_ids:
|
||||
normalized["attachment_ids"] = attachment_ids
|
||||
body_hash = hashlib.sha256(
|
||||
json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode()
|
||||
).hexdigest()
|
||||
|
||||
async def do(conn):
|
||||
return await repo.create_message_idempotent(conn, cid, user["id"], text, attachment_ids, idem_key, body_hash)
|
||||
|
||||
status, message = await db.in_transaction(do)
|
||||
|
||||
if status == "conflict":
|
||||
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
|
||||
if status == "not_member":
|
||||
raise not_found()
|
||||
if status == "attachment_not_found":
|
||||
raise not_found()
|
||||
if status == "attachment_failed":
|
||||
raise ApiError(409, "conflict", "Вложение в состоянии failed, нельзя прикрепить")
|
||||
return JSONResponse(message, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def list_messages(chat_id: str, request: Request):
|
||||
chat = await member_chat(await actor(request), chat_id)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "messages:" + chat["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201)
|
||||
async def upload_attachment(chat_id: str, request: Request):
|
||||
user = await actor(request)
|
||||
chat = await member_chat(user, chat_id)
|
||||
|
||||
filename = request.query_params.get("filename", "")
|
||||
validate_filename(filename)
|
||||
|
||||
declared_ct = request.headers.get("content-type", "").split(";")[0].strip().lower()
|
||||
if declared_ct not in ALLOWED_CONTENT_TYPES:
|
||||
raise ApiError(415, "unsupported_media_type", "Допускается только image/png или image/jpeg")
|
||||
|
||||
declared_length = request.headers.get("content-length")
|
||||
too_large = ApiError(413, "payload_too_large", "Файл превышает 10 MiB")
|
||||
if declared_length and declared_length.isdigit() and int(declared_length) > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
ensure_capacity()
|
||||
|
||||
sha = hashlib.sha256()
|
||||
size = 0
|
||||
with tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES) as spool:
|
||||
head = b""
|
||||
async for chunk in request.stream():
|
||||
size += len(chunk)
|
||||
if size > config.MAX_UPLOAD_BYTES:
|
||||
raise too_large
|
||||
if len(head) < 8:
|
||||
head += chunk[: 8 - len(head)]
|
||||
if len(head) == 8 and detect_content_type(head) is None:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||||
sha.update(chunk)
|
||||
spool.write(chunk)
|
||||
if size == 0:
|
||||
raise bad("Пустой файл")
|
||||
actual_ct = detect_content_type(head)
|
||||
if actual_ct is None:
|
||||
raise ApiError(415, "unsupported_media_type", "Содержимое не является PNG/JPEG")
|
||||
if actual_ct != declared_ct:
|
||||
raise ApiError(415, "unsupported_media_type", "Content-Type не соответствует содержимому файла")
|
||||
async with upload_slots:
|
||||
await run_in_threadpool(inspect_image, spool, size, actual_ct)
|
||||
|
||||
attachment_id = str(uuid.uuid4())
|
||||
object_key = "attachments/" + attachment_id
|
||||
async with db.connection() as conn:
|
||||
await repo.begin_attachment(
|
||||
conn, attachment_id, chat["id"], user["id"], filename, actual_ct, size, sha.hexdigest(), object_key,
|
||||
)
|
||||
try:
|
||||
async with upload_slots:
|
||||
await run_in_threadpool(s3.put, object_key, spool, size, actual_ct)
|
||||
except BaseException:
|
||||
defer(compensate(attachment_id, object_key))
|
||||
raise
|
||||
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
att = await repo.finish_attachment(conn, attachment_id)
|
||||
except BaseException:
|
||||
await compensate(attachment_id, object_key)
|
||||
raise
|
||||
if att is None:
|
||||
raise unavailable("Загрузка отменена сверкой, повторите запрос")
|
||||
log.info("attachment_uploaded id=%s chat=%s size=%d", att["id"], chat["id"], size)
|
||||
return JSONResponse(att, status_code=201)
|
||||
|
||||
|
||||
def defer(coro) -> None:
|
||||
task = asyncio.ensure_future(coro)
|
||||
background.add(task)
|
||||
task.add_done_callback(background.discard)
|
||||
|
||||
|
||||
async def compensate(attachment_id: str, object_key: str) -> None:
|
||||
"""Best effort; whatever is left behind is collected by the reconciler after the grace period."""
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
await repo.abort_attachment(conn, attachment_id)
|
||||
await run_in_threadpool(s3.delete, object_key)
|
||||
except BaseException:
|
||||
log.warning("compensation incomplete attachment=%s, left for reconciler", attachment_id)
|
||||
|
||||
|
||||
async def accessible_attachment(request: Request, attachment_id: str) -> dict:
|
||||
user = await actor(request)
|
||||
aid = parse_uuid(attachment_id)
|
||||
if aid is None:
|
||||
raise bad("attachment_id должен быть UUID")
|
||||
async with db.connection() as conn:
|
||||
row = await repo.get_attachment(conn, aid)
|
||||
if row is None:
|
||||
raise not_found()
|
||||
chat = await repo.chat_for_member(conn, str(row["chat_id"]), user["id"])
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return row
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}")
|
||||
async def get_attachment(attachment_id: str, request: Request):
|
||||
return repo._attachment(await accessible_attachment(request, attachment_id))
|
||||
|
||||
|
||||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||||
async def download_attachment(attachment_id: str, request: Request):
|
||||
row = await accessible_attachment(request, attachment_id)
|
||||
variant = request.query_params.get("variant", "original")
|
||||
if variant != "original":
|
||||
raise not_found()
|
||||
if row["status"] != "ready":
|
||||
raise ApiError(409, "conflict", "Вложение недоступно для скачивания")
|
||||
opened = await run_in_threadpool(s3.open_stream, row["object_key"])
|
||||
if opened is None:
|
||||
log.error("object_missing attachment=%s", row["id"])
|
||||
raise unavailable("Оригинал временно недоступен")
|
||||
chunks, length = opened
|
||||
return StreamingResponse(
|
||||
iterate_in_threadpool(chunks), media_type=row["content_type"], headers={"Content-Length": str(length)},
|
||||
)
|
||||
|
||||
|
||||
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def fallback(request: Request):
|
||||
raise not_found()
|
||||
@@ -0,0 +1,16 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import Argon2Error, VerifyMismatchError
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = HASHER.hash("there-is-no-such-user-password")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return HASHER.hash(password)
|
||||
|
||||
|
||||
def verify_password(stored: str, password: str) -> bool:
|
||||
try:
|
||||
return HASHER.verify(stored, password)
|
||||
except (VerifyMismatchError, Argon2Error, ValueError):
|
||||
return False
|
||||
@@ -0,0 +1,115 @@
|
||||
"""DB <-> S3 reconciliation for attachments.
|
||||
|
||||
Invariant: a row is inserted as 'uploading' BEFORE the object is written, and becomes 'ready' only AFTER the
|
||||
PUT succeeded. So every object has a row, and a client only ever sees 'ready' rows.
|
||||
|
||||
Cases (all gated by a grace period so in-flight uploads are never touched):
|
||||
stale_uploading row 'uploading' older than grace AND its object (if any) older than grace -> delete both;
|
||||
a fresh object under an old row is left alone (re-checked via LastModified)
|
||||
orphan_object object without any row, older than grace -> delete object
|
||||
missing_object 'ready' row confirmed longer than grace ago whose object is gone (re-checked by HEAD) -> mark 'failed'/object_missing, never delete the row
|
||||
Rows referenced by messages are never deleted: only 'uploading' rows can be removed, and those cannot be linked.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
log = logging.getLogger("messenger.reconcile")
|
||||
|
||||
LOCK_ID = 2026_0104
|
||||
PREFIX = "attachments/"
|
||||
BATCH = 500
|
||||
|
||||
|
||||
async def run(db, s3, grace_seconds: int, dry_run: bool = True) -> dict:
|
||||
started = time.monotonic()
|
||||
result = {
|
||||
"dry_run": dry_run, "grace_seconds": grace_seconds,
|
||||
"stale_uploading": 0, "orphan_objects": 0, "missing_objects": 0,
|
||||
"deleted_objects": 0, "removed_rows": 0, "marked_failed": 0,
|
||||
"objects_scanned": 0, "rows_scanned": 0, "protected_fresh_objects": 0,
|
||||
}
|
||||
async with db.connection() as conn:
|
||||
cur = await conn.execute("SELECT pg_try_advisory_lock(%s) AS got", (LOCK_ID,))
|
||||
if not (await cur.fetchone())["got"]:
|
||||
return {"skipped": "another reconciler is running"}
|
||||
try:
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=grace_seconds)
|
||||
await _stale_uploading(conn, s3, cutoff, dry_run, result)
|
||||
await _orphan_objects(conn, s3, cutoff, dry_run, result)
|
||||
await _missing_objects(conn, s3, cutoff, dry_run, result)
|
||||
finally:
|
||||
await conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
result["seconds"] = round(time.monotonic() - started, 3)
|
||||
return result
|
||||
|
||||
|
||||
async def _stale_uploading(conn, s3, cutoff, dry_run, result):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE status = 'uploading' AND created_at < %s", (cutoff,),
|
||||
)
|
||||
for row in await cur.fetchall():
|
||||
modified = await run_in_threadpool(s3.head, row["object_key"])
|
||||
if modified is not None and modified >= cutoff:
|
||||
result["protected_fresh_objects"] += 1
|
||||
continue
|
||||
result["stale_uploading"] += 1
|
||||
if dry_run:
|
||||
continue
|
||||
if modified is not None:
|
||||
await run_in_threadpool(s3.delete, row["object_key"])
|
||||
result["deleted_objects"] += 1
|
||||
done = await conn.execute(
|
||||
"DELETE FROM attachments WHERE id = %s AND status = 'uploading' AND created_at < %s", (row["id"], cutoff),
|
||||
)
|
||||
result["removed_rows"] += done.rowcount
|
||||
|
||||
|
||||
async def _orphan_objects(conn, s3, cutoff, dry_run, result):
|
||||
pages = await run_in_threadpool(lambda: iter(s3.list_pages(PREFIX)))
|
||||
while True:
|
||||
page = await run_in_threadpool(next, pages, None)
|
||||
if page is None:
|
||||
break
|
||||
result["objects_scanned"] += len(page)
|
||||
keys = [o["Key"] for o in page]
|
||||
if not keys:
|
||||
continue
|
||||
cur = await conn.execute("SELECT object_key FROM attachments WHERE object_key = ANY(%s)", (keys,))
|
||||
known = {row["object_key"] for row in await cur.fetchall()}
|
||||
for obj in page:
|
||||
if obj["Key"] in known or obj["LastModified"] >= cutoff:
|
||||
continue
|
||||
result["orphan_objects"] += 1
|
||||
if not dry_run:
|
||||
again = await conn.execute("SELECT 1 FROM attachments WHERE object_key = %s", (obj["Key"],))
|
||||
if await again.fetchone() is None:
|
||||
await run_in_threadpool(s3.delete, obj["Key"])
|
||||
result["deleted_objects"] += 1
|
||||
|
||||
|
||||
async def _missing_objects(conn, s3, cutoff, dry_run, result):
|
||||
last = None
|
||||
while True:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, object_key FROM attachments WHERE status = 'ready' AND updated_at < %s"
|
||||
" AND (%s::uuid IS NULL OR id > %s::uuid) ORDER BY id LIMIT %s",
|
||||
(cutoff, last, last, BATCH),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
if not rows:
|
||||
return
|
||||
for row in rows:
|
||||
result["rows_scanned"] += 1
|
||||
if await run_in_threadpool(s3.exists, row["object_key"]):
|
||||
continue
|
||||
result["missing_objects"] += 1
|
||||
if not dry_run:
|
||||
done = await conn.execute(
|
||||
"UPDATE attachments SET status = 'failed', error_code = 'object_missing',"
|
||||
" updated_at = clock_timestamp() WHERE id = %s AND status = 'ready'", (row["id"],),
|
||||
)
|
||||
result["marked_failed"] += done.rowcount
|
||||
last = str(rows[-1]["id"])
|
||||
+293
@@ -0,0 +1,293 @@
|
||||
import hashlib
|
||||
import json
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import psycopg
|
||||
|
||||
from app.cursor import fmt
|
||||
|
||||
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
|
||||
CHAT_QUERY = (
|
||||
"SELECT c.id, c.title, c.created_at,"
|
||||
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
|
||||
)
|
||||
|
||||
|
||||
class UnknownMember(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def decoded(value) -> str:
|
||||
return bytes(value).decode("utf-8")
|
||||
|
||||
|
||||
def public_user(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": decoded(row["display_name"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _chat(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": decoded(row["title"]),
|
||||
"member_ids": [str(value) for value in row["member_ids"]],
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": decoded(row["text"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message_with_attachments(row, attachment_ids: List[str]) -> dict:
|
||||
msg = _message(row)
|
||||
msg["attachment_ids"] = attachment_ids
|
||||
return msg
|
||||
|
||||
|
||||
def _attachment(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"owner_id": str(row["owner_id"]),
|
||||
"filename": row["filename"],
|
||||
"content_type": row["content_type"],
|
||||
"size_bytes": row["size_bytes"],
|
||||
"sha256": row["sha256"],
|
||||
"status": row["status"],
|
||||
"variants": [],
|
||||
"error_code": row["error_code"],
|
||||
"created_at": fmt(row["created_at"]),
|
||||
"updated_at": fmt(row["updated_at"]),
|
||||
}
|
||||
|
||||
|
||||
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
|
||||
try:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
|
||||
" RETURNING id, username, display_name, created_at",
|
||||
(username, display_name.encode("utf-8"), password_hash),
|
||||
)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
return None
|
||||
return public_user(await cur.fetchone())
|
||||
|
||||
|
||||
async def credentials(conn, username: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
|
||||
" RETURNING id, title, created_at",
|
||||
(title.encode("utf-8"),),
|
||||
)
|
||||
chat = await cur.fetchone()
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
|
||||
(chat["id"], member_ids),
|
||||
)
|
||||
if cur.rowcount != len(member_ids):
|
||||
raise UnknownMember()
|
||||
chat["member_ids"] = list(member_ids)
|
||||
return _chat(chat)
|
||||
|
||||
|
||||
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
|
||||
(user_id, chat_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _chat(row) if row else None
|
||||
|
||||
|
||||
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY
|
||||
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
|
||||
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY c.created_at, c.id LIMIT %s",
|
||||
(user_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [_chat(row) for row in rows[:limit]], len(rows) > limit
|
||||
|
||||
|
||||
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _message(row) if row else None
|
||||
|
||||
|
||||
async def _fetch_message_attachment_ids(conn, message_id: str) -> List[str]:
|
||||
cur = await conn.execute(
|
||||
"SELECT attachment_id FROM message_attachments WHERE message_id = %s ORDER BY position",
|
||||
(message_id,),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [str(row["attachment_id"]) for row in rows]
|
||||
|
||||
|
||||
async def _link_attachments(conn, message_id: str, chat_id: str, attachment_ids: List[str]) -> None:
|
||||
for pos, aid in enumerate(attachment_ids):
|
||||
await conn.execute(
|
||||
"INSERT INTO message_attachments (message_id, attachment_id, position) VALUES (%s, %s, %s)",
|
||||
(message_id, aid, pos),
|
||||
)
|
||||
|
||||
|
||||
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
|
||||
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY created_at, id LIMIT %s",
|
||||
(chat_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
messages = []
|
||||
for row in rows[:limit]:
|
||||
aids = await _fetch_message_attachment_ids(conn, row["id"])
|
||||
messages.append(_message_with_attachments(row, aids))
|
||||
return messages, len(rows) > limit
|
||||
|
||||
|
||||
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
|
||||
raw = "%s:%s:%s" % (user_id, chat_id, key)
|
||||
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
|
||||
|
||||
|
||||
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str,
|
||||
attachment_ids: List[str], idem_key: str, body_hash: str):
|
||||
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
)
|
||||
if await cur.fetchone() is None:
|
||||
return "not_member", None
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT request_body_hash, response_json FROM idempotency_keys"
|
||||
" WHERE user_id = %s AND chat_id = %s AND key = %s",
|
||||
(sender_id, chat_id, idem_key),
|
||||
)
|
||||
existing = await cur.fetchone()
|
||||
|
||||
if existing:
|
||||
if existing["request_body_hash"] != body_hash:
|
||||
return "conflict", None
|
||||
return "replay", json.loads(existing["response_json"])
|
||||
|
||||
if attachment_ids:
|
||||
cur = await conn.execute(
|
||||
"SELECT id, chat_id, status FROM attachments WHERE id = ANY(%s::uuid[]) AND status <> 'uploading'",
|
||||
(attachment_ids,),
|
||||
)
|
||||
att_rows = await cur.fetchall()
|
||||
found_ids = {str(row["id"]) for row in att_rows}
|
||||
for aid in attachment_ids:
|
||||
if aid not in found_ids:
|
||||
return "attachment_not_found", None
|
||||
for row in att_rows:
|
||||
if str(row["chat_id"]) != chat_id:
|
||||
return "attachment_not_found", None
|
||||
if row["status"] == "failed":
|
||||
return "attachment_failed", None
|
||||
|
||||
text_bytes = text.encode("utf-8") if text else b""
|
||||
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text_bytes, chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "not_member", None
|
||||
|
||||
if attachment_ids:
|
||||
await _link_attachments(conn, row["id"], chat_id, attachment_ids)
|
||||
|
||||
message = _message_with_attachments(row, list(attachment_ids))
|
||||
|
||||
await conn.execute(
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
|
||||
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
return "created", message
|
||||
|
||||
|
||||
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
|
||||
(ttl_seconds,),
|
||||
)
|
||||
return cur.rowcount
|
||||
|
||||
|
||||
async def begin_attachment(conn, attachment_id: str, chat_id: str, owner_id: str, filename: str,
|
||||
content_type: str, size_bytes: int, sha256: str, object_key: str) -> None:
|
||||
await conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
|
||||
" status, error_code, created_at, updated_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, %s, %s, %s, 'uploading', NULL, clock_timestamp(), clock_timestamp())",
|
||||
(attachment_id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key),
|
||||
)
|
||||
|
||||
|
||||
async def finish_attachment(conn, attachment_id: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
"UPDATE attachments SET status = 'ready', updated_at = clock_timestamp()"
|
||||
" WHERE id = %s AND status = 'uploading'"
|
||||
" RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, error_code,"
|
||||
" created_at, updated_at",
|
||||
(attachment_id,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _attachment(row) if row else None
|
||||
|
||||
|
||||
async def abort_attachment(conn, attachment_id: str) -> None:
|
||||
await conn.execute("DELETE FROM attachments WHERE id = %s AND status = 'uploading'", (attachment_id,))
|
||||
|
||||
|
||||
async def get_attachment(conn, attachment_id: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key, status, error_code,"
|
||||
" created_at, updated_at FROM attachments WHERE id = %s AND status <> 'uploading'",
|
||||
(attachment_id,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
@@ -0,0 +1,97 @@
|
||||
import logging
|
||||
from typing import Iterator, Optional, Tuple
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app import config
|
||||
from app.errors import unavailable
|
||||
|
||||
log = logging.getLogger("messenger.s3")
|
||||
|
||||
MISSING = ("NoSuchKey", "404", "NotFound")
|
||||
|
||||
|
||||
class S3Storage:
|
||||
def __init__(self) -> None:
|
||||
self.client = None
|
||||
self.probe_client = None
|
||||
|
||||
def open(self) -> None:
|
||||
self.probe_client = self._client(1.0, 1.5)
|
||||
self.client = self._client(config.S3_CONNECT_TIMEOUT, config.S3_READ_TIMEOUT)
|
||||
|
||||
def _client(self, connect: float, read: float):
|
||||
return boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_ENDPOINT,
|
||||
region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY_ID,
|
||||
aws_secret_access_key=config.S3_SECRET_ACCESS_KEY,
|
||||
config=BotoConfig(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
connect_timeout=connect,
|
||||
read_timeout=read,
|
||||
retries={"max_attempts": 0},
|
||||
),
|
||||
)
|
||||
|
||||
def probe(self) -> bool:
|
||||
try:
|
||||
self.probe_client.head_bucket(Bucket=config.S3_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def put(self, key: str, fileobj, size: int, content_type: str) -> None:
|
||||
try:
|
||||
fileobj.seek(0)
|
||||
self.client.put_object(
|
||||
Bucket=config.S3_BUCKET, Key=key, Body=fileobj, ContentLength=size, ContentType=content_type,
|
||||
)
|
||||
except (BotoCoreError, ClientError) as exc:
|
||||
log.error("s3_put_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
|
||||
def open_stream(self, key: str) -> Optional[Tuple[Iterator[bytes], int]]:
|
||||
try:
|
||||
response = self.client.get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return None
|
||||
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
except BotoCoreError as exc:
|
||||
log.error("s3_get_failed key=%s error=%s", key, type(exc).__name__)
|
||||
raise unavailable("Хранилище файлов недоступно") from None
|
||||
body = response["Body"]
|
||||
return body.iter_chunks(65536), int(response["ContentLength"])
|
||||
|
||||
def head(self, key: str):
|
||||
"""LastModified of the object, or None when it does not exist."""
|
||||
try:
|
||||
return self.client.head_object(Bucket=config.S3_BUCKET, Key=key)["LastModified"]
|
||||
except ClientError as exc:
|
||||
if exc.response["Error"].get("Code", "") in MISSING:
|
||||
return None
|
||||
raise
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self.head(key) is not None
|
||||
|
||||
def delete(self, key: str) -> None:
|
||||
self.client.delete_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
|
||||
def list_pages(self, prefix: str):
|
||||
paginator = self.client.get_paginator("list_objects_v2")
|
||||
for page in paginator.paginate(Bucket=config.S3_BUCKET, Prefix=prefix, PaginationConfig={"PageSize": 500}):
|
||||
yield page.get("Contents", [])
|
||||
|
||||
def ready(self) -> bool:
|
||||
try:
|
||||
self.client.head_bucket(Bucket=config.S3_BUCKET)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
import valkey.asyncio as valkey
|
||||
|
||||
from app import config, repo
|
||||
from app.cursor import fmt
|
||||
from app.errors import unavailable
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, db) -> None:
|
||||
self.db = db
|
||||
self.client = None
|
||||
self.stats = {"hits": 0, "misses": 0, "errors": 0}
|
||||
|
||||
async def open(self) -> None:
|
||||
if not config.CACHE_URL:
|
||||
log.info("session cache disabled: CACHE_URL is empty")
|
||||
return
|
||||
self.client = valkey.Valkey.from_url(
|
||||
config.CACHE_URL,
|
||||
socket_timeout=config.CACHE_TIMEOUT,
|
||||
socket_connect_timeout=config.CACHE_TIMEOUT,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
self.client = None
|
||||
|
||||
@staticmethod
|
||||
def digest(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session:" + digest.hex()
|
||||
|
||||
async def create(self, user: dict) -> tuple:
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = self.digest(token)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at)"
|
||||
" VALUES (%s, %s, clock_timestamp(), %s)",
|
||||
(digest, user["id"], expires_at),
|
||||
)
|
||||
await self._cache_put(digest, user, expires_at)
|
||||
return token, fmt(expires_at)
|
||||
|
||||
async def resolve(self, token: str) -> Optional[dict]:
|
||||
digest = self.digest(token)
|
||||
cached = await self._cache_get(digest)
|
||||
if cached == REVOKED:
|
||||
self.stats["hits"] += 1
|
||||
return None
|
||||
if cached is not None:
|
||||
self.stats["hits"] += 1
|
||||
return cached
|
||||
self.stats["misses"] += 1
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
|
||||
" FROM sessions s JOIN users u ON u.id = s.user_id"
|
||||
" WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(digest,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
user = repo.public_user(row)
|
||||
await self._cache_put(digest, user, row["expires_at"])
|
||||
return user
|
||||
|
||||
async def revoke(self, token: str) -> None:
|
||||
digest = self.digest(token)
|
||||
await self._cache_revoke(digest)
|
||||
try:
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
|
||||
except Exception:
|
||||
await self._cache_release(digest)
|
||||
raise
|
||||
|
||||
async def sweep(self) -> int:
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()")
|
||||
return cur.rowcount
|
||||
|
||||
def _cache_ttl(self, expires_at: datetime) -> int:
|
||||
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
|
||||
return min(config.SESSION_CACHE_TTL_SECONDS, remaining)
|
||||
|
||||
async def _cache_get(self, digest: bytes) -> Optional[dict]:
|
||||
if self.client is None:
|
||||
return None
|
||||
try:
|
||||
raw = await self.client.get(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("get")
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
if raw == REVOKED:
|
||||
return REVOKED
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
ttl = self._cache_ttl(expires_at)
|
||||
if ttl <= 0:
|
||||
return
|
||||
payload = json.dumps(user)
|
||||
try:
|
||||
await self.client.set(self.key(digest), payload, ex=ttl, nx=True)
|
||||
except Exception:
|
||||
self._cache_failed("set")
|
||||
|
||||
async def _cache_release(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.delete(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("release")
|
||||
|
||||
async def _cache_revoke(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS)
|
||||
except Exception:
|
||||
self._cache_failed("revoke")
|
||||
raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None
|
||||
|
||||
def _cache_failed(self, operation: str) -> None:
|
||||
self.stats["errors"] += 1
|
||||
if self.stats["errors"] % 50 == 1:
|
||||
log.warning("session cache %s failed, falling back to postgres (errors=%d)", operation, self.stats["errors"])
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
name: mtusi-lab4
|
||||
|
||||
x-api: &api-defaults
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab4
|
||||
env_file:
|
||||
- .env
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
minio-init:
|
||||
condition: service_completed_successfully
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
nginx:
|
||||
image: nginx:1.27.3-alpine
|
||||
command: ["/bin/sh", "/opt/nginx-health.sh"]
|
||||
environment:
|
||||
API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2}
|
||||
HEALTH_INTERVAL: "1"
|
||||
HEALTH_FALL: "2"
|
||||
HEALTH_RISE: "2"
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/nginx-health.sh:/opt/nginx-health.sh:ro
|
||||
depends_on:
|
||||
api-1:
|
||||
condition: service_healthy
|
||||
api-2:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 64M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
api-1:
|
||||
<<: *api-defaults
|
||||
environment:
|
||||
INSTANCE_ID: api-1
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
|
||||
api-2:
|
||||
<<: *api-defaults
|
||||
environment:
|
||||
INSTANCE_ID: api-2
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
S3_ENDPOINT: http://minio:9000
|
||||
|
||||
postgres:
|
||||
image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_INITDB_ARGS: --data-checksums
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- synchronous_commit=on
|
||||
- -c
|
||||
- shared_buffers=128MB
|
||||
- -c
|
||||
- work_mem=8MB
|
||||
- -c
|
||||
- max_connections=50
|
||||
- -c
|
||||
- track_io_timing=on
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
command:
|
||||
- valkey-server
|
||||
- --save
|
||||
- ""
|
||||
- --appendonly
|
||||
- "no"
|
||||
- --maxmemory
|
||||
- 64mb
|
||||
- --maxmemory-policy
|
||||
- allkeys-lru
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
minio:
|
||||
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
|
||||
command: ["server", "/data"]
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password
|
||||
MINIO_BROWSER: "off"
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
volumes:
|
||||
- miniodata:/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "mc ready local || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
minio-init:
|
||||
image: madlabcg/minio@sha256:ad134b2dd572ae0d989d6807c3e45516cc75a34883a8021db2ff04be5d1d496a
|
||||
entrypoint: ["/bin/sh", "/opt/init.sh"]
|
||||
environment:
|
||||
S3_BUCKET: messenger-attachments
|
||||
secrets:
|
||||
- minio_root_user
|
||||
- minio_root_password
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
volumes:
|
||||
- ./infra/minio/init.sh:/opt/init.sh:ro
|
||||
- ./infra/minio/app-policy.json:/etc/minio-policy/app-policy.json:ro
|
||||
depends_on:
|
||||
minio:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab4
|
||||
command: ["python", "scripts/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
miniodata:
|
||||
|
||||
secrets:
|
||||
minio_root_user:
|
||||
file: ./secrets/minio_root_user
|
||||
minio_root_password:
|
||||
file: ./secrets/minio_root_password
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key
|
||||
@@ -0,0 +1,4 @@
|
||||
metadata served by api-1
|
||||
ok uploaded via api-2, downloaded via api-1, identical sha256
|
||||
ok restart of all stateful services keeps metadata and original
|
||||
PASS: attachment failover + persistence
|
||||
@@ -0,0 +1,9 @@
|
||||
ok auth and membership on upload
|
||||
ok file over 10 MiB -> 413
|
||||
ok 48 Mpx image -> 413 before decoding
|
||||
ok corrupted image -> 415
|
||||
ok MIME spoofing and unsupported types -> 415
|
||||
ok dangerous filenames -> 400
|
||||
ok unknown / foreign / other-chat attachment ids rejected, no 500
|
||||
ok client abort mid-upload sent; server must stay healthy
|
||||
PASS: attachment negative scenarios
|
||||
@@ -0,0 +1,2 @@
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 0.441}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,11 @@
|
||||
ok truncated PNG, bad filter byte, broken IDAT, 27-byte JPEG stub -> 415
|
||||
ok real JPEG accepted; first half + EOI and first half alone -> 415
|
||||
ok same defects in images above the full-decode threshold (6 Mpx) -> 415
|
||||
ok valid 6 Mpx PNG accepted
|
||||
ok valid 36 Mpx PNG (116 KiB on the wire, 108 MiB decoded) accepted in 0.1s
|
||||
ok 36 Mpx PNG with bad filter byte -> 415
|
||||
ok whitespace-only, DEL, C1 and dot filenames -> 400
|
||||
ok attachment order preserved in POST, replay and history; reordered body -> 409
|
||||
ok idempotency: [] == absent, invalid keys -> 400, scoped per user
|
||||
ok 12 MiB body -> JSON 413
|
||||
PASS: input hardening
|
||||
@@ -0,0 +1,19 @@
|
||||
valid baseline 675893 bytes accepted 0.01s ok
|
||||
valid progressive 616591 bytes accepted 0.02s ok
|
||||
valid grayscale 359415 bytes accepted 0.00s ok
|
||||
valid CMYK 1262509 bytes accepted 0.01s ok
|
||||
valid, junk after EOI 675901 bytes accepted 0.01s ok
|
||||
valid 40 Mpx baseline 626630 bytes accepted 0.01s ok
|
||||
valid 40 Mpx progressive 352914 bytes accepted 0.07s ok
|
||||
valid 40 Mpx progressive 444 704086 bytes accepted 0.13s ok
|
||||
valid 40 Mpx progressive CMYK 626581 bytes accepted 0.17s ok
|
||||
27-byte stub 27 bytes rejected 415 0.00s ok
|
||||
first half + EOI 337948 bytes rejected 415 0.00s ok
|
||||
first half, no EOI 337946 bytes rejected 415 0.00s ok
|
||||
progressive half + EOI 308297 bytes rejected 415 0.00s ok
|
||||
headers only + EOI 611 bytes rejected 415 0.01s ok
|
||||
␍ ␍checker peak RSS, 40 Mpx baseline 1.7 MiB
|
||||
␍ ␍checker peak RSS, 40 Mpx progressive 116.3 MiB
|
||||
␍ ␍checker peak RSS, 40 Mpx progressive 444 230.6 MiB
|
||||
␍ ␍checker peak RSS, 40 Mpx progressive CMYK 306.9 MiB
|
||||
RESULT: 14/14
|
||||
@@ -0,0 +1,25 @@
|
||||
== 10 parallel uploads of 8674403 bytes, 3 rounds ==
|
||||
upload statuses: {201: 30}
|
||||
== 10 parallel downloads ==
|
||||
download statuses: {200: 20}
|
||||
== 10 parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), 3 rounds ==
|
||||
upload statuses: {201: 30}
|
||||
== 10 MiB + 1 byte, Content-Length: 100 requests, 10 at a time ==
|
||||
{'413 application/json': 100}
|
||||
== 10 MiB + 1 byte, chunked: 100 requests, 10 at a time ==
|
||||
{'413 application/json': 100}
|
||||
== 10 MiB + 1 byte, chunked in 64 KiB pieces: 100 requests, 10 at a time ==
|
||||
{'413 application/json': 100}
|
||||
== 8 MiB to a foreign chat: 100 requests, 10 at a time ==
|
||||
{'404 application/json': 100}
|
||||
== 8 MiB without session: 100 requests, 10 at a time ==
|
||||
{'401 application/json': 100}
|
||||
== 8 MiB with wrong magic: 100 requests, 10 at a time ==
|
||||
{'415 application/json': 100}
|
||||
== 8 MiB with wrong Content-Type: 100 requests, 10 at a time ==
|
||||
{'415 application/json': 100}
|
||||
== 70 KiB JSON message: 100 requests, 10 at a time ==
|
||||
{'413 application/json': 100}
|
||||
container events during the run: none
|
||||
cgroup memory.peak, MiB: {'mtusi-lab4-api-1-1': 475.1, 'mtusi-lab4-api-2-1': 490.5, 'mtusi-lab4-nginx-1': 34.3}
|
||||
PASS: LB stress
|
||||
@@ -0,0 +1,25 @@
|
||||
== 30 parallel uploads of 8674403 bytes, 2 rounds ==
|
||||
upload statuses: {201: 60}
|
||||
== 30 parallel downloads ==
|
||||
download statuses: {200: 60}
|
||||
== 30 parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), 2 rounds ==
|
||||
upload statuses: {201: 60}
|
||||
== 10 MiB + 1 byte, Content-Length: 30 requests, 30 at a time ==
|
||||
{'413 application/json': 30}
|
||||
== 10 MiB + 1 byte, chunked: 30 requests, 30 at a time ==
|
||||
{'413 application/json': 30}
|
||||
== 10 MiB + 1 byte, chunked in 64 KiB pieces: 30 requests, 30 at a time ==
|
||||
{'413 application/json': 30}
|
||||
== 8 MiB to a foreign chat: 30 requests, 30 at a time ==
|
||||
{'404 application/json': 30}
|
||||
== 8 MiB without session: 30 requests, 30 at a time ==
|
||||
{'401 application/json': 30}
|
||||
== 8 MiB with wrong magic: 30 requests, 30 at a time ==
|
||||
{'415 application/json': 30}
|
||||
== 8 MiB with wrong Content-Type: 30 requests, 30 at a time ==
|
||||
{'415 application/json': 30}
|
||||
== 70 KiB JSON message: 30 requests, 30 at a time ==
|
||||
{'413 application/json': 30}
|
||||
container events during the run: none
|
||||
cgroup memory.peak, MiB: {'mtusi-lab4-api-1-1': 465.3, 'mtusi-lab4-api-2-1': 463.0, 'mtusi-lab4-nginx-1': 34.3}
|
||||
PASS: LB stress
|
||||
@@ -0,0 +1,25 @@
|
||||
payload bytes: 8674403
|
||||
round 1 peak MiB per api: {'mtusi-lab4-api-2-1': 95.5, 'mtusi-lab4-api-1-1': 115.6} statuses=[201]
|
||||
round 2 peak MiB per api: {'mtusi-lab4-api-2-1': 104.8, 'mtusi-lab4-api-1-1': 115.6} statuses=[201]
|
||||
round 3 peak MiB per api: {'mtusi-lab4-api-2-1': 106.6, 'mtusi-lab4-api-1-1': 115.3} statuses=[201]
|
||||
{
|
||||
"payload_mib": 8.27,
|
||||
"concurrency": 10,
|
||||
"rounds": 3,
|
||||
"idle_mib": {
|
||||
"mtusi-lab4-api-2-1": 87.2,
|
||||
"mtusi-lab4-api-1-1": 87.8
|
||||
},
|
||||
"peak_mib": {
|
||||
"mtusi-lab4-api-2-1": 106.6,
|
||||
"mtusi-lab4-api-1-1": 115.6
|
||||
},
|
||||
"container_limit_mib": 512,
|
||||
"statuses": {
|
||||
"201": 30
|
||||
},
|
||||
"upload_seconds_median": 0.63,
|
||||
"upload_seconds_max": 0.82
|
||||
}
|
||||
peak=163 MiB oom_kill 0
|
||||
peak=154 MiB oom_kill 0
|
||||
@@ -0,0 +1,23 @@
|
||||
payload bytes: 8674403
|
||||
round 1 peak MiB per api: {'mtusi-lab4-api-1-1': 143.7, 'mtusi-lab4-api-2-1': 142.4} statuses=[201]
|
||||
round 2 peak MiB per api: {'mtusi-lab4-api-1-1': 166.1, 'mtusi-lab4-api-2-1': 153.2} statuses=[201]
|
||||
round 3 peak MiB per api: {'mtusi-lab4-api-1-1': 182.3, 'mtusi-lab4-api-2-1': 154.7} statuses=[201]
|
||||
{
|
||||
"payload_mib": 8.27,
|
||||
"concurrency": 10,
|
||||
"rounds": 3,
|
||||
"idle_mib": {
|
||||
"mtusi-lab4-api-1-1": 88.8,
|
||||
"mtusi-lab4-api-2-1": 88.4
|
||||
},
|
||||
"peak_mib": {
|
||||
"mtusi-lab4-api-1-1": 182.3,
|
||||
"mtusi-lab4-api-2-1": 154.7
|
||||
},
|
||||
"container_limit_mib": 256,
|
||||
"statuses": {
|
||||
"201": 30
|
||||
},
|
||||
"upload_seconds_median": 0.82,
|
||||
"upload_seconds_max": 1.28
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,12 @@
|
||||
== published ports of minio (expect none) ==
|
||||
no port 9000/tcp for container mtusi-lab4-minio-1:
|
||||
no port 9001/tcp for container mtusi-lab4-minio-1:
|
||||
== anonymous requests from inside the network ==
|
||||
GET /messenger-attachments/ -> 403
|
||||
GET /messenger-attachments/attachments/does-not-matter -> 403
|
||||
GET /messenger-attachments?list-type=2 -> 403
|
||||
== bucket policy ==
|
||||
Container mtusi-lab4-minio-init-run-85f57841572d Creating
|
||||
Container mtusi-lab4-minio-init-run-85f57841572d Created
|
||||
Access permission for `local/messenger-attachments` is `private`
|
||||
PASS: bucket is private
|
||||
@@ -0,0 +1,30 @@
|
||||
baseline (data left by earlier tests): {"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 366, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 203, "seconds": 0.253, "stale_uploading": 0}
|
||||
== dry-run, grace=900 ==
|
||||
{"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 2, "objects_scanned": 371, "orphan_objects": 0, "protected_fresh_objects": 2, "removed_rows": 0, "rows_scanned": 206, "seconds": 0.264, "stale_uploading": 1}
|
||||
ok dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)
|
||||
ok dry-run: 2 new missing objects (e, g)
|
||||
ok dry-run changes nothing
|
||||
== apply, grace=900 ==
|
||||
{"deleted_objects": 0, "dry_run": false, "grace_seconds": 900, "marked_failed": 2, "missing_objects": 2, "objects_scanned": 371, "orphan_objects": 0, "protected_fresh_objects": 2, "removed_rows": 1, "rows_scanned": 206, "seconds": 0.273, "stale_uploading": 1}
|
||||
ok old row whose object is young is kept (a)
|
||||
ok stale row without object removed
|
||||
ok fresh in-flight upload untouched
|
||||
ok fresh orphan object kept inside grace
|
||||
ok old row with a fresh object is left alone
|
||||
ok recently confirmed row is not marked missing
|
||||
ok missing unused object marked failed
|
||||
ok used attachment with object untouched
|
||||
ok used attachment with lost object kept, marked failed
|
||||
== apply after grace expired (grace=2 after 3s) ==
|
||||
{"deleted_objects": 4, "dry_run": false, "grace_seconds": 2, "marked_failed": 1, "missing_objects": 1, "objects_scanned": 368, "orphan_objects": 1, "protected_fresh_objects": 0, "removed_rows": 3, "rows_scanned": 368, "seconds": 0.428, "stale_uploading": 3}
|
||||
ok orphan object deleted after grace
|
||||
ok expired in-flight row collected after grace
|
||||
ok old row + object collected once the object is older than grace (a, h)
|
||||
ok used attachment still untouched
|
||||
== second apply is idempotent ==
|
||||
{"deleted_objects": 0, "dry_run": false, "grace_seconds": 2, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 367, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 367, "seconds": 0.457, "stale_uploading": 0}
|
||||
ok repeat run changes nothing
|
||||
== cost of a full walk, 1000 ready attachments ==
|
||||
{"deleted_objects": 0, "dry_run": true, "grace_seconds": 900, "marked_failed": 0, "missing_objects": 0, "objects_scanned": 1367, "orphan_objects": 0, "protected_fresh_objects": 0, "removed_rows": 0, "rows_scanned": 1204, "seconds": 1.605, "stale_uploading": 0}
|
||||
ok bulk walk finds nothing wrong
|
||||
RESULT: 18/18 checks passed
|
||||
@@ -0,0 +1,5 @@
|
||||
ok S3 down: upload/download/ready -> 503 JSON, metadata and text messages still served through the LB (no 502)
|
||||
ok recovered 2s after S3 start
|
||||
ok ready is 200 again
|
||||
ok hung S3: upload answered 503 in 7.0s
|
||||
PASS: S3 outage behaviour
|
||||
@@ -0,0 +1 @@
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments/attachments/*"]
|
||||
},
|
||||
{
|
||||
"Effect": "Allow",
|
||||
"Action": ["s3:ListBucket", "s3:GetBucketLocation"],
|
||||
"Resource": ["arn:aws:s3:::messenger-attachments"]
|
||||
}
|
||||
]
|
||||
}
|
||||
Executable
+19
@@ -0,0 +1,19 @@
|
||||
#!/bin/sh
|
||||
# Idempotent: safe to run on every `compose up`. Root credentials are used only here, never by the API.
|
||||
set -eu
|
||||
BUCKET="${S3_BUCKET:-messenger-attachments}"
|
||||
ROOT_USER="$(cat /run/secrets/minio_root_user)"
|
||||
ROOT_PASSWORD="$(cat /run/secrets/minio_root_password)"
|
||||
APP_USER="$(cat /run/secrets/s3_access_key_id)"
|
||||
APP_PASSWORD="$(cat /run/secrets/s3_secret_access_key)"
|
||||
|
||||
until mc alias set local http://minio:9000 "$ROOT_USER" "$ROOT_PASSWORD" >/dev/null 2>&1; do
|
||||
echo "waiting for minio"; sleep 2
|
||||
done
|
||||
mc mb --ignore-existing "local/$BUCKET"
|
||||
mc anonymous set none "local/$BUCKET"
|
||||
mc admin policy create local app-attachments /etc/minio-policy/app-policy.json 2>/dev/null \
|
||||
|| mc admin policy create local app-attachments /etc/minio-policy/app-policy.json --overwrite 2>/dev/null || true
|
||||
mc admin user add local "$APP_USER" "$APP_PASSWORD" 2>/dev/null || true
|
||||
mc admin policy attach local app-attachments --user "$APP_USER" 2>/dev/null || true
|
||||
echo "minio bootstrap complete: bucket=$BUCKET private, app user limited to attachments/*"
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/bin/sh
|
||||
set -u
|
||||
|
||||
SERVERS="${API_UPSTREAMS:-api-1 api-2}"
|
||||
INTERVAL="${HEALTH_INTERVAL:-1}"
|
||||
FALL="${HEALTH_FALL:-2}"
|
||||
RISE="${HEALTH_RISE:-2}"
|
||||
OUT=/etc/nginx/upstream/servers.conf
|
||||
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
|
||||
var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; }
|
||||
|
||||
render() {
|
||||
any_up=0
|
||||
for s in $SERVERS; do
|
||||
eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1
|
||||
done
|
||||
for s in $SERVERS; do
|
||||
mark=""
|
||||
if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then
|
||||
mark=" down"
|
||||
fi
|
||||
echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;"
|
||||
done
|
||||
}
|
||||
|
||||
for s in $SERVERS; do
|
||||
v=$(var "$s")
|
||||
eval "up_$v=1 ok_$v=0 bad_$v=0"
|
||||
done
|
||||
render > "$OUT"
|
||||
|
||||
watch() {
|
||||
pending=0
|
||||
while sleep "$INTERVAL"; do
|
||||
changed=0
|
||||
for s in $SERVERS; do
|
||||
v=$(var "$s")
|
||||
if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then
|
||||
eval "ok_$v=\$((ok_$v + 1)) bad_$v=0"
|
||||
if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then
|
||||
eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream"
|
||||
fi
|
||||
else
|
||||
eval "bad_$v=\$((bad_$v + 1)) ok_$v=0"
|
||||
if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then
|
||||
eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "$changed" = 1 ]; then
|
||||
render > "$OUT"
|
||||
pending=1
|
||||
fi
|
||||
if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then
|
||||
pending=0
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
watch &
|
||||
exec nginx -g 'daemon off;'
|
||||
@@ -0,0 +1,75 @@
|
||||
worker_processes 2;
|
||||
|
||||
events {
|
||||
worker_connections 256;
|
||||
}
|
||||
|
||||
http {
|
||||
resolver 127.0.0.11 valid=5s ipv6=off;
|
||||
|
||||
upstream api {
|
||||
zone api 64k;
|
||||
include /etc/nginx/upstream/servers.conf;
|
||||
keepalive 32;
|
||||
keepalive_timeout 4s;
|
||||
}
|
||||
|
||||
log_format main '$remote_addr - $upstream_addr [$time_local] '
|
||||
'"$request" $status $body_bytes_sent '
|
||||
'"$upstream_response_time" "$request_time"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 64k;
|
||||
error_page 413 @payload_too_large;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 5s;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/messages$ {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
|
||||
}
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/attachments$ {
|
||||
client_max_body_size 10m;
|
||||
proxy_request_buffering off;
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream off;
|
||||
}
|
||||
|
||||
location @payload_too_large {
|
||||
default_type application/json;
|
||||
add_header X-Request-Id $request_id always;
|
||||
add_header X-Instance-Id lb always;
|
||||
add_header Connection close always;
|
||||
return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}';
|
||||
}
|
||||
|
||||
location /health/ {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout;
|
||||
}
|
||||
|
||||
location ~ ^/api/v1/attachments/ {
|
||||
proxy_max_temp_file_size 0;
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout http_502 http_503;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name bytea NOT NULL
|
||||
CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title bytea NOT NULL
|
||||
CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX chats_created_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
text bytea NOT NULL
|
||||
CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
key text NOT NULL
|
||||
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'),
|
||||
request_body_hash text NOT NULL,
|
||||
response_json text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,28 @@
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
filename text NOT NULL
|
||||
CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL
|
||||
CONSTRAINT attachments_ct CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL
|
||||
CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL
|
||||
CONSTRAINT attachments_sha CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
object_key text NOT NULL,
|
||||
status text NOT NULL DEFAULT 'ready'
|
||||
CONSTRAINT attachments_status CHECK (status IN ('ready', 'failed')),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL,
|
||||
updated_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_idx ON attachments (chat_id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
attachment_id uuid NOT NULL REFERENCES attachments (id) ON DELETE RESTRICT,
|
||||
position smallint NOT NULL,
|
||||
PRIMARY KEY (message_id, attachment_id)
|
||||
);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE messages DROP CONSTRAINT messages_text_size;
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 0 AND 16000);
|
||||
@@ -0,0 +1,5 @@
|
||||
ALTER TABLE attachments DROP CONSTRAINT attachments_status;
|
||||
ALTER TABLE attachments ADD CONSTRAINT attachments_status CHECK (status IN ('uploading', 'ready', 'failed'));
|
||||
ALTER TABLE attachments ALTER COLUMN status DROP DEFAULT;
|
||||
CREATE UNIQUE INDEX attachments_object_key_idx ON attachments (object_key);
|
||||
CREATE INDEX attachments_status_created_idx ON attachments (status, created_at);
|
||||
@@ -0,0 +1,30 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
boto3==1.35.81
|
||||
botocore==1.35.81
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
fastapi==0.115.6
|
||||
h11==0.16.0
|
||||
httptools==0.9.0
|
||||
idna==3.20
|
||||
jmespath==1.0.1
|
||||
pillow==11.0.0
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
pycparser==3.11
|
||||
pydantic==2.14.1
|
||||
pydantic-core==2.50.1
|
||||
python-dateutil==2.9.0.post0
|
||||
s3transfer==0.10.4
|
||||
six==1.17.0
|
||||
starlette==0.41.3
|
||||
typing-extensions==4.16.0
|
||||
typing-inspection==0.4.4
|
||||
urllib3==2.3.0
|
||||
uvicorn==0.34.0
|
||||
uvloop==0.23.0
|
||||
valkey==6.2.0
|
||||
@@ -0,0 +1,91 @@
|
||||
#!/usr/bin/env python3
|
||||
"""JPEG integrity matrix and peak memory of the checker.
|
||||
docker run --rm --entrypoint python mtusi-messenger:lab4 scripts/jpeg_check_experiment.py"""
|
||||
import io
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from PIL import Image # noqa: E402
|
||||
|
||||
from app.errors import ApiError # noqa: E402
|
||||
from app.imagecheck import DJPEG, inspect_image # noqa: E402
|
||||
|
||||
|
||||
def encode(img, **kw):
|
||||
out = io.BytesIO()
|
||||
img.save(out, "JPEG", **kw)
|
||||
return out.getvalue()
|
||||
|
||||
|
||||
def verdict(data):
|
||||
with tempfile.SpooledTemporaryFile(max_size=1048576) as spool:
|
||||
spool.write(data)
|
||||
try:
|
||||
inspect_image(spool, len(data), "image/jpeg")
|
||||
return "accepted"
|
||||
except ApiError as exc:
|
||||
return "rejected %d" % exc.status
|
||||
|
||||
|
||||
def child_peak_kib(data):
|
||||
with tempfile.NamedTemporaryFile() as handle:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
handle.seek(0)
|
||||
proc = subprocess.Popen(DJPEG, stdin=handle)
|
||||
peak = 0
|
||||
while proc.poll() is None:
|
||||
try:
|
||||
with open("/proc/%d/status" % proc.pid) as status:
|
||||
for line in status:
|
||||
if line.startswith("VmHWM"):
|
||||
peak = max(peak, int(line.split()[1]))
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.002)
|
||||
return peak
|
||||
|
||||
|
||||
noise = Image.frombytes("RGB", (1000, 750), os.urandom(1000 * 750 * 3))
|
||||
base = encode(noise, quality=90)
|
||||
progressive = encode(noise, quality=90, progressive=True)
|
||||
flat = Image.new("RGB", (8000, 5000), (120, 30, 200))
|
||||
big_base = encode(flat, quality=90)
|
||||
big_progressive = encode(flat, quality=90, progressive=True)
|
||||
big_444 = encode(flat, quality=90, progressive=True, subsampling=0)
|
||||
big_cmyk = encode(flat.convert("CMYK"), quality=90, progressive=True)
|
||||
del flat
|
||||
sos = base.index(b"\xff\xda")
|
||||
cases = [
|
||||
("valid baseline", base, "accepted"),
|
||||
("valid progressive", progressive, "accepted"),
|
||||
("valid grayscale", encode(noise.convert("L")), "accepted"),
|
||||
("valid CMYK", encode(noise.convert("CMYK")), "accepted"),
|
||||
("valid, junk after EOI", base + b"trailing", "accepted"),
|
||||
("valid 40 Mpx baseline", big_base, "accepted"),
|
||||
("valid 40 Mpx progressive", big_progressive, "accepted"),
|
||||
("valid 40 Mpx progressive 444", big_444, "accepted"),
|
||||
("valid 40 Mpx progressive CMYK", big_cmyk, "accepted"),
|
||||
("27-byte stub", bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9"), "rejected 415"),
|
||||
("first half + EOI", base[: len(base) // 2] + b"\xff\xd9", "rejected 415"),
|
||||
("first half, no EOI", base[: len(base) // 2], "rejected 415"),
|
||||
("progressive half + EOI", progressive[: len(progressive) // 2] + b"\xff\xd9", "rejected 415"),
|
||||
("headers only + EOI", base[:sos] + b"\xff\xd9", "rejected 415"),
|
||||
]
|
||||
failed = 0
|
||||
for name, data, expected in cases:
|
||||
started = time.monotonic()
|
||||
got = verdict(data)
|
||||
ok = got == expected
|
||||
failed += not ok
|
||||
print("%-28s %9d bytes %-13s %.2fs %s" % (name, len(data), got, time.monotonic() - started, "ok" if ok else "FAIL"))
|
||||
for name, data in (("40 Mpx baseline", big_base), ("40 Mpx progressive", big_progressive),
|
||||
("40 Mpx progressive 444", big_444), ("40 Mpx progressive CMYK", big_cmyk)):
|
||||
print("checker peak RSS, %-24s %6.1f MiB" % (name, child_peak_kib(data) / 1024))
|
||||
print("RESULT: %d/%d" % (len(cases) - failed, len(cases)))
|
||||
sys.exit(1 if failed else 0)
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
MIGRATIONS = ROOT / "migrations"
|
||||
LOCK_ID = 2026_0004
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
path = os.environ.get("DATABASE_URL_FILE")
|
||||
if path:
|
||||
return pathlib.Path(path).read_text(encoding="utf-8").strip()
|
||||
url = os.environ.get("DATABASE_URL", "").strip()
|
||||
if not url:
|
||||
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||
return url
|
||||
|
||||
|
||||
def main() -> None:
|
||||
files = sorted(MIGRATIONS.glob("*.sql"))
|
||||
if not files:
|
||||
sys.exit("no migration files found in %s" % MIGRATIONS)
|
||||
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY,"
|
||||
" checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
|
||||
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
|
||||
applied = {row[0]: row[1] for row in rows}
|
||||
for path in files:
|
||||
version = path.stem
|
||||
body = path.read_text(encoding="utf-8")
|
||||
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
sys.exit("migration %s changed after it was applied" % version)
|
||||
print("skip %s" % version)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(body)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
|
||||
(version, checksum),
|
||||
)
|
||||
print("apply %s" % version)
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
print("migrations complete: %d file(s)" % len(files))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run one DB<->S3 reconciliation pass. Default is dry-run; pass --apply to change anything."""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import config, reconcile # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--apply", action="store_true", help="delete/mark; without it only reports")
|
||||
parser.add_argument("--grace-seconds", type=int, default=config.RECONCILE_GRACE_SECONDS)
|
||||
args = parser.parse_args()
|
||||
db, s3 = Database(), S3Storage()
|
||||
await db.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
try:
|
||||
print(json.dumps(await reconcile.run(db, s3, args.grace_seconds, dry_run=not args.apply), sort_keys=True))
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fault injection for the DB<->S3 reconciler. Run inside an api container:
|
||||
docker compose exec -T api-1 python scripts/reconcile_experiment.py [--bulk 1000]"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import io
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, "/app")
|
||||
|
||||
from starlette.concurrency import run_in_threadpool # noqa: E402
|
||||
|
||||
from app import config, reconcile # noqa: E402
|
||||
from app.db import Database # noqa: E402
|
||||
from app.s3 import S3Storage # noqa: E402
|
||||
|
||||
ZERO = "0" * 64
|
||||
checks = []
|
||||
|
||||
|
||||
def check(name, condition):
|
||||
checks.append((name, bool(condition)))
|
||||
print(("ok " if condition else "FAIL ") + name)
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--bulk", type=int, default=1000)
|
||||
args = parser.parse_args()
|
||||
db, s3 = Database(), S3Storage()
|
||||
await db.open()
|
||||
await run_in_threadpool(s3.open)
|
||||
put = lambda key: run_in_threadpool(s3.put, key, io.BytesIO(b"x"), 1, "image/png")
|
||||
exists = lambda key: run_in_threadpool(s3.exists, key)
|
||||
|
||||
baseline = await reconcile.run(db, s3, 900, dry_run=True)
|
||||
print("baseline (data left by earlier tests):", json.dumps(baseline, sort_keys=True))
|
||||
|
||||
async with db.connection() as conn:
|
||||
suffix = uuid.uuid4().hex[:10]
|
||||
user = (await (await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, 'x', now()) RETURNING id", ("rx_" + suffix, b"rx"))).fetchone())["id"]
|
||||
chat = (await (await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at) VALUES (gen_random_uuid(), %s, now()) RETURNING id", (b"rx",))).fetchone())["id"]
|
||||
await conn.execute("INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", (chat, user))
|
||||
|
||||
async def row(status, age_s, with_object, linked=False):
|
||||
aid = str(uuid.uuid4())
|
||||
key = "attachments/" + aid
|
||||
await conn.execute(
|
||||
"INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, object_key,"
|
||||
" status, created_at, updated_at) VALUES (%s,%s,%s,'f.png','image/png',1,%s,%s,%s,"
|
||||
" now() - make_interval(secs => %s), now() - make_interval(secs => %s))", (aid, chat, user, ZERO, key, status, age_s, age_s))
|
||||
if with_object:
|
||||
await put(key)
|
||||
if linked:
|
||||
mid = (await (await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, ''::bytea, now()) RETURNING id", (chat, user))).fetchone())["id"]
|
||||
await conn.execute("INSERT INTO message_attachments VALUES (%s,%s,0)", (mid, aid))
|
||||
return aid, key
|
||||
|
||||
async def status_of(aid):
|
||||
r = await (await conn.execute("SELECT status, error_code FROM attachments WHERE id=%s", (aid,))).fetchone()
|
||||
return (r["status"], r["error_code"]) if r else None
|
||||
|
||||
old = 7200
|
||||
a_id, a_key = await row("uploading", old, True) # crash after PUT, before ready (object is young)
|
||||
b_id, b_key = await row("uploading", old, False) # crash before PUT
|
||||
c_id, c_key = await row("uploading", 5, True) # in-flight upload, must survive
|
||||
e_id, e_key = await row("ready", old, False) # object lost
|
||||
f_id, f_key = await row("ready", old, True, linked=True) # healthy, used
|
||||
g_id, g_key = await row("ready", old, False, linked=True) # object lost, used
|
||||
h_id, h_key = await row("uploading", old, False) # old intent row, but a NEW object under its key
|
||||
await put(h_key)
|
||||
i_id, i_key = await row("ready", 5, False) # freshly confirmed, object not yet visible: must not be marked
|
||||
d_key = "attachments/" + str(uuid.uuid4()) # PUT done, row never written
|
||||
await put(d_key)
|
||||
|
||||
print("== dry-run, grace=900 ==")
|
||||
r = await reconcile.run(db, s3, 900, dry_run=True)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("dry-run: 1 new stale row (b), 2 old rows protected by young objects (a, h)",
|
||||
r["stale_uploading"] - baseline["stale_uploading"] == 1
|
||||
and r["protected_fresh_objects"] - baseline["protected_fresh_objects"] == 2)
|
||||
check("dry-run: 2 new missing objects (e, g)", r["missing_objects"] - baseline["missing_objects"] == 2)
|
||||
check("dry-run changes nothing", await exists(a_key) and await status_of(a_id) == ("uploading", None)
|
||||
and await status_of(e_id) == ("ready", None))
|
||||
|
||||
print("== apply, grace=900 ==")
|
||||
r = await reconcile.run(db, s3, 900, dry_run=False)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("old row whose object is young is kept (a)", await status_of(a_id) == ("uploading", None) and await exists(a_key))
|
||||
check("stale row without object removed", await status_of(b_id) is None)
|
||||
check("fresh in-flight upload untouched", await status_of(c_id) == ("uploading", None) and await exists(c_key))
|
||||
check("fresh orphan object kept inside grace", await exists(d_key))
|
||||
check("old row with a fresh object is left alone", await status_of(h_id) == ("uploading", None) and await exists(h_key))
|
||||
check("recently confirmed row is not marked missing", await status_of(i_id) == ("ready", None))
|
||||
check("missing unused object marked failed", await status_of(e_id) == ("failed", "object_missing"))
|
||||
check("used attachment with object untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
|
||||
check("used attachment with lost object kept, marked failed", await status_of(g_id) == ("failed", "object_missing"))
|
||||
|
||||
print("== apply after grace expired (grace=2 after 3s) ==")
|
||||
await asyncio.sleep(3)
|
||||
r = await reconcile.run(db, s3, 2, dry_run=False)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("orphan object deleted after grace", not await exists(d_key))
|
||||
check("expired in-flight row collected after grace", await status_of(c_id) is None and not await exists(c_key))
|
||||
check("old row + object collected once the object is older than grace (a, h)",
|
||||
await status_of(a_id) is None and not await exists(a_key)
|
||||
and await status_of(h_id) is None and not await exists(h_key))
|
||||
check("used attachment still untouched", await status_of(f_id) == ("ready", None) and await exists(f_key))
|
||||
|
||||
print("== second apply is idempotent ==")
|
||||
r = await reconcile.run(db, s3, 2, dry_run=False)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("repeat run changes nothing", r["deleted_objects"] == 0 and r["removed_rows"] == 0 and r["marked_failed"] == 0)
|
||||
|
||||
print("== cost of a full walk, %d ready attachments ==" % args.bulk)
|
||||
keys = []
|
||||
for _ in range(args.bulk):
|
||||
aid, key = await row("ready", old, True)
|
||||
keys.append(aid)
|
||||
r = await reconcile.run(db, s3, 900, dry_run=True)
|
||||
print(json.dumps(r, sort_keys=True))
|
||||
check("bulk walk finds nothing wrong", r["missing_objects"] == 0 and r["orphan_objects"] == 0)
|
||||
|
||||
for aid in keys + [e_id, f_id, g_id, h_id, i_id]:
|
||||
await run_in_threadpool(s3.delete, "attachments/" + aid)
|
||||
await conn.execute("DELETE FROM chats WHERE id = %s", (chat,))
|
||||
await conn.execute("DELETE FROM users WHERE id = %s", (user,))
|
||||
await db.close()
|
||||
failed = [n for n, ok in checks if not ok]
|
||||
print("RESULT: %d/%d checks passed" % (len(checks) - len(failed), len(checks)))
|
||||
sys.exit(1 if failed else 0)
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart -t 5 postgres valkey minio api-1 api-2 nginx
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/bin/sh
|
||||
# Proves the bucket is private without going through the API: anonymous S3 requests from inside the compose
|
||||
# network are refused, and the MinIO console/ports are not published on the host.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
NET="$(docker compose ps -q minio | xargs docker inspect --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}}{{end}}')"
|
||||
BUCKET=messenger-attachments
|
||||
echo "== published ports of minio (expect none) =="
|
||||
docker compose port minio 9000 2>&1 || true
|
||||
docker compose port minio 9001 2>&1 || true
|
||||
echo "== anonymous requests from inside the network =="
|
||||
for path in "$BUCKET/" "$BUCKET/attachments/does-not-matter" "$BUCKET?list-type=2"; do
|
||||
code="$(docker run --rm -i --network "$NET" python:3.12.8-slim-bookworm python - "$path" <<'PY'
|
||||
import sys, urllib.request, urllib.error
|
||||
try:
|
||||
print(urllib.request.urlopen("http://minio:9000/" + sys.argv[1], timeout=5).status)
|
||||
except urllib.error.HTTPError as e:
|
||||
print(e.code)
|
||||
PY
|
||||
)"
|
||||
echo "GET /$path -> $code"
|
||||
[ "$code" = 403 ] || { echo "FAIL: expected 403"; exit 1; }
|
||||
done
|
||||
echo "== bucket policy =="
|
||||
docker compose run --rm --no-deps --entrypoint sh minio-init -c \
|
||||
'mc alias set local http://minio:9000 "$(cat /run/secrets/minio_root_user)" "$(cat /run/secrets/minio_root_password)" >/dev/null && mc anonymous get local/messenger-attachments'
|
||||
echo "PASS: bucket is private"
|
||||
@@ -0,0 +1,93 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Peak API memory under N concurrent ~8.7 MB uploads. Samples `docker stats` for the api containers.
|
||||
python3 student-tests/memory_experiment.py --concurrency 10 --rounds 3"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
import struct
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client # noqa: E402
|
||||
|
||||
|
||||
def png(width, height):
|
||||
raw = b"".join(b"\0" + os.urandom(width * 3) for _ in range(height))
|
||||
def chunk(kind, body):
|
||||
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
|
||||
return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0))
|
||||
+ chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b""))
|
||||
|
||||
|
||||
UNITS = {"B": 1, "KiB": 1024, "MiB": 1048576, "GiB": 1073741824}
|
||||
|
||||
|
||||
def mib(text):
|
||||
value, unit = re.match(r"([\d.]+)\s*([A-Za-z]+)", text).groups()
|
||||
return float(value) * UNITS[unit] / 1048576
|
||||
|
||||
|
||||
def sample(stop, out):
|
||||
while not stop.is_set():
|
||||
result = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.MemUsage}}"],
|
||||
cwd=ROOT, capture_output=True, text=True)
|
||||
for line in result.stdout.splitlines():
|
||||
name, usage = line.split(" ", 1)
|
||||
if "api-" in name:
|
||||
out.setdefault(name, []).append(mib(usage.split("/")[0].strip()))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--rounds", type=int, default=3)
|
||||
args = parser.parse_args()
|
||||
body = png(1700, 1700)
|
||||
print("payload bytes:", len(body))
|
||||
c = Client(timeout=60)
|
||||
alice = c.user("mem")
|
||||
chat = c.call("POST", "/api/v1/chats", 201, actor=alice, body={"title": "mem", "member_ids": [alice["id"]]}, schema="Chat")["json"]
|
||||
path = "/api/v1/chats/%s/attachments?filename=big.png" % chat["id"]
|
||||
|
||||
def one(_):
|
||||
r = c.call("POST", path, [201, 429, 503], actor=alice, raw=body, headers={"Content-Type": "image/png"})
|
||||
return r["status"], r["elapsed"]
|
||||
|
||||
idle, peaks, statuses, times = {}, [], [], []
|
||||
probe = {}
|
||||
stop = threading.Event()
|
||||
t = threading.Thread(target=sample, args=(stop, probe)); t.start(); time.sleep(3); stop.set(); t.join()
|
||||
idle = {k: max(v) for k, v in probe.items()}
|
||||
for round_no in range(args.rounds):
|
||||
probe = {}
|
||||
stop = threading.Event()
|
||||
t = threading.Thread(target=sample, args=(stop, probe)); t.start()
|
||||
with ThreadPoolExecutor(args.concurrency) as pool:
|
||||
results = list(pool.map(one, range(args.concurrency)))
|
||||
stop.set(); t.join()
|
||||
peak = {k: max(v) for k, v in probe.items()}
|
||||
peaks.append(peak)
|
||||
statuses += [s for s, _ in results]
|
||||
times += [e for _, e in results]
|
||||
print("round %d peak MiB per api: %s statuses=%s" % (round_no + 1, {k: round(v, 1) for k, v in peak.items()}, sorted(set(s for s, _ in results))))
|
||||
summary = {
|
||||
"payload_mib": round(len(body) / 1048576, 2), "concurrency": args.concurrency, "rounds": args.rounds,
|
||||
"idle_mib": {k: round(v, 1) for k, v in idle.items()},
|
||||
"peak_mib": {k: round(max(p.get(k, 0) for p in peaks), 1) for k in idle},
|
||||
"container_limit_mib": 512,
|
||||
"statuses": {str(s): statuses.count(s) for s in sorted(set(statuses))},
|
||||
"upload_seconds_median": round(statistics.median(times), 2), "upload_seconds_max": round(max(times), 2),
|
||||
}
|
||||
print(json.dumps(summary, indent=2))
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload -> stop the serving replica -> download via the other; then restart everything and download again."""
|
||||
import hashlib
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check, run # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True)
|
||||
|
||||
|
||||
def wait_ready(c):
|
||||
for _ in range(60):
|
||||
try:
|
||||
c.call("GET", "/health/ready")
|
||||
return
|
||||
except Exception:
|
||||
time.sleep(2)
|
||||
check(False, "stack did not become ready")
|
||||
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
att, content = c.attachment(alice, chat)
|
||||
served_by = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["headers"]["X-Instance-Id"]
|
||||
print("metadata served by", served_by)
|
||||
first = c.call("POST", "/api/v1/chats/%s/attachments?filename=x.png" % chat["id"], 201, actor=alice, raw=content,
|
||||
headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
uploader = first["headers"]["X-Instance-Id"]
|
||||
compose("stop", "-t", "3", uploader)
|
||||
try:
|
||||
time.sleep(3)
|
||||
got = c.call("GET", "/api/v1/attachments/%s/content" % first["json"]["id"], actor=bob)
|
||||
check(got["headers"]["X-Instance-Id"] != uploader, "download must be served by another replica")
|
||||
check(hashlib.sha256(got["bytes"]).hexdigest() == first["json"]["sha256"], "bytes differ after uploader stopped")
|
||||
print("ok uploaded via %s, downloaded via %s, identical sha256" % (uploader, got["headers"]["X-Instance-Id"]))
|
||||
finally:
|
||||
compose("start", uploader)
|
||||
wait_ready(c)
|
||||
compose("restart", "-t", "5", "postgres", "valkey", "minio", "api-1", "api-2", "nginx")
|
||||
wait_ready(c)
|
||||
got = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", actor=bob)
|
||||
check(got["bytes"] == content, "original lost after full restart")
|
||||
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
|
||||
check(meta == att, "metadata changed after restart")
|
||||
print("ok restart of all stateful services keeps metadata and original")
|
||||
print("PASS: attachment failover + persistence")
|
||||
|
||||
|
||||
run(main)
|
||||
@@ -0,0 +1,115 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Negative and boundary scenarios for attachments (grade 4). Needs the stack from README on BASE_URL."""
|
||||
import hashlib
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import sys
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urlparse
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check, run # noqa: E402
|
||||
|
||||
SAMPLE = (ROOT / "tests/fixtures/sample.png").read_bytes()
|
||||
|
||||
|
||||
def chunk(kind, body):
|
||||
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def header_only_png(width, height):
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def upload(c, actor, chat, body, ctype, expected, filename="a.png"):
|
||||
path = "/api/v1/chats/%s/attachments?filename=%s" % (chat["id"], quote(filename, safe=""))
|
||||
return c.call("POST", path, expected, actor=actor, raw=body, headers={"Content-Type": ctype},
|
||||
schema="Attachment" if expected == 201 else None)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
other = c.call("POST", "/api/v1/chats", 201, actor=alice,
|
||||
body={"title": "Второй", "member_ids": [alice["id"]]}, schema="Chat")["json"]
|
||||
eve_chat = c.call("POST", "/api/v1/chats", 201, actor=eve,
|
||||
body={"title": "Чат Евы", "member_ids": [eve["id"]]}, schema="Chat")["json"]
|
||||
|
||||
ok = upload(c, alice, chat, SAMPLE, "image/png", 201)["json"]
|
||||
check(ok["status"] == "ready" and ok["variants"] == [] and ok["error_code"] is None, "ready upload shape")
|
||||
got = c.call("GET", "/api/v1/attachments/" + ok["id"] + "/content", actor=bob)
|
||||
check(hashlib.sha256(got["bytes"]).hexdigest() == ok["sha256"], "bob downloads identical bytes")
|
||||
check(int(got["headers"]["Content-Length"]) == len(SAMPLE), "download Content-Length")
|
||||
c.call("GET", "/api/v1/attachments/" + ok["id"] + "/content?variant=thumbnail", 404, actor=bob)
|
||||
|
||||
upload(c, None, chat, SAMPLE, "image/png", 401)
|
||||
upload(c, eve, chat, SAMPLE, "image/png", 404)
|
||||
print("ok auth and membership on upload")
|
||||
|
||||
big = header_only_png(10, 10) + b"\0" * (10 * 1024 * 1024)
|
||||
upload(c, alice, chat, big, "image/png", 413)
|
||||
print("ok file over 10 MiB -> 413")
|
||||
upload(c, alice, chat, header_only_png(8000, 6000), "image/png", 413)
|
||||
print("ok 48 Mpx image -> 413 before decoding")
|
||||
upload(c, alice, chat, SAMPLE[: len(SAMPLE) // 2], "image/png", 415)
|
||||
upload(c, alice, chat, SAMPLE[:40] + b"garbage" * 50, "image/png", 415)
|
||||
print("ok corrupted image -> 415")
|
||||
upload(c, alice, chat, SAMPLE, "image/jpeg", 415)
|
||||
upload(c, alice, chat, b"\xff\xd8\xff\xe0" + b"\0" * 200, "image/png", 415)
|
||||
upload(c, alice, chat, b"MZ not an image at all", "image/png", 415)
|
||||
upload(c, alice, chat, b"<svg xmlns='http://www.w3.org/2000/svg'/>", "image/png", 415)
|
||||
upload(c, alice, chat, SAMPLE, "text/plain", 415)
|
||||
upload(c, alice, chat, SAMPLE, "image/gif", 415)
|
||||
print("ok MIME spoofing and unsupported types -> 415")
|
||||
upload(c, alice, chat, b"", "image/png", 400)
|
||||
for bad in ("../x.png", "a/b.png", "a\\b.png", "a\x00b.png", "a\nb.png", "", "x" * 256):
|
||||
upload(c, alice, chat, SAMPLE, "image/png", 400, filename=bad)
|
||||
print("ok dangerous filenames -> 400")
|
||||
unicode_name = upload(c, alice, chat, SAMPLE, "image/png", 201, filename="фото 🙂.png")["json"]
|
||||
check(unicode_name["filename"] == "фото 🙂.png", "display filename preserved")
|
||||
|
||||
own = upload(c, alice, other, SAMPLE, "image/png", 201)["json"]
|
||||
foreign = upload(c, eve, eve_chat, SAMPLE, "image/png", 201)["json"]
|
||||
msg = lambda actor, ch, ids, expected: c.call(
|
||||
"POST", "/api/v1/chats/%s/messages" % ch["id"], expected, actor=actor,
|
||||
body={"text": "", "attachment_ids": ids}, headers={"Idempotency-Key": uuid.uuid4().hex}, schema="Message" if expected == 201 else None)
|
||||
msg(alice, chat, [str(uuid.uuid4())], 404)
|
||||
msg(alice, chat, [foreign["id"]], 404)
|
||||
msg(alice, chat, [own["id"]], 404)
|
||||
msg(alice, chat, [ok["id"], ok["id"]], 400)
|
||||
msg(alice, chat, ["not-a-uuid"], 400)
|
||||
msg(alice, chat, [str(uuid.uuid4()) for _ in range(11)], 400)
|
||||
msg(eve, chat, [ok["id"]], 404)
|
||||
c.call("POST", "/api/v1/chats/%s/messages" % chat["id"], 400, actor=alice, body={"text": " "},
|
||||
headers={"Idempotency-Key": uuid.uuid4().hex})
|
||||
print("ok unknown / foreign / other-chat attachment ids rejected, no 500")
|
||||
sent = msg(bob, chat, [ok["id"]], 201)["json"]
|
||||
check(sent["attachment_ids"] == [ok["id"]] and sent["text"] == "", "attachment-only message")
|
||||
history = c.call("GET", "/api/v1/chats/%s/messages" % chat["id"], actor=alice, schema="MessagePage")["json"]
|
||||
check(any(m["attachment_ids"] == [ok["id"]] for m in history["items"]), "history keeps attachment_ids")
|
||||
for suffix in ("", "/content"):
|
||||
c.call("GET", "/api/v1/attachments/" + foreign["id"] + suffix, 404, actor=alice)
|
||||
c.call("GET", "/api/v1/attachments/" + str(uuid.uuid4()) + suffix, 404, actor=alice)
|
||||
c.call("GET", "/api/v1/attachments/not-a-uuid" + suffix, 400, actor=alice)
|
||||
c.call("GET", "/api/v1/attachments/" + ok["id"] + suffix, 401)
|
||||
|
||||
parsed = urlparse(c.base)
|
||||
sock = socket.create_connection((parsed.hostname, parsed.port or 80), timeout=10)
|
||||
request = ("POST /api/v1/chats/%s/attachments?filename=cut.png HTTP/1.1\r\nHost: x\r\nAuthorization: Bearer %s\r\n"
|
||||
"Content-Type: image/png\r\nContent-Length: %d\r\n\r\n" % (chat["id"], alice["token"], len(SAMPLE)))
|
||||
sock.sendall(request.encode() + SAMPLE[: len(SAMPLE) // 2])
|
||||
sock.close()
|
||||
print("ok client abort mid-upload sent; server must stay healthy")
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
after = upload(c, alice, chat, SAMPLE, "image/png", 201)["json"]
|
||||
check(after["sha256"] == hashlib.sha256(SAMPLE).hexdigest(), "upload works after aborted one")
|
||||
print("PASS: attachment negative scenarios")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Hardening scenarios: corrupted images, filenames, attachment order, idempotency keys, oversized bodies."""
|
||||
import hashlib
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import uuid
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
sys.path.insert(0, str(ROOT / "student-tests"))
|
||||
from client import Client, check, run # noqa: E402
|
||||
from test_attachments import SAMPLE, chunk, header_only_png # noqa: E402
|
||||
|
||||
JPEG_STUB = bytes.fromhex("ffd8ffc0000b080001000101011100ffda0008010100003f00ffd9")
|
||||
|
||||
|
||||
def png(width, height, filter_byte=0, level=1, corrupt_idat=False, noise=False):
|
||||
row = bytes([filter_byte]) + (os.urandom(width * 3) if noise else b"\x10" * (width * 3))
|
||||
raw = row * height
|
||||
data = zlib.compress(raw, level)
|
||||
if corrupt_idat:
|
||||
mid = len(data) // 2
|
||||
data = data[:mid] + bytes(b ^ 0xFF for b in data[mid:mid + 40]) + data[mid + 40:]
|
||||
ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", data) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
path = "/api/v1/chats/%s/attachments?filename=" % chat["id"]
|
||||
|
||||
def up(body, ctype, expected, name="a.png", **kw):
|
||||
return c.call("POST", path + quote(name, safe=""), expected, actor=alice, raw=body,
|
||||
headers={"Content-Type": ctype}, schema="Attachment" if expected == 201 else None)
|
||||
|
||||
up(SAMPLE[:33], "image/png", 415)
|
||||
up(png(1, 1, filter_byte=5), "image/png", 415)
|
||||
up(png(1, 1, filter_byte=0xFF), "image/png", 415)
|
||||
up(png(64, 64, corrupt_idat=True), "image/png", 415)
|
||||
up(JPEG_STUB, "image/jpeg", 415)
|
||||
print("ok truncated PNG, bad filter byte, broken IDAT, 27-byte JPEG stub -> 415")
|
||||
real = subprocess.run(
|
||||
["docker", "run", "--rm", "--entrypoint", "python", "mtusi-messenger:lab4", "-c",
|
||||
"import io,os,sys;from PIL import Image;b=io.BytesIO();"
|
||||
"Image.frombytes('RGB',(1000,750),os.urandom(2250000)).save(b,'JPEG',quality=90);"
|
||||
"sys.stdout.buffer.write(b.getvalue())"], capture_output=True, check=True).stdout
|
||||
whole = up(real, "image/jpeg", 201, name="real.jpg")["json"]
|
||||
check(c.call("GET", "/api/v1/attachments/" + whole["id"] + "/content", actor=bob)["bytes"] == real, "JPEG bytes differ")
|
||||
up(real[: len(real) // 2] + b"\xff\xd9", "image/jpeg", 415, name="cut.jpg")
|
||||
up(real[: len(real) // 2], "image/jpeg", 415, name="cut.jpg")
|
||||
print("ok real JPEG accepted; first half + EOI and first half alone -> 415")
|
||||
up(png(3000, 2000, filter_byte=7), "image/png", 415)
|
||||
up(png(3000, 2000, corrupt_idat=True, noise=True), "image/png", 415)
|
||||
print("ok same defects in images above the full-decode threshold (6 Mpx) -> 415")
|
||||
up(png(3000, 2000), "image/png", 201)
|
||||
print("ok valid 6 Mpx PNG accepted")
|
||||
import time
|
||||
big = png(6000, 6000, level=9)
|
||||
started = time.monotonic()
|
||||
r = up(big, "image/png", 201, name="36mpx.png")
|
||||
took = time.monotonic() - started
|
||||
check(took < 15, "36 Mpx PNG must be checked quickly, took %.1fs" % took)
|
||||
print("ok valid 36 Mpx PNG (%d KiB on the wire, 108 MiB decoded) accepted in %.1fs" % (len(big) // 1024, took))
|
||||
up(png(6000, 6000, filter_byte=9, level=9), "image/png", 415)
|
||||
print("ok 36 Mpx PNG with bad filter byte -> 415")
|
||||
|
||||
for name in (" ", "\t", "bad\x7f.png", "bad\x85.png", ".", "..", "a
b.png"):
|
||||
up(SAMPLE, "image/png", 400, name=name)
|
||||
print("ok whitespace-only, DEL, C1 and dot filenames -> 400")
|
||||
|
||||
a = up(SAMPLE, "image/png", 201, name="a.png")["json"]
|
||||
b = up(SAMPLE, "image/png", 201, name="b.png")["json"]
|
||||
url = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
key = uuid.uuid4().hex
|
||||
first = c.call("POST", url, 201, actor=alice, body={"text": "", "attachment_ids": [b["id"], a["id"]]},
|
||||
headers={"Idempotency-Key": key}, schema="Message")["json"]
|
||||
check(first["attachment_ids"] == [b["id"], a["id"]], "POST keeps attachment order")
|
||||
replay = c.call("POST", url, 201, actor=alice, body={"attachment_ids": [b["id"], a["id"]], "text": ""},
|
||||
headers={"Idempotency-Key": key}, schema="Message")["json"]
|
||||
check(replay == first, "replay returns the same message")
|
||||
c.call("POST", url, 409, actor=alice, body={"text": "", "attachment_ids": [a["id"], b["id"]]},
|
||||
headers={"Idempotency-Key": key})
|
||||
history = c.call("GET", url, actor=bob, schema="MessagePage")["json"]["items"]
|
||||
check([m for m in history if m["id"] == first["id"]][0]["attachment_ids"] == [b["id"], a["id"]],
|
||||
"history keeps attachment order")
|
||||
print("ok attachment order preserved in POST, replay and history; reordered body -> 409")
|
||||
|
||||
key2 = uuid.uuid4().hex
|
||||
plain = c.call("POST", url, 201, actor=alice, body={"text": "x"}, headers={"Idempotency-Key": key2}, schema="Message")["json"]
|
||||
same = c.call("POST", url, 201, actor=alice, body={"text": "x", "attachment_ids": []},
|
||||
headers={"Idempotency-Key": key2}, schema="Message")["json"]
|
||||
check(plain == same, "absent attachment_ids equals []")
|
||||
for bad in ("a b", "a/b", "k" * 129, ""):
|
||||
c.call("POST", url, 400, actor=alice, body={"text": "k"}, headers={"Idempotency-Key": bad})
|
||||
other = c.call("POST", url, 201, actor=bob, body={"text": "x"}, headers={"Idempotency-Key": key2}, schema="Message")["json"]
|
||||
check(other["id"] != plain["id"] and other["sender_id"] == bob["id"], "idempotency key is per user")
|
||||
print("ok idempotency: [] == absent, invalid keys -> 400, scoped per user")
|
||||
|
||||
huge = b"\x89PNG\r\n\x1a\n" + b"\0" * (12 * 1024 * 1024)
|
||||
r = c.call("POST", path + "big.png", 413, actor=alice, raw=huge, headers={"Content-Type": "image/png"})
|
||||
check(r["headers"].get_content_type() == "application/json", "413 above the nginx limit is API-shaped JSON")
|
||||
print("ok 12 MiB body -> JSON 413")
|
||||
print("PASS: input hardening")
|
||||
|
||||
|
||||
run(main)
|
||||
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""LB under upload pressure: no OOM/restart of any container, no HTML 502, every rejection is API-shaped JSON.
|
||||
python3 student-tests/test_lb_stress.py --concurrency 10 --rounds 3 --oversize 200"""
|
||||
import argparse
|
||||
import base64
|
||||
import collections
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
import zlib
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
HOST, PORT = "localhost", 8080
|
||||
LIMIT = 10485760
|
||||
|
||||
|
||||
def noise_png(side):
|
||||
raw = b"".join(b"\0" + os.urandom(side * 3) for _ in range(side))
|
||||
|
||||
def chunk(kind, body):
|
||||
return struct.pack(">I", len(body)) + kind + body + struct.pack(">I", zlib.crc32(kind + body) & 0xFFFFFFFF)
|
||||
|
||||
ihdr = struct.pack(">IIBBBBB", side, side, 8, 2, 0, 0, 0)
|
||||
return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", zlib.compress(raw, 1)) + chunk(b"IEND", b"")
|
||||
|
||||
|
||||
def request(method, path, body=None, headers=None, timeout=90):
|
||||
"""Returns (status, content_type, payload) or ('EXC:<name>', '', b'')."""
|
||||
conn = http.client.HTTPConnection(HOST, PORT, timeout=timeout)
|
||||
try:
|
||||
conn.request(method, path, body=body, headers=headers or {})
|
||||
response = conn.getresponse()
|
||||
return response.status, response.getheader("Content-Type", ""), response.read()
|
||||
except Exception as exc:
|
||||
return "EXC:" + type(exc).__name__, "", b""
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def actor(label):
|
||||
username = "st_%s_%s" % (label, uuid.uuid4().hex[:10])
|
||||
password = "T3st-" + uuid.uuid4().hex
|
||||
body = {"username": username, "display_name": label, "password": password}
|
||||
status, _, _ = request("POST", "/api/v1/users", json.dumps(body), {"Content-Type": "application/json"})
|
||||
assert status == 201, status
|
||||
basic = base64.b64encode((username + ":" + password).encode()).decode()
|
||||
status, _, data = request("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + basic})
|
||||
assert status == 201, status
|
||||
session = json.loads(data)
|
||||
return session["user"]["id"], {"Authorization": "Bearer " + session["token"]}
|
||||
|
||||
|
||||
def container_ids():
|
||||
out = subprocess.run(["docker", "compose", "ps", "-q"], cwd=ROOT, capture_output=True, text=True).stdout.split()
|
||||
names = {}
|
||||
for cid in out:
|
||||
name = subprocess.run(["docker", "inspect", "--format", "{{.Name}}", cid], capture_output=True, text=True).stdout
|
||||
names[cid] = name.strip().lstrip("/")
|
||||
return names
|
||||
|
||||
|
||||
def peak_mib(name):
|
||||
out = subprocess.run(["docker", "exec", name, "cat", "/sys/fs/cgroup/memory.peak"], capture_output=True, text=True)
|
||||
return round(int(out.stdout) / 1048576, 1) if out.stdout.strip().isdigit() else None
|
||||
|
||||
|
||||
class Events(threading.Thread):
|
||||
def __init__(self):
|
||||
super().__init__(daemon=True)
|
||||
self.lines = []
|
||||
self.proc = subprocess.Popen(
|
||||
["docker", "events", "--filter", "type=container", "--filter", "event=oom", "--filter", "event=die",
|
||||
"--filter", "event=restart", "--format", "{{.Action}} {{.Actor.Attributes.name}} exit={{.Actor.Attributes.exitCode}}"],
|
||||
stdout=subprocess.PIPE, text=True,
|
||||
)
|
||||
|
||||
def run(self):
|
||||
for line in self.proc.stdout:
|
||||
self.lines.append(line.strip())
|
||||
|
||||
def stop(self):
|
||||
self.proc.terminate()
|
||||
|
||||
|
||||
def is_json_error(result, code):
|
||||
status, ctype, data = result
|
||||
if status != code or not ctype.startswith("application/json"):
|
||||
return False
|
||||
return json.loads(data)["error"]["code"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--concurrency", type=int, default=10)
|
||||
parser.add_argument("--rounds", type=int, default=3)
|
||||
parser.add_argument("--oversize", type=int, default=200)
|
||||
args = parser.parse_args()
|
||||
# memory.peak is cumulative since container start, so read it after a fresh recreate.
|
||||
|
||||
failures = []
|
||||
events = Events()
|
||||
events.start()
|
||||
uid, auth = actor("a")
|
||||
_, outsider = actor("b")
|
||||
status, _, data = request("POST", "/api/v1/chats", json.dumps({"title": "stress", "member_ids": [uid]}),
|
||||
{**auth, "Content-Type": "application/json"})
|
||||
assert status == 201, status
|
||||
chat = json.loads(data)["id"]
|
||||
upload = "/api/v1/chats/%s/attachments?filename=s.png" % chat
|
||||
png_headers = {**auth, "Content-Type": "image/png"}
|
||||
|
||||
body = noise_png(1700)
|
||||
print("== %d parallel uploads of %d bytes, %d rounds ==" % (args.concurrency, len(body), args.rounds))
|
||||
ids, tally = [], collections.Counter()
|
||||
for _ in range(args.rounds):
|
||||
with ThreadPoolExecutor(args.concurrency) as pool:
|
||||
results = list(pool.map(lambda _: request("POST", upload, body, png_headers), range(args.concurrency)))
|
||||
for status, _, data in results:
|
||||
tally[status] += 1
|
||||
if status == 201:
|
||||
ids.append(json.loads(data)["id"])
|
||||
print("upload statuses:", dict(tally))
|
||||
if set(tally) != {201}:
|
||||
failures.append("parallel uploads: %s" % dict(tally))
|
||||
|
||||
print("== %d parallel downloads ==" % args.concurrency)
|
||||
with ThreadPoolExecutor(args.concurrency) as pool:
|
||||
results = list(pool.map(lambda i: request("GET", "/api/v1/attachments/%s/content" % i, headers=auth),
|
||||
ids[:args.concurrency * 2]))
|
||||
tally = collections.Counter(r[0] for r in results)
|
||||
print("download statuses:", dict(tally))
|
||||
if set(tally) != {200} or any(r[2] != body for r in results):
|
||||
failures.append("parallel downloads: %s" % dict(tally))
|
||||
|
||||
print("== %d parallel valid 40 Mpx progressive CMYK JPEGs (worst case for the checker), %d rounds ==" % (args.concurrency, args.rounds))
|
||||
jpeg = subprocess.run(
|
||||
["docker", "run", "--rm", "--entrypoint", "python", "mtusi-messenger:lab4", "-c",
|
||||
"import io,sys;from PIL import Image;b=io.BytesIO();"
|
||||
"Image.new('CMYK',(8000,5000),(120,30,200,5)).save(b,'JPEG',quality=90,progressive=True);"
|
||||
"sys.stdout.buffer.write(b.getvalue())"], capture_output=True, check=True).stdout
|
||||
tally = collections.Counter()
|
||||
for _ in range(args.rounds):
|
||||
with ThreadPoolExecutor(args.concurrency) as pool:
|
||||
results = list(pool.map(lambda _: request("POST", upload, jpeg, {**auth, "Content-Type": "image/jpeg"}),
|
||||
range(args.concurrency)))
|
||||
tally.update(r[0] for r in results)
|
||||
print("upload statuses:", dict(tally))
|
||||
if set(tally) != {201}:
|
||||
failures.append("parallel progressive JPEG: %s" % dict(tally))
|
||||
|
||||
cases = [
|
||||
("10 MiB + 1 byte, Content-Length", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (LIMIT - 7), png_headers, 413, "payload_too_large"),
|
||||
("10 MiB + 1 byte, chunked", upload, iter([b"\x89PNG\r\n\x1a\n" + b"\0" * (LIMIT - 7)]), png_headers, 413, "payload_too_large"),
|
||||
("10 MiB + 1 byte, chunked in 64 KiB pieces", upload,
|
||||
[b"\x89PNG\r\n\x1a\n"] + [b"\0" * 65536] * 160, png_headers, 413, "payload_too_large"),
|
||||
("8 MiB to a foreign chat", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (8 << 20), {**outsider, "Content-Type": "image/png"}, 404, "not_found"),
|
||||
("8 MiB without session", upload, b"\x89PNG\r\n\x1a\n" + b"\0" * (8 << 20), {"Content-Type": "image/png"}, 401, "unauthorized"),
|
||||
("8 MiB with wrong magic", upload, b"X" * (8 << 20), png_headers, 415, "unsupported_media_type"),
|
||||
("8 MiB with wrong Content-Type", upload, b"X" * (8 << 20), {**auth, "Content-Type": "image/gif"}, 415, "unsupported_media_type"),
|
||||
("70 KiB JSON message", "/api/v1/chats/%s/messages" % chat, b'{"text":"' + b"x" * (70 << 10) + b'"}',
|
||||
{**auth, "Content-Type": "application/json", "Idempotency-Key": "k"}, 413, "payload_too_large"),
|
||||
]
|
||||
for name, path, payload, headers, code, error in cases:
|
||||
print("== %s: %d requests, %d at a time ==" % (name, args.oversize, args.concurrency))
|
||||
materialized = list(payload) if not isinstance(payload, bytes) else None
|
||||
|
||||
def one(_):
|
||||
data = payload if materialized is None else iter(materialized)
|
||||
extra = {} if materialized is None else {"Transfer-Encoding": "chunked"}
|
||||
conn_headers = {**headers, **extra}
|
||||
conn = http.client.HTTPConnection(HOST, PORT, timeout=90)
|
||||
try:
|
||||
conn.request("POST", path, body=data, headers=conn_headers, encode_chunked=materialized is not None)
|
||||
response = conn.getresponse()
|
||||
return response.status, response.getheader("Content-Type", ""), response.read()
|
||||
except Exception as exc:
|
||||
return "EXC:" + type(exc).__name__, "", b""
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
with ThreadPoolExecutor(args.concurrency) as pool:
|
||||
results = list(pool.map(one, range(args.oversize)))
|
||||
tally = collections.Counter((r[0], r[1].split(";")[0]) for r in results)
|
||||
print(" ", {"%s %s" % k: v for k, v in tally.items()})
|
||||
if not all(is_json_error(r, code) == error for r in results):
|
||||
failures.append("%s: %s" % (name, dict(tally)))
|
||||
|
||||
status, _, _ = request("GET", "/health/ready")
|
||||
if status != 200:
|
||||
failures.append("not ready after the run: %s" % status)
|
||||
events.stop()
|
||||
bad = [line for line in events.lines if " mtusi-lab4-" in line]
|
||||
print("container events during the run:", bad or "none")
|
||||
if bad:
|
||||
failures.append("container events: %s" % bad)
|
||||
print("cgroup memory.peak, MiB:", {n: peak_mib(n) for n in container_ids().values() if "api-" in n or "nginx" in n})
|
||||
for failure in failures:
|
||||
print("FAIL:", failure)
|
||||
print("PASS: LB stress" if not failures else "FAILED: %d" % len(failures))
|
||||
sys.exit(1 if failures else 0)
|
||||
|
||||
|
||||
main()
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""S3 down: upload/download answer 503 (never 500), no partial rows are exposed, service recovers."""
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, check, run # noqa: E402
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, capture_output=True)
|
||||
|
||||
|
||||
def main():
|
||||
c = Client()
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
att, content = c.attachment(alice, chat)
|
||||
path = "/api/v1/chats/%s/attachments?filename=x.png" % chat["id"]
|
||||
compose("stop", "-t", "2", "minio")
|
||||
try:
|
||||
time.sleep(4)
|
||||
c.call("POST", path, 503, actor=alice, raw=content, headers={"Content-Type": "image/png"})
|
||||
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 503, actor=bob)
|
||||
c.call("GET", "/health/ready", 503)
|
||||
c.call("GET", "/health/ready", 503)
|
||||
c.message(alice, chat, "текст без S3")
|
||||
c.call("GET", "/health/live", schema="Health")
|
||||
c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")
|
||||
c.call("GET", "/api/v1/chats/%s/messages" % chat["id"], actor=bob, schema="MessagePage")
|
||||
print("ok S3 down: upload/download/ready -> 503 JSON, metadata and text messages still served through the LB (no 502)")
|
||||
finally:
|
||||
compose("start", "minio")
|
||||
got = None
|
||||
started = time.monotonic()
|
||||
while time.monotonic() - started < 90:
|
||||
got = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", [200, 503], actor=bob)
|
||||
if got["status"] == 200:
|
||||
break
|
||||
time.sleep(2)
|
||||
check(got["status"] == 200, "download did not recover after S3 came back")
|
||||
print("ok recovered %.0fs after S3 start" % (time.monotonic() - started))
|
||||
check(got["bytes"] == content, "original intact after S3 outage")
|
||||
streak = 0
|
||||
for _ in range(60):
|
||||
streak = streak + 1 if c.call("GET", "/health/ready", [200, 503])["status"] == 200 else 0
|
||||
if streak >= 6:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
check(streak >= 6, "both replicas must report ready again")
|
||||
print("ok ready is 200 again")
|
||||
|
||||
compose("pause", "minio")
|
||||
try:
|
||||
started = time.monotonic()
|
||||
c.timeout = 40
|
||||
c.call("POST", path, 503, actor=alice, raw=content, headers={"Content-Type": "image/png"})
|
||||
took = time.monotonic() - started
|
||||
check(took < 20, "upload to a hung S3 must fail fast, took %.1fs" % took)
|
||||
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 503, actor=bob)
|
||||
print("ok hung S3: upload answered 503 in %.1fs" % took)
|
||||
finally:
|
||||
compose("unpause", "minio")
|
||||
time.sleep(3)
|
||||
c.call("POST", path, 201, actor=alice, raw=content, headers={"Content-Type": "image/png"}, schema="Attachment")
|
||||
print("PASS: S3 outage behaviour")
|
||||
|
||||
|
||||
run(main)
|
||||
Reference in New Issue
Block a user