diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..1ee77dce --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +*.sh text eol=lf +Dockerfile text eol=lf +*.conf text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..ca8b4b92 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,18 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Казанцева Виктория / БПИ2301 +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: трек 2, согласованность PostgreSQL и S3 -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: минимум lab3 (пользователи, Argon2id-сессии в PostgreSQL, чаты, keyset-история, Idempotency-Key, две API-реплики за HAProxy) плюс синхронные PNG/JPEG-вложения. Метаданные в PostgreSQL, байты в приватном MinIO. Есть компенсация неудачного commit и reconciler с grace 120 с, dry-run и повторным запуском. +Команды запуска из чистого clone: `python scripts/init_env.py`, затем `docker compose --env-file .env up -d --build`. На Windows для публичных тестов нужен `PYTHONUTF8=1`. Подробности и версии образов — в REPORT.md. +Результаты проверок и ссылка на REPORT.md: smoke, persistence и failover PASS; негативные вложения, гонка идемпотентности, приватный bucket, память, reconciler и отказ S3 PASS. См. REPORT.md и `evidence/`. +Ограничения / известные ошибки: полный decode только до 1 млн пикселей; reconciler запускается оператором и делает полный обход bucket; один Docker-host не переживает потерю машины. +Источники: документация S3/MinIO и контракт репозитория; FastAPI, asyncpg, argon2-cffi, boto3, Pillow. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Проверки из make check и make test выполнены напрямую: на машине нет make. Реальный вывод — в REPORT.md. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/.gitignore b/.gitignore index b5a71896..d5cff84a 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,6 @@ coverage/ .DS_Store .idea/ .vscode/ +scripts/*.exe # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..39da85c1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# mc is copied from the pinned MinIO image already used by the stand (static binary). +FROM minio/minio:RELEASE.2025-09-07T16-13-09Z AS mc + +FROM python:3.12-alpine@sha256:d09d15e60962ca365d1cd544a48773bac9d33f2fb1b00f2aa0deec78ade7dc31 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +COPY --from=mc /usr/bin/mc /usr/local/bin/mc + +RUN adduser -D -u 10001 -h /home/app app \ + && mkdir -p /tmp/uploads /tmp/mc \ + && chown app:app /tmp/uploads /tmp/mc + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app +COPY infra/migrate.sql ./infra/migrate.sql +COPY infra/bootstrap-s3.sh ./infra/bootstrap-s3.sh + +RUN chmod 755 /usr/local/bin/mc /app/infra/bootstrap-s3.sh \ + && chown -R app:app /app + +USER app +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=4s --start-period=25s --retries=20 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "2"] diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..dab14abd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,107 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Казанцева Виктория / БПИ2301 +- Целевая оценка; выбранный трек 5: **5, трек 2 — согласованность PostgreSQL и S3** +- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab4/pulls/88 +- Источники и библиотеки: контракт репозитория, документация S3 и MinIO; FastAPI, uvicorn, asyncpg, argon2-cffi (Argon2id), boto3, Pillow. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Машина проверки: Windows, Docker Desktop 29.2.1, CPU Intel i7-12650H, 16 ГБ RAM. Лимит каждой API-реплики — 768 МиБ. Публичные тесты на Windows нужно запускать с `PYTHONUTF8=1`: `tests/client.py` читает OpenAPI через `Path.read_text()` без кодировки. + +Образы, все с фиксированной версией или digest: + +| Роль | Образ | Digest | +| --- | --- | --- | +| API | `python:3.12-alpine@sha256:d09d15e60962ca365d1cd544a48773bac9d33f2fb1b00f2aa0deec78ade7dc31` | Python 3.12.14 внутри контейнера | +| PostgreSQL | `postgres:16.8-bookworm` | `sha256:301bcb60b8a3ee4ab7e147932723e3abd1cef53516ce5210b39fd9fe5e3602ae` | +| S3 | `minio/minio:RELEASE.2025-09-07T16-13-09Z` | `sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e` | +| Клиент bootstrap | `mc` `RELEASE.2025-08-13T08-35-41Z`, скопирован из образа MinIO | тот же digest MinIO | +| Балансировщик | `haproxy:2.8.14-alpine` | `sha256:72030a2e37d5c5790e13387da24f890bf860b8dfef10fab1b1fd0fd2e974ff75` | + +Порты MinIO, включая консоль `:9001`, наружу не публикуются. Снаружи только `127.0.0.1:8080` балансировщика. + +```sh +python scripts/init_env.py +docker compose --env-file .env up -d --build +``` + +`scripts/init_env.py` создаёт локальный `.env` и не перезаписывает существующий. Секреты в отчёт и git не входят. Схема создаётся при старте API из `infra/migrate.sql` под advisory lock. Bucket и пользователь API создаёт одноразовый `s3-init` (`infra/bootstrap-s3.sh`); повторный запуск идемпотентен. + +Дождаться `GET /health/ready` → `200 {"status":"ok"}`. Затем: + +```sh +python tests/smoke.py --base-url http://127.0.0.1:8080 +python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +``` + +На Windows тот же сценарий выполняет `scripts/restart.exe` и `scripts/stop-one.exe` (исходник `scripts/labhook.cs`): команды совпадают с `.sh`, потому что `subprocess` не запускает shell-скрипт. После failover остановленную реплику возвращают отдельно: `docker compose --env-file .env start api-1 api-2`. + +Остановка без удаления данных: `docker compose --env-file .env down`. Учебные volume удаляет только явная команда `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy (`lb`, round-robin, проверка `/health/live`) → `api-1` и `api-2` → PostgreSQL и приватный bucket MinIO. + +Из lab3 сохранены регистрация, Argon2id-сессии в PostgreSQL, чаты, keyset-история, `Idempotency-Key` с advisory lock на пользователя+метод+путь+ключ и заголовок `X-Instance-Id` (`api-1` / `api-2`). Сессии и сообщения переживают рестарт, TTL сессии рестарт не продлевает. + +Вложение: байты лежат в S3 по ключу `o/`, который назначает сервер. `filename` только отображаемое имя. Метаданные, связь с чатом и ссылки из сообщений — в PostgreSQL. У API нет volume. `201` возвращается после успешного `PUT` и `COMMIT` строки `attachments` вместе с переводом ledger в `committed`. Если `PUT` прошёл, а commit нет, процесс удаляет объект и помечает ledger `aborted`. Если удаление тоже не удалось, объект остаётся `staging`: reconciler заберёт его только после grace period 120 секунд. Свежий staging не трогается, потому что загрузка может ещё идти. + +Переменные `.env.example` отображаются так: `DATABASE_URL` собирается в Compose из `POSTGRES_*`; `S3_ENDPOINT=http://minio:9000` только во внутренней сети; ключ API — `S3_ACCESS_KEY_ID` / `S3_SECRET_ACCESS_KEY`. Root MinIO есть у `minio` и `s3-init`, у процесса API его нет. Политика пользователя API: `GetObject`, `PutObject`, `DeleteObject` на объекты bucket и `ListBucket`/`GetBucketLocation` на сам bucket. + +Пароль: Argon2id, time_cost 2, memory 19 МиБ, parallelism 1. Для отсутствующего пользователя выполняется verify фиктивного хеша. + +Курсор истории — непрозрачная строка с chat_id. Чужой чат даёт 400. Порядок `(created_at, id)`. Снимок между запросами не обещается: `created_at` берётся из `clock_timestamp()` в момент вставки. + +Чтение изображения ограничено 10 МиБ до разбора. Число пикселей читается из заголовка PNG/JPEG и сравнивается с 40 млн до Pillow. Полный `load()` делается только до 1 млн пикселей, чтобы одна загрузка не разворачивала 40-мегапиксельный буфер. Файл на время проверки пишется во временный файл контейнера и удаляется; каноническая копия — объект S3. + +`/health/live` не смотрит зависимости. `/health/ready` требует `SELECT 1` и доступность bucket. Балансировщик проверяет live, поэтому остановленный процесс выпадает из пула, а живой процесс при недоступном S3 продолжает отвечать `503` на ready. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда | Результат | Файл | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | `PASS: lab4 public API scenario` на финальной сборке | вывод команды | +| Контракт skeleton | `python scripts/check_contract.py` | `PASS: offline skeleton structure` | вывод команды | +| JPEG, checksum, участник чата | `python tests_student/negative.py` | `201`, SHA-256 совпал, скачивание вторым участником побайтово | вывод `PASS: negative...` | +| Слишком большой файл | тот же прогон | `413 payload_too_large` | там же | +| Повреждённый PNG и подмена MIME | тот же прогон | `415 unsupported_media_type`, не 500 | там же | +| Опасные filename | тот же прогон | `400 invalid_request` | там же | +| Чужой attachment и файл другого чата | тот же прогон | `404 not_found` для метаданных, байтов и `attachment_ids` | там же | +| Пиксельная бомба 20000×20000 | тот же прогон | `413` до декодирования | там же | +| Параллельный idempotency | `python tests_student/idempotency_race.py` | 8 повторов, один id в истории | `PASS: eight parallel...` | +| Приватный bucket | `python tests_student/private_bucket.py` | анонимные GET и List → 403; порты MinIO не опубликованы; у API нет mounts и нет `MINIO_ROOT_USER` | вывод команды | +| 10 одновременных загрузок | `python scripts/memory_experiment.py` | 10/10 для 12420 байт и для 3241218 байт | `evidence/memory.json` | +| Reconciler, трек 2 | `python scripts/reconcile_experiment.py` | dry-run 2 действия, повторный dry-run 0; свежий orphan и in-use metadata сохранены | `evidence/reconcile.json` | +| Недоступность S3 | `python scripts/s3_outage.py` | live 200, ready 503, после `start` тот же SHA-256 | `evidence/s3_outage.json` | +| Рестарт с volume | `tests/resilience.py --mode persistence --action-script scripts/restart.exe` | `PASS: persistence preserved confirmed data and existing session` | вывод команды | +| Отказ одной реплики | `tests/resilience.py --mode failover --action-script scripts/stop-one.exe` | до отказа 2 реплики; 106 успешных записей, 1 ошибка; первое восстановление 2.390 с; `PASS` | вывод команды | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Память. Гипотеза: десять загрузок не занимают неограниченную RAM, потому что тело режется лимитом 10 МиБ, пишется во временный файл чанками и не декодируется в bitmap, если пикселей больше 1 млн. Волна 12 КиБ заняла около 0.37 с, пик `docker stats` 56.2 и 57.2 МиБ, VmHWM около 73 МиБ и за волну не вырос. Волна 3.24 МиБ × 10 заняла 0.74 с. Пик `docker stats` 65.2 МиБ на `api-1` и 53.6 МиБ на `api-2`. VmHWM вырос с 74892 до 77184 КиБ и с 74776 до 76492 КиБ, то есть примерно на 2 МиБ на процесс, а не на 10 × 3.24 МиБ. Ограничение измерения: волна короче секунды, `docker stats` снимался каждые 0.25 с и включает page cache; VmHWM — честный максимум RSS процесса после рестарта перед волной. + +Трек 2. Гипотеза: полный обход с grace 120 с удаляет только просроченный orphan и метаданные без объекта, на которые не ссылается сообщение; повторный запуск ничего не удаляет. Методика: через API сохранены свежее вложение в сообщении и свежее вложение без сообщения; `app.admin inject` добавил свежий staging-объект, staging возрастом 2 часа, строку без объекта возрастом 2 часа и такую же строку, на которую ссылается сообщение. Dry-run увидел 27 объектов за 1 страницу ListObjects, 129.4 мс, из них 127.2 мс — список S3 и 2.0 мс — запросы к PostgreSQL. Действия: удалить `old-orphan` и `fault-stale-missing`. Пропущены `fresh_orphan` и `in_use_missing`. Apply повторил те же два действия за 154.3 мс. Второй dry-run: `action_count=0`, объектов 26. Скачивание свежего вложения из сообщения совпало с fixture. Ограничения: это полный обход одного bucket, не инкрементальный; возраст orphan считается по ledger, потому что LastModified MinIO из лаборатории не переписывается; пропавшие байты используемого вложения не восстанавливаются, строка остаётся, content отвечает 404; reconciler запускается оператором, а не постоянным циклом. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдали | Восстановление | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Рестарт API, PostgreSQL, MinIO и LB с сохранением volume | прежний токен, сообщение и байты вложения | PASS публичного persistence-теста | стек снова `ready`, данные на месте | +| `stop` реплики, обслужившей пробный POST | запись продолжается через вторую реплику, подтверждённые сообщения без дублей | 2 реплики до отказа, 106 успехов, 1 ошибка, восстановление 2.390 с, остановленная реплика не в последних ответах | реплика остаётся остановленной, пока её явно не запустить | +| Остановка MinIO | live 200, ready 503, после старта те же байты | именно так | оригиналы в volume MinIO | + +Один Docker-host не переживает потерю этой машины. Один PostgreSQL и один MinIO не являются отказоустойчивой парой. Эксперимент показывает потерю одной API-реплики и краткую недоступность S3, не потерю диска. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Пользовательские ошибки загрузки остаются 400/404/413/415. Полный decode сознательно ограничен 1 млн пикселей: файл до 40 млн пикселей принимается после заголовка и `verify()`, но редкое повреждение большого файла может быть не поймано так же строго, как у маленького. Grace 120 с защищает незавершённую загрузку и одновременно откладывает уборку мусора на этот срок. Секреты живут в локальном `.env`, не в образе. + +## Ответы, которые пригодятся на защите + +Файл в volume одной API-реплики не равен объектному хранилищу: у второй реплики этого диска нет, рестарт контейнера без общего volume его теряет, а bucket с собственным volume читают обе реплики. + +`201` честно отдаётся только когда объект уже записан в S3 и строка метаданных закоммичена. До commit клиент получает ошибку и может повторить загрузку отдельным запросом: у загрузки нет Idempotency-Key. + +Если S3 PUT прошёл, а COMMIT нет, ответ не 201. Компенсация удаляет объект. Если удалить не удалось, reconciler после 120 с видит staging без строки `attachments` и удаляет объект. Повторный dry-run после apply пустой. + +Случайный ключ и закрытый bucket не заменяют ACL: любой авторизованный пользователь, не состоящий в чате, должен получать 404 и на метаданные, и на байты. Проверка членства делается в API. Анонимный GET в MinIO и так отвечает 403, это отдельная граница. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..293c3dff --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Messenger API for laboratory 4.""" diff --git a/app/admin.py b/app/admin.py new file mode 100644 index 00000000..3f7504f2 --- /dev/null +++ b/app/admin.py @@ -0,0 +1,226 @@ +"""Operator tools. Not part of the public HTTP API.""" + +import argparse +import asyncio +import hashlib +import json +import sys +import urllib.error +import urllib.request +import uuid + +from app.config import load_settings +from app.db import close, connect, migrate, pool +from app.reconcile import reconcile +from app import s3client + + +async def _prepare(settings): + await connect(settings) + await migrate(settings) + + +async def _inject(settings, chat_id: str, owner_id: str) -> dict: + if settings.app_env != "lab": + raise SystemExit("inject is allowed only when APP_ENV=lab") + chat_uuid = uuid.UUID(chat_id) + owner_uuid = uuid.UUID(owner_id) + member = await pool().fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + chat_uuid, + owner_uuid, + ) + if member is None: + raise SystemExit("owner is not a member of the chat") + + def staged(note: str) -> tuple[str, str, bytes]: + attachment_id = str(uuid.uuid4()) + key = f"o/{attachment_id}" + payload = f"fault-{note}-{attachment_id}".encode("utf-8") + return attachment_id, key, payload + + fresh_id, fresh_key, fresh_body = staged("fresh-orphan") + old_id, old_key, old_body = staged("old-orphan") + stale_id, stale_key, stale_body = staged("stale-missing") + used_id, used_key, used_body = staged("in-use-missing") + + for key, body in ( + (fresh_key, fresh_body), + (old_key, old_body), + (stale_key, stale_body), + (used_key, used_body), + ): + s3client.put_bytes(settings, key, body) + + async with pool().acquire() as conn: + async with conn.transaction(): + for attachment_id, key, body, note in ( + (fresh_id, fresh_key, fresh_body, "fresh-orphan"), + (old_id, old_key, old_body, "old-orphan"), + ): + await conn.execute( + """ + INSERT INTO object_ledger ( + object_key, attachment_id, chat_id, state, note, size_bytes, created_at, updated_at + ) + VALUES ($1, $2, $3, 'staging', $4, $5, clock_timestamp(), clock_timestamp()) + """, + key, + uuid.UUID(attachment_id), + chat_uuid, + note, + len(body), + ) + await conn.execute( + """ + UPDATE object_ledger + SET created_at = clock_timestamp() - interval '2 hours', + updated_at = clock_timestamp() - interval '2 hours' + WHERE object_key = $1 + """, + old_key, + ) + for attachment_id, key, body, filename in ( + (stale_id, stale_key, stale_body, "fault-stale-missing"), + (used_id, used_key, used_body, "fault-in-use-missing"), + ): + digest = hashlib.sha256(body).hexdigest() + await conn.execute( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + status, object_key, error_code, created_at, updated_at + ) + VALUES ( + $1, $2, $3, $4, 'image/png', $5, $6, 'ready', $7, NULL, + clock_timestamp() - interval '2 hours', + clock_timestamp() - interval '2 hours' + ) + """, + uuid.UUID(attachment_id), + chat_uuid, + owner_uuid, + filename, + len(body), + digest, + key, + ) + await conn.execute( + """ + INSERT INTO object_ledger ( + object_key, attachment_id, chat_id, state, note, size_bytes, created_at, updated_at + ) + VALUES ( + $1, $2, $3, 'committed', $4, $5, + clock_timestamp() - interval '2 hours', + clock_timestamp() - interval '2 hours' + ) + """, + key, + uuid.UUID(attachment_id), + chat_uuid, + filename, + len(body), + ) + message_id = uuid.uuid4() + await conn.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, clock_timestamp() - interval '2 hours') + """, + message_id, + chat_uuid, + owner_uuid, + "fault-in-use", + ) + await conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES ($1, $2, 0) + """, + message_id, + uuid.UUID(used_id), + ) + + s3client.delete_object(settings, stale_key) + s3client.delete_object(settings, used_key) + return { + "fresh_orphan": {"attachment_id": fresh_id, "object_key": fresh_key, "note": "fresh-orphan"}, + "old_orphan": {"attachment_id": old_id, "object_key": old_key, "note": "old-orphan"}, + "stale_missing": {"attachment_id": stale_id, "object_key": stale_key, "note": "fault-stale-missing"}, + "in_use_missing": {"attachment_id": used_id, "object_key": used_key, "note": "fault-in-use-missing"}, + } + + +async def _probe(settings, attachment_id: str | None, object_key: str | None) -> dict: + result = {} + if attachment_id: + row = await pool().fetchval("SELECT 1 FROM attachments WHERE id = $1::uuid", uuid.UUID(attachment_id)) + result["metadata"] = row is not None + if object_key: + result["object"] = s3client.object_exists(settings, object_key) + return result + + +def _anonymous_status(url: str) -> int: + request = urllib.request.Request(url, method="GET") + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + +async def _check_private(settings) -> dict: + endpoint = settings.s3_endpoint.rstrip("/") + object_status = _anonymous_status(f"{endpoint}/{settings.s3_bucket}/o/private-probe-missing") + list_status = _anonymous_status(f"{endpoint}/{settings.s3_bucket}?list-type=2") + return { + "anonymous_object_status": object_status, + "anonymous_list_status": list_status, + "private": object_status == 403 and list_status == 403, + } + + +async def _run(args) -> dict: + settings = load_settings() + await _prepare(settings) + try: + if args.cmd == "reconcile": + if args.apply and args.dry_run: + raise SystemExit("choose either --apply or --dry-run") + return await reconcile(settings, apply=bool(args.apply)) + if args.cmd == "inject": + return await _inject(settings, args.chat_id, args.owner_id) + if args.cmd == "probe": + return await _probe(settings, args.attachment_id, args.object_key) + if args.cmd == "check-private": + return await _check_private(settings) + raise SystemExit("unknown command") + finally: + await close() + + +def main() -> None: + parser = argparse.ArgumentParser(description="Lab operator tools") + sub = parser.add_subparsers(dest="cmd", required=True) + reconcile_parser = sub.add_parser("reconcile") + reconcile_parser.add_argument("--apply", action="store_true") + reconcile_parser.add_argument("--dry-run", action="store_true") + inject_parser = sub.add_parser("inject") + inject_parser.add_argument("--chat-id", required=True) + inject_parser.add_argument("--owner-id", required=True) + probe_parser = sub.add_parser("probe") + probe_parser.add_argument("--attachment-id") + probe_parser.add_argument("--object-key") + sub.add_parser("check-private") + args = parser.parse_args() + if args.cmd == "reconcile" and not args.apply and not args.dry_run: + args.dry_run = True + report = asyncio.run(_run(args)) + json.dump(report, sys.stdout, ensure_ascii=False) + sys.stdout.write("\n") + + +if __name__ == "__main__": + main() diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..43fa4110 --- /dev/null +++ b/app/config.py @@ -0,0 +1,58 @@ +"""Process configuration. Secrets are read from the environment, never logged.""" + +import os +from dataclasses import dataclass + + +def _required(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise SystemExit(f"missing required environment variable {name}") + return value + + +def _int(name: str, default: int) -> int: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + try: + return int(raw) + except ValueError as exc: + raise SystemExit(f"environment variable {name} must be an integer") from exc + + +@dataclass(frozen=True) +class Settings: + instance_id: str + database_url: str + s3_endpoint: str + s3_region: str + s3_bucket: str + s3_access_key: str + s3_secret_key: str + session_ttl_seconds: int + idempotency_ttl_seconds: int + max_upload_bytes: int + max_image_pixels: int + orphan_grace_seconds: int + app_env: str + # Full bitmap decode is bounded so one upload cannot expand to 40e6 pixels in RAM. + full_decode_pixels: int = 1_000_000 + + +def load_settings() -> Settings: + return Settings( + instance_id=_required("INSTANCE_ID"), + database_url=_required("DATABASE_URL"), + s3_endpoint=_required("S3_ENDPOINT"), + s3_region=os.environ.get("S3_REGION", "us-east-1").strip() or "us-east-1", + s3_bucket=_required("S3_BUCKET"), + s3_access_key=_required("S3_ACCESS_KEY_ID"), + s3_secret_key=_required("S3_SECRET_ACCESS_KEY"), + session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600), + idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400), + max_upload_bytes=_int("MAX_UPLOAD_BYTES", 10_485_760), + max_image_pixels=_int("MAX_IMAGE_PIXELS", 40_000_000), + orphan_grace_seconds=_int("ORPHAN_GRACE_SECONDS", 120), + app_env=os.environ.get("APP_ENV", "lab").strip() or "lab", + ) diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..6d3f3b61 --- /dev/null +++ b/app/db.py @@ -0,0 +1,61 @@ +"""PostgreSQL pool, JSON codec, and idempotent schema migration.""" + +import json +from pathlib import Path + +import asyncpg + +from app.config import Settings + +_pool: asyncpg.Pool | None = None +_MIGRATION = Path(__file__).resolve().parents[1] / "infra" / "migrate.sql" + + +async def _init_connection(conn: asyncpg.Connection) -> None: + await conn.execute("SET TIME ZONE 'UTC'") + await conn.execute("SET statement_timeout = '30s'") + await conn.set_type_codec( + "jsonb", + encoder=json.dumps, + decoder=json.loads, + schema="pg_catalog", + format="text", + ) + + +async def connect(settings: Settings) -> asyncpg.Pool: + global _pool + if _pool is None: + _pool = await asyncpg.create_pool( + settings.database_url, + min_size=1, + max_size=12, + timeout=10, + command_timeout=30, + init=_init_connection, + ) + return _pool + + +async def migrate(settings: Settings) -> None: + pool = await connect(settings) + sql = _MIGRATION.read_text(encoding="utf-8") + statements = [part.strip() for part in sql.split(";") if part.strip()] + async with pool.acquire() as conn: + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock(874521)") + for statement in statements: + await conn.execute(statement) + + +async def close() -> None: + global _pool + if _pool is not None: + await _pool.close() + _pool = None + + +def pool() -> asyncpg.Pool: + if _pool is None: + raise RuntimeError("database pool is not connected") + return _pool diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..2cdba228 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,23 @@ +"""API errors mapped to the contract body. Details stay in server logs.""" + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +def unauthorized(scheme: str) -> ApiError: + if scheme == "basic": + challenge = 'Basic realm="messenger", charset="UTF-8"' + else: + challenge = 'Bearer realm="messenger"' + return ApiError( + 401, + "unauthorized", + "Требуется действительная аутентификация", + {"WWW-Authenticate": challenge}, + ) diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..20d112f7 --- /dev/null +++ b/app/images.py @@ -0,0 +1,92 @@ +"""Bounded PNG/JPEG checks. Pixel count is taken from headers before bitmap decode.""" + +from PIL import Image, ImageFile, UnidentifiedImageError + +from app.errors import ApiError + +ImageFile.LOAD_TRUNCATED_IMAGES = False + +_PNG = b"\x89PNG\r\n\x1a\n" +_JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} + + +def _png_size(path: str) -> tuple[int, int] | None: + with open(path, "rb") as handle: + if handle.read(8) != _PNG: + return None + length = handle.read(4) + kind = handle.read(4) + if kind != b"IHDR" or len(length) != 4 or int.from_bytes(length, "big") < 13: + return None + header = handle.read(8) + if len(header) != 8: + return None + width = int.from_bytes(header[0:4], "big") + height = int.from_bytes(header[4:8], "big") + return width, height + + +def _jpeg_size(path: str) -> tuple[int, int] | None: + with open(path, "rb") as handle: + if handle.read(2) != b"\xff\xd8": + return None + while True: + marker_byte = handle.read(1) + if marker_byte != b"\xff": + return None + while marker_byte == b"\xff": + marker_byte = handle.read(1) + if not marker_byte: + return None + marker = marker_byte[0] + if marker in {0xD8, 0xD9, 0x01} or 0xD0 <= marker <= 0xD7: + continue + length_bytes = handle.read(2) + if len(length_bytes) != 2: + return None + length = int.from_bytes(length_bytes, "big") + if length < 2: + return None + if marker in _JPEG_SOF: + payload = handle.read(length - 2) + if len(payload) < 5: + return None + height = int.from_bytes(payload[1:3], "big") + width = int.from_bytes(payload[3:5], "big") + return width, height + handle.seek(length - 2, 1) + + +def _detect(path: str) -> str | None: + with open(path, "rb") as handle: + header = handle.read(12) + if header.startswith(_PNG): + return "image/png" + if header.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + return None + + +def validate_image(path: str, declared: str, max_pixels: int, full_decode_pixels: int) -> None: + detected = _detect(path) + if detected is None or detected != declared: + raise ApiError(415, "unsupported_media_type", "Файл не является PNG или JPEG заявленного типа") + size = _png_size(path) if detected == "image/png" else _jpeg_size(path) + if size is None: + raise ApiError(415, "unsupported_media_type", "Не удалось прочитать размеры изображения") + width, height = size + if width <= 0 or height <= 0: + raise ApiError(415, "unsupported_media_type", "Некорректные размеры изображения") + if width > max_pixels or height > max_pixels or width * height > max_pixels: + raise ApiError(413, "payload_too_large", "Изображение превышает лимит пикселей") + Image.MAX_IMAGE_PIXELS = max_pixels + try: + with Image.open(path) as image: + image.verify() + if width * height <= full_decode_pixels: + with Image.open(path) as image: + image.load() + except Image.DecompressionBombError as exc: + raise ApiError(413, "payload_too_large", "Изображение превышает лимит пикселей") from exc + except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: + raise ApiError(415, "unsupported_media_type", "Повреждённое или неподдерживаемое изображение") from exc diff --git a/app/logic.py b/app/logic.py new file mode 100644 index 00000000..eac1dd8d --- /dev/null +++ b/app/logic.py @@ -0,0 +1,792 @@ +"""Users, sessions, chats, messages, and synchronous attachment upload.""" + +import base64 +import hashlib +import json +import logging +import os +import re +import secrets +import tempfile +import uuid +from datetime import datetime, timezone + +import asyncpg +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError, InvalidHashError + +from app.config import Settings +from app.db import pool +from app.errors import ApiError, unauthorized +from app.images import validate_image +from app import s3client + +logger = logging.getLogger("messenger") + +_HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +_DUMMY_HASH = _HASHER.hash("timing-only-not-a-login-secret") +_USERNAME = re.compile(r"^[a-z0-9_]{3,32}$") +_IDEMPOTENCY = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +_CONTROLS = set(range(32)) + + +def iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + else: + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def parse_iso(value: str) -> datetime: + if not isinstance(value, str) or not value.endswith("Z"): + raise ValueError("timestamp") + return datetime.fromisoformat(value[:-1] + "+00:00") + + +def parse_uuid(value: str) -> str: + try: + return str(uuid.UUID(str(value))) + except (ValueError, AttributeError, TypeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный идентификатор") from exc + + +def _code_points(value: str) -> int: + return len(value) + + +def _require_object(data, allowed: set[str], required: set[str]) -> None: + if not isinstance(data, dict): + raise ApiError(400, "invalid_request", "Ожидается JSON-объект") + extra = set(data) - allowed + if extra: + raise ApiError(400, "invalid_request", "Неизвестные поля") + missing = required - set(data) + if missing: + raise ApiError(400, "invalid_request", "Отсутствуют обязательные поля") + + +def _nonblank(value: str) -> bool: + return value.strip() != "" + + +def _lock_keys(scope: str) -> tuple[int, int]: + digest = hashlib.sha256(scope.encode("utf-8")).digest() + return int.from_bytes(digest[0:4], "big") & 0x7FFFFFFF, int.from_bytes(digest[4:8], "big") & 0x7FFFFFFF + + +def _cursor_encode(payload: dict) -> str: + raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + +def _cursor_decode(raw: str) -> dict: + if not isinstance(raw, str) or raw == "": + raise ApiError(400, "invalid_request", "Некорректный cursor") + padding = "=" * (-len(raw) % 4) + try: + data = json.loads(base64.urlsafe_b64decode(raw + padding).decode("utf-8")) + except (ValueError, UnicodeError, json.JSONDecodeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + if not isinstance(data, dict): + raise ApiError(400, "invalid_request", "Некорректный cursor") + return data + + +def user_dict(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": iso(row["created_at"]), + } + + +def message_dict(row, attachment_ids: list[str]) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": iso(row["created_at"]), + "attachment_ids": attachment_ids, + } + + +def attachment_dict(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "owner_id": str(row["owner_id"]), + "filename": row["filename"], + "content_type": row["content_type"], + "size_bytes": row["size_bytes"], + "sha256": row["sha256"], + "status": row["status"], + "variants": [], + "error_code": None, + "created_at": iso(row["created_at"]), + "updated_at": iso(row["updated_at"]), + } + + +async def _attachments_for(conn, message_ids: list) -> dict[str, list[str]]: + grouped: dict[str, list[str]] = {str(item): [] for item in message_ids} + if not message_ids: + return grouped + rows = await conn.fetch( + """ + SELECT message_id, attachment_id + FROM message_attachments + WHERE message_id = ANY($1::uuid[]) + ORDER BY message_id, position + """, + message_ids, + ) + for row in rows: + grouped.setdefault(str(row["message_id"]), []).append(str(row["attachment_id"])) + return grouped + + +async def _members_for(conn, chat_ids: list) -> dict[str, list[str]]: + grouped: dict[str, list[str]] = {str(item): [] for item in chat_ids} + if not chat_ids: + return grouped + rows = await conn.fetch( + """ + SELECT chat_id, user_id + FROM chat_members + WHERE chat_id = ANY($1::uuid[]) + ORDER BY chat_id, position + """, + chat_ids, + ) + for row in rows: + grouped.setdefault(str(row["chat_id"]), []).append(str(row["user_id"])) + return grouped + + +def chat_dict(row, member_ids: list[str]) -> dict: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": iso(row["created_at"]), + } + + +async def register_user(body: dict) -> dict: + _require_object(body, {"username", "display_name", "password"}, {"username", "display_name", "password"}) + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if not isinstance(username, str) or _USERNAME.fullmatch(username) is None: + raise ApiError(400, "invalid_request", "Некорректное имя пользователя") + if not isinstance(display_name, str) or not _nonblank(display_name) or _code_points(display_name) > 100: + raise ApiError(400, "invalid_request", "Некорректное отображаемое имя") + if not isinstance(password, str) or not (12 <= _code_points(password) <= 128) or "\x00" in password: + raise ApiError(400, "invalid_request", "Некорректный пароль") + password_hash = _HASHER.hash(password) + try: + row = await pool().fetchrow( + """ + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, clock_timestamp()) + RETURNING id, username, display_name, created_at + """, + uuid.uuid4(), + username, + display_name, + password_hash, + ) + except asyncpg.UniqueViolationError as exc: + raise ApiError(409, "conflict", "Имя пользователя занято") from exc + return user_dict(row) + + +async def create_session(username: str, password: str, settings: Settings) -> dict: + if not isinstance(username, str) or _USERNAME.fullmatch(username) is None: + raise unauthorized("basic") + if not isinstance(password, str) or "\x00" in password: + raise unauthorized("basic") + row = await pool().fetchrow( + """ + SELECT id, username, display_name, created_at, password_hash + FROM users + WHERE username = $1 + """, + username, + ) + if row is None: + try: + _HASHER.verify(_DUMMY_HASH, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + pass + raise unauthorized("basic") + try: + ok = _HASHER.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + ok = False + if not ok: + raise unauthorized("basic") + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + expires = await pool().fetchval( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES ($1, $2, clock_timestamp() + make_interval(secs => $3::int)) + RETURNING expires_at + """, + token_hash, + row["id"], + settings.session_ttl_seconds, + ) + return { + "token": token, + "token_type": "Bearer", + "expires_at": iso(expires), + "user": user_dict(row), + } + + +async def authenticate(token: str) -> dict: + if not token: + raise unauthorized("bearer") + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + row = await pool().fetchrow( + """ + SELECT u.id, u.username, u.display_name, u.created_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 + AND s.revoked_at IS NULL + AND s.expires_at > clock_timestamp() + """, + token_hash, + ) + if row is None: + raise unauthorized("bearer") + return user_dict(row) + + +async def revoke_session(token: str) -> None: + token_hash = hashlib.sha256(token.encode("utf-8")).hexdigest() + await pool().execute( + """ + UPDATE sessions + SET revoked_at = clock_timestamp() + WHERE token_hash = $1 AND revoked_at IS NULL + """, + token_hash, + ) + + +async def create_chat(user_id: str, body: dict) -> dict: + _require_object(body, {"title", "member_ids"}, {"title", "member_ids"}) + title = body["title"] + member_ids = body["member_ids"] + if not isinstance(title, str) or not _nonblank(title) or _code_points(title) > 100 or "\x00" in title: + raise ApiError(400, "invalid_request", "Некорректное название чата") + if not isinstance(member_ids, list) or not (1 <= len(member_ids) <= 100): + raise ApiError(400, "invalid_request", "Некорректный список участников") + canonical = [] + for item in member_ids: + if not isinstance(item, str): + raise ApiError(400, "invalid_request", "Некорректный список участников") + canonical.append(parse_uuid(item)) + if len(canonical) != len(set(canonical)): + raise ApiError(400, "invalid_request", "Список участников содержит дубликаты") + if user_id not in canonical: + raise ApiError(400, "invalid_request", "Создатель должен входить в список участников") + member_uuids = [uuid.UUID(item) for item in canonical] + async with pool().acquire() as conn: + async with conn.transaction(): + found = await conn.fetch("SELECT id FROM users WHERE id = ANY($1::uuid[])", member_uuids) + if len(found) != len(canonical): + raise ApiError(400, "invalid_request", "Участник не найден") + row = await conn.fetchrow( + """ + INSERT INTO chats (id, title, created_at) + VALUES ($1, $2, clock_timestamp()) + RETURNING id, title, created_at + """, + uuid.uuid4(), + title, + ) + for position, member_id in enumerate(member_uuids): + await conn.execute( + """ + INSERT INTO chat_members (chat_id, user_id, position) + VALUES ($1, $2, $3) + """, + row["id"], + member_id, + position, + ) + return chat_dict(row, canonical) + + +def parse_limit(raw: str | None) -> int: + if raw is None: + return 50 + if not re.fullmatch(r"[1-9][0-9]*", raw): + raise ApiError(400, "invalid_request", "Некорректный limit") + value = int(raw) + if value > 100: + raise ApiError(400, "invalid_request", "Некорректный limit") + return value + + +async def list_chats(user_id: str, limit: int, cursor: str | None) -> dict: + created_at = None + cursor_id = None + if cursor is not None: + payload = _cursor_decode(cursor) + if payload.get("kind") != "chat" or payload.get("user_id") != user_id: + raise ApiError(400, "invalid_request", "Некорректный cursor") + try: + created_at = parse_iso(payload["created_at"]) + cursor_id = uuid.UUID(parse_uuid(payload["id"])) + except (KeyError, ValueError, TypeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + rows = await pool().fetch( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $1 + WHERE ($2::timestamptz IS NULL OR (c.created_at, c.id) > ($2, $3)) + ORDER BY c.created_at, c.id + LIMIT $4 + """, + uuid.UUID(user_id), + created_at, + cursor_id, + limit + 1, + ) + has_more = len(rows) > limit + rows = rows[:limit] + members = await _members_for(pool(), [row["id"] for row in rows]) + items = [chat_dict(row, members.get(str(row["id"]), [])) for row in rows] + next_cursor = None + if has_more and rows: + last = rows[-1] + next_cursor = _cursor_encode( + { + "v": 1, + "kind": "chat", + "user_id": user_id, + "created_at": iso(last["created_at"]), + "id": str(last["id"]), + } + ) + return {"items": items, "next_cursor": next_cursor} + + +async def get_chat(user_id: str, chat_id: str) -> dict: + chat_uuid = uuid.UUID(parse_uuid(chat_id)) + row = await pool().fetchrow( + """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = $2 + WHERE c.id = $1 + """, + chat_uuid, + uuid.UUID(user_id), + ) + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + members = await _members_for(pool(), [row["id"]]) + return chat_dict(row, members.get(str(row["id"]), [])) + + +def _message_parts(body: dict) -> tuple[str, list[str]]: + _require_object(body, {"text", "attachment_ids"}, {"text"}) + text = body["text"] + if not isinstance(text, str) or "\x00" in text or _code_points(text) > 4000: + raise ApiError(400, "invalid_request", "Некорректный текст сообщения") + raw_ids = body.get("attachment_ids", []) + if raw_ids is None or not isinstance(raw_ids, list) or len(raw_ids) > 10: + raise ApiError(400, "invalid_request", "Некорректный список вложений") + attachment_ids = [] + for item in raw_ids: + if not isinstance(item, str): + raise ApiError(400, "invalid_request", "Некорректный список вложений") + attachment_ids.append(parse_uuid(item)) + if len(attachment_ids) != len(set(attachment_ids)): + raise ApiError(400, "invalid_request", "Список вложений содержит дубликаты") + if not _nonblank(text) and not attachment_ids: + raise ApiError(400, "invalid_request", "Пустое сообщение") + return text, attachment_ids + + +def _canonical_hash(text: str, attachment_ids: list[str]) -> str: + payload = json.dumps( + {"attachment_ids": attachment_ids, "text": text}, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +async def create_message(user_id: str, chat_id: str, body: dict, idem_key: str | None, settings: Settings) -> dict: + chat_id = parse_uuid(chat_id) + if idem_key is None or _IDEMPOTENCY.fullmatch(idem_key) is None: + raise ApiError(400, "invalid_request", "Требуется корректный Idempotency-Key") + text, attachment_ids = _message_parts(body) + request_hash = _canonical_hash(text, attachment_ids) + path = f"/api/v1/chats/{chat_id}/messages" + k1, k2 = _lock_keys(f"{user_id}|POST|{path}|{idem_key}") + async with pool().acquire() as conn: + async with conn.transaction(): + await conn.execute("SELECT pg_advisory_xact_lock($1::int, $2::int)", k1, k2) + member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + uuid.UUID(chat_id), + uuid.UUID(user_id), + ) + if member is None: + raise ApiError(404, "not_found", "Объект не найден") + existing = await conn.fetchrow( + """ + SELECT request_hash, response_json, + created_at > clock_timestamp() - make_interval(secs => $4::int) AS fresh + FROM idempotency + WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3 + """, + uuid.UUID(user_id), + path, + idem_key, + settings.idempotency_ttl_seconds, + ) + if existing is not None and existing["fresh"]: + if existing["request_hash"] != request_hash: + raise ApiError(409, "conflict", "Ключ идемпотентности уже использован") + stored = existing["response_json"] + if isinstance(stored, str): + stored = json.loads(stored) + return stored + if existing is not None and not existing["fresh"]: + await conn.execute( + """ + DELETE FROM idempotency + WHERE user_id = $1 AND http_method = 'POST' AND path = $2 AND idem_key = $3 + """, + uuid.UUID(user_id), + path, + idem_key, + ) + if attachment_ids: + found = await conn.fetch( + "SELECT id, chat_id, status FROM attachments WHERE id = ANY($1::uuid[])", + [uuid.UUID(item) for item in attachment_ids], + ) + by_id = {str(row["id"]): row for row in found} + if len(by_id) != len(attachment_ids): + raise ApiError(404, "not_found", "Объект не найден") + for attachment_id in attachment_ids: + row = by_id[attachment_id] + if str(row["chat_id"]) != chat_id or row["status"] != "ready": + if str(row["chat_id"]) != chat_id: + raise ApiError(404, "not_found", "Объект не найден") + raise ApiError(409, "conflict", "Вложение нельзя прикрепить") + message = await conn.fetchrow( + """ + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, clock_timestamp()) + RETURNING id, chat_id, sender_id, text, created_at + """, + uuid.uuid4(), + uuid.UUID(chat_id), + uuid.UUID(user_id), + text, + ) + for position, attachment_id in enumerate(attachment_ids): + await conn.execute( + """ + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES ($1, $2, $3) + """, + message["id"], + uuid.UUID(attachment_id), + position, + ) + payload = message_dict(message, attachment_ids) + await conn.execute( + """ + INSERT INTO idempotency ( + user_id, http_method, path, idem_key, request_hash, status_code, response_json, created_at + ) + VALUES ($1, 'POST', $2, $3, $4, 201, $5::jsonb, clock_timestamp()) + """, + uuid.UUID(user_id), + path, + idem_key, + request_hash, + payload, + ) + return payload + + +async def list_messages(user_id: str, chat_id: str, limit: int, cursor: str | None) -> dict: + chat_id = parse_uuid(chat_id) + member = await pool().fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + uuid.UUID(chat_id), + uuid.UUID(user_id), + ) + if member is None: + raise ApiError(404, "not_found", "Объект не найден") + created_at = None + cursor_id = None + if cursor is not None: + payload = _cursor_decode(cursor) + if payload.get("kind") != "message" or payload.get("chat_id") != chat_id: + raise ApiError(400, "invalid_request", "Некорректный cursor") + try: + created_at = parse_iso(payload["created_at"]) + cursor_id = uuid.UUID(parse_uuid(payload["id"])) + except (KeyError, ValueError, TypeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный cursor") from exc + rows = await pool().fetch( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + AND ($2::timestamptz IS NULL OR (created_at, id) > ($2, $3)) + ORDER BY created_at, id + LIMIT $4 + """, + uuid.UUID(chat_id), + created_at, + cursor_id, + limit + 1, + ) + has_more = len(rows) > limit + rows = rows[:limit] + grouped = await _attachments_for(pool(), [row["id"] for row in rows]) + items = [message_dict(row, grouped.get(str(row["id"]), [])) for row in rows] + next_cursor = None + if has_more and rows: + last = rows[-1] + next_cursor = _cursor_encode( + { + "v": 1, + "kind": "message", + "chat_id": chat_id, + "created_at": iso(last["created_at"]), + "id": str(last["id"]), + } + ) + return {"items": items, "next_cursor": next_cursor} + + +def validate_filename(filename: str | None) -> str: + if filename is None: + raise ApiError(400, "invalid_request", "Требуется filename") + if not isinstance(filename, str) or not (1 <= _code_points(filename) <= 255): + raise ApiError(400, "invalid_request", "Некорректное имя файла") + if not _nonblank(filename) or filename in {".", ".."}: + raise ApiError(400, "invalid_request", "Некорректное имя файла") + for char in filename: + code = ord(char) + if char in {"/", "\\"} or code in _CONTROLS or code == 127: + raise ApiError(400, "invalid_request", "Некорректное имя файла") + return filename + + +def _media_type(value: str | None) -> str: + if not value: + raise ApiError(415, "unsupported_media_type", "Требуется image/png или image/jpeg") + media = value.split(";", 1)[0].strip().lower() + if media not in {"image/png", "image/jpeg"}: + raise ApiError(415, "unsupported_media_type", "Поддерживаются только image/png и image/jpeg") + return media + + +async def _mark_ledger(key: str, state: str) -> None: + await pool().execute( + """ + UPDATE object_ledger + SET state = $2, updated_at = clock_timestamp() + WHERE object_key = $1 + """, + key, + state, + ) + + +async def _compensate(settings: Settings, key: str) -> None: + """Drop a staged object after a failed metadata commit. A failed delete stays for the reconciler.""" + try: + await _to_thread(s3client.delete_object, settings, key) + await _mark_ledger(key, "aborted") + except Exception: + logger.exception("compensation failed for staged object; reconciler will retry after the grace period") + + +async def _to_thread(func, *args): + import asyncio + + return await asyncio.to_thread(func, *args) + + +async def upload_attachment( + user_id: str, + chat_id: str, + filename: str | None, + content_type_header: str | None, + content_length: str | None, + byte_iter, + settings: Settings, +) -> dict: + chat_id = parse_uuid(chat_id) + member = await pool().fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + uuid.UUID(chat_id), + uuid.UUID(user_id), + ) + if member is None: + raise ApiError(404, "not_found", "Объект не найден") + filename = validate_filename(filename) + media = _media_type(content_type_header) + if content_length is not None and not re.fullmatch(r"\d+", content_length): + raise ApiError(400, "invalid_request", "Некорректный Content-Length") + attachment_id = str(uuid.uuid4()) + object_key = f"o/{attachment_id}" + os.makedirs("/tmp/uploads", exist_ok=True) + fd, temp_path = tempfile.mkstemp(prefix="up-", dir="/tmp/uploads") + os.close(fd) + digest = hashlib.sha256() + size = 0 + try: + with open(temp_path, "wb") as handle: + async for chunk in byte_iter: + if not chunk: + continue + size += len(chunk) + if size > settings.max_upload_bytes: + raise ApiError(413, "payload_too_large", "Файл превышает 10 МиБ") + digest.update(chunk) + handle.write(chunk) + if size < 1: + raise ApiError(415, "unsupported_media_type", "Пустой файл") + await _to_thread( + validate_image, + temp_path, + media, + settings.max_image_pixels, + settings.full_decode_pixels, + ) + sha256 = digest.hexdigest() + await pool().execute( + """ + INSERT INTO object_ledger ( + object_key, attachment_id, chat_id, state, note, size_bytes, created_at, updated_at + ) + VALUES ($1, $2, $3, 'staging', 'upload', $4, clock_timestamp(), clock_timestamp()) + """, + object_key, + uuid.UUID(attachment_id), + uuid.UUID(chat_id), + size, + ) + try: + await _to_thread(s3client.put_file, settings, object_key, temp_path, media, size) + except Exception: + await _compensate(settings, object_key) + raise + try: + async with pool().acquire() as conn: + async with conn.transaction(): + row = await conn.fetchrow( + """ + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + status, object_key, error_code, created_at, updated_at + ) + VALUES ( + $1, $2, $3, $4, $5, $6, $7, 'ready', $8, NULL, + clock_timestamp(), clock_timestamp() + ) + RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + status, created_at, updated_at + """, + uuid.UUID(attachment_id), + uuid.UUID(chat_id), + uuid.UUID(user_id), + filename, + media, + size, + sha256, + object_key, + ) + updated = await conn.execute( + """ + UPDATE object_ledger + SET state = 'committed', updated_at = clock_timestamp() + WHERE object_key = $1 AND state = 'staging' + """, + object_key, + ) + if not updated.endswith("1"): + raise RuntimeError("ledger was not in staging state") + except Exception: + logger.exception("metadata commit failed after object upload") + await _compensate(settings, object_key) + raise ApiError(503, "unavailable", "Не удалось сохранить метаданные вложения") + return attachment_dict(row) + finally: + try: + os.remove(temp_path) + except OSError: + pass + + +async def get_attachment(user_id: str, attachment_id: str) -> dict: + attachment_id = parse_uuid(attachment_id) + row = await pool().fetchrow( + """ + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, + a.status, a.created_at, a.updated_at + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 + WHERE a.id = $1 + """, + uuid.UUID(attachment_id), + uuid.UUID(user_id), + ) + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + return attachment_dict(row) + + +async def download_attachment(user_id: str, attachment_id: str, variant: str | None, settings: Settings) -> tuple[bytes, str]: + if variant not in {None, "original"}: + raise ApiError(400, "invalid_request", "Некорректный variant") + attachment_id = parse_uuid(attachment_id) + row = await pool().fetchrow( + """ + SELECT a.object_key, a.content_type, a.size_bytes + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 + WHERE a.id = $1 + """, + uuid.UUID(attachment_id), + uuid.UUID(user_id), + ) + if row is None: + raise ApiError(404, "not_found", "Объект не найден") + data = await _to_thread(s3client.get_bytes, settings, row["object_key"], settings.max_upload_bytes) + if len(data) != row["size_bytes"]: + raise ApiError(404, "not_found", "Объект не найден") + return data, row["content_type"] + + +async def ready(settings: Settings) -> None: + try: + await pool().fetchval("SELECT 1") + except Exception as exc: + raise ApiError(503, "unavailable", "База данных недоступна") from exc + await _to_thread(s3client.head_bucket, settings) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..8b0f9b05 --- /dev/null +++ b/app/main.py @@ -0,0 +1,293 @@ +"""HTTP API. Every response carries X-Request-Id and X-Instance-Id.""" + +import asyncio +import base64 +import contextvars +import json +import logging +import uuid +from contextlib import asynccontextmanager + +import asyncpg +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from starlette.datastructures import MutableHeaders +from starlette.exceptions import HTTPException as StarletteHTTPException + +from app.config import load_settings +from app.db import close, connect, migrate +from app.errors import ApiError, unauthorized +from app import logic + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +logger = logging.getLogger("messenger") + +settings = load_settings() +request_id_var: contextvars.ContextVar[str] = contextvars.ContextVar("request_id", default="") + +_ERROR_CODES = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", +} + + +def _error_body(status: int, code: str, message: str) -> dict: + return {"error": {"code": code, "message": message, "request_id": request_id_var.get() or "error"}} + + +def _error_response(status: int, code: str, message: str, headers: dict | None = None) -> JSONResponse: + response = JSONResponse(status_code=status, content=_error_body(status, code, message)) + for key, value in (headers or {}).items(): + response.headers[key] = value + return response + + +class ContextMiddleware: + def __init__(self, app): + self.app = app + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + await self.app(scope, receive, send) + return + request_id = str(uuid.uuid4()) + token = request_id_var.set(request_id) + started = False + + async def send_wrapper(message): + nonlocal started + if message["type"] == "http.response.start": + started = True + headers = MutableHeaders(raw=message["headers"]) + headers["X-Request-Id"] = request_id + headers["X-Instance-Id"] = settings.instance_id + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + except Exception: + logger.exception("unhandled ASGI error") + if not started: + body = json.dumps(_error_body(500, "internal_error", "Внутренняя ошибка")).encode("utf-8") + await send( + { + "type": "http.response.start", + "status": 500, + "headers": [ + (b"content-type", b"application/json"), + (b"x-request-id", request_id.encode("ascii")), + (b"x-instance-id", settings.instance_id.encode("utf-8")), + ], + } + ) + await send({"type": "http.response.body", "body": body}) + finally: + request_id_var.reset(token) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + delay = 1 + for attempt in range(30): + try: + await connect(settings) + await migrate(settings) + logger.info("instance %s ready to serve", settings.instance_id) + break + except Exception: + logger.exception("startup attempt %s failed", attempt + 1) + await asyncio.sleep(delay) + delay = min(delay * 2, 5) + else: + raise RuntimeError("database startup failed") + try: + yield + finally: + await close() + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None, redirect_slashes=False) +app.add_middleware(ContextMiddleware) + + +@app.exception_handler(ApiError) +async def handle_api_error(_request: Request, exc: ApiError): + return _error_response(exc.status, exc.code, exc.message, exc.headers) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http(_request: Request, exc: StarletteHTTPException): + status = exc.status_code if exc.status_code in _ERROR_CODES else 500 + return _error_response(status, _ERROR_CODES.get(status, "internal_error"), "Запрос отклонён") + + +@app.exception_handler(asyncpg.PostgresConnectionError) +async def handle_db_down(_request: Request, exc: asyncpg.PostgresConnectionError): + logger.exception("database connection failed") + return _error_response(503, "unavailable", "База данных недоступна") + + +@app.exception_handler(Exception) +async def handle_unexpected(_request: Request, exc: Exception): + logger.exception("unhandled error") + return _error_response(500, "internal_error", "Внутренняя ошибка") + + +def _bearer(request: Request) -> str: + header = request.headers.get("authorization", "") + if not header.lower().startswith("bearer "): + raise unauthorized("bearer") + token = header.split(" ", 1)[1].strip() + if not token: + raise unauthorized("bearer") + return token + + +def _basic(request: Request) -> tuple[str, str]: + header = request.headers.get("authorization", "") + if not header.lower().startswith("basic "): + raise unauthorized("basic") + encoded = header.split(" ", 1)[1].strip() + try: + decoded = base64.b64decode(encoded).decode("utf-8") + except (ValueError, UnicodeError) as exc: + raise unauthorized("basic") from exc + if ":" not in decoded: + raise unauthorized("basic") + username, password = decoded.split(":", 1) + return username, password + + +async def _current_user(request: Request) -> dict: + return await logic.authenticate(_bearer(request)) + + +async def _json_object(request: Request) -> dict: + media = (request.headers.get("content-type") or "").split(";", 1)[0].strip().lower() + if media != "application/json": + raise ApiError(400, "invalid_request", "Ожидается application/json") + raw = bytearray() + async for chunk in request.stream(): + if len(raw) + len(chunk) > 65536: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое") + raw.extend(chunk) + try: + data = json.loads(bytes(raw) if raw else b"") + except (json.JSONDecodeError, UnicodeError) as exc: + raise ApiError(400, "invalid_request", "Некорректный JSON") from exc + if not isinstance(data, dict): + raise ApiError(400, "invalid_request", "Ожидается JSON-объект") + return data + + +@app.get("/health/live") +async def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def ready(): + await logic.ready(settings) + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def register(request: Request): + return await logic.register_user(await _json_object(request)) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + username, password = _basic(request) + return await logic.create_session(username, password, settings) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request): + token = _bearer(request) + await logic.authenticate(token) + await logic.revoke_session(token) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + return await _current_user(request) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await _current_user(request) + return await logic.create_chat(user["id"], await _json_object(request)) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await _current_user(request) + limit = logic.parse_limit(request.query_params.get("limit")) + return await logic.list_chats(user["id"], limit, request.query_params.get("cursor")) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + user = await _current_user(request) + return await logic.get_chat(user["id"], chat_id) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + user = await _current_user(request) + body = await _json_object(request) + return await logic.create_message( + user["id"], + chat_id, + body, + request.headers.get("idempotency-key"), + settings, + ) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + user = await _current_user(request) + limit = logic.parse_limit(request.query_params.get("limit")) + return await logic.list_messages(user["id"], chat_id, limit, request.query_params.get("cursor")) + + +@app.post("/api/v1/chats/{chat_id}/attachments", status_code=201) +async def upload(chat_id: str, request: Request): + user = await _current_user(request) + payload = await logic.upload_attachment( + user["id"], + chat_id, + request.query_params.get("filename"), + request.headers.get("content-type"), + request.headers.get("content-length"), + request.stream(), + settings, + ) + return JSONResponse(status_code=201, content=payload) + + +@app.get("/api/v1/attachments/{attachment_id}") +async def attachment_meta(attachment_id: str, request: Request): + user = await _current_user(request) + return await logic.get_attachment(user["id"], attachment_id) + + +@app.get("/api/v1/attachments/{attachment_id}/content") +async def attachment_content(attachment_id: str, request: Request): + user = await _current_user(request) + data, media = await logic.download_attachment( + user["id"], + attachment_id, + request.query_params.get("variant"), + settings, + ) + return Response(content=data, media_type=media, headers={"Content-Length": str(len(data))}) diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..912ccbd3 --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,235 @@ +"""Reconcile PostgreSQL metadata with the private bucket. + +Grace period: objects and rows younger than ORPHAN_GRACE_SECONDS are left alone, +because an upload may have written S3 and not committed metadata yet. +Attachments referenced by a message are never deleted. +A dry-run reports the same actions apply would perform, then apply is safe to repeat. +""" + +import time +from datetime import datetime, timezone + +from app.config import Settings +from app.db import pool +from app import s3client + + +def _aware(value: datetime) -> datetime: + if value.tzinfo is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def _age(moment: datetime, now: datetime) -> float: + return (_aware(now) - _aware(moment)).total_seconds() + + +async def reconcile(settings: Settings, apply: bool) -> dict: + started = time.perf_counter() + db_started = time.perf_counter() + async with pool().acquire() as conn: + now = await conn.fetchval("SELECT clock_timestamp()") + attachments = await conn.fetch( + "SELECT id, object_key, filename, created_at FROM attachments" + ) + ledger_rows = await conn.fetch( + "SELECT object_key, attachment_id, state, note, created_at FROM object_ledger" + ) + referenced = { + str(row["attachment_id"]) + for row in await conn.fetch("SELECT attachment_id FROM message_attachments") + } + db_ms = (time.perf_counter() - db_started) * 1000 + list_started = time.perf_counter() + objects, pages = s3client.list_objects(settings, "o/") + list_ms = (time.perf_counter() - list_started) * 1000 + + by_key = {row["object_key"]: row for row in attachments} + ledger = {row["object_key"]: row for row in ledger_rows} + object_keys = {item["Key"] for item in objects} + grace = settings.orphan_grace_seconds + actions = [] + skipped: dict[str, int] = {} + + def skip(reason: str) -> None: + skipped[reason] = skipped.get(reason, 0) + 1 + + for item in objects: + key = item["Key"] + led = ledger.get(key) + if key in by_key: + skip("accounted") + continue + moment = led["created_at"] if led is not None else item["LastModified"] + note = (led["note"] if led is not None and led["note"] else None) or ( + led["state"] if led is not None else "untracked" + ) + age = _age(moment, now) + attachment_id = str(led["attachment_id"]) if led is not None and led["attachment_id"] else None + if led is not None and led["state"] in {"aborted", "purged"} and age >= grace: + actions.append( + { + "op": "delete_orphan_object", + "note": note, + "object_key": key, + "attachment_id": attachment_id, + "age_seconds": int(age), + } + ) + continue + if age < grace: + skip("fresh_orphan") + continue + actions.append( + { + "op": "delete_orphan_object", + "note": note, + "object_key": key, + "attachment_id": attachment_id, + "age_seconds": int(age), + } + ) + + for row in attachments: + if row["object_key"] in object_keys: + continue + attachment_id = str(row["id"]) + age = _age(row["created_at"], now) + if attachment_id in referenced: + skip("in_use_missing") + continue + if age < grace: + skip("fresh_missing") + continue + actions.append( + { + "op": "delete_stale_metadata", + "note": row["filename"], + "object_key": row["object_key"], + "attachment_id": attachment_id, + "age_seconds": int(age), + } + ) + + for row in ledger_rows: + if row["object_key"] in object_keys or row["object_key"] in by_key: + continue + if row["state"] in {"aborted", "purged"}: + skip("terminal_ledger") + continue + age = _age(row["created_at"], now) + if age < grace: + skip("fresh_intent") + continue + actions.append( + { + "op": "abort_stale_intent", + "note": row["note"] or row["state"], + "object_key": row["object_key"], + "attachment_id": str(row["attachment_id"]) if row["attachment_id"] else None, + "age_seconds": int(age), + } + ) + + applied = [] + if apply: + for action in actions: + result = await _apply_one(settings, action, grace) + if result is not None: + applied.append(result) + + public_actions = [_public(item) for item in (applied if apply else actions)] + return { + "mode": "apply" if apply else "dry-run", + "duration_ms": round((time.perf_counter() - started) * 1000, 3), + "db_ms": round(db_ms, 3), + "list_ms": round(list_ms, 3), + "list_pages": pages, + "objects_seen": len(objects), + "attachment_rows": len(attachments), + "ledger_rows": len(ledger_rows), + "referenced_attachments": len(referenced), + "grace_seconds": grace, + "skipped": skipped, + "actions": public_actions, + "action_count": len(public_actions), + } + + +def _public(action: dict) -> dict: + return { + "op": action["op"], + "note": action.get("note"), + "attachment_id": action.get("attachment_id"), + "age_seconds": action.get("age_seconds"), + } + + +async def _apply_one(settings: Settings, action: dict, grace: int) -> dict | None: + key = action["object_key"] + op = action["op"] + async with pool().acquire() as conn: + now = await conn.fetchval("SELECT clock_timestamp()") + if op == "delete_orphan_object": + owner = await conn.fetchrow("SELECT id, created_at FROM attachments WHERE object_key = $1", key) + if owner is not None: + return None + led = await conn.fetchrow("SELECT created_at, state FROM object_ledger WHERE object_key = $1", key) + moment = led["created_at"] if led is not None else now + if led is not None and _age(moment, now) < grace and led["state"] not in {"aborted", "purged"}: + return None + s3client.delete_object(settings, key) + if led is not None: + await conn.execute( + """ + UPDATE object_ledger + SET state = 'purged', updated_at = clock_timestamp() + WHERE object_key = $1 + """, + key, + ) + return action + if op == "delete_stale_metadata": + attachment_id = action["attachment_id"] + row = await conn.fetchrow( + "SELECT object_key, created_at FROM attachments WHERE id = $1::uuid", + attachment_id, + ) + if row is None or _age(row["created_at"], now) < grace: + return None + used = await conn.fetchval( + "SELECT 1 FROM message_attachments WHERE attachment_id = $1::uuid", + attachment_id, + ) + if used is not None or s3client.object_exists(settings, row["object_key"]): + return None + async with conn.transaction(): + await conn.execute("DELETE FROM attachments WHERE id = $1::uuid", attachment_id) + await conn.execute( + """ + UPDATE object_ledger + SET state = 'purged', updated_at = clock_timestamp() + WHERE object_key = $1 + """, + row["object_key"], + ) + return action + if op == "abort_stale_intent": + led = await conn.fetchrow( + "SELECT state, created_at FROM object_ledger WHERE object_key = $1", + key, + ) + if led is None or led["state"] in {"aborted", "purged"} or _age(led["created_at"], now) < grace: + return None + if s3client.object_exists(settings, key): + s3client.delete_object(settings, key) + await conn.execute( + """ + UPDATE object_ledger + SET state = 'aborted', updated_at = clock_timestamp() + WHERE object_key = $1 AND state IN ('staging', 'committed') + """, + key, + ) + return action + return None diff --git a/app/s3client.py b/app/s3client.py new file mode 100644 index 00000000..6e20333c --- /dev/null +++ b/app/s3client.py @@ -0,0 +1,152 @@ +"""S3 client for the private attachment bucket. Object keys are chosen by the server.""" + +import socket +from urllib.parse import urlparse + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError + +from app.config import Settings +from app.errors import ApiError + +_client = None + + +def client(settings: Settings): + global _client + if _client is None: + _client = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint, + aws_access_key_id=settings.s3_access_key, + aws_secret_access_key=settings.s3_secret_key, + region_name=settings.s3_region, + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path"}, + connect_timeout=1, + read_timeout=20, + retries={"max_attempts": 1, "mode": "standard"}, + ), + ) + return _client + + +def _unavailable(exc: Exception) -> ApiError: + return ApiError(503, "unavailable", "Хранилище недоступно") + + +def _ensure_reachable(settings: Settings) -> None: + """Fail in about a second when the endpoint black-holes a stopped container.""" + parsed = urlparse(settings.s3_endpoint) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + try: + with socket.create_connection((parsed.hostname, port), timeout=1): + return + except OSError as exc: + raise _unavailable(exc) from exc + + +def head_bucket(settings: Settings) -> None: + _ensure_reachable(settings) + try: + client(settings).head_bucket(Bucket=settings.s3_bucket) + except (ClientError, EndpointConnectionError, BotoCoreError, OSError) as exc: + raise _unavailable(exc) from exc + + +def put_file(settings: Settings, key: str, path: str, content_type: str, size: int) -> None: + _ensure_reachable(settings) + try: + with open(path, "rb") as handle: + client(settings).put_object( + Bucket=settings.s3_bucket, + Key=key, + Body=handle, + ContentLength=size, + ContentType=content_type, + ) + except (ClientError, EndpointConnectionError, BotoCoreError, OSError) as exc: + raise _unavailable(exc) from exc + + +def put_bytes(settings: Settings, key: str, payload: bytes, content_type: str = "application/octet-stream") -> None: + _ensure_reachable(settings) + try: + client(settings).put_object( + Bucket=settings.s3_bucket, + Key=key, + Body=payload, + ContentLength=len(payload), + ContentType=content_type, + ) + except (ClientError, EndpointConnectionError, BotoCoreError, OSError) as exc: + raise _unavailable(exc) from exc + + +def get_bytes(settings: Settings, key: str, limit: int) -> bytes: + _ensure_reachable(settings) + try: + obj = client(settings).get_object(Bucket=settings.s3_bucket, Key=key) + body = obj["Body"] + try: + data = body.read(limit + 1) + finally: + body.close() + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if code in {"NoSuchKey", "404", "NotFound", "NoSuchBucket"} or status == 404: + raise ApiError(404, "not_found", "Объект не найден") from exc + raise _unavailable(exc) from exc + except (EndpointConnectionError, BotoCoreError, OSError) as exc: + raise _unavailable(exc) from exc + if len(data) > limit: + raise ApiError(500, "internal_error", "Внутренняя ошибка") + return data + + +def delete_object(settings: Settings, key: str) -> None: + _ensure_reachable(settings) + try: + client(settings).delete_object(Bucket=settings.s3_bucket, Key=key) + except (ClientError, EndpointConnectionError, BotoCoreError, OSError) as exc: + raise _unavailable(exc) from exc + + +def object_exists(settings: Settings, key: str) -> bool: + _ensure_reachable(settings) + try: + client(settings).head_object(Bucket=settings.s3_bucket, Key=key) + return True + except ClientError as exc: + code = exc.response.get("Error", {}).get("Code", "") + status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + if code in {"NoSuchKey", "404", "NotFound", "NoSuchBucket"} or status in {404, 400}: + return False + raise _unavailable(exc) from exc + except (EndpointConnectionError, BotoCoreError, OSError) as exc: + raise _unavailable(exc) from exc + + +def list_objects(settings: Settings, prefix: str = "o/") -> tuple[list[dict], int]: + _ensure_reachable(settings) + found = [] + pages = 0 + token = None + while True: + kwargs = {"Bucket": settings.s3_bucket, "Prefix": prefix, "MaxKeys": 1000} + if token: + kwargs["ContinuationToken"] = token + try: + response = client(settings).list_objects_v2(**kwargs) + except (ClientError, EndpointConnectionError, BotoCoreError, OSError) as exc: + raise _unavailable(exc) from exc + pages += 1 + found.extend(response.get("Contents") or []) + if not response.get("IsTruncated"): + return found, pages + token = response.get("NextContinuationToken") + if not token: + raise ApiError(503, "unavailable", "Хранилище вернуло неполный список") diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e2871166 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,131 @@ +# Local lab stack. Pinned image tags. Secrets come from an untracked .env. +name: mtusi2026lab4 + +services: + postgres: + image: postgres:16.8-bookworm + container_name: mtusi-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB} + volumes: + - postgres-data:/var/lib/postgresql/data + networks: + - lab + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER} -d ${POSTGRES_DB}"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + restart: unless-stopped + + minio: + image: minio/minio:RELEASE.2025-09-07T16-13-09Z + container_name: mtusi-minio + command: ["server", "/data", "--console-address", ":9001"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + volumes: + - minio-data:/data + networks: + - lab + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 2s + timeout: 5s + retries: 30 + start_period: 5s + restart: unless-stopped + + s3-init: + build: + context: . + dockerfile: Dockerfile + image: mtusi2026lab4-api:local + pull_policy: build + container_name: mtusi-s3-init + entrypoint: ["/bin/sh", "/app/infra/bootstrap-s3.sh"] + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_BUCKET: ${S3_BUCKET} + depends_on: + minio: + condition: service_healthy + networks: + - lab + restart: "no" + + api-1: &api + image: mtusi2026lab4-api:local + pull_policy: never + container_name: mtusi-api-1 + environment: &api-env + INSTANCE_ID: api-1 + DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} + S3_ENDPOINT: http://minio:9000 + S3_REGION: ${S3_REGION} + S3_BUCKET: ${S3_BUCKET} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS} + ORPHAN_GRACE_SECONDS: ${ORPHAN_GRACE_SECONDS} + APP_ENV: ${APP_ENV} + depends_on: + postgres: + condition: service_healthy + s3-init: + condition: service_completed_successfully + networks: + - lab + stop_grace_period: 3s + deploy: + resources: + limits: + memory: 768M + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=3)"] + interval: 3s + timeout: 5s + retries: 25 + start_period: 20s + restart: unless-stopped + + api-2: + <<: *api + container_name: mtusi-api-2 + environment: + <<: *api-env + INSTANCE_ID: api-2 + + lb: + image: haproxy:2.8.14-alpine + container_name: mtusi-lb + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:8080:8080" + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + networks: + - lab + restart: unless-stopped + +networks: + lab: + name: mtusi-lab + +volumes: + postgres-data: + minio-data: diff --git a/evidence/memory.json b/evidence/memory.json new file mode 100644 index 00000000..e601993d --- /dev/null +++ b/evidence/memory.json @@ -0,0 +1,51 @@ +{ + "hypothesis": "A single upload is capped at 10 MiB and does not decode bitmaps above 1_000_000 pixels; ten concurrent uploads stay inside the 768 MiB container limit.", + "limits": { + "container_memory": "768MiB", + "max_upload_bytes": 10485760, + "full_decode_pixels": 1000000 + }, + "waves": [ + { + "label": "small", + "file_bytes": 12420, + "concurrency": 10, + "successes": 10, + "errors": [], + "elapsed_seconds": 0.367, + "docker_stats_peak_bytes": { + "mtusi-api-1": 56235130, + "mtusi-api-2": 57199820 + }, + "vmhwm_kib_before": { + "mtusi-api-1": 74468, + "mtusi-api-2": 74984 + }, + "vmhwm_kib_after": { + "mtusi-api-1": 74468, + "mtusi-api-2": 74984 + } + }, + { + "label": "large", + "file_bytes": 3241218, + "concurrency": 10, + "successes": 10, + "errors": [], + "elapsed_seconds": 0.74, + "docker_stats_peak_bytes": { + "mtusi-api-1": 68304240, + "mtusi-api-2": 56214159 + }, + "vmhwm_kib_before": { + "mtusi-api-1": 74892, + "mtusi-api-2": 74776 + }, + "vmhwm_kib_after": { + "mtusi-api-1": 77184, + "mtusi-api-2": 76492 + } + } + ], + "note": "docker stats includes page cache. VmHWM is the process high-water RSS in KiB after a fresh API restart, so the two waves are comparable." +} diff --git a/evidence/reconcile.json b/evidence/reconcile.json new file mode 100644 index 00000000..a589fdd1 --- /dev/null +++ b/evidence/reconcile.json @@ -0,0 +1,100 @@ +{ + "hypothesis": "A full-scan reconciler with a 120s grace period removes only expired orphans and unreferenced metadata whose object is already gone.", + "method": "Inject a fresh orphan, a 2h orphan, a 2h missing unreferenced row, and a 2h missing row referenced by a message. Dry-run, apply, dry-run again.", + "dry_run": { + "mode": "dry-run", + "duration_ms": 129.367, + "db_ms": 2.036, + "list_ms": 127.228, + "list_pages": 1, + "objects_seen": 27, + "attachment_rows": 27, + "ledger_rows": 29, + "referenced_attachments": 3, + "grace_seconds": 120, + "skipped": { + "fresh_orphan": 1, + "accounted": 25, + "in_use_missing": 1 + }, + "actions": [ + { + "op": "delete_orphan_object", + "note": "old-orphan", + "attachment_id": "b763d1e0-5694-4816-b035-3bcd748d4e48", + "age_seconds": 7200 + }, + { + "op": "delete_stale_metadata", + "note": "fault-stale-missing", + "attachment_id": "93a9f66b-b559-4053-8094-189d09a19864", + "age_seconds": 7200 + } + ], + "action_count": 2 + }, + "apply": { + "mode": "apply", + "duration_ms": 154.264, + "db_ms": 2.987, + "list_ms": 128.676, + "list_pages": 1, + "objects_seen": 27, + "attachment_rows": 27, + "ledger_rows": 29, + "referenced_attachments": 3, + "grace_seconds": 120, + "skipped": { + "fresh_orphan": 1, + "accounted": 25, + "in_use_missing": 1 + }, + "actions": [ + { + "op": "delete_orphan_object", + "note": "old-orphan", + "attachment_id": "b763d1e0-5694-4816-b035-3bcd748d4e48", + "age_seconds": 7202 + }, + { + "op": "delete_stale_metadata", + "note": "fault-stale-missing", + "attachment_id": "93a9f66b-b559-4053-8094-189d09a19864", + "age_seconds": 7202 + } + ], + "action_count": 2 + }, + "second_dry_run": { + "mode": "dry-run", + "duration_ms": 129.352, + "db_ms": 1.791, + "list_ms": 127.518, + "list_pages": 1, + "objects_seen": 26, + "attachment_rows": 26, + "ledger_rows": 29, + "referenced_attachments": 3, + "grace_seconds": 120, + "skipped": { + "fresh_orphan": 1, + "accounted": 25, + "in_use_missing": 1, + "terminal_ledger": 2 + }, + "actions": [], + "action_count": 0 + }, + "survived": { + "fresh_orphan_object": true, + "in_use_metadata": true, + "fresh_message_attachment_sha256": "99ea7533c441f79cd630d15ce4d58d1ec4e06f6f0efb3b77a5c088305cd1bd75", + "loose_unreferenced_attachment": "a8b23e26-ee4f-49c6-8490-16fe5a1ef9b8" + }, + "limitations": [ + "Walk cost is one full ListObjects plus one pass over attachment and ledger rows; this stand has one page.", + "Grace uses the ledger or metadata timestamp, not a backdated S3 LastModified, because MinIO does not let the lab rewrite LastModified.", + "In-use metadata whose bytes are gone is reported and kept; the reconciler does not invent replacement bytes.", + "A second dry-run being empty shows the repair is repeatable, not that a continuous watcher is running." + ] +} diff --git a/evidence/s3_outage.json b/evidence/s3_outage.json new file mode 100644 index 00000000..8810b08a --- /dev/null +++ b/evidence/s3_outage.json @@ -0,0 +1,5 @@ +{ + "live_while_s3_down": 200, + "ready_while_s3_down": 503, + "download_after_start": "match" +} diff --git a/infra/bootstrap-s3.sh b/infra/bootstrap-s3.sh new file mode 100644 index 00000000..63ba1276 --- /dev/null +++ b/infra/bootstrap-s3.sh @@ -0,0 +1,54 @@ +#!/bin/sh +# Idempotent private bucket and least-privilege API user. Root credentials stay in this init container. +set -eu + +: "${MINIO_ROOT_USER:?}" +: "${MINIO_ROOT_PASSWORD:?}" +: "${S3_ACCESS_KEY_ID:?}" +: "${S3_SECRET_ACCESS_KEY:?}" +: "${S3_BUCKET:?}" + +export MC_CONFIG_DIR=/tmp/mc +export MC_HOST_local="http://${MINIO_ROOT_USER}:${MINIO_ROOT_PASSWORD}@minio:9000" + +i=0 +until mc mb --ignore-existing "local/${S3_BUCKET}" >/dev/null 2>&1; do + i=$((i + 1)) + if [ "$i" -gt 40 ]; then + echo "bucket bootstrap failed" >&2 + exit 1 + fi + sleep 1 +done + +mc anonymous set none "local/${S3_BUCKET}" >/dev/null + +cat > /tmp/messenger-api-policy.json </dev/null 2>&1; then + mc admin policy create local messenger-api /tmp/messenger-api-policy.json >/dev/null +fi + +if ! mc admin user info local "${S3_ACCESS_KEY_ID}" >/dev/null 2>&1; then + mc admin user add local "${S3_ACCESS_KEY_ID}" "${S3_SECRET_ACCESS_KEY}" >/dev/null +fi + +mc admin policy attach local messenger-api --user "${S3_ACCESS_KEY_ID}" >/dev/null 2>&1 || true + +echo "bucket bootstrap ok" diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..ed93424e --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,22 @@ +global + maxconn 256 + +defaults + mode http + timeout connect 2s + timeout client 120s + timeout server 120s + option http-server-close + +frontend messenger + bind *:8080 + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/live + http-check expect status 200 + default-server inter 1s fall 2 rise 1 + server api-1 api-1:8080 check + server api-2 api-2:8080 check diff --git a/infra/migrate.sql b/infra/migrate.sql new file mode 100644 index 00000000..b84ca273 --- /dev/null +++ b/infra/migrate.sql @@ -0,0 +1,92 @@ +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); + +CREATE INDEX IF NOT EXISTS sessions_user_id_idx ON sessions (user_id); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position integer NOT NULL, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + owner_id uuid NOT NULL REFERENCES users (id), + filename text NOT NULL, + content_type text NOT NULL, + size_bytes integer NOT NULL CHECK (size_bytes > 0 AND size_bytes <= 10485760), + sha256 text NOT NULL, + status text NOT NULL CHECK (status = 'ready'), + object_key text NOT NULL UNIQUE, + error_code text, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE INDEX IF NOT EXISTS attachments_chat_idx ON attachments (chat_id, created_at); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id), + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE INDEX IF NOT EXISTS messages_chat_order_idx ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS message_attachments ( + message_id uuid NOT NULL REFERENCES messages (id), + attachment_id uuid NOT NULL REFERENCES attachments (id), + position integer NOT NULL, + PRIMARY KEY (message_id, position) +); + +CREATE INDEX IF NOT EXISTS message_attachments_attachment_idx ON message_attachments (attachment_id); + +CREATE TABLE IF NOT EXISTS idempotency ( + user_id uuid NOT NULL REFERENCES users (id), + http_method text NOT NULL, + path text NOT NULL, + idem_key text NOT NULL, + request_hash text NOT NULL, + status_code integer NOT NULL, + response_json jsonb NOT NULL, + created_at timestamptz NOT NULL, + PRIMARY KEY (user_id, http_method, path, idem_key) +); + +CREATE TABLE IF NOT EXISTS object_ledger ( + object_key text PRIMARY KEY, + attachment_id uuid, + chat_id uuid, + state text NOT NULL CHECK (state IN ('staging', 'committed', 'aborted', 'purged')), + note text, + size_bytes bigint, + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE INDEX IF NOT EXISTS object_ledger_state_idx ON object_ledger (state, created_at); diff --git a/infra/minio-policy.json b/infra/minio-policy.json new file mode 100644 index 00000000..108e26e9 --- /dev/null +++ b/infra/minio-policy.json @@ -0,0 +1,15 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": ["s3:GetObject", "s3:PutObject", "s3:DeleteObject"], + "Resource": ["arn:aws:s3:::messenger-attachments/*"] + }, + { + "Effect": "Allow", + "Action": ["s3:ListBucket", "s3:GetBucketLocation"], + "Resource": ["arn:aws:s3:::messenger-attachments"] + } + ] +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..a0dd334c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +asyncpg==0.30.0 +argon2-cffi==23.1.0 +boto3==1.35.99 +pillow==11.1.0 diff --git a/scripts/init_env.py b/scripts/init_env.py new file mode 100644 index 00000000..80979af7 --- /dev/null +++ b/scripts/init_env.py @@ -0,0 +1,44 @@ +"""Create a local .env with synthetic secrets. Refuses to overwrite an existing file.""" + +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +TARGET = ROOT / ".env" + + +def token(length: int = 32) -> str: + alphabet = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" + return "".join(secrets.choice(alphabet) for _ in range(length)) + + +def main() -> None: + if TARGET.exists(): + print(".env already exists; left unchanged") + return + content = "\n".join( + [ + "POSTGRES_USER=messenger", + f"POSTGRES_PASSWORD={token()}", + "POSTGRES_DB=messenger", + "MINIO_ROOT_USER=labminio", + f"MINIO_ROOT_PASSWORD={token(40)}", + "S3_BUCKET=messenger-attachments", + "S3_REGION=us-east-1", + "S3_ACCESS_KEY_ID=messengerapi", + f"S3_SECRET_ACCESS_KEY={token(40)}", + "SESSION_TTL_SECONDS=3600", + "IDEMPOTENCY_TTL_SECONDS=86400", + "MAX_UPLOAD_BYTES=10485760", + "MAX_IMAGE_PIXELS=40000000", + "ORPHAN_GRACE_SECONDS=120", + "APP_ENV=lab", + "", + ] + ) + TARGET.write_text(content, encoding="utf-8") + print("wrote .env with local synthetic secrets; do not commit it") + + +if __name__ == "__main__": + main() diff --git a/scripts/labhook.cs b/scripts/labhook.cs new file mode 100644 index 00000000..2c2c49d4 --- /dev/null +++ b/scripts/labhook.cs @@ -0,0 +1,33 @@ +using System; +using System.Diagnostics; +using System.IO; + +class Program +{ + static int Run(string args) + { + string dir = Path.GetFullPath(Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "..")); + var start = new ProcessStartInfo("docker", args) + { + WorkingDirectory = dir, + UseShellExecute = false + }; + var process = Process.Start(start); + process.WaitForExit(); + return process.ExitCode; + } + + static int Main() + { + string name = Path.GetFileName(Environment.GetCommandLineArgs()[0]).ToLowerInvariant(); + if (name.Contains("stop")) + { + string id = Environment.GetEnvironmentVariable("TARGET_INSTANCE_ID"); + if (id != "api-1" && id != "api-2") return 2; + return Run("compose --env-file .env stop -t 2 " + id); + } + int up = Run("compose --env-file .env up -d --no-recreate --no-build postgres minio api-1 api-2 lb"); + if (up != 0) return up; + return Run("compose --env-file .env restart postgres minio api-1 api-2 lb"); + } +} diff --git a/scripts/memory_experiment.py b/scripts/memory_experiment.py new file mode 100644 index 00000000..febd8f81 --- /dev/null +++ b/scripts/memory_experiment.py @@ -0,0 +1,171 @@ +"""Ten concurrent uploads at two sizes. Records container RSS, not a universal capacity claim.""" + +import json +import random +import struct +import subprocess +import sys +import threading +import time +import urllib.request +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 + + +def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) + + +def make_png(width: int, height: int, seed: int) -> bytes: + rng = random.Random(seed) + raw = bytearray() + for _y in range(height): + raw.append(0) + raw.extend(rng.randbytes(width * 3)) + compressor = zlib.compressobj(level=0) + payload = compressor.compress(bytes(raw)) + compressor.flush() + ihdr = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", payload) + chunk(b"IEND", b"") + + +def parse_bytes(text: str) -> int: + """docker stats MemUsage looks like '12.3MiB / 768MiB'.""" + value = text.split("/")[0].strip() + number = "" + unit = "" + for char in value: + if char.isdigit() or char == ".": + number += char + elif char != " ": + unit += char + scale = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3, "kB": 1000, "MB": 1000**2, "GB": 1000**3} + return int(float(number) * scale[unit]) + + +def sample(stop: threading.Event, peaks: dict) -> None: + while not stop.is_set(): + try: + raw = subprocess.check_output( + ["docker", "stats", "--no-stream", "--format", "{{.Name}}\t{{.MemUsage}}"], + text=True, + ) + except subprocess.CalledProcessError: + time.sleep(0.3) + continue + for line in raw.splitlines(): + name, usage = line.split("\t", 1) + if name not in {"mtusi-api-1", "mtusi-api-2"}: + continue + peaks[name] = max(peaks.get(name, 0), parse_bytes(usage)) + time.sleep(0.25) + + +def vmhwm() -> dict: + result = {} + for name in ("mtusi-api-1", "mtusi-api-2"): + text = subprocess.check_output(["docker", "exec", name, "sh", "-c", "grep VmHWM /proc/1/status"], text=True) + result[name] = int(text.split()[1]) + return result + + +def wave(client: Client, actor, chat, payload: bytes, label: str) -> dict: + stop = threading.Event() + peaks = {} + thread = threading.Thread(target=sample, args=(stop, peaks)) + before = vmhwm() + thread.start() + errors = [] + + def once(index): + local = Client("http://127.0.0.1:8080", timeout=60) + path = f"/api/v1/chats/{chat['id']}/attachments?filename={label}-{index}.png" + try: + response = local.call( + "POST", + path, + 201, + actor=actor, + raw=payload, + headers={"Content-Type": "image/png"}, + schema="Attachment", + ) + return response["json"]["id"] + except Exception as exc: + errors.append(str(exc)) + return None + + started = time.perf_counter() + from concurrent.futures import ThreadPoolExecutor + + with ThreadPoolExecutor(max_workers=10) as pool: + ids = list(pool.map(once, range(10))) + elapsed = time.perf_counter() - started + time.sleep(0.6) + stop.set() + thread.join() + after = vmhwm() + return { + "label": label, + "file_bytes": len(payload), + "concurrency": 10, + "successes": sum(1 for item in ids if item), + "errors": errors[:5], + "elapsed_seconds": round(elapsed, 3), + "docker_stats_peak_bytes": peaks, + "vmhwm_kib_before": before, + "vmhwm_kib_after": after, + } + + +def restart_apis() -> None: + subprocess.check_call(["docker", "compose", "--env-file", ".env", "restart", "api-1", "api-2"], cwd=ROOT) + deadline = time.time() + 90 + while time.time() < deadline: + try: + urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=2).read() + return + except Exception: + time.sleep(1) + raise SystemExit("API did not become ready after restart") + + +def main() -> None: + client = Client("http://127.0.0.1:8080", timeout=30) + alice = client.user("mem_alice") + bob = client.user("mem_bob") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Память", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + small = make_png(64, 64, 1) + large = make_png(1200, 900, 2) + restart_apis() + first = wave(client, alice, chat, small, "small") + restart_apis() + second = wave(client, alice, chat, large, "large") + report = { + "hypothesis": "A single upload is capped at 10 MiB and does not decode bitmaps above 1_000_000 pixels; ten concurrent uploads stay inside the 768 MiB container limit.", + "limits": {"container_memory": "768MiB", "max_upload_bytes": 10485760, "full_decode_pixels": 1000000}, + "waves": [first, second], + "note": "docker stats includes page cache. VmHWM is the process high-water RSS in KiB after a fresh API restart, so the two waves are comparable.", + } + dest = ROOT / "evidence" / "memory.json" + dest.parent.mkdir(exist_ok=True) + dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + if first["successes"] != 10 or second["successes"] != 10: + raise SystemExit("not all concurrent uploads succeeded") + print("PASS: ten concurrent uploads at two sizes") + + +if __name__ == "__main__": + main() diff --git a/scripts/reconcile_experiment.py b/scripts/reconcile_experiment.py new file mode 100644 index 00000000..6ce63278 --- /dev/null +++ b/scripts/reconcile_experiment.py @@ -0,0 +1,126 @@ +"""Inject faults on both sides, dry-run, apply, and repeat. Fresh and in-use rows survive.""" + +import hashlib +import json +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check # noqa: E402 + +FIXTURE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def admin(*args) -> dict: + raw = subprocess.check_output( + ["docker", "exec", "mtusi-api-1", "python", "-m", "app.admin", *args], + text=True, + ) + return json.loads(raw) + + +def redact(report: dict) -> dict: + cloned = json.loads(json.dumps(report)) + for action in cloned.get("actions", []): + action.pop("object_key", None) + return cloned + + +def main() -> None: + client = Client("http://127.0.0.1:8080", timeout=30) + alice = client.user("rec_alice") + bob = client.user("rec_bob") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Сверка", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + fresh = client.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=kept.png", + 201, + actor=alice, + raw=FIXTURE, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + client.message(alice, chat, "", attachments=[fresh["id"]]) + loose = client.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=loose.png", + 201, + actor=alice, + raw=FIXTURE, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + faults = admin("inject", "--chat-id", chat["id"], "--owner-id", alice["id"]) + dry = admin("reconcile", "--dry-run") + notes = {item.get("note") for item in dry["actions"]} + check("old-orphan" in notes, "dry-run missed the old orphan") + check("fault-stale-missing" in notes, "dry-run missed stale metadata") + check("fresh-orphan" not in notes, "dry-run wanted to delete a fresh orphan") + check("fault-in-use-missing" not in notes, "dry-run wanted to delete an in-use attachment") + check(dry["skipped"].get("fresh_orphan", 0) >= 1, "fresh orphan was not skipped") + check(dry["skipped"].get("in_use_missing", 0) >= 1, "in-use missing attachment was not skipped") + applied = admin("reconcile", "--apply") + again = admin("reconcile", "--dry-run") + check(again["action_count"] == 0, "second dry-run was not a no-op") + old_object = admin("probe", "--object-key", faults["old_orphan"]["object_key"]) + fresh_object = admin("probe", "--object-key", faults["fresh_orphan"]["object_key"]) + stale_meta = admin("probe", "--attachment-id", faults["stale_missing"]["attachment_id"]) + used_meta = admin("probe", "--attachment-id", faults["in_use_missing"]["attachment_id"]) + check(old_object["object"] is False, "old orphan object survived apply") + check(fresh_object["object"] is True, "fresh orphan object was deleted") + check(stale_meta["metadata"] is False, "stale unreferenced metadata survived apply") + check(used_meta["metadata"] is True, "in-use metadata was deleted") + downloaded = client.call("GET", "/api/v1/attachments/" + fresh["id"] + "/content", actor=bob) + check(downloaded["bytes"] == FIXTURE, "fresh in-use original changed") + check(hashlib.sha256(downloaded["bytes"]).hexdigest() == fresh["sha256"], "checksum changed") + client.call("GET", "/api/v1/attachments/" + loose["id"], actor=bob, schema="Attachment") + client.call("GET", "/api/v1/attachments/" + faults["in_use_missing"]["attachment_id"], actor=bob, schema="Attachment") + missing_bytes = client.call( + "GET", + "/api/v1/attachments/" + faults["in_use_missing"]["attachment_id"] + "/content", + 404, + actor=bob, + ) + check(missing_bytes["json"]["error"]["code"] == "not_found", "missing in-use bytes were not 404") + client.call("GET", "/api/v1/attachments/" + faults["stale_missing"]["attachment_id"], 404, actor=alice) + report = { + "hypothesis": "A full-scan reconciler with a 120s grace period removes only expired orphans and unreferenced metadata whose object is already gone.", + "method": "Inject a fresh orphan, a 2h orphan, a 2h missing unreferenced row, and a 2h missing row referenced by a message. Dry-run, apply, dry-run again.", + "dry_run": redact(dry), + "apply": redact(applied), + "second_dry_run": redact(again), + "survived": { + "fresh_orphan_object": True, + "in_use_metadata": True, + "fresh_message_attachment_sha256": fresh["sha256"], + "loose_unreferenced_attachment": loose["id"], + }, + "limitations": [ + "Walk cost is one full ListObjects plus one pass over attachment and ledger rows; this stand has one page.", + "Grace uses the ledger or metadata timestamp, not a backdated S3 LastModified, because MinIO does not let the lab rewrite LastModified.", + "In-use metadata whose bytes are gone is reported and kept; the reconciler does not invent replacement bytes.", + "A second dry-run being empty shows the repair is repeatable, not that a continuous watcher is running.", + ], + } + dest = ROOT / "evidence" / "reconcile.json" + dest.parent.mkdir(exist_ok=True) + dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({"action_count": dry["action_count"], "second": again["action_count"], "duration_ms": dry["duration_ms"], "objects_seen": dry["objects_seen"], "list_pages": dry["list_pages"]}, ensure_ascii=False)) + print("PASS: reconciler dry-run/apply/repeat kept fresh and in-use attachments") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print("FAIL:", exc) + raise SystemExit(1) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..e7c06621 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Restart API and every store used by the lab. Volumes are preserved. +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env up -d --no-recreate --no-build postgres minio api-1 api-2 lb +docker compose --env-file .env restart postgres minio api-1 api-2 lb diff --git a/scripts/s3_outage.py b/scripts/s3_outage.py new file mode 100644 index 00000000..41efe494 --- /dev/null +++ b/scripts/s3_outage.py @@ -0,0 +1,85 @@ +"""Stop MinIO, observe ready=503, start it again, and read the same bytes.""" + +import hashlib +import json +import subprocess +import time +import urllib.error +import urllib.request +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, check # noqa: E402 + +FIXTURE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def status(path: str) -> int: + try: + with urllib.request.urlopen("http://127.0.0.1:8080" + path, timeout=15) as response: + return response.status + except urllib.error.HTTPError as exc: + return exc.code + + +def wait_ready(expected: int) -> None: + deadline = time.time() + 90 + last = None + while time.time() < deadline: + try: + last = status("/health/ready") + except Exception: + last = None + if last == expected: + return + time.sleep(1) + raise SystemExit(f"ready did not become {expected}; last={last}") + + +def main() -> None: + client = Client("http://127.0.0.1:8080", timeout=20) + alice = client.user("out_alice") + bob = client.user("out_bob") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Отказ S3", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + uploaded = client.call( + "POST", + f"/api/v1/chats/{chat['id']}/attachments?filename=outage.png", + 201, + actor=alice, + raw=FIXTURE, + headers={"Content-Type": "image/png"}, + schema="Attachment", + )["json"] + subprocess.check_call(["docker", "compose", "--env-file", ".env", "stop", "-t", "2", "minio"], cwd=ROOT) + time.sleep(2) + live = status("/health/live") + ready = status("/health/ready") + check(live == 200, "liveness failed while S3 was stopped") + check(ready == 503, "readiness stayed 200 while S3 was stopped") + subprocess.check_call(["docker", "compose", "--env-file", ".env", "start", "minio"], cwd=ROOT) + wait_ready(200) + downloaded = client.call("GET", "/api/v1/attachments/" + uploaded["id"] + "/content", actor=bob) + check(downloaded["bytes"] == FIXTURE, "original changed after the S3 restart") + check(hashlib.sha256(downloaded["bytes"]).hexdigest() == uploaded["sha256"], "checksum changed after S3 restart") + report = {"live_while_s3_down": live, "ready_while_s3_down": ready, "download_after_start": "match"} + dest = ROOT / "evidence" / "s3_outage.json" + dest.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False)) + print("PASS: S3 outage fails readiness and preserves the original") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print("FAIL:", exc) + raise SystemExit(1) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..cbb79a41 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/bin/sh +# Stop exactly one API replica and leave it stopped. Do not touch volumes or stores. +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?}" +case "$TARGET_INSTANCE_ID" in + api-1|api-2) ;; + *) + echo "unexpected instance" >&2 + exit 2 + ;; +esac +docker compose --env-file .env stop -t 2 "$TARGET_INSTANCE_ID" diff --git a/tests_student/idempotency_race.py b/tests_student/idempotency_race.py new file mode 100644 index 00000000..c3012486 --- /dev/null +++ b/tests_student/idempotency_race.py @@ -0,0 +1,50 @@ +"""Eight parallel retries with one key must create one message.""" + +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 + + +def main() -> None: + client = Client("http://127.0.0.1:8080", timeout=20) + alice = client.user("race_alice") + bob = client.user("race_bob") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Гонка", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + key = "race-" + alice["id"][:8] + text = "одно сообщение " + key + + def once(_index): + local = Client("http://127.0.0.1:8080", timeout=20) + return local.message(alice, chat, text, key=key)["json"] + + with ThreadPoolExecutor(max_workers=8) as pool: + bodies = list(pool.map(once, range(8))) + check(len({item["id"] for item in bodies}) == 1, "parallel retries created more than one message") + check(all(item == bodies[0] for item in bodies), "parallel retries returned different messages") + seen = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=100", + actor=bob, + schema="MessagePage", + )["json"]["items"] + check(sum(1 for item in seen if item["text"] == text) == 1, "history contains duplicate race messages") + print("PASS: eight parallel idempotent retries created one message") + + +if __name__ == "__main__": + try: + main() + except (Failure, Exception) as exc: + print("FAIL:", exc) + raise SystemExit(1) diff --git a/tests_student/negative.py b/tests_student/negative.py new file mode 100644 index 00000000..30b7649f --- /dev/null +++ b/tests_student/negative.py @@ -0,0 +1,171 @@ +"""Grade-4 negative cases. User mistakes must stay 4xx, never 500.""" + +import base64 +import hashlib +import json +import struct +import subprocess +import sys +import uuid +import zlib +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 + +FIXTURE = (ROOT / "tests" / "fixtures" / "sample.png").read_bytes() + + +def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", crc) + + +def pixel_bomb() -> bytes: + ihdr = struct.pack(">IIBBBBB", 20000, 20000, 8, 2, 0, 0, 0) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IEND", b"") + + +def jpeg_bytes() -> bytes: + raw = subprocess.check_output( + [ + "docker", + "exec", + "mtusi-api-1", + "python", + "-c", + "import io,base64; from PIL import Image; b=io.BytesIO(); " + "Image.new('RGB',(8,8),(12,34,56)).save(b,format='JPEG',quality=85); " + "print(base64.b64encode(b.getvalue()).decode())", + ], + text=True, + ) + return base64.b64decode(raw.strip()) + + +def main() -> None: + client = Client("http://127.0.0.1:8080", timeout=60) + client.call("GET", "/health/live", schema="Health") + alice = client.user("n_alice") + bob = client.user("n_bob") + eve = client.user("n_eve") + chat = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Негативные", "member_ids": [alice["id"], bob["id"]]}, + schema="Chat", + )["json"] + other = client.call( + "POST", + "/api/v1/chats", + 201, + actor=alice, + body={"title": "Чужой", "member_ids": [alice["id"], eve["id"]]}, + schema="Chat", + )["json"] + + def upload(actor, chat_id, payload, filename, content_type, expected): + path = f"/api/v1/chats/{chat_id}/attachments?filename=" + quote(filename, safe="") + schema = "Attachment" if expected == 201 else None + return client.call( + "POST", + path, + expected, + actor=actor, + raw=payload, + headers={"Content-Type": content_type}, + schema=schema, + ) + + jpeg = jpeg_bytes() + created = upload(alice, chat["id"], jpeg, "фото.jpg", "image/jpeg", 201)["json"] + check(created["content_type"] == "image/jpeg", "jpeg content type was not stored") + check(created["sha256"] == hashlib.sha256(jpeg).hexdigest(), "jpeg checksum mismatch") + downloaded = client.call("GET", "/api/v1/attachments/" + created["id"] + "/content", actor=bob) + check(downloaded["bytes"] == jpeg, "jpeg download mismatch") + check(downloaded["headers"].get_content_type() == "image/jpeg", "jpeg response type mismatch") + + big = upload(alice, chat["id"], b"\x89PNG\r\n\x1a\n" + b"0" * (10 * 1024 * 1024), "big.png", "image/png", 413) + check(big["json"]["error"]["code"] == "payload_too_large", "oversize code") + bomb = upload(alice, chat["id"], pixel_bomb(), "bomb.png", "image/png", 413) + check(bomb["json"]["error"]["code"] == "payload_too_large", "pixel bomb code") + corrupt = upload(alice, chat["id"], FIXTURE[:-80], "broken.png", "image/png", 415) + check(corrupt["json"]["error"]["code"] == "unsupported_media_type", "corrupt code") + spoof = upload(alice, chat["id"], FIXTURE, "spoof.jpg", "image/jpeg", 415) + check(spoof["json"]["error"]["code"] == "unsupported_media_type", "mime spoof code") + garbage = upload(alice, chat["id"], b"not an image", "note.png", "image/png", 415) + check(garbage["json"]["error"]["code"] == "unsupported_media_type", "garbage code") + plain = upload(alice, chat["id"], FIXTURE, "note.png", "text/plain", 415) + check(plain["json"]["error"]["code"] == "unsupported_media_type", "declared text code") + + for name in ("../secret.png", "a/b.png", "a\\b.png", "bad\nname.png", "bad\x00name.png", " ", "."): + refused = upload(alice, chat["id"], FIXTURE, name, "image/png", 400) + check(refused["json"]["error"]["code"] == "invalid_request", "dangerous filename was not 400") + + missing = client.call("GET", "/api/v1/attachments/" + str(uuid.uuid4()), 404, actor=alice) + check(missing["json"]["error"]["code"] == "not_found", "random attachment code") + foreign = upload(alice, other["id"], FIXTURE, "other.png", "image/png", 201)["json"] + attached = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 404, + actor=alice, + body={"text": "чужое вложение", "attachment_ids": [foreign["id"]]}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + check(attached["json"]["error"]["code"] == "not_found", "cross-chat attachment code") + hidden = client.call("GET", "/api/v1/attachments/" + created["id"], 404, actor=eve) + check(hidden["json"]["error"]["code"] == "not_found", "outsider metadata code") + hidden_bytes = client.call("GET", "/api/v1/attachments/" + created["id"] + "/content", 404, actor=eve) + check(hidden_bytes["json"]["error"]["code"] == "not_found", "outsider bytes code") + + client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=alice, + body={"text": " "}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 400, + actor=alice, + body={"text": "привет", "extra": 1}, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ) + client.call("POST", "/api/v1/users", 400, body={"username": "ab", "display_name": "Имя", "password": "short-pass"}) + client.call( + "POST", + "/api/v1/chats", + 400, + actor=alice, + body={"title": "Дубли", "member_ids": [alice["id"], alice["id"]]}, + ) + page = client.call( + "GET", + "/api/v1/chats/" + chat["id"] + "/messages?limit=1", + actor=alice, + schema="MessagePage", + )["json"] + if page["next_cursor"]: + client.call( + "GET", + "/api/v1/chats/" + other["id"] + "/messages?limit=1&cursor=" + quote(page["next_cursor"], safe=""), + 400, + actor=alice, + ) + print("PASS: negative attachment and validation cases stayed in 4xx") + + +if __name__ == "__main__": + try: + main() + except (Failure, Exception) as exc: + print("FAIL:", exc) + raise SystemExit(1) diff --git a/tests_student/private_bucket.py b/tests_student/private_bucket.py new file mode 100644 index 00000000..9c2420e0 --- /dev/null +++ b/tests_student/private_bucket.py @@ -0,0 +1,38 @@ +"""Anonymous S3 access is denied, and the console/API ports are not published.""" + +import json +import subprocess +import sys + + +def run(args): + return subprocess.check_output(args, text=True) + + +def main() -> None: + report = json.loads(run(["docker", "exec", "mtusi-api-1", "python", "-m", "app.admin", "check-private"])) + if not report.get("private"): + raise SystemExit("anonymous S3 access was not denied: " + json.dumps(report)) + ports = json.loads(run(["docker", "inspect", "mtusi-minio", "--format", "{{json .NetworkSettings.Ports}}"])) + published = [] + for bindings in (ports or {}).values(): + if bindings: + published.extend(bindings) + if published: + raise SystemExit("MinIO published a host port, including the console if 9001 is mapped") + mounts = json.loads(run(["docker", "inspect", "mtusi-api-1", "--format", "{{json .Mounts}}"])) + if mounts: + raise SystemExit("API replica has a volume; originals must not live on the API filesystem") + env = run(["docker", "exec", "mtusi-api-1", "python", "-c", "import os; print('MINIO_ROOT_USER' in os.environ)"]) + if env.strip() != "False": + raise SystemExit("API process can see the MinIO root user") + print(json.dumps({"private_bucket": report, "minio_published_ports": published, "api_mounts": mounts}, ensure_ascii=False)) + print("PASS: private bucket, no published MinIO ports, API has no volume and no root credentials") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print("FAIL:", exc) + raise SystemExit(1)