diff --git a/.env.example b/.env.example index d99d26a2..8dba1057 100644 --- a/.env.example +++ b/.env.example @@ -2,11 +2,15 @@ # These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 +# Host port published by the LB. Default 8080; change if that port is already taken. +LB_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +# compose.yaml gives api/migrate DB_HOST/DB_USER/DB_NAME and +# DB_PASSWORD_FILE=/run/secrets/postgres_password (secret from POSTGRES_PASSWORD). +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-locally +POSTGRES_DB=messenger SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 @@ -14,7 +18,17 @@ IDEMPOTENCY_TTL_SECONDS=86400 S3_ENDPOINT=http://s3:9000 S3_REGION=us-east-1 S3_BUCKET=messenger-attachments +# compose.yaml gives minio MINIO_ROOT_USER_FILE / MINIO_ROOT_PASSWORD_FILE +# (secrets from MINIO_ROOT_USER / MINIO_ROOT_PASSWORD). Root is bootstrap-only. +# The API user is created by minio-init and limited to S3_BUCKET. +# App containers should read the files below, not these values as env vars. +MINIO_ROOT_USER=messenger-root +MINIO_ROOT_PASSWORD=change-me-minio-root +S3_ACCESS_KEY_ID=messenger-api +S3_SECRET_ACCESS_KEY=change-me-minio-api S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +# Reconciler does not delete originals younger than this (seconds). Overridable per run with --grace-seconds. +RECONCILE_GRACE_SECONDS=900 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..dfdb8b77 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..7b306b09 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# Закреплённая среда выполнения. Тег называет версию, digest — то, что Docker реально скачает. +FROM python:3.14.8-slim-bookworm@sha256:c8137f4c460908c8763f281c8f22c431eb5c538514ba9553fc3a89c06b7cfb88 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +# Значения по умолчанию, секретов нет. `docker run --env-file .env` перекрывает их при запуске. +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + LOG_LEVEL=info \ + DRAIN_SECONDS=4 \ + SHUTDOWN_TIMEOUT_SECONDS=3 \ + SESSION_TTL_SECONDS=3600 + +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid app --create-home --shell /usr/sbin/nologin app + +WORKDIR /app + +COPY requirements.txt . +RUN python -m pip install --no-cache-dir --disable-pip-version-check --root-user-action=ignore -r requirements.txt + +COPY --chown=app:app app ./app +COPY --chown=app:app migrations ./migrations +COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json + +USER app + +# Visible as X-App-Version: shows which build served a request during a rolling update. +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ['APP_PORT'], timeout=2)"] + +STOPSIGNAL SIGTERM +CMD ["python", "-m", "app"] diff --git a/Makefile b/Makefile index 5d86d171..7beec481 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,12 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-persistence test-failover test-attachments check-private-bucket upload-mem reconcile-track2 help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "test-attachments: student negative cases; check-private-bucket: MinIO ACL; upload-mem: 10 concurrent uploads" + @echo "reconcile-track2: inject both drifts, then time the walk at 10/100/400 extra objects" check: $(PYTHON) scripts/check_contract.py @@ -28,3 +30,17 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachments: + $(PYTHON) tests/attachments_negative.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +check-private-bucket: + $(PYTHON) scripts/check_private_bucket.py + +upload-mem: + $(PYTHON) scripts/upload_mem.py --base-url "$(BASE_URL)" --output evidence/upload-concurrency.json + +reconcile-track2: + docker compose --env-file .env --profile ops build reconcile + docker compose --env-file .env --profile ops run --rm -T reconcile --experiment > evidence/reconcile-track2.json + docker compose --env-file .env --profile ops run --rm -T reconcile --bench 10,100,400 > evidence/reconcile-walk.json diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..1aa18106 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,111 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Тимаева М.П., БПИ2302 +- Целевая оценка; выбранный трек 5, если нужен: 5, трек 2 +- Commit/PR: TODO +- Использованные источники, библиотеки и помощь ИИ: OpenAPI/семантика курса; документация MinIO, boto3, HAProxy 3.2; FastAPI/Starlette; Cursor помогал переносить старый код в новый репозиторий и с составление отчетов ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Docker Compose v2, Python 3.9+ для публичных тестов, свободный порт для LB. + +```sh +cp .env.example .env +# Задайте уникальные POSTGRES_PASSWORD, MINIO_ROOT_PASSWORD, S3_SECRET_ACCESS_KEY. +# Если 8080 занят (другая лабораторная), задайте LB_PORT, например 8084. +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождаться healthy у lb/api. Миграции и minio-init — one-shot. +make test BASE_URL=http://localhost:${LB_PORT:-8080} +make test-attachments BASE_URL=http://localhost:${LB_PORT:-8080} +make check-private-bucket +make reconcile-track2 +# Остановка с сохранением volumes: +docker compose --env-file .env down +# Учебный сброс данных (уничтожает pgdata и miniodata): +docker compose --env-file .env down -v +``` + +Образы (закреплены тегом+digest в `compose.yaml` / `Dockerfile`): + +| Роль | Service | Образ | Volume | Сеть | Порт на host | Credentials | +| ------------- | ------------------------------------ | ----------------------------------------- | -------------- | ---------------- | ------------------------- | --------------------------------- | +| LB | lb | haproxy:3.2.26-alpine@sha256:96ec… | — | edge | 127.0.0.1 / ::1 `LB_PORT` | нет | +| API | api ×2 | messenger-lab4-app (python:3.14.8-slim) | нет оригиналов | edge+data | нет | DB и S3 из secret files uid 10001 | +| PostgreSQL | postgres | postgres:16.10-alpine@sha256:0296… | pgdata | data (internal) | нет | POSTGRES_PASSWORD_FILE | +| S3 | minio | minio RELEASE.2025-09-07…@sha256:14ce… | miniodata | data, alias `s3` | нет | root только у minio-init | +| Bootstrap | minio-init | тот же MinIO + `infra/minio/bootstrap.sh` | — | data | — | root + API keys | +| Сверка | reconcile (profile `ops`) | тот же app image | — | data | — | те же API S3 keys | +| Private probe | check-private-bucket (profile `ops`) | тот же MinIO | — | data | — | root + API keys | + +Локально на этой машине `LB_PORT=8084`, потому что 8080 занят стендом lab3. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HAProxy → API ×2 → PostgreSQL (метаданные) + приватный bucket MinIO (байты). Object key назначает сервер: `originals/{uuid}`. Filename только для отображения. + +Источник истины для «файл принят»: строка в `attachments` **и** объект в S3. `201 ready` возвращается только после PUT и INSERT. Неудачный INSERT компенсируется `delete_object`. Обрыв клиента до PUT не создаёт объект. Crash между PUT и INSERT оставляет orphan; его забирает сверка после grace period (по умолчанию 900 с). + +Сверка сравнивает `ListObjectsV2` по `originals/` со строками `attachments`. Dry-run по умолчанию. `--apply` удаляет объект без строки и строку без объекта, только если каждый не моложе grace (возраст меньше grace не трогаем) и строка не входит в `message_attachments`. Перед удалением объекта ключ читается из PostgreSQL ещё раз; строка удаляется одним `DELETE` с `NOT EXISTS` по `message_attachments` и `object_key LIKE 'originals/%'`. Ключ вне `originals/` только попадает в `ignored_outside_prefix`. Строка, на которую ссылается сообщение, не удаляется: объекта из метаданных не восстановить, а FK без `ON DELETE CASCADE` порвал бы историю. One-shot `reconcile` ограничен 256 MiB: эксперимент создаёт двух пользователей и считает Argon2id. + +Upload не держит весь файл в Python heap: чанки считаются, SHA-256 копится на лету, заголовок PNG/JPEG смотрится без распаковки bitmap (`inspect_prefix`), тело уходит в `SpooledTemporaryFile` (64 KiB RAM, остальное ephemeral tempfile) и streaming `put_object`. Лимит 10 MiB применяется по `Content-Length` и по счётчику чанков; HAProxy отсекает `Content-Length > 10485760` до API. PUT S3 ограничен 30 с; в backend HAProxy `set-timeout server 30s` только для `POST .../attachments`. + +Перенос из lab3: сессии, идемпотентность сообщений, две реплики, LB. Новое: S3, Attachment, сверка, негативные тесты файлов. + +Трек 2 поверх этого: те же prefix, dry-run и proxy `POST`, плюс исправление обеих сторон и замер обхода (`make reconcile-track2`). ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| ------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| Публичный smoke | `python tests/smoke.py --base-url http://localhost:8084` | PASS lab4 public API | [evidence/smoke.txt](evidence/smoke.txt) | +| S3 в стенде, байты не в БД/диске API | compose: minio без host ports, API без volume оригиналов, ключ `originals/{uuid}` | minio ports `[]`, API не публикует 9000 | [evidence/compose-minio-exposure.json](evidence/compose-minio-exposure.json) | +| PNG/JPEG ≤10 MiB / 40M px, MIME по содержимому | smoke fixture + negative suite | 201 на sample.png; 413/415 на отказ | [evidence/attachments-negative.txt](evidence/attachments-negative.txt) | +| Другая реплика / restart volumes | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` и `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`, BASE_URL `http://127.0.0.1:8084` | persistence: PASS, тот же токен и тот же оригинал; id контейнеров не сменились, StartedAt обновился у postgres, minio, обеих API и LB. failover: 2 реплики до отказа, 128 успешных записей, 0 ошибок, восстановление за 5.699 с; api-2 осталась exited, postgres/minio/lb/api-1 не перезапускались | [evidence/persistence.txt](evidence/persistence.txt), [evidence/failover.txt](evidence/failover.txt) | +| Негативные вложения, не 500 | `make test-attachments BASE_URL=http://localhost:8084` | oversized CL и chunked → 413; corrupt/MIME spoof → 415; pixel bomb → 413; чужой id и чужой чат → 404 | [evidence/attachments-negative.txt](evidence/attachments-negative.txt) | +| Компенсация / сверка / grace | `docker compose --profile ops run --rm -T reconcile --seed-orphan`, затем `--apply --grace-seconds 0` | dry-run, grace 900: посеянный ключ в `orphans_protected_by_grace`, `deleted` пуст. apply, grace 0: удалён только `originals/ecc25a63-0271-4944-bd3a-d9c83c78c4ab`. Строка сообщения без объекта осталась в `missing_in_use`, ключи вне `originals/` — в `ignored_outside_prefix` | [evidence/reconcile-dry-run.json](evidence/reconcile-dry-run.json), [evidence/reconcile-apply.json](evidence/reconcile-apply.json) | +| RAM, ≥10 одновременных upload | `python scripts/upload_mem.py --base-url http://localhost:8084` | 10/10 ок, ~1.9 MiB PNG, пик RSS api 93.66 MiB при лимите 256M (N×size ≈ 19 MiB) | [evidence/upload-concurrency.json](evidence/upload-concurrency.json) | +| Идемпотентный bootstrap | два `docker compose --env-file .env run --rm --no-deps minio-init` | оба: `bucket messenger-attachments is private`, exit 0 | [evidence/minio-init-rerun.txt](evidence/minio-init-rerun.txt) | +| Узкие S3 creds, консоль закрыта, private bucket вне API | `python scripts/check_private_bucket.py` | anonymous GET 403; API user не листит чужой bucket; MINIO_BROWSER=off; нет host ports; не в edge | [evidence/private-bucket.txt](evidence/private-bucket.txt) | +| Трек 2: обе стороны, dry-run, повтор, защита | `make reconcile-track2` | `ok: true`. Grace 900 не удалил свежий orphan и свежую непривязанную строку; привязанная строка без объекта осталась. После сдвига `created_at` на 2 ч `--apply` удалил только её, второй прогон пуст. Grace 0 после preflight удалил только посеянный orphan; повтор снова пуст. SHA-256 здорового объекта совпал со строкой | [evidence/reconcile-track2.json](evidence/reconcile-track2.json) | +| Трек 2: стоимость обхода | тот же прогон, `--bench 10,100,400` | 51 ключ: list объектов 13 ms; 141 ключ: 180 ms; 441 ключ: 75 ms. Метаданные 1–5 ms. Свои ключи бенча удалены, в `--apply` не передавались | [evidence/reconcile-walk.json](evidence/reconcile-walk.json) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза (RAM): десять одновременных ~1.9 MiB PNG не разгоняют RSS API пропорционально `N * size` и укладываются в 256 MiB. Стенд: Windows AMD64, Intel, compose limits api 256M × 2 реплики, MinIO 512M. Профиль: 10 POST, файл 1 994 104 байт, сгенерирован в рантайме. Результат: 10 успехов, 0 ошибок, p50 ≈ 0.52 с, p95 ≈ 0.55 с, пик RSS контейнера api-2 **93.66 MiB**. Ограничение: `docker stats` — RSS контейнера с шагом десятых долей секунды, не профилировщик кучи; в выборке 3 снимка, пик внутри запроса мог быть выше. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +Гипотеза (трек 2): один `ListObjectsV2` по `originals/` и один `SELECT` по `attachments` остаются в пределах секунды на сотнях ключей и растут примерно линейно; `--apply` удаляет только устаревший orphan и устаревшую непривязанную строку без объекта, повторный прогон пуст. Методика: `make reconcile-track2` на том же стенде. Сценарий сам создаёт чат, здоровое вложение в сообщении, привязанное вложение без объекта, непривязанное вложение без объекта и один orphan. Результат безопасности: `ok: true` в [evidence/reconcile-track2.json](evidence/reconcile-track2.json). Свежие (age 0, grace 900) не удалены; после `created_at` минус 2 часа удалена только непривязанная строка (`age_seconds` 7200); привязанная строка осталась в `missing_in_use`; grace 0 разрешён preflight-ом, потому что в `orphans_stale` был только посеянный ключ, и удалён только он. Повтор обоих apply вернул пустые `deleted` и `metadata_deleted`. Байты здорового объекта совпали с `sha256` строки. + +Результат обхода: [evidence/reconcile-walk.json](evidence/reconcile-walk.json). К уже лежавшим ~41 ключам добавлялись 10, 100 и 400 объектов, каждый набор удалялся по списку своих ключей. + +| Посеяно | Ключей в list | list объектов, ms | list метаданных, ms | весь проход, ms | +| ------- | ------------: | ----------------: | ------------------: | --------------: | +| 10 | 51 | 13 | 5 | 24 | +| 100 | 141 | 180 | 1 | 188 | +| 400 | 441 | 75 | 2 | 82 | + +Верхняя граница подтвердилась: все три прохода короче 200 ms. Линейный рост на этих трёх точках не подтвердился: 441 ключ обошёлся дешевле 141. `SELECT` оставался в единицах миллисекунд. Ограничение замера: один прогон, время boto/MinIO внутри compose, `elapsed_ms` включает ещё и `ListMultipartUploads`. Это не стоимость обхода миллиона ключей. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| -------------------------------------------------------------- | ------------------------------------------ | -------------------------------------------------------------- | --------------------------------------------- | +| Content-Length > 10 MiB | 413 на LB, тело не читается | 413 `payload_too_large` | нет записи | +| Chunked тело > 10 MiB | 413 на API по счётчику чанков | 413 `payload_too_large` | нет объекта | +| Повреждённый PNG/JPEG, подмена MIME | 415 | 415 `unsupported_media_type` | нет объекта | +| IHDR 10000×5000 (50M px) | 413 до decode | 413 | нет объекта | +| GET чужого attachment_id | 404, без утечки | 404 `not_found` | — | +| attachment_id другого чата в POST message | 404 | 404 | сообщение не создано | +| S3 PUT прошёл, INSERT нет | компенсация DELETE; иначе orphan до сверки | `_discard_object`; сирота удаляется `--apply` после grace | объект без метаданных не вечный | +| Объект есть, строки нет, объект моложе grace | не удалять | age 0 остался в `orphans_protected_by_grace` при `--apply` | объект на месте | +| Строка есть, объекта нет, строка в сообщении | не удалять даже при grace 0 | осталась в `missing_in_use`; `metadata_deleted` её не содержит | строка на месте, байтов нет | +| Строка есть, объекта нет, строка старше grace и не в сообщении | удалить строку; повтор пуст | удалена одна строка с age 7200; второй `--apply` пуст | строка снята | +| Grace 0 при чужом orphan | не удалять ничего | preflight сравнивает `orphans_stale` с посеянным ключом | в этом прогоне чужих не было, удаление прошло | +| Повтор minio-init | no-op, bucket private | два exit 0 | нет | +| Unauthenticated GET объекта | 403 мимо API | 403 | — | + +Две старые строки с ключами без prefix `originals/` на каждом проходе остаются в `ignored_outside_prefix` и не удаляются. Живая пара «строка + объект» в списки расхождений не попадает. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Выдерживаются клиентские ошибки вложений (4xx, не 500), обрыв до PUT, неудачный INSERT, stale orphan после grace, недоступность bucket с хоста, повтор bootstrap. Не выдерживается потеря всего Docker-host; одна PostgreSQL и один MinIO не HA. Download оригинала пока буферизуется целиком (критерий 4 про загрузку). Spool — ephemeral tempfile запроса, не источник истины. + +Сверка не чинит ключи вне `originals/` и не восстанавливает байты пропавшего объекта: привязанная строка без объекта остаётся навсегда, `GET .../content` для неё даёт 404. Grace 0 для orphan — подмена часов, потому что MinIO не даёт сдвинуть `LastModified`; если в бакете уже есть другой orphan, эксперимент выходит с ошибкой и этот `--apply` не запускает. Один MinIO и одна PostgreSQL. Presigned upload и фон lab5 не делались. Prefix `originals/` по-прежнему не трогает будущий `staging/` или `variants/`. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/__main__.py b/app/__main__.py new file mode 100644 index 00000000..5396aabb --- /dev/null +++ b/app/__main__.py @@ -0,0 +1,49 @@ +import logging +import threading + +import uvicorn + +from .config import load_settings +from .drain import draining + +logger = logging.getLogger('messenger') + + +class DrainingServer(uvicorn.Server): + """First SIGTERM/SIGINT: keep serving for drain_seconds with readiness 503, then uvicorn's graceful stop. + + drain_seconds must exceed the LB detection time (HAProxy inter 1s x fall 2 + timeout check 1s), + so the replica is out of rotation before it stops accepting connections. + """ + + def __init__(self, config: uvicorn.Config, drain_seconds: float): + super().__init__(config) + self.drain_seconds = drain_seconds + + def handle_exit(self, sig, frame) -> None: + if draining.is_set() or self.drain_seconds <= 0: + draining.set() + super().handle_exit(sig, frame) + return + draining.set() + logger.warning('signal %s: draining for %.1f s before shutdown', sig, self.drain_seconds) + timer = threading.Timer(self.drain_seconds, super().handle_exit, (sig, frame)) + timer.daemon = True + timer.start() + + +def main() -> None: + settings = load_settings() + config = uvicorn.Config( + 'app.main:app', + host=settings.host, + port=settings.port, + log_level=settings.log_level, + timeout_graceful_shutdown=settings.shutdown_timeout, + timeout_keep_alive=settings.keepalive_timeout, + ) + DrainingServer(config, settings.drain_seconds).run() + + +if __name__ == '__main__': + main() diff --git a/app/auth.py b/app/auth.py new file mode 100644 index 00000000..1006e055 --- /dev/null +++ b/app/auth.py @@ -0,0 +1,30 @@ +"""Basic is only for creating a session. Every other route accepts only Bearer.""" +import base64 +import binascii +from typing import Optional, Tuple + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def parse_basic(header: str) -> Optional[Tuple[str, str]]: + """username:password. The password may itself contain a colon.""" + scheme, separator, param = header.partition(' ') + if separator != ' ' or scheme.lower() != 'basic' or not param or param != param.strip(): + return None + try: + decoded = base64.b64decode(param, validate=True) + text = decoded.decode('utf-8') + except (binascii.Error, UnicodeDecodeError, ValueError): + return None + username, separator, password = text.partition(':') + if separator != ':': + return None + return username, password + + +def parse_bearer(header: str) -> Optional[str]: + scheme, separator, token = header.partition(' ') + if separator != ' ' or scheme.lower() != 'bearer' or not token or token != token.strip() or ' ' in token: + return None + return token diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..a71beb94 --- /dev/null +++ b/app/config.py @@ -0,0 +1,113 @@ +import os +import socket +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import quote + + +@dataclass(frozen=True) +class ObjectSettings: + endpoint: str + region: str + bucket: str + access_key: str + secret_key: str + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + env: str + log_level: str + shutdown_timeout: int + drain_seconds: float + keepalive_timeout: int + session_ttl_seconds: int + idempotency_ttl_seconds: int + database_url: str + instance_id: str + app_version: str + + +def _read_secret_file(file_var: str, file_name: str) -> str: + try: + value = Path(file_name).read_text(encoding='utf-8').strip() + except OSError: + raise RuntimeError('%s could not be read' % file_var) from None + if not value: + raise RuntimeError('%s is empty' % file_var) + return value + + +def _database_url() -> str: + """DATABASE_URL_FILE, then DATABASE_URL, then DB_* parts with the password from DB_PASSWORD_FILE. + + Secret values are never logged. + """ + file_name = os.getenv('DATABASE_URL_FILE', '').strip() + if file_name: + return _read_secret_file('DATABASE_URL_FILE', file_name) + value = os.getenv('DATABASE_URL', '').strip() + if value: + return value + password_file = os.getenv('DB_PASSWORD_FILE', '').strip() + if not password_file: + raise RuntimeError('DATABASE_URL, DATABASE_URL_FILE or DB_PASSWORD_FILE is required') + password = _read_secret_file('DB_PASSWORD_FILE', password_file) + parts = {name: os.getenv(name, '').strip() for name in ('DB_HOST', 'DB_USER', 'DB_NAME')} + missing = [name for name, part in parts.items() if not part] + if missing: + raise RuntimeError('%s required with DB_PASSWORD_FILE' % ', '.join(missing)) + return 'postgresql://%s:%s@%s:%s/%s' % ( + quote(parts['DB_USER'], safe=''), + quote(password, safe=''), + parts['DB_HOST'], + int(os.getenv('DB_PORT', '5432')), + quote(parts['DB_NAME'], safe=''), + ) + + +def load_object_settings() -> ObjectSettings: + """S3 credentials come from files. The values are not logged.""" + endpoint = os.getenv('S3_ENDPOINT', '').strip() + region = os.getenv('S3_REGION', 'us-east-1').strip() or 'us-east-1' + bucket = os.getenv('S3_BUCKET', '').strip() + key_file = os.getenv('S3_ACCESS_KEY_ID_FILE', '').strip() + secret_file = os.getenv('S3_SECRET_ACCESS_KEY_FILE', '').strip() + missing = [ + name for name, value in ( + ('S3_ENDPOINT', endpoint), + ('S3_BUCKET', bucket), + ('S3_ACCESS_KEY_ID_FILE', key_file), + ('S3_SECRET_ACCESS_KEY_FILE', secret_file), + ) if not value + ] + if missing: + raise RuntimeError('%s is required' % ', '.join(missing)) + return ObjectSettings( + endpoint, + region, + bucket, + _read_secret_file('S3_ACCESS_KEY_ID_FILE', key_file), + _read_secret_file('S3_SECRET_ACCESS_KEY_FILE', secret_file), + ) + + +def load_settings() -> Settings: + return Settings( + host=os.getenv('APP_HOST', '0.0.0.0'), + port=int(os.getenv('APP_PORT', '8080')), + env=os.getenv('APP_ENV', 'lab'), + log_level=os.getenv('LOG_LEVEL', 'info').lower(), + shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '3')), + drain_seconds=float(os.getenv('DRAIN_SECONDS', '4')), + # Longer than HAProxy's idle-connection purge (pool-purge-delay 5 s), so the LB usually closes them first. + keepalive_timeout=int(os.getenv('KEEPALIVE_TIMEOUT_SECONDS', '10')), + session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')), + idempotency_ttl_seconds=int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400')), + database_url=_database_url(), + # Docker sets the hostname to the short container ID, which stop-one.sh maps back to a container. + instance_id=os.getenv('INSTANCE_ID', '').strip() or socket.gethostname(), + app_version=os.getenv('APP_VERSION', 'dev').strip() or 'dev', + ) diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..caa8abf8 --- /dev/null +++ b/app/db.py @@ -0,0 +1,47 @@ +"""Connection pool. The URL is never written to the log.""" +import asyncio +import logging +import os + +import asyncpg + +logger = logging.getLogger('messenger') + +_ATTEMPTS = 40 + + +def _pool_size(name: str, default: int) -> int: + raw = os.getenv(name, '').strip() + return int(raw) if raw else default + + +async def open_pool(database_url: str) -> asyncpg.Pool: + delay = 0.5 + last_type = 'error' + min_size = _pool_size('DB_POOL_MIN', 1) + max_size = max(min_size, _pool_size('DB_POOL_MAX', 10)) + for attempt in range(1, _ATTEMPTS + 1): + pool = None + try: + pool = await asyncpg.create_pool( + database_url, + min_size=min_size, + max_size=max_size, + timeout=5, + # Uploads are capped at 10 MiB; give that write room past the old 5 s JSON timeout. + command_timeout=30, + server_settings={'statement_timeout': '30000'}, + ) + async with pool.acquire() as conn: + await conn.execute('SELECT 1') + return pool + except Exception as exc: + last_type = type(exc).__name__ + if pool is not None: + await pool.close() + if attempt == _ATTEMPTS: + break + logger.warning('database is not ready (%s), retrying', last_type) + await asyncio.sleep(delay) + delay = min(delay * 1.5, 2) + raise RuntimeError('database did not become reachable (%s)' % last_type) diff --git a/app/drain.py b/app/drain.py new file mode 100644 index 00000000..f23a67c3 --- /dev/null +++ b/app/drain.py @@ -0,0 +1,4 @@ +"""Set on SIGTERM: readiness turns 503 so the LB stops routing here while requests are still served.""" +import threading + +draining = threading.Event() diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..83b14be4 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,37 @@ +class ApiError(Exception): + """Expected failure: the message never contains internal details.""" + + def __init__(self, status: int, code: str, message: str, challenge: str | None = None) -> None: + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.challenge = challenge + + +def invalid_request(message: str) -> ApiError: + return ApiError(400, 'invalid_request', message) + + +def unauthorized(message: str, challenge: str) -> ApiError: + return ApiError(401, 'unauthorized', message, challenge) + + +def not_found(message: str) -> ApiError: + return ApiError(404, 'not_found', message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, 'conflict', message) + + +def payload_too_large(message: str) -> ApiError: + return ApiError(413, 'payload_too_large', message) + + +def unsupported_media_type(message: str) -> ApiError: + return ApiError(415, 'unsupported_media_type', message) + + +def unavailable(message: str) -> ApiError: + return ApiError(503, 'unavailable', message) diff --git a/app/images.py b/app/images.py new file mode 100644 index 00000000..edd9f1f2 --- /dev/null +++ b/app/images.py @@ -0,0 +1,119 @@ +"""PNG/JPEG checks that look at the header only. + +Dimensions come from IHDR or a JPEG SOF marker. The compressed pixel data is never unpacked, +so a small file cannot expand into an unbounded bitmap before the pixel limit is applied. +""" +from typing import NoReturn, Optional + +from .errors import payload_too_large, unsupported_media_type + +MAX_UPLOAD_BYTES = 10_485_760 +MAX_IMAGE_PIXELS = 40_000_000 +# SOF/IHDR almost always sit in the first tens of kilobytes. The rest of the +# object is hashed and stored without being held as one Python bytes object. +INSPECT_PREFIX = 256 * 1024 + +_PNG = b'\x89PNG\r\n\x1a\n' +# SOF markers that carry width and height. DHT/DAC/JPG are skipped by the length field. +_JPEG_SOF = {0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7, 0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF} + + +class _NeedMore(Exception): + """The prefix does not yet contain a complete PNG IHDR or JPEG SOF.""" + + +def _reject() -> NoReturn: + raise unsupported_media_type('Файл не является PNG или JPEG') + + +def _check_pixels(width: int, height: int) -> None: + if width <= 0 or height <= 0: + _reject() + if width > MAX_IMAGE_PIXELS or height > MAX_IMAGE_PIXELS or width * height > MAX_IMAGE_PIXELS: + raise payload_too_large('Изображение превышает 40 миллионов пикселей') + + +def _png_size(data: bytes) -> str: + if len(data) < 8: + if _PNG.startswith(data): + raise _NeedMore() + _reject() + if not data.startswith(_PNG): + _reject() + if len(data) < 24: + raise _NeedMore() + if int.from_bytes(data[8:12], 'big') != 13 or data[12:16] != b'IHDR': + _reject() + width = int.from_bytes(data[16:20], 'big') + height = int.from_bytes(data[20:24], 'big') + _check_pixels(width, height) + return 'image/png' + + +def _jpeg_size(data: bytes) -> str: + if len(data) < 2: + if b'\xff\xd8'.startswith(data): + raise _NeedMore() + _reject() + if data[0:2] != b'\xff\xd8': + _reject() + index = 2 + while index < len(data): + if data[index] != 0xFF: + _reject() + index += 1 + while index < len(data) and data[index] == 0xFF: + index += 1 + if index >= len(data): + raise _NeedMore() + marker = data[index] + index += 1 + if marker in (0x01, 0xD8, 0xD9) or 0xD0 <= marker <= 0xD7: + if marker == 0xD9: + break + continue + if index + 2 > len(data): + raise _NeedMore() + segment_length = int.from_bytes(data[index:index + 2], 'big') + if segment_length < 2: + _reject() + if index + segment_length > len(data): + raise _NeedMore() + # Start of scan: entropy-coded bytes follow. Dimensions must already have been seen. + if marker == 0xDA: + break + if marker in _JPEG_SOF: + if segment_length < 7: + _reject() + height = int.from_bytes(data[index + 3:index + 5], 'big') + width = int.from_bytes(data[index + 5:index + 7], 'big') + _check_pixels(width, height) + return 'image/jpeg' + index += segment_length + _reject() + + +def inspect_prefix(data: bytes, *, complete: bool) -> Optional[str]: + """Return image/png or image/jpeg, or None if `complete` is false and more bytes are required.""" + if not data: + if complete: + _reject() + return None + try: + if data[0] == 0x89: + return _png_size(data) + if data[0] == 0xFF: + return _jpeg_size(data) + _reject() + except _NeedMore: + if complete: + _reject() + return None + + +def inspect_image(data: bytes) -> str: + """Return image/png or image/jpeg after a header check. Raises 413 or 415.""" + result = inspect_prefix(data, complete=True) + if result is None: + _reject() + return result diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..63587886 --- /dev/null +++ b/app/main.py @@ -0,0 +1,126 @@ +import json +import logging +import re +import uuid +from contextlib import asynccontextmanager +from pathlib import Path + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + +from .config import load_object_settings, load_settings +from .db import open_pool +from .errors import ApiError +from .objects import ObjectStore +from .routes import router +from .store import Store + +logger = logging.getLogger('messenger') + +_SAFE_REQUEST_ID = re.compile(r'^[A-Za-z0-9._:-]{1,128}$') +_CODES = { + 400: 'invalid_request', + 401: 'unauthorized', + 404: 'not_found', + 409: 'conflict', + 413: 'payload_too_large', + 415: 'unsupported_media_type', + 429: 'rate_limited', + 503: 'unavailable', +} +_MESSAGES = { + 'invalid_request': 'Некорректный запрос', + 'unauthorized': 'Требуется идентификация', + 'not_found': 'Объект не найден', + 'internal_error': 'Внутренняя ошибка', +} +_BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def _error(status: int, code: str, message: str, request_id: str, challenge: str | None = None) -> JSONResponse: + body = {'error': {'code': code, 'message': message, 'request_id': request_id}} + headers = {'X-Request-Id': request_id} + if challenge: + headers['WWW-Authenticate'] = challenge + return JSONResponse(body, status_code=status, headers=headers) + + +def _request_id(request: Request) -> str: + return getattr(request.state, 'request_id', None) or str(uuid.uuid4()) + + +_OPENAPI_PATH = Path(__file__).resolve().parents[1] / 'contracts' / 'openapi.json' + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """Migrations are not applied here: the one-shot migrate service runs them before replicas start.""" + settings = load_settings() + pool = await open_pool(settings.database_url) + try: + app.state.store = Store( + pool, + settings.session_ttl_seconds, + settings.idempotency_ttl_seconds, + objects=ObjectStore(load_object_settings()), + ) + logger.info('instance %s started', settings.instance_id) + yield + finally: + await pool.close() + + +def create_app() -> FastAPI: + app = FastAPI( + title='MTUSI Messenger lab4', + version='1.4.0', + lifespan=lifespan, + ) + settings = load_settings() + instance_id = settings.instance_id + app_version = settings.app_version + + def openapi(): + if app.openapi_schema is None: + app.openapi_schema = json.loads(_OPENAPI_PATH.read_text(encoding='utf-8')) + return app.openapi_schema + + app.openapi = openapi + + @app.middleware('http') + async def request_context(request: Request, call_next): + """Gives every response, including unexpected failures, an X-Request-Id, X-Instance-Id and X-App-Version.""" + incoming = request.headers.get('X-Request-Id') + request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception: + logger.exception('request %s failed: %s %s', request_id, request.method, request.url.path) + response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id) + response.headers['X-Request-Id'] = request_id + response.headers['X-Instance-Id'] = instance_id + response.headers['X-App-Version'] = app_version + return response + + @app.exception_handler(ApiError) + async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse: + return _error(exc.status, exc.code, exc.message, _request_id(request), exc.challenge) + + @app.exception_handler(RequestValidationError) + async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse: + return _error(400, 'invalid_request', _MESSAGES['invalid_request'], _request_id(request)) + + @app.exception_handler(StarletteHTTPException) + async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse: + code = _CODES.get(exc.status_code, 'internal_error' if exc.status_code >= 500 else 'invalid_request') + challenge = _BEARER_CHALLENGE if exc.status_code == 401 else None + return _error(exc.status_code, code, _MESSAGES.get(code, _MESSAGES['invalid_request']), _request_id(request), challenge) + + app.include_router(router) + return app + + +app = create_app() diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..dc1b1c89 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,64 @@ +"""Apply numbered SQL files once. Run as a separate one-shot step: python -m app.migrate.""" +import asyncio +import logging +from pathlib import Path + +import asyncpg + +from .config import load_settings +from .db import open_pool + +logger = logging.getLogger('messenger') + +_MIGRATIONS = Path(__file__).resolve().parents[1] / 'migrations' +# Any constant shared by all runners; serializes concurrent migrators across processes. +_LOCK_ID = 4_2026_0004 + +_BOOTSTRAP = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +) +""" + + +def expected_versions() -> list[str]: + return sorted(path.name for path in _MIGRATIONS.glob('*.sql')) + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + async with pool.acquire() as conn: + await conn.execute('SET statement_timeout = 0') + await conn.execute('SELECT pg_advisory_lock($1)', _LOCK_ID) + try: + await conn.execute(_BOOTSTRAP) + rows = await conn.fetch('SELECT version FROM schema_migrations') + applied = {row['version'] for row in rows} + for version in expected_versions(): + if version in applied: + logger.info('migration %s already applied', version) + continue + sql = (_MIGRATIONS / version).read_text(encoding='utf-8') + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + 'INSERT INTO schema_migrations (version) VALUES ($1)', + version, + ) + logger.info('applied migration %s', version) + finally: + await conn.execute('SELECT pg_advisory_unlock($1)', _LOCK_ID) + + +async def _main() -> None: + settings = load_settings() + pool = await open_pool(settings.database_url) + try: + await apply_migrations(pool) + finally: + await pool.close() + + +if __name__ == '__main__': + logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s') + asyncio.run(_main()) diff --git a/app/objects.py b/app/objects.py new file mode 100644 index 00000000..762c1161 --- /dev/null +++ b/app/objects.py @@ -0,0 +1,206 @@ +"""Private S3 bucket. Callers pass a server-generated key; the display filename never becomes one.""" +from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import TimeoutError as FutureTimeout +from dataclasses import dataclass +from datetime import datetime +from typing import BinaryIO, List + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from .config import ObjectSettings + +ORIGINAL_PREFIX = 'originals/' + +# Docker DNS can spend several seconds on a stopped MinIO name. The HTTP handler +# must still answer inside HAProxy's one-second health check. +_POOL = ThreadPoolExecutor(max_workers=32, thread_name_prefix='s3') + + +class MissingObject(Exception): + """The key is not in the bucket.""" + + +class StorageError(Exception): + """The object store could not complete the call. The message stays internal.""" + + +@dataclass(frozen=True) +class StoredObject: + key: str + last_modified: datetime + size: int + + +@dataclass(frozen=True) +class MultipartUpload: + key: str + upload_id: str + initiated: datetime + + +class ObjectStore: + def __init__(self, settings: ObjectSettings) -> None: + self._bucket = settings.bucket + self._client = boto3.client( + 's3', + endpoint_url=settings.endpoint, + region_name=settings.region, + aws_access_key_id=settings.access_key, + aws_secret_access_key=settings.secret_key, + config=Config( + signature_version='s3v4', + s3={'addressing_style': 'path'}, + connect_timeout=1, + read_timeout=30, + retries={'max_attempts': 1}, + ), + ) + + def ready(self) -> None: + self._bounded(self._head, 0.7) + + def put(self, key: str, body: bytes, content_type: str) -> None: + self._bounded(lambda: self._put_bytes(key, body, content_type), 30) + + def put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None: + self._bounded(lambda: self._put_stream(key, body, size, content_type), 30) + + def get(self, key: str) -> bytes: + return self._bounded(lambda: self._get(key), 15) + + def delete(self, key: str) -> None: + self._bounded(lambda: self._delete(key), 8) + + def list_originals(self, timeout: float = 15) -> List[StoredObject]: + return self._bounded(self._list_originals, timeout) + + def list_multipart(self) -> List[MultipartUpload]: + return self._bounded(self._list_multipart, 15) + + def abort_multipart(self, key: str, upload_id: str) -> None: + self._bounded(lambda: self._abort_multipart(key, upload_id), 8) + + def _bounded(self, fn, timeout: float): + future = _POOL.submit(fn) + try: + return future.result(timeout=timeout) + except FutureTimeout: + raise StorageError() from None + + def _head(self) -> None: + try: + self._client.head_bucket(Bucket=self._bucket) + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _put_bytes(self, key: str, body: bytes, content_type: str) -> None: + try: + self._client.put_object( + Bucket=self._bucket, + Key=key, + Body=body, + ContentType=content_type, + ContentLength=len(body), + ) + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _put_stream(self, key: str, body: BinaryIO, size: int, content_type: str) -> None: + body.seek(0) + try: + self._client.put_object( + Bucket=self._bucket, + Key=key, + Body=body, + ContentType=content_type, + ContentLength=size, + ) + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _get(self, key: str) -> bytes: + try: + response = self._client.get_object(Bucket=self._bucket, Key=key) + except ClientError as exc: + code = exc.response.get('Error', {}).get('Code', '') + if code in ('NoSuchKey', '404', 'NotFound'): + raise MissingObject() from None + raise StorageError() from None + except BotoCoreError: + raise StorageError() from None + payload = response['Body'] + try: + return payload.read() + finally: + payload.close() + + def _delete(self, key: str) -> None: + try: + self._client.delete_object(Bucket=self._bucket, Key=key) + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _list_originals(self) -> List[StoredObject]: + objects: List[StoredObject] = [] + token = None + try: + while True: + kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX} + if token: + kwargs['ContinuationToken'] = token + response = self._client.list_objects_v2(**kwargs) + for item in response.get('Contents') or (): + key = item['Key'] + if not key.startswith(ORIGINAL_PREFIX) or key.endswith('/'): + continue + objects.append(StoredObject(key, item['LastModified'], int(item['Size']))) + if not response.get('IsTruncated'): + return objects + token = response.get('NextContinuationToken') + if not token: + return objects + except (BotoCoreError, ClientError): + raise StorageError() from None + + def _list_multipart(self) -> List[MultipartUpload]: + uploads: List[MultipartUpload] = [] + key_marker = '' + upload_marker = '' + try: + while True: + kwargs = {'Bucket': self._bucket, 'Prefix': ORIGINAL_PREFIX} + if key_marker: + kwargs['KeyMarker'] = key_marker + if upload_marker: + kwargs['UploadIdMarker'] = upload_marker + response = self._client.list_multipart_uploads(**kwargs) + for item in response.get('Uploads') or (): + key = item['Key'] + if not key.startswith(ORIGINAL_PREFIX): + continue + uploads.append(MultipartUpload(key, item['UploadId'], item['Initiated'])) + if not response.get('IsTruncated'): + return uploads + key_marker = response.get('NextKeyMarker') or '' + upload_marker = response.get('NextUploadIdMarker') or '' + if not key_marker and not upload_marker: + return uploads + except ClientError as exc: + code = exc.response.get('Error', {}).get('Code', '') + if code in ('NotImplemented', 'AccessDenied'): + return [] + raise StorageError() from None + except BotoCoreError: + raise StorageError() from None + + def _abort_multipart(self, key: str, upload_id: str) -> None: + try: + self._client.abort_multipart_upload( + Bucket=self._bucket, + Key=key, + UploadId=upload_id, + ) + except (BotoCoreError, ClientError): + raise StorageError() from None diff --git a/app/pagination.py b/app/pagination.py new file mode 100644 index 00000000..20c8957f --- /dev/null +++ b/app/pagination.py @@ -0,0 +1,43 @@ +import base64 +import json +import re +from typing import Mapping, Optional, Tuple + +from .errors import invalid_request + +# Position of the last item returned to the client. +Key = Tuple[str, str] + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 100 +_LIMIT = re.compile(r'^\d{1,3}$') + + +def encode_cursor(scope: str, key: Key) -> str: + """Scope binds the cursor to one resource, so a foreign cursor is detectable.""" + payload = json.dumps({'s': scope, 't': key[0], 'i': key[1]}, separators=(',', ':')) + return base64.urlsafe_b64encode(payload.encode('utf-8')).decode('ascii').rstrip('=') + + +def decode_cursor(value: str, scope: str) -> Key: + try: + raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4)) + payload = json.loads(raw) + created_at, item_id = payload['t'], payload['i'] + belongs = payload['s'] == scope + except (ValueError, TypeError, KeyError): + raise invalid_request('Некорректный cursor') from None + if not belongs or not isinstance(created_at, str) or not isinstance(item_id, str): + raise invalid_request('Cursor относится к другому запросу') from None + return created_at, item_id + + +def parse_page(params: Mapping[str, str], scope: str) -> Tuple[int, Optional[Key]]: + raw_limit = params.get('limit') + limit = DEFAULT_LIMIT + if raw_limit is not None: + if not _LIMIT.match(raw_limit) or not 1 <= int(raw_limit) <= MAX_LIMIT: + raise invalid_request('limit должен быть целым числом от 1 до %d' % MAX_LIMIT) + limit = int(raw_limit) + raw_cursor = params.get('cursor') + return limit, decode_cursor(raw_cursor, scope) if raw_cursor is not None else None diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..37b3f2d1 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,27 @@ +"""Argon2id password hashes. The hash never leaves this module in an API response.""" +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError + +_hasher = PasswordHasher() +# Same cost as a real hash, so a missing user is not faster than a wrong password. +_DUMMY_HASH = _hasher.hash('not-a-user-password') +_DUMMY_PASSWORD = 'not-a-user-password' + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password_hash: str, password: str) -> bool: + try: + return _hasher.verify(password_hash, password) + except VerificationError: + return False + + +def password_matches(stored_hash: str | None, password: str) -> bool: + """One Argon2 check either way. An out-of-range secret is never the stored password.""" + if stored_hash is None or not 12 <= len(password) <= 128 or not password.strip(): + verify_password(_DUMMY_HASH, _DUMMY_PASSWORD) + return False + return verify_password(stored_hash, password) diff --git a/app/reconcile.py b/app/reconcile.py new file mode 100644 index 00000000..d74378d1 --- /dev/null +++ b/app/reconcile.py @@ -0,0 +1,591 @@ +"""Compare originals/ in the private bucket with attachment metadata. + +Dry-run is the default. --apply deletes two kinds of stale drift, and nothing else: + +- an originals/ object with no attachment row, older than the grace period; +- an attachment row whose originals/ object is gone, that no message references, + older than the grace period. + +A row linked from message_attachments is never deleted: the object cannot be +rebuilt from metadata, and the message must keep its attachment id. A young +object or row stays until grace passes, because a replica may still be between +PUT and INSERT. Keys outside originals/ are reported and left alone. + +LastModified cannot be moved backward, so the experiment's stale-orphan step +uses --grace-seconds 0 only after a preflight that the seeded key is the only +object that would be deleted. +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import io +import json +import logging +import os +import time +import uuid +import zlib +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Sequence, Set + +from .config import load_object_settings, load_settings +from .db import open_pool +from .objects import ORIGINAL_PREFIX, ObjectStore, StorageError +from .store import Attachment, AttachmentRef, Store + +logger = logging.getLogger('messenger.reconcile') + +_DEFAULT_GRACE = 900 +_LIST_TIMEOUT = 60 +_EXPERIMENT_GRACE = 900 + + +class ExperimentError(Exception): + """An invariant failed. `outcome` is the JSON already safe to print.""" + + def __init__(self, outcome: Dict[str, Any]) -> None: + super().__init__(outcome.get('error', 'experiment failed')) + self.outcome = outcome + + +def _grace_default() -> int: + raw = os.getenv('RECONCILE_GRACE_SECONDS', '').strip() + return int(raw) if raw else _DEFAULT_GRACE + + +def _age_seconds(moment: datetime, now: datetime) -> float: + if moment.tzinfo is None: + moment = moment.replace(tzinfo=timezone.utc) + return max(0.0, (now - moment).total_seconds()) + + +def _ms(started: float) -> int: + return int((time.perf_counter() - started) * 1000) + + +def _keys(entries: Sequence[Dict[str, Any]]) -> Set[str]: + return {item['key'] for item in entries} + + +def _one_pixel_png() -> bytes: + raw = b'\x00\xff\x00\x00' + + def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xffffffff + return len(data).to_bytes(4, 'big') + tag + data + crc.to_bytes(4, 'big') + + ihdr = (1).to_bytes(4, 'big') + (1).to_bytes(4, 'big') + bytes((8, 2, 0, 0, 0)) + compressed = zlib.compress(raw) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', compressed) + chunk(b'IEND', b'') + + +async def _seed(objects: ObjectStore) -> str: + key = ORIGINAL_PREFIX + str(uuid.uuid4()) + await asyncio.to_thread(objects.put, key, b'orphan-probe', 'application/octet-stream') + return key + + +def _blank_report(apply: bool, grace_seconds: int) -> Dict[str, Any]: + return { + 'apply': apply, + 'grace_seconds': grace_seconds, + 'seeded': None, + 'scanned_originals': 0, + 'referenced': 0, + 'orphans_protected_by_grace': [], + 'orphans_stale': [], + 'multipart_protected_by_grace': [], + 'multipart_stale': [], + 'deleted': [], + 'aborted_multipart': [], + 'missing_protected_by_grace': [], + 'missing_in_use': [], + 'missing_stale': [], + 'ignored_outside_prefix': [], + 'metadata_deleted': [], + 'list_objects_ms': 0, + 'list_metadata_ms': 0, + 'elapsed_ms': 0, + } + + +async def _scan( + store: Store, + objects: ObjectStore, + *, + apply: bool, + grace_seconds: int, + now: datetime, + seeded: str | None = None, +) -> Dict[str, Any]: + started = time.perf_counter() + report = _blank_report(apply, grace_seconds) + report['seeded'] = seeded + cutoff = now - timedelta(seconds=grace_seconds) + + listed = time.perf_counter() + index = await store.attachment_index() + report['list_metadata_ms'] = _ms(listed) + report['referenced'] = len(index) + + listed = time.perf_counter() + stored = await asyncio.to_thread(objects.list_originals, _LIST_TIMEOUT) + report['list_objects_ms'] = _ms(listed) + report['scanned_originals'] = len(stored) + stored_keys = {item.key for item in stored} + referenced = {item.object_key for item in index} + + for item in index: + if not item.object_key.startswith(ORIGINAL_PREFIX): + report['ignored_outside_prefix'].append({'id': item.id, 'key': item.object_key}) + continue + if item.object_key in stored_keys: + continue + age = _age_seconds(item.created_at, now) + entry = { + 'id': item.id, + 'key': item.object_key, + 'age_seconds': int(age), + 'in_use': item.in_use, + } + if item.in_use: + report['missing_in_use'].append(entry) + continue + if age < grace_seconds: + report['missing_protected_by_grace'].append(entry) + continue + report['missing_stale'].append(entry) + if not apply: + continue + removed = await store.delete_attachment_if_unlinked(item.id, cutoff) + if removed: + report['metadata_deleted'].append({'id': item.id, 'key': item.object_key}) + else: + logger.warning('skipped metadata delete for %s', item.object_key) + + for item in stored: + if item.key in referenced: + continue + age = _age_seconds(item.last_modified, now) + entry = {'key': item.key, 'age_seconds': int(age), 'size': item.size} + if age < grace_seconds: + report['orphans_protected_by_grace'].append(entry) + continue + report['orphans_stale'].append(entry) + if not apply: + continue + if await store.object_key_referenced(item.key): + logger.warning('skipped orphan delete; metadata appeared for %s', item.key) + continue + try: + await asyncio.to_thread(objects.delete, item.key) + except StorageError: + logger.warning('failed to delete orphan %s', item.key) + continue + report['deleted'].append(item.key) + + uploads = await asyncio.to_thread(objects.list_multipart) + for item in uploads: + age = _age_seconds(item.initiated, now) + entry = {'key': item.key, 'upload_id': item.upload_id, 'age_seconds': int(age)} + if age < grace_seconds: + report['multipart_protected_by_grace'].append(entry) + continue + report['multipart_stale'].append(entry) + if not apply: + continue + try: + await asyncio.to_thread(objects.abort_multipart, item.key, item.upload_id) + report['aborted_multipart'].append(entry) + except StorageError: + logger.warning('failed to abort multipart %s', item.key) + + report['elapsed_ms'] = _ms(started) + return report + + +async def _run(args: argparse.Namespace) -> Dict[str, Any]: + settings = load_settings() + objects = ObjectStore(load_object_settings()) + pool = await open_pool(settings.database_url) + now = datetime.now(timezone.utc) + try: + store = Store( + pool, + settings.session_ttl_seconds, + settings.idempotency_ttl_seconds, + objects=objects, + ) + seeded = await _seed(objects) if args.seed_orphan else None + return await _scan( + store, + objects, + apply=args.apply, + grace_seconds=args.grace_seconds, + now=now, + seeded=seeded, + ) + finally: + await pool.close() + + +def _username(prefix: str) -> str: + return prefix + uuid.uuid4().hex[:8] + + +async def _fixture_attachment(store: Store, chat_id: str, owner_id: str, filename: str) -> Attachment: + body = _one_pixel_png() + digest = hashlib.sha256(body).hexdigest() + return await store.create_attachment( + chat_id, + owner_id, + filename, + 'image/png', + io.BytesIO(body), + len(body), + digest, + ) + + +def _check(outcome: Dict[str, Any], ok: bool, message: str) -> None: + if ok: + return + outcome['ok'] = False + outcome['error'] = message + raise ExperimentError(outcome) + + +def _problem_keys(report: Dict[str, Any]) -> Set[str]: + names = ( + 'orphans_protected_by_grace', + 'orphans_stale', + 'missing_protected_by_grace', + 'missing_in_use', + 'missing_stale', + 'ignored_outside_prefix', + ) + found: Set[str] = set() + for name in names: + found.update(_keys(report[name])) + found.update(report['deleted']) + found.update(_keys(report['metadata_deleted'])) + return found + + +async def _object_sha256(objects: ObjectStore, key: str) -> str: + body = await asyncio.to_thread(objects.get, key) + return hashlib.sha256(body).hexdigest() + + +async def _row(store: Store, attachment_id: str) -> AttachmentRef | None: + for item in await store.attachment_index(): + if item.id == attachment_id: + return item + return None + + +async def _experiment() -> Dict[str, Any]: + settings = load_settings() + objects = ObjectStore(load_object_settings()) + pool = await open_pool(settings.database_url) + outcome: Dict[str, Any] = {'ok': False, 'grace_seconds': _EXPERIMENT_GRACE} + try: + store = Store( + pool, + settings.session_ttl_seconds, + settings.idempotency_ttl_seconds, + objects=objects, + ) + suffix_a = _username('r2a') + suffix_b = _username('r2b') + alice = await store.create_user(suffix_a, 'Reconcile A', 'reconcile-track2') + bob = await store.create_user(suffix_b, 'Reconcile B', 'reconcile-track2') + chat = await store.create_chat('reconcile', [alice.id, bob.id]) + healthy = await _fixture_attachment(store, chat.id, alice.id, 'healthy.png') + linked = await _fixture_attachment(store, chat.id, alice.id, 'linked-missing.png') + unlinked = await _fixture_attachment(store, chat.id, alice.id, 'unlinked-missing.png') + await store.add_message( + chat.id, + alice.id, + 'track2', + [healthy.id, linked.id], + path='/api/v1/chats/%s/messages' % chat.id, + key=uuid.uuid4().hex, + request_hash=hashlib.sha256(b'track2').hexdigest(), + ) + healthy_key = ORIGINAL_PREFIX + healthy.id + linked_key = ORIGINAL_PREFIX + linked.id + unlinked_key = ORIGINAL_PREFIX + unlinked.id + await asyncio.to_thread(objects.delete, linked_key) + await asyncio.to_thread(objects.delete, unlinked_key) + orphan_key = await _seed(objects) + outcome['fixtures'] = { + 'healthy_key': healthy_key, + 'linked_missing_key': linked_key, + 'unlinked_missing_key': unlinked_key, + 'orphan_key': orphan_key, + } + + dry = await _scan( + store, + objects, + apply=False, + grace_seconds=_EXPERIMENT_GRACE, + now=datetime.now(timezone.utc), + ) + outcome['dry_run'] = dry + _check(outcome, orphan_key in _keys(dry['orphans_protected_by_grace']), 'fresh orphan was not protected') + _check(outcome, linked_key in _keys(dry['missing_in_use']), 'in-use missing row was not reported') + _check( + outcome, + unlinked_key in _keys(dry['missing_protected_by_grace']), + 'fresh unlinked missing row was not protected', + ) + _check(outcome, healthy_key not in _problem_keys(dry), 'healthy key appeared in a drift list') + + fresh = await _scan( + store, + objects, + apply=True, + grace_seconds=_EXPERIMENT_GRACE, + now=datetime.now(timezone.utc), + ) + outcome['apply_fresh'] = fresh + _check(outcome, orphan_key not in fresh['deleted'], 'fresh orphan was deleted') + _check(outcome, healthy_key not in fresh['deleted'], 'healthy object was deleted') + _check( + outcome, + _keys(fresh['metadata_deleted']).isdisjoint({healthy_key, linked_key, unlinked_key}), + 'fresh or in-use metadata was deleted', + ) + healthy_row = await _row(store, healthy.id) + linked_row = await _row(store, linked.id) + _check(outcome, healthy_row is not None and linked_row is not None, 'protected row disappeared') + assert healthy_row is not None + digest = await _object_sha256(objects, healthy_key) + outcome['healthy_sha256_matches'] = digest == healthy_row.sha256 + _check(outcome, digest == healthy_row.sha256, 'healthy object bytes do not match the row') + + await store.rewind_attachment_created_at( + unlinked.id, + datetime.now(timezone.utc) - timedelta(hours=2), + ) + aged_preview = await _scan( + store, + objects, + apply=False, + grace_seconds=_EXPERIMENT_GRACE, + now=datetime.now(timezone.utc), + ) + outcome['aged_dry_run'] = aged_preview + expected = _keys(aged_preview['missing_stale']) + _check(outcome, unlinked_key in expected, 'aged unlinked row was not stale') + _check( + outcome, + {healthy_key, linked_key, orphan_key}.isdisjoint(expected), + 'protected key classified as stale metadata', + ) + aged = await _scan( + store, + objects, + apply=True, + grace_seconds=_EXPERIMENT_GRACE, + now=datetime.now(timezone.utc), + ) + outcome['apply_aged_metadata'] = aged + deleted_meta = _keys(aged['metadata_deleted']) + _check(outcome, unlinked_key in deleted_meta, 'aged unlinked row was not deleted') + _check(outcome, deleted_meta == expected, 'apply deleted a different metadata set than dry-run') + _check(outcome, healthy_key not in aged['deleted'], 'healthy object deleted while aging metadata') + _check(outcome, orphan_key not in aged['deleted'], 'fresh orphan deleted while aging metadata') + _check(outcome, await _row(store, linked.id) is not None, 'in-use row deleted with the stale row') + _check(outcome, await _row(store, unlinked.id) is None, 'stale unlinked row still present') + + again = await _scan( + store, + objects, + apply=True, + grace_seconds=_EXPERIMENT_GRACE, + now=datetime.now(timezone.utc), + ) + outcome['apply_aged_metadata_again'] = again + _check(outcome, again['metadata_deleted'] == [], 'second metadata apply was not empty') + _check(outcome, again['deleted'] == [], 'second apply deleted an object') + + preview = await _scan( + store, + objects, + apply=False, + grace_seconds=0, + now=datetime.now(timezone.utc), + ) + outcome['grace0_preflight'] = { + 'orphans_stale': [item['key'] for item in preview['orphans_stale']], + 'missing_stale': [item['key'] for item in preview['missing_stale']], + } + stale_orphans = _keys(preview['orphans_stale']) + if stale_orphans != {orphan_key} or preview['missing_stale']: + outcome['error'] = ( + 'grace 0 would delete something other than the seeded orphan; apply was not run' + ) + raise ExperimentError(outcome) + stale = await _scan( + store, + objects, + apply=True, + grace_seconds=0, + now=datetime.now(timezone.utc), + ) + outcome['apply_stale_orphan'] = stale + _check(outcome, stale['deleted'] == [orphan_key], 'stale orphan delete did not remove only the seed') + _check(outcome, linked_key not in _keys(stale['metadata_deleted']), 'in-use row deleted at grace 0') + _check(outcome, healthy_key not in stale['deleted'], 'healthy object deleted at grace 0') + + stale_again = await _scan( + store, + objects, + apply=True, + grace_seconds=0, + now=datetime.now(timezone.utc), + ) + outcome['apply_stale_orphan_again'] = { + 'deleted': stale_again['deleted'], + 'metadata_deleted': stale_again['metadata_deleted'], + } + _check(outcome, stale_again['deleted'] == [], 'repeat orphan apply was not empty') + _check(outcome, stale_again['metadata_deleted'] == [], 'repeat orphan apply deleted metadata') + healthy_row = await _row(store, healthy.id) + _check(outcome, healthy_row is not None, 'healthy row missing after orphan apply') + assert healthy_row is not None + digest = await _object_sha256(objects, healthy_key) + outcome['healthy_sha256_matches'] = digest == healthy_row.sha256 + _check(outcome, digest == healthy_row.sha256, 'healthy bytes changed after orphan apply') + _check(outcome, await _row(store, linked.id) is not None, 'in-use row missing after orphan apply') + outcome['linked_row_remains'] = True + outcome['ok'] = True + return outcome + except ExperimentError: + raise + finally: + await pool.close() + + +async def _bench(counts: Sequence[int]) -> Dict[str, Any]: + settings = load_settings() + objects = ObjectStore(load_object_settings()) + pool = await open_pool(settings.database_url) + samples: List[Dict[str, Any]] = [] + try: + store = Store( + pool, + settings.session_ttl_seconds, + settings.idempotency_ttl_seconds, + objects=objects, + ) + for count in counts: + seeded: List[str] = [] + try: + for _ in range(count): + key = ORIGINAL_PREFIX + str(uuid.uuid4()) + await asyncio.to_thread(objects.put, key, b'b', 'application/octet-stream') + seeded.append(key) + report = await _scan( + store, + objects, + apply=False, + grace_seconds=_DEFAULT_GRACE, + now=datetime.now(timezone.utc), + ) + samples.append({ + 'seeded': count, + 'scanned_originals': report['scanned_originals'], + 'referenced': report['referenced'], + 'list_objects_ms': report['list_objects_ms'], + 'list_metadata_ms': report['list_metadata_ms'], + 'elapsed_ms': report['elapsed_ms'], + }) + finally: + for key in seeded: + try: + await asyncio.to_thread(objects.delete, key) + except StorageError: + logger.warning('bench object left in the bucket: %s', key) + return {'samples': samples} + finally: + await pool.close() + + +def _bench_counts(raw: str) -> List[int]: + counts: List[int] = [] + for part in raw.split(','): + part = part.strip() + if not part: + continue + try: + value = int(part) + except ValueError: + raise SystemExit('bench counts must be integers') from None + if value < 1 or value > 5000: + raise SystemExit('bench count must be from 1 to 5000') + counts.append(value) + if not counts: + raise SystemExit('bench counts required') + return counts + + +def main() -> None: + logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s') + parser = argparse.ArgumentParser(description='Reconcile originals/ with attachment metadata') + parser.add_argument( + '--apply', + action='store_true', + help='delete stale orphans and stale unlinked rows; default is dry-run', + ) + parser.add_argument( + '--grace-seconds', + type=int, + default=_grace_default(), + help='do not delete objects or rows younger than this (default 900, or RECONCILE_GRACE_SECONDS)', + ) + parser.add_argument( + '--seed-orphan', + action='store_true', + help='put an unreferenced originals/ object, then scan (evidence only)', + ) + parser.add_argument( + '--experiment', + action='store_true', + help='inject both drifts, apply, and exit non-zero if a protected attachment is removed', + ) + parser.add_argument( + '--bench', + default='', + help='comma-separated object counts; dry-run each, then delete only those keys', + ) + args = parser.parse_args() + if args.grace_seconds < 0: + parser.error('grace-seconds must be >= 0') + if args.experiment and (args.bench or args.apply or args.seed_orphan): + parser.error('--experiment does not combine with --apply, --seed-orphan, or --bench') + if args.bench and (args.apply or args.seed_orphan): + parser.error('--bench does not combine with --apply or --seed-orphan') + if args.experiment: + try: + outcome = asyncio.run(_experiment()) + except ExperimentError as exc: + print(json.dumps(exc.outcome, ensure_ascii=False, indent=2)) + raise SystemExit(1) from None + print(json.dumps(outcome, ensure_ascii=False, indent=2)) + return + if args.bench: + outcome = asyncio.run(_bench(_bench_counts(args.bench))) + print(json.dumps(outcome, ensure_ascii=False, indent=2)) + return + report = asyncio.run(_run(args)) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/app/routes.py b/app/routes.py new file mode 100644 index 00000000..b9e28513 --- /dev/null +++ b/app/routes.py @@ -0,0 +1,337 @@ +import hashlib +import json +import re +import tempfile +import uuid +from typing import BinaryIO, Optional, Sequence, Tuple, Type, TypeVar + +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse +from pydantic import BaseModel, ValidationError +from starlette.responses import Response + +from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer +from .drain import draining +from .errors import invalid_request, not_found, payload_too_large, unauthorized, unavailable, unsupported_media_type +from .images import INSPECT_PREFIX, MAX_UPLOAD_BYTES, inspect_image, inspect_prefix +from .pagination import encode_cursor, parse_page +from .schemas import CreateChat, CreateMessage, CreateUser +from .store import Chat, Store, User + +# Bytes kept in RAM before SpooledTemporaryFile rolls over to an ephemeral file. +# The original is never persisted on the API volume; the spool dies with the request. +_SPOOL_RAM = 64 * 1024 + +router = APIRouter() +Body = TypeVar('Body', bound=BaseModel) + +_IDEMPOTENCY_KEY = re.compile(r'[A-Za-z0-9._:-]{1,128}') + + +async def get_store(request: Request) -> Store: + return request.app.state.store + + +async def current_user(request: Request) -> User: + """Bearer only. X-User-Id is ignored and does not identify anyone.""" + token = parse_bearer(request.headers.get('Authorization', '')) + if token is None: + raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE) + user = await request.app.state.store.user_for_token(token) + if user is None: + raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE) + return user + + +async def read_body(request: Request, model: Type[Body]) -> Body: + try: + payload = json.loads(await request.body()) + except (UnicodeDecodeError, ValueError): + raise invalid_request('Тело запроса не является корректным JSON') from None + if not isinstance(payload, dict): + raise invalid_request('Тело запроса должно быть JSON-объектом') + try: + return model.model_validate(payload) + except ValidationError: + raise invalid_request('Тело запроса не соответствует контракту') from None + + +def _request_hash(payload: BaseModel) -> str: + """Key order in the client JSON does not matter: the validated body is re-serialized canonically. + + CreateMessage always includes attachment_ids, so a missing field and [] are the same body. + """ + canonical = json.dumps(payload.model_dump(), sort_keys=True, separators=(',', ':'), ensure_ascii=False) + return hashlib.sha256(canonical.encode('utf-8')).hexdigest() + + +def _filename(value: Optional[str]) -> str: + if value is None or not value.strip() or len(value) > 255: + raise invalid_request('Некорректное filename') + if any(ord(char) < 32 or ord(char) == 127 or char in '/\\' for char in value): + raise invalid_request('Некорректное filename') + return value + + +def _declared_image_type(value: Optional[str]) -> str: + if not value: + raise unsupported_media_type('Требуется Content-Type image/png или image/jpeg') + media = value.split(';', 1)[0].strip().lower() + if media not in ('image/png', 'image/jpeg'): + raise unsupported_media_type('Требуется Content-Type image/png или image/jpeg') + return media + + +async def _spool_upload(request: Request, declared: str) -> Tuple[BinaryIO, int, str, str]: + """Stream the body into a size-capped spool, hash it, and inspect the header. + + Returns (spool, size, sha256, sniffed_type). The caller must close the spool. + One request never holds more than INSPECT_PREFIX plus _SPOOL_RAM in Python heap; + the rest, if any, is an ephemeral tempfile deleted in finally. + """ + length = request.headers.get('content-length') + if length is not None: + try: + size = int(length) + except ValueError: + raise invalid_request('Некорректный Content-Length') from None + if size > MAX_UPLOAD_BYTES: + raise payload_too_large('Файл больше 10 МиБ') + spool = tempfile.SpooledTemporaryFile(max_size=_SPOOL_RAM) + hasher = hashlib.sha256() + head = bytearray() + total = 0 + sniffed: Optional[str] = None + try: + async for chunk in request.stream(): + if not chunk: + continue + if total + len(chunk) > MAX_UPLOAD_BYTES: + raise payload_too_large('Файл больше 10 МиБ') + hasher.update(chunk) + spool.write(chunk) + total += len(chunk) + if len(head) < INSPECT_PREFIX: + head.extend(chunk[: INSPECT_PREFIX - len(head)]) + if sniffed is None and len(head) >= 24: + sniffed = inspect_prefix(bytes(head), complete=False) + if sniffed is not None and sniffed != declared: + raise unsupported_media_type('Содержимое не соответствует Content-Type') + if sniffed is None: + sniffed = inspect_image(bytes(head)) + if sniffed != declared: + raise unsupported_media_type('Содержимое не соответствует Content-Type') + spool.seek(0) + return spool, total, hasher.hexdigest(), sniffed + except BaseException: + spool.close() + raise + + +async def _member_chat(store: Store, chat_id: str, actor: User) -> Chat: + try: + canonical = str(uuid.UUID(chat_id)) + except ValueError: + raise invalid_request('Некорректный chat_id') from None + chat = await store.find_chat(canonical, actor.id) + if chat is None: + raise not_found('Чат не найден') + return chat + + +def _page(items: Sequence, has_more: bool, scope: str) -> dict: + cursor: Optional[str] = None + if has_more and items: + cursor = encode_cursor(scope, (items[-1].created_at, items[-1].id)) + return {'items': [item.to_dict() for item in items], 'next_cursor': cursor} + + +@router.get('/health/live') +async def liveness() -> JSONResponse: + return JSONResponse({'status': 'ok'}) + + +@router.get('/health/ready') +async def readiness(store: Store = Depends(get_store)) -> JSONResponse: + if draining.is_set(): + raise unavailable('Реплика завершает работу') + await store.ready() + return JSONResponse({'status': 'ok'}) + + +@router.post('/api/v1/users') +async def register_user(request: Request, store: Store = Depends(get_store)) -> JSONResponse: + payload = await read_body(request, CreateUser) + user = await store.create_user(payload.username, payload.display_name, payload.password) + return JSONResponse(user.to_dict(), status_code=201) + + +@router.post('/api/v1/auth/sessions') +async def create_session(request: Request, store: Store = Depends(get_store)) -> JSONResponse: + parsed = parse_basic(request.headers.get('Authorization', '')) + if parsed is None: + raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE) + user = await store.authenticate(*parsed) + if user is None: + raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE) + token, expires_at = await store.create_session(user.id) + body = { + 'token': token, + 'token_type': 'Bearer', + 'expires_at': expires_at, + 'user': user.to_dict(), + } + return JSONResponse(body, status_code=201) + + +@router.delete('/api/v1/auth/sessions/current') +async def revoke_session(request: Request, actor: User = Depends(current_user)) -> Response: + token = parse_bearer(request.headers.get('Authorization', '')) + if token is None or not await request.app.state.store.revoke_session(token): + raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE) + return Response(status_code=204) + + +@router.get('/api/v1/users/me') +async def show_current_user(actor: User = Depends(current_user)) -> JSONResponse: + return JSONResponse(actor.to_dict()) + + +@router.post('/api/v1/chats') +async def create_chat( + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + payload = await read_body(request, CreateChat) + if actor.id not in payload.member_ids: + raise invalid_request('Создатель чата должен быть указан в member_ids') + chat = await store.create_chat(payload.title, payload.member_ids) + return JSONResponse(chat.to_dict(), status_code=201) + + +@router.get('/api/v1/chats') +async def list_chats( + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + scope = 'chats:' + actor.id + limit, after = parse_page(request.query_params, scope) + chats, has_more = await store.list_chats(actor.id, limit, after) + return JSONResponse(_page(chats, has_more, scope)) + + +@router.get('/api/v1/chats/{chat_id}') +async def show_chat( + chat_id: str, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + return JSONResponse((await _member_chat(store, chat_id, actor)).to_dict()) + + +@router.post('/api/v1/chats/{chat_id}/messages') +async def send_message( + chat_id: str, + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + chat = await _member_chat(store, chat_id, actor) + payload = await read_body(request, CreateMessage) + key = request.headers.get('Idempotency-Key') + if key is None or not _IDEMPOTENCY_KEY.fullmatch(key): + raise invalid_request('Требуется корректный Idempotency-Key') + message = await store.add_message( + chat.id, + actor.id, + payload.text, + payload.attachment_ids, + path='/api/v1/chats/%s/messages' % chat.id, + key=key, + request_hash=_request_hash(payload), + ) + return JSONResponse(message.to_dict(), status_code=201) + + +@router.post('/api/v1/chats/{chat_id}/attachments') +async def upload_attachment( + chat_id: str, + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + chat = await _member_chat(store, chat_id, actor) + filename = _filename(request.query_params.get('filename')) + declared = _declared_image_type(request.headers.get('content-type')) + spool = None + try: + spool, size, digest, actual = await _spool_upload(request, declared) + attachment = await store.create_attachment( + chat.id, + actor.id, + filename, + actual, + spool, + size, + digest, + ) + finally: + if spool is not None: + spool.close() + return JSONResponse(attachment.to_dict(), status_code=201) + + +def _variant(request: Request) -> str: + raw = request.query_params.get('variant') + variant = 'original' if raw is None else raw + if variant not in ('original', 'thumbnail', 'crop', 'optimized'): + raise invalid_request('Некорректный variant') + return variant + + +@router.get('/api/v1/attachments/{attachment_id}') +async def show_attachment( + attachment_id: str, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + attachment = await store.find_attachment(attachment_id, actor.id) + if attachment is None: + raise not_found('Вложение не найдено') + return JSONResponse(attachment.to_dict()) + + +@router.get('/api/v1/attachments/{attachment_id}/content') +async def download_attachment( + attachment_id: str, + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> Response: + variant = _variant(request) + if variant != 'original': + attachment = await store.find_attachment(attachment_id, actor.id) + if attachment is None: + raise not_found('Вложение не найдено') + raise not_found('Вариант не найден') + found = await store.read_original(attachment_id, actor.id) + if found is None: + raise not_found('Вложение не найдено') + content_type, body = found + return Response(content=body, media_type=content_type) + + +@router.get('/api/v1/chats/{chat_id}/messages') +async def list_messages( + chat_id: str, + request: Request, + actor: User = Depends(current_user), + store: Store = Depends(get_store), +) -> JSONResponse: + chat = await _member_chat(store, chat_id, actor) + scope = 'messages:' + chat.id + limit, after = parse_page(request.query_params, scope) + messages, has_more = await store.list_messages(chat.id, limit, after) + return JSONResponse(_page(messages, has_more, scope)) diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 00000000..44049bd7 --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,73 @@ +"""Request bodies. The contract forbids unknown fields, wrong types and blank strings.""" +import uuid +from typing import List + +from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator, model_validator +from typing_extensions import Annotated + + +def _not_blank(value: str) -> str: + if not value.strip(): + raise ValueError('value must not be whitespace only') + return value + + +def _canonical_uuid(value: str) -> str: + parsed = uuid.UUID(value) + if str(parsed) != value.lower(): + raise ValueError('value must be a canonical UUID') + return str(parsed) + + +Username = Annotated[str, Field(pattern=r'^[a-z0-9_]{3,32}$')] +DisplayName = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)] +Title = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)] +Password = Annotated[str, Field(min_length=12, max_length=128), AfterValidator(_not_blank)] +UserId = Annotated[str, AfterValidator(_canonical_uuid)] + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra='forbid') + + username: Username + display_name: DisplayName + password: Password + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra='forbid') + + title: Title + member_ids: Annotated[List[UserId], Field(min_length=1, max_length=100)] + + @field_validator('member_ids') + @classmethod + def _unique(cls, value: List[str]) -> List[str]: + if len(set(value)) != len(value): + raise ValueError('member_ids must be unique') + return value + + +class CreateMessage(BaseModel): + """text is required. attachment_ids defaults to [], so a lab3 body {"text": "..."} stays valid. + + A blank text is accepted only together with at least one attachment. + """ + + model_config = ConfigDict(extra='forbid') + + text: Annotated[str, Field(max_length=4000)] + attachment_ids: Annotated[List[UserId], Field(default_factory=list, max_length=10)] + + @field_validator('attachment_ids') + @classmethod + def _unique_attachments(cls, value: List[str]) -> List[str]: + if len(set(value)) != len(value): + raise ValueError('attachment_ids must be unique') + return value + + @model_validator(mode='after') + def _has_content(self) -> 'CreateMessage': + if not self.text.strip() and not self.attachment_ids: + raise ValueError('text or attachment_ids is required') + return self diff --git a/app/store.py b/app/store.py new file mode 100644 index 00000000..8fef6651 --- /dev/null +++ b/app/store.py @@ -0,0 +1,783 @@ +import asyncio +import hashlib +import json +import logging +import secrets +import threading +import uuid +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from typing import BinaryIO, Dict, List, Optional, Sequence, Tuple + +import asyncpg + +from .errors import conflict, invalid_request, not_found, unavailable +from .migrate import expected_versions +from .objects import ORIGINAL_PREFIX, MissingObject, ObjectStore, StorageError +from .pagination import Key +from .passwords import hash_password, password_matches + +logger = logging.getLogger('messenger') + + +@dataclass(frozen=True) +class User: + id: str + username: str + display_name: str + created_at: str + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'username': self.username, + 'display_name': self.display_name, + 'created_at': self.created_at, + } + + +@dataclass(frozen=True) +class Chat: + id: str + title: str + member_ids: Tuple[str, ...] + created_at: str + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'title': self.title, + 'member_ids': list(self.member_ids), + 'created_at': self.created_at, + } + + +@dataclass(frozen=True) +class Message: + id: str + chat_id: str + sender_id: str + text: str + created_at: str + attachment_ids: Tuple[str, ...] = () + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'chat_id': self.chat_id, + 'sender_id': self.sender_id, + 'text': self.text, + 'created_at': self.created_at, + 'attachment_ids': list(self.attachment_ids), + } + + +@dataclass(frozen=True) +class Attachment: + id: str + chat_id: str + owner_id: str + filename: str + content_type: str + size_bytes: int + sha256: str + created_at: str + updated_at: str + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'chat_id': self.chat_id, + 'owner_id': self.owner_id, + 'filename': self.filename, + 'content_type': self.content_type, + 'size_bytes': self.size_bytes, + 'sha256': self.sha256, + 'status': 'ready', + 'variants': [], + 'error_code': None, + 'created_at': self.created_at, + 'updated_at': self.updated_at, + } + + +@dataclass(frozen=True) +class AttachmentRef: + """One metadata row as the reconciler sees it. Not an API response.""" + + id: str + object_key: str + created_at: datetime + in_use: bool + sha256: str + + +def _token_key(token: str) -> str: + return hashlib.sha256(token.encode('utf-8')).hexdigest() + + +def _stamp(moment: datetime) -> str: + return moment.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def _parse_stamp(value: str) -> datetime: + if value.endswith('Z'): + value = value[:-1] + '+00:00' + return datetime.fromisoformat(value) + + +def _user(row: asyncpg.Record) -> User: + return User(str(row['id']), row['username'], row['display_name'], _stamp(row['created_at'])) + + +def _as_uuid(value: str, message: str) -> uuid.UUID: + try: + return uuid.UUID(value) + except ValueError: + raise invalid_request(message) from None + + +def _db_down(exc: BaseException) -> bool: + return isinstance(exc, ( + asyncpg.PostgresConnectionError, + asyncpg.InterfaceError, + asyncpg.CannotConnectNowError, + OSError, + TimeoutError, + )) + + +class Store: + """PostgreSQL holds users, chats, sessions, messages and attachment metadata. + + Original bytes are in the private bucket. Any replica can read them by the server-assigned key. + """ + + def __init__( + self, + pool: asyncpg.Pool, + session_ttl_seconds: int = 3600, + idempotency_ttl_seconds: int = 86400, + *, + objects: ObjectStore, + ) -> None: + self._pool = pool + self._objects = objects + self._session_ttl = timedelta(seconds=session_ttl_seconds) + self._idempotency_ttl = timedelta(seconds=idempotency_ttl_seconds) + self._lock = threading.Lock() + self._last_moment: Optional[datetime] = None + self._migrations = expected_versions() + + def _now(self) -> datetime: + """Strictly increasing within one process; across replicas order relies on the shared host clock.""" + with self._lock: + moment = datetime.now(timezone.utc) + if self._last_moment is not None and moment <= self._last_moment: + moment = self._last_moment + timedelta(microseconds=1) + self._last_moment = moment + return moment + + async def ready(self) -> None: + """Ready when PostgreSQL has this build's schema and the private bucket answers.""" + try: + async with self._pool.acquire() as conn: + applied = await conn.fetchval( + 'SELECT count(*) FROM schema_migrations WHERE version = ANY($1::text[])', + self._migrations, + ) + except Exception: + raise unavailable('Хранилище недоступно') from None + if applied != len(self._migrations): + raise unavailable('Схема БД ещё не применена') + try: + await asyncio.to_thread(self._objects.ready) + except StorageError: + raise unavailable('Хранилище недоступно') from None + + async def create_user(self, username: str, display_name: str, password: str) -> User: + password_hash = hash_password(password) + user_id = uuid.uuid4() + created = self._now() + try: + async with self._pool.acquire() as conn: + await conn.execute( + ''' + INSERT INTO users (id, username, display_name, password_hash, created_at) + VALUES ($1, $2, $3, $4, $5) + ''', + user_id, + username, + display_name, + password_hash, + created, + ) + except asyncpg.UniqueViolationError: + raise conflict('Username уже занят') from None + return User(str(user_id), username, display_name, _stamp(created)) + + async def authenticate(self, username: str, password: str) -> Optional[User]: + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT id, username, display_name, created_at, password_hash + FROM users + WHERE username = $1 + ''', + username, + ) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + stored_hash = row['password_hash'] if row is not None else None + if not password_matches(stored_hash, password) or row is None: + return None + return _user(row) + + async def create_session(self, user_id: str) -> Tuple[str, str]: + token = secrets.token_urlsafe(32) + expires = datetime.now(timezone.utc) + self._session_ttl + try: + async with self._pool.acquire() as conn: + await conn.execute( + 'INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, $3)', + _token_key(token), + _as_uuid(user_id, 'Некорректный user_id'), + expires, + ) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + return token, _stamp(expires) + + async def user_for_token(self, token: str) -> Optional[User]: + now = datetime.now(timezone.utc) + key = _token_key(token) + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at + FROM sessions s + JOIN users u ON u.id = s.user_id + WHERE s.token_hash = $1 + ''', + key, + ) + if row is None: + return None + if row['expires_at'] <= now: + await conn.execute('DELETE FROM sessions WHERE token_hash = $1', key) + return None + return _user(row) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + + async def revoke_session(self, token: str) -> bool: + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + DELETE FROM sessions + WHERE token_hash = $1 AND expires_at > $2 + RETURNING token_hash + ''', + _token_key(token), + datetime.now(timezone.utc), + ) + except Exception as exc: + if _db_down(exc): + raise unavailable('Хранилище недоступно') from None + raise + return row is not None + + async def create_chat(self, title: str, member_ids: Sequence[str]) -> Chat: + ids = [_as_uuid(member_id, 'Некорректный user_id') for member_id in member_ids] + chat_id = uuid.uuid4() + created = self._now() + async with self._pool.acquire() as conn: + async with conn.transaction(): + found = await conn.fetch('SELECT id FROM users WHERE id = ANY($1::uuid[])', ids) + if len(found) != len(ids): + raise invalid_request('Все участники должны быть существующими пользователями') + await conn.execute( + 'INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)', + chat_id, + title, + created, + ) + await conn.execute( + ''' + INSERT INTO chat_members (chat_id, user_id, position) + SELECT $1, member_id, (ordinality - 1)::integer + FROM unnest($2::uuid[]) WITH ORDINALITY AS members(member_id, ordinality) + ''', + chat_id, + ids, + ) + return Chat(str(chat_id), title, tuple(member_ids), _stamp(created)) + + async def find_chat(self, chat_id: str, member_id: str) -> Optional[Chat]: + """A chat of somebody else is indistinguishable from a missing one.""" + chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') + member_uuid = _as_uuid(member_id, 'Некорректный user_id') + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2 + WHERE c.id = $1 + ''', + chat_uuid, + member_uuid, + ) + if row is None: + return None + members = await self._members(conn, [chat_uuid]) + return Chat(str(row['id']), row['title'], members.get(chat_uuid, ()), _stamp(row['created_at'])) + + async def list_chats(self, member_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Chat], bool]: + member_uuid = _as_uuid(member_id, 'Некорректный user_id') + after_at, after_id = self._cursor(after) + async with self._pool.acquire() as conn: + if after_at is None: + rows = await conn.fetch( + ''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 + ORDER BY c.created_at, c.id + LIMIT $2 + ''', + member_uuid, + limit + 1, + ) + else: + rows = await conn.fetch( + ''' + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1 + WHERE (c.created_at, c.id) > ($2::timestamptz, $3::uuid) + ORDER BY c.created_at, c.id + LIMIT $4 + ''', + member_uuid, + after_at, + after_id, + limit + 1, + ) + page = rows[:limit] + members = await self._members(conn, [row['id'] for row in page]) + chats = [ + Chat(str(row['id']), row['title'], members.get(row['id'], ()), _stamp(row['created_at'])) + for row in page + ] + return chats, len(rows) > limit + + async def create_attachment( + self, + chat_id: str, + owner_id: str, + filename: str, + content_type: str, + body: BinaryIO, + size_bytes: int, + sha256: str, + ) -> Attachment: + chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') + owner_uuid = _as_uuid(owner_id, 'Некорректный user_id') + attachment_id = uuid.uuid4() + # The key is not derived from filename. A crash after PUT and before INSERT leaves an + # unreferenced object; a failed INSERT deletes it. 201 is returned only after both succeed. + object_key = ORIGINAL_PREFIX + str(attachment_id) + created = self._now() + try: + await asyncio.to_thread(self._objects.put_stream, object_key, body, size_bytes, content_type) + except StorageError: + await self._discard_object(object_key) + raise unavailable('Хранилище недоступно') from None + try: + async with self._pool.acquire() as conn: + await conn.execute( + ''' + INSERT INTO attachments ( + id, chat_id, owner_id, filename, content_type, size_bytes, sha256, + object_key, created_at, updated_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $9) + ''', + attachment_id, + chat_uuid, + owner_uuid, + filename, + content_type, + size_bytes, + sha256, + object_key, + created, + ) + except Exception: + await self._discard_object(object_key) + raise + stamp = _stamp(created) + return Attachment( + str(attachment_id), + str(chat_uuid), + str(owner_uuid), + filename, + content_type, + size_bytes, + sha256, + stamp, + stamp, + ) + + async def attachment_index(self) -> List[AttachmentRef]: + """Every attachment row. `in_use` means a message still points at it. + + The reconciler uses this instead of the HTTP API. A row linked from + message_attachments must survive even when the object is gone: the + foreign key has no ON DELETE CASCADE, and the message would lose its file. + """ + async with self._pool.acquire() as conn: + rows = await conn.fetch( + ''' + SELECT a.id, a.object_key, a.created_at, a.sha256, + EXISTS ( + SELECT 1 FROM message_attachments m + WHERE m.attachment_id = a.id + ) AS in_use + FROM attachments a + ''' + ) + return [ + AttachmentRef( + str(row['id']), + row['object_key'], + row['created_at'], + bool(row['in_use']), + row['sha256'], + ) + for row in rows + ] + + async def object_key_referenced(self, object_key: str) -> bool: + """True when a metadata row owns this key. Checked again immediately before delete.""" + async with self._pool.acquire() as conn: + found = await conn.fetchval( + 'SELECT 1 FROM attachments WHERE object_key = $1', + object_key, + ) + return found is not None + + async def delete_attachment_if_unlinked(self, attachment_id: str, created_before: datetime) -> bool: + """Delete a stale row that no message references. A link or a young row keeps it. + + `created_before` is the scan clock minus grace, so the row deleted here is the + same row the scan classified as stale. The originals/ predicate matches the + prefix the reconciler is allowed to own. + """ + async with self._pool.acquire() as conn: + result = await conn.execute( + ''' + DELETE FROM attachments + WHERE id = $1 + AND object_key LIKE 'originals/%' + AND created_at <= $2 + AND NOT EXISTS ( + SELECT 1 FROM message_attachments WHERE attachment_id = $1 + ) + ''', + _as_uuid(attachment_id, 'Некорректный attachment_id'), + created_before, + ) + return result == 'DELETE 1' + + async def rewind_attachment_created_at(self, attachment_id: str, created_at: datetime) -> None: + """Move one row's clock back. The track-2 experiment uses this to age a fixture.""" + async with self._pool.acquire() as conn: + result = await conn.execute( + 'UPDATE attachments SET created_at = $2 WHERE id = $1', + _as_uuid(attachment_id, 'Некорректный attachment_id'), + created_at, + ) + if result != 'UPDATE 1': + raise RuntimeError('attachment row missing') + + async def find_attachment(self, attachment_id: str, user_id: str) -> Optional[Attachment]: + """Metadata for a chat member. Missing and foreign attachments are both absent.""" + attachment_uuid = _as_uuid(attachment_id, 'Некорректный attachment_id') + user_uuid = _as_uuid(user_id, 'Некорректный user_id') + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT a.id, a.chat_id, a.owner_id, a.filename, a.content_type, + a.size_bytes, a.sha256, a.created_at, a.updated_at + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 + WHERE a.id = $1 + ''', + attachment_uuid, + user_uuid, + ) + if row is None: + return None + return Attachment( + str(row['id']), + str(row['chat_id']), + str(row['owner_id']), + row['filename'], + row['content_type'], + row['size_bytes'], + row['sha256'], + _stamp(row['created_at']), + _stamp(row['updated_at']), + ) + + async def read_original(self, attachment_id: str, user_id: str) -> Optional[Tuple[str, bytes]]: + attachment_uuid = _as_uuid(attachment_id, 'Некорректный attachment_id') + user_uuid = _as_uuid(user_id, 'Некорректный user_id') + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + ''' + SELECT a.content_type, a.size_bytes, a.sha256, a.object_key + FROM attachments a + JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 + WHERE a.id = $1 + ''', + attachment_uuid, + user_uuid, + ) + if row is None: + return None + try: + body = await asyncio.to_thread(self._objects.get, row['object_key']) + except MissingObject: + return None + except StorageError: + raise unavailable('Хранилище недоступно') from None + if len(body) != row['size_bytes'] or hashlib.sha256(body).hexdigest() != row['sha256']: + logger.warning('object bytes do not match attachment metadata') + raise unavailable('Хранилище недоступно') + return row['content_type'], body + + async def _discard_object(self, object_key: str) -> None: + try: + await asyncio.to_thread(self._objects.delete, object_key) + except StorageError: + logger.warning('object left in the bucket after a failed metadata write') + + async def add_message( + self, + chat_id: str, + sender_id: str, + text: str, + attachment_ids: Sequence[str], + *, + path: str, + key: str, + request_hash: str, + ) -> Message: + """Exactly one message per (sender, POST, path, key), whichever replica receives the request. + + Concurrent duplicates block on the primary key of idempotency_keys until the first + transaction commits or rolls back, so a rolled-back attempt leaves the key free. + """ + chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') + sender_uuid = _as_uuid(sender_id, 'Некорректный user_id') + linked = tuple(_as_uuid(item, 'Некорректный attachment_id') for item in attachment_ids) + message = Message( + str(uuid.uuid4()), + str(chat_uuid), + sender_id, + text, + _stamp(self._now()), + tuple(str(item) for item in linked), + ) + cutoff = datetime.now(timezone.utc) - self._idempotency_ttl + async with self._pool.acquire() as conn: + async with conn.transaction(): + await conn.execute( + ''' + DELETE FROM idempotency_keys + WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3 AND created_at < $4 + ''', + sender_uuid, + path, + key, + cutoff, + ) + claimed = await conn.fetchval( + ''' + INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response) + VALUES ($1, 'POST', $2, $3, $4, 201, $5::jsonb) + ON CONFLICT (user_id, method, path, key) DO NOTHING + RETURNING 1 + ''', + sender_uuid, + path, + key, + request_hash, + json.dumps(message.to_dict()), + ) + if claimed is None: + return await self._replay(conn, sender_uuid, path, key, request_hash) + if linked: + rows = await conn.fetch( + 'SELECT id, chat_id FROM attachments WHERE id = ANY($1::uuid[])', + list(linked), + ) + found = {row['id']: row['chat_id'] for row in rows} + if len(found) != len(linked) or any(found[item] != chat_uuid for item in linked): + raise not_found('Вложение не найдено') + await conn.execute( + ''' + INSERT INTO messages (id, chat_id, sender_id, text, created_at) + VALUES ($1, $2, $3, $4, $5) + ''', + uuid.UUID(message.id), + chat_uuid, + sender_uuid, + text, + _parse_stamp(message.created_at), + ) + if linked: + await conn.executemany( + ''' + INSERT INTO message_attachments (message_id, attachment_id, position) + VALUES ($1, $2, $3) + ''', + [(uuid.UUID(message.id), item, index) for index, item in enumerate(linked)], + ) + return message + + async def _replay( + self, + conn: asyncpg.Connection, + sender_uuid: uuid.UUID, + path: str, + key: str, + request_hash: str, + ) -> Message: + row = await conn.fetchrow( + ''' + SELECT request_hash, response + FROM idempotency_keys + WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3 + ''', + sender_uuid, + path, + key, + ) + if row is None: + raise unavailable('Повторите запрос с тем же Idempotency-Key') + if row['request_hash'] != request_hash: + raise conflict('Idempotency-Key уже использован с другим сообщением') + data = json.loads(row['response']) + return Message( + data['id'], + data['chat_id'], + data['sender_id'], + data['text'], + data['created_at'], + tuple(data.get('attachment_ids') or ()), + ) + + async def list_messages(self, chat_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Message], bool]: + chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id') + after_at, after_id = self._cursor(after) + async with self._pool.acquire() as conn: + if after_at is None: + rows = await conn.fetch( + ''' + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + ORDER BY created_at, id + LIMIT $2 + ''', + chat_uuid, + limit + 1, + ) + else: + rows = await conn.fetch( + ''' + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 + AND (created_at, id) > ($2::timestamptz, $3::uuid) + ORDER BY created_at, id + LIMIT $4 + ''', + chat_uuid, + after_at, + after_id, + limit + 1, + ) + page = rows[:limit] + links = await self._attachment_ids(conn, [row['id'] for row in page]) + messages = [ + Message( + str(row['id']), + str(row['chat_id']), + str(row['sender_id']), + row['text'], + _stamp(row['created_at']), + links.get(row['id'], ()), + ) + for row in page + ] + return messages, len(rows) > limit + + async def _attachment_ids( + self, + conn: asyncpg.Connection, + message_ids: Sequence[uuid.UUID], + ) -> Dict[uuid.UUID, Tuple[str, ...]]: + if not message_ids: + return {} + rows = await conn.fetch( + ''' + SELECT message_id, attachment_id + FROM message_attachments + WHERE message_id = ANY($1::uuid[]) + ORDER BY message_id, position + ''', + list(message_ids), + ) + grouped: Dict[uuid.UUID, List[str]] = {} + for row in rows: + grouped.setdefault(row['message_id'], []).append(str(row['attachment_id'])) + return {message_id: tuple(ids) for message_id, ids in grouped.items()} + + async def _members(self, conn: asyncpg.Connection, chat_ids: Sequence[uuid.UUID]) -> Dict[uuid.UUID, Tuple[str, ...]]: + if not chat_ids: + return {} + rows = await conn.fetch( + ''' + SELECT chat_id, user_id + FROM chat_members + WHERE chat_id = ANY($1::uuid[]) + ORDER BY chat_id, position + ''', + list(chat_ids), + ) + grouped: Dict[uuid.UUID, List[str]] = {} + for row in rows: + grouped.setdefault(row['chat_id'], []).append(str(row['user_id'])) + return {chat_id: tuple(member_ids) for chat_id, member_ids in grouped.items()} + + def _cursor(self, after: Optional[Key]) -> Tuple[Optional[datetime], Optional[uuid.UUID]]: + if after is None: + return None, None + try: + return _parse_stamp(after[0]), uuid.UUID(after[1]) + except ValueError: + raise invalid_request('Некорректный cursor') from None diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..8de23afc --- /dev/null +++ b/compose.yaml @@ -0,0 +1,329 @@ +# Client -> lb (HAProxy) -> api x API_REPLICAS -> postgres (metadata) + minio (private bucket). +# Only the LB publishes a port. Networks: edge = lb<->api, data = api/migrate/minio (no LB, no internet). +# DB and MinIO passwords come from .env (not committed) as Compose secret files, never as container env vars. +name: messenger-lab4 + +x-db-client: &db-client + APP_ENV: ${APP_ENV:-lab} + LOG_LEVEL: ${LOG_LEVEL:-info} + DB_HOST: postgres + DB_USER: ${POSTGRES_USER:?set in .env} + DB_NAME: ${POSTGRES_DB:?set in .env} + DB_PASSWORD_FILE: /run/secrets/postgres_password + +x-app-build: &app-build + context: . + dockerfile: Dockerfile + args: + APP_VERSION: ${APP_VERSION:-dev} + +services: + postgres: + image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 + environment: + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_DB: ${POSTGRES_DB} + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + networks: + - data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 2s + timeout: 5s + retries: 30 + deploy: + resources: + limits: + cpus: "${POSTGRES_CPUS:-1.0}" + memory: "${POSTGRES_MEMORY:-512M}" + reservations: + cpus: "0.25" + memory: 256M + restart: unless-stopped + + # One-shot schema step. Replicas never migrate on startup, so there is no race between them; + # an advisory lock additionally serializes accidental parallel runs. + migrate: + image: messenger-lab4-app:${APP_VERSION:-dev} + build: *app-build + command: ["python", "-m", "app.migrate"] + environment: + <<: *db-client + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - data + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: "no" + + # MinIO Community RELEASE.2025-09-07T16-13-09Z. The digest is the official + # multi-arch manifest list (linux/amd64 inside it is sha256:a1a8bd4a…). + # docker.io/minio/minio and quay.io/minio/minio no longer allow an anonymous + # pull; this GHCR mirror still serves that same manifest list. + # No host ports. MINIO_BROWSER=off: the admin console is not started. + # S3 API is http://minio:9000, also http://s3:9000 (network alias). + minio: + image: ghcr.io/tsouza/cerberus-mirror/minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + command: ["server", "/data"] + environment: + MINIO_ROOT_USER_FILE: /run/secrets/minio_root_user + MINIO_ROOT_PASSWORD_FILE: /run/secrets/minio_root_password + MINIO_BROWSER: "off" + MINIO_UPDATE: "off" + secrets: + - minio_root_user + - minio_root_password + volumes: + - miniodata:/data + networks: + data: + aliases: + - s3 + healthcheck: + test: ["CMD", "curl", "-fsS", "http://127.0.0.1:9000/minio/health/ready"] + interval: 2s + timeout: 5s + retries: 30 + start_period: 10s + deploy: + resources: + limits: + cpus: "${MINIO_CPUS:-1.0}" + memory: "${MINIO_MEMORY:-512M}" + reservations: + cpus: "0.25" + memory: 256M + restart: unless-stopped + + # Creates the private bucket and the bucket-scoped API user. Re-running is a no-op. + minio-init: + image: ghcr.io/tsouza/cerberus-mirror/minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + entrypoint: ["/bin/sh", "/bootstrap/minio-bootstrap.sh"] + environment: + S3_BUCKET: ${S3_BUCKET:?set in .env} + S3_REGION: ${S3_REGION:-us-east-1} + MINIO_ENDPOINT: http://minio:9000 + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio/bootstrap.sh:/bootstrap/minio-bootstrap.sh:ro + networks: + - data + depends_on: + minio: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: "no" + + # Operator one-shot: python -m app.reconcile (dry-run). + # Args: --apply, --seed-orphan, --experiment, --bench 10,100,400. + # Not started by `up`; use: docker compose --env-file .env --profile ops run --rm reconcile + reconcile: + image: messenger-lab4-app:${APP_VERSION:-dev} + build: *app-build + profiles: ["ops"] + entrypoint: ["python", "-m", "app.reconcile"] + environment: + <<: *db-client + S3_ENDPOINT: ${S3_ENDPOINT:?set in .env} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:?set in .env} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + RECONCILE_GRACE_SECONDS: ${RECONCILE_GRACE_SECONDS:-900} + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + - source: s3_access_key_id + uid: "10001" + gid: "10001" + mode: 0400 + - source: s3_secret_access_key + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - data + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + restart: "no" + + # Unauthenticated GET and console probe from the internal data network. + check-private-bucket: + image: ghcr.io/tsouza/cerberus-mirror/minio/minio:RELEASE.2025-09-07T16-13-09Z@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e + profiles: ["ops"] + entrypoint: ["/bin/sh", "/bootstrap/check-private-bucket.sh"] + environment: + S3_BUCKET: ${S3_BUCKET:?set in .env} + MINIO_ENDPOINT: http://minio:9000 + secrets: + - minio_root_user + - minio_root_password + - s3_access_key_id + - s3_secret_access_key + volumes: + - ./infra/minio/check-private-bucket.sh:/bootstrap/check-private-bucket.sh:ro + networks: + - data + depends_on: + minio: + condition: service_healthy + minio-init: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + restart: "no" + + api: + image: messenger-lab4-app:${APP_VERSION:-dev} + build: *app-build + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPUS:-1.0}" + memory: "${API_MEMORY:-256M}" + reservations: + cpus: "0.25" + memory: 128M + environment: + <<: *db-client + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DRAIN_SECONDS: ${DRAIN_SECONDS:-4} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + S3_ENDPOINT: ${S3_ENDPOINT:?set in .env} + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:?set in .env} + S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id + S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + - source: s3_access_key_id + uid: "10001" + gid: "10001" + mode: 0400 + - source: s3_secret_access_key + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - edge + - data + expose: + - "8080" + depends_on: + postgres: + condition: service_healthy + migrate: + condition: service_completed_successfully + minio-init: + condition: service_completed_successfully + # Docker health gates startup order and rolling update; the LB runs its own checks (infra/haproxy/haproxy.cfg). + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2) + interval: 3s + timeout: 3s + retries: 3 + start_period: 20s + # SIGTERM -> readiness 503 for DRAIN_SECONDS (LB drops the replica in <= 3 s) -> uvicorn finishes + # in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. 4 + 3 < 10, so SIGKILL is not reached. + stop_signal: SIGTERM + stop_grace_period: 10s + restart: unless-stopped + + lb: + image: haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 + # Loopback only. Both families: on Windows "localhost" tries ::1 first and waits ~2 s if nothing listens. + ports: + - "127.0.0.1:${LB_PORT:-8080}:8080" + - "[::1]:${LB_PORT:-8080}:8080" + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: + - edge + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/lb/health"] + interval: 5s + timeout: 3s + retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + reservations: + cpus: "0.1" + memory: 32M + restart: unless-stopped + +networks: + edge: {} + data: + internal: true + +secrets: + postgres_password: + environment: POSTGRES_PASSWORD + minio_root_user: + environment: MINIO_ROOT_USER + minio_root_password: + environment: MINIO_ROOT_PASSWORD + s3_access_key_id: + environment: S3_ACCESS_KEY_ID + s3_secret_access_key: + environment: S3_SECRET_ACCESS_KEY + +volumes: + pgdata: + miniodata: diff --git a/evidence/attachments-negative.txt b/evidence/attachments-negative.txt new file mode 100644 index 00000000..238a6bbd Binary files /dev/null and b/evidence/attachments-negative.txt differ diff --git a/evidence/compose-minio-exposure.json b/evidence/compose-minio-exposure.json new file mode 100644 index 00000000..331e066f --- /dev/null +++ b/evidence/compose-minio-exposure.json @@ -0,0 +1,23 @@ +{ + "minio_ports": [], + "MINIO_BROWSER": "off", + "networks": [ + "data" + ], + "lb_ports": [ + { + "mode": "ingress", + "host_ip": "127.0.0.1", + "target": 8080, + "published": "8084", + "protocol": "tcp" + }, + { + "mode": "ingress", + "host_ip": "::1", + "target": 8080, + "published": "8084", + "protocol": "tcp" + } + ] +} diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..ad74a618 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,26 @@ +# python tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8084 +# Same public test as: make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8084 +# On this Windows host the hook process was Git bash, because CreateProcess cannot exec a #!/bin/sh file. +# The script and its arguments are scripts/stop-one.sh. git index mode of the hook is 100755. + +=== before === +api-1 606d8436f093 status=running started=2026-11-12T15:27:01.000430042Z +api-2 2d9cb0ed8adc status=running started=2026-11-12T15:27:00.68343251Z +lb-1 a708afda6620 status=running started=2026-11-12T15:27:06.923950206Z +postgres-1 9c54ddd911ba status=running started=2026-11-12T15:26:49.628819651Z +minio-1 4aa69397a7e9 status=running started=2026-11-12T15:26:49.732842597Z + +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 5.699} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +=== after the hook, before any restore === +api-1 606d8436f093 status=running started=2026-11-12T15:27:01.000430042Z +api-2 2d9cb0ed8adc status=exited finished=2026-11-12T15:27:30.226687078Z +lb-1 a708afda6620 status=running started=2026-11-12T15:27:06.923950206Z +postgres-1 9c54ddd911ba status=running started=2026-11-12T15:26:49.628819651Z +minio-1 4aa69397a7e9 status=running started=2026-11-12T15:26:49.732842597Z + +The test saw two instance ids before the stop. Writes recovered in 5.699 s, with 128 successes and 0 failures. api-2 stayed exited. PostgreSQL, MinIO, the LB and api-1 kept the start times from before the hook. + +After this snapshot the stopped replica was started again with `docker compose --env-file .env up -d --wait api`. Both API replicas were healthy. That restore is not part of the hook: the hook left api-2 exited. diff --git a/evidence/minio-init-rerun.txt b/evidence/minio-init-rerun.txt new file mode 100644 index 00000000..eb812aca --- /dev/null +++ b/evidence/minio-init-rerun.txt @@ -0,0 +1,7 @@ +# Idempotent MinIO bootstrap. Secrets are not printed. +# docker compose --env-file .env run --rm --no-deps minio-init +# (two sequential runs after the 256M memory limit on minio-init) + +run 1: bucket messenger-attachments is private +run 2: bucket messenger-attachments is private +exit: 0 both times diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..0f28ded2 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,23 @@ +# python tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8084 +# Same public test as: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8084 +# On this Windows host the hook process was Git bash, because CreateProcess cannot exec a #!/bin/sh file. +# The script and its arguments are scripts/restart.sh. git index mode of the hook is 100755. + +=== before, docker inspect StartedAt === +api-1 606d8436f093 status=running started=2026-11-12T15:25:28.188004778Z +api-2 2d9cb0ed8adc status=running started=2026-11-12T15:25:27.827643637Z +lb-1 a708afda6620 status=running started=2026-11-12T15:25:34.521198734Z +postgres-1 9c54ddd911ba status=running started=2026-11-12T15:25:18.149534309Z +minio-1 4aa69397a7e9 status=running started=2026-11-12T15:25:18.110830021Z + +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +=== after, same container ids, new StartedAt === +api-1 606d8436f093 status=running started=2026-11-12T15:27:01.000430042Z +api-2 2d9cb0ed8adc status=running started=2026-11-12T15:27:00.68343251Z +lb-1 a708afda6620 status=running started=2026-11-12T15:27:06.923950206Z +postgres-1 9c54ddd911ba status=running started=2026-11-12T15:26:49.628819651Z +minio-1 4aa69397a7e9 status=running started=2026-11-12T15:26:49.732842597Z + +Container ids did not change, so the named volumes stayed. PostgreSQL, MinIO, both API replicas and the LB all received a new start time. diff --git a/evidence/private-bucket.txt b/evidence/private-bucket.txt new file mode 100644 index 00000000..26018cb4 --- /dev/null +++ b/evidence/private-bucket.txt @@ -0,0 +1,7 @@ +# python scripts/check_private_bucket.py +# (stack already up; in-network probe via compose profile ops) + +PASS: bucket messenger-attachments is private; unauthenticated GET is 403; API user cannot list another bucket; console is closed +PASS: minio has no host ports, console env is off, in-network GET is 403 + +See also evidence/compose-minio-exposure.json diff --git a/evidence/reconcile-apply.json b/evidence/reconcile-apply.json new file mode 100644 index 00000000..2ac245a5 --- /dev/null +++ b/evidence/reconcile-apply.json @@ -0,0 +1,45 @@ +{ + "apply": true, + "grace_seconds": 0, + "seeded": null, + "scanned_originals": 48, + "referenced": 50, + "orphans_protected_by_grace": [], + "orphans_stale": [ + { + "key": "originals/ecc25a63-0271-4944-bd3a-d9c83c78c4ab", + "age_seconds": 648, + "size": 12 + } + ], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [ + "originals/ecc25a63-0271-4944-bd3a-d9c83c78c4ab" + ], + "aborted_multipart": [], + "missing_protected_by_grace": [], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 168553, + "in_use": true + } + ], + "missing_stale": [], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [], + "list_objects_ms": 14, + "list_metadata_ms": 2, + "elapsed_ms": 31 +} diff --git a/evidence/reconcile-dry-run.json b/evidence/reconcile-dry-run.json new file mode 100644 index 00000000..e62f21e3 --- /dev/null +++ b/evidence/reconcile-dry-run.json @@ -0,0 +1,43 @@ +{ + "apply": false, + "grace_seconds": 900, + "seeded": "originals/ecc25a63-0271-4944-bd3a-d9c83c78c4ab", + "scanned_originals": 45, + "referenced": 47, + "orphans_protected_by_grace": [ + { + "key": "originals/ecc25a63-0271-4944-bd3a-d9c83c78c4ab", + "age_seconds": 0, + "size": 12 + } + ], + "orphans_stale": [], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [], + "aborted_multipart": [], + "missing_protected_by_grace": [], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 167904, + "in_use": true + } + ], + "missing_stale": [], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [], + "list_objects_ms": 20, + "list_metadata_ms": 7, + "elapsed_ms": 30 +} diff --git a/evidence/reconcile-track2.json b/evidence/reconcile-track2.json new file mode 100644 index 00000000..bd62ae6e --- /dev/null +++ b/evidence/reconcile-track2.json @@ -0,0 +1,315 @@ +{ + "ok": true, + "grace_seconds": 900, + "fixtures": { + "healthy_key": "originals/7764b26b-6b40-4379-9de5-74fedda9bbfd", + "linked_missing_key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "unlinked_missing_key": "originals/5a68bef7-8d1c-4625-b11e-a3df9566224e", + "orphan_key": "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae" + }, + "dry_run": { + "apply": false, + "grace_seconds": 900, + "seeded": null, + "scanned_originals": 42, + "referenced": 45, + "orphans_protected_by_grace": [ + { + "key": "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae", + "age_seconds": 0, + "size": 12 + } + ], + "orphans_stale": [], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [], + "aborted_multipart": [], + "missing_protected_by_grace": [ + { + "id": "5a68bef7-8d1c-4625-b11e-a3df9566224e", + "key": "originals/5a68bef7-8d1c-4625-b11e-a3df9566224e", + "age_seconds": 0, + "in_use": false + } + ], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 0, + "in_use": true + } + ], + "missing_stale": [], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [], + "list_objects_ms": 76, + "list_metadata_ms": 4, + "elapsed_ms": 84 + }, + "apply_fresh": { + "apply": true, + "grace_seconds": 900, + "seeded": null, + "scanned_originals": 42, + "referenced": 45, + "orphans_protected_by_grace": [ + { + "key": "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae", + "age_seconds": 0, + "size": 12 + } + ], + "orphans_stale": [], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [], + "aborted_multipart": [], + "missing_protected_by_grace": [ + { + "id": "5a68bef7-8d1c-4625-b11e-a3df9566224e", + "key": "originals/5a68bef7-8d1c-4625-b11e-a3df9566224e", + "age_seconds": 0, + "in_use": false + } + ], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 0, + "in_use": true + } + ], + "missing_stale": [], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [], + "list_objects_ms": 12, + "list_metadata_ms": 1, + "elapsed_ms": 89 + }, + "healthy_sha256_matches": true, + "aged_dry_run": { + "apply": false, + "grace_seconds": 900, + "seeded": null, + "scanned_originals": 42, + "referenced": 45, + "orphans_protected_by_grace": [ + { + "key": "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae", + "age_seconds": 0, + "size": 12 + } + ], + "orphans_stale": [], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [], + "aborted_multipart": [], + "missing_protected_by_grace": [], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 0, + "in_use": true + } + ], + "missing_stale": [ + { + "id": "5a68bef7-8d1c-4625-b11e-a3df9566224e", + "key": "originals/5a68bef7-8d1c-4625-b11e-a3df9566224e", + "age_seconds": 7200, + "in_use": false + } + ], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [], + "list_objects_ms": 14, + "list_metadata_ms": 1, + "elapsed_ms": 89 + }, + "apply_aged_metadata": { + "apply": true, + "grace_seconds": 900, + "seeded": null, + "scanned_originals": 42, + "referenced": 45, + "orphans_protected_by_grace": [ + { + "key": "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae", + "age_seconds": 0, + "size": 12 + } + ], + "orphans_stale": [], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [], + "aborted_multipart": [], + "missing_protected_by_grace": [], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 0, + "in_use": true + } + ], + "missing_stale": [ + { + "id": "5a68bef7-8d1c-4625-b11e-a3df9566224e", + "key": "originals/5a68bef7-8d1c-4625-b11e-a3df9566224e", + "age_seconds": 7200, + "in_use": false + } + ], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [ + { + "id": "5a68bef7-8d1c-4625-b11e-a3df9566224e", + "key": "originals/5a68bef7-8d1c-4625-b11e-a3df9566224e" + } + ], + "list_objects_ms": 15, + "list_metadata_ms": 1, + "elapsed_ms": 98 + }, + "apply_aged_metadata_again": { + "apply": true, + "grace_seconds": 900, + "seeded": null, + "scanned_originals": 42, + "referenced": 44, + "orphans_protected_by_grace": [ + { + "key": "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae", + "age_seconds": 0, + "size": 12 + } + ], + "orphans_stale": [], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [], + "aborted_multipart": [], + "missing_protected_by_grace": [], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 0, + "in_use": true + } + ], + "missing_stale": [], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [], + "list_objects_ms": 10, + "list_metadata_ms": 1, + "elapsed_ms": 16 + }, + "grace0_preflight": { + "orphans_stale": [ + "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae" + ], + "missing_stale": [] + }, + "apply_stale_orphan": { + "apply": true, + "grace_seconds": 0, + "seeded": null, + "scanned_originals": 42, + "referenced": 44, + "orphans_protected_by_grace": [], + "orphans_stale": [ + { + "key": "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae", + "age_seconds": 0, + "size": 12 + } + ], + "multipart_protected_by_grace": [], + "multipart_stale": [], + "deleted": [ + "originals/1f0ce87a-b473-41fc-ac17-63c014d4f6ae" + ], + "aborted_multipart": [], + "missing_protected_by_grace": [], + "missing_in_use": [ + { + "id": "c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "key": "originals/c9877ebc-08b1-46ba-9cb9-d9d42240db71", + "age_seconds": 0, + "in_use": true + } + ], + "missing_stale": [], + "ignored_outside_prefix": [ + { + "id": "28bbf1a8-866e-4b03-a22f-d7daa2e36157", + "key": "c96e4c9da2114c9e837c372ef9e75e15" + }, + { + "id": "69c98e0c-0660-4dc2-ae9f-1ecb52650b60", + "key": "a93d1b7ab79245ec95f792b569c3ee46" + } + ], + "metadata_deleted": [], + "list_objects_ms": 10, + "list_metadata_ms": 2, + "elapsed_ms": 95 + }, + "apply_stale_orphan_again": { + "deleted": [], + "metadata_deleted": [] + }, + "linked_row_remains": true +} diff --git a/evidence/reconcile-walk.json b/evidence/reconcile-walk.json new file mode 100644 index 00000000..5339311a --- /dev/null +++ b/evidence/reconcile-walk.json @@ -0,0 +1,28 @@ +{ + "samples": [ + { + "seeded": 10, + "scanned_originals": 51, + "referenced": 44, + "list_objects_ms": 13, + "list_metadata_ms": 5, + "elapsed_ms": 24 + }, + { + "seeded": 100, + "scanned_originals": 141, + "referenced": 44, + "list_objects_ms": 180, + "list_metadata_ms": 1, + "elapsed_ms": 188 + }, + { + "seeded": 400, + "scanned_originals": 441, + "referenced": 44, + "list_objects_ms": 75, + "list_metadata_ms": 2, + "elapsed_ms": 82 + } + ] +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..450bb652 Binary files /dev/null and b/evidence/smoke.txt differ diff --git a/evidence/upload-concurrency.json b/evidence/upload-concurrency.json new file mode 100644 index 00000000..34a72728 --- /dev/null +++ b/evidence/upload-concurrency.json @@ -0,0 +1,37 @@ +{ + "hypothesis": "Ten concurrent ~1.9 MiB PNG uploads stay inside the 256 MiB API limit and peak RSS is not proportional to N * file_size because the body is spooled, not held twice as bytes.", + "when": "2026-11-09T17:15:07Z", + "commit": "c594b88", + "host": { + "system": "Windows", + "machine": "AMD64", + "processor": "Intel64 Family 6 Model 165 Stepping 2, GenuineIntel" + }, + "compose_limits": { + "api_memory": "256M", + "replicas": 2 + }, + "profile": { + "concurrency": 10, + "width": 900, + "height": 700, + "size_bytes": 1994104, + "sha256": "03f35c154abaa690cfc41d6d73715992df84d10f45ff48dc14722bd64e49a263" + }, + "results": { + "ok": 10, + "errors": 0, + "elapsed_s": 1.995, + "p50_s": 0.5158, + "p95_s": 0.5466, + "p99_s": 0.5466, + "peak_api_rss_mib": 93.66, + "peak_rss_by_container_mib": { + "messenger-lab4-api-2": 93.66, + "messenger-lab4-api-1": 85.37 + }, + "samples": 3, + "naive_n_times_size_mib": 19.02 + }, + "limits_of_measurement": "docker stats RSS is container-level and sampled ~200 ms; it is not a cgroup memory profiler. Python heap is not separated from MinIO/HAProxy. Coordinated start of N POSTs from one client." +} diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..c28adc3a --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,92 @@ +# Client -> HAProxy -> api x N. Replicas are discovered through Docker DNS and checked actively. +# +# Timing contract (keep in sync with compose.yaml and app settings): +# check period inter 1s, timeout check 1s +# marked DOWN after fall 2 failed checks -> hung/draining replica leaves the pool in <= 3 s +# marked UP after rise 2 good checks -> new replica gets traffic ~2 s after it is ready +# DNS refresh every 1 s (hold valid 1s); a removed container disappears within ~1-2 s +# crashed process connect refused -> request redispatched to another replica at once +# app DRAIN_SECONDS=4 > 3 s, so a replica receiving SIGTERM is out of rotation before it closes its socket. + +global + log stdout format raw local0 info + maxconn 2048 + # Runtime API for evidence (scripts/lb-status.sh); loopback inside the container only. + stats socket ipv4@127.0.0.1:9999 level operator + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 0s + +defaults + mode http + log global + option httplog + option dontlognull + option forwardfor + timeout connect 1s + timeout client 30s + timeout server 10s + # Body arrival for a 10 MiB attachment; JSON posts finish well under this. + timeout http-request 30s + timeout http-keep-alive 5s + timeout queue 2s + timeout check 1s + # A request whose connection to a replica failed is sent to another replica. + retries 2 + option redispatch 1 + +frontend public + bind :8080 + monitor-uri /lb/health + # Attachment uploads are raw bytes, up to 10 MiB. A larger Content-Length is + # rejected here, before the body is read or an API replica is contacted. + # The body matches the API error envelope; request_id echoes a safe client + # X-Request-Id and otherwise stays "lb", like the other gateway errors. + acl upload_too_large req.hdr_val(content-length) gt 10485760 + acl safe_request_id req.hdr(X-Request-Id) -m reg ^[A-Za-z0-9._:-]{1,128}$ + http-request set-var(txn.reqid) req.hdr(X-Request-Id) if upload_too_large safe_request_id + http-request set-var(txn.reqid) str(lb) if upload_too_large !safe_request_id + http-request return status 413 content-type application/json lf-string "{\"error\":{\"code\":\"payload_too_large\",\"message\":\"Файл больше 10 МиБ\",\"request_id\":\"%[var(txn.reqid)]\"}}" hdr X-Request-Id %[var(txn.reqid)] hdr X-Instance-Id %[str(lb)] if upload_too_large + + # Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe. + # Other POSTs (users, chats, sessions) are retried only when the connection was never established. + acl message_post method POST + acl message_post_path path_reg ^/api/v1/chats/[^/]+/messages$ + http-request disable-l7-retry if message_post !message_post_path + + default_backend api + +backend api + # Round robin, no stickiness: any replica serves any session from PostgreSQL. + # Attachment PUT to S3 can outlast the default 10 s server timeout under concurrent load. + acl attachment_upload method POST + acl attachment_upload_path path_reg ^/api/v1/chats/[^/]+/attachments$ + http-request set-timeout server 30s if attachment_upload attachment_upload_path + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + retry-on conn-failure empty-response response-timeout + http-error status 503 content-type application/json string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API replica is available\",\"request_id\":\"lb\"}}" + http-error status 502 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Bad gateway\",\"request_id\":\"lb\"}}" + http-error status 504 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Gateway timeout\",\"request_id\":\"lb\"}}" + default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none + server-template api 1-10 api:8080 + +# Read-only stats for evidence; not published, reachable only inside the container. +frontend stats + bind 127.0.0.1:8404 + stats enable + stats uri /stats + stats refresh 5s + http-request set-log-level silent diff --git a/infra/minio/bootstrap.sh b/infra/minio/bootstrap.sh new file mode 100644 index 00000000..a5c8024f --- /dev/null +++ b/infra/minio/bootstrap.sh @@ -0,0 +1,116 @@ +#!/bin/sh +# Idempotent private-bucket bootstrap for MinIO. Root credentials are read from +# files and are not printed. A second run leaves the bucket, policy and API user in place. +set -eu + +bucket=${S3_BUCKET:?S3_BUCKET is required} +region=${S3_REGION:-us-east-1} +endpoint=${MINIO_ENDPOINT:-http://minio:9000} +policy_name=attachments-api + +case "$bucket" in + [a-z0-9][a-z0-9.-]*[a-z0-9]) ;; + *) + echo "S3_BUCKET must be a DNS-style name" >&2 + exit 1 + ;; +esac + +read_secret() { + tr -d '\r\n' < "$1" +} + +root_user=$(read_secret /run/secrets/minio_root_user) +root_password=$(read_secret /run/secrets/minio_root_password) +access_key=$(read_secret /run/secrets/s3_access_key_id) +secret_key=$(read_secret /run/secrets/s3_secret_access_key) + +if [ -z "$root_user" ] || [ -z "$root_password" ] || [ -z "$access_key" ] || [ -z "$secret_key" ]; then + echo "MinIO bootstrap secrets are empty" >&2 + exit 1 +fi + +i=0 +until mc alias set local "$endpoint" "$root_user" "$root_password" >/dev/null; do + i=$((i + 1)) + if [ "$i" -ge 30 ]; then + echo "MinIO is not accepting the root credentials" >&2 + exit 1 + fi + sleep 1 +done + +# A new MinIO bucket has no anonymous policy. Set none explicitly so a later +# public policy cannot survive a repeated bootstrap. +mc mb --ignore-existing --region "$region" "local/${bucket}" >/dev/null +mc anonymous set none "local/${bucket}" >/dev/null + +anon=$(mc anonymous get "local/${bucket}") +case "$anon" in + *private*) ;; + *) + echo "bucket ${bucket} is not private" >&2 + exit 1 + ;; +esac + +probe_key=".bootstrap/private-probe" +printf 'probe' | mc pipe "local/${bucket}/${probe_key}" >/dev/null +code=$(curl -s -o /dev/null -w '%{http_code}' "${endpoint}/${bucket}/${probe_key}") +mc rm --force "local/${bucket}/${probe_key}" >/dev/null +if [ "$code" != "403" ]; then + echo "unauthenticated GET returned ${code}, expected 403" >&2 + exit 1 +fi + +policy_file=/tmp/attachments-api-policy.json +cat > "$policy_file" </dev/null 2>&1; then + mc admin policy detach local "$policy_name" --user "$access_key" >/dev/null 2>&1 || true +fi +mc admin policy remove local "$policy_name" >/dev/null 2>&1 || \ + mc admin policy rm local "$policy_name" >/dev/null 2>&1 || true +mc admin policy create local "$policy_name" "$policy_file" >/dev/null + +if ! mc admin user info local "$access_key" >/dev/null 2>&1; then + mc admin user add local "$access_key" "$secret_key" >/dev/null +fi + +mc admin policy attach local "$policy_name" --user "$access_key" >/dev/null + +mc alias set api "$endpoint" "$access_key" "$secret_key" >/dev/null +api_key=".bootstrap/api-probe" +printf 'ok' | mc pipe "api/${bucket}/${api_key}" >/dev/null +mc cat "api/${bucket}/${api_key}" >/dev/null +mc rm --force "api/${bucket}/${api_key}" >/dev/null + +if mc ls "api/not-this-bucket" >/dev/null 2>&1; then + echo "API credentials can list a different bucket" >&2 + exit 1 +fi + +echo "bucket ${bucket} is private" diff --git a/infra/minio/check-private-bucket.sh b/infra/minio/check-private-bucket.sh new file mode 100644 index 00000000..2bb22f97 --- /dev/null +++ b/infra/minio/check-private-bucket.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# In-network proof that the bucket is private and API credentials cannot see other buckets. +# Run via: docker compose --env-file .env --profile ops run --rm check-private-bucket +set -eu + +bucket=${S3_BUCKET:?S3_BUCKET is required} +endpoint=${MINIO_ENDPOINT:-http://minio:9000} + +read_secret() { + tr -d '\r\n' < "$1" +} + +root_user=$(read_secret /run/secrets/minio_root_user) +root_password=$(read_secret /run/secrets/minio_root_password) +access_key=$(read_secret /run/secrets/s3_access_key_id) +secret_key=$(read_secret /run/secrets/s3_secret_access_key) + +mc alias set local "$endpoint" "$root_user" "$root_password" >/dev/null + +# MINIO_BROWSER=off: nothing should answer on the console port. +if curl -sf --connect-timeout 2 "http://minio:9001/" >/dev/null 2>&1; then + echo "MinIO console answered on :9001" >&2 + exit 1 +fi + +probe_key=".bootstrap/private-check" +printf 'probe' | mc pipe "local/${bucket}/${probe_key}" >/dev/null +code=$(curl -s -o /dev/null -w '%{http_code}' "${endpoint}/${bucket}/${probe_key}") +mc rm --force "local/${bucket}/${probe_key}" >/dev/null +if [ "$code" != "403" ]; then + echo "unauthenticated GET returned ${code}, expected 403" >&2 + exit 1 +fi + +mc alias set api "$endpoint" "$access_key" "$secret_key" >/dev/null +if mc ls "api/not-this-bucket" >/dev/null 2>&1; then + echo "API credentials can list a different bucket" >&2 + exit 1 +fi + +echo "PASS: bucket ${bucket} is private; unauthenticated GET is 403; API user cannot list another bucket; console is closed" diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..5933542d --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,53 @@ +-- Initial schema. The runner records this file in schema_migrations and skips it on the next start. + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT users_username_unique UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); + +CREATE TABLE sessions ( + token_hash TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users (id), + expires_at TIMESTAMPTZ NOT NULL +); + +CREATE TABLE chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +CREATE TABLE chat_members ( + chat_id UUID NOT NULL REFERENCES chats (id), + user_id UUID NOT NULL REFERENCES users (id), + position INTEGER NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_nonnegative CHECK (position >= 0) +); + +CREATE TABLE messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + sender_id UUID NOT NULL REFERENCES users (id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); + +-- Keyset history: WHERE chat_id = ? AND (created_at, id) > cursor ORDER BY created_at, id. +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +-- Chat list is filtered by the caller, then ordered by the chat's own (created_at, id). +CREATE INDEX chat_members_user_idx ON chat_members (user_id); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..c9f4da7f --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,16 @@ +-- Result of POST /chats/{id}/messages per (user, method, path, Idempotency-Key). +-- The row and the message are written in one transaction, so a failed attempt leaves no key behind. +CREATE TABLE idempotency_keys ( + user_id UUID NOT NULL REFERENCES users (id), + method TEXT NOT NULL, + path TEXT NOT NULL, + key TEXT NOT NULL, + request_hash TEXT NOT NULL, + status_code INTEGER NOT NULL, + response JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, method, path, key), + CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$') +); + +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/migrations/003_session_indexes.sql b/migrations/003_session_indexes.sql new file mode 100644 index 00000000..0b7eb784 --- /dev/null +++ b/migrations/003_session_indexes.sql @@ -0,0 +1,4 @@ +-- Expand-only: new indexes, no column/table removal or rename, so replicas of the previous build keep working +-- while the rolling update is in progress. Destructive changes go to a later release, after old replicas are gone. +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); +CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/004_attachments.sql b/migrations/004_attachments.sql new file mode 100644 index 00000000..2f4673d4 --- /dev/null +++ b/migrations/004_attachments.sql @@ -0,0 +1,36 @@ +-- Lab4: message may be text, attachments, or both. Empty text is stored only when +-- an attachment is linked; the API rejects a blank message that has neither. +ALTER TABLE messages DROP CONSTRAINT messages_text_len; +ALTER TABLE messages ADD CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 0 AND 4000); + +-- filename is the display name only. object_key is assigned by the server and is never taken from filename. +-- The original bytes live in the private S3 bucket under that key, not in this table. +CREATE TABLE attachments ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats (id), + owner_id UUID NOT NULL REFERENCES users (id), + filename TEXT NOT NULL, + content_type TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + sha256 TEXT NOT NULL, + object_key TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + CONSTRAINT attachments_filename_len CHECK (char_length(filename) BETWEEN 1 AND 255), + CONSTRAINT attachments_content_type CHECK (content_type IN ('image/png', 'image/jpeg')), + CONSTRAINT attachments_size CHECK (size_bytes BETWEEN 1 AND 10485760), + CONSTRAINT attachments_sha256 CHECK (sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT attachments_object_key_unique UNIQUE (object_key) +); + +CREATE INDEX attachments_chat_idx ON attachments (chat_id); + +CREATE TABLE message_attachments ( + message_id UUID NOT NULL REFERENCES messages (id), + attachment_id UUID NOT NULL REFERENCES attachments (id), + position INTEGER NOT NULL, + PRIMARY KEY (message_id, attachment_id), + CONSTRAINT message_attachments_position CHECK (position >= 0 AND position < 10) +); + +CREATE INDEX message_attachments_message_idx ON message_attachments (message_id, position); diff --git a/migrations/005_attachment_bytes_in_s3.sql b/migrations/005_attachment_bytes_in_s3.sql new file mode 100644 index 00000000..81fdfdef --- /dev/null +++ b/migrations/005_attachment_bytes_in_s3.sql @@ -0,0 +1,4 @@ +-- Databases that already applied the earlier 004 kept the original in a BYTEA column. +-- Drop it. Fresh installs create the table without that column; both statements are then no-ops. +ALTER TABLE attachments DROP CONSTRAINT IF EXISTS attachments_body_len; +ALTER TABLE attachments DROP COLUMN IF EXISTS body; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..3298a46c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,26 @@ +# Direct: fastapi, pydantic, uvicorn, argon2-cffi, boto3. The rest are their dependencies, pinned. +boto3==1.43.126 +botocore==1.43.126 +asyncpg==0.32.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi==25.1.0 +argon2-cffi-bindings==25.1.0 +cffi==2.0.0 +click==8.5.0 +fastapi==0.141.1 +h11==0.16.0 +idna==3.20 +jmespath==1.1.0 +pycparser==3.0 +python-dateutil==2.9.0.post0 +pydantic==2.13.5 +pydantic_core==2.46.5 +s3transfer==0.19.3 +six==1.17.0 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.53.0 diff --git a/scripts/check_private_bucket.py b/scripts/check_private_bucket.py new file mode 100644 index 00000000..dcb0e6c9 --- /dev/null +++ b/scripts/check_private_bucket.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Host-side private-bucket checks plus the in-network probe. + +Confirms MinIO publishes no host ports, the console env is off, then runs the +compose service that performs an unauthenticated GET from the data network. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def _compose(args): + env_file = ROOT / '.env' + cmd = ['docker', 'compose'] + if env_file.is_file(): + cmd.extend(['--env-file', str(env_file)]) + cmd.extend(args) + return subprocess.run(cmd, cwd=str(ROOT), check=True, capture_output=True, text=True) + + +def main() -> None: + os.chdir(ROOT) + result = _compose(['config', '--format', 'json']) + config = json.loads(result.stdout) + services = config.get('services') or {} + minio = services.get('minio') + if minio is None: + raise SystemExit('FAIL: compose config has no minio service') + ports = minio.get('ports') or [] + if ports: + raise SystemExit('FAIL: minio publishes host ports: %s' % ports) + env = minio.get('environment') or {} + browser = str(env.get('MINIO_BROWSER', '')).strip().strip('"').lower() + if browser not in ('off', 'false', '0'): + raise SystemExit('FAIL: MINIO_BROWSER is %r, expected off' % env.get('MINIO_BROWSER')) + networks = (minio.get('networks') or {}) + if 'edge' in networks: + raise SystemExit('FAIL: minio is attached to the edge network') + probe = subprocess.run( + [ + 'docker', 'compose', + *(['--env-file', str(ROOT / '.env')] if (ROOT / '.env').is_file() else []), + '--profile', 'ops', + 'run', '--rm', 'check-private-bucket', + ], + cwd=str(ROOT), + check=False, + text=True, + ) + if probe.returncode != 0: + raise SystemExit(probe.returncode) + print('PASS: minio has no host ports, console env is off, in-network GET is 403') + + +if __name__ == '__main__': + try: + main() + except subprocess.CalledProcessError as exc: + sys.stderr.write(exc.stderr or exc.stdout or str(exc)) + raise SystemExit(exc.returncode) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..efc12d4e --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Restart the LB, every API replica, PostgreSQL and MinIO. Volumes stay; nothing is re-seeded. +docker compose --env-file .env stop lb api +docker compose --env-file .env restart postgres minio +# Brings back all replicas (including one left stopped by stop-one.sh); migrate re-runs as a no-op. +docker compose --env-file .env up -d --wait diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..0cdb2a01 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,29 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Stop exactly ONE API replica and leave it stopped. DB, LB and volumes are untouched. +# X-Instance-Id is the container hostname, i.e. the short Docker container ID. +# Restore afterwards with: docker compose --env-file .env up -d --wait +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + *[!0-9a-f]*) valid=no ;; + *) valid=yes ;; +esac +if [ "$valid" = no ] || [ "${#TARGET_INSTANCE_ID}" -lt 12 ]; then + echo "TARGET_INSTANCE_ID is not a container ID" >&2 + exit 2 +fi +project="${COMPOSE_PROJECT_NAME:-messenger-lab4}" +matches=$(docker ps -q --no-trunc \ + --filter "label=com.docker.compose.project=$project" \ + --filter "label=com.docker.compose.service=api" | grep "^$TARGET_INSTANCE_ID" || true) +count=$(printf '%s' "$matches" | grep -c . || true) +if [ "$count" -ne 1 ]; then + echo "expected one running api container for $TARGET_INSTANCE_ID, found $count" >&2 + exit 1 +fi +name=$(docker inspect --format '{{.Name}}' "$matches") +# Graceful: SIGTERM -> 4 s drain (readiness 503, LB drops it) -> <= 3 s in-flight finish. +# SIGKILL only after 9 s, so the hook still returns before the test's 10 s limit. +docker stop --time 9 "$matches" >/dev/null +echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)" diff --git a/scripts/upload_mem.py b/scripts/upload_mem.py new file mode 100644 index 00000000..e22cc932 --- /dev/null +++ b/scripts/upload_mem.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Concurrent attachment uploads with peak RSS of API containers. + +Generates a valid PNG at runtime (not stored in Git). Measures docker stats while +N workers POST the same image. Writes a sanitized JSON summary. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import subprocess +import sys +import threading +import time +import uuid +import zlib +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure # noqa: E402 + + +def make_png(width: int, height: int) -> bytes: + rows = [] + for _y in range(height): + row = bytearray(1 + width * 3) + row[0] = 0 + row[1:] = os.urandom(width * 3) + rows.append(bytes(row)) + raw = b''.join(rows) + compressed = zlib.compress(raw, 1) + + def chunk(tag: bytes, data: bytes) -> bytes: + crc = zlib.crc32(tag + data) & 0xffffffff + return len(data).to_bytes(4, 'big') + tag + data + crc.to_bytes(4, 'big') + + ihdr = width.to_bytes(4, 'big') + height.to_bytes(4, 'big') + bytes([8, 2, 0, 0, 0]) + return b'\x89PNG\r\n\x1a\n' + chunk(b'IHDR', ihdr) + chunk(b'IDAT', compressed) + chunk(b'IEND', b'') + + +def parse_mib(usage: str) -> float: + # "45.23MiB / 256MiB" or "123.4MB / 256MB" + left = usage.split('/', 1)[0].strip() + number = '' + unit = '' + for char in left: + if char.isdigit() or char in '.': + number += char + else: + unit += char + value = float(number or '0') + unit = unit.strip().lower() + if unit in ('gib', 'gb'): + return value * 1024 + if unit in ('kib', 'kb'): + return value / 1024 + if unit in ('b',): + return value / (1024 * 1024) + return value + + +def git_commit() -> str: + try: + return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=str(ROOT), text=True).strip() + except (OSError, subprocess.CalledProcessError): + return 'unknown' + + +def docker_stats() -> list: + try: + ids = subprocess.check_output( + ['docker', 'ps', '-q', '--filter', 'name=messenger-lab4-api'], + cwd=str(ROOT), + text=True, + timeout=5, + ).split() + if not ids: + return [] + output = subprocess.check_output( + ['docker', 'stats', '--no-stream', '--format', '{{.Name}}\t{{.MemUsage}}\t{{.MemPerc}}', *ids], + cwd=str(ROOT), + text=True, + timeout=8, + ) + except (OSError, subprocess.CalledProcessError, subprocess.TimeoutExpired): + return [] + rows = [] + for line in output.splitlines(): + parts = line.split('\t') + if len(parts) < 2: + continue + name, usage = parts[0], parts[1] + rows.append({'name': name, 'mem_usage': usage, 'rss_mib': round(parse_mib(usage), 2)}) + return rows + + +def main() -> None: + parser = argparse.ArgumentParser(description='Concurrent upload memory experiment') + parser.add_argument('--base-url', default=os.environ.get('BASE_URL', 'http://localhost:8080')) + parser.add_argument('--concurrency', type=int, default=10) + parser.add_argument('--width', type=int, default=900) + parser.add_argument('--height', type=int, default=700) + parser.add_argument('--output', default=str(ROOT / 'evidence' / 'upload-concurrency.json')) + args = parser.parse_args() + if args.concurrency < 10: + raise SystemExit('concurrency must be >= 10') + + image = make_png(args.width, args.height) + size = len(image) + digest = hashlib.sha256(image).hexdigest() + client = Client(args.base_url, timeout=60) + alice, _bob, _eve, chat = client.bootstrap() + path = '/api/v1/chats/%s/attachments?filename=load.png' % chat['id'] + + stop = threading.Event() + samples = [] + + def watch(): + while not stop.is_set(): + snapshot = docker_stats() + if snapshot: + samples.append({'ts': time.time(), 'containers': snapshot}) + stop.wait(0.05) + + watcher = threading.Thread(target=watch, daemon=True) + watcher.start() + baseline = docker_stats() + if baseline: + samples.append({'ts': time.time(), 'containers': baseline, 'phase': 'baseline'}) + started = time.monotonic() + latencies = [] + errors = 0 + + def one(_index: int): + response = client.call( + 'POST', + path, + 201, + actor=alice, + raw=image, + headers={'Content-Type': 'image/png'}, + schema='Attachment', + ) + body = response['json'] + if body.get('sha256') != digest or body.get('size_bytes') != size: + raise Failure('checksum mismatch on concurrent upload') + return response['elapsed'] + + try: + with ThreadPoolExecutor(max_workers=args.concurrency) as pool: + futs = [pool.submit(one, i) for i in range(args.concurrency)] + for fut in as_completed(futs): + try: + latencies.append(fut.result()) + except Exception: + errors += 1 + finally: + stop.set() + watcher.join(timeout=2) + + elapsed = time.monotonic() - started + latencies.sort() + + def pct(p): + if not latencies: + return None + index = min(len(latencies) - 1, max(0, int(round((p / 100) * (len(latencies) - 1))))) + return round(latencies[index], 4) + + peak = 0.0 + per_container = {} + for sample in samples: + for row in sample['containers']: + peak = max(peak, row['rss_mib']) + prev = per_container.get(row['name'], 0.0) + per_container[row['name']] = max(prev, row['rss_mib']) + + naive_bound_mib = round((args.concurrency * size) / (1024 * 1024), 2) + summary = { + 'hypothesis': ( + 'Ten concurrent ~%.1f MiB PNG uploads stay inside the 256 MiB API limit and ' + 'peak RSS is not proportional to N * file_size because the body is spooled, not held twice as bytes.' + % (size / (1024 * 1024)) + ), + 'when': datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'commit': git_commit(), + 'host': { + 'system': platform.system(), + 'machine': platform.machine(), + 'processor': platform.processor(), + }, + 'compose_limits': {'api_memory': '256M', 'replicas': 2}, + 'profile': { + 'concurrency': args.concurrency, + 'width': args.width, + 'height': args.height, + 'size_bytes': size, + 'sha256': digest, + }, + 'results': { + 'ok': len(latencies), + 'errors': errors, + 'elapsed_s': round(elapsed, 3), + 'p50_s': pct(50), + 'p95_s': pct(95), + 'p99_s': pct(99), + 'peak_api_rss_mib': round(peak, 2), + 'peak_rss_by_container_mib': per_container, + 'samples': len(samples), + 'naive_n_times_size_mib': naive_bound_mib, + }, + 'limits_of_measurement': ( + 'docker stats RSS is container-level and sampled ~200 ms; it is not a cgroup memory profiler. ' + 'Python heap is not separated from MinIO/HAProxy. Coordinated start of N POSTs from one client.' + ), + } + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(summary, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print('ok=%s errors=%s peak_rss_mib=%s naive_N_x_size_mib=%s file=%s' % ( + len(latencies), errors, summary['results']['peak_api_rss_mib'], naive_bound_mib, out, + )) + if errors: + raise SystemExit(1) + + +if __name__ == '__main__': + try: + main() + except Failure as exc: + print('FAIL:', exc) + raise SystemExit(1) diff --git a/tests/attachments_negative.py b/tests/attachments_negative.py new file mode 100644 index 00000000..5f6aaba2 --- /dev/null +++ b/tests/attachments_negative.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Student negative suite for lab4 attachments. Public smoke.py is not modified.""" +import http.client +import json +import os +import struct +import uuid +import zlib +from urllib.parse import urlparse + +from client import Client, Failure, arguments, check, run, shape + +MAX_UPLOAD_BYTES = 10_485_760 +ROOT = os.path.dirname(os.path.abspath(__file__)) + + +def png_header(width, height): + ihdr = struct.pack('>IIBBBBB', width, height, 8, 2, 0, 0, 0) + data = ihdr + crc = zlib.crc32(b'IHDR' + data) & 0xffffffff + chunk = struct.pack('>I', len(data)) + b'IHDR' + data + struct.pack('>I', crc) + return b'\x89PNG\r\n\x1a\n' + chunk + + +def error_payload(status, headers, body, where): + check(status != 500, where + ' returned 500 for a client error') + check(bool(headers.get('X-Request-Id')), where + ': missing X-Request-Id') + content_type = headers.get_content_type() if hasattr(headers, 'get_content_type') else (headers.get('Content-Type') or '').split(';', 1)[0].strip().lower() + check(content_type == 'application/json', where + ': expected JSON error') + try: + payload = json.loads(body) + except (ValueError, UnicodeError): + raise Failure(where + ': invalid error JSON') from None + shape(payload, 'Error') + check(payload['error']['request_id'] == headers.get('X-Request-Id'), where + ': request_id mismatch') + return payload + + +def raw_post(base, path, headers, body, extra_headers=None, chunked=False): + parsed = urlparse(base) + conn = http.client.HTTPConnection(parsed.hostname, parsed.port or 80, timeout=60) + try: + hdrs = dict(headers) + if extra_headers: + hdrs.update(extra_headers) + conn.putrequest('POST', path) + if chunked: + hdrs.pop('Content-Length', None) + for name, value in hdrs.items(): + conn.putheader(name, value) + conn.putheader('Transfer-Encoding', 'chunked') + conn.endheaders() + if isinstance(body, (bytes, bytearray)): + pieces = [bytes(body)] + else: + pieces = body + for piece in pieces: + conn.send(('%x\r\n' % len(piece)).encode('ascii') + piece + b'\r\n') + conn.send(b'0\r\n\r\n') + else: + payload = body if isinstance(body, (bytes, bytearray)) else b'' + if not any(name.lower() == 'content-length' for name in hdrs): + hdrs['Content-Length'] = str(len(payload)) + for name, value in hdrs.items(): + conn.putheader(name, value) + conn.endheaders() + if payload: + conn.send(payload) + response = conn.getresponse() + data = response.read(1024 * 1024) + hdr = response.headers + return response.status, hdr, data + finally: + conn.close() + + +def expect_error(c, actor, chat, *, filename='photo.png', content_type='image/png', body=b'', status=None, extra_headers=None, chunked=False, pieces=None, where='upload'): + path = '/api/v1/chats/%s/attachments?filename=%s' % (chat['id'], filename) + headers = { + 'Authorization': 'Bearer ' + actor['token'], + 'Content-Type': content_type, + 'Accept': 'application/json', + } + payload_body = pieces if chunked and pieces is not None else body + code, hdr, data = raw_post(c.base, path, headers, payload_body, extra_headers=extra_headers, chunked=chunked) + payload = error_payload(code, hdr, data, where) + if status is not None: + check(code == status, '%s: expected %s, received %s (%s)' % (where, status, code, payload['error']['code'])) + return code, payload + + +def main(): + args = arguments('Lab4 attachment negative cases') + c = Client(args.base_url, args.ca_file, timeout=60) + alice, bob, eve, chat = c.bootstrap() + sample = open(os.path.join(ROOT, 'fixtures', 'sample.png'), 'rb').read() + + code, payload = expect_error( + c, alice, chat, + extra_headers={'Content-Length': str(MAX_UPLOAD_BYTES + 1)}, + body=b'x', + status=413, + where='oversized Content-Length', + ) + check(payload['error']['code'] == 'payload_too_large', 'oversized Content-Length: wrong code') + + header = png_header(32, 32) + remain = MAX_UPLOAD_BYTES + 1 - len(header) + chunk = b'\x00' * 65536 + pieces = [header] + sent = len(header) + while sent < len(header) + remain: + take = min(len(chunk), len(header) + remain - sent) + pieces.append(chunk[:take]) + sent += take + code, payload = expect_error( + c, alice, chat, + chunked=True, + pieces=pieces, + status=413, + where='oversized chunked body', + ) + check(payload['error']['code'] == 'payload_too_large', 'oversized chunked body: wrong code') + + expect_error(c, alice, chat, body=b'\x89PNG\r\n\x1a\n\x00\x00\x00\x0dnotH', status=415, where='corrupt PNG') + expect_error(c, alice, chat, body=b'\xff\xd8\xff\xe0\x00', status=415, where='truncated JPEG') + expect_error(c, alice, chat, body=b'GIF89a-not-an-image', status=415, where='foreign format') + + code, payload = expect_error( + c, alice, chat, + content_type='image/jpeg', + body=sample, + status=415, + where='MIME spoof', + ) + check(payload['error']['code'] == 'unsupported_media_type', 'MIME spoof: wrong code') + + bomb = png_header(10000, 5000) + code, payload = expect_error(c, alice, chat, body=bomb, status=413, where='pixel bomb') + check(payload['error']['code'] == 'payload_too_large', 'pixel bomb: wrong code') + + att, _content = c.attachment(alice, chat) + for suffix in ('', '/content'): + unknown = c.call('GET', '/api/v1/attachments/' + str(uuid.uuid4()) + suffix, 404, actor=alice) + check(unknown['json']['error']['code'] == 'not_found', 'random id: wrong code') + check(unknown['status'] != 500, 'random id returned 500') + foreign = c.call('GET', '/api/v1/attachments/' + att['id'] + suffix, 404, actor=eve) + check(foreign['json']['error']['code'] == 'not_found', 'outsider: wrong code') + + other = c.call( + 'POST', + '/api/v1/chats', + 201, + actor=alice, + body={'title': 'Чужой чат', 'member_ids': [alice['id']]}, + schema='Chat', + )['json'] + linked = c.call( + 'POST', + '/api/v1/chats/' + other['id'] + '/messages', + 404, + actor=alice, + body={'text': '', 'attachment_ids': [att['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}, + ) + check(linked['json']['error']['code'] == 'not_found', 'cross-chat attach: wrong code') + check(linked['status'] != 500, 'cross-chat attach returned 500') + + print('PASS: lab4 attachment negative cases (413/415/404, never 500)') + + +if __name__ == '__main__': + run(main)