From b99ad646e824c9c2c79aed8afb3618b4839861a0 Mon Sep 17 00:00:00 2001 From: lenovo Date: Mon, 2 Nov 2026 23:40:07 +0300 Subject: [PATCH] =?UTF-8?q?=D0=A0=D0=B5=D0=B0=D0=BB=D0=B8=D0=B7=D0=BE?= =?UTF-8?q?=D0=B2=D0=B0=D0=BD=D0=BE=20=D1=85=D1=80=D0=B0=D0=BD=D0=B8=D0=BB?= =?UTF-8?q?=D0=B8=D1=89=D0=B5=20=D0=B2=D0=BB=D0=BE=D0=B6=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D0=B9=20=D0=B4=D0=BB=D1=8F=20=D0=BB=D0=B0=D0=B1=D0=BE=D1=80?= =?UTF-8?q?=D0=B0=D1=82=D0=BE=D1=80=D0=BD=D0=BE=D0=B9=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 30 +- .gitattributes | 2 + .github/pull_request_template.md | 31 ++- Dockerfile | 16 ++ Makefile | 23 +- REPORT.md | 96 +++++-- app/__init__.py | 0 app/api/__init__.py | 0 app/api/deps.py | 42 +++ app/api/routes/__init__.py | 0 app/api/routes/attachments.py | 155 +++++++++++ app/api/routes/auth.py | 31 +++ app/api/routes/chats.py | 33 +++ app/api/routes/health.py | 17 ++ app/api/routes/me.py | 11 + app/api/routes/messages.py | 52 ++++ app/api/routes/users.py | 14 + app/api/serialize.py | 17 ++ app/core/__init__.py | 0 app/core/config.py | 55 ++++ app/core/cursors.py | 37 +++ app/core/errors.py | 72 +++++ app/core/security.py | 39 +++ app/db/__init__.py | 1 + app/db/migrate.py | 56 ++++ app/main.py | 57 ++++ app/repositories/__init__.py | 0 app/repositories/memory.py | 148 ++++++++++ app/repositories/object_store.py | 109 ++++++++ app/repositories/postgres.py | 278 +++++++++++++++++++ app/repositories/redis_sessions.py | 66 +++++ app/schemas.py | 85 ++++++ compose.yaml | 130 +++++++++ evidence/compose-config.txt | 1 + evidence/compose-final.txt | 7 + evidence/compose-yaml.txt | 1 + evidence/diff-check.txt | 1 + evidence/failover-before.txt | 8 + evidence/failover-restore.txt | 1 + evidence/failover-restored.txt | 8 + evidence/failover-stopped.txt | 8 + evidence/hooks-syntax.txt | 1 + evidence/make-check.txt | 2 + evidence/make-test.txt | 2 + evidence/make-validate-first-attempt.txt | 6 + evidence/make-validate.txt | 2 + evidence/migrations-repeat.txt | 15 + evidence/persistence-after.txt | 7 + evidence/persistence-before.txt | 7 + evidence/python-compileall.txt | 1 + evidence/restore-api.txt | 1 + evidence/restore-final.txt | 21 ++ evidence/static-checks.txt | 7 + evidence/test-attachment-failover.txt | 2 + evidence/test-failover.txt | 4 + evidence/test-idempotency.txt | 2 + evidence/test-persistence.txt | 3 + evidence/test-private-bucket.txt | 3 + evidence/test-s3-failure-analysis.txt | 7 + evidence/test-s3-failure-first-attempt.txt | 3 + evidence/test-s3-failure.txt | 3 + evidence/test-student.txt | 2 + evidence/verification-metadata.txt | 12 + migrations/0001_initial.sql | 37 +++ migrations/0002_remove_postgres_sessions.sql | 3 + migrations/0003_message_idempotency.sql | 10 + migrations/0004_attachments.sql | 37 +++ nginx/default.conf | 29 ++ requirements-app.txt | 8 + scripts/restart.sh | 9 + scripts/stop-one.sh | 15 + student-tests/README.md | 31 +++ student-tests/attachment_failover.py | 74 +++++ student-tests/attachments.py | 79 ++++++ student-tests/idempotency.py | 53 ++++ student-tests/private_bucket.py | 43 +++ student-tests/s3_failure.py | 73 +++++ 77 files changed, 2300 insertions(+), 52 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/api/__init__.py create mode 100644 app/api/deps.py create mode 100644 app/api/routes/__init__.py create mode 100644 app/api/routes/attachments.py create mode 100644 app/api/routes/auth.py create mode 100644 app/api/routes/chats.py create mode 100644 app/api/routes/health.py create mode 100644 app/api/routes/me.py create mode 100644 app/api/routes/messages.py create mode 100644 app/api/routes/users.py create mode 100644 app/api/serialize.py create mode 100644 app/core/__init__.py create mode 100644 app/core/config.py create mode 100644 app/core/cursors.py create mode 100644 app/core/errors.py create mode 100644 app/core/security.py create mode 100644 app/db/__init__.py create mode 100644 app/db/migrate.py create mode 100644 app/main.py create mode 100644 app/repositories/__init__.py create mode 100644 app/repositories/memory.py create mode 100644 app/repositories/object_store.py create mode 100644 app/repositories/postgres.py create mode 100644 app/repositories/redis_sessions.py create mode 100644 app/schemas.py create mode 100644 compose.yaml create mode 100644 evidence/compose-config.txt create mode 100644 evidence/compose-final.txt create mode 100644 evidence/compose-yaml.txt create mode 100644 evidence/diff-check.txt create mode 100644 evidence/failover-before.txt create mode 100644 evidence/failover-restore.txt create mode 100644 evidence/failover-restored.txt create mode 100644 evidence/failover-stopped.txt create mode 100644 evidence/hooks-syntax.txt create mode 100644 evidence/make-check.txt create mode 100644 evidence/make-test.txt create mode 100644 evidence/make-validate-first-attempt.txt create mode 100644 evidence/make-validate.txt create mode 100644 evidence/migrations-repeat.txt create mode 100644 evidence/persistence-after.txt create mode 100644 evidence/persistence-before.txt create mode 100644 evidence/python-compileall.txt create mode 100644 evidence/restore-api.txt create mode 100644 evidence/restore-final.txt create mode 100644 evidence/static-checks.txt create mode 100644 evidence/test-attachment-failover.txt create mode 100644 evidence/test-failover.txt create mode 100644 evidence/test-idempotency.txt create mode 100644 evidence/test-persistence.txt create mode 100644 evidence/test-private-bucket.txt create mode 100644 evidence/test-s3-failure-analysis.txt create mode 100644 evidence/test-s3-failure-first-attempt.txt create mode 100644 evidence/test-s3-failure.txt create mode 100644 evidence/test-student.txt create mode 100644 evidence/verification-metadata.txt create mode 100644 migrations/0001_initial.sql create mode 100644 migrations/0002_remove_postgres_sessions.sql create mode 100644 migrations/0003_message_idempotency.sql create mode 100644 migrations/0004_attachments.sql create mode 100644 nginx/default.conf create mode 100644 requirements-app.txt create mode 100755 scripts/restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 student-tests/README.md create mode 100644 student-tests/attachment_failover.py create mode 100644 student-tests/attachments.py create mode 100644 student-tests/idempotency.py create mode 100644 student-tests/private_bucket.py create mode 100644 student-tests/s3_failure.py diff --git a/.env.example b/.env.example index d99d26a2..a6054e0f 100644 --- a/.env.example +++ b/.env.example @@ -1,20 +1,20 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Copy to .env and replace every placeholder with a local random value. +POSTGRES_USER=messenger +POSTGRES_PASSWORD=replace_with_random_hex +POSTGRES_DB=messenger SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 -S3_ENDPOINT=http://s3:9000 +DB_POOL_MIN_SIZE=1 +DB_POOL_MAX_SIZE=8 +DB_POOL_ACQUIRE_TIMEOUT_SECONDS=3 +DB_COMMAND_TIMEOUT_SECONDS=5 +REDIS_TIMEOUT_SECONDS=2 +S3_ACCESS_KEY_ID=replace_with_random_access_id +S3_SECRET_ACCESS_KEY=replace_with_random_secret S3_REGION=us-east-1 S3_BUCKET=messenger-attachments -S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id -S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key +S3_CONNECT_TIMEOUT_SECONDS=3 +S3_READ_TIMEOUT_SECONDS=10 MAX_UPLOAD_BYTES=10485760 MAX_IMAGE_PIXELS=40000000 +API_REPLICAS=2 +LB_PORT=8080 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..efdba876 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +* text=auto +*.sh text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 1bec5472..48809a80 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,21 @@ ## Лабораторная 4 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Федотов Михаил, БВТ2302 +Целевая оценка: 3 +Трек 5: не заявлен -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и перенесено: обязательная основа Lab 3 (PostgreSQL, Argon2id и Redis-сессии, LB × 2 API, идемпотентность) и Lab 4 (RustFS S3, versioned migration 0004, metadata в PostgreSQL, оригиналы в приватном bucket, authenticated upload/download, attachment_ids в сообщениях). Собственные проверки размещены в student-tests/. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Команды запуска из чистого clone: см. раздел «Запуск из чистого clone» в [REPORT.md](../REPORT.md). Кратко: скопировать .env.example в .env, заменить secrets, выполнить docker compose config и up --build, затем make check/validate/test и student tests. + +Результаты проверок: make check, make validate PYTHON=.venv/bin/python, make test, student tests, S3 outage/recovery, restart, API failover и повторный запуск миграций прошли. Конкретный stdout лежит в evidence/; сводка результатов и ограничения — в REPORT.md. + +Ограничения / известные ошибки: одноузловый RustFS и один Docker host; автоматическая сверка orphan S3 objects и отдельная минимальная S3 policy не реализованы, так как целевой уровень — 3. TLS и отдельный load benchmark не заявлены; стенд рассчитан на localhost и синтетические данные. + +Источники и использование ИИ: исходники Lab 3 — собственная предыдущая работа; RustFS использован по официальной документации. При переносе и реализации использовался ИИ-помощник. + +- [x] Контракт и публичные tests сохранены без ослабления. +- [x] Отчёт содержит инструкцию запуска именно этого решения. +- [x] Secrets и локальные volumes исключены из Git через .gitignore. +- [x] make check, make validate и make test выполнены; приложен реальный вывод. +- [x] Сценарии отказа и сохранности уровня 3 воспроизведены. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..18e9f75e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.12.8-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + AWS_EC2_METADATA_DISABLED=true +WORKDIR /app + +COPY requirements-app.txt ./ +RUN pip install --no-cache-dir -r requirements-app.txt +COPY app ./app +COPY migrations ./migrations + +RUN useradd --create-home --uid 10001 appuser +USER appuser +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Makefile b/Makefile index 5d86d171..7fb845f3 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +.PHONY: help check validate test load test-student test-idempotency test-private-bucket test-s3-failure test-attachment-failover test-persistence test-failover help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI; test: public smoke; test-student: own Lab 4 checks" check: $(PYTHON) scripts/check_contract.py @@ -18,13 +18,28 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +test-student: + $(PYTHON) student-tests/attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-idempotency: + $(PYTHON) student-tests/idempotency.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-private-bucket: + $(PYTHON) student-tests/private_bucket.py + +test-s3-failure: + $(PYTHON) student-tests/s3_failure.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-attachment-failover: + $(PYTHON) student-tests/attachment_failover.py --base-url "$(BASE_URL)" $(TLS_ARGS) + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to scripts/restart.sh"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to scripts/stop-one.sh"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index 9ad2b84f..55c0c2ca 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,95 @@ # Отчёт — лабораторная 4 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Федотов Михаил, БВТ2302 +- Целевая оценка: 3 +- Трек 5: не заявлен +- Commit/PR: ещё не создано по просьбе автора; текущая ветка `solution/BVT2302-Fedotov` +- Использованные источники и помощь: исходный код и миграции собственной Lab 3; [официальная документация RustFS](https://docs.rustfs.com/en/installation/container/docker) и [закреплённый образ 1.0.4-glibc](https://github.com/rustfs/rustfs/pkgs/container/rustfs); при разработке использовался ИИ-помощник. Публичные README, контракты и тесты Lab 4 не копировались поверх новой версии и не ослаблялись. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Engine с Compose V2 и Python 3.9+ для публичных проверок. Приложение собирается в Docker на Python 3.12.8. Подготовьте локальные значения secrets; файл `.env` игнорируется Git: + +```sh +cp .env.example .env +sed -i "s/replace_with_random_hex/$(openssl rand -hex 24)/" .env +sed -i "s/replace_with_random_access_id/$(openssl rand -hex 12)/" .env +sed -i "s/replace_with_random_secret/$(openssl rand -hex 32)/" .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +until curl -fsS http://localhost:8080/health/ready; do sleep 2; done +``` + +Проверки: + +```sh +make check +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +make test BASE_URL=http://localhost:8080 +make test-student BASE_URL=http://localhost:8080 +make test-idempotency BASE_URL=http://localhost:8080 +make test-private-bucket +make test-s3-failure +make test-attachment-failover BASE_URL=http://localhost:8080 +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080 +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080 +docker compose --env-file .env up -d --scale api=2 +``` + +Обычная остановка без удаления данных: `docker compose --env-file .env down`. Намеренное удаление учебной БД, сессий и S3-объектов: `docker compose --env-file .env down -v`; выполнять только при осознанном сбросе стенда. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → Nginx LB → две API-реплики → PostgreSQL, Redis и приватный RustFS S3. Наружу опубликован только LB на localhost:8080; PostgreSQL, Redis, API и S3 доступны только во внутренней Compose-сети. Консоль RustFS выключена и порт S3 на хост не опубликован. + +Из Lab 3 перенесены FastAPI-маршруты и схема Basic → Argon2id/Bearer-сессия, PostgreSQL/Redis-хранилище, миграции 0001–0003, cursor pagination, идемпотентность сообщений, Nginx и hooks отказа/перезапуска. Публичные `tests/`, `contracts/`, README и шаблоны преподавателя не заменялись кодом из предыдущих лабораторных. + +В Lab 4 добавлена миграция 0004: +- `attachments` хранит ID чата/владельца, имя для отображения, MIME, размер, SHA-256, статус и серверный object key; +- `message_attachments` связывает сообщение и вложение, а составные внешние ключи запрещают ссылку на файл другого чата; +- пустой текст сообщения разрешён только если в запросе есть хотя бы одно вложение; проверка выполняется API. + +Оригинальные байты находятся только в S3. API ограниченно читает тело до 10 MiB, распознаёт PNG/JPEG через Pillow, сопоставляет MIME с реальным форматом и проверяет пиксели до декодирования. S3 объект записывается до метаданных; если запись метаданных в PostgreSQL не проходит, API пытается удалить только что записанный объект. HTTP 201 возвращается после записи объекта и метаданных. Object key генерируется как UUID и не зависит от filename. + +Bucket создаётся при старте API, отдельная политика публичного доступа не назначается; S3 endpoint не опубликован наружу. Участник чата получает metadata и оригинал через API, посторонний получает 404. Все сервисы состояния используют именованные volumes: `postgres_data`, `redis_data`, `s3_data`. + +Образы закреплены: Python 3.12.8-slim, PostgreSQL 16.4-alpine, Redis 7.4.2-alpine, Nginx 1.27.4-alpine и RustFS 1.0.4-glibc по digest. Версии Python-зависимостей приложения закреплены в `requirements-app.txt`. + +| Роль | Service | Данные/готовность | Порт и credentials | +| --- | --- | --- | --- | +| API | `api`, 2 реплики | readiness проверяет PostgreSQL, Redis и S3; локального состояния нет | доступ только из Compose-сети | +| Метаданные | `postgres`, 16.4 | именованный volume; `pg_isready` | внутренний 5432; пароль только в локальном `.env` | +| Сессии | `redis`, 7.4.2 | AOF и `appendfsync always`, именованный volume | внутренний 6379 | +| Бинарные объекты | `s3`, RustFS 1.0.4 | приватный bucket, именованный volume; API проверяет `head_bucket` | внутренний 9000; console выключена | +| Балансировщик | `lb`, Nginx 1.27.4 | healthcheck через API readiness | localhost:8080 | ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Критерий | Проверка | Результат финального прогона | +| --- | --- | --- | +| Структура и сохранность контракта | `make check`, `git diff --check` | PASS; `evidence/make-check.txt`, `evidence/diff-check.txt` | +| OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS; `evidence/make-validate.txt` | +| Синтаксис приложения, тестов и hooks | `compileall`, YAML parse, `sh -n`, Compose config | PASS; отдельные stdout-файлы в `evidence/` | +| Публичный API и вложения | `make test`, `make test-student` | PASS; `evidence/make-test.txt`, `evidence/test-student.txt` | +| Идемпотентность и доступ к bucket | `make test-idempotency`, `make test-private-bucket` | PASS; 16 конкурентных запросов, приватный bucket отвечает 403 на анонимный листинг | +| Отказ S3 и чтение между репликами | `make test-s3-failure`, `make test-attachment-failover` | PASS; API вернул 503 при отказе S3, метаданные сохранились, файл читается после восстановления и через вторую API-реплику | +| Перезапуск и API failover | `make test-persistence`, `make test-failover` | PASS; подтверждённые данные и действующая сессия сохранены | +| Повторный запуск миграций | два запуска `migrate` и запрос `schema_migrations` | PASS; в таблице ровно версии 1, 2, 3, 4 | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Первый запуск нового S3-теста оборвался по 5-секундному таймауту тестового HTTP-клиента до завершения повторных попыток S3 SDK. Причина и исправление тестового таймаута записаны в `evidence/test-s3-failure-analysis.txt`; следующий прогон прошёл полностью. Ошибки приложения этим первым запуском не маскируются. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +`scripts/restart.sh` пересоздаёт контейнеры и сеть Compose, не удаляя volumes. Финальный `make test-persistence` подтвердил чтение существующих чатов/сообщений, действующей сессии и оригинала вложения после перезапуска стека. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +`scripts/stop-one.sh` находит API контейнер по `X-Instance-Id` и останавливает только его. Межрепличный тест загрузил вложение, остановил обслужившую запрос API-реплику и прочитал metadata и оригинальные байты через оставшуюся реплику. Нагрузочный failover подтвердил 2 реплики до отказа, 140 успешных записей, 0 неуспешных попыток и восстановление за 0.556 секунды; подтверждённая запись и действующая сессия сохранились. Финальный статус контейнеров записан в `evidence/compose-final.txt`. -## Ограничения и следующие шаги +Тест отказа S3 остановил только service `s3`: readiness и чтение оригинала вернули ограниченный HTTP 503, PostgreSQL metadata оставались доступны, после перезапуска S3 исходные байты читались без изменений. Проверка неавторизованного доступа к bucket получила HTTP 403. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Аппаратный профиль WSL на момент проверки: 32 CPU threads и 15 GiB памяти; Docker Engine 29.8.0. Для каждой API-реплики установлены пределы 1 CPU и 512 MiB; для остальных контейнеров явные resource limits не заданы. Отдельный throughput/load benchmark не запускался и не заявляется. Все фактические stdout-проверки и профиль сохранены в `evidence/`. + +## Ограничения + +Цель этой работы — уровень 3. Один узел RustFS и один Docker host сами по себе не обеспечивают отказоустойчивость к потере диска/хоста. API использует ключ доступа RustFS из локального `.env`; отдельный пользователь S3 с минимальными правами и механизм сверки orphan объектов относятся к дополнительным практикам уровня 4 и не заявляются. Если компенсирующее удаление после ошибки PostgreSQL также не удастся, может остаться orphan-объект; автоматическая сверка не реализована. TLS отсутствует, стенд предназначен для localhost и синтетических данных. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 00000000..b1e5fb59 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,42 @@ +from fastapi import Request + +from app.core.errors import ApiError +from app.repositories.memory import SessionRow, UserRow + +import uuid as _uuid + +BEARER_CHALLENGE = {'WWW-Authenticate': 'Bearer realm="messenger"'} + + +def storage_of(request: Request): + return request.app.state.storage + + +async def current_session(request: Request) -> SessionRow: + header = request.headers.get('authorization', '') + token = header[7:].strip() if header[:7].lower() == 'bearer ' else '' + if not token: + raise ApiError(401, 'Bearer session required', headers=BEARER_CHALLENGE) + row = await storage_of(request).session(token) + if row is None: + raise ApiError(401, 'Session is missing, expired or revoked', headers=BEARER_CHALLENGE) + return row + + +async def current_user(request: Request) -> UserRow: + session = await current_session(request) + user = await storage_of(request).user(session.user_id) + if user is None: + raise ApiError(401, 'Session user no longer exists', headers=BEARER_CHALLENGE) + return user + + +async def require_chat_member(request: Request, chat_id: str, user: UserRow): + try: + _uuid.UUID(chat_id) + except ValueError: + raise ApiError(400, 'chat_id must be a UUID') + row = await storage_of(request).chat(chat_id) + if row is None or user.id not in row.member_ids: + raise ApiError(404, 'Chat is not available for this user') + return row \ No newline at end of file diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/routes/attachments.py b/app/api/routes/attachments.py new file mode 100644 index 00000000..13067420 --- /dev/null +++ b/app/api/routes/attachments.py @@ -0,0 +1,155 @@ +import hashlib +import io +import uuid +import warnings + +from fastapi import APIRouter, Query, Request +from fastapi.responses import Response +from PIL import Image, UnidentifiedImageError + +from app.api.deps import current_user, require_chat_member +from app.core.config import settings +from app.core.errors import ApiError + +router = APIRouter() +MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'} + + +def _validate_filename(filename: str) -> None: + if not filename or len(filename) > 255 or not filename.strip(): + raise ApiError(400, 'Invalid filename') + if any(ch in filename for ch in ('/', chr(92))): + raise ApiError(400, 'Invalid filename') + if any(ord(ch) == 0 or __import__('unicodedata').category(ch).startswith('C') for ch in filename): + raise ApiError(400, 'Invalid filename') + + +async def _read_bounded_body(request: Request) -> bytes: + length = request.headers.get('content-length') + if length: + try: + size = int(length) + except ValueError: + raise ApiError(400, 'Invalid Content-Length') from None + if size < 0: + raise ApiError(400, 'Invalid Content-Length') + if size > settings.max_upload_bytes: + raise ApiError(413, 'Image exceeds the maximum size') + chunks = [] + total = 0 + async for chunk in request.stream(): + total += len(chunk) + if total > settings.max_upload_bytes: + raise ApiError(413, 'Image exceeds the maximum size') + chunks.append(chunk) + if total == 0: + raise ApiError(415, 'Empty or invalid image') + return b''.join(chunks) + + +def _validate_image(data: bytes, declared_type: str) -> None: + if declared_type not in {'image/png', 'image/jpeg'}: + raise ApiError(415, 'Only PNG and JPEG images are accepted') + try: + with warnings.catch_warnings(): + warnings.simplefilter('error', Image.DecompressionBombWarning) + with Image.open(io.BytesIO(data)) as image: + actual_type = MIME_BY_FORMAT.get(image.format) + width, height = image.size + if actual_type is None or actual_type != declared_type: + raise ApiError(415, 'Image content does not match Content-Type') + if width < 1 or height < 1: + raise ApiError(415, 'Invalid image dimensions') + if width * height > settings.max_image_pixels: + raise ApiError(413, 'Image exceeds the maximum pixel count') + image.verify() + except ApiError: + raise + except (UnidentifiedImageError, Image.DecompressionBombError, + Image.DecompressionBombWarning, OSError, SyntaxError, ValueError): + raise ApiError(415, 'Image content is invalid or unsupported') from None + + +def attachment_out(row) -> dict: + return { + 'id': row.id, + 'chat_id': row.chat_id, + 'owner_id': row.owner_id, + 'filename': row.filename, + 'content_type': row.content_type, + 'size_bytes': row.size_bytes, + 'sha256': row.sha256, + 'status': 'ready', + 'variants': [], + 'error_code': None, + 'created_at': row.created_at, + 'updated_at': row.updated_at, + } + + +@router.post('/api/v1/chats/{chat_id}/attachments', status_code=201) +async def upload_attachment( + chat_id: str, + request: Request, + filename: str = Query(min_length=1, max_length=255), +): + user = await current_user(request) + chat = await require_chat_member(request, chat_id, user) + _validate_filename(filename) + content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower() + if content_type not in {'image/png', 'image/jpeg'}: + raise ApiError(415, 'Only PNG and JPEG images are accepted') + data = await _read_bounded_body(request) + _validate_image(data, content_type) + + attachment_id = str(uuid.uuid4()) + object_key = 'original/' + uuid.uuid4().hex + digest = hashlib.sha256(data).hexdigest() + object_store = request.app.state.object_store + await object_store.put(object_key, data, content_type) + try: + row = await request.app.state.storage.create_attachment( + attachment_id=attachment_id, + chat_id=chat.id, + owner_id=user.id, + object_key=object_key, + filename=filename, + content_type=content_type, + size_bytes=len(data), + sha256=digest, + ) + except Exception: + # Если запись метаданных не зафиксирована, пытаемся убрать только что загруженный объект. + try: + await object_store.delete(object_key) + except ApiError: + pass + raise + return attachment_out(row) + + +@router.get('/api/v1/attachments/{attachment_id}') +async def get_attachment(attachment_id: str, request: Request): + user = await current_user(request) + row = await request.app.state.storage.attachment_for_member(attachment_id, user.id) + if row is None: + raise ApiError(404, 'Attachment is not available for this user') + return attachment_out(row) + + +@router.get('/api/v1/attachments/{attachment_id}/content') +async def download_attachment( + attachment_id: str, + request: Request, + variant: str = Query(default='original', pattern=r'^original$'), +): + user = await current_user(request) + row = await request.app.state.storage.attachment_for_member(attachment_id, user.id) + if row is None: + raise ApiError(404, 'Attachment is not available for this user') + data = await request.app.state.object_store.get(row.object_key) + return Response( + content=data, + media_type=row.content_type, + headers={'Cache-Control': 'private, no-store'}, + ) diff --git a/app/api/routes/auth.py b/app/api/routes/auth.py new file mode 100644 index 00000000..93d9d10a --- /dev/null +++ b/app/api/routes/auth.py @@ -0,0 +1,31 @@ +from fastapi import APIRouter, Request +from fastapi.responses import Response + +from app.api.deps import current_session +from app.api.serialize import user_out +from app.core.config import settings +from app.core.errors import ApiError +from app.core.security import parse_basic, verify_password +from app.repositories.memory import iso_z + +router = APIRouter() +BASIC_CHALLENGE = {'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'} + + +@router.post('/api/v1/auth/sessions', status_code=201) +async def login(request: Request): + creds = parse_basic(request.headers.get('authorization', '')) + storage = request.app.state.storage + row = await storage.user_by_username(creds[0]) if creds else None + if creds is None or row is None or not verify_password(row.password_hash, creds[1]): + raise ApiError(401, 'Invalid credentials', headers=BASIC_CHALLENGE) + session = await storage.create_session(row.id, settings.session_ttl_seconds) + return {'token': session.token, 'token_type': 'Bearer', + 'expires_at': iso_z(session.expires_at), 'user': user_out(row)} + + +@router.delete('/api/v1/auth/sessions/current', status_code=204) +async def logout(request: Request): + session = await current_session(request) + await request.app.state.storage.revoke(session.token) + return Response(status_code=204) \ No newline at end of file diff --git a/app/api/routes/chats.py b/app/api/routes/chats.py new file mode 100644 index 00000000..b2e4e66b --- /dev/null +++ b/app/api/routes/chats.py @@ -0,0 +1,33 @@ +from fastapi import APIRouter, Query, Request + +from app.api.deps import current_user, require_chat_member +from app.api.serialize import chat_out +from app.core.cursors import decode_cursor, paginate +from app.schemas import CreateChat + +router = APIRouter() + + +@router.post('/api/v1/chats', status_code=201) +async def create_chat(body: CreateChat, request: Request): + user = await current_user(request) + member_ids = [str(mid) for mid in body.member_ids] + row = await request.app.state.storage.create_chat(body.title, member_ids, user.id) + return chat_out(row) + + +@router.get('/api/v1/chats') +async def list_chats(request: Request, limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None, min_length=1)): + user = await current_user(request) + resource = 'chats:' + user.id + key = decode_cursor(cursor, resource) if cursor else None + rows = await request.app.state.storage.chats_of(user.id) + page, next_cursor = paginate(rows, key, limit, resource) + return {'items': [chat_out(r) for r in page], 'next_cursor': next_cursor} + + +@router.get('/api/v1/chats/{chat_id}') +async def get_chat(chat_id: str, request: Request): + user = await current_user(request) + return chat_out(await require_chat_member(request, chat_id, user)) \ No newline at end of file diff --git a/app/api/routes/health.py b/app/api/routes/health.py new file mode 100644 index 00000000..9547683c --- /dev/null +++ b/app/api/routes/health.py @@ -0,0 +1,17 @@ +from fastapi import APIRouter, Request + +from app.core.errors import ApiError + +router = APIRouter() + + +@router.get('/health/live') +async def live(): + return {'status': 'ok'} + + +@router.get('/health/ready') +async def ready(request: Request): + if not await request.app.state.storage.ready(): + raise ApiError(503, 'Storage is unavailable') + return {'status': 'ok'} \ No newline at end of file diff --git a/app/api/routes/me.py b/app/api/routes/me.py new file mode 100644 index 00000000..a510d63c --- /dev/null +++ b/app/api/routes/me.py @@ -0,0 +1,11 @@ +from fastapi import APIRouter, Request + +from app.api.deps import current_user +from app.api.serialize import user_out + +router = APIRouter() + + +@router.get('/api/v1/users/me') +async def me(request: Request): + return user_out(await current_user(request)) \ No newline at end of file diff --git a/app/api/routes/messages.py b/app/api/routes/messages.py new file mode 100644 index 00000000..1c291042 --- /dev/null +++ b/app/api/routes/messages.py @@ -0,0 +1,52 @@ +import hashlib +import json + +from fastapi import APIRouter, Header, Query, Request + +from app.api.deps import current_user, require_chat_member +from app.api.serialize import message_out +from app.core.cursors import decode_cursor, encode_cursor +from app.core.errors import ApiError +from app.schemas import CreateMessage + +router = APIRouter() + + +@router.post('/api/v1/chats/{chat_id}/messages', status_code=201) +async def create_message( + body: CreateMessage, + chat_id: str, + request: Request, + idempotency_key: str | None = Header( + default=None, alias='Idempotency-Key', min_length=1, max_length=128, + pattern=r'^[A-Za-z0-9._:-]{1,128}$', + ), +): + if idempotency_key is None: + raise ApiError(400, 'Idempotency-Key is required') + if not body.text.strip() and not body.attachment_ids: + raise ApiError(400, 'Message must contain text or at least one attachment') + user = await current_user(request) + chat = await require_chat_member(request, chat_id, user) + normalized_body = json.dumps( + body.model_dump(mode='json'), ensure_ascii=False, sort_keys=True, separators=(',', ':') + ).encode('utf-8') + body_hash = hashlib.sha256(normalized_body).digest() + row = await request.app.state.storage.add_idempotent_message( + chat.id, user.id, body.text, [str(value) for value in body.attachment_ids], + idempotency_key, body_hash, + ) + return message_out(row) + + +@router.get('/api/v1/chats/{chat_id}/messages') +async def list_messages(request: Request, chat_id: str, + limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None, min_length=1)): + user = await current_user(request) + chat = await require_chat_member(request, chat_id, user) + resource = 'messages:' + chat.id + key = decode_cursor(cursor, resource) if cursor else None + rows, has_more = await request.app.state.storage.messages_page(chat.id, key, limit) + next_cursor = encode_cursor(resource, rows[-1].created_at, rows[-1].id) if has_more else None + return {'items': [message_out(row) for row in rows], 'next_cursor': next_cursor} diff --git a/app/api/routes/users.py b/app/api/routes/users.py new file mode 100644 index 00000000..efd3d1d7 --- /dev/null +++ b/app/api/routes/users.py @@ -0,0 +1,14 @@ +from fastapi import APIRouter, Request + +from app.api.serialize import user_out +from app.core.security import hash_password +from app.schemas import CreateUser + +router = APIRouter() + + +@router.post('/api/v1/users', status_code=201) +async def register(body: CreateUser, request: Request): + row = await request.app.state.storage.create_user( + body.username, body.display_name, hash_password(body.password)) + return user_out(row) \ No newline at end of file diff --git a/app/api/serialize.py b/app/api/serialize.py new file mode 100644 index 00000000..b72027cf --- /dev/null +++ b/app/api/serialize.py @@ -0,0 +1,17 @@ +from app.repositories.memory import UserRow + + +def user_out(row: UserRow) -> dict: + return {'id': row.id, 'username': row.username, + 'display_name': row.display_name, 'created_at': row.created_at} + + +def chat_out(row) -> dict: + return {'id': row.id, 'title': row.title, + 'member_ids': list(row.member_ids), 'created_at': row.created_at} + + +def message_out(row) -> dict: + return {'id': row.id, 'chat_id': row.chat_id, 'sender_id': row.sender_id, + 'text': row.text, 'created_at': row.created_at, + 'attachment_ids': list(row.attachment_ids)} diff --git a/app/core/__init__.py b/app/core/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/core/config.py b/app/core/config.py new file mode 100644 index 00000000..0c99dcd9 --- /dev/null +++ b/app/core/config.py @@ -0,0 +1,55 @@ +from pathlib import Path + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file='.env', extra='ignore') + + app_host: str = '0.0.0.0' + app_port: int = 8080 + session_ttl_seconds: int = 3600 + database_url: str = '' + database_url_file: str | None = None + redis_url: str = 'redis://localhost:6379/0' + redis_url_file: str | None = None + db_pool_min_size: int = 1 + db_pool_max_size: int = 10 + db_pool_acquire_timeout_seconds: float = 3.0 + db_command_timeout_seconds: float = 5.0 + redis_timeout_seconds: float = 2.0 + s3_endpoint_url: str = 'http://localhost:9000' + s3_region: str = 'us-east-1' + s3_bucket: str = 'messenger-attachments' + s3_access_key_id: str = '' + s3_access_key_id_file: str | None = None + s3_secret_access_key: str = '' + s3_secret_access_key_file: str | None = None + s3_connect_timeout_seconds: float = 3.0 + s3_read_timeout_seconds: float = 10.0 + max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760) + max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000) + + def resolved_database_url(self) -> str: + if self.database_url_file: + return Path(self.database_url_file).read_text(encoding='utf-8').strip() + return self.database_url + + def resolved_redis_url(self) -> str: + if self.redis_url_file: + return Path(self.redis_url_file).read_text(encoding='utf-8').strip() + return self.redis_url + + def resolved_s3_access_key_id(self) -> str: + if self.s3_access_key_id_file: + return Path(self.s3_access_key_id_file).read_text(encoding='utf-8').strip() + return self.s3_access_key_id + + def resolved_s3_secret_access_key(self) -> str: + if self.s3_secret_access_key_file: + return Path(self.s3_secret_access_key_file).read_text(encoding='utf-8').strip() + return self.s3_secret_access_key + + +settings = Settings() diff --git a/app/core/cursors.py b/app/core/cursors.py new file mode 100644 index 00000000..c5b1443c --- /dev/null +++ b/app/core/cursors.py @@ -0,0 +1,37 @@ +import base64 +import binascii +import json +import uuid +from datetime import datetime + +from app.core.errors import ApiError +from app.repositories.memory import order_key + + +def encode_cursor(resource: str, created_at: str, row_id: str) -> str: + payload = json.dumps({'r': resource, 't': created_at, 'i': row_id}, separators=(',', ':')) + return base64.urlsafe_b64encode(payload.encode()).decode() + + +def decode_cursor(value: str, resource: str) -> tuple[datetime, int]: + """Курсор от другого ресурса или битый -> 400 invalid_request.""" + try: + payload = json.loads(base64.urlsafe_b64decode(value.encode())) + if payload['r'] != resource: + raise ValueError('cursor belongs to another resource') + moment = datetime.fromisoformat(payload['t'].replace('Z', '+00:00')) + return moment, uuid.UUID(payload['i']).int + except (ValueError, KeyError, TypeError, binascii.Error): + raise ApiError(400, 'Invalid cursor') + + +def paginate(rows: list, cursor_key, limit: int, resource: str): + """Keyset: берём строки строго после (created_at, id) курсора, без offset-скана.""" + if cursor_key is not None: + rows = [r for r in rows if order_key(r.created_dt, r.id) > cursor_key] + page = rows[:limit] + next_cursor = None + if len(rows) > limit: # следующая запись есть на момент чтения + last = page[-1] + next_cursor = encode_cursor(resource, last.created_at, last.id) + return page, next_cursor \ No newline at end of file diff --git a/app/core/errors.py b/app/core/errors.py new file mode 100644 index 00000000..50effe7a --- /dev/null +++ b/app/core/errors.py @@ -0,0 +1,72 @@ +import os +import socket +import uuid + +import asyncpg +from redis.exceptions import RedisError + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.middleware.base import BaseHTTPMiddleware + +CODES = {400: 'invalid_request', 401: 'unauthorized', 404: 'not_found', 409: 'conflict', + 413: 'payload_too_large', 415: 'unsupported_media_type', 429: 'rate_limited', + 503: 'unavailable', 500: 'internal_error'} + + +class ApiError(Exception): + """Бизнес-ошибка: HTTP-статус + код контракта + безопасное сообщение.""" + + def __init__(self, status_code: int, message: str, code: str | None = None, + headers: dict | None = None): + self.status_code = status_code + self.code = code or CODES.get(status_code, 'internal_error') + self.message = message + self.headers = headers or {} + + +class RequestIdMiddleware(BaseHTTPMiddleware): + """Каждый ответ, включая 204 и ошибки, несёт непустой X-Request-Id.""" + + async def dispatch(self, request: Request, call_next): + request.state.request_id = str(uuid.uuid4()) + response = await call_next(request) + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = os.environ.get('INSTANCE_ID') or socket.gethostname() + return response + + +def _body(request: Request, status_code: int, code: str, message: str, + headers: dict | None = None) -> JSONResponse: + rid = getattr(request.state, 'request_id', str(uuid.uuid4())) + return JSONResponse(status_code=status_code, headers=headers, + content={'error': {'code': code, 'message': message, 'request_id': rid}}) + + +def register_error_handlers(app: FastAPI) -> None: + @app.exception_handler(ApiError) + async def on_api_error(request: Request, exc: ApiError): + return _body(request, exc.status_code, exc.code, exc.message, exc.headers) + + @app.exception_handler(RequestValidationError) + async def on_validation(request: Request, exc: RequestValidationError): + return _body(request, 400, 'invalid_request', 'Request validation failed') + + @app.exception_handler(StarletteHTTPException) + async def on_http(request: Request, exc: StarletteHTTPException): + code = CODES.get(exc.status_code, 'invalid_request') + return _body(request, exc.status_code, code, 'Route or method not found') + + async def on_dependency_failure(request: Request, exc: Exception): + return _body(request, 503, 'unavailable', 'A required service is unavailable') + + app.add_exception_handler(asyncpg.PostgresError, on_dependency_failure) + app.add_exception_handler(RedisError, on_dependency_failure) + app.add_exception_handler(OSError, on_dependency_failure) + app.add_exception_handler(TimeoutError, on_dependency_failure) + + @app.exception_handler(Exception) + async def on_unexpected(request: Request, exc: Exception): + return _body(request, 500, 'internal_error', 'Internal error') diff --git a/app/core/security.py b/app/core/security.py new file mode 100644 index 00000000..d526774d --- /dev/null +++ b/app/core/security.py @@ -0,0 +1,39 @@ +import base64 +import binascii +import secrets + +from argon2 import PasswordHasher +from argon2.exceptions import Argon2Error + +_ph = PasswordHasher() # по умолчанию Argon2id + + +def hash_password(password: str) -> str: + return _ph.hash(password) + + +def verify_password(digest: str, password: str) -> bool: + try: + return _ph.verify(digest, password) + except (Argon2Error, ValueError): + return False + + +def new_token() -> str: + return secrets.token_hex(32) # 32 байта = 256 бит энтропии + + +def parse_basic(header: str) -> tuple[str, str] | None: + """Преобразовать 'Basic base64(username:password)' в пару (username, password) или вернуть None.""" + parts = header.split(' ', 1) + if len(parts) != 2 or parts[0].lower() != 'basic': + return None + try: + raw = base64.b64decode(parts[1].strip(), validate=True) + text = raw.decode('utf-8') + except (binascii.Error, ValueError, UnicodeDecodeError): + return None + if ':' not in text: + return None + username, password = text.split(':', 1) # двоеточие в пароле допустимо + return username, password diff --git a/app/db/__init__.py b/app/db/__init__.py new file mode 100644 index 00000000..c0df4427 --- /dev/null +++ b/app/db/__init__.py @@ -0,0 +1 @@ +"""Настройка базы данных и миграции схемы.""" diff --git a/app/db/migrate.py b/app/db/migrate.py new file mode 100644 index 00000000..a026c493 --- /dev/null +++ b/app/db/migrate.py @@ -0,0 +1,56 @@ +from pathlib import Path + +import asyncpg + +MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / 'migrations' + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + """Применить каждую нумерованную SQL-миграцию один раз в транзакции под advisory lock.""" + async with pool.acquire() as connection: + await connection.execute('SELECT pg_advisory_lock($1)', 762019002) + try: + await connection.execute( + 'CREATE TABLE IF NOT EXISTS schema_migrations (' + ' version integer PRIMARY KEY,' + ' applied_at timestamptz NOT NULL DEFAULT now())' + ) + applied = { + row['version'] + for row in await connection.fetch('SELECT version FROM schema_migrations') + } + for path in sorted(MIGRATIONS_DIR.glob('[0-9]*_*.sql')): + version = int(path.name.split('_', 1)[0]) + if version in applied: + continue + async with connection.transaction(): + await connection.execute(path.read_text(encoding='utf-8')) + await connection.execute( + 'INSERT INTO schema_migrations (version) VALUES ($1)', version + ) + finally: + await connection.execute('SELECT pg_advisory_unlock($1)', 762019002) + +async def _main() -> None: + import asyncio + from app.core.config import settings + + dsn = settings.resolved_database_url() + if not dsn: + raise RuntimeError('Set DATABASE_URL or DATABASE_URL_FILE before running migrations') + pool = await asyncpg.create_pool( + dsn, + min_size=settings.db_pool_min_size, + max_size=settings.db_pool_max_size, + timeout=settings.db_pool_acquire_timeout_seconds, + command_timeout=settings.db_command_timeout_seconds, + ) + try: + await apply_migrations(pool) + finally: + await pool.close() + + +if __name__ == '__main__': + import asyncio + asyncio.run(_main()) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..3b54dd50 --- /dev/null +++ b/app/main.py @@ -0,0 +1,57 @@ +from contextlib import asynccontextmanager + +import asyncpg +from fastapi import FastAPI +from redis.asyncio import Redis + +from app.api.routes import attachments, auth, health, me, users, chats, messages +from app.core.config import settings +from app.core.errors import RequestIdMiddleware, register_error_handlers +from app.repositories.object_store import S3ObjectStore +from app.repositories.postgres import PgStorage +from app.repositories.redis_sessions import RedisSessionStore + + +def create_app() -> FastAPI: + @asynccontextmanager + async def lifespan(app: FastAPI): + dsn = settings.resolved_database_url() + if not dsn: + raise RuntimeError('Set DATABASE_URL or DATABASE_URL_FILE before starting the API') + pool = await asyncpg.create_pool( + dsn, + min_size=settings.db_pool_min_size, + max_size=settings.db_pool_max_size, + timeout=settings.db_pool_acquire_timeout_seconds, + command_timeout=settings.db_command_timeout_seconds, + ) + redis_client = Redis.from_url( + settings.resolved_redis_url(), + decode_responses=True, + socket_connect_timeout=settings.redis_timeout_seconds, + socket_timeout=settings.redis_timeout_seconds, + ) + object_store = S3ObjectStore() + try: + await object_store.ensure_bucket() + app.state.object_store = object_store + app.state.storage = PgStorage(pool, RedisSessionStore(redis_client), object_store) + yield + finally: + await redis_client.aclose() + await pool.close() + + app = FastAPI(title='MTUSI Messenger lab4', docs_url=None, redoc_url=None, lifespan=lifespan) + app.add_middleware(RequestIdMiddleware) + register_error_handlers(app) + app.include_router(health.router) + app.include_router(users.router) + app.include_router(me.router) + app.include_router(auth.router) + app.include_router(chats.router) + app.include_router(messages.router) + app.include_router(attachments.router) + return app + + +app = create_app() diff --git a/app/repositories/__init__.py b/app/repositories/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/repositories/memory.py b/app/repositories/memory.py new file mode 100644 index 00000000..5e52148a --- /dev/null +++ b/app/repositories/memory.py @@ -0,0 +1,148 @@ +import threading +import uuid +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone + +from app.core.errors import ApiError + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def iso_z(moment: datetime) -> str: + """RFC 3339 UTC с Z и микросекундами: порядок (created_at, id) не теряет точность.""" + return moment.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + +def order_key(created_dt: datetime, row_id: str): + """Контракт: порядок по (created_at, id), id сравнивается как значение UUID.""" + return (created_dt, uuid.UUID(row_id).int) + + +@dataclass +class UserRow: + id: str + username: str + display_name: str + password_hash: str + created_at: str + + +@dataclass +class ChatRow: + id: str + title: str + member_ids: list + created_dt: datetime + created_at: str + + +@dataclass +class MessageRow: + id: str + chat_id: str + sender_id: str + text: str + created_dt: datetime + created_at: str + attachment_ids: list[str] = field(default_factory=list) + + +@dataclass +class AttachmentRow: + id: str + chat_id: str + owner_id: str + object_key: str + filename: str + content_type: str + size_bytes: int + sha256: str + status: str + created_at: str + updated_at: str + + +@dataclass +class SessionRow: + token: str + user_id: str + expires_at: datetime + + +class MemoryStorage: + """Хранилище в памяти из скелета lab1. В lab2 его заменяет PgStorage.""" + + def __init__(self): + self._lock = threading.RLock() + self._users: dict[str, UserRow] = {} + self._by_username: dict[str, str] = {} + self._chats: dict[str, ChatRow] = {} + self._messages: dict[str, list[MessageRow]] = {} + self._sessions: dict[str, SessionRow] = {} + + def ready(self) -> bool: + return True + + def create_user(self, username: str, display_name: str, password_hash: str) -> UserRow: + with self._lock: + if username in self._by_username: + raise ApiError(409, 'Username is already taken') + row = UserRow(str(uuid.uuid4()), username, display_name, password_hash, + iso_z(now_utc())) + self._users[row.id] = row + self._by_username[username] = row.id + return row + + def user_by_username(self, username: str) -> UserRow | None: + with self._lock: + uid = self._by_username.get(username) + return self._users.get(uid) if uid else None + + def user(self, user_id: str) -> UserRow | None: + with self._lock: + return self._users.get(user_id) + + def create_session(self, user_id: str, ttl_seconds: int) -> SessionRow: + from app.core.security import new_token + with self._lock: + row = SessionRow(new_token(), user_id, now_utc() + timedelta(seconds=ttl_seconds)) + self._sessions[row.token] = row + return row + + def session(self, token: str) -> SessionRow | None: + with self._lock: + row = self._sessions.get(token) + if row is None: + return None + if now_utc() >= row.expires_at: + del self._sessions[token] + return None + return row + + def revoke(self, token: str) -> bool: + with self._lock: + return self._sessions.pop(token, None) is not None + + def create_chat(self, title: str, member_ids: list, caller_id: str) -> ChatRow: + with self._lock: + if caller_id not in member_ids: + raise ApiError(400, 'Creator must be included in member_ids') + for mid in member_ids: + if mid not in self._users: + raise ApiError(400, 'Unknown member id') + moment = now_utc() + row = ChatRow(str(uuid.uuid4()), title, list(member_ids), moment, iso_z(moment)) + self._chats[row.id] = row + self._messages[row.id] = [] + return row + + def chat(self, chat_id: str) -> ChatRow | None: + with self._lock: + return self._chats.get(chat_id) + + def chats_of(self, user_id: str) -> list[ChatRow]: + with self._lock: + rows = [c for c in self._chats.values() if user_id in c.member_ids] + return sorted(rows, key=lambda c: order_key(c.created_dt, c.id)) diff --git a/app/repositories/object_store.py b/app/repositories/object_store.py new file mode 100644 index 00000000..2d657be2 --- /dev/null +++ b/app/repositories/object_store.py @@ -0,0 +1,109 @@ +import asyncio +import time + +import boto3 +from botocore.config import Config +from botocore.exceptions import BotoCoreError, ClientError + +from app.core.config import settings +from app.core.errors import ApiError + + +class S3ObjectStore: + """Хранит байты в приватном S3 bucket, не в процессе API.""" + + def __init__(self): + self.bucket = settings.s3_bucket + self.client = boto3.client( + "s3", + endpoint_url=settings.s3_endpoint_url, + region_name=settings.s3_region, + aws_access_key_id=settings.resolved_s3_access_key_id(), + aws_secret_access_key=settings.resolved_s3_secret_access_key(), + config=Config( + connect_timeout=settings.s3_connect_timeout_seconds, + read_timeout=settings.s3_read_timeout_seconds, + max_pool_connections=16, + retries={"max_attempts": 2, "mode": "standard"}, + s3={"addressing_style": "path"}, + ), + ) + + @staticmethod + def _unavailable(): + return ApiError(503, "Object storage is unavailable") + + def _ensure_bucket(self): + for attempt in range(30): + try: + self.client.head_bucket(Bucket=self.bucket) + return + except ClientError as exc: + code = str(exc.response.get("Error", {}).get("Code", "")) + if code not in {"404", "NoSuchBucket", "NotFound"}: + if code in {"403", "AccessDenied"} or attempt == 29: + raise self._unavailable() from None + else: + try: + self.client.create_bucket(Bucket=self.bucket) + except ClientError as create_exc: + create_code = str(create_exc.response.get("Error", {}).get("Code", "")) + if create_code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}: + if attempt == 29: + raise self._unavailable() from None + else: + return + except (BotoCoreError, OSError, TimeoutError): + if attempt == 29: + raise self._unavailable() from None + time.sleep(0.5) + raise self._unavailable() + + async def ensure_bucket(self): + await asyncio.to_thread(self._ensure_bucket) + + def _head_bucket(self): + self.client.head_bucket(Bucket=self.bucket) + + async def ready(self) -> bool: + try: + await asyncio.to_thread(self._head_bucket) + return True + except (ClientError, BotoCoreError, OSError, TimeoutError): + return False + + def _put(self, key: str, data: bytes, content_type: str): + self.client.put_object(Bucket=self.bucket, Key=key, Body=data, ContentType=content_type) + + async def put(self, key: str, data: bytes, content_type: str): + try: + await asyncio.to_thread(self._put, key, data, content_type) + except (ClientError, BotoCoreError, OSError, TimeoutError): + raise self._unavailable() from None + + def _get(self, key: str) -> bytes: + result = self.client.get_object(Bucket=self.bucket, Key=key) + try: + return result["Body"].read(settings.max_upload_bytes + 1) + finally: + result["Body"].close() + + async def get(self, key: str) -> bytes: + try: + data = await asyncio.to_thread(self._get, key) + if len(data) > settings.max_upload_bytes: + raise self._unavailable() + return data + except ApiError: + raise + except (ClientError, BotoCoreError, OSError, TimeoutError): + raise self._unavailable() from None + + def _delete(self, key: str): + self.client.delete_object(Bucket=self.bucket, Key=key) + + async def delete(self, key: str): + try: + await asyncio.to_thread(self._delete, key) + except (ClientError, BotoCoreError, OSError, TimeoutError): + raise self._unavailable() from None diff --git a/app/repositories/postgres.py b/app/repositories/postgres.py new file mode 100644 index 00000000..03825f0c --- /dev/null +++ b/app/repositories/postgres.py @@ -0,0 +1,278 @@ +import uuid + +import asyncpg + +from app.core.errors import ApiError +from app.repositories.memory import ( + AttachmentRow, ChatRow, MessageRow, UserRow, iso_z, +) + + +def _user(row: asyncpg.Record | None) -> UserRow | None: + if row is None: + return None + return UserRow( + id=str(row['id']), username=row['username'], display_name=row['display_name'], + password_hash=row['password_hash'], created_at=iso_z(row['created_at']), + ) + + +def _chat(row: asyncpg.Record) -> ChatRow: + moment = row['created_at'] + return ChatRow( + id=str(row['id']), title=row['title'], member_ids=[str(x) for x in row['member_ids']], + created_dt=moment, created_at=iso_z(moment), + ) + + +def _message(row: asyncpg.Record) -> MessageRow: + moment = row['created_at'] + attachment_ids = row['attachment_ids'] if 'attachment_ids' in row else [] + return MessageRow( + id=str(row['id']), chat_id=str(row['chat_id']), sender_id=str(row['sender_id']), + text=row['text'], created_dt=moment, created_at=iso_z(moment), + attachment_ids=[str(value) for value in (attachment_ids or [])], + ) + + +def _attachment(row: asyncpg.Record) -> AttachmentRow: + return AttachmentRow( + id=str(row['id']), chat_id=str(row['chat_id']), owner_id=str(row['owner_id']), + object_key=row['object_key'], filename=row['filename'], + content_type=row['content_type'], size_bytes=row['size_bytes'], + sha256=row['sha256'], status=row['status'], + created_at=iso_z(row['created_at']), updated_at=iso_z(row['updated_at']), + ) + + +_MESSAGE_SELECT = ( + 'SELECT m.*, COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) ' + "FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::uuid[]) AS attachment_ids " + 'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id ' +) + + +class PgStorage: + """PostgreSQL — источник истины для метаданных; S3 хранит только байты.""" + + def __init__(self, pool: asyncpg.Pool, session_store, object_store): + self.pool = pool + self.session_store = session_store + self.object_store = object_store + + async def ready(self) -> bool: + try: + async with self.pool.acquire() as connection: + database_ready = await connection.fetchval('SELECT 1') == 1 + return ( + database_ready + and await self.session_store.ready() + and await self.object_store.ready() + ) + except (asyncpg.PostgresError, OSError, TimeoutError): + return False + + async def create_user(self, username: str, display_name: str, password_hash: str) -> UserRow: + try: + async with self.pool.acquire() as connection: + row = await connection.fetchrow( + 'INSERT INTO users (id, username, display_name, password_hash) ' + 'VALUES ($1, $2, $3, $4) RETURNING *', + uuid.uuid4(), username, display_name, password_hash, + ) + except asyncpg.UniqueViolationError as exc: + if exc.constraint_name == 'users_username_key': + raise ApiError(409, 'Username is already taken') from None + raise + return _user(row) + + async def user_by_username(self, username: str) -> UserRow | None: + async with self.pool.acquire() as connection: + return _user(await connection.fetchrow( + 'SELECT * FROM users WHERE username = $1', username + )) + + async def user(self, user_id: str) -> UserRow | None: + async with self.pool.acquire() as connection: + return _user(await connection.fetchrow( + 'SELECT * FROM users WHERE id = $1', uuid.UUID(user_id) + )) + + async def create_session(self, user_id: str, ttl_seconds: int): + return await self.session_store.create_session(user_id, ttl_seconds) + + async def session(self, token: str): + return await self.session_store.session(token) + + async def revoke(self, token: str) -> bool: + return await self.session_store.revoke(token) + + async def create_chat(self, title: str, member_ids: list[str], caller_id: str) -> ChatRow: + if caller_id not in member_ids: + raise ApiError(400, 'Creator must be included in member_ids') + chat_id = uuid.uuid4() + try: + async with self.pool.acquire() as connection: + async with connection.transaction(): + created_at = await connection.fetchval( + 'INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at', + chat_id, title, + ) + await connection.executemany( + 'INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)', + [(chat_id, uuid.UUID(member_id)) for member_id in member_ids], + ) + except asyncpg.ForeignKeyViolationError: + raise ApiError(400, 'Unknown member id') from None + return ChatRow( + id=str(chat_id), title=title, member_ids=list(member_ids), + created_dt=created_at, created_at=iso_z(created_at), + ) + + async def chat(self, chat_id: str) -> ChatRow | None: + async with self.pool.acquire() as connection: + row = await connection.fetchrow( + 'SELECT c.id, c.title, c.created_at, ' + 'array_agg(cm.user_id ORDER BY cm.user_id) AS member_ids ' + 'FROM chats c JOIN chat_members cm ON cm.chat_id = c.id ' + 'WHERE c.id = $1 GROUP BY c.id', uuid.UUID(chat_id), + ) + return _chat(row) if row else None + + async def chats_of(self, user_id: str) -> list[ChatRow]: + async with self.pool.acquire() as connection: + rows = await connection.fetch( + 'SELECT c.id, c.title, c.created_at, ' + 'array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids ' + 'FROM chats c ' + 'JOIN chat_members mine ON mine.chat_id = c.id ' + 'JOIN chat_members cm_all ON cm_all.chat_id = c.id ' + 'WHERE mine.user_id = $1 ' + 'GROUP BY c.id ORDER BY c.created_at, c.id', uuid.UUID(user_id), + ) + return [_chat(row) for row in rows] + + async def create_attachment( + self, attachment_id: str, chat_id: str, owner_id: str, object_key: str, + filename: str, content_type: str, size_bytes: int, sha256: str, + ) -> AttachmentRow: + async with self.pool.acquire() as connection: + row = await connection.fetchrow( + 'INSERT INTO attachments ' + '(id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256) ' + 'VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *', + uuid.UUID(attachment_id), uuid.UUID(chat_id), uuid.UUID(owner_id), + object_key, filename, content_type, size_bytes, sha256, + ) + return _attachment(row) + + async def attachment_for_member(self, attachment_id: str, user_id: str) -> AttachmentRow | None: + try: + attachment_uuid = uuid.UUID(attachment_id) + except ValueError: + raise ApiError(400, 'attachment_id must be a UUID') from None + async with self.pool.acquire() as connection: + row = await connection.fetchrow( + 'SELECT a.* FROM attachments a ' + 'JOIN chat_members cm ON cm.chat_id = a.chat_id ' + 'WHERE a.id = $1 AND cm.user_id = $2', + attachment_uuid, uuid.UUID(user_id), + ) + return _attachment(row) if row else None + + async def add_idempotent_message( + self, chat_id: str, sender_id: str, text: str, attachment_ids: list[str], + idempotency_key: str, request_hash: bytes, + ) -> MessageRow: + """Создать сообщение, его вложения и ключ повтора одной транзакцией.""" + chat_uuid = uuid.UUID(chat_id) + sender_uuid = uuid.UUID(sender_id) + attachment_uuids = [uuid.UUID(value) for value in attachment_ids] + lock_scope = f'{sender_uuid}:POST:/api/v1/chats/{chat_uuid}/messages:{idempotency_key}' + async with self.pool.acquire() as connection: + async with connection.transaction(): + await connection.fetchval( + 'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', lock_scope, + ) + existing = await connection.fetchrow( + 'SELECT k.request_hash, m.id FROM message_idempotency k ' + 'JOIN messages m ON m.id = k.message_id ' + 'WHERE k.user_id = $1 AND k.chat_id = $2 AND k.idempotency_key = $3 ' + "AND k.created_at > now() - interval '24 hours'", + sender_uuid, chat_uuid, idempotency_key, + ) + if existing is not None: + if bytes(existing['request_hash']) != request_hash: + raise ApiError(409, 'Idempotency-Key was already used with another request') + row = await connection.fetchrow( + _MESSAGE_SELECT + 'WHERE m.id = $1 GROUP BY m.id', existing['id'], + ) + return _message(row) + + if attachment_uuids: + available = await connection.fetch( + "SELECT id FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[]) " + "AND status = 'ready' FOR SHARE", + chat_uuid, attachment_uuids, + ) + if len(available) != len(attachment_uuids): + raise ApiError(404, 'Attachment is not available in this chat') + + await connection.execute( + 'DELETE FROM message_idempotency ' + 'WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3', + sender_uuid, chat_uuid, idempotency_key, + ) + await connection.execute( + "DELETE FROM message_idempotency WHERE ctid IN (" + 'SELECT ctid FROM message_idempotency ' + "WHERE created_at <= now() - interval '24 hours' " + 'ORDER BY created_at LIMIT 100)' + ) + message_id = uuid.uuid4() + await connection.execute( + 'INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4)', + message_id, chat_uuid, sender_uuid, text, + ) + if attachment_uuids: + await connection.executemany( + 'INSERT INTO message_attachments ' + '(message_id, attachment_id, chat_id, position) VALUES ($1, $2, $3, $4)', + [(message_id, attachment_id, chat_uuid, index) + for index, attachment_id in enumerate(attachment_uuids)], + ) + row = await connection.fetchrow( + _MESSAGE_SELECT + 'WHERE m.id = $1 GROUP BY m.id', message_id, + ) + await connection.execute( + 'INSERT INTO message_idempotency ' + '(user_id, chat_id, idempotency_key, request_hash, message_id) ' + 'VALUES ($1, $2, $3, $4, $5)', + sender_uuid, chat_uuid, idempotency_key, request_hash, message_id, + ) + return _message(row) + + async def messages_page(self, chat_id: str, cursor_key, limit: int): + """Прочитать ограниченную keyset-страницу вместе с ID прикреплённых файлов.""" + chat_uuid = uuid.UUID(chat_id) + base = ( + 'SELECT m.*, COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) ' + "FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::uuid[]) AS attachment_ids " + 'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id ' + 'WHERE m.chat_id = $1 ' + ) + async with self.pool.acquire() as connection: + if cursor_key is None: + rows = await connection.fetch( + base + 'GROUP BY m.id ORDER BY m.created_at, m.id LIMIT $2', + chat_uuid, limit + 1, + ) + else: + created_at, id_value = cursor_key + rows = await connection.fetch( + base + 'AND (m.created_at, m.id) > ($2, $3) ' + 'GROUP BY m.id ORDER BY m.created_at, m.id LIMIT $4', + chat_uuid, created_at, uuid.UUID(int=id_value), limit + 1, + ) + has_more = len(rows) > limit + return [_message(row) for row in rows[:limit]], has_more diff --git a/app/repositories/redis_sessions.py b/app/repositories/redis_sessions.py new file mode 100644 index 00000000..cfd18de3 --- /dev/null +++ b/app/repositories/redis_sessions.py @@ -0,0 +1,66 @@ +import hashlib +import time +from datetime import datetime, timedelta, timezone + +from redis.asyncio import Redis +from redis.exceptions import RedisError + +from app.core.errors import ApiError +from app.core.security import new_token +from app.repositories.memory import SessionRow, now_utc + + +class RedisSessionStore: + """Redis — источник истины для сессий; PXAT сохраняет исходный срок истечения после перезапуска.""" + + def __init__(self, client: Redis): + self.client = client + + @staticmethod + def _key(token: str) -> str: + digest = hashlib.sha256(token.encode('utf-8')).hexdigest() + return 'session:' + digest + + async def ready(self) -> bool: + try: + return bool(await self.client.ping()) + except (RedisError, OSError, TimeoutError): + return False + + async def create_session(self, user_id: str, ttl_seconds: int) -> SessionRow: + token = new_token() + current = now_utc() + expires_at = current.replace(microsecond=(current.microsecond // 1000) * 1000) + timedelta(seconds=ttl_seconds) + expires_ms = int(expires_at.timestamp() * 1000) + try: + await self.client.set( + self._key(token), f'{user_id}|{expires_ms}', pxat=expires_ms + ) + except RedisError: + raise ApiError(503, 'Session storage is unavailable') from None + return SessionRow(token=token, user_id=user_id, expires_at=expires_at) + + async def session(self, token: str) -> SessionRow | None: + try: + value = await self.client.get(self._key(token)) + except RedisError: + raise ApiError(503, 'Session storage is unavailable') from None + if value is None: + return None + try: + user_id, expires_ms_text = value.split('|', 1) + expires_ms = int(expires_ms_text) + if expires_ms <= int(time.time() * 1000): + await self.client.delete(self._key(token)) + return None + expires_at = datetime.fromtimestamp(expires_ms / 1000, timezone.utc) + return SessionRow(token=token, user_id=user_id, expires_at=expires_at) + except (ValueError, OverflowError): + await self.client.delete(self._key(token)) + return None + + async def revoke(self, token: str) -> bool: + try: + return bool(await self.client.delete(self._key(token))) + except RedisError: + raise ApiError(503, 'Session storage is unavailable') from None diff --git a/app/schemas.py b/app/schemas.py new file mode 100644 index 00000000..03976c78 --- /dev/null +++ b/app/schemas.py @@ -0,0 +1,85 @@ +import uuid as _uuid +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + +STRICT = ConfigDict(extra='forbid') + + +class CreateUser(BaseModel): + model_config = STRICT + username: str = Field(pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator('display_name', 'password') + @classmethod + def _not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError('string must not consist of whitespace only') + return value + + +class User(BaseModel): + model_config = STRICT + id: _uuid.UUID + username: str + display_name: str + created_at: str + + +class Session(BaseModel): + model_config = STRICT + token: str + token_type: Literal['Bearer'] + expires_at: str + user: User + + +class CreateChat(BaseModel): + model_config = STRICT + title: str = Field(min_length=1, max_length=100) + member_ids: list[_uuid.UUID] = Field(min_length=1, max_length=100) + + @field_validator('title') + @classmethod + def _title_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError('string must not consist of whitespace only') + return value + + @field_validator('member_ids') + @classmethod + def _unique_members(cls, value): + if len(set(value)) != len(value): + raise ValueError('duplicate member ids') + return value + + +class CreateMessage(BaseModel): + model_config = STRICT + text: str = Field(min_length=0, max_length=4000) + attachment_ids: list[_uuid.UUID] = Field(default_factory=list, max_length=10) + + @field_validator('attachment_ids') + @classmethod + def _unique_attachments(cls, value): + if len(set(value)) != len(value): + raise ValueError('duplicate attachment ids') + return value + + +class Attachment(BaseModel): + model_config = STRICT + id: _uuid.UUID + chat_id: _uuid.UUID + owner_id: _uuid.UUID + filename: str = Field(min_length=1, max_length=255) + content_type: Literal['image/png', 'image/jpeg'] + size_bytes: int = Field(ge=1, le=10_485_760) + sha256: str = Field(pattern=r'^[a-f0-9]{64}$') + status: Literal['ready'] + variants: list = Field(default_factory=list) + error_code: str | None = None + created_at: str + updated_at: str diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..c7832421 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,130 @@ +name: mtusi-lab4 + +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER:?Set POSTGRES_USER in .env} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + POSTGRES_DB: ${POSTGRES_DB:?Set POSTGRES_DB in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 3s + timeout: 3s + retries: 30 + start_period: 5s + restart: unless-stopped + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", ""] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 3s + retries: 30 + start_period: 5s + restart: unless-stopped + + s3: + image: ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d + command: ["/data"] + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: 0.0.0.0:9000 + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env} + RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env} + RUSTFS_OBS_LOGGER_LEVEL: error + volumes: + - s3_data:/data + restart: unless-stopped + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "app.db.migrate"] + environment: + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} + DB_POOL_MIN_SIZE: 1 + DB_POOL_MAX_SIZE: 2 + DB_POOL_ACQUIRE_TIMEOUT_SECONDS: 3 + DB_COMMAND_TIMEOUT_SECONDS: 10 + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB} + DB_POOL_MIN_SIZE: ${DB_POOL_MIN_SIZE:-1} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} + DB_POOL_ACQUIRE_TIMEOUT_SECONDS: ${DB_POOL_ACQUIRE_TIMEOUT_SECONDS:-3} + DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-5} + REDIS_URL: redis://redis:6379/0 + REDIS_TIMEOUT_SECONDS: ${REDIS_TIMEOUT_SECONDS:-2} + S3_ENDPOINT_URL: http://s3:9000 + S3_REGION: ${S3_REGION:-us-east-1} + S3_BUCKET: ${S3_BUCKET:-messenger-attachments} + S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID} + S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY} + S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-3} + S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10} + MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760} + MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000} + depends_on: + migrate: + condition: service_completed_successfully + postgres: + condition: service_healthy + redis: + condition: service_healthy + s3: + condition: service_started + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 3s + timeout: 3s + retries: 20 + start_period: 10s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "1.0" + memory: 512M + stop_grace_period: 20s + restart: unless-stopped + + lb: + image: nginx:1.27.4-alpine + ports: + - "127.0.0.1:${LB_PORT:-8080}:8080" + volumes: + - ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"] + interval: 3s + timeout: 3s + retries: 10 + restart: unless-stopped + +volumes: + postgres_data: + redis_data: + s3_data: diff --git a/evidence/compose-config.txt b/evidence/compose-config.txt new file mode 100644 index 00000000..9fe6cb69 --- /dev/null +++ b/evidence/compose-config.txt @@ -0,0 +1 @@ +PASS: docker compose config --quiet diff --git a/evidence/compose-final.txt b/evidence/compose-final.txt new file mode 100644 index 00000000..730bcc63 --- /dev/null +++ b/evidence/compose-final.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 4 minutes ago Up 3 minutes (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 4 minutes ago Up 4 minutes (healthy) 8080/tcp +mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 4 minutes ago Up 4 minutes (healthy) 5432/tcp +mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 4 minutes ago Up 4 minutes (healthy) 6379/tcp +mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 4 minutes ago Up 4 minutes 9000-9001/tcp diff --git a/evidence/compose-yaml.txt b/evidence/compose-yaml.txt new file mode 100644 index 00000000..47700d92 --- /dev/null +++ b/evidence/compose-yaml.txt @@ -0,0 +1 @@ +PASS: compose.yaml parses diff --git a/evidence/diff-check.txt b/evidence/diff-check.txt new file mode 100644 index 00000000..77494d37 --- /dev/null +++ b/evidence/diff-check.txt @@ -0,0 +1 @@ +PASS: final git diff --check diff --git a/evidence/failover-before.txt b/evidence/failover-before.txt new file mode 100644 index 00000000..877cfc83 --- /dev/null +++ b/evidence/failover-before.txt @@ -0,0 +1,8 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 26 seconds ago Up 19 seconds (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 26 seconds ago Up 19 seconds (healthy) 8080/tcp +mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 26 seconds ago Up 13 seconds (healthy) 127.0.0.1:8080->8080/tcp +mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 26 seconds ago Exited (0) 19 seconds ago +mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 26 seconds ago Up 25 seconds (healthy) 5432/tcp +mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 26 seconds ago Up 25 seconds (healthy) 6379/tcp +mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 26 seconds ago Up 25 seconds 9000-9001/tcp diff --git a/evidence/failover-restore.txt b/evidence/failover-restore.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/evidence/failover-restore.txt @@ -0,0 +1 @@ + diff --git a/evidence/failover-restored.txt b/evidence/failover-restored.txt new file mode 100644 index 00000000..3aca3bb1 --- /dev/null +++ b/evidence/failover-restored.txt @@ -0,0 +1,8 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 49 seconds ago Up 42 seconds (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 49 seconds ago Up 5 seconds (healthy) 8080/tcp +mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 49 seconds ago Up 36 seconds (healthy) 127.0.0.1:8080->8080/tcp +mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 49 seconds ago Exited (0) 5 seconds ago +mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 49 seconds ago Up 48 seconds (healthy) 5432/tcp +mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 49 seconds ago Up 48 seconds (healthy) 6379/tcp +mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 49 seconds ago Up 48 seconds 9000-9001/tcp diff --git a/evidence/failover-stopped.txt b/evidence/failover-stopped.txt new file mode 100644 index 00000000..510803d8 --- /dev/null +++ b/evidence/failover-stopped.txt @@ -0,0 +1,8 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 42 seconds ago Up 35 seconds (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 42 seconds ago Exited (0) 14 seconds ago +mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 42 seconds ago Up 29 seconds (healthy) 127.0.0.1:8080->8080/tcp +mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 42 seconds ago Exited (0) 35 seconds ago +mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 42 seconds ago Up 41 seconds (healthy) 5432/tcp +mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 42 seconds ago Up 41 seconds (healthy) 6379/tcp +mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 42 seconds ago Up 41 seconds 9000-9001/tcp diff --git a/evidence/hooks-syntax.txt b/evidence/hooks-syntax.txt new file mode 100644 index 00000000..17c508d9 --- /dev/null +++ b/evidence/hooks-syntax.txt @@ -0,0 +1 @@ +PASS: restart/failover hooks syntax diff --git a/evidence/make-check.txt b/evidence/make-check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/make-check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/make-test.txt b/evidence/make-test.txt new file mode 100644 index 00000000..b10270e9 --- /dev/null +++ b/evidence/make-test.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab4 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/make-validate-first-attempt.txt b/evidence/make-validate-first-attempt.txt new file mode 100644 index 00000000..8c0007c5 --- /dev/null +++ b/evidence/make-validate-first-attempt.txt @@ -0,0 +1,6 @@ +python3 scripts/validate_openapi.py +Traceback (most recent call last): + File "/home/lenovo/lab4/mtusi2026lab4-fork/scripts/validate_openapi.py", line 5, in + from openapi_spec_validator import validate +ModuleNotFoundError: No module named 'openapi_spec_validator' +make: *** [Makefile:16: validate] Error 1 diff --git a/evidence/make-validate.txt b/evidence/make-validate.txt new file mode 100644 index 00000000..37e81884 --- /dev/null +++ b/evidence/make-validate.txt @@ -0,0 +1,2 @@ +.venv/bin/python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/migrations-repeat.txt b/evidence/migrations-repeat.txt new file mode 100644 index 00000000..cff291ba --- /dev/null +++ b/evidence/migrations-repeat.txt @@ -0,0 +1,15 @@ + Container mtusi-lab4-postgres-1 Running + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-migrate-run-3a566e7f2cbd Creating + Container mtusi-lab4-migrate-run-3a566e7f2cbd Created + Container mtusi-lab4-postgres-1 Running + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-migrate-run-5e9470435d61 Creating + Container mtusi-lab4-migrate-run-5e9470435d61 Created +Migration versions: +1 +2 +3 +4 diff --git a/evidence/persistence-after.txt b/evidence/persistence-after.txt new file mode 100644 index 00000000..7124204d --- /dev/null +++ b/evidence/persistence-after.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 12 seconds ago Up 6 seconds (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 12 seconds ago Up 5 seconds (healthy) 8080/tcp +mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 12 seconds ago Up Less than a second (health: starting) 127.0.0.1:8080->8080/tcp +mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 12 seconds ago Up 11 seconds (healthy) 5432/tcp +mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 12 seconds ago Up 11 seconds (healthy) 6379/tcp +mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 12 seconds ago Up 12 seconds 9000-9001/tcp diff --git a/evidence/persistence-before.txt b/evidence/persistence-before.txt new file mode 100644 index 00000000..dfcc403d --- /dev/null +++ b/evidence/persistence-before.txt @@ -0,0 +1,7 @@ +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp +mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 2 minutes ago Up About a minute (healthy) 8080/tcp +mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 2 minutes ago Up 2 minutes (healthy) 5432/tcp +mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 2 minutes ago Up 2 minutes (healthy) 6379/tcp +mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 2 minutes ago Up 2 minutes 9000-9001/tcp diff --git a/evidence/python-compileall.txt b/evidence/python-compileall.txt new file mode 100644 index 00000000..9d0a02bf --- /dev/null +++ b/evidence/python-compileall.txt @@ -0,0 +1 @@ +PASS: compileall app student-tests scripts diff --git a/evidence/restore-api.txt b/evidence/restore-api.txt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/evidence/restore-api.txt @@ -0,0 +1 @@ + diff --git a/evidence/restore-final.txt b/evidence/restore-final.txt new file mode 100644 index 00000000..003d8283 --- /dev/null +++ b/evidence/restore-final.txt @@ -0,0 +1,21 @@ + Container mtusi-lab4-postgres-1 Running + Container mtusi-lab4-redis-1 Running + Container mtusi-lab4-s3-1 Running + Container mtusi-lab4-api-2 Running + Container mtusi-lab4-lb-1 Running + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-migrate-1 Starting + Container mtusi-lab4-migrate-1 Started + Container mtusi-lab4-migrate-1 Waiting + Container mtusi-lab4-postgres-1 Waiting + Container mtusi-lab4-redis-1 Waiting + Container mtusi-lab4-postgres-1 Healthy + Container mtusi-lab4-redis-1 Healthy + Container mtusi-lab4-migrate-1 Exited + Container mtusi-lab4-api-1 Starting + Container mtusi-lab4-api-1 Started + Container mtusi-lab4-api-1 Waiting + Container mtusi-lab4-api-2 Waiting + Container mtusi-lab4-api-1 Healthy + Container mtusi-lab4-api-2 Healthy diff --git a/evidence/static-checks.txt b/evidence/static-checks.txt new file mode 100644 index 00000000..33149b4f --- /dev/null +++ b/evidence/static-checks.txt @@ -0,0 +1,7 @@ +Static checks (2026-11-02; base commit recorded in verification-metadata.txt) +make-check: PASS +make-validate: PASS +python-compileall: PASS +hooks-syntax: PASS +compose-yaml: PASS +diff-check: PASS diff --git a/evidence/test-attachment-failover.txt b/evidence/test-attachment-failover.txt new file mode 100644 index 00000000..c7cf3dc9 --- /dev/null +++ b/evidence/test-attachment-failover.txt @@ -0,0 +1,2 @@ +python3 student-tests/attachment_failover.py --base-url "http://localhost:8080" +PASS: upload instance stopped; metadata and original bytes read via peer diff --git a/evidence/test-failover.txt b/evidence/test-failover.txt new file mode 100644 index 00000000..2bba4b1e --- /dev/null +++ b/evidence/test-failover.txt @@ -0,0 +1,4 @@ +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 0, "first_recovery_seconds": 0.556} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/test-idempotency.txt b/evidence/test-idempotency.txt new file mode 100644 index 00000000..3e3aae30 --- /dev/null +++ b/evidence/test-idempotency.txt @@ -0,0 +1,2 @@ +python3 student-tests/idempotency.py --base-url "http://localhost:8080" +PASS: 16 concurrent requests across 2 API replicas returned one Message; changed body returned 409. diff --git a/evidence/test-persistence.txt b/evidence/test-persistence.txt new file mode 100644 index 00000000..933c59c7 --- /dev/null +++ b/evidence/test-persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/test-private-bucket.txt b/evidence/test-private-bucket.txt new file mode 100644 index 00000000..808e9a90 --- /dev/null +++ b/evidence/test-private-bucket.txt @@ -0,0 +1,3 @@ +python3 student-tests/private_bucket.py +HTTP 403 +PASS: private bucket rejects unauthenticated listing diff --git a/evidence/test-s3-failure-analysis.txt b/evidence/test-s3-failure-analysis.txt new file mode 100644 index 00000000..f63a99a9 --- /dev/null +++ b/evidence/test-s3-failure-analysis.txt @@ -0,0 +1,7 @@ +# Анализ первого запуска S3 outage-теста + +Дата: 2026-11-02. + +Первый запуск остановил S3 и успел создать и прочитать синтетическое вложение. Проверочный клиент имел timeout=5 s и прервал GET /health/ready раньше, чем завершились ограниченные connect/read/retry таймауты S3 SDK. Сценарий восстановления в finally запустил S3 обратно; readiness снова стала 200, исходные байты остались доступны. + +В тесте увеличен только клиентский timeout до 35 s, чтобы наблюдать фактический HTTP-ответ после bounded retries. Настройки production S3 не менялись. Повторный прогон получил 503 на readiness и скачивание во время отказа, сохранил доступ к metadata PostgreSQL и подтвердил чтение оригинальных байтов после восстановления S3. Итоговый stdout: test-s3-failure.txt; первоначальный stdout: test-s3-failure-first-attempt.txt. diff --git a/evidence/test-s3-failure-first-attempt.txt b/evidence/test-s3-failure-first-attempt.txt new file mode 100644 index 00000000..6517af4b --- /dev/null +++ b/evidence/test-s3-failure-first-attempt.txt @@ -0,0 +1,3 @@ +python3 student-tests/s3_failure.py --base-url "http://localhost:8080" +FAIL: GET /health/ready: connection/TLS failure (TimeoutError) +make: *** [Makefile:31: test-s3-failure] Error 1 diff --git a/evidence/test-s3-failure.txt b/evidence/test-s3-failure.txt new file mode 100644 index 00000000..fe5b35ce --- /dev/null +++ b/evidence/test-s3-failure.txt @@ -0,0 +1,3 @@ +python3 student-tests/s3_failure.py --base-url "http://localhost:8080" +PASS: S3 outage returned 503 for readiness/download; metadata remained available +PASS: original attachment bytes were readable after S3 recovery diff --git a/evidence/test-student.txt b/evidence/test-student.txt new file mode 100644 index 00000000..f9951d74 --- /dev/null +++ b/evidence/test-student.txt @@ -0,0 +1,2 @@ +python3 student-tests/attachments.py --base-url "http://localhost:8080" +PASS: upload/checksum/download, chat ACL, message binding and upload limits diff --git a/evidence/verification-metadata.txt b/evidence/verification-metadata.txt new file mode 100644 index 00000000..1bf74649 --- /dev/null +++ b/evidence/verification-metadata.txt @@ -0,0 +1,12 @@ +Date: 2026-11-02 +Base commit: ec540a1 +Branch: solution/BVT2302-Fedotov +Working tree: uncommitted at author's request; no commit or push created +Docker Engine: 29.8.0 +WSL kernel: 6.18.33.2-microsoft-standard-WSL2 +CPU threads: 32 +Memory available to WSL: 15 GiB +Compose limits: each API replica 1 CPU / 512 MiB; other containers have no explicit resource limits +Verification profile: make check; full OpenAPI validation; public smoke; upload/download, checksum, ACL and upload limits; 16-request idempotency race across two APIs; private bucket anonymous listing; S3 outage and recovery; cross-replica attachment read; full persistence restart; 140-write API failover; two repeated migration runs (versions 1–4). +Load benchmark: not run. +Evidence contents: small sanitized command/stdout summaries only; no .env, credentials, tokens, user payloads, or raw dumps. diff --git a/migrations/0001_initial.sql b/migrations/0001_initial.sql new file mode 100644 index 00000000..296fc4b0 --- /dev/null +++ b/migrations/0001_initial.sql @@ -0,0 +1,37 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL CHECK (length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''), + created_at timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); + +-- A leaked session table must not contain usable bearer credentials. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/migrations/0002_remove_postgres_sessions.sql b/migrations/0002_remove_postgres_sessions.sql new file mode 100644 index 00000000..23a244c2 --- /dev/null +++ b/migrations/0002_remove_postgres_sessions.sql @@ -0,0 +1,3 @@ +-- Active sessions move to Redis in lab2 grade 4; old token digests cannot be +-- reconstructed and are intentionally invalidated during this one-time migration. +DROP TABLE IF EXISTS sessions; diff --git a/migrations/0003_message_idempotency.sql b/migrations/0003_message_idempotency.sql new file mode 100644 index 00000000..eb04798b --- /dev/null +++ b/migrations/0003_message_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE message_idempotency ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key text NOT NULL CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash bytea NOT NULL CHECK (octet_length(request_hash) = 32), + message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, idempotency_key) +); +CREATE INDEX message_idempotency_expiry_idx ON message_idempotency (created_at); diff --git a/migrations/0004_attachments.sql b/migrations/0004_attachments.sql new file mode 100644 index 00000000..994a03ca --- /dev/null +++ b/migrations/0004_attachments.sql @@ -0,0 +1,37 @@ +ALTER TABLE messages + DROP CONSTRAINT messages_text_check, + ADD CONSTRAINT messages_text_check CHECK (length(text) BETWEEN 0 AND 4000); + +CREATE TABLE attachments ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + owner_id uuid NOT NULL REFERENCES users(id), + object_key text NOT NULL UNIQUE, + filename text NOT NULL CHECK (length(filename) BETWEEN 1 AND 255), + content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')), + size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760), + sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'), + status text NOT NULL DEFAULT 'ready' CHECK (status = 'ready'), + variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (variants = '[]'::jsonb), + error_code text, + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now(), + UNIQUE (id, chat_id) +); + +CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id); + +ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id); + +CREATE TABLE message_attachments ( + message_id uuid NOT NULL, + attachment_id uuid NOT NULL, + chat_id uuid NOT NULL, + position smallint NOT NULL CHECK (position BETWEEN 0 AND 9), + PRIMARY KEY (message_id, attachment_id), + UNIQUE (message_id, position), + FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE, + FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) ON DELETE RESTRICT +); + +CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id); diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 00000000..e110a7d6 --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,29 @@ +# DNS в Compose обновляется после замены остановленной API-реплики. +resolver 127.0.0.11 valid=1s ipv6=off; + +upstream api_backend { + zone api_backend 64k; + server api:8080 resolve max_fails=1 fail_timeout=2s; + keepalive 32; +} + +server { + listen 8080; + server_name _; + client_max_body_size 12m; + + location / { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 2s; + proxy_send_timeout 30s; + proxy_read_timeout 30s; + proxy_request_buffering off; + proxy_next_upstream error timeout http_502 http_503 http_504; + proxy_next_upstream_tries 2; + } +} diff --git a/requirements-app.txt b/requirements-app.txt new file mode 100644 index 00000000..7c9968a7 --- /dev/null +++ b/requirements-app.txt @@ -0,0 +1,8 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +pydantic-settings==2.7.1 +argon2-cffi==23.1.0 +asyncpg==0.30.0 +redis==5.2.1 +boto3==1.36.26 +Pillow==11.1.0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..5534aaac --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +# compose down удаляет контейнеры и сеть, но оставляет именованные volumes. +docker compose --env-file .env down +docker compose --env-file .env up -d diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..7b1fc46e --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,15 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides X-Instance-Id of the target replica}" + +for container in $(docker compose ps -q api); do + instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then + # Manual stop leaves this replica out of service until the operator restores it. + docker stop --timeout 1 "$container" >/dev/null + exit 0 + fi +done +printf '%s\n' 'target API replica was not found; no container was stopped' >&2 +exit 1 diff --git a/student-tests/README.md b/student-tests/README.md new file mode 100644 index 00000000..1f7b6be4 --- /dev/null +++ b/student-tests/README.md @@ -0,0 +1,31 @@ +# Собственные проверки Lab 4 + +Этот каталог принадлежит студенту. Публичные contract tests в `tests/` сохранены без изменений. + +Запуск после старта стенда: + +```sh +make test-student BASE_URL=http://localhost:8080 +``` + +Проверка создаёт синтетических пользователей и чат, загружает публичный PNG fixture, читает его как второй участник, проверяет ACL для постороннего, привязку вложения к чату и типовые ошибки upload. Она не печатает токены и не сохраняет пользовательские данные. + +Для обязательного эксперимента чтения между API-репликами выполните после smoke: + +```sh +make test-attachment-failover BASE_URL=http://localhost:8080 +``` + +Скрипт останавливает только API-реплику, которая обслужила upload, читает оригинал через LB, затем восстанавливает две реплики. Все хранилища и volumes остаются на месте. + +Проверка приватности bucket из сети API: + +```sh +make test-private-bucket +``` + +Проверка недоступности и восстановления S3 (временно останавливает только сервис s3, затем запускает его обратно): + +```sh +make test-s3-failure BASE_URL=http://localhost:8080 +``` diff --git a/student-tests/attachment_failover.py b/student-tests/attachment_failover.py new file mode 100644 index 00000000..7abcaa22 --- /dev/null +++ b/student-tests/attachment_failover.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Upload through one API instance, stop it, then download through its peer.""" +import os +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Lab 4 attachment cross-replica check') + client = Client(args.base_url, args.ca_file, timeout=3) + alice, bob, eve, chat = client.bootstrap() + content = (ROOT / 'tests/fixtures/sample.png').read_bytes() + uploaded = client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 201, actor=alice, raw=content, headers={'Content-Type': 'image/png'}, + schema='Attachment', + ) + source_instance = uploaded['headers']['X-Instance-Id'] + observed = set() + for _ in range(50): + observed.add(client.call( + 'GET', '/health/live', schema='Health', + )['headers']['X-Instance-Id']) + check(len(observed) >= 2, 'expected at least two API instances before stopping one') + check(source_instance in observed, 'upload instance was not observed by the load balancer') + + hook = ROOT / 'scripts/stop-one.sh' + env = dict(os.environ) + env['TARGET_INSTANCE_ID'] = source_instance + subprocess_result = subprocess.run( + [str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=10, check=False, + ) + check(subprocess_result.returncode == 0, 'stop-one hook could not stop the upload instance') + + try: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + try: + metadata = client.call( + 'GET', '/api/v1/attachments/' + uploaded['json']['id'], + actor=bob, schema='Attachment', + ) + download = client.call( + 'GET', '/api/v1/attachments/' + uploaded['json']['id'] + '/content', + actor=bob, + ) + if (metadata['headers']['X-Instance-Id'] != source_instance + and download['headers']['X-Instance-Id'] != source_instance): + check(download['bytes'] == content, 'cross-replica bytes differ from uploaded file') + check(metadata['json']['sha256'] == uploaded['json']['sha256'], + 'cross-replica metadata checksum changed') + print('PASS: upload instance stopped; metadata and original bytes read via peer') + return + except Failure: + time.sleep(0.25) + raise Failure('attachment was not readable from the other API instance within 10 seconds') + finally: + # Restore the two-replica local test stack after this explicit failure experiment. + subprocess.run( + ['docker', 'compose', '--env-file', '.env', 'up', '-d', '--scale', 'api=2'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=60, check=False, + ) + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/attachments.py b/student-tests/attachments.py new file mode 100644 index 00000000..6ffedfa2 --- /dev/null +++ b/student-tests/attachments.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Собственные интеграционные проверки вложений поверх неизменённого публичного API.""" +import hashlib +from pathlib import Path +import sys +from urllib.parse import quote +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments('Lab 4 student attachment checks') + client = Client(args.base_url, args.ca_file) + alice, bob, eve, chat = client.bootstrap() + fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes() + uploaded = client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png', + 201, actor=alice, raw=fixture, headers={'Content-Type': 'image/png'}, + schema='Attachment', + )['json'] + check(uploaded['status'] == 'ready', 'upload was not ready') + check(uploaded['variants'] == [] and uploaded['error_code'] is None, + 'Lab 4 ready metadata must have no variants or error') + check(uploaded['sha256'] == hashlib.sha256(fixture).hexdigest(), + 'metadata checksum differs from fixture') + check(uploaded['size_bytes'] == len(fixture), 'metadata size differs from fixture') + + metadata = client.call( + 'GET', '/api/v1/attachments/' + uploaded['id'], + actor=bob, schema='Attachment', + )['json'] + check(metadata == uploaded, 'another chat participant got different attachment metadata') + downloaded = client.call( + 'GET', '/api/v1/attachments/' + uploaded['id'] + '/content', actor=bob, + ) + check(downloaded['bytes'] == fixture, 'downloaded bytes are not identical') + check(downloaded['headers'].get_content_type() == 'image/png', + 'download Content-Type is not the detected image type') + client.call('GET', '/api/v1/attachments/' + uploaded['id'], 404, actor=eve) + client.call('GET', '/api/v1/attachments/' + uploaded['id'] + '/content', 404, actor=eve) + + # Нельзя привязать существующий файл чата к сообщению другого чата. + other_chat = client.call( + 'POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]}, + schema='Chat', + )['json'] + client.call( + 'POST', '/api/v1/chats/' + other_chat['id'] + '/messages', 404, + actor=alice, body={'text': '', 'attachment_ids': [uploaded['id']]}, + headers={'Idempotency-Key': uuid.uuid4().hex}, + ) + + prefix = '/api/v1/chats/' + chat['id'] + '/attachments?filename=' + client.call( + 'POST', prefix + 'bad.png', 415, actor=alice, raw=b'not an image', + headers={'Content-Type': 'image/png'}, + ) + client.call( + 'POST', prefix + 'mismatch.jpg', 415, actor=alice, raw=fixture, + headers={'Content-Type': 'image/jpeg'}, + ) + client.call( + 'POST', prefix + quote('../sample.png', safe=''), 400, actor=alice, + raw=fixture, headers={'Content-Type': 'image/png'}, + ) + client.call( + 'POST', prefix + 'too-large.png', 413, actor=alice, + raw=b'\x00' * (10 * 1024 * 1024 + 1), + headers={'Content-Type': 'image/png'}, + ) + print('PASS: upload/checksum/download, chat ACL, message binding and upload limits') + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/idempotency.py b/student-tests/idempotency.py new file mode 100644 index 00000000..465f5ee0 --- /dev/null +++ b/student-tests/idempotency.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Проверка атомарного повтора одного сообщения через разные API-реплики.""" +from concurrent.futures import ThreadPoolExecutor +import uuid +from pathlib import Path +import sys +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, arguments, check, run + + +def main(): + args = arguments('Concurrent cross-replica idempotency check') + setup = Client(args.base_url, args.ca_file) + alice, _bob, _eve, chat = setup.bootstrap() + key = 'concurrent-' + uuid.uuid4().hex + text = 'Одно сообщение при конкурентном повторе ' + uuid.uuid4().hex + count = 16 + + def send(_): + client = Client(args.base_url, args.ca_file, timeout=10) + return client.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=201, + actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message', + ) + + with ThreadPoolExecutor(max_workers=count) as pool: + responses = list(pool.map(send, range(count))) + messages = [response['json'] for response in responses] + instance_ids = {response['headers'].get('X-Instance-Id') for response in responses} + check(all(message == messages[0] for message in messages), 'concurrent replay returned different Message objects') + check(len(instance_ids) >= 2, 'concurrent requests did not reach at least two observable API replicas') + + replay = setup.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=201, + actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message', + )['json'] + check(replay == messages[0], 'retry returned a different status payload') + setup.call( + 'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=409, + actor=alice, body={'text': text + ' другой текст'}, headers={'Idempotency-Key': key}, + ) + history = setup.call( + 'GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', + actor=alice, schema='MessagePage', + )['json']['items'] + matching = [message for message in history if message['text'] == text] + check(matching == [messages[0]], 'history contains a missing or duplicate side effect') + print('PASS: %d concurrent requests across %d API replicas returned one Message; changed body returned 409.' % (count, len(instance_ids))) + + +if __name__ == '__main__': + run(main) diff --git a/student-tests/private_bucket.py b/student-tests/private_bucket.py new file mode 100644 index 00000000..9fb87f92 --- /dev/null +++ b/student-tests/private_bucket.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Отправить анонимный ListBucket из сети приложения и подтвердить private bucket.""" +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def main(): + env_path = ROOT / '.env' + if not env_path.is_file(): + raise SystemExit('Create local .env from .env.example first') + values = {} + for line in env_path.read_text().splitlines(): + if '=' in line and not line.lstrip().startswith('#'): + key, value = line.split('=', 1) + values[key.strip()] = value.strip() + bucket = values.get('S3_BUCKET', 'messenger-attachments') + url = 'http://s3:9000/' + bucket + '?list-type=2' + code = ( + 'import urllib.error, urllib.request\n' + 'url = ' + repr(url) + '\n' + 'try:\n' + ' urllib.request.urlopen(url, timeout=5)\n' + 'except urllib.error.HTTPError as error:\n' + ' print("HTTP", error.code)\n' + ' raise SystemExit(0 if error.code in (401, 403) else 1)\n' + 'except Exception as error:\n' + ' print(type(error).__name__)\n' + ' raise SystemExit(1)\n' + 'raise SystemExit("anonymous bucket listing unexpectedly succeeded")\n' + ) + result = subprocess.run( + ['docker', 'compose', '--env-file', '.env', 'exec', '-T', 'api', 'python', '-c', code], + cwd=ROOT, check=False, + ) + if result.returncode: + raise SystemExit('FAIL: anonymous S3 bucket listing was not denied') + print('PASS: private bucket rejects unauthenticated listing') + + +if __name__ == '__main__': + main() diff --git a/student-tests/s3_failure.py b/student-tests/s3_failure.py new file mode 100644 index 00000000..e009376c --- /dev/null +++ b/student-tests/s3_failure.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Проверить ограниченную ошибку API при отказе S3 и восстановить его.""" +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments('Lab 4 S3 outage and recovery check') + # Дождаться ограниченных повторов boto3 при остановленном S3. + client = Client(args.base_url, args.ca_file, timeout=35) + alice, bob, eve, chat = client.bootstrap() + attachment, content = client.attachment(alice, chat) + initial = client.call( + 'GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob, + ) + check(initial['bytes'] == content, 'uploaded bytes differ before the outage') + + stopped = subprocess.run( + ['docker', 'compose', '--env-file', '.env', 'stop', 's3'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=20, check=False, + ) + check(stopped.returncode == 0, 'could not stop S3 for the outage test') + try: + ready = client.call('GET', '/health/ready', expected=503, schema='Error') + check(ready['json']['error']['code'] == 'unavailable', + 'readiness did not report unavailable S3') + failed = client.call( + 'GET', '/api/v1/attachments/' + attachment['id'] + '/content', + expected=503, actor=bob, schema='Error', + ) + check(failed['json']['error']['code'] == 'unavailable', + 'S3 failure was not translated to a bounded API error') + metadata = client.call( + 'GET', '/api/v1/attachments/' + attachment['id'], + actor=bob, schema='Attachment', + ) + check(metadata['json']['id'] == attachment['id'], + 'PostgreSQL metadata was lost while S3 was unavailable') + print('PASS: S3 outage returned 503 for readiness/download; metadata remained available') + finally: + started = subprocess.run( + ['docker', 'compose', '--env-file', '.env', 'start', 's3'], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + timeout=20, check=False, + ) + check(started.returncode == 0, 'could not restart S3 after the outage test') + + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + try: + client.call('GET', '/health/ready', schema='Health') + break + except Failure: + time.sleep(0.25) + else: + raise Failure('API did not become ready after S3 restart') + + recovered = client.call( + 'GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob, + ) + check(recovered['bytes'] == content, 'S3 restart changed or lost the original bytes') + print('PASS: original attachment bytes were readable after S3 recovery') + + +if __name__ == '__main__': + run(main) -- 2.54.0