forked from mtusi2026/mtusi2026lab4
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
b99ad646e8 |
+15
-15
@@ -1,20 +1,20 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# Copy to .env and replace every placeholder with a local random value.
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=replace_with_random_hex
|
||||
POSTGRES_DB=messenger
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
S3_ENDPOINT=http://s3:9000
|
||||
DB_POOL_MIN_SIZE=1
|
||||
DB_POOL_MAX_SIZE=8
|
||||
DB_POOL_ACQUIRE_TIMEOUT_SECONDS=3
|
||||
DB_COMMAND_TIMEOUT_SECONDS=5
|
||||
REDIS_TIMEOUT_SECONDS=2
|
||||
S3_ACCESS_KEY_ID=replace_with_random_access_id
|
||||
S3_SECRET_ACCESS_KEY=replace_with_random_secret
|
||||
S3_REGION=us-east-1
|
||||
S3_BUCKET=messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
S3_CONNECT_TIMEOUT_SECONDS=3
|
||||
S3_READ_TIMEOUT_SECONDS=10
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
API_REPLICAS=2
|
||||
LB_PORT=8080
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
* text=auto
|
||||
*.sh text eol=lf
|
||||
@@ -1,18 +1,21 @@
|
||||
## Лабораторная 4
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Федотов Михаил, БВТ2302
|
||||
Целевая оценка: 3
|
||||
Трек 5: не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и перенесено: обязательная основа Lab 3 (PostgreSQL, Argon2id и Redis-сессии, LB × 2 API, идемпотентность) и Lab 4 (RustFS S3, versioned migration 0004, metadata в PostgreSQL, оригиналы в приватном bucket, authenticated upload/download, attachment_ids в сообщениях). Собственные проверки размещены в student-tests/.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone: см. раздел «Запуск из чистого clone» в [REPORT.md](../REPORT.md). Кратко: скопировать .env.example в .env, заменить secrets, выполнить docker compose config и up --build, затем make check/validate/test и student tests.
|
||||
|
||||
Результаты проверок: make check, make validate PYTHON=.venv/bin/python, make test, student tests, S3 outage/recovery, restart, API failover и повторный запуск миграций прошли. Конкретный stdout лежит в evidence/; сводка результатов и ограничения — в REPORT.md.
|
||||
|
||||
Ограничения / известные ошибки: одноузловый RustFS и один Docker host; автоматическая сверка orphan S3 objects и отдельная минимальная S3 policy не реализованы, так как целевой уровень — 3. TLS и отдельный load benchmark не заявлены; стенд рассчитан на localhost и синтетические данные.
|
||||
|
||||
Источники и использование ИИ: исходники Lab 3 — собственная предыдущая работа; RustFS использован по официальной документации. При переносе и реализации использовался ИИ-помощник.
|
||||
|
||||
- [x] Контракт и публичные tests сохранены без ослабления.
|
||||
- [x] Отчёт содержит инструкцию запуска именно этого решения.
|
||||
- [x] Secrets и локальные volumes исключены из Git через .gitignore.
|
||||
- [x] make check, make validate и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказа и сохранности уровня 3 воспроизведены.
|
||||
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
FROM python:3.12.8-slim
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
AWS_EC2_METADATA_DISABLED=true
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements-app.txt ./
|
||||
RUN pip install --no-cache-dir -r requirements-app.txt
|
||||
COPY app ./app
|
||||
COPY migrations ./migrations
|
||||
|
||||
RUN useradd --create-home --uid 10001 appuser
|
||||
USER appuser
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
@@ -4,10 +4,10 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
.PHONY: help check validate test load test-student test-idempotency test-private-bucket test-s3-failure test-attachment-failover test-persistence test-failover
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "check: offline structure; validate: full OpenAPI; test: public smoke; test-student: own Lab 4 checks"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -18,13 +18,28 @@ validate:
|
||||
test:
|
||||
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-student:
|
||||
$(PYTHON) student-tests/attachments.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-idempotency:
|
||||
$(PYTHON) student-tests/idempotency.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-private-bucket:
|
||||
$(PYTHON) student-tests/private_bucket.py
|
||||
|
||||
test-s3-failure:
|
||||
$(PYTHON) student-tests/s3_failure.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-attachment-failover:
|
||||
$(PYTHON) student-tests/attachment_failover.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
load:
|
||||
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-persistence:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to scripts/restart.sh"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to scripts/stop-one.sh"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
@@ -1,37 +1,95 @@
|
||||
# Отчёт — лабораторная 4
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Федотов Михаил, БВТ2302
|
||||
- Целевая оценка: 3
|
||||
- Трек 5: не заявлен
|
||||
- Commit/PR: ещё не создано по просьбе автора; текущая ветка `solution/BVT2302-Fedotov`
|
||||
- Использованные источники и помощь: исходный код и миграции собственной Lab 3; [официальная документация RustFS](https://docs.rustfs.com/en/installation/container/docker) и [закреплённый образ 1.0.4-glibc](https://github.com/rustfs/rustfs/pkgs/container/rustfs); при разработке использовался ИИ-помощник. Публичные README, контракты и тесты Lab 4 не копировались поверх новой версии и не ослаблялись.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требуются Docker Engine с Compose V2 и Python 3.9+ для публичных проверок. Приложение собирается в Docker на Python 3.12.8. Подготовьте локальные значения secrets; файл `.env` игнорируется Git:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
sed -i "s/replace_with_random_hex/$(openssl rand -hex 24)/" .env
|
||||
sed -i "s/replace_with_random_access_id/$(openssl rand -hex 12)/" .env
|
||||
sed -i "s/replace_with_random_secret/$(openssl rand -hex 32)/" .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build
|
||||
until curl -fsS http://localhost:8080/health/ready; do sleep 2; done
|
||||
```
|
||||
|
||||
Проверки:
|
||||
|
||||
```sh
|
||||
make check
|
||||
python3 -m venv .venv
|
||||
.venv/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv/bin/python
|
||||
make test BASE_URL=http://localhost:8080
|
||||
make test-student BASE_URL=http://localhost:8080
|
||||
make test-idempotency BASE_URL=http://localhost:8080
|
||||
make test-private-bucket
|
||||
make test-s3-failure
|
||||
make test-attachment-failover BASE_URL=http://localhost:8080
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:8080
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:8080
|
||||
docker compose --env-file .env up -d --scale api=2
|
||||
```
|
||||
|
||||
Обычная остановка без удаления данных: `docker compose --env-file .env down`. Намеренное удаление учебной БД, сессий и S3-объектов: `docker compose --env-file .env down -v`; выполнять только при осознанном сбросе стенда.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab3, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
Клиент → Nginx LB → две API-реплики → PostgreSQL, Redis и приватный RustFS S3. Наружу опубликован только LB на localhost:8080; PostgreSQL, Redis, API и S3 доступны только во внутренней Compose-сети. Консоль RustFS выключена и порт S3 на хост не опубликован.
|
||||
|
||||
Из Lab 3 перенесены FastAPI-маршруты и схема Basic → Argon2id/Bearer-сессия, PostgreSQL/Redis-хранилище, миграции 0001–0003, cursor pagination, идемпотентность сообщений, Nginx и hooks отказа/перезапуска. Публичные `tests/`, `contracts/`, README и шаблоны преподавателя не заменялись кодом из предыдущих лабораторных.
|
||||
|
||||
В Lab 4 добавлена миграция 0004:
|
||||
- `attachments` хранит ID чата/владельца, имя для отображения, MIME, размер, SHA-256, статус и серверный object key;
|
||||
- `message_attachments` связывает сообщение и вложение, а составные внешние ключи запрещают ссылку на файл другого чата;
|
||||
- пустой текст сообщения разрешён только если в запросе есть хотя бы одно вложение; проверка выполняется API.
|
||||
|
||||
Оригинальные байты находятся только в S3. API ограниченно читает тело до 10 MiB, распознаёт PNG/JPEG через Pillow, сопоставляет MIME с реальным форматом и проверяет пиксели до декодирования. S3 объект записывается до метаданных; если запись метаданных в PostgreSQL не проходит, API пытается удалить только что записанный объект. HTTP 201 возвращается после записи объекта и метаданных. Object key генерируется как UUID и не зависит от filename.
|
||||
|
||||
Bucket создаётся при старте API, отдельная политика публичного доступа не назначается; S3 endpoint не опубликован наружу. Участник чата получает metadata и оригинал через API, посторонний получает 404. Все сервисы состояния используют именованные volumes: `postgres_data`, `redis_data`, `s3_data`.
|
||||
|
||||
Образы закреплены: Python 3.12.8-slim, PostgreSQL 16.4-alpine, Redis 7.4.2-alpine, Nginx 1.27.4-alpine и RustFS 1.0.4-glibc по digest. Версии Python-зависимостей приложения закреплены в `requirements-app.txt`.
|
||||
|
||||
| Роль | Service | Данные/готовность | Порт и credentials |
|
||||
| --- | --- | --- | --- |
|
||||
| API | `api`, 2 реплики | readiness проверяет PostgreSQL, Redis и S3; локального состояния нет | доступ только из Compose-сети |
|
||||
| Метаданные | `postgres`, 16.4 | именованный volume; `pg_isready` | внутренний 5432; пароль только в локальном `.env` |
|
||||
| Сессии | `redis`, 7.4.2 | AOF и `appendfsync always`, именованный volume | внутренний 6379 |
|
||||
| Бинарные объекты | `s3`, RustFS 1.0.4 | приватный bucket, именованный volume; API проверяет `head_bucket` | внутренний 9000; console выключена |
|
||||
| Балансировщик | `lb`, Nginx 1.27.4 | healthcheck через API readiness | localhost:8080 |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий | Проверка | Результат финального прогона |
|
||||
| --- | --- | --- |
|
||||
| Структура и сохранность контракта | `make check`, `git diff --check` | PASS; `evidence/make-check.txt`, `evidence/diff-check.txt` |
|
||||
| OpenAPI 3.1 и JSON Schema | `make validate PYTHON=.venv/bin/python` | PASS; `evidence/make-validate.txt` |
|
||||
| Синтаксис приложения, тестов и hooks | `compileall`, YAML parse, `sh -n`, Compose config | PASS; отдельные stdout-файлы в `evidence/` |
|
||||
| Публичный API и вложения | `make test`, `make test-student` | PASS; `evidence/make-test.txt`, `evidence/test-student.txt` |
|
||||
| Идемпотентность и доступ к bucket | `make test-idempotency`, `make test-private-bucket` | PASS; 16 конкурентных запросов, приватный bucket отвечает 403 на анонимный листинг |
|
||||
| Отказ S3 и чтение между репликами | `make test-s3-failure`, `make test-attachment-failover` | PASS; API вернул 503 при отказе S3, метаданные сохранились, файл читается после восстановления и через вторую API-реплику |
|
||||
| Перезапуск и API failover | `make test-persistence`, `make test-failover` | PASS; подтверждённые данные и действующая сессия сохранены |
|
||||
| Повторный запуск миграций | два запуска `migrate` и запрос `schema_migrations` | PASS; в таблице ровно версии 1, 2, 3, 4 |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Первый запуск нового S3-теста оборвался по 5-секундному таймауту тестового HTTP-клиента до завершения повторных попыток S3 SDK. Причина и исправление тестового таймаута записаны в `evidence/test-s3-failure-analysis.txt`; следующий прогон прошёл полностью. Ошибки приложения этим первым запуском не маскируются.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
`scripts/restart.sh` пересоздаёт контейнеры и сеть Compose, не удаляя volumes. Финальный `make test-persistence` подтвердил чтение существующих чатов/сообщений, действующей сессии и оригинала вложения после перезапуска стека.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
`scripts/stop-one.sh` находит API контейнер по `X-Instance-Id` и останавливает только его. Межрепличный тест загрузил вложение, остановил обслужившую запрос API-реплику и прочитал metadata и оригинальные байты через оставшуюся реплику. Нагрузочный failover подтвердил 2 реплики до отказа, 140 успешных записей, 0 неуспешных попыток и восстановление за 0.556 секунды; подтверждённая запись и действующая сессия сохранились. Финальный статус контейнеров записан в `evidence/compose-final.txt`.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
Тест отказа S3 остановил только service `s3`: readiness и чтение оригинала вернули ограниченный HTTP 503, PostgreSQL metadata оставались доступны, после перезапуска S3 исходные байты читались без изменений. Проверка неавторизованного доступа к bucket получила HTTP 403.
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
Аппаратный профиль WSL на момент проверки: 32 CPU threads и 15 GiB памяти; Docker Engine 29.8.0. Для каждой API-реплики установлены пределы 1 CPU и 512 MiB; для остальных контейнеров явные resource limits не заданы. Отдельный throughput/load benchmark не запускался и не заявляется. Все фактические stdout-проверки и профиль сохранены в `evidence/`.
|
||||
|
||||
## Ограничения
|
||||
|
||||
Цель этой работы — уровень 3. Один узел RustFS и один Docker host сами по себе не обеспечивают отказоустойчивость к потере диска/хоста. API использует ключ доступа RustFS из локального `.env`; отдельный пользователь S3 с минимальными правами и механизм сверки orphan объектов относятся к дополнительным практикам уровня 4 и не заявляются. Если компенсирующее удаление после ошибки PostgreSQL также не удастся, может остаться orphan-объект; автоматическая сверка не реализована. TLS отсутствует, стенд предназначен для localhost и синтетических данных.
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
from fastapi import Request
|
||||
|
||||
from app.core.errors import ApiError
|
||||
from app.repositories.memory import SessionRow, UserRow
|
||||
|
||||
import uuid as _uuid
|
||||
|
||||
BEARER_CHALLENGE = {'WWW-Authenticate': 'Bearer realm="messenger"'}
|
||||
|
||||
|
||||
def storage_of(request: Request):
|
||||
return request.app.state.storage
|
||||
|
||||
|
||||
async def current_session(request: Request) -> SessionRow:
|
||||
header = request.headers.get('authorization', '')
|
||||
token = header[7:].strip() if header[:7].lower() == 'bearer ' else ''
|
||||
if not token:
|
||||
raise ApiError(401, 'Bearer session required', headers=BEARER_CHALLENGE)
|
||||
row = await storage_of(request).session(token)
|
||||
if row is None:
|
||||
raise ApiError(401, 'Session is missing, expired or revoked', headers=BEARER_CHALLENGE)
|
||||
return row
|
||||
|
||||
|
||||
async def current_user(request: Request) -> UserRow:
|
||||
session = await current_session(request)
|
||||
user = await storage_of(request).user(session.user_id)
|
||||
if user is None:
|
||||
raise ApiError(401, 'Session user no longer exists', headers=BEARER_CHALLENGE)
|
||||
return user
|
||||
|
||||
|
||||
async def require_chat_member(request: Request, chat_id: str, user: UserRow):
|
||||
try:
|
||||
_uuid.UUID(chat_id)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'chat_id must be a UUID')
|
||||
row = await storage_of(request).chat(chat_id)
|
||||
if row is None or user.id not in row.member_ids:
|
||||
raise ApiError(404, 'Chat is not available for this user')
|
||||
return row
|
||||
@@ -0,0 +1,155 @@
|
||||
import hashlib
|
||||
import io
|
||||
import uuid
|
||||
import warnings
|
||||
|
||||
from fastapi import APIRouter, Query, Request
|
||||
from fastapi.responses import Response
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
|
||||
from app.api.deps import current_user, require_chat_member
|
||||
from app.core.config import settings
|
||||
from app.core.errors import ApiError
|
||||
|
||||
router = APIRouter()
|
||||
MIME_BY_FORMAT = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}
|
||||
|
||||
|
||||
def _validate_filename(filename: str) -> None:
|
||||
if not filename or len(filename) > 255 or not filename.strip():
|
||||
raise ApiError(400, 'Invalid filename')
|
||||
if any(ch in filename for ch in ('/', chr(92))):
|
||||
raise ApiError(400, 'Invalid filename')
|
||||
if any(ord(ch) == 0 or __import__('unicodedata').category(ch).startswith('C') for ch in filename):
|
||||
raise ApiError(400, 'Invalid filename')
|
||||
|
||||
|
||||
async def _read_bounded_body(request: Request) -> bytes:
|
||||
length = request.headers.get('content-length')
|
||||
if length:
|
||||
try:
|
||||
size = int(length)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'Invalid Content-Length') from None
|
||||
if size < 0:
|
||||
raise ApiError(400, 'Invalid Content-Length')
|
||||
if size > settings.max_upload_bytes:
|
||||
raise ApiError(413, 'Image exceeds the maximum size')
|
||||
chunks = []
|
||||
total = 0
|
||||
async for chunk in request.stream():
|
||||
total += len(chunk)
|
||||
if total > settings.max_upload_bytes:
|
||||
raise ApiError(413, 'Image exceeds the maximum size')
|
||||
chunks.append(chunk)
|
||||
if total == 0:
|
||||
raise ApiError(415, 'Empty or invalid image')
|
||||
return b''.join(chunks)
|
||||
|
||||
|
||||
def _validate_image(data: bytes, declared_type: str) -> None:
|
||||
if declared_type not in {'image/png', 'image/jpeg'}:
|
||||
raise ApiError(415, 'Only PNG and JPEG images are accepted')
|
||||
try:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
||||
with Image.open(io.BytesIO(data)) as image:
|
||||
actual_type = MIME_BY_FORMAT.get(image.format)
|
||||
width, height = image.size
|
||||
if actual_type is None or actual_type != declared_type:
|
||||
raise ApiError(415, 'Image content does not match Content-Type')
|
||||
if width < 1 or height < 1:
|
||||
raise ApiError(415, 'Invalid image dimensions')
|
||||
if width * height > settings.max_image_pixels:
|
||||
raise ApiError(413, 'Image exceeds the maximum pixel count')
|
||||
image.verify()
|
||||
except ApiError:
|
||||
raise
|
||||
except (UnidentifiedImageError, Image.DecompressionBombError,
|
||||
Image.DecompressionBombWarning, OSError, SyntaxError, ValueError):
|
||||
raise ApiError(415, 'Image content is invalid or unsupported') from None
|
||||
|
||||
|
||||
def attachment_out(row) -> dict:
|
||||
return {
|
||||
'id': row.id,
|
||||
'chat_id': row.chat_id,
|
||||
'owner_id': row.owner_id,
|
||||
'filename': row.filename,
|
||||
'content_type': row.content_type,
|
||||
'size_bytes': row.size_bytes,
|
||||
'sha256': row.sha256,
|
||||
'status': 'ready',
|
||||
'variants': [],
|
||||
'error_code': None,
|
||||
'created_at': row.created_at,
|
||||
'updated_at': row.updated_at,
|
||||
}
|
||||
|
||||
|
||||
@router.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
|
||||
async def upload_attachment(
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
filename: str = Query(min_length=1, max_length=255),
|
||||
):
|
||||
user = await current_user(request)
|
||||
chat = await require_chat_member(request, chat_id, user)
|
||||
_validate_filename(filename)
|
||||
content_type = request.headers.get('content-type', '').split(';', 1)[0].strip().lower()
|
||||
if content_type not in {'image/png', 'image/jpeg'}:
|
||||
raise ApiError(415, 'Only PNG and JPEG images are accepted')
|
||||
data = await _read_bounded_body(request)
|
||||
_validate_image(data, content_type)
|
||||
|
||||
attachment_id = str(uuid.uuid4())
|
||||
object_key = 'original/' + uuid.uuid4().hex
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
object_store = request.app.state.object_store
|
||||
await object_store.put(object_key, data, content_type)
|
||||
try:
|
||||
row = await request.app.state.storage.create_attachment(
|
||||
attachment_id=attachment_id,
|
||||
chat_id=chat.id,
|
||||
owner_id=user.id,
|
||||
object_key=object_key,
|
||||
filename=filename,
|
||||
content_type=content_type,
|
||||
size_bytes=len(data),
|
||||
sha256=digest,
|
||||
)
|
||||
except Exception:
|
||||
# Если запись метаданных не зафиксирована, пытаемся убрать только что загруженный объект.
|
||||
try:
|
||||
await object_store.delete(object_key)
|
||||
except ApiError:
|
||||
pass
|
||||
raise
|
||||
return attachment_out(row)
|
||||
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}')
|
||||
async def get_attachment(attachment_id: str, request: Request):
|
||||
user = await current_user(request)
|
||||
row = await request.app.state.storage.attachment_for_member(attachment_id, user.id)
|
||||
if row is None:
|
||||
raise ApiError(404, 'Attachment is not available for this user')
|
||||
return attachment_out(row)
|
||||
|
||||
|
||||
@router.get('/api/v1/attachments/{attachment_id}/content')
|
||||
async def download_attachment(
|
||||
attachment_id: str,
|
||||
request: Request,
|
||||
variant: str = Query(default='original', pattern=r'^original$'),
|
||||
):
|
||||
user = await current_user(request)
|
||||
row = await request.app.state.storage.attachment_for_member(attachment_id, user.id)
|
||||
if row is None:
|
||||
raise ApiError(404, 'Attachment is not available for this user')
|
||||
data = await request.app.state.object_store.get(row.object_key)
|
||||
return Response(
|
||||
content=data,
|
||||
media_type=row.content_type,
|
||||
headers={'Cache-Control': 'private, no-store'},
|
||||
)
|
||||
@@ -0,0 +1,31 @@
|
||||
from fastapi import APIRouter, Request
|
||||
from fastapi.responses import Response
|
||||
|
||||
from app.api.deps import current_session
|
||||
from app.api.serialize import user_out
|
||||
from app.core.config import settings
|
||||
from app.core.errors import ApiError
|
||||
from app.core.security import parse_basic, verify_password
|
||||
from app.repositories.memory import iso_z
|
||||
|
||||
router = APIRouter()
|
||||
BASIC_CHALLENGE = {'WWW-Authenticate': 'Basic realm="messenger", charset="UTF-8"'}
|
||||
|
||||
|
||||
@router.post('/api/v1/auth/sessions', status_code=201)
|
||||
async def login(request: Request):
|
||||
creds = parse_basic(request.headers.get('authorization', ''))
|
||||
storage = request.app.state.storage
|
||||
row = await storage.user_by_username(creds[0]) if creds else None
|
||||
if creds is None or row is None or not verify_password(row.password_hash, creds[1]):
|
||||
raise ApiError(401, 'Invalid credentials', headers=BASIC_CHALLENGE)
|
||||
session = await storage.create_session(row.id, settings.session_ttl_seconds)
|
||||
return {'token': session.token, 'token_type': 'Bearer',
|
||||
'expires_at': iso_z(session.expires_at), 'user': user_out(row)}
|
||||
|
||||
|
||||
@router.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
async def logout(request: Request):
|
||||
session = await current_session(request)
|
||||
await request.app.state.storage.revoke(session.token)
|
||||
return Response(status_code=204)
|
||||
@@ -0,0 +1,33 @@
|
||||
from fastapi import APIRouter, Query, Request
|
||||
|
||||
from app.api.deps import current_user, require_chat_member
|
||||
from app.api.serialize import chat_out
|
||||
from app.core.cursors import decode_cursor, paginate
|
||||
from app.schemas import CreateChat
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post('/api/v1/chats', status_code=201)
|
||||
async def create_chat(body: CreateChat, request: Request):
|
||||
user = await current_user(request)
|
||||
member_ids = [str(mid) for mid in body.member_ids]
|
||||
row = await request.app.state.storage.create_chat(body.title, member_ids, user.id)
|
||||
return chat_out(row)
|
||||
|
||||
|
||||
@router.get('/api/v1/chats')
|
||||
async def list_chats(request: Request, limit: int = Query(50, ge=1, le=100),
|
||||
cursor: str | None = Query(None, min_length=1)):
|
||||
user = await current_user(request)
|
||||
resource = 'chats:' + user.id
|
||||
key = decode_cursor(cursor, resource) if cursor else None
|
||||
rows = await request.app.state.storage.chats_of(user.id)
|
||||
page, next_cursor = paginate(rows, key, limit, resource)
|
||||
return {'items': [chat_out(r) for r in page], 'next_cursor': next_cursor}
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}')
|
||||
async def get_chat(chat_id: str, request: Request):
|
||||
user = await current_user(request)
|
||||
return chat_out(await require_chat_member(request, chat_id, user))
|
||||
@@ -0,0 +1,17 @@
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.core.errors import ApiError
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get('/health/live')
|
||||
async def live():
|
||||
return {'status': 'ok'}
|
||||
|
||||
|
||||
@router.get('/health/ready')
|
||||
async def ready(request: Request):
|
||||
if not await request.app.state.storage.ready():
|
||||
raise ApiError(503, 'Storage is unavailable')
|
||||
return {'status': 'ok'}
|
||||
@@ -0,0 +1,11 @@
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.api.deps import current_user
|
||||
from app.api.serialize import user_out
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get('/api/v1/users/me')
|
||||
async def me(request: Request):
|
||||
return user_out(await current_user(request))
|
||||
@@ -0,0 +1,52 @@
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Header, Query, Request
|
||||
|
||||
from app.api.deps import current_user, require_chat_member
|
||||
from app.api.serialize import message_out
|
||||
from app.core.cursors import decode_cursor, encode_cursor
|
||||
from app.core.errors import ApiError
|
||||
from app.schemas import CreateMessage
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post('/api/v1/chats/{chat_id}/messages', status_code=201)
|
||||
async def create_message(
|
||||
body: CreateMessage,
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
idempotency_key: str | None = Header(
|
||||
default=None, alias='Idempotency-Key', min_length=1, max_length=128,
|
||||
pattern=r'^[A-Za-z0-9._:-]{1,128}$',
|
||||
),
|
||||
):
|
||||
if idempotency_key is None:
|
||||
raise ApiError(400, 'Idempotency-Key is required')
|
||||
if not body.text.strip() and not body.attachment_ids:
|
||||
raise ApiError(400, 'Message must contain text or at least one attachment')
|
||||
user = await current_user(request)
|
||||
chat = await require_chat_member(request, chat_id, user)
|
||||
normalized_body = json.dumps(
|
||||
body.model_dump(mode='json'), ensure_ascii=False, sort_keys=True, separators=(',', ':')
|
||||
).encode('utf-8')
|
||||
body_hash = hashlib.sha256(normalized_body).digest()
|
||||
row = await request.app.state.storage.add_idempotent_message(
|
||||
chat.id, user.id, body.text, [str(value) for value in body.attachment_ids],
|
||||
idempotency_key, body_hash,
|
||||
)
|
||||
return message_out(row)
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}/messages')
|
||||
async def list_messages(request: Request, chat_id: str,
|
||||
limit: int = Query(50, ge=1, le=100),
|
||||
cursor: str | None = Query(None, min_length=1)):
|
||||
user = await current_user(request)
|
||||
chat = await require_chat_member(request, chat_id, user)
|
||||
resource = 'messages:' + chat.id
|
||||
key = decode_cursor(cursor, resource) if cursor else None
|
||||
rows, has_more = await request.app.state.storage.messages_page(chat.id, key, limit)
|
||||
next_cursor = encode_cursor(resource, rows[-1].created_at, rows[-1].id) if has_more else None
|
||||
return {'items': [message_out(row) for row in rows], 'next_cursor': next_cursor}
|
||||
@@ -0,0 +1,14 @@
|
||||
from fastapi import APIRouter, Request
|
||||
|
||||
from app.api.serialize import user_out
|
||||
from app.core.security import hash_password
|
||||
from app.schemas import CreateUser
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post('/api/v1/users', status_code=201)
|
||||
async def register(body: CreateUser, request: Request):
|
||||
row = await request.app.state.storage.create_user(
|
||||
body.username, body.display_name, hash_password(body.password))
|
||||
return user_out(row)
|
||||
@@ -0,0 +1,17 @@
|
||||
from app.repositories.memory import UserRow
|
||||
|
||||
|
||||
def user_out(row: UserRow) -> dict:
|
||||
return {'id': row.id, 'username': row.username,
|
||||
'display_name': row.display_name, 'created_at': row.created_at}
|
||||
|
||||
|
||||
def chat_out(row) -> dict:
|
||||
return {'id': row.id, 'title': row.title,
|
||||
'member_ids': list(row.member_ids), 'created_at': row.created_at}
|
||||
|
||||
|
||||
def message_out(row) -> dict:
|
||||
return {'id': row.id, 'chat_id': row.chat_id, 'sender_id': row.sender_id,
|
||||
'text': row.text, 'created_at': row.created_at,
|
||||
'attachment_ids': list(row.attachment_ids)}
|
||||
@@ -0,0 +1,55 @@
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(env_file='.env', extra='ignore')
|
||||
|
||||
app_host: str = '0.0.0.0'
|
||||
app_port: int = 8080
|
||||
session_ttl_seconds: int = 3600
|
||||
database_url: str = ''
|
||||
database_url_file: str | None = None
|
||||
redis_url: str = 'redis://localhost:6379/0'
|
||||
redis_url_file: str | None = None
|
||||
db_pool_min_size: int = 1
|
||||
db_pool_max_size: int = 10
|
||||
db_pool_acquire_timeout_seconds: float = 3.0
|
||||
db_command_timeout_seconds: float = 5.0
|
||||
redis_timeout_seconds: float = 2.0
|
||||
s3_endpoint_url: str = 'http://localhost:9000'
|
||||
s3_region: str = 'us-east-1'
|
||||
s3_bucket: str = 'messenger-attachments'
|
||||
s3_access_key_id: str = ''
|
||||
s3_access_key_id_file: str | None = None
|
||||
s3_secret_access_key: str = ''
|
||||
s3_secret_access_key_file: str | None = None
|
||||
s3_connect_timeout_seconds: float = 3.0
|
||||
s3_read_timeout_seconds: float = 10.0
|
||||
max_upload_bytes: int = Field(default=10_485_760, ge=1, le=10_485_760)
|
||||
max_image_pixels: int = Field(default=40_000_000, ge=1, le=40_000_000)
|
||||
|
||||
def resolved_database_url(self) -> str:
|
||||
if self.database_url_file:
|
||||
return Path(self.database_url_file).read_text(encoding='utf-8').strip()
|
||||
return self.database_url
|
||||
|
||||
def resolved_redis_url(self) -> str:
|
||||
if self.redis_url_file:
|
||||
return Path(self.redis_url_file).read_text(encoding='utf-8').strip()
|
||||
return self.redis_url
|
||||
|
||||
def resolved_s3_access_key_id(self) -> str:
|
||||
if self.s3_access_key_id_file:
|
||||
return Path(self.s3_access_key_id_file).read_text(encoding='utf-8').strip()
|
||||
return self.s3_access_key_id
|
||||
|
||||
def resolved_s3_secret_access_key(self) -> str:
|
||||
if self.s3_secret_access_key_file:
|
||||
return Path(self.s3_secret_access_key_file).read_text(encoding='utf-8').strip()
|
||||
return self.s3_secret_access_key
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,37 @@
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from app.core.errors import ApiError
|
||||
from app.repositories.memory import order_key
|
||||
|
||||
|
||||
def encode_cursor(resource: str, created_at: str, row_id: str) -> str:
|
||||
payload = json.dumps({'r': resource, 't': created_at, 'i': row_id}, separators=(',', ':'))
|
||||
return base64.urlsafe_b64encode(payload.encode()).decode()
|
||||
|
||||
|
||||
def decode_cursor(value: str, resource: str) -> tuple[datetime, int]:
|
||||
"""Курсор от другого ресурса или битый -> 400 invalid_request."""
|
||||
try:
|
||||
payload = json.loads(base64.urlsafe_b64decode(value.encode()))
|
||||
if payload['r'] != resource:
|
||||
raise ValueError('cursor belongs to another resource')
|
||||
moment = datetime.fromisoformat(payload['t'].replace('Z', '+00:00'))
|
||||
return moment, uuid.UUID(payload['i']).int
|
||||
except (ValueError, KeyError, TypeError, binascii.Error):
|
||||
raise ApiError(400, 'Invalid cursor')
|
||||
|
||||
|
||||
def paginate(rows: list, cursor_key, limit: int, resource: str):
|
||||
"""Keyset: берём строки строго после (created_at, id) курсора, без offset-скана."""
|
||||
if cursor_key is not None:
|
||||
rows = [r for r in rows if order_key(r.created_dt, r.id) > cursor_key]
|
||||
page = rows[:limit]
|
||||
next_cursor = None
|
||||
if len(rows) > limit: # следующая запись есть на момент чтения
|
||||
last = page[-1]
|
||||
next_cursor = encode_cursor(resource, last.created_at, last.id)
|
||||
return page, next_cursor
|
||||
@@ -0,0 +1,72 @@
|
||||
import os
|
||||
import socket
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
CODES = {400: 'invalid_request', 401: 'unauthorized', 404: 'not_found', 409: 'conflict',
|
||||
413: 'payload_too_large', 415: 'unsupported_media_type', 429: 'rate_limited',
|
||||
503: 'unavailable', 500: 'internal_error'}
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
"""Бизнес-ошибка: HTTP-статус + код контракта + безопасное сообщение."""
|
||||
|
||||
def __init__(self, status_code: int, message: str, code: str | None = None,
|
||||
headers: dict | None = None):
|
||||
self.status_code = status_code
|
||||
self.code = code or CODES.get(status_code, 'internal_error')
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
|
||||
class RequestIdMiddleware(BaseHTTPMiddleware):
|
||||
"""Каждый ответ, включая 204 и ошибки, несёт непустой X-Request-Id."""
|
||||
|
||||
async def dispatch(self, request: Request, call_next):
|
||||
request.state.request_id = str(uuid.uuid4())
|
||||
response = await call_next(request)
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = os.environ.get('INSTANCE_ID') or socket.gethostname()
|
||||
return response
|
||||
|
||||
|
||||
def _body(request: Request, status_code: int, code: str, message: str,
|
||||
headers: dict | None = None) -> JSONResponse:
|
||||
rid = getattr(request.state, 'request_id', str(uuid.uuid4()))
|
||||
return JSONResponse(status_code=status_code, headers=headers,
|
||||
content={'error': {'code': code, 'message': message, 'request_id': rid}})
|
||||
|
||||
|
||||
def register_error_handlers(app: FastAPI) -> None:
|
||||
@app.exception_handler(ApiError)
|
||||
async def on_api_error(request: Request, exc: ApiError):
|
||||
return _body(request, exc.status_code, exc.code, exc.message, exc.headers)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def on_validation(request: Request, exc: RequestValidationError):
|
||||
return _body(request, 400, 'invalid_request', 'Request validation failed')
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def on_http(request: Request, exc: StarletteHTTPException):
|
||||
code = CODES.get(exc.status_code, 'invalid_request')
|
||||
return _body(request, exc.status_code, code, 'Route or method not found')
|
||||
|
||||
async def on_dependency_failure(request: Request, exc: Exception):
|
||||
return _body(request, 503, 'unavailable', 'A required service is unavailable')
|
||||
|
||||
app.add_exception_handler(asyncpg.PostgresError, on_dependency_failure)
|
||||
app.add_exception_handler(RedisError, on_dependency_failure)
|
||||
app.add_exception_handler(OSError, on_dependency_failure)
|
||||
app.add_exception_handler(TimeoutError, on_dependency_failure)
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def on_unexpected(request: Request, exc: Exception):
|
||||
return _body(request, 500, 'internal_error', 'Internal error')
|
||||
@@ -0,0 +1,39 @@
|
||||
import base64
|
||||
import binascii
|
||||
import secrets
|
||||
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import Argon2Error
|
||||
|
||||
_ph = PasswordHasher() # по умолчанию Argon2id
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _ph.hash(password)
|
||||
|
||||
|
||||
def verify_password(digest: str, password: str) -> bool:
|
||||
try:
|
||||
return _ph.verify(digest, password)
|
||||
except (Argon2Error, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def new_token() -> str:
|
||||
return secrets.token_hex(32) # 32 байта = 256 бит энтропии
|
||||
|
||||
|
||||
def parse_basic(header: str) -> tuple[str, str] | None:
|
||||
"""Преобразовать 'Basic base64(username:password)' в пару (username, password) или вернуть None."""
|
||||
parts = header.split(' ', 1)
|
||||
if len(parts) != 2 or parts[0].lower() != 'basic':
|
||||
return None
|
||||
try:
|
||||
raw = base64.b64decode(parts[1].strip(), validate=True)
|
||||
text = raw.decode('utf-8')
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
return None
|
||||
if ':' not in text:
|
||||
return None
|
||||
username, password = text.split(':', 1) # двоеточие в пароле допустимо
|
||||
return username, password
|
||||
@@ -0,0 +1 @@
|
||||
"""Настройка базы данных и миграции схемы."""
|
||||
@@ -0,0 +1,56 @@
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / 'migrations'
|
||||
|
||||
|
||||
async def apply_migrations(pool: asyncpg.Pool) -> None:
|
||||
"""Применить каждую нумерованную SQL-миграцию один раз в транзакции под advisory lock."""
|
||||
async with pool.acquire() as connection:
|
||||
await connection.execute('SELECT pg_advisory_lock($1)', 762019002)
|
||||
try:
|
||||
await connection.execute(
|
||||
'CREATE TABLE IF NOT EXISTS schema_migrations ('
|
||||
' version integer PRIMARY KEY,'
|
||||
' applied_at timestamptz NOT NULL DEFAULT now())'
|
||||
)
|
||||
applied = {
|
||||
row['version']
|
||||
for row in await connection.fetch('SELECT version FROM schema_migrations')
|
||||
}
|
||||
for path in sorted(MIGRATIONS_DIR.glob('[0-9]*_*.sql')):
|
||||
version = int(path.name.split('_', 1)[0])
|
||||
if version in applied:
|
||||
continue
|
||||
async with connection.transaction():
|
||||
await connection.execute(path.read_text(encoding='utf-8'))
|
||||
await connection.execute(
|
||||
'INSERT INTO schema_migrations (version) VALUES ($1)', version
|
||||
)
|
||||
finally:
|
||||
await connection.execute('SELECT pg_advisory_unlock($1)', 762019002)
|
||||
|
||||
async def _main() -> None:
|
||||
import asyncio
|
||||
from app.core.config import settings
|
||||
|
||||
dsn = settings.resolved_database_url()
|
||||
if not dsn:
|
||||
raise RuntimeError('Set DATABASE_URL or DATABASE_URL_FILE before running migrations')
|
||||
pool = await asyncpg.create_pool(
|
||||
dsn,
|
||||
min_size=settings.db_pool_min_size,
|
||||
max_size=settings.db_pool_max_size,
|
||||
timeout=settings.db_pool_acquire_timeout_seconds,
|
||||
command_timeout=settings.db_command_timeout_seconds,
|
||||
)
|
||||
try:
|
||||
await apply_migrations(pool)
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import asyncio
|
||||
asyncio.run(_main())
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import asyncpg
|
||||
from fastapi import FastAPI
|
||||
from redis.asyncio import Redis
|
||||
|
||||
from app.api.routes import attachments, auth, health, me, users, chats, messages
|
||||
from app.core.config import settings
|
||||
from app.core.errors import RequestIdMiddleware, register_error_handlers
|
||||
from app.repositories.object_store import S3ObjectStore
|
||||
from app.repositories.postgres import PgStorage
|
||||
from app.repositories.redis_sessions import RedisSessionStore
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
dsn = settings.resolved_database_url()
|
||||
if not dsn:
|
||||
raise RuntimeError('Set DATABASE_URL or DATABASE_URL_FILE before starting the API')
|
||||
pool = await asyncpg.create_pool(
|
||||
dsn,
|
||||
min_size=settings.db_pool_min_size,
|
||||
max_size=settings.db_pool_max_size,
|
||||
timeout=settings.db_pool_acquire_timeout_seconds,
|
||||
command_timeout=settings.db_command_timeout_seconds,
|
||||
)
|
||||
redis_client = Redis.from_url(
|
||||
settings.resolved_redis_url(),
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=settings.redis_timeout_seconds,
|
||||
socket_timeout=settings.redis_timeout_seconds,
|
||||
)
|
||||
object_store = S3ObjectStore()
|
||||
try:
|
||||
await object_store.ensure_bucket()
|
||||
app.state.object_store = object_store
|
||||
app.state.storage = PgStorage(pool, RedisSessionStore(redis_client), object_store)
|
||||
yield
|
||||
finally:
|
||||
await redis_client.aclose()
|
||||
await pool.close()
|
||||
|
||||
app = FastAPI(title='MTUSI Messenger lab4', docs_url=None, redoc_url=None, lifespan=lifespan)
|
||||
app.add_middleware(RequestIdMiddleware)
|
||||
register_error_handlers(app)
|
||||
app.include_router(health.router)
|
||||
app.include_router(users.router)
|
||||
app.include_router(me.router)
|
||||
app.include_router(auth.router)
|
||||
app.include_router(chats.router)
|
||||
app.include_router(messages.router)
|
||||
app.include_router(attachments.router)
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,148 @@
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from app.core.errors import ApiError
|
||||
|
||||
|
||||
def now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def iso_z(moment: datetime) -> str:
|
||||
"""RFC 3339 UTC с Z и микросекундами: порядок (created_at, id) не теряет точность."""
|
||||
return moment.strftime('%Y-%m-%dT%H:%M:%S.%fZ')
|
||||
|
||||
|
||||
def order_key(created_dt: datetime, row_id: str):
|
||||
"""Контракт: порядок по (created_at, id), id сравнивается как значение UUID."""
|
||||
return (created_dt, uuid.UUID(row_id).int)
|
||||
|
||||
|
||||
@dataclass
|
||||
class UserRow:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
password_hash: str
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ChatRow:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: list
|
||||
created_dt: datetime
|
||||
created_at: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class MessageRow:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_dt: datetime
|
||||
created_at: str
|
||||
attachment_ids: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttachmentRow:
|
||||
id: str
|
||||
chat_id: str
|
||||
owner_id: str
|
||||
object_key: str
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str
|
||||
created_at: str
|
||||
updated_at: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class SessionRow:
|
||||
token: str
|
||||
user_id: str
|
||||
expires_at: datetime
|
||||
|
||||
|
||||
class MemoryStorage:
|
||||
"""Хранилище в памяти из скелета lab1. В lab2 его заменяет PgStorage."""
|
||||
|
||||
def __init__(self):
|
||||
self._lock = threading.RLock()
|
||||
self._users: dict[str, UserRow] = {}
|
||||
self._by_username: dict[str, str] = {}
|
||||
self._chats: dict[str, ChatRow] = {}
|
||||
self._messages: dict[str, list[MessageRow]] = {}
|
||||
self._sessions: dict[str, SessionRow] = {}
|
||||
|
||||
def ready(self) -> bool:
|
||||
return True
|
||||
|
||||
def create_user(self, username: str, display_name: str, password_hash: str) -> UserRow:
|
||||
with self._lock:
|
||||
if username in self._by_username:
|
||||
raise ApiError(409, 'Username is already taken')
|
||||
row = UserRow(str(uuid.uuid4()), username, display_name, password_hash,
|
||||
iso_z(now_utc()))
|
||||
self._users[row.id] = row
|
||||
self._by_username[username] = row.id
|
||||
return row
|
||||
|
||||
def user_by_username(self, username: str) -> UserRow | None:
|
||||
with self._lock:
|
||||
uid = self._by_username.get(username)
|
||||
return self._users.get(uid) if uid else None
|
||||
|
||||
def user(self, user_id: str) -> UserRow | None:
|
||||
with self._lock:
|
||||
return self._users.get(user_id)
|
||||
|
||||
def create_session(self, user_id: str, ttl_seconds: int) -> SessionRow:
|
||||
from app.core.security import new_token
|
||||
with self._lock:
|
||||
row = SessionRow(new_token(), user_id, now_utc() + timedelta(seconds=ttl_seconds))
|
||||
self._sessions[row.token] = row
|
||||
return row
|
||||
|
||||
def session(self, token: str) -> SessionRow | None:
|
||||
with self._lock:
|
||||
row = self._sessions.get(token)
|
||||
if row is None:
|
||||
return None
|
||||
if now_utc() >= row.expires_at:
|
||||
del self._sessions[token]
|
||||
return None
|
||||
return row
|
||||
|
||||
def revoke(self, token: str) -> bool:
|
||||
with self._lock:
|
||||
return self._sessions.pop(token, None) is not None
|
||||
|
||||
def create_chat(self, title: str, member_ids: list, caller_id: str) -> ChatRow:
|
||||
with self._lock:
|
||||
if caller_id not in member_ids:
|
||||
raise ApiError(400, 'Creator must be included in member_ids')
|
||||
for mid in member_ids:
|
||||
if mid not in self._users:
|
||||
raise ApiError(400, 'Unknown member id')
|
||||
moment = now_utc()
|
||||
row = ChatRow(str(uuid.uuid4()), title, list(member_ids), moment, iso_z(moment))
|
||||
self._chats[row.id] = row
|
||||
self._messages[row.id] = []
|
||||
return row
|
||||
|
||||
def chat(self, chat_id: str) -> ChatRow | None:
|
||||
with self._lock:
|
||||
return self._chats.get(chat_id)
|
||||
|
||||
def chats_of(self, user_id: str) -> list[ChatRow]:
|
||||
with self._lock:
|
||||
rows = [c for c in self._chats.values() if user_id in c.member_ids]
|
||||
return sorted(rows, key=lambda c: order_key(c.created_dt, c.id))
|
||||
@@ -0,0 +1,109 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import boto3
|
||||
from botocore.config import Config
|
||||
from botocore.exceptions import BotoCoreError, ClientError
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.errors import ApiError
|
||||
|
||||
|
||||
class S3ObjectStore:
|
||||
"""Хранит байты в приватном S3 bucket, не в процессе API."""
|
||||
|
||||
def __init__(self):
|
||||
self.bucket = settings.s3_bucket
|
||||
self.client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.s3_endpoint_url,
|
||||
region_name=settings.s3_region,
|
||||
aws_access_key_id=settings.resolved_s3_access_key_id(),
|
||||
aws_secret_access_key=settings.resolved_s3_secret_access_key(),
|
||||
config=Config(
|
||||
connect_timeout=settings.s3_connect_timeout_seconds,
|
||||
read_timeout=settings.s3_read_timeout_seconds,
|
||||
max_pool_connections=16,
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
s3={"addressing_style": "path"},
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _unavailable():
|
||||
return ApiError(503, "Object storage is unavailable")
|
||||
|
||||
def _ensure_bucket(self):
|
||||
for attempt in range(30):
|
||||
try:
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
return
|
||||
except ClientError as exc:
|
||||
code = str(exc.response.get("Error", {}).get("Code", ""))
|
||||
if code not in {"404", "NoSuchBucket", "NotFound"}:
|
||||
if code in {"403", "AccessDenied"} or attempt == 29:
|
||||
raise self._unavailable() from None
|
||||
else:
|
||||
try:
|
||||
self.client.create_bucket(Bucket=self.bucket)
|
||||
except ClientError as create_exc:
|
||||
create_code = str(create_exc.response.get("Error", {}).get("Code", ""))
|
||||
if create_code not in {"BucketAlreadyOwnedByYou", "BucketAlreadyExists"}:
|
||||
if attempt == 29:
|
||||
raise self._unavailable() from None
|
||||
else:
|
||||
return
|
||||
except (BotoCoreError, OSError, TimeoutError):
|
||||
if attempt == 29:
|
||||
raise self._unavailable() from None
|
||||
time.sleep(0.5)
|
||||
raise self._unavailable()
|
||||
|
||||
async def ensure_bucket(self):
|
||||
await asyncio.to_thread(self._ensure_bucket)
|
||||
|
||||
def _head_bucket(self):
|
||||
self.client.head_bucket(Bucket=self.bucket)
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
await asyncio.to_thread(self._head_bucket)
|
||||
return True
|
||||
except (ClientError, BotoCoreError, OSError, TimeoutError):
|
||||
return False
|
||||
|
||||
def _put(self, key: str, data: bytes, content_type: str):
|
||||
self.client.put_object(Bucket=self.bucket, Key=key, Body=data, ContentType=content_type)
|
||||
|
||||
async def put(self, key: str, data: bytes, content_type: str):
|
||||
try:
|
||||
await asyncio.to_thread(self._put, key, data, content_type)
|
||||
except (ClientError, BotoCoreError, OSError, TimeoutError):
|
||||
raise self._unavailable() from None
|
||||
|
||||
def _get(self, key: str) -> bytes:
|
||||
result = self.client.get_object(Bucket=self.bucket, Key=key)
|
||||
try:
|
||||
return result["Body"].read(settings.max_upload_bytes + 1)
|
||||
finally:
|
||||
result["Body"].close()
|
||||
|
||||
async def get(self, key: str) -> bytes:
|
||||
try:
|
||||
data = await asyncio.to_thread(self._get, key)
|
||||
if len(data) > settings.max_upload_bytes:
|
||||
raise self._unavailable()
|
||||
return data
|
||||
except ApiError:
|
||||
raise
|
||||
except (ClientError, BotoCoreError, OSError, TimeoutError):
|
||||
raise self._unavailable() from None
|
||||
|
||||
def _delete(self, key: str):
|
||||
self.client.delete_object(Bucket=self.bucket, Key=key)
|
||||
|
||||
async def delete(self, key: str):
|
||||
try:
|
||||
await asyncio.to_thread(self._delete, key)
|
||||
except (ClientError, BotoCoreError, OSError, TimeoutError):
|
||||
raise self._unavailable() from None
|
||||
@@ -0,0 +1,278 @@
|
||||
import uuid
|
||||
|
||||
import asyncpg
|
||||
|
||||
from app.core.errors import ApiError
|
||||
from app.repositories.memory import (
|
||||
AttachmentRow, ChatRow, MessageRow, UserRow, iso_z,
|
||||
)
|
||||
|
||||
|
||||
def _user(row: asyncpg.Record | None) -> UserRow | None:
|
||||
if row is None:
|
||||
return None
|
||||
return UserRow(
|
||||
id=str(row['id']), username=row['username'], display_name=row['display_name'],
|
||||
password_hash=row['password_hash'], created_at=iso_z(row['created_at']),
|
||||
)
|
||||
|
||||
|
||||
def _chat(row: asyncpg.Record) -> ChatRow:
|
||||
moment = row['created_at']
|
||||
return ChatRow(
|
||||
id=str(row['id']), title=row['title'], member_ids=[str(x) for x in row['member_ids']],
|
||||
created_dt=moment, created_at=iso_z(moment),
|
||||
)
|
||||
|
||||
|
||||
def _message(row: asyncpg.Record) -> MessageRow:
|
||||
moment = row['created_at']
|
||||
attachment_ids = row['attachment_ids'] if 'attachment_ids' in row else []
|
||||
return MessageRow(
|
||||
id=str(row['id']), chat_id=str(row['chat_id']), sender_id=str(row['sender_id']),
|
||||
text=row['text'], created_dt=moment, created_at=iso_z(moment),
|
||||
attachment_ids=[str(value) for value in (attachment_ids or [])],
|
||||
)
|
||||
|
||||
|
||||
def _attachment(row: asyncpg.Record) -> AttachmentRow:
|
||||
return AttachmentRow(
|
||||
id=str(row['id']), chat_id=str(row['chat_id']), owner_id=str(row['owner_id']),
|
||||
object_key=row['object_key'], filename=row['filename'],
|
||||
content_type=row['content_type'], size_bytes=row['size_bytes'],
|
||||
sha256=row['sha256'], status=row['status'],
|
||||
created_at=iso_z(row['created_at']), updated_at=iso_z(row['updated_at']),
|
||||
)
|
||||
|
||||
|
||||
_MESSAGE_SELECT = (
|
||||
'SELECT m.*, COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) '
|
||||
"FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::uuid[]) AS attachment_ids "
|
||||
'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id '
|
||||
)
|
||||
|
||||
|
||||
class PgStorage:
|
||||
"""PostgreSQL — источник истины для метаданных; S3 хранит только байты."""
|
||||
|
||||
def __init__(self, pool: asyncpg.Pool, session_store, object_store):
|
||||
self.pool = pool
|
||||
self.session_store = session_store
|
||||
self.object_store = object_store
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
async with self.pool.acquire() as connection:
|
||||
database_ready = await connection.fetchval('SELECT 1') == 1
|
||||
return (
|
||||
database_ready
|
||||
and await self.session_store.ready()
|
||||
and await self.object_store.ready()
|
||||
)
|
||||
except (asyncpg.PostgresError, OSError, TimeoutError):
|
||||
return False
|
||||
|
||||
async def create_user(self, username: str, display_name: str, password_hash: str) -> UserRow:
|
||||
try:
|
||||
async with self.pool.acquire() as connection:
|
||||
row = await connection.fetchrow(
|
||||
'INSERT INTO users (id, username, display_name, password_hash) '
|
||||
'VALUES ($1, $2, $3, $4) RETURNING *',
|
||||
uuid.uuid4(), username, display_name, password_hash,
|
||||
)
|
||||
except asyncpg.UniqueViolationError as exc:
|
||||
if exc.constraint_name == 'users_username_key':
|
||||
raise ApiError(409, 'Username is already taken') from None
|
||||
raise
|
||||
return _user(row)
|
||||
|
||||
async def user_by_username(self, username: str) -> UserRow | None:
|
||||
async with self.pool.acquire() as connection:
|
||||
return _user(await connection.fetchrow(
|
||||
'SELECT * FROM users WHERE username = $1', username
|
||||
))
|
||||
|
||||
async def user(self, user_id: str) -> UserRow | None:
|
||||
async with self.pool.acquire() as connection:
|
||||
return _user(await connection.fetchrow(
|
||||
'SELECT * FROM users WHERE id = $1', uuid.UUID(user_id)
|
||||
))
|
||||
|
||||
async def create_session(self, user_id: str, ttl_seconds: int):
|
||||
return await self.session_store.create_session(user_id, ttl_seconds)
|
||||
|
||||
async def session(self, token: str):
|
||||
return await self.session_store.session(token)
|
||||
|
||||
async def revoke(self, token: str) -> bool:
|
||||
return await self.session_store.revoke(token)
|
||||
|
||||
async def create_chat(self, title: str, member_ids: list[str], caller_id: str) -> ChatRow:
|
||||
if caller_id not in member_ids:
|
||||
raise ApiError(400, 'Creator must be included in member_ids')
|
||||
chat_id = uuid.uuid4()
|
||||
try:
|
||||
async with self.pool.acquire() as connection:
|
||||
async with connection.transaction():
|
||||
created_at = await connection.fetchval(
|
||||
'INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at',
|
||||
chat_id, title,
|
||||
)
|
||||
await connection.executemany(
|
||||
'INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)',
|
||||
[(chat_id, uuid.UUID(member_id)) for member_id in member_ids],
|
||||
)
|
||||
except asyncpg.ForeignKeyViolationError:
|
||||
raise ApiError(400, 'Unknown member id') from None
|
||||
return ChatRow(
|
||||
id=str(chat_id), title=title, member_ids=list(member_ids),
|
||||
created_dt=created_at, created_at=iso_z(created_at),
|
||||
)
|
||||
|
||||
async def chat(self, chat_id: str) -> ChatRow | None:
|
||||
async with self.pool.acquire() as connection:
|
||||
row = await connection.fetchrow(
|
||||
'SELECT c.id, c.title, c.created_at, '
|
||||
'array_agg(cm.user_id ORDER BY cm.user_id) AS member_ids '
|
||||
'FROM chats c JOIN chat_members cm ON cm.chat_id = c.id '
|
||||
'WHERE c.id = $1 GROUP BY c.id', uuid.UUID(chat_id),
|
||||
)
|
||||
return _chat(row) if row else None
|
||||
|
||||
async def chats_of(self, user_id: str) -> list[ChatRow]:
|
||||
async with self.pool.acquire() as connection:
|
||||
rows = await connection.fetch(
|
||||
'SELECT c.id, c.title, c.created_at, '
|
||||
'array_agg(cm_all.user_id ORDER BY cm_all.user_id) AS member_ids '
|
||||
'FROM chats c '
|
||||
'JOIN chat_members mine ON mine.chat_id = c.id '
|
||||
'JOIN chat_members cm_all ON cm_all.chat_id = c.id '
|
||||
'WHERE mine.user_id = $1 '
|
||||
'GROUP BY c.id ORDER BY c.created_at, c.id', uuid.UUID(user_id),
|
||||
)
|
||||
return [_chat(row) for row in rows]
|
||||
|
||||
async def create_attachment(
|
||||
self, attachment_id: str, chat_id: str, owner_id: str, object_key: str,
|
||||
filename: str, content_type: str, size_bytes: int, sha256: str,
|
||||
) -> AttachmentRow:
|
||||
async with self.pool.acquire() as connection:
|
||||
row = await connection.fetchrow(
|
||||
'INSERT INTO attachments '
|
||||
'(id, chat_id, owner_id, object_key, filename, content_type, size_bytes, sha256) '
|
||||
'VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *',
|
||||
uuid.UUID(attachment_id), uuid.UUID(chat_id), uuid.UUID(owner_id),
|
||||
object_key, filename, content_type, size_bytes, sha256,
|
||||
)
|
||||
return _attachment(row)
|
||||
|
||||
async def attachment_for_member(self, attachment_id: str, user_id: str) -> AttachmentRow | None:
|
||||
try:
|
||||
attachment_uuid = uuid.UUID(attachment_id)
|
||||
except ValueError:
|
||||
raise ApiError(400, 'attachment_id must be a UUID') from None
|
||||
async with self.pool.acquire() as connection:
|
||||
row = await connection.fetchrow(
|
||||
'SELECT a.* FROM attachments a '
|
||||
'JOIN chat_members cm ON cm.chat_id = a.chat_id '
|
||||
'WHERE a.id = $1 AND cm.user_id = $2',
|
||||
attachment_uuid, uuid.UUID(user_id),
|
||||
)
|
||||
return _attachment(row) if row else None
|
||||
|
||||
async def add_idempotent_message(
|
||||
self, chat_id: str, sender_id: str, text: str, attachment_ids: list[str],
|
||||
idempotency_key: str, request_hash: bytes,
|
||||
) -> MessageRow:
|
||||
"""Создать сообщение, его вложения и ключ повтора одной транзакцией."""
|
||||
chat_uuid = uuid.UUID(chat_id)
|
||||
sender_uuid = uuid.UUID(sender_id)
|
||||
attachment_uuids = [uuid.UUID(value) for value in attachment_ids]
|
||||
lock_scope = f'{sender_uuid}:POST:/api/v1/chats/{chat_uuid}/messages:{idempotency_key}'
|
||||
async with self.pool.acquire() as connection:
|
||||
async with connection.transaction():
|
||||
await connection.fetchval(
|
||||
'SELECT pg_advisory_xact_lock(hashtextextended($1, 0))', lock_scope,
|
||||
)
|
||||
existing = await connection.fetchrow(
|
||||
'SELECT k.request_hash, m.id FROM message_idempotency k '
|
||||
'JOIN messages m ON m.id = k.message_id '
|
||||
'WHERE k.user_id = $1 AND k.chat_id = $2 AND k.idempotency_key = $3 '
|
||||
"AND k.created_at > now() - interval '24 hours'",
|
||||
sender_uuid, chat_uuid, idempotency_key,
|
||||
)
|
||||
if existing is not None:
|
||||
if bytes(existing['request_hash']) != request_hash:
|
||||
raise ApiError(409, 'Idempotency-Key was already used with another request')
|
||||
row = await connection.fetchrow(
|
||||
_MESSAGE_SELECT + 'WHERE m.id = $1 GROUP BY m.id', existing['id'],
|
||||
)
|
||||
return _message(row)
|
||||
|
||||
if attachment_uuids:
|
||||
available = await connection.fetch(
|
||||
"SELECT id FROM attachments WHERE chat_id = $1 AND id = ANY($2::uuid[]) "
|
||||
"AND status = 'ready' FOR SHARE",
|
||||
chat_uuid, attachment_uuids,
|
||||
)
|
||||
if len(available) != len(attachment_uuids):
|
||||
raise ApiError(404, 'Attachment is not available in this chat')
|
||||
|
||||
await connection.execute(
|
||||
'DELETE FROM message_idempotency '
|
||||
'WHERE user_id = $1 AND chat_id = $2 AND idempotency_key = $3',
|
||||
sender_uuid, chat_uuid, idempotency_key,
|
||||
)
|
||||
await connection.execute(
|
||||
"DELETE FROM message_idempotency WHERE ctid IN ("
|
||||
'SELECT ctid FROM message_idempotency '
|
||||
"WHERE created_at <= now() - interval '24 hours' "
|
||||
'ORDER BY created_at LIMIT 100)'
|
||||
)
|
||||
message_id = uuid.uuid4()
|
||||
await connection.execute(
|
||||
'INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4)',
|
||||
message_id, chat_uuid, sender_uuid, text,
|
||||
)
|
||||
if attachment_uuids:
|
||||
await connection.executemany(
|
||||
'INSERT INTO message_attachments '
|
||||
'(message_id, attachment_id, chat_id, position) VALUES ($1, $2, $3, $4)',
|
||||
[(message_id, attachment_id, chat_uuid, index)
|
||||
for index, attachment_id in enumerate(attachment_uuids)],
|
||||
)
|
||||
row = await connection.fetchrow(
|
||||
_MESSAGE_SELECT + 'WHERE m.id = $1 GROUP BY m.id', message_id,
|
||||
)
|
||||
await connection.execute(
|
||||
'INSERT INTO message_idempotency '
|
||||
'(user_id, chat_id, idempotency_key, request_hash, message_id) '
|
||||
'VALUES ($1, $2, $3, $4, $5)',
|
||||
sender_uuid, chat_uuid, idempotency_key, request_hash, message_id,
|
||||
)
|
||||
return _message(row)
|
||||
|
||||
async def messages_page(self, chat_id: str, cursor_key, limit: int):
|
||||
"""Прочитать ограниченную keyset-страницу вместе с ID прикреплённых файлов."""
|
||||
chat_uuid = uuid.UUID(chat_id)
|
||||
base = (
|
||||
'SELECT m.*, COALESCE(array_agg(ma.attachment_id ORDER BY ma.position) '
|
||||
"FILTER (WHERE ma.attachment_id IS NOT NULL), ARRAY[]::uuid[]) AS attachment_ids "
|
||||
'FROM messages m LEFT JOIN message_attachments ma ON ma.message_id = m.id '
|
||||
'WHERE m.chat_id = $1 '
|
||||
)
|
||||
async with self.pool.acquire() as connection:
|
||||
if cursor_key is None:
|
||||
rows = await connection.fetch(
|
||||
base + 'GROUP BY m.id ORDER BY m.created_at, m.id LIMIT $2',
|
||||
chat_uuid, limit + 1,
|
||||
)
|
||||
else:
|
||||
created_at, id_value = cursor_key
|
||||
rows = await connection.fetch(
|
||||
base + 'AND (m.created_at, m.id) > ($2, $3) '
|
||||
'GROUP BY m.id ORDER BY m.created_at, m.id LIMIT $4',
|
||||
chat_uuid, created_at, uuid.UUID(int=id_value), limit + 1,
|
||||
)
|
||||
has_more = len(rows) > limit
|
||||
return [_message(row) for row in rows[:limit]], has_more
|
||||
@@ -0,0 +1,66 @@
|
||||
import hashlib
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from redis.asyncio import Redis
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
from app.core.errors import ApiError
|
||||
from app.core.security import new_token
|
||||
from app.repositories.memory import SessionRow, now_utc
|
||||
|
||||
|
||||
class RedisSessionStore:
|
||||
"""Redis — источник истины для сессий; PXAT сохраняет исходный срок истечения после перезапуска."""
|
||||
|
||||
def __init__(self, client: Redis):
|
||||
self.client = client
|
||||
|
||||
@staticmethod
|
||||
def _key(token: str) -> str:
|
||||
digest = hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
return 'session:' + digest
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
return bool(await self.client.ping())
|
||||
except (RedisError, OSError, TimeoutError):
|
||||
return False
|
||||
|
||||
async def create_session(self, user_id: str, ttl_seconds: int) -> SessionRow:
|
||||
token = new_token()
|
||||
current = now_utc()
|
||||
expires_at = current.replace(microsecond=(current.microsecond // 1000) * 1000) + timedelta(seconds=ttl_seconds)
|
||||
expires_ms = int(expires_at.timestamp() * 1000)
|
||||
try:
|
||||
await self.client.set(
|
||||
self._key(token), f'{user_id}|{expires_ms}', pxat=expires_ms
|
||||
)
|
||||
except RedisError:
|
||||
raise ApiError(503, 'Session storage is unavailable') from None
|
||||
return SessionRow(token=token, user_id=user_id, expires_at=expires_at)
|
||||
|
||||
async def session(self, token: str) -> SessionRow | None:
|
||||
try:
|
||||
value = await self.client.get(self._key(token))
|
||||
except RedisError:
|
||||
raise ApiError(503, 'Session storage is unavailable') from None
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
user_id, expires_ms_text = value.split('|', 1)
|
||||
expires_ms = int(expires_ms_text)
|
||||
if expires_ms <= int(time.time() * 1000):
|
||||
await self.client.delete(self._key(token))
|
||||
return None
|
||||
expires_at = datetime.fromtimestamp(expires_ms / 1000, timezone.utc)
|
||||
return SessionRow(token=token, user_id=user_id, expires_at=expires_at)
|
||||
except (ValueError, OverflowError):
|
||||
await self.client.delete(self._key(token))
|
||||
return None
|
||||
|
||||
async def revoke(self, token: str) -> bool:
|
||||
try:
|
||||
return bool(await self.client.delete(self._key(token)))
|
||||
except RedisError:
|
||||
raise ApiError(503, 'Session storage is unavailable') from None
|
||||
@@ -0,0 +1,85 @@
|
||||
import uuid as _uuid
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
|
||||
STRICT = ConfigDict(extra='forbid')
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = STRICT
|
||||
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('display_name', 'password')
|
||||
@classmethod
|
||||
def _not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError('string must not consist of whitespace only')
|
||||
return value
|
||||
|
||||
|
||||
class User(BaseModel):
|
||||
model_config = STRICT
|
||||
id: _uuid.UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
|
||||
class Session(BaseModel):
|
||||
model_config = STRICT
|
||||
token: str
|
||||
token_type: Literal['Bearer']
|
||||
expires_at: str
|
||||
user: User
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = STRICT
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[_uuid.UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator('title')
|
||||
@classmethod
|
||||
def _title_not_blank(cls, value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError('string must not consist of whitespace only')
|
||||
return value
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def _unique_members(cls, value):
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('duplicate member ids')
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = STRICT
|
||||
text: str = Field(min_length=0, max_length=4000)
|
||||
attachment_ids: list[_uuid.UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
@field_validator('attachment_ids')
|
||||
@classmethod
|
||||
def _unique_attachments(cls, value):
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('duplicate attachment ids')
|
||||
return value
|
||||
|
||||
|
||||
class Attachment(BaseModel):
|
||||
model_config = STRICT
|
||||
id: _uuid.UUID
|
||||
chat_id: _uuid.UUID
|
||||
owner_id: _uuid.UUID
|
||||
filename: str = Field(min_length=1, max_length=255)
|
||||
content_type: Literal['image/png', 'image/jpeg']
|
||||
size_bytes: int = Field(ge=1, le=10_485_760)
|
||||
sha256: str = Field(pattern=r'^[a-f0-9]{64}$')
|
||||
status: Literal['ready']
|
||||
variants: list = Field(default_factory=list)
|
||||
error_code: str | None = None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
name: mtusi-lab4
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER:?Set POSTGRES_USER in .env}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
|
||||
POSTGRES_DB: ${POSTGRES_DB:?Set POSTGRES_DB in .env}
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
image: redis:7.4.2-alpine
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--save", ""]
|
||||
volumes:
|
||||
- redis_data:/data
|
||||
healthcheck:
|
||||
test: ["CMD", "redis-cli", "ping"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
s3:
|
||||
image: ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d
|
||||
command: ["/data"]
|
||||
environment:
|
||||
RUSTFS_VOLUMES: /data
|
||||
RUSTFS_ADDRESS: 0.0.0.0:9000
|
||||
RUSTFS_CONSOLE_ENABLE: "false"
|
||||
RUSTFS_ACCESS_KEY: ${S3_ACCESS_KEY_ID:?Set S3_ACCESS_KEY_ID in .env}
|
||||
RUSTFS_SECRET_KEY: ${S3_SECRET_ACCESS_KEY:?Set S3_SECRET_ACCESS_KEY in .env}
|
||||
RUSTFS_OBS_LOGGER_LEVEL: error
|
||||
volumes:
|
||||
- s3_data:/data
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
command: ["python", "-m", "app.db.migrate"]
|
||||
environment:
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
|
||||
DB_POOL_MIN_SIZE: 1
|
||||
DB_POOL_MAX_SIZE: 2
|
||||
DB_POOL_ACQUIRE_TIMEOUT_SECONDS: 3
|
||||
DB_COMMAND_TIMEOUT_SECONDS: 10
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
environment:
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: 8080
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
|
||||
DB_POOL_MIN_SIZE: ${DB_POOL_MIN_SIZE:-1}
|
||||
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8}
|
||||
DB_POOL_ACQUIRE_TIMEOUT_SECONDS: ${DB_POOL_ACQUIRE_TIMEOUT_SECONDS:-3}
|
||||
DB_COMMAND_TIMEOUT_SECONDS: ${DB_COMMAND_TIMEOUT_SECONDS:-5}
|
||||
REDIS_URL: redis://redis:6379/0
|
||||
REDIS_TIMEOUT_SECONDS: ${REDIS_TIMEOUT_SECONDS:-2}
|
||||
S3_ENDPOINT_URL: http://s3:9000
|
||||
S3_REGION: ${S3_REGION:-us-east-1}
|
||||
S3_BUCKET: ${S3_BUCKET:-messenger-attachments}
|
||||
S3_ACCESS_KEY_ID: ${S3_ACCESS_KEY_ID}
|
||||
S3_SECRET_ACCESS_KEY: ${S3_SECRET_ACCESS_KEY}
|
||||
S3_CONNECT_TIMEOUT_SECONDS: ${S3_CONNECT_TIMEOUT_SECONDS:-3}
|
||||
S3_READ_TIMEOUT_SECONDS: ${S3_READ_TIMEOUT_SECONDS:-10}
|
||||
MAX_UPLOAD_BYTES: ${MAX_UPLOAD_BYTES:-10485760}
|
||||
MAX_IMAGE_PIXELS: ${MAX_IMAGE_PIXELS:-40000000}
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
redis:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_started
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
stop_grace_period: 20s
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: nginx:1.27.4-alpine
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./nginx/default.conf:/etc/nginx/conf.d/default.conf:ro
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/ready"]
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
s3_data:
|
||||
@@ -0,0 +1 @@
|
||||
PASS: docker compose config --quiet
|
||||
@@ -0,0 +1,7 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 4 minutes ago Up 3 minutes (healthy) 8080/tcp
|
||||
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 4 minutes ago Up 4 minutes (healthy) 8080/tcp
|
||||
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 4 minutes ago Up 4 minutes (healthy) 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 4 minutes ago Up 4 minutes (healthy) 5432/tcp
|
||||
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 4 minutes ago Up 4 minutes (healthy) 6379/tcp
|
||||
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 4 minutes ago Up 4 minutes 9000-9001/tcp
|
||||
@@ -0,0 +1 @@
|
||||
PASS: compose.yaml parses
|
||||
@@ -0,0 +1 @@
|
||||
PASS: final git diff --check
|
||||
@@ -0,0 +1,8 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 26 seconds ago Up 19 seconds (healthy) 8080/tcp
|
||||
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 26 seconds ago Up 19 seconds (healthy) 8080/tcp
|
||||
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 26 seconds ago Up 13 seconds (healthy) 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 26 seconds ago Exited (0) 19 seconds ago
|
||||
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 26 seconds ago Up 25 seconds (healthy) 5432/tcp
|
||||
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 26 seconds ago Up 25 seconds (healthy) 6379/tcp
|
||||
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 26 seconds ago Up 25 seconds 9000-9001/tcp
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 49 seconds ago Up 42 seconds (healthy) 8080/tcp
|
||||
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 49 seconds ago Up 5 seconds (healthy) 8080/tcp
|
||||
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 49 seconds ago Up 36 seconds (healthy) 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 49 seconds ago Exited (0) 5 seconds ago
|
||||
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 49 seconds ago Up 48 seconds (healthy) 5432/tcp
|
||||
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 49 seconds ago Up 48 seconds (healthy) 6379/tcp
|
||||
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 49 seconds ago Up 48 seconds 9000-9001/tcp
|
||||
@@ -0,0 +1,8 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 42 seconds ago Up 35 seconds (healthy) 8080/tcp
|
||||
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 42 seconds ago Exited (0) 14 seconds ago
|
||||
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 42 seconds ago Up 29 seconds (healthy) 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab4-migrate-1 mtusi-lab4-migrate "python -m app.db.mi…" migrate 42 seconds ago Exited (0) 35 seconds ago
|
||||
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 42 seconds ago Up 41 seconds (healthy) 5432/tcp
|
||||
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 42 seconds ago Up 41 seconds (healthy) 6379/tcp
|
||||
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 42 seconds ago Up 41 seconds 9000-9001/tcp
|
||||
@@ -0,0 +1 @@
|
||||
PASS: restart/failover hooks syntax
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab4 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,6 @@
|
||||
python3 scripts/validate_openapi.py
|
||||
Traceback (most recent call last):
|
||||
File "/home/lenovo/lab4/mtusi2026lab4-fork/scripts/validate_openapi.py", line 5, in <module>
|
||||
from openapi_spec_validator import validate
|
||||
ModuleNotFoundError: No module named 'openapi_spec_validator'
|
||||
make: *** [Makefile:16: validate] Error 1
|
||||
@@ -0,0 +1,2 @@
|
||||
.venv/bin/python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,15 @@
|
||||
Container mtusi-lab4-postgres-1 Running
|
||||
Container mtusi-lab4-postgres-1 Waiting
|
||||
Container mtusi-lab4-postgres-1 Healthy
|
||||
Container mtusi-lab4-migrate-run-3a566e7f2cbd Creating
|
||||
Container mtusi-lab4-migrate-run-3a566e7f2cbd Created
|
||||
Container mtusi-lab4-postgres-1 Running
|
||||
Container mtusi-lab4-postgres-1 Waiting
|
||||
Container mtusi-lab4-postgres-1 Healthy
|
||||
Container mtusi-lab4-migrate-run-5e9470435d61 Creating
|
||||
Container mtusi-lab4-migrate-run-5e9470435d61 Created
|
||||
Migration versions:
|
||||
1
|
||||
2
|
||||
3
|
||||
4
|
||||
@@ -0,0 +1,7 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 12 seconds ago Up 6 seconds (healthy) 8080/tcp
|
||||
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 12 seconds ago Up 5 seconds (healthy) 8080/tcp
|
||||
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 12 seconds ago Up Less than a second (health: starting) 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 12 seconds ago Up 11 seconds (healthy) 5432/tcp
|
||||
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 12 seconds ago Up 11 seconds (healthy) 6379/tcp
|
||||
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 12 seconds ago Up 12 seconds 9000-9001/tcp
|
||||
@@ -0,0 +1,7 @@
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
mtusi-lab4-api-1 mtusi-lab4-api "uvicorn app.main:ap…" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
|
||||
mtusi-lab4-api-2 mtusi-lab4-api "uvicorn app.main:ap…" api 2 minutes ago Up About a minute (healthy) 8080/tcp
|
||||
mtusi-lab4-lb-1 nginx:1.27.4-alpine "/docker-entrypoint.…" lb 2 minutes ago Up 2 minutes (healthy) 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab4-postgres-1 postgres:16.4-alpine "docker-entrypoint.s…" postgres 2 minutes ago Up 2 minutes (healthy) 5432/tcp
|
||||
mtusi-lab4-redis-1 redis:7.4.2-alpine "docker-entrypoint.s…" redis 2 minutes ago Up 2 minutes (healthy) 6379/tcp
|
||||
mtusi-lab4-s3-1 ghcr.io/rustfs/rustfs:1.0.4-glibc@sha256:a9b705b9256be756802a60f3498abdc543918b5b5a4437a57a541e89e733c74d "/entrypoint.sh /data" s3 2 minutes ago Up 2 minutes 9000-9001/tcp
|
||||
@@ -0,0 +1 @@
|
||||
PASS: compileall app student-tests scripts
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
Container mtusi-lab4-postgres-1 Running
|
||||
Container mtusi-lab4-redis-1 Running
|
||||
Container mtusi-lab4-s3-1 Running
|
||||
Container mtusi-lab4-api-2 Running
|
||||
Container mtusi-lab4-lb-1 Running
|
||||
Container mtusi-lab4-postgres-1 Waiting
|
||||
Container mtusi-lab4-postgres-1 Healthy
|
||||
Container mtusi-lab4-migrate-1 Starting
|
||||
Container mtusi-lab4-migrate-1 Started
|
||||
Container mtusi-lab4-migrate-1 Waiting
|
||||
Container mtusi-lab4-postgres-1 Waiting
|
||||
Container mtusi-lab4-redis-1 Waiting
|
||||
Container mtusi-lab4-postgres-1 Healthy
|
||||
Container mtusi-lab4-redis-1 Healthy
|
||||
Container mtusi-lab4-migrate-1 Exited
|
||||
Container mtusi-lab4-api-1 Starting
|
||||
Container mtusi-lab4-api-1 Started
|
||||
Container mtusi-lab4-api-1 Waiting
|
||||
Container mtusi-lab4-api-2 Waiting
|
||||
Container mtusi-lab4-api-1 Healthy
|
||||
Container mtusi-lab4-api-2 Healthy
|
||||
@@ -0,0 +1,7 @@
|
||||
Static checks (2026-11-02; base commit recorded in verification-metadata.txt)
|
||||
make-check: PASS
|
||||
make-validate: PASS
|
||||
python-compileall: PASS
|
||||
hooks-syntax: PASS
|
||||
compose-yaml: PASS
|
||||
diff-check: PASS
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 student-tests/attachment_failover.py --base-url "http://localhost:8080"
|
||||
PASS: upload instance stopped; metadata and original bytes read via peer
|
||||
@@ -0,0 +1,4 @@
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 140, "failed_attempts": 0, "first_recovery_seconds": 0.556}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 student-tests/idempotency.py --base-url "http://localhost:8080"
|
||||
PASS: 16 concurrent requests across 2 API replicas returned one Message; changed body returned 409.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 student-tests/private_bucket.py
|
||||
HTTP 403
|
||||
PASS: private bucket rejects unauthenticated listing
|
||||
@@ -0,0 +1,7 @@
|
||||
# Анализ первого запуска S3 outage-теста
|
||||
|
||||
Дата: 2026-11-02.
|
||||
|
||||
Первый запуск остановил S3 и успел создать и прочитать синтетическое вложение. Проверочный клиент имел timeout=5 s и прервал GET /health/ready раньше, чем завершились ограниченные connect/read/retry таймауты S3 SDK. Сценарий восстановления в finally запустил S3 обратно; readiness снова стала 200, исходные байты остались доступны.
|
||||
|
||||
В тесте увеличен только клиентский timeout до 35 s, чтобы наблюдать фактический HTTP-ответ после bounded retries. Настройки production S3 не менялись. Повторный прогон получил 503 на readiness и скачивание во время отказа, сохранил доступ к metadata PostgreSQL и подтвердил чтение оригинальных байтов после восстановления S3. Итоговый stdout: test-s3-failure.txt; первоначальный stdout: test-s3-failure-first-attempt.txt.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 student-tests/s3_failure.py --base-url "http://localhost:8080"
|
||||
FAIL: GET /health/ready: connection/TLS failure (TimeoutError)
|
||||
make: *** [Makefile:31: test-s3-failure] Error 1
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 student-tests/s3_failure.py --base-url "http://localhost:8080"
|
||||
PASS: S3 outage returned 503 for readiness/download; metadata remained available
|
||||
PASS: original attachment bytes were readable after S3 recovery
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 student-tests/attachments.py --base-url "http://localhost:8080"
|
||||
PASS: upload/checksum/download, chat ACL, message binding and upload limits
|
||||
@@ -0,0 +1,12 @@
|
||||
Date: 2026-11-02
|
||||
Base commit: ec540a1
|
||||
Branch: solution/BVT2302-Fedotov
|
||||
Working tree: uncommitted at author's request; no commit or push created
|
||||
Docker Engine: 29.8.0
|
||||
WSL kernel: 6.18.33.2-microsoft-standard-WSL2
|
||||
CPU threads: 32
|
||||
Memory available to WSL: 15 GiB
|
||||
Compose limits: each API replica 1 CPU / 512 MiB; other containers have no explicit resource limits
|
||||
Verification profile: make check; full OpenAPI validation; public smoke; upload/download, checksum, ACL and upload limits; 16-request idempotency race across two APIs; private bucket anonymous listing; S3 outage and recovery; cross-replica attachment read; full persistence restart; 140-write API failover; two repeated migration runs (versions 1–4).
|
||||
Load benchmark: not run.
|
||||
Evidence contents: small sanitized command/stdout summaries only; no .env, credentials, tokens, user payloads, or raw dumps.
|
||||
@@ -0,0 +1,37 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (length(title) BETWEEN 1 AND 100 AND btrim(title) <> ''),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users(id),
|
||||
text text NOT NULL CHECK (length(text) BETWEEN 1 AND 4000 AND btrim(text) <> ''),
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
-- A leaked session table must not contain usable bearer credentials.
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Active sessions move to Redis in lab2 grade 4; old token digests cannot be
|
||||
-- reconstructed and are intentionally invalidated during this one-time migration.
|
||||
DROP TABLE IF EXISTS sessions;
|
||||
@@ -0,0 +1,10 @@
|
||||
CREATE TABLE message_idempotency (
|
||||
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
idempotency_key text NOT NULL CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash bytea NOT NULL CHECK (octet_length(request_hash) = 32),
|
||||
message_id uuid NOT NULL REFERENCES messages(id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (user_id, chat_id, idempotency_key)
|
||||
);
|
||||
CREATE INDEX message_idempotency_expiry_idx ON message_idempotency (created_at);
|
||||
@@ -0,0 +1,37 @@
|
||||
ALTER TABLE messages
|
||||
DROP CONSTRAINT messages_text_check,
|
||||
ADD CONSTRAINT messages_text_check CHECK (length(text) BETWEEN 0 AND 4000);
|
||||
|
||||
CREATE TABLE attachments (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
owner_id uuid NOT NULL REFERENCES users(id),
|
||||
object_key text NOT NULL UNIQUE,
|
||||
filename text NOT NULL CHECK (length(filename) BETWEEN 1 AND 255),
|
||||
content_type text NOT NULL CHECK (content_type IN ('image/png', 'image/jpeg')),
|
||||
size_bytes integer NOT NULL CHECK (size_bytes BETWEEN 1 AND 10485760),
|
||||
sha256 text NOT NULL CHECK (sha256 ~ '^[a-f0-9]{64}$'),
|
||||
status text NOT NULL DEFAULT 'ready' CHECK (status = 'ready'),
|
||||
variants jsonb NOT NULL DEFAULT '[]'::jsonb CHECK (variants = '[]'::jsonb),
|
||||
error_code text,
|
||||
created_at timestamptz NOT NULL DEFAULT now(),
|
||||
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||
UNIQUE (id, chat_id)
|
||||
);
|
||||
|
||||
CREATE INDEX attachments_chat_created_idx ON attachments (chat_id, created_at, id);
|
||||
|
||||
ALTER TABLE messages ADD CONSTRAINT messages_id_chat_unique UNIQUE (id, chat_id);
|
||||
|
||||
CREATE TABLE message_attachments (
|
||||
message_id uuid NOT NULL,
|
||||
attachment_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
position smallint NOT NULL CHECK (position BETWEEN 0 AND 9),
|
||||
PRIMARY KEY (message_id, attachment_id),
|
||||
UNIQUE (message_id, position),
|
||||
FOREIGN KEY (message_id, chat_id) REFERENCES messages(id, chat_id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (attachment_id, chat_id) REFERENCES attachments(id, chat_id) ON DELETE RESTRICT
|
||||
);
|
||||
|
||||
CREATE INDEX message_attachments_attachment_idx ON message_attachments (attachment_id);
|
||||
@@ -0,0 +1,29 @@
|
||||
# DNS в Compose обновляется после замены остановленной API-реплики.
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
|
||||
upstream api_backend {
|
||||
zone api_backend 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
keepalive 32;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
client_max_body_size 12m;
|
||||
|
||||
location / {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_send_timeout 30s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_request_buffering off;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
fastapi==0.115.6
|
||||
uvicorn==0.34.0
|
||||
pydantic-settings==2.7.1
|
||||
argon2-cffi==23.1.0
|
||||
asyncpg==0.30.0
|
||||
redis==5.2.1
|
||||
boto3==1.36.26
|
||||
Pillow==11.1.0
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$ROOT"
|
||||
|
||||
# compose down удаляет контейнеры и сеть, но оставляет именованные volumes.
|
||||
docker compose --env-file .env down
|
||||
docker compose --env-file .env up -d
|
||||
Executable
+15
@@ -0,0 +1,15 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides X-Instance-Id of the target replica}"
|
||||
|
||||
for container in $(docker compose ps -q api); do
|
||||
instance_id=$(docker inspect --format '{{.Config.Hostname}}' "$container")
|
||||
if [ "$instance_id" = "$TARGET_INSTANCE_ID" ]; then
|
||||
# Manual stop leaves this replica out of service until the operator restores it.
|
||||
docker stop --timeout 1 "$container" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
printf '%s\n' 'target API replica was not found; no container was stopped' >&2
|
||||
exit 1
|
||||
@@ -0,0 +1,31 @@
|
||||
# Собственные проверки Lab 4
|
||||
|
||||
Этот каталог принадлежит студенту. Публичные contract tests в `tests/` сохранены без изменений.
|
||||
|
||||
Запуск после старта стенда:
|
||||
|
||||
```sh
|
||||
make test-student BASE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
Проверка создаёт синтетических пользователей и чат, загружает публичный PNG fixture, читает его как второй участник, проверяет ACL для постороннего, привязку вложения к чату и типовые ошибки upload. Она не печатает токены и не сохраняет пользовательские данные.
|
||||
|
||||
Для обязательного эксперимента чтения между API-репликами выполните после smoke:
|
||||
|
||||
```sh
|
||||
make test-attachment-failover BASE_URL=http://localhost:8080
|
||||
```
|
||||
|
||||
Скрипт останавливает только API-реплику, которая обслужила upload, читает оригинал через LB, затем восстанавливает две реплики. Все хранилища и volumes остаются на месте.
|
||||
|
||||
Проверка приватности bucket из сети API:
|
||||
|
||||
```sh
|
||||
make test-private-bucket
|
||||
```
|
||||
|
||||
Проверка недоступности и восстановления S3 (временно останавливает только сервис s3, затем запускает его обратно):
|
||||
|
||||
```sh
|
||||
make test-s3-failure BASE_URL=http://localhost:8080
|
||||
```
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Upload through one API instance, stop it, then download through its peer."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Lab 4 attachment cross-replica check')
|
||||
client = Client(args.base_url, args.ca_file, timeout=3)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
content = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
uploaded = client.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
||||
201, actor=alice, raw=content, headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment',
|
||||
)
|
||||
source_instance = uploaded['headers']['X-Instance-Id']
|
||||
observed = set()
|
||||
for _ in range(50):
|
||||
observed.add(client.call(
|
||||
'GET', '/health/live', schema='Health',
|
||||
)['headers']['X-Instance-Id'])
|
||||
check(len(observed) >= 2, 'expected at least two API instances before stopping one')
|
||||
check(source_instance in observed, 'upload instance was not observed by the load balancer')
|
||||
|
||||
hook = ROOT / 'scripts/stop-one.sh'
|
||||
env = dict(os.environ)
|
||||
env['TARGET_INSTANCE_ID'] = source_instance
|
||||
subprocess_result = subprocess.run(
|
||||
[str(hook)], cwd=ROOT, env=env, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL, timeout=10, check=False,
|
||||
)
|
||||
check(subprocess_result.returncode == 0, 'stop-one hook could not stop the upload instance')
|
||||
|
||||
try:
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
metadata = client.call(
|
||||
'GET', '/api/v1/attachments/' + uploaded['json']['id'],
|
||||
actor=bob, schema='Attachment',
|
||||
)
|
||||
download = client.call(
|
||||
'GET', '/api/v1/attachments/' + uploaded['json']['id'] + '/content',
|
||||
actor=bob,
|
||||
)
|
||||
if (metadata['headers']['X-Instance-Id'] != source_instance
|
||||
and download['headers']['X-Instance-Id'] != source_instance):
|
||||
check(download['bytes'] == content, 'cross-replica bytes differ from uploaded file')
|
||||
check(metadata['json']['sha256'] == uploaded['json']['sha256'],
|
||||
'cross-replica metadata checksum changed')
|
||||
print('PASS: upload instance stopped; metadata and original bytes read via peer')
|
||||
return
|
||||
except Failure:
|
||||
time.sleep(0.25)
|
||||
raise Failure('attachment was not readable from the other API instance within 10 seconds')
|
||||
finally:
|
||||
# Restore the two-replica local test stack after this explicit failure experiment.
|
||||
subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'up', '-d', '--scale', 'api=2'],
|
||||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=60, check=False,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Собственные интеграционные проверки вложений поверх неизменённого публичного API."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import quote
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Lab 4 student attachment checks')
|
||||
client = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
fixture = (ROOT / 'tests/fixtures/sample.png').read_bytes()
|
||||
uploaded = client.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/attachments?filename=sample.png',
|
||||
201, actor=alice, raw=fixture, headers={'Content-Type': 'image/png'},
|
||||
schema='Attachment',
|
||||
)['json']
|
||||
check(uploaded['status'] == 'ready', 'upload was not ready')
|
||||
check(uploaded['variants'] == [] and uploaded['error_code'] is None,
|
||||
'Lab 4 ready metadata must have no variants or error')
|
||||
check(uploaded['sha256'] == hashlib.sha256(fixture).hexdigest(),
|
||||
'metadata checksum differs from fixture')
|
||||
check(uploaded['size_bytes'] == len(fixture), 'metadata size differs from fixture')
|
||||
|
||||
metadata = client.call(
|
||||
'GET', '/api/v1/attachments/' + uploaded['id'],
|
||||
actor=bob, schema='Attachment',
|
||||
)['json']
|
||||
check(metadata == uploaded, 'another chat participant got different attachment metadata')
|
||||
downloaded = client.call(
|
||||
'GET', '/api/v1/attachments/' + uploaded['id'] + '/content', actor=bob,
|
||||
)
|
||||
check(downloaded['bytes'] == fixture, 'downloaded bytes are not identical')
|
||||
check(downloaded['headers'].get_content_type() == 'image/png',
|
||||
'download Content-Type is not the detected image type')
|
||||
client.call('GET', '/api/v1/attachments/' + uploaded['id'], 404, actor=eve)
|
||||
client.call('GET', '/api/v1/attachments/' + uploaded['id'] + '/content', 404, actor=eve)
|
||||
|
||||
# Нельзя привязать существующий файл чата к сообщению другого чата.
|
||||
other_chat = client.call(
|
||||
'POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'Другой чат', 'member_ids': [alice['id'], bob['id']]},
|
||||
schema='Chat',
|
||||
)['json']
|
||||
client.call(
|
||||
'POST', '/api/v1/chats/' + other_chat['id'] + '/messages', 404,
|
||||
actor=alice, body={'text': '', 'attachment_ids': [uploaded['id']]},
|
||||
headers={'Idempotency-Key': uuid.uuid4().hex},
|
||||
)
|
||||
|
||||
prefix = '/api/v1/chats/' + chat['id'] + '/attachments?filename='
|
||||
client.call(
|
||||
'POST', prefix + 'bad.png', 415, actor=alice, raw=b'not an image',
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
client.call(
|
||||
'POST', prefix + 'mismatch.jpg', 415, actor=alice, raw=fixture,
|
||||
headers={'Content-Type': 'image/jpeg'},
|
||||
)
|
||||
client.call(
|
||||
'POST', prefix + quote('../sample.png', safe=''), 400, actor=alice,
|
||||
raw=fixture, headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
client.call(
|
||||
'POST', prefix + 'too-large.png', 413, actor=alice,
|
||||
raw=b'\x00' * (10 * 1024 * 1024 + 1),
|
||||
headers={'Content-Type': 'image/png'},
|
||||
)
|
||||
print('PASS: upload/checksum/download, chat ACL, message binding and upload limits')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверка атомарного повтора одного сообщения через разные API-реплики."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
import sys
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Concurrent cross-replica idempotency check')
|
||||
setup = Client(args.base_url, args.ca_file)
|
||||
alice, _bob, _eve, chat = setup.bootstrap()
|
||||
key = 'concurrent-' + uuid.uuid4().hex
|
||||
text = 'Одно сообщение при конкурентном повторе ' + uuid.uuid4().hex
|
||||
count = 16
|
||||
|
||||
def send(_):
|
||||
client = Client(args.base_url, args.ca_file, timeout=10)
|
||||
return client.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=201,
|
||||
actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message',
|
||||
)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=count) as pool:
|
||||
responses = list(pool.map(send, range(count)))
|
||||
messages = [response['json'] for response in responses]
|
||||
instance_ids = {response['headers'].get('X-Instance-Id') for response in responses}
|
||||
check(all(message == messages[0] for message in messages), 'concurrent replay returned different Message objects')
|
||||
check(len(instance_ids) >= 2, 'concurrent requests did not reach at least two observable API replicas')
|
||||
|
||||
replay = setup.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=201,
|
||||
actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message',
|
||||
)['json']
|
||||
check(replay == messages[0], 'retry returned a different status payload')
|
||||
setup.call(
|
||||
'POST', '/api/v1/chats/' + chat['id'] + '/messages', expected=409,
|
||||
actor=alice, body={'text': text + ' другой текст'}, headers={'Idempotency-Key': key},
|
||||
)
|
||||
history = setup.call(
|
||||
'GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100',
|
||||
actor=alice, schema='MessagePage',
|
||||
)['json']['items']
|
||||
matching = [message for message in history if message['text'] == text]
|
||||
check(matching == [messages[0]], 'history contains a missing or duplicate side effect')
|
||||
print('PASS: %d concurrent requests across %d API replicas returned one Message; changed body returned 409.' % (count, len(instance_ids)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Отправить анонимный ListBucket из сети приложения и подтвердить private bucket."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def main():
|
||||
env_path = ROOT / '.env'
|
||||
if not env_path.is_file():
|
||||
raise SystemExit('Create local .env from .env.example first')
|
||||
values = {}
|
||||
for line in env_path.read_text().splitlines():
|
||||
if '=' in line and not line.lstrip().startswith('#'):
|
||||
key, value = line.split('=', 1)
|
||||
values[key.strip()] = value.strip()
|
||||
bucket = values.get('S3_BUCKET', 'messenger-attachments')
|
||||
url = 'http://s3:9000/' + bucket + '?list-type=2'
|
||||
code = (
|
||||
'import urllib.error, urllib.request\n'
|
||||
'url = ' + repr(url) + '\n'
|
||||
'try:\n'
|
||||
' urllib.request.urlopen(url, timeout=5)\n'
|
||||
'except urllib.error.HTTPError as error:\n'
|
||||
' print("HTTP", error.code)\n'
|
||||
' raise SystemExit(0 if error.code in (401, 403) else 1)\n'
|
||||
'except Exception as error:\n'
|
||||
' print(type(error).__name__)\n'
|
||||
' raise SystemExit(1)\n'
|
||||
'raise SystemExit("anonymous bucket listing unexpectedly succeeded")\n'
|
||||
)
|
||||
result = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'exec', '-T', 'api', 'python', '-c', code],
|
||||
cwd=ROOT, check=False,
|
||||
)
|
||||
if result.returncode:
|
||||
raise SystemExit('FAIL: anonymous S3 bucket listing was not denied')
|
||||
print('PASS: private bucket rejects unauthenticated listing')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Проверить ограниченную ошибку API при отказе S3 и восстановить его."""
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, arguments, check, run
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments('Lab 4 S3 outage and recovery check')
|
||||
# Дождаться ограниченных повторов boto3 при остановленном S3.
|
||||
client = Client(args.base_url, args.ca_file, timeout=35)
|
||||
alice, bob, eve, chat = client.bootstrap()
|
||||
attachment, content = client.attachment(alice, chat)
|
||||
initial = client.call(
|
||||
'GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob,
|
||||
)
|
||||
check(initial['bytes'] == content, 'uploaded bytes differ before the outage')
|
||||
|
||||
stopped = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'stop', 's3'],
|
||||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=20, check=False,
|
||||
)
|
||||
check(stopped.returncode == 0, 'could not stop S3 for the outage test')
|
||||
try:
|
||||
ready = client.call('GET', '/health/ready', expected=503, schema='Error')
|
||||
check(ready['json']['error']['code'] == 'unavailable',
|
||||
'readiness did not report unavailable S3')
|
||||
failed = client.call(
|
||||
'GET', '/api/v1/attachments/' + attachment['id'] + '/content',
|
||||
expected=503, actor=bob, schema='Error',
|
||||
)
|
||||
check(failed['json']['error']['code'] == 'unavailable',
|
||||
'S3 failure was not translated to a bounded API error')
|
||||
metadata = client.call(
|
||||
'GET', '/api/v1/attachments/' + attachment['id'],
|
||||
actor=bob, schema='Attachment',
|
||||
)
|
||||
check(metadata['json']['id'] == attachment['id'],
|
||||
'PostgreSQL metadata was lost while S3 was unavailable')
|
||||
print('PASS: S3 outage returned 503 for readiness/download; metadata remained available')
|
||||
finally:
|
||||
started = subprocess.run(
|
||||
['docker', 'compose', '--env-file', '.env', 'start', 's3'],
|
||||
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
timeout=20, check=False,
|
||||
)
|
||||
check(started.returncode == 0, 'could not restart S3 after the outage test')
|
||||
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
client.call('GET', '/health/ready', schema='Health')
|
||||
break
|
||||
except Failure:
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
raise Failure('API did not become ready after S3 restart')
|
||||
|
||||
recovered = client.call(
|
||||
'GET', '/api/v1/attachments/' + attachment['id'] + '/content', actor=bob,
|
||||
)
|
||||
check(recovered['bytes'] == content, 'S3 restart changed or lost the original bytes')
|
||||
print('PASS: original attachment bytes were readable after S3 recovery')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user